diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml deleted file mode 100644 index 6898a80fdc..0000000000 --- a/.builds/freebsd.yml +++ /dev/null @@ -1,31 +0,0 @@ -# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd -image: freebsd/latest -packages: -- databases/sqlite3 -- devel/boehm-gc-threaded -- devel/pcre -- devel/sdl20 -- devel/sfml -- www/node -- devel/gmake -sources: -- https://github.com/nim-lang/Nim -environment: - CC: /usr/bin/clang -tasks: -- setup: | - cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git - gmake -C csources -j $(sysctl -n hw.ncpu) - bin/nim c --skipUserCfg --skipParentCfg koch - echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv -- test: | - cd Nim - if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim - exit 1 - fi -triggers: -- action: email - condition: failure - to: Andreas Rumpf diff --git a/.builds/openbsd_0.yml b/.builds/openbsd_0.yml deleted file mode 100644 index 146ad81657..0000000000 --- a/.builds/openbsd_0.yml +++ /dev/null @@ -1,34 +0,0 @@ -## do not edit directly; auto-generated by `nim r tools/ci_generate.nim` - -image: openbsd/latest -packages: -- gmake -- sqlite3 -- node -- boehm-gc -- pcre -- sfml -- sdl2 -- libffi -sources: -- https://github.com/nim-lang/Nim -environment: - NIM_TESTAMENT_BATCH: "0_2" - CC: /usr/bin/clang -tasks: -- setup: | - cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git - gmake -C csources -j $(sysctl -n hw.ncpuonline) - bin/nim c koch - echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv -- test: | - cd Nim - if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim - exit 1 - fi -triggers: -- action: email - condition: failure - to: Andreas Rumpf diff --git a/.builds/openbsd_1.yml b/.builds/openbsd_1.yml deleted file mode 100644 index ad37340083..0000000000 --- a/.builds/openbsd_1.yml +++ /dev/null @@ -1,34 +0,0 @@ -## do not edit directly; auto-generated by `nim r tools/ci_generate.nim` - -image: openbsd/latest -packages: -- gmake -- sqlite3 -- node -- boehm-gc -- pcre -- sfml -- sdl2 -- libffi -sources: -- https://github.com/nim-lang/Nim -environment: - NIM_TESTAMENT_BATCH: "1_2" - CC: /usr/bin/clang -tasks: -- setup: | - cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git - gmake -C csources -j $(sysctl -n hw.ncpuonline) - bin/nim c koch - echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv -- test: | - cd Nim - if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim - exit 1 - fi -triggers: -- action: email - condition: failure - to: Andreas Rumpf diff --git a/.gitattributes b/.gitattributes index 674da79398..2c9ff103cd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,6 @@ # duplicated, which is easily identifiable and fixable. /changelog.md merge=union +# bug https://github.com/dom96/choosenim/issues/256 for WSL CRLF +*.sh text eol=lf +/config/build_config.txt text eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 9571896d89..0000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -name: Bug report -about: Have you found an unexpected behavior? Use this template. -title: Think about the title, twice -labels: '' -assignees: '' - ---- - -Function `echo` outputs the wrong string. - -### Example -```nim -echo "Hello World!" -# This code should be a minimum reproducible example: -# try to simplify and minimize as much as possible. If it's a compiler -# issue, try to minimize further by removing any imports if possible. -``` - -### Current Output -please check whether the problem still exists in git head before posting, -see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler). -``` -Hola mundo! -``` - -### Expected Output -``` -Hello World! -``` - -### Possible Solution - -* In file xyz there is a call that might be the cause of it. - -### Additional Information -If it's a regression, you can help us by identifying which version introduced -the bug, see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions), -or at least try known past releases (eg `choosenim 1.2.0`). - -If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler) -which should give more context on a compiler crash. - -* Issue #abc is related, but different because of ... -* This issue is blocking my project xyz - -``` -$ nim -v -Nim Compiler Version 0.1.2 -# make sure to include the git hash if not using a tagged release -``` diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000000..0145138650 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,76 @@ +name: "Bug Report" +description: "Create a new bug report. Have you found an unexpected behavior? Use this form." +title: "Think about the title, twice." +labels: ["unconfirmed"] +body: + +- type: markdown + attributes: + value: | + - **Please provide a minimal code example that reproduces the Bug!** :bug: + Reports with a reproducible example and descriptive detailed information will likely receive fixes faster. + +- type: textarea + id: description + attributes: + label: Description + description: | + Use DETAILED DESCRIPTIVE information about the problem. + Here, you go into more details about your Bug report. This section can be a few paragraphs long. + placeholder: Bug reports with reproducible code and detailed information will be fixed faster. + validations: + required: true + +- type: textarea + id: nim-version + attributes: + label: Nim Version + description: Copy and paste the output of `nim -v` on the command line. For development versions, make sure to include the commit hash. + validations: + required: true + +- type: textarea + id: current-logs + attributes: + label: Current Output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + placeholder: Bug reports with reproducible code and detailed information will be fixed faster. + render: text + +- type: textarea + id: expected-logs + attributes: + label: Expected Output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + placeholder: Bug reports with reproducible code and detailed information will be fixed faster. + render: text + +- type: textarea + id: possible-solution + attributes: + label: Possible Solution + description: Propose a possible solution. + validations: + required: false + +- type: textarea + id: extra-info + attributes: + label: Additional Information + description: Any additional relevant information. + validations: + required: false + +- type: markdown + attributes: + value: | + - Thanks for your contributions!, your Bug report will receive feedback from the community soon... + - Please check whether the problem still exists in the devel branch, see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler). + - Consider writing a PR targetting devel branch after filing this, see [contributing](https://nim-lang.github.io/Nim/contributing.html). + - If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler) + which should give more context on a compiler crash. + - If it's a regression, you can help us by identifying which version introduced the bug, + see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions), + or at least try known past releases (e.g. `choosenim 2.0.0`). The Nim repo also supports online bisecting + via making a comment, which contains a code block starting by `!nim c`, `!nim js` etc. , see [nimrun-action](https://github.com/juancarlospaco/nimrun-action). + - [Please, consider a Donation for the Nim project.](https://nim-lang.org/donate.html) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index b9f7caad58..0000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -name: Feature request -about: Do you want to suggest a new feature? Use this template. -title: '' -labels: '' -assignees: '' - ---- - - - - -### Summary - - - -### Description - - - -### Alternatives - - - -### Additional Information - diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000000..5cb4586260 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,72 @@ +name: "Feature request" +description: "Create a new Feature Request. Do you want to suggest a new feature? Use this form." +title: "Think about the title, twice." +labels: ["unconfirmed"] +body: + +- type: markdown + attributes: + value: | + - Please provide a minimal code example that illustrates the basic idea behind your feature request. + Reports with full repro code and descriptive detailed information will likely receive feedback faster. + - There is a separate repository for the detailed RFCs and proposals: https://github.com/nim-lang/RFCs. + If you have a simple feature request, you can post it here using this form, + but bear in mind that adding new features to the language is currently a low priority. + +- type: textarea + id: summary + attributes: + label: Summary + description: | + Use DETAILED DESCRIPTIVE information about the feature request. + Here, you go into more details about your ideas. This section can be a few paragraphs long. + placeholder: Short summary of your proposed feature. + validations: + required: true + +- type: textarea + id: description + attributes: + label: Description + description: Describe your solution, what problem does it fix? + validations: + required: true + +- type: textarea + id: alternatives + attributes: + label: Alternatives + description: Are there any alternatives you've considered? + validations: + required: false + +- type: textarea + id: Examples + attributes: + label: Examples + description: Provide examples for your feature request here. + placeholder: Example code here. + +- type: textarea + id: incompatibility + attributes: + label: Backwards Compatibility + description: If your Feature Request introduces backward-incompatible changes, describe them and propose how to deal with them. + validations: + required: false + +- type: textarea + id: links + attributes: + label: Links + description: Please copy and paste any relevant links to projects, issues, discussions, technical documentations, code samples, etc. + validations: + required: false + +- type: markdown + attributes: + value: | + - Thanks for your contributions!, your ideas will receive feedback from the community soon... + - **Remember to :star: Star the Nim project on GitHub!.** + - Consider writing a PR targetting devel branch after filing this, see [contributing](https://nim-lang.github.io/Nim/contributing.html). + - [Please, consider a Donation for the Nim project.](https://nim-lang.org/donate.html) diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index c175f64a0f..0000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,69 +0,0 @@ -# Configuration for probot-stale - https://github.com/probot/stale - -# Number of days of inactivity before an Issue or Pull Request becomes stale -daysUntilStale: 365 - -# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. -# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. -daysUntilClose: 30 - -# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) -onlyLabels: [] - -# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable -exemptLabels: - - ARC - - bounty - - Codegen - - Crash - - Generics - - High Priority - - Macros - - Next release - - Showstopper - - Static[T] - -# Set to true to ignore issues in a project (defaults to false) -exemptProjects: false - -# Set to true to ignore issues in a milestone (defaults to false) -exemptMilestones: false - -# Set to true to ignore issues with an assignee (defaults to false) -exemptAssignees: false - -# Label to use when marking as stale -staleLabel: stale - -# Comment to post when marking as stale. Set to `false` to disable -markComment: > - This pull request has been automatically marked as stale because it has not had - recent activity. - If you think it is still a valid PR, please rebase it on the latest devel; - otherwise it will be closed. Thank you for your contributions. - -# Comment to post when removing the stale label. -# unmarkComment: > -# Your comment here. - -# Comment to post when closing a stale Issue or Pull Request. -# closeComment: > -# Your comment here. - -# Limit the number of actions per hour, from 1-30. Default is 30 -limitPerRun: 20 - -# Limit to only `issues` or `pulls` -only: pulls - -# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': -# pulls: -# daysUntilStale: 30 -# markComment: > -# This pull request has been automatically marked as stale because it has not had -# recent activity. It will be closed if no further activity occurs. Thank you -# for your contributions. - -# issues: -# exemptLabels: -# - confirmed diff --git a/.github/workflows/bisects.yml b/.github/workflows/bisects.yml new file mode 100644 index 0000000000..4a6a92f018 --- /dev/null +++ b/.github/workflows/bisects.yml @@ -0,0 +1,31 @@ +# See https://github.com/juancarlospaco/nimrun-action/issues/3#issuecomment-1607344901 +name: issue comments bisects +on: + issue_comment: + types: created + +jobs: + bisects: + if: | + github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '!nim ') && github.event.issue.pull_request == null && github.event.comment.author_association != 'NONE' + strategy: + fail-fast: false + matrix: + platform: [ubuntu-latest, windows-latest, macos-latest] + name: ${{ matrix.platform }}-bisects + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v4 + + - uses: jiro4989/setup-nim-action@v1 + with: + nim-version: 'devel' + + - uses: juancarlospaco/nimrun-action@nim + if: | + runner.os == 'Linux' && contains(github.event.comment.body, '-d:linux' ) || + runner.os == 'Windows' && contains(github.event.comment.body, '-d:windows') || + runner.os == 'macOS' && contains(github.event.comment.body, '-d:osx' ) || + runner.os == 'Linux' && !contains(github.event.comment.body, '-d:linux') && !contains(github.event.comment.body, '-d:windows') && !contains(github.event.comment.body, '-d:osx') + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml.disabled b/.github/workflows/ci.yml.disabled deleted file mode 100644 index 1835764563..0000000000 --- a/.github/workflows/ci.yml.disabled +++ /dev/null @@ -1,122 +0,0 @@ -name: Continous Integration -on: [push, pull_request] - -jobs: - build: - strategy: - matrix: - os: [ubuntu-18.04, macos-10.15, windows-2019] - cpu: [amd64, i386] - cpp: ['false', 'true'] - pkg: ['false', 'true'] - exclude: - - os: ubuntu-18.04 - cpp: 'true' - - os: ubuntu-18.04 - pkg: 'true' - - os: macos-10.15 - cpu: i386 - - os: macos-10.15 - pkg: 'true' - - os: windows-2019 - cpu: i386 - - os: windows-2019 - cpp: 'true' - name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})' - runs-on: ${{ matrix.os }} - env: - NIM_COMPILE_TO_CPP: ${{ matrix.cpp }} - NIM_TEST_PACKAGES: ${{ matrix.pkg }} - steps: - - name: 'Checkout' - uses: actions/checkout@v2 - - name: 'Checkout csources' - uses: actions/checkout@v2 - with: - repository: nim-lang/csources - path: csources - - - name: 'Install node.js 8.x' - uses: actions/setup-node@v1 - with: - node-version: '8.x' - - name: 'Install dependencies (Linux amd64)' - if: runner.os == 'Linux' && matrix.cpu == 'amd64' - run: | - sudo apt-fast update -qq - DEBIAN_FRONTEND='noninteractive' \ - sudo apt-fast install --no-install-recommends -yq \ - libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \ - valgrind libc6-dbg - - name: 'Install dependencies (Linux i386)' - if: runner.os == 'Linux' && matrix.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - - sudo apt-fast update -qq - DEBIAN_FRONTEND='noninteractive' \ - sudo apt-fast install --no-install-recommends --allow-downgrades -yq \ - g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \ - libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \ - libffi-dev:i386 - - cat << EOF > bin/gcc - #!/bin/bash - - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > bin/g++ - #!/bin/bash - - exec $(which g++) -m32 "\$@" - EOF - - chmod 755 bin/gcc - chmod 755 bin/g++ - - name: 'Install dependencies (macOS)' - if: runner.os == 'macOS' - run: brew install boehmgc make sfml - - name: 'Install dependencies (Windows)' - if: runner.os == 'Windows' - shell: bash - run: | - mkdir dist - curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - 7z x dist/mingw64.7z -odist - 7z x dist/dlls.zip -obin - echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" - - - name: 'Add build binaries to PATH' - shell: bash - run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - - name: 'Build csources' - shell: bash - run: | - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - - make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}' - - name: 'Build koch' - shell: bash - run: nim c koch - - name: 'Run CI' - shell: bash - run: ./koch runCI - - - name: 'Show failed tests' - if: failure() - shell: bash - run: nim c -r tools/ci_testresults.nim diff --git a/.github/workflows/ci_bench.yml b/.github/workflows/ci_bench.yml new file mode 100644 index 0000000000..7c76ed89bc --- /dev/null +++ b/.github/workflows/ci_bench.yml @@ -0,0 +1,115 @@ +name: Benchmarks CI +on: + pull_request: + push: + branches: + - 'devel' + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + cpu: [amd64] + name: '${{ matrix.os }}' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 # refs bug #18178 + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: 'Install node.js 20.x' + uses: actions/setup-node@v4 + with: + node-version: '20.x' + + - name: 'Install dependencies (Linux amd64)' + if: runner.os == 'Linux' && matrix.cpu == 'amd64' + run: | + sudo apt-fast update -qq + DEBIAN_FRONTEND='noninteractive' \ + sudo apt-fast install --no-install-recommends -yq \ + libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \ + valgrind libc6-dbg libblas-dev xorg-dev + + - name: 'Add build binaries to PATH' + shell: bash + run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" + + - name: 'Build csourcesAny' + shell: bash + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' + + - name: 'Build koch' + shell: bash + run: nim c koch + + - name: 'Build Nim' + shell: bash + run: ./koch boot -d:release -d:nimStrictMode --lib:lib + + - name: 'Build Nimble' + shell: bash + run: ./koch nimble + + - name: 'Action' + shell: bash + run: nim c -r -d:release ci/action.nim + + - name: 'Checkout minimize' + uses: actions/checkout@v4 + with: + repository: 'nim-lang/ci_bench' + path: minimize + + - name: 'Run minimize benchmarks' + shell: bash + run: ./minimize/minimize ci-bench + + - name: 'Restore minimize cached database' + uses: actions/cache/restore@v3 + with: + path: minimize.csv + key: minimize-db-key + + - name: 'Update minimize db' + shell: bash + run: ./minimize/minimize update-db + + - name: 'Save minimize cached database' + if: | + github.event_name == 'push' && github.ref == 'refs/heads/devel' && + matrix.target == 'linux' + uses: actions/cache/save@v3 + with: + path: minimize.csv + key: minimize-db-key + + - name: 'Generate minimize report' + shell: bash + run: ./minimize/minimize generate-report + + - name: 'Archive minimize report' + uses: actions/upload-artifact@v3 + with: + name: minimize-report + path: | + minimize/minimize.html + minimize/minimize.csv + + # Requires additional permissions, see: + # https://github.com/nim-lang/Nim/actions/runs/4778177321/jobs/8494423792?pr=21566 + # - name: 'Publish HTML report' + # uses: rossjrw/pr-preview-action@v1 + # with: + # source-dir: minimize + # umbrella-dir: minimize + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract md summary + run: | + cat minimize/summary.md >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 22d425b103..2faa37ce0a 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -2,24 +2,10 @@ name: Nim Docs CI on: push: paths: - - 'compiler/docgen.nim' - - 'compiler/renderverbatim.nim' - - 'config/nimdoc.cfg' - - 'doc/**.rst' - - 'doc/nimdoc.css' - - 'lib/**.nim' - - 'nimdoc/testproject/expected/testproject.html' - - 'tools/dochack/dochack.nim' - - 'tools/kochdocs.nim' - - '.github/workflows/ci_docs.yml' - - 'koch.nim' - pull_request: - # Run only on changes on these files. - paths: - - 'compiler/docgen.nim' - - 'compiler/renderverbatim.nim' + - 'compiler/**.nim' - 'config/nimdoc.cfg' - 'doc/**.rst' + - 'doc/**.md' - 'doc/nimdoc.css' - 'lib/**.nim' - 'nimdoc/testproject/expected/testproject.html' @@ -28,28 +14,48 @@ on: - '.github/workflows/ci_docs.yml' - 'koch.nim' + pull_request: + # Run only on changes on these files. + paths: + - 'compiler/**.nim' + - 'config/nimdoc.cfg' + - 'doc/**.rst' + - 'doc/**.md' + - 'doc/nimdoc.css' + - 'lib/**.nim' + - 'nimdoc/testproject/expected/testproject.html' + - 'tools/dochack/dochack.nim' + - 'tools/kochdocs.nim' + - '.github/workflows/ci_docs.yml' + - 'koch.nim' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: - if: | - !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]') strategy: fail-fast: false matrix: target: [linux, windows, osx] include: - target: linux - os: ubuntu-18.04 + os: ubuntu-20.04 - target: windows os: windows-2019 - target: osx - os: macos-10.15 + os: macos-12 name: ${{ matrix.target }} runs-on: ${{ matrix.os }} + timeout-minutes: 60 # refs bug #18178 steps: - name: 'Checkout' - uses: actions/checkout@v2 + uses: actions/checkout@v4 + with: + fetch-depth: 2 - name: 'Install build dependencies (macOS)' if: runner.os == 'macOS' @@ -59,64 +65,33 @@ jobs: if: runner.os == 'Windows' shell: bash run: | - mkdir dist - curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - 7z x dist/mingw64.7z -odist - 7z x dist/dlls.zip -obin + set -e + . ci/funs.sh + nimInternalInstallDepsWindows echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" - name: 'Add build binaries to PATH' shell: bash run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - name: 'Get current csources version' - id: csources-version + - name: 'System information' shell: bash - run: | - sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1) - echo "::set-output name=sha::$sha" + run: . ci/funs.sh && nimCiSystemInfo - - name: 'Get prebuilt csources from cache' - id: csources-cache - uses: actions/cache@v1 - with: - path: bin - key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}' - - - name: 'Checkout csources' - if: steps.csources-cache.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: nim-lang/csources - path: csources - - - name: 'Build 1-stage compiler from csources' + - name: 'Build csourcesAny (posix)' + # this would work on windows and other CI use this on windows, + # but we ensure here that `ci/build_autogen.bat` keeps working on windows. + if: runner.os != 'Windows' shell: bash - run: | - ext= - [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe - if [[ ! -x bin/nim-csources$ext ]]; then - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc + # was previously using caching via `actions/cache@v1` but this wasn't + # used in other CI pipelines and it's unclear the added complexity + # was worth the saving; can be revisited if needed. - make -C csources -j $ncpu CC=gcc - cp bin/nim{,-csources}$ext - else - echo 'Cache hit, using prebuilt csources' - cp bin/nim{-csources,}$ext - fi + - name: 'Build csourcesAny (windows)' + if: runner.os == 'Windows' + shell: cmd + run: ci/build_autogen.bat - name: 'Build koch' shell: bash @@ -134,7 +109,7 @@ jobs: if: | github.event_name == 'push' && github.ref == 'refs/heads/devel' && matrix.target == 'linux' - uses: crazy-max/ghaction-github-pages@v1 + uses: crazy-max/ghaction-github-pages@v4 with: build_dir: doc/html env: diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index dd42db0c46..2ea5092e3e 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -1,34 +1,42 @@ name: Packages CI -on: [push, pull_request] +on: + pull_request: + push: + branches: + - 'devel' + - 'version-2-0' + - 'version-1-6' + - 'version-1-2' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: - if: | - !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]') strategy: fail-fast: false matrix: - os: [ubuntu-18.04, macos-10.15] + os: [ubuntu-20.04, macos-12] cpu: [amd64] - batch: ["0_3", "1_3", "2_3"] # list of `index_num` + batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num` name: '${{ matrix.os }} (batch: ${{ matrix.batch }})' runs-on: ${{ matrix.os }} + timeout-minutes: 60 # refs bug #18178 env: NIM_TEST_PACKAGES: "1" NIM_TESTAMENT_BATCH: ${{ matrix.batch }} steps: - name: 'Checkout' - uses: actions/checkout@v2 - - name: 'Checkout csources' - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: - repository: nim-lang/csources - path: csources + fetch-depth: 2 - - name: 'Install node.js 12.x' - uses: actions/setup-node@v1 + - name: 'Install node.js 20.x' + uses: actions/setup-node@v4 with: - node-version: '12.x' + node-version: '20.x' + - name: 'Install dependencies (Linux amd64)' if: runner.os == 'Linux' && matrix.cpu == 'amd64' run: | @@ -44,43 +52,23 @@ jobs: if: runner.os == 'Windows' shell: bash run: | - mkdir dist - curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - 7z x dist/mingw64.7z -odist - 7z x dist/dlls.zip -obin - echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" + set -e + . ci/funs.sh + nimInternalInstallDepsWindows + echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" - name: 'Add build binaries to PATH' shell: bash run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - name: 'Build csources' + - name: 'System information' shell: bash - run: | - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + run: . ci/funs.sh && nimCiSystemInfo - make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}' - - name: 'Build koch' + - name: 'Build csourcesAny' shell: bash - run: nim c koch - - name: 'Run CI' - shell: bash - run: ./koch runCI + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' - - name: 'Show failed tests' - if: failure() + - name: 'koch, Run CI' shell: bash - run: nim c -r tools/ci_testresults.nim + run: . ci/funs.sh && nimInternalBuildKochAndRunCI diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml new file mode 100644 index 0000000000..decfe953ec --- /dev/null +++ b/.github/workflows/ci_publish.yml @@ -0,0 +1,90 @@ +name: Tracking orc-booting compiler memory usage + +on: + pull_request_target: + types: [closed] + + +jobs: + build: + if: github.event.pull_request.merged == true + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + cpu: [amd64] + name: '${{ matrix.os }}' + runs-on: ${{ matrix.os }} + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: 'Install node.js 20.x' + uses: actions/setup-node@v4 + with: + node-version: '20.x' + + - name: 'Install dependencies (Linux amd64)' + if: runner.os == 'Linux' && matrix.cpu == 'amd64' + run: | + sudo apt-fast update -qq + DEBIAN_FRONTEND='noninteractive' \ + sudo apt-fast install --no-install-recommends -yq \ + libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \ + valgrind libc6-dbg libblas-dev xorg-dev + - name: 'Install dependencies (macOS)' + if: runner.os == 'macOS' + run: brew install boehmgc make sfml gtk+3 + - name: 'Install dependencies (Windows)' + if: runner.os == 'Windows' + shell: bash + run: | + set -e + . ci/funs.sh + nimInternalInstallDepsWindows + echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" + + - name: 'Add build binaries to PATH' + shell: bash + run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" + + - name: 'System information' + shell: bash + run: . ci/funs.sh && nimCiSystemInfo + + - name: 'Build csourcesAny' + shell: bash + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' + + - name: 'Build koch' + shell: bash + run: nim c koch + + - name: 'Build Nim' + shell: bash + run: ./koch boot -d:release -d:nimStrictMode --lib:lib + + - name: 'Action' + shell: bash + run: nim c -r -d:release ci/action.nim + + - name: 'Comment' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + try { + const data = fs.readFileSync('ci/nimcache/results.txt', 'utf8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: data + }) + } catch (err) { + console.error(err); + } + diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..0c5a533e1d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,25 @@ +# https://github.com/actions/stale#usage +name: Stale pull requests + +on: + schedule: + - cron: '0 0 * * *' # Midnight. + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + days-before-pr-stale: 365 + days-before-pr-close: 30 + days-before-issue-stale: -1 + days-before-issue-close: -1 + exempt-pr-labels: "ARC,bounty,Codegen,Crash,Generics,High Priority,Macros,Next release,Showstopper,Static[T]" + exempt-issue-labels: "Showstopper,Severe,bounty,Compiler Crash,Medium Priority" + stale-pr-message: > + This pull request is stale because it has been open for 1 year with no activity. + Contribute more commits on the pull request and rebase it on the latest devel, + or it will be closed in 30 days. Thank you for your contributions. + close-pr-message: > + This pull request has been marked as stale and closed due to inactivity after 395 days. diff --git a/.gitignore b/.gitignore index e71fdf352a..fad7909bd8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,9 @@ !*.* # Cache -nimcache/ -rnimcache/ -dnimcache/ +nimcache*/ +rnimcache*/ +dnimcache*/ *.o !/icons/*.o @@ -64,7 +64,11 @@ lib/**/*.html testament.db /tests/**/*.json /tests/**/*.js + /csources +/csources_v1 +/csources_v2 + /dist/ # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it @@ -72,14 +76,18 @@ testament.db .*/ ~* -# testament cruft; TODO: generate these in a gitignore'd dir in the first place. +# testament cruft; TODO: generate these in a gitignore'd dir (./build) in the first place. testresults/ test.txt /test.ini tweeter.db tweeter_test.db -megatest.nim + +/tests/megatest.nim +/tests/ic/*_temp.nim +/tests/navigator/*_temp.nim + /outputExpected.txt /outputGotten.txt @@ -102,3 +110,5 @@ htmldocs nimdoc.out.css # except here: !/nimdoc/testproject/expected/* +pkgs/ +/compiler/compiler/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index ad9e565a6c..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,60 +0,0 @@ -image: ubuntu:16.04 - -stages: - - pre-build - - build - - deploy - - test - -.linux_set_path: &linux_set_path_def - before_script: - - export PATH=$(pwd)/bin${PATH:+:$PATH} - tags: - - linux - -.windows_set_path: &win_set_path_def - before_script: - - set PATH=%CD%\bin;%PATH% - tags: - - windows - - -build-windows: - stage: build - script: - - ci\build.bat - artifacts: - paths: - - bin\nim.exe - - bin\nimd.exe - - compiler\nim.exe - - koch.exe - expire_in: 1 week - tags: - - windows - -deploy-windows: - stage: deploy - script: - - koch.exe winrelease - artifacts: - paths: - - build/*.exe - - build/*.zip - expire_in: 1 week - tags: - - windows - - fast - - - -test-windows: - stage: test - <<: *win_set_path_def - script: - - call ci\deps.bat - - nim c testament\tester - - testament\tester.exe all - tags: - - windows - - fast diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4c4c35334b..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,52 +0,0 @@ -sudo: false -language: c - -dist: xenial - -matrix: - include: - - os: linux - env: - - NIM_COMPILE_TO_CPP=false - - CPU=amd64 - -addons: - apt: - # update the list above if more deps are introduced - packages: - - libcurl4-openssl-dev - - libsdl1.2-dev - - libgc-dev - - libsfml-dev - - libc6-dbg - - valgrind - -before_script: - - git clone --depth 1 https://github.com/nim-lang/csources.git - - export PATH="$PWD/bin${PATH:+:$PATH}" - - make -C csources -j 2 LD=$CC ucpu=$CPU - -script: - - echo "travis_fold:start:nim_c_koch" - - nim c koch - - echo "travis_fold:end:nim_c_koch" - - echo "travis_fold:start:koch_boot" - - ./koch boot - - echo "travis_fold:end:koch_boot" - - echo "travis_fold:start:koch_doc" - - ./koch doc - - echo "travis_fold:end:koch_doc" - -before_deploy: - # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html - - cp -f ./doc/html/overview.html ./doc/html/index.html - -deploy: # https://nim-lang.github.io/Nim - provider: pages - # local-dir *has* to be a relative path from the repo root. - local-dir: "doc/html" - skip-cleanup: true - github-token: "$GITHUB_OAUTH_TOKEN" - keep-history: false - on: - branch: devel diff --git a/appveyor.yml.disabled b/appveyor.yml.disabled deleted file mode 100644 index 5468ac88af..0000000000 --- a/appveyor.yml.disabled +++ /dev/null @@ -1,37 +0,0 @@ -version: '{build}' - -environment: - DLLS_URL: https://nim-lang.org/download/dlls.zip - DLLS_ARCHIVE: dlls.zip - MINGW_DIR: mingw64 - MINGW_URL: https://nim-lang.org/download/mingw64.7z - MINGW_ARCHIVE: mingw64.7z - - matrix: - - NIM_TEST_PACKAGES: false - - NIM_TEST_PACKAGES: true - -cache: - - '%MINGW_ARCHIVE%' - - '%DLLS_ARCHIVE%' - - -install: - - ps: Install-Product node 8 # node 8 or later is required to test js async stuff - - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" - - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\BIN"> nul - - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH% - - git clone --depth 1 https://github.com/nim-lang/csources - - cd csources - - build64.bat - - cd .. - -build_script: - - openssl version - - openssl version -d - - bin\nim c koch - - koch runCI - -deploy: off diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f65f8ca11e..9ef202948f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,12 +1,17 @@ trigger: branches: include: - - '*' + - 'devel' + - 'version-*' pr: branches: include: - '*' +variables: +- name: skipci + value: false + jobs: - job: packages @@ -15,18 +20,19 @@ jobs: strategy: matrix: Linux_amd64: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-20.04' CPU: amd64 - # Linux_i386: - # # bug #17325: fails on 'ubuntu-16.04' because it now errors with: - # # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed - # vmImage: 'ubuntu-18.04' - # CPU: i386 + # regularly breaks, refs bug #17325 +# Linux_i386: +# # on 'ubuntu-16.04' (not supported anymore anyways) it errored with: +# # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed +# vmImage: 'ubuntu-18.04' +# CPU: i386 OSX_amd64: - vmImage: 'macOS-10.15' + vmImage: 'macOS-12' CPU: amd64 OSX_amd64_cpp: - vmImage: 'macOS-10.15' + vmImage: 'macOS-12' CPU: amd64 NIM_COMPILE_TO_CPP: true Windows_amd64_batch0_3: @@ -52,18 +58,24 @@ jobs: steps: - bash: git config --global core.autocrlf false displayName: 'Disable auto conversion to CRLF by git (Windows-only)' - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - checkout: self - fetchDepth: 1 + fetchDepth: 2 # see D20210329T004830 - - bash: git clone --depth 1 https://github.com/nim-lang/csources - displayName: 'Checkout Nim csources' + - bash: | + set -e + . ci/funs.sh + if nimIsCiSkip; then + echo '##vso[task.setvariable variable=skipci]true' + fi + displayName: 'Check whether to skip CI' - task: NodeTool@0 inputs: - versionSpec: '12.x' - displayName: 'Install node.js 12.x' + versionSpec: '20.x' + displayName: 'Install node.js 20.x' + condition: and(succeeded(), eq(variables['skipci'], 'false')) - bash: | set -e @@ -73,7 +85,7 @@ jobs: echo_run sudo apt-fast install --no-install-recommends -yq \ libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg displayName: 'Install dependencies (amd64 Linux)' - condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64')) + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64')) - bash: | set -e @@ -113,82 +125,44 @@ jobs: echo_run chmod 755 bin/g++ displayName: 'Install dependencies (i386 Linux)' - condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386')) + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386')) - bash: brew install boehmgc make sfml displayName: 'Install dependencies (OSX)' - condition: eq(variables['Agent.OS'], 'Darwin') + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Darwin')) - bash: | set -e . ci/funs.sh - echo_run mkdir dist - echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - echo_run 7z x dist/mingw64.7z -odist - echo_run 7z x dist/dlls.zip -obin + nimInternalInstallDepsWindows echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin' displayName: 'Install dependencies (Windows)' - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Windows_NT')) - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin' + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'Add build binaries to PATH' - - bash: | - set -e - . ci/funs.sh - echo_run echo 'PATH:' "$PATH" - echo_run echo '##[section]gcc version' - echo_run gcc -v - echo_run echo '##[section]nodejs version' - echo_run node -v - echo_run echo '##[section]make version' - echo_run make -v + - bash: . ci/funs.sh && nimCiSystemInfo + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'System information' - - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)" - displayName: 'Get csources version' + - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu=$(CPU) + condition: and(succeeded(), eq(variables['skipci'], 'false')) + displayName: 'Build csourcesAny' - - task: Cache@2 - inputs: - key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)' - path: csources/bin - displayName: 'Restore built csources' + # this could be revived if performance justifies it (needs a few updates) + # - task: Cache@2 + # inputs: + # key: 'csourcesAny | "$(Agent.OS)" | $(CPU) | $(csources_version)' + # path: $(nim_csources) + # condition: and(succeeded(), eq(variables['skipci'], 'false')) + # displayName: 'Restore built csourcesAny' - - bash: | - set -e - . ci/funs.sh - ncpu= - ext= - case '$(Agent.OS)' in - 'Linux') - ncpu=$(nproc) - ;; - 'Darwin') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows_NT') - ncpu=$NUMBER_OF_PROCESSORS - ext=.exe - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - - if [[ -x csources/bin/nim$ext ]]; then - echo_run echo "Found cached compiler, skipping build" - else - echo_run make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no - fi - - echo_run cp csources/bin/nim$ext bin - displayName: 'Build 1-stage compiler from csources' - - - bash: nim c koch - displayName: 'Build koch' - - # set result to omit the "bash exited with error code '1'" message - - bash: ./koch runCI || echo '##vso[task.complete result=Failed]' - displayName: 'Run CI' + - bash: . ci/funs.sh && nimInternalBuildKochAndRunCI + # would could also show on failure: echo '##vso[task.complete result=Failed]' + condition: and(succeeded(), eq(variables['skipci'], 'false')) + displayName: 'koch, Run CI' env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/bin/nim-gdb b/bin/nim-gdb index f7f9a79c62..d4d60e4320 100755 --- a/bin/nim-gdb +++ b/bin/nim-gdb @@ -7,14 +7,14 @@ which nim > /dev/null || (echo "nim not in PATH"; exit 1) which gdb > /dev/null || (echo "gdb not in PATH"; exit 1) which readlink > /dev/null || (echo "readlink not in PATH."; exit 1) -if [[ "$(uname -s)" == "Darwin" || "(uname -s)" == *"BSD" ]]; then +if [[ $(uname -s) == Darwin || $(uname -s) == *BSD ]]; then NIM_SYSROOT=$(dirname $(dirname $(readlink -f $(which nim)))) else NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim)))) fi # Find out where the pretty printer Python module is -GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py" +GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/debug/nim-gdb.py" # Run GDB with the additional arguments that load the pretty printers # Set the environment variable `NIM_GDB` to overwrite the call to a diff --git a/bin/nim-gdb.bash b/bin/nim-gdb.bash deleted file mode 120000 index f6eba4b0c4..0000000000 --- a/bin/nim-gdb.bash +++ /dev/null @@ -1 +0,0 @@ -nim-gdb \ No newline at end of file diff --git a/bin/nim-gdb.bat b/bin/nim-gdb.bat index e98a2063c8..b5537dd9d3 100644 --- a/bin/nim-gdb.bat +++ b/bin/nim-gdb.bat @@ -3,7 +3,7 @@ for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i) for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi) -set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py +set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\debug\nim-gdb.py set @NIM_GDB=gdb.exe @echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt diff --git a/build_all.bat b/build_all.bat index 7967e2584d..ef4a5f6a4e 100644 --- a/build_all.bat +++ b/build_all.bat @@ -1,17 +1,29 @@ @echo off -rem build development version of the compiler; can be rerun safely -if not exist csources ( - git clone --depth 1 https://github.com/nim-lang/csources.git +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% ) -if not exist bin\nim.exe ( - cd csources - if PROCESSOR_ARCHITECTURE == AMD64 ( + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( SET ARCH=64 ) CALL build.bat cd .. + copy /y bin\nim.exe %nim_csources% ) -bin\nim.exe c --skipUserCfg --skipParentCfg koch -koch.exe boot -d:release --skipUserCfg --skipParentCfg -koch.exe tools --skipUserCfg --skipParentCfg - + bin\nim.exe c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch + koch boot -d:release --skipUserCfg --skipParentCfg --hints:off + koch tools --skipUserCfg --skipParentCfg --hints:off diff --git a/build_all.sh b/build_all.sh index e66980e569..83848f41a1 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,52 +1,17 @@ -#! /bin/sh +#!/bin/sh +# DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` # build development version of the compiler; can be rerun safely. -# arguments can be passed, e.g. `--os freebsd` +# arguments can be passed, e.g.: +# CC=gcc ucpu=amd64 uos=darwin set -u # error on undefined variables set -e # exit on first error -echo_run(){ - echo "$*" - "$@" -} +. ci/funs.sh +nimBuildCsourcesIfNeeded "$@" -[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git - -nim_csources=bin/nim_csources - -build_nim_csources_via_script(){ - echo_run cd csources - echo_run sh build.sh "$@" -} - -build_nim_csources(){ - # avoid changing dir in case of failure - ( - if [ $# -ne 0 ]; then - # some args were passed (e.g.: `--cpu i386`), need to call build.sh - build_nim_csources_via_script "$@" - else - # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s) - makeX=make - unamestr=$(uname) - if [ "$unamestr" = 'FreeBSD' ]; then - makeX=gmake - fi - nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1) - which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script - fi - ) - # keep $nim_csources in case needed to investigate bootstrap issues - # without having to rebuild from csources - echo_run cp bin/nim $nim_csources -} - -[ -f $nim_csources ] || echo_run build_nim_csources $@ - -# Note: if fails, may need to `cd csources && git pull` -echo_run bin/nim c --skipUserCfg --skipParentCfg koch - -echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg -echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools. +echo_run bin/nim c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch +echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off +echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off diff --git a/changelog.md b/changelog.md index 219a66ae7a..a137b3e426 100644 --- a/changelog.md +++ b/changelog.md @@ -1,313 +1,110 @@ -# v1.6.x - yyyy-mm-dd +# v2.2.0 - yyyy-mm-dd +## Changes affecting backward compatibility + +- `-d:nimStrictDelete` becomes the default. An index error is produced when the index passed to `system.delete` was out of bounds. Use `-d:nimAuditDelete` to mimic the old behavior for backwards compatibility. +- The default user-agent in `std/httpclient` has been changed to `Nim-httpclient/` instead of `Nim httpclient/` which was incorrect according to the HTTP spec. +- Methods now support implementations based on a VTable by using `--experimental:vtables`. Methods are then confined to be in the same module where their type has been defined. +- With `-d:nimPreviewNonVarDestructor`, non-var destructors become the default. +- A bug where tuple unpacking assignment with a longer tuple on the RHS than the LHS was allowed has been fixed, i.e. code like: + ```nim + var a, b: int + (a, b) = (1, 2, 3, 4) + ``` + will no longer compile. +- `internalNew` is removed from system, use `new` instead. + +- `bindMethod` in `std/jsffi` is deprecated, don't use it with closures. ## Standard library additions and changes -- Make custom op in macros.quote work for all statements. +[//]: # "Changes:" -- On Windows the SSL library now checks for valid certificates. - It uses the `cacert.pem` file for this purpose which was extracted - from `https://curl.se/ca/cacert.pem`. Besides - the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you - now also need to ship `cacert.pem` with your `.exe` file. +- Changed `std/osfiles.copyFile` to allow to specify `bufferSize` instead of a hardcoded one. +- Changed `std/osfiles.copyFile` to use `POSIX_FADV_SEQUENTIAL` hints for kernel-level aggressive sequential read-aheads. +- `std/htmlparser` has been moved to a nimble package, use `nimble` or `atlas` to install it. +[//]: # "Additions:" -- Make `{.requiresInit.}` pragma to work for `distinct` types. +- Added `newStringUninit` to system, which creates a new string of length `len` like `newString` but with uninitialized content. +- Added `setLenUninit` to system, which doesn't initalize +slots when enlarging a sequence. +- Added `hasDefaultValue` to `std/typetraits` to check if a type has a valid default value. +- Added Viewport API for the JavaScript targets in the `dom` module. +- Added `toSinglyLinkedRing` and `toDoublyLinkedRing` to `std/lists` to convert from `openArray`s. +- ORC: To be enabled via `nimOrcStats` there is a new API called `GC_orcStats` that can be used to query how many + objects the cyclic collector did free. If the number is zero that is a strong indicator that you can use `--mm:arc` + instead of `--mm:orc`. +- A `$` template is provided for `Path` in `std/paths`. -- Added a macros `enumLen` for returning the number of items in an enum to the - `typetraits.nim` module. +[//]: # "Deprecations:" -- `prelude` now works with the JavaScript target. - Added `sequtils` import to `prelude`. - `prelude` can now be used via `include std/prelude`, but `include prelude` still works. +- Deprecates `system.newSeqUninitialized`, which is replaced by `newSeqUninit`. -- Added `almostEqual` in `math` for comparing two float values using a machine epsilon. - -- Added `clamp` in `math` which allows using a `Slice` to clamp to a value. - -- The JSON module can now handle integer literals and floating point literals of - arbitrary length and precision. - Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are - kept as string literals and one can use external BigNum libraries to handle these. - The `parseFloat` family of functions also has now optional `rawIntegers` and - `rawFloats` parameters that can be used to enforce that all integer or float - literals remain in the "raw" string form so that client code can easily treat - small and large numbers uniformly. - -- Added `BackwardsIndex` overload for `JsonNode`. - -- added `jsonutils.jsonTo` overload with `opt = Joptions()` param. - -- `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64` - instead of raising (as in 1.4) or giving wrong results (as in 1.2). - -- Added an overload for the `collect` macro that inferes the container type based - on the syntax of the last expression. Works with std seqs, tables and sets. - -- Added `randState` template that exposes the default random number generator. - Useful for library authors. - -- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum. - Added `items` for enums with holes. - Added `symbolName` to return the enum symbol name ignoring the human readable name. - -- Added `typetraits.HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. - -- Removed deprecated `iup` module from stdlib, it has already moved to - [nimble](https://github.com/nim-lang/iup). - -- various functions in `httpclient` now accept `url` of type `Uri`. Moreover `request` function's - `httpMethod` argument of type `string` was deprecated in favor of `HttpMethod` enum type. - -- `nodejs` backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`. - -- Added `cmpMem` to `system`. - -- `doAssertRaises` now correctly handles foreign exceptions. - -- Added `asyncdispatch.activeDescriptors` that returns the number of currently - active async event handles/file descriptors. - -- `--gc:orc` is now 10% faster than previously for common workloads. If - you have trouble with its changed behavior, compile with `-d:nimOldOrc`. - - -- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`, - determining preferred I/O block size for this file object. - -- Added a simpler to use `io.readChars` overload. - -- `repr` now doesn't insert trailing newline; previous behavior was very inconsistent, - see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior. - -- Added `**` to jsffi. - -- `writeStackTrace` is available in JS backend now. - -- Added `decodeQuery` to `std/uri`. - -- `strscans.scanf` now supports parsing single characters. - -- `strscans.scanTuple` added which uses `strscans.scanf` internally, - returning a tuple which can be unpacked for easier usage of `scanf`. - -- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`, - if the iterable yields a built-in settable type. - -- Added `setutils.fullSet` which returns a full built-in `set` for a valid type. - -- Added `setutils.complement` which returns the complement of a built-in `set`. - -- Added `setutils.[]=`. - -- Added `math.isNaN`. - -- `echo` and `debugEcho` will now raise `IOError` if writing to stdout fails. Previous behavior - silently ignored errors. See #16366. Use `-d:nimLegacyEchoNoRaise` for previous behavior. - -- Added `jsbigints` module, arbitrary precision integers for JavaScript target. - -- Added `math.copySign`. - -- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList` - and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements - shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes - its argument, `addMoved`, is also supplied. - -- Added `euclDiv` and `euclMod` to `math`. - -- Added `httpcore.is1xx` and missing HTTP codes. - -- Added `jsconsole.jsAssert` for JavaScript target. - -- Added `posix_utils.osReleaseFile` to get system identification from `os-release` file on Linux and the BSDs. - https://www.freedesktop.org/software/systemd/man/os-release.html - -- `math.round` now is rounded "away from zero" in JS backend which is consistent - with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior. - -- Added `socketstream` module that wraps sockets in the stream interface - -- Changed the behavior of `uri.decodeQuery` when there are unencoded `=` - characters in the decoded values. Prior versions would raise an error. This is - no longer the case to comply with the HTML spec and other languages - implementations. Old behavior can be obtained with - `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same - underlying code is also updated the same way. - -- Added `sugar.dumpToString` which improves on `sugar.dump`. - -- Added `math.signbit`. - -- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` iterators (it never worked reliably) - -- In `lists`: renamed `append` to `add` and retained `append` as an alias; - added `prepend` and `prependMoved` analogously to `add` and `addMoved`; - added `remove` for `SinglyLinkedList`s. - -- Deprecated `any`. See https://github.com/nim-lang/RFCs/issues/281 - -- Added `std/sysrand` module to get random numbers from a secure source - provided by the operating system. - -- Added optional `options` argument to `copyFile`, `copyFileToDir`, and - `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are - followed (copy files symlinks point to); on Windows, `options` argument is - ignored and symlinks are skipped. - -- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as - symlinks (instead of skipping them as it was before); on Windows symlinks are - skipped. - -- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks - (instead of skipping them sometimes as it was before). - -- Added optional `followSymlinks` argument to `setFilePermissions`. - -- Added `os.isAdmin` to tell whether the caller's process is a member of the - Administrators local group (on Windows) or a root (on POSIX). - -- Added `random.initRand()` overload with no argument which uses the current time as a seed. - -- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C). - -- Added `compilesettings.SingleValueSetting.libPath`. - -- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding - issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages) - for modules importing `std/wrapnils`. - Added `??.` macro which returns an `Option`. - -- Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. - -- `parseopt.initOptParser` has been made available and `parseopt` has been - added back to `prelude` for all backends. Previously `initOptParser` was - unavailable if the `os` module did not have `paramCount` or `paramStr`, - but the use of these in `initOptParser` were conditionally to the runtime - arguments passed to it, so `initOptParser` has been changed to raise - `ValueError` when the real command line is not available. `parseopt` was - previously excluded from `prelude` for JS, as it could not be imported. - -- On POSIX systems, the default signal handlers used for Nim programs (it's - used for printing the stacktrace on fatal signals) will now re-raise the - signal for the OS default handlers to handle. - - This lets the OS perform its default actions, which might include core - dumping (on select signals) and notifying the parent process about the cause - of termination. - -- Added `system.prepareStrMutation` for better support of low - level `moveMem`, `copyMem` operations for Orc's copy-on-write string - implementation. - -- `hashes.hash` now supports `object`, but can be overloaded. - -- Added `std/strbasics` for high performance string operations. - Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. - -- `hashes.hash` now supports `object`, but can be overloaded. - -- Added to `wrapnils` an option-like API via `??.`, `isSome`, `get`. - -- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` - and `$none(int)` to `"none(int)"` instead of `"None[int]"`. - -- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target. - -- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target. - -- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) wrapper for JavaScript target. - -- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible - with most other languages. - -- Removed support for named procs in `sugar.=>`. - -- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). - - -- Added `std/channels`. - -- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). - -- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time - zone offsets without colons, e.g. `UTC+7 -> +0700`. - -- In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`, - unless `-d:nimLegacyHomeDir` is specified (for a transition period). - -- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. +[//]: # "Removals:" ## Language changes -- `nimscript` now handles `except Exception as e`. +- `noInit` can be used in types and fields to disable member initializers in the C++ backend. +- C++ custom constructors initializers see https://nim-lang.org/docs/manual_experimental.htm#constructor-initializer +- `member` can be used to attach a procedure to a C++ type. +- C++ `constructor` now reuses `result` instead creating `this`. -- The `cstring` doesn't support `[]=` operator in JS backend. +- Tuple unpacking changes: + - Tuple unpacking assignment now supports using underscores to discard values. + ```nim + var a, c: int + (a, _, c) = (1, 2, 3) + ``` + - Tuple unpacking variable declarations now support type annotations, but + only for the entire tuple. + ```nim + let (a, b): (int, int) = (1, 2) + let (a, (b, c)): (byte, (float, cstring)) = (1, (2, "abc")) + ``` -- nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. +- An experimental option `genericsOpenSym` has been added to allow captured + symbols in generic routine bodies to be replaced by symbols injected locally + by templates/macros at instantiation time. `bind` may be used to keep the + captured symbols over the injected ones regardless of enabling the option. -- `typetraits.distinctBase` now is identity instead of error for non distinct types. + Since this change may affect runtime behavior, the experimental switch + `genericsOpenSym` needs to be enabled, and a warning is given in the case + where an injected symbol would replace a captured symbol not bound by `bind` + and the experimental switch isn't enabled. -- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; - use `-d:nimLegacyCopyFile` for OSX < 10.5. + ```nim + const value = "captured" + template foo(x: int, body: untyped) = + let value {.inject.} = "injected" + body -- The required name of case statement macros for the experimental - `caseStmtMacros` feature has changed from `match` to `` `case` ``. + proc old[T](): string = + foo(123): + return value # warning: a new `value` has been injected, use `bind` or turn on `experimental:genericsOpenSym` + echo old[int]() # "captured" -- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages. + {.experimental: "genericsOpenSym".} + proc bar[T](): string = + foo(123): + return value + assert bar[int]() == "injected" # previously it would be "captured" + proc baz[T](): string = + bind value + foo(123): + return value + assert baz[int]() == "captured" + ``` ## Compiler changes -- Added `--declaredlocs` to show symbol declaration location in messages. - -- Deprecated `TaintedString` and `--taintmode`. - -- Deprecated `--nilseqs` which is now a noop. - -- Added `--spellSuggest` to show spelling suggestions on typos. - -- Source+Edit links now appear on top of every docgen'd page when - `nim doc --git.url:url ...` is given. - -- Added `nim --eval:cmd` to evaluate a command directly, see `nim --help`. - -- VM now supports `addr(mystring[ind])` (index + index assignment) - -- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous - behavior. - -- Added `--hintAsError` with similar semantics as `--warningAsError`. - -- TLS: OSX now uses native TLS (`--tlsEmulation:off`), TLS now works with importcpp non-POD types, - such types must use `.cppNonPod` and `--tlsEmulation:off`should be used. - -- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists, for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`. - -- docgen: rst files can now use single backticks instead of double backticks and correctly render - in both rst2html (as before) as well as common tools rendering rst directly (e.g. github), by - adding: `default-role:: code` directive inside the rst file, which is now handled by rst2html. - -- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have certain hints/warnings - -- Added `then`, `catch` to `asyncjs`, for now hidden behind `-d:nimExperimentalAsyncjsThen`. - -- `--newruntime` and `--refchecks` are deprecated. - -- Added `unsafeIsolate` and `extract` to `std/isolation`. - - +- `--nimcache` using a relative path as the argument in a config file is now relative to the config file instead of the current directory. ## Tool changes -- The rst parser now supports markdown table syntax. - Known limitations: - - cell alignment is not supported, i.e. alignment annotations in a delimiter - row (`:---`, `:--:`, `---:`) are ignored, - - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`. +- koch now allows bootstrapping with `-d:nimHasLibFFI`, replacing the older option of building the compiler directly w/ the `libffi` nimble package in tow. -- `fusion` is now un-bundled from nim, `./koch fusion` will - install it via nimble at a fixed hash. diff --git a/changelogs/changelog_0_19_0.md b/changelogs/changelog_0_19_0.md index 18d3ca2b33..45cefe933c 100644 --- a/changelogs/changelog_0_19_0.md +++ b/changelogs/changelog_0_19_0.md @@ -15,7 +15,7 @@ at runtime. Use ``ref object`` with inheritance rather than ``object`` with inheritance to prevent this issue. - The ``not nil`` type annotation now has to be enabled explicitly - via ``{.experimental: "notnil"}`` as we are still not pleased with how this + via ``{.experimental: "notnil".}`` as we are still not pleased with how this feature works with Nim's containers. - The parser now warns about inconsistent spacing around binary operators as these can easily be confused with unary operators. This warning will likely diff --git a/changelogs/changelog_1_4_0.md b/changelogs/changelog_1_4_0.md index 091048ad19..77f66ffdeb 100644 --- a/changelogs/changelog_1_4_0.md +++ b/changelogs/changelog_1_4_0.md @@ -282,7 +282,7 @@ - Removed the deprecated `asyncdispatch.newAsyncNativeSocket`. - Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`. -- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour. +- Removed `sharedlist.initSharedList`, was deprecated and produces undefined behaviour. - There is a new experimental feature called "strictFuncs" which makes the definition of `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs) diff --git a/changelogs/changelog_1_6_0.md b/changelogs/changelog_1_6_0.md new file mode 100644 index 0000000000..4a1047d205 --- /dev/null +++ b/changelogs/changelog_1_6_0.md @@ -0,0 +1,957 @@ +# v1.6.0 - 2021-10-14 + + + +# Major new features + +With so many new features, pinpointing the most salient ones is a subjective exercise, +but here are a select few: + + +## `iterable[T]` + +The `iterable[T]` type class was added to match called iterators, +which solves a number of long-standing issues related to iterators. +Example: + +```nim +iterator iota(n: int): int = + for i in 0.. response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) +discard fn() +``` + + +## New `std/tasks` module + +Provides basic primitives for creating parallel programs. +Example: +```nim +import std/tasks +var num = 0 +proc hello(a: int) = num+=a + +let b = toTask hello(13) # arguments must be isolated, see `std/isolation` +b.invoke() +assert num == 13 +b.invoke() # can be called again +assert num == 26 +``` + + +## New module: `std/genasts` + +Provides an API `genAst` that avoids the problems inherent with `quote do` and can +be used as a replacement. +Example showing how this could be used for writing a simplified version of `unittest.check`: +```nim +import std/[genasts, macros, strutils] +macro check2(cond: bool): untyped = + assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind + result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): + # each local symbol we access must be explicitly captured + if not cond: + doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] +let a = 3 +check2 a*2 == a+3 +if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' +``` + +See PR [#17426](https://github.com/nim-lang/Nim/pull/17426) for details. + + +## New module: `std/setutils` + +- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`, + if the iterable yields a built-in settable type. +- Added `setutils.fullSet` which returns a full built-in `set` for a valid type. +- Added `setutils.complement` which returns the complement of a built-in `set`. +- Added `setutils.[]=`. + + +## New module: `std/enumutils` + +- Added `genEnumCaseStmt` macro that generates + case statement to parse string to enum. +- Added `items` for enums with holes. +- Added `symbolName` to return the `enum` symbol name ignoring the human-readable name. +- Added `symbolRank` to return the index in which an `enum` member is listed in an enum. + + +## `system` + +- Added `system.prepareMutation` for better support of low + level `moveMem`, `copyMem` operations for `gc:orc`'s copy-on-write string + implementation. +- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible + with most other languages. +- Added `cmpMem` to `system`. +- `doAssertRaises` now correctly handles foreign exceptions. +- `addInt` now supports unsigned integers. + +Compatibility notes: +- `system.delete` had surprising behavior when the index passed to it was out of + bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so + that an index error is produced instead. Be aware however that your code might depend on + this quirky behavior so a review process is required on your part before you can + use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete` + switch, which pretends that `system.delete` is deprecated so that it is easier + to see where it was used in your code. + `-d:nimStrictDelete` will become the default in upcoming versions. +- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`. + Please use `char` or `uint8` instead. +- `repr` now doesn't insert trailing newlines; the previous behavior was very inconsistent, + see [#16034](https://github.com/nim-lang/Nim/pull/16034). + Use `-d:nimLegacyReprWithNewline` for the previous behavior. + `repr` now also renders ASTs correctly for user defined literals, setters, `do`, etc. +- Deprecated `any`. See RFC [#281](https://github.com/nim-lang/RFCs/issues/281). +- The unary slice `..b` was deprecated, use `0..b` instead. + + +## `std/math` + +- Added `almostEqual` for comparing two float values using a machine epsilon. +- Added `clamp` which allows using a `Slice` to clamp to a value. +- Added `ceilDiv` for integer division that rounds up. +- Added `isNaN`. +- Added `copySign`. +- Added `euclDiv` and `euclMod`. +- Added `signbit`. +- Added `frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. + +Compatibility notes: +- `math.round` now rounds "away from zero" in the JS backend, which is consistent + with other backends. See [#9125](https://github.com/nim-lang/Nim/pull/9125). + Use `-d:nimLegacyJsRound` for the previous behavior. + + +## Random number generators: `std/random`, `std/sysrand`, `std/oids` + +- Added `std/sysrand` module (see details above). +- Added `randState` template that exposes the default random number generator. + Useful for library authors. +- Added `initRand()` overload with no argument which uses the current time as a seed. +- `initRand(seed)` now allows `seed == 0`. +- Fixed overflow bugs. +- Fix `initRand` to avoid random number sequences overlapping, refs + [#18744](https://github.com/nim-lang/Nim/pull/18744). +- `std/oids` now uses `std/random`. + +Compatibility notes: +- Deprecated `std/mersenne`. +- `random.initRand(seed)` now produces non-skewed values for the first call to + `rand()` after initialization with a small (< 30000) seed. + Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition + time, see PR [#17467](https://github.com/nim-lang/Nim/pull/17467). + + +## `std/json`, `std/jsonutils` + +- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize + holey enums as regular enums (via `ord`) instead of as strings. + It is expected that this behavior becomes the new default in upcoming versions. + `toJson` now serializes `JsonNode` as is via reference (without a deep copy) + instead of treating `JsonNode` as a regular ref object, + this can be customized via `jsonNodeMode`. +- `std/json` and `std/jsonutils` now serialize `NaN`, `Inf`, `-Inf` as strings, + so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` + which was invalid JSON. +- `std/json` can now handle integer literals and floating point literals of + arbitrary length and precision. + Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are + kept as string literals and one can use external BigNum libraries to handle these. + The `parseFloat` family of functions also has now optional `rawIntegers` and + `rawFloats` parameters that can be used to enforce that all integer or float + literals remain in the "raw" string form so that client code can easily treat + small and large numbers uniformly. +- Added `BackwardsIndex` overload for `JsonNode`. +- `json.%`,`json.to`, `jsonutils.fromJson`,`jsonutils.toJson` now work with `uint|uint64` + instead of raising (as in 1.4) or giving wrong results (as in 1.2). +- `std/jsonutils` now handles `cstring` (including as Table key), and `set`. +- Added `jsonutils.jsonTo` overload with `opt = Joptions()` param. +- `jsonutils.toJson` now supports customization via `ToJsonOptions`. +- `std/json`, `std/jsonutils` now support round-trip serialization when + `-d:nimPreviewFloatRoundtrip` is used. + + +## `std/typetraits`, `std/compilesettings` + +- `distinctBase` now is identity instead of error for non distinct types. +- `distinctBase` now allows controlling whether to be recursive or not. +- Added `enumLen` to return the number of elements in an enum. +- Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. +- Added `hasClosure`. +- Added `pointerBase` to return `T` for `ref T | ptr T`. +- Added `compilesettings.SingleValueSetting.libPath`. + + +## networking: `std/net`, `std/asyncnet`, `std/htmlgen`, `std/httpclient`, `std/asyncdispatch`, `std/asynchttpserver`, `std/httpcore` + +- Fixed buffer overflow bugs in `std/net`. +- Exported `sslHandle` from `std/net` and `std/asyncnet`. +- Added `hasDataBuffered` to `std/asyncnet`. +- Various functions in `std/httpclient` now accept `url` of type `Uri`. + Moreover, the `request` function's `httpMethod` argument of type `string` was + deprecated in favor of `HttpMethod` `enum` type; see + [#15919](https://github.com/nim-lang/Nim/pull/15919). +- Added `asyncdispatch.activeDescriptors` that returns the number of currently + active async event handles/file descriptors. +- Added `getPort` to `std/asynchttpserver` to resolve OS-assigned `Port(0)`; + this is usually recommended instead of hardcoding ports which can lead to + "Address already in use" errors. +- Fixed premature garbage collection in `std/asyncdispatch`, when a stacktrace + override is in place. +- Added `httpcore.is1xx` and missing HTTP codes. +- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). + +Compatibility notes: +- On Windows, the SSL library now checks for valid certificates. + For this purpose it uses the `cacert.pem` file, which was extracted + from `https://curl.se/ca/cacert.pem`. Besides + the OpenSSL DLLs (e.g. `libssl-1_1-x64.dll`, `libcrypto-1_1-x64.dll`) you + now also need to ship `cacert.pem` with your `.exe` file. + + +## `std/hashes` + +- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code), + if `-d:nimPreviewHashRef` is used. It is expected that this behavior + becomes the new default in upcoming versions. +- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`. + `hashes.hash(closure)` has also been improved. + + +## OS: `std/os`, `std/io`, `std/socketstream`, `std/linenoise`, `std/tempfiles` + +- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`, + determining preferred I/O block size for this file object. +- Added `os.getCacheDir()` to return platform specific cache directory. +- Improved `os.getTempDir()`, see PR [#16914](https://github.com/nim-lang/Nim/pull/16914). +- Added `os.isAdmin` to tell whether the caller's process is a member of the + Administrators local group (on Windows) or a root (on POSIX). +- Added optional `options` argument to `copyFile`, `copyFileToDir`, and + `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are + followed (copy files symlinks point to); on Windows, `options` argument is + ignored and symlinks are skipped. +- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as + symlinks (instead of skipping them as it was before); on Windows symlinks are + skipped. +- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks + (instead of skipping them sometimes as it was before). +- Added optional `followSymlinks` argument to `setFilePermissions`. +- Added a simpler to use `io.readChars` overload. +- Added `socketstream` module that wraps sockets in the stream interface. +- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C). + + +## Environment variable handling + +- Empty environment variable values are now supported across OS's and backends. +- Environment variable APIs now work in multithreaded scenarios, by delegating to + direct OS calls instead of trying to keep track of the environment. +- `putEnv`, `delEnv` now work at compile time. +- NodeJS backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`. + +Compatibility notes: +- `std/os`: `putEnv` now raises if the first argument contains a `=`. + + +## POSIX + +- On POSIX systems, the default signal handlers used for Nim programs (it's + used for printing the stacktrace on fatal signals) will now re-raise the + signal for the OS default handlers to handle. + This lets the OS perform its default actions, which might include core + dumping (on select signals) and notifying the parent process about the cause + of termination. +- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler` + for previous behavior. +- Added `posix_utils.osReleaseFile` to get system identification from `os-release` + file on Linux and the BSDs. + ([link](https://www.freedesktop.org/software/systemd/man/os-release.html)) +- Removed undefined behavior for `posix.open`. + + +## `std/prelude` + +- `std/strformat` is now part of `include std/prelude`. +- Added `std/sequtils` import to `std/prelude`. +- `std/prelude` now works with the JS target. +- `std/prelude` can now be used via `include std/prelude`, but `include prelude` still works. + + +## String manipulation: `std/strformat`, `std/strbasics` + +- Added support for parenthesized expressions. +- Added support for const strings instead of just string literals. +- Added `std/strbasics` for high-performance string operations. +- Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. + + +## `std/wrapnils` + +- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding + issues like bug [#13063](https://github.com/nim-lang/Nim/issues/13063) + (which affected error messages) for modules importing `std/wrapnils`. +- Added `??.` macro which returns an `Option`. +- `std/wrapnils` can now be used to protect against `FieldDefect` errors in + case objects, generates optimal code (no overhead compared to manual + if-else branches), and preserves lvalue semantics which allows modifying + an expression. + + +## Containers: `std/algorithm`, `std/lists`, `std/sequtils`, `std/options`, `std/packedsets` + +- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` + iterators (it never worked reliably). +- Added `algorithm.merge`. +- In `std/lists`: renamed `append` to `add` and retained `append` as an alias; + added `prepend` and `prependMoved` analogously to `add` and `addMoved`; + added `remove` for `SinglyLinkedList`s. +- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList` + and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements + shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes + its argument, `addMoved`, is also supplied. + See PRs [#16362](https://github.com/nim-lang/Nim/pull/16362), + [#16536](https://github.com/nim-lang/Nim/pull/16536). +- New module: `std/packedsets`. + Generalizes `std/intsets`, see PR [#15564](https://github.com/nim-lang/Nim/pull/15564). + +Compatibility notes: +- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises + a defect if the slice is out of bounds, likewise with `strutils.delete`. +- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` + in `std/algorithm`. +- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` + and `$none(int)` to `"none(int)"` instead of `"None[int]"`. + + +## `std/times` + +- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time + zone offsets without colons, e.g. `UTC+7 -> +0700`. +- Added `dateTime` and deprecated `initDateTime`. + + +## `std/macros` and AST + +- New module `std/genasts`, see description above. +- The required name of case statement macros for the experimental + `caseStmtMacros` feature has changed from `match` to `` `case` ``. +- Tuple expressions are now parsed consistently as + `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`. +- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes + in a collapsed way. + Use `-d:nimLegacyMacrosCollapseSymChoice` to get the previous behavior. +- Made custom op in `macros.quote` work for all statements. + + +## `std/sugar` + +- Added `sugar.dumpToString` which improves on `sugar.dump`. +- Added an overload for the `collect` macro that infers the container type based + on the syntax of the last expression. Works with std seqs, tables and sets. + +Compatibility notes: +- Removed support for named procs in `sugar.=>`. + + +## Parsing: `std/parsecfg`, `std/strscans`, `std/uri` + +- Added `sections` iterator in `parsecfg`. +- `strscans.scanf` now supports parsing single characters. +- Added `strscans.scanTuple` which uses `strscans.scanf` internally, + returning a tuple which can be unpacked for easier usage of `scanf`. +- Added `decodeQuery` to `std/uri`. +- `parseopt.initOptParser` has been made available and `parseopt` has been + added back to `std/prelude` for all backends. Previously `initOptParser` was + unavailable if the `std/os` module did not have `paramCount` or `paramStr`, + but the use of these in `initOptParser` were conditionally to the runtime + arguments passed to it, so `initOptParser` has been changed to raise + `ValueError` when the real command line is not available. `parseopt` was + previously excluded from `std/prelude` for JS, as it could not be imported. + +Compatibility notes: +- Changed the behavior of `uri.decodeQuery` when there are unencoded `=` + characters in the decoded values. Prior versions would raise an error. This is + no longer the case to comply with the HTML spec and other languages' + implementations. Old behavior can be obtained with + `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same + underlying code is also updated the same way. + + +## JS stdlib changes + +- Added `std/jsbigints` module, which provides arbitrary precision integers for the JS target. +- Added `setCurrentException` for the JS backend. +- `writeStackTrace` is available in the JS backend now. +- Added `then`, `catch` to `std/asyncjs` for promise pipelining, for now hidden + behind `-d:nimExperimentalAsyncjsThen`. +- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) + wrapper for the JS target. +- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) + wrapper for the JS target. +- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) + wrapper for the JS target. +- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). +- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. +- Added dollar `$` and `len` for `jsre.RegExp`. +- Added `jsconsole.jsAssert` for the JS target. +- Added `**` to `std/jsffi`. +- Added `copyWithin` [for `seq` and `array` for JS targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin). +- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. + Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. +- Added `dom.scrollIntoView` proc with options. +- Added `dom.setInterval`, `dom.clearInterval` overloads. +- Merged `std/dom_extensions` into the `std/dom` module, + as it was a module with a single line, see RFC [#413](https://github.com/nim-lang/RFCs/issues/413). +- `$` now gives more correct results on the JS backend. + + +## JS compiler changes + +- `cstring` doesn't support the `[]=` operator anymore in the JS backend. +- Array literals now use JS typed arrays when the corresponding JS typed array exists, + for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`. + + +## VM and nimscript backend + +- VM now supports `addr(mystring[ind])` (index + index assignment). +- `nimscript` now handles `except Exception as e`. +- `nil` dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. +- `static[T]` now works better, refs [#17590](https://github.com/nim-lang/Nim/pull/17590), + [#15853](https://github.com/nim-lang/Nim/pull/15853). +- `distinct T` conversions now work in VM. +- `items(cstring)` now works in VM. +- Fix `addr`, `len`, `high` in VM ([#16002](https://github.com/nim-lang/Nim/pull/16002), + [#16610](https://github.com/nim-lang/Nim/pull/16610)). +- `std/cstrutils` now works in VM. + + +## OS/platform-specific notes + +- Support for Apple silicon/M1. +- Support for 32-bit RISC-V, refs [#16231](https://github.com/nim-lang/Nim/pull/16231). +- Support for armv8l, refs [#18901](https://github.com/nim-lang/Nim/pull/18901). +- Support for CROSSOS, refs [#18889](https://github.com/nim-lang/Nim/pull/18889). +- The allocator for Nintendo Switch, which was nonfunctional because + of breaking changes in libnx, was removed, in favor of the new `-d:nimAllocPagesViaMalloc` option. +- Allow reading parameters when compiling for Nintendo Switch. +- `--nimcache` now correctly works in a cross-compilation setting. +- Cross compilation targeting Windows was improved. +- This now works from macOS/Linux: `nim r -d:mingw main` + + + +## Performance / memory optimizations + +- The comment field in PNode AST was moved to a side channel, reducing overall + memory usage during compilation by a factor 1.25x +- `std/jsonutils` deserialization is now up to 20x faster. +- `os.copyFile` is now 2.5x faster on macOS, by using `copyfile` from `copyfile.h`; + use `-d:nimLegacyCopyFile` for macOS < 10.5. +- Float to string conversion is now 10x faster thanks to the Dragonbox algorithm, + with `-d:nimPreviewFloatRoundtrip`. +- `newSeqWith` is 3x faster. +- CI now supports batching (making Windows CI 2.3X faster). +- Sets now uses the optimized `countSetBits` proc, see PR + [#17334](https://github.com/nim-lang/Nim/pull/17334). + + +## Debugging + +- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. +- `koch tools` now builds `bin/nim_dbg` which allows easy access to a debug version + of Nim without recompiling. +- Added new module `compiler/debugutils` to help with debugging Nim compiler. +- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints` and + used it in more contexts; this flag which works in tandem with `--stackTraceMsgs` + to show user code context in compiler stacktraces. + + +## Type system + +- `typeof(voidStmt)` now works and returns `void`. +- `enum` values can now be overloaded. This needs to be enabled + via `{.experimental: "overloadableEnums".}`. We hope that this feature allows + for the development of more fluent (less ugly) APIs. + See RFC [#373](https://github.com/nim-lang/RFCs/issues/373) for more details. +- A type conversion from one `enum` type to another now produces an `[EnumConv]` warning. + You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead. + ```nim + type A = enum a1, a2 + type B = enum b1, b2 + echo a1.B # produces a warning + echo a1.ord.B # produces no warning + ``` +- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning. + This warning will become an error in future versions! Use an explicit conversion + like `cstring(x)` in order to silence the warning. +- There is a new warning for *any* type conversion to `enum` that can be enabled via + `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`. +- Reusing a type name in a different scope now works, refs [#17710](https://github.com/nim-lang/Nim/pull/17710). +- Fixed implicit and explicit generics in procedures, refs [#18808](https://github.com/nim-lang/Nim/pull/18808). + + +## New-style concepts + +Example: +```nim +type + Comparable = concept # no T, an atom + proc cmp(a, b: Self): int +``` +The new design does not rely on `system.compiles` and may compile faster. +See PR [#15251](https://github.com/nim-lang/Nim/pull/15251) +and RFC [#168](https://github.com/nim-lang/RFCs/issues/168) for details. + + +## Lexical / syntactic + +- Nim now supports a small subset of Unicode operators as operator symbols. + The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". + To enable this feature, use `--experimental:unicodeOperators`. Note that due + to parser limitations you **cannot** enable this feature via a + pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable + it via the command line or in a configuration file. +- `var a {.foo.} = expr` now works inside templates (except when `foo` is overloaded). + + +## Compiler messages, error messages, hints, warnings + +- Significant improvement to error messages involving effect mismatches, + see PRs [#18384](https://github.com/nim-lang/Nim/pull/18384), + [#18418](https://github.com/nim-lang/Nim/pull/18418). +- Added `--declaredLocs` to show symbol declaration location in error messages. +- Added `--spellSuggest` to show spelling suggestions on typos. +- Added `--processing:dots|filenames|off` which customizes `hintProcessing`; + `--processing:filenames` shows which include/import modules are being compiled as an import stack. +- `FieldDefect` messages now shows discriminant value + lineinfo, in all backends (C, JS, VM) +- Added `--hintAsError` with similar semantics as `--warningAsError`. +- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31` + before a newline for every generated message (potentially multiline), + so tooling can tell when messages start and end. +- Added `--filenames:abs|canonical|legacyRelProj` which replaces `--listFullPaths:on|off` +- `--hint:all:on|off` is now supported to select or deselect all hints; it + differs from `--hints:on|off` which acts as a (reversible) gate. + Likewise with `--warning:all:on|off`. +- The style checking of the compiler now supports a `--styleCheck:usages` switch. + This switch enforces that every symbol is written as it was declared, not enforcing + the official Nim style guide. To be enabled, this has to be combined either + with `--styleCheck:error` or `--styleCheck:hint`. +- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for + previous behavior. +- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages. +- `runnableExamples` now show originating location in stacktraces on failure. +- `SuccessX` message now shows more useful information. +- New `DuplicateModuleImport` warning; improved `UnusedImport` and + `XDeclaredButNotUsed` accuracy. + +Compatibility notes: +- `--hint:CC` now prints to stderr (like all other hints) instead of stdout. + + +## Building and running Nim programs, configuration system + +- JSON build instructions are now generated in `$nimcache/outFileBasename.json` + instead of `$nimcache/projectName.json`. This allows avoiding recompiling a + given project compiled with different options if the output file differs. +- `--usenimcache` (implied by `nim r main`) now generates an output file that includes + a hash of some of the compilation options, which allows caching generated binaries: + ```bash + nim r main # recompiles + nim r -d:foo main # recompiles + nim r main # uses cached binary + nim r main arg1 arg2 # likewise (runtime arguments are irrelevant) + ``` +- `nim r` now supports cross compilation from unix to windows when specifying + `-d:mingw` by using Wine, e.g.: + `nim r --eval:'import os; echo "a" / "b"'` prints `a\b`. +- `nim` can compile version 1.4.0 as follows: + `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`. + `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel. +- `nim e` now accepts arbitrary file extensions for the nimscript file, + although `.nims` is still the preferred extension in general. +- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. + The downside is that these defines now have custom logic that doesn't apply for + other defines. + + +## Multithreading + +- TLS: macOS now uses native TLS (`--tlsEmulation:off`). TLS now works with + `importcpp` non-POD types; such types must use `.cppNonPod` and + `--tlsEmulation:off`should be used. +- Added `unsafeIsolate` and `extract` to `std/isolation`. +- Added `std/tasks`, see description above. + + +## Memory management + +- `--gc:arc` now bootstraps (PR [#17342](https://github.com/nim-lang/Nim/pull/17342)). +- Lots of improvements to `gc:arc`, `gc:orc`, see PR + [#15697](https://github.com/nim-lang/Nim/pull/15697), + [#16849](https://github.com/nim-lang/Nim/pull/16849), + [#17993](https://github.com/nim-lang/Nim/pull/17993). +- `--gc:orc` is now 10% faster than previously for common workloads. + If you have trouble with its changed behavior, compile with `-d:nimOldOrc`. +- The `--gc:orc` algorithm was refined so that custom container types can participate in the + cycle collection process. See the documentation of `=trace` for more details. +- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. + This is only supported for `--gc:orc` or `--gc:arc`. + +Compatibility notes: +- `--newruntime` and `--refchecks` are deprecated, + use `--gc:arc`, `--gc:orc`, or `--gc:none` as appropriate instead. + + +## Docgen + +- docgen: RST files can now use single backticks instead of double backticks and + correctly render in both `nim rst2html` (as before) as well as common tools rendering + RST directly (e.g. GitHub). + This is done by adding the `default-role:: code` directive inside the RST file + (which is now handled by `nim rst2html`). +- Source+Edit links now appear on top of every docgen'd page when + `nim doc --git.url:url ...` is given. +- Latex doc generation is revised: output `.tex` files should be compiled + by `xelatex` (not by `pdflatex` as before). Now default Latex settings + provide support for Unicode and better avoid margin overflows. + The minimum required version is TeXLive 2018 (or an equivalent MikTeX version). +- The RST parser now supports footnotes, citations, admonitions, and short style + references with symbols. +- The RST parser now supports Markdown table syntax. + Known limitations: + - cell alignment is not supported, i.e. alignment annotations in a delimiter + row (`:---`, `:--:`, `---:`) are ignored + - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`. +- Implemented `doc2tex` compiler command which converts documentation in + `.nim` files to Latex. +- docgen now supports syntax highlighting for inline code. +- docgen now supports same-line doc comments: + ```nim + func fn*(a: int): int = 42 ## Doc comment + ``` +- docgen now renders `deprecated` and other pragmas. +- `runnableExamples` now works with templates and nested templates. +- `runnableExamples: "-r:off"` now works for examples that should compile but not run. +- `runnableExamples` now renders code verbatim, and produces correct code in all cases. +- docgen now shows correct, canonical import paths in docs. +- docgen now shows all routines in sidebar, and the proc signature is now shown in sidebar. + + +## Effects and checks + +- Significant improvement to error messages involving effect mismatches +- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some + compiler checks regarding `case objects`. This allows serialization libraries + to avoid ugly, non-portable solutions. See RFC + [#407](https://github.com/nim-lang/RFCs/issues/407) for more details. + +Compatibility notes: +- Fixed effect tracking for borrowed procs (see [#18882](https://github.com/nim-lang/Nim/pull/18882)). + One consequence is that, under some circumstances, Nim could previously permit a procedure with side effects to be written with `func` - you may need to change some occurrences of `func` to `proc`. + To illustrate, Nim versions before 1.6.0 compile the below without error + ```nim + proc print(s: string) = + echo s + + type + MyString = distinct string + + proc print(s: MyString) {.borrow.} + + func foo(s: MyString) = + print(s) + ``` + but Nim 1.6.0 produces the error + ``` + Error: 'foo' can have side effects + ``` + similar to how we expect that + ```nim + func print(s: string) = + echo s + ``` + produces + ``` + Error: 'print' can have side effects + ``` + + +## Tools + +- Major improvements to `nimgrep`, see PR [#15612 +](https://github.com/nim-lang/Nim/pull/15612). +- `fusion` is now un-bundled from Nim, `./koch fusion` will + install it via Nimble at a fixed hash. +- `testament`: added `nimoutFull: bool` spec to compare full output of compiler + instead of a subset; many bugfixes to testament. + + +## Misc/cleanups + +- Deprecated `TaintedString` and `--taintmode`. +- Deprecated `--nilseqs` which is now a noop. +- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have + certain hints/warnings. +- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. +- `[skip ci]` now works in azure and CI pipelines, see detail in PR + [#17561](https://github.com/nim-lang/Nim/pull/17561). diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md new file mode 100644 index 0000000000..457cc62a6b --- /dev/null +++ b/changelogs/changelog_2_0_0.md @@ -0,0 +1,330 @@ +# v2.0.0 - 2023-08-01 + +Version 2.0 is a big milestone with too many changes to list them all here. + +For a full list see [details](changelog_2_0_0_details.html). + + +## New features + +### Better tuple unpacking + +Tuple unpacking for variables is now treated as syntax sugar that directly +expands into multiple assignments. Along with this, tuple unpacking for +variables can now be nested. + +```nim +proc returnsNestedTuple(): (int, (int, int), int, int) = (4, (5, 7), 2, 3) + +# Now nesting is supported! +let (x, (_, y), _, z) = returnsNestedTuple() + +``` + +### Improved type inference + +A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference) has been implemented for a variety of basic cases. + +For example, code like the following now compiles: + +```nim +let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")] +``` + +### Forbidden Tags + +[Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) now supports the definition +of forbidden tags by the `.forbids` pragma which can be used to disable certain effects in proc types. + +For example: + +```nim + +type IO = object ## input/output effect +proc readLine(): string {.tags: [IO].} = discard +proc echoLine(): void = discard + +proc no_IO_please() {.forbids: [IO].} = + # this is OK because it didn't define any tag: + echoLine() + # the compiler prevents this: + let y = readLine() + +``` + +### New standard library modules + +The famous `os` module got an overhaul. Several of its features are available +under a new interface that introduces a `Path` abstraction. A `Path` is +a `distinct string`, which improves the type safety when dealing with paths, files +and directories. + +Use: + +- `std/oserrors` for OS error reporting. +- `std/envvars` for environment variables handling. +- `std/paths` for path handling. +- `std/dirs` for directory creation/deletion/traversal. +- `std/files` for file existence checking, file deletions and moves. +- `std/symlinks` for symlink handling. +- `std/appdirs` for accessing configuration/home/temp directories. +- `std/cmdline` for reading command line parameters. + +### Consistent underscore handling + +The underscore identifier (`_`) is now generally not added to scope when +used as the name of a definition. While this was already the case for +variables, it is now also the case for routine parameters, generic +parameters, routine declarations, type declarations, etc. This means that the following code now does not compile: + +```nim +proc foo(_: int): int = _ + 1 +echo foo(1) + +proc foo[_](t: typedesc[_]): seq[_] = @[default(_)] +echo foo[int]() + +proc _() = echo "_" +_() + +type _ = int +let x: _ = 3 +``` + +Whereas the following code now compiles: + +```nim +proc foo(_, _: int): int = 123 +echo foo(1, 2) + +proc foo[_, _](): int = 123 +echo foo[int, bool]() + +proc foo[T, U](_: typedesc[T], _: typedesc[U]): (T, U) = (default(T), default(U)) +echo foo(int, bool) + +proc _() = echo "one" +proc _() = echo "two" + +type _ = int +type _ = float +``` + +### JavaScript codegen improvement + +The JavaScript backend now uses [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt) +for 64-bit integer types (`int64` and `uint64`) by default. As this affects +JS code generation, code using these types to interface with the JS backend +may need to be updated. Note that `int` and `uint` are not affected. + +For compatibility with [platforms that do not support BigInt](https://caniuse.com/bigint) +and in the case of potential bugs with the new implementation, the +old behavior is currently still supported with the command line option +`--jsbigint64:off`. + + +## Docgen improvements + +`Markdown` is now the default markup language of doc comments (instead +of the legacy `RstMarkdown` mode). In this release we begin to separate +RST and Markdown features to better follow specification of each +language, with the focus on Markdown development. +See also [the docs](https://nim-lang.github.io/Nim/markdown_rst.html). + +* Added a `{.doctype: Markdown | RST | RstMarkdown.}` pragma allowing to + select the markup language mode in the doc comments of the current `.nim` + file for processing by `nim doc`: + + 1. `Markdown` (default) is basically CommonMark (standard Markdown) + + some Pandoc Markdown features + some RST features that are missing + in our current implementation of CommonMark and Pandoc Markdown. + 2. `RST` closely follows the RST spec with few additional Nim features. + 3. `RstMarkdown` is a maximum mix of RST and Markdown features, which + is kept for the sake of compatibility and ease of migration. + +* Added separate `md2html` and `rst2html` commands for processing + standalone `.md` and `.rst` files respectively (and also `md2tex`/`rst2tex`). + +* Added Pandoc Markdown bracket syntax `[...]` for making anchor-less links. +* Docgen now supports concise syntax for referencing Nim symbols: + instead of specifying HTML anchors directly one can use original + Nim symbol declarations (adding the aforementioned link brackets + `[...]` around them). + * To use this feature across modules, a new `importdoc` directive was added. + Using this feature for referencing also helps to ensure that links + (inside one module or the whole project) are not broken. +* Added support for RST & Markdown quote blocks (blocks starting with `>`). +* Added a popular Markdown definition lists extension. +* Added Markdown indented code blocks (blocks indented by >= 4 spaces). +* Added syntax for additional parameters to Markdown code blocks: + + ```nim test="nim c $1" + ... + ``` + + +## C++ interop enhancements + +Nim 2.0 takes C++ interop to the next level. With the new [virtual](https://nim-lang.github.io/Nim/manual_experimental.html#virtual-pragma) pragma and the extended [constructor](https://nim-lang.github.io/Nim/manual_experimental.html#constructor-pragma) pragma. +Now one can define constructors and virtual procs that maps to C++ constructors and virtual methods, allowing one to further customize +the interoperability. There is also extended support for the [codeGenDecl](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-codegendecl-pragma) pragma, so that it works on types. + +It's a common pattern in C++ to use inheritance to extend a library. Some even use multiple inheritance as a mechanism to make interfaces. + +Consider the following example: + +```cpp + +struct Base { + int someValue; + Base(int inValue) { + someValue = inValue; + }; +}; + +class IPrinter { +public: + virtual void print() = 0; +}; +``` + +```nim + +type + Base* {.importcpp, inheritable.} = object + someValue*: int32 + IPrinter* {.importcpp.} = object + +const objTemplate = """ + struct $1 : public $3, public IPrinter { + $2 + }; +"""; + +type NimChild {.codegenDecl: objTemplate .} = object of Base + +proc makeNimChild(val: int32): NimChild {.constructor: "NimClass('1 #1) : Base(#1)".} = + echo "It calls the base constructor passing " & $this.someValue + this.someValue = val * 2 # Notice how we can access `this` inside the constructor. It's of the type `ptr NimChild`. + +proc print*(self: NimChild) {.virtual.} = + echo "Some value is " & $self.someValue + +let child = makeNimChild(10) +child.print() +``` + +It outputs: + +``` +It calls the base constructor passing 10 +Some value is 20 +``` + + +## ARC/ORC refinements + +With the 2.0 release, the ARC/ORC model got refined once again and is now finally complete: + +1. Programmers now have control over the "item was moved from" state as `=wasMoved` is overridable. +2. There is a new `=dup` hook which is more efficient than the old combination of `=wasMoved(tmp); =copy(tmp, x)` operations. +3. Destructors now take a parameter of the attached object type `T` directly and don't have to take a `var T` parameter. + +With these important optimizations we improved the runtime of the compiler and important benchmarks by 0%! Wait ... what? +Yes, unfortunately it turns out that for a modern optimizer like in GCC or LLVM there is no difference. + +But! This refined model is more efficient once separate compilation enters the picture. In other words, as we think of +providing a stable ABI it is important not to lose any efficiency in the calling conventions. + + +## Tool changes + +- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs` before). Use `nimble develop --global` to create an old style link file in the special links directory documented at https://github.com/nim-lang/nimble#nimble-develop. +- nimgrep now offers the option `--inContext` (and `--notInContext`), which + allows to filter only matches with the context block containing a given pattern. +- nimgrep: names of options containing "include/exclude" are deprecated, + e.g. instead of `--includeFile` and `--excludeFile` we have + `--filename` and `--notFilename` respectively. + Also, the semantics are now consistent for such positive/negative filters. +- Nim now ships with an alternative package manager called Atlas. More on this in upcoming versions. + + +## Porting guide + +### Block and Break + +Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead. In other words, turn: + +```nim + +block: + a() + if cond: + break + b() + +``` + +Into: + +```nim + +block maybePerformB: + a() + if cond: + break maybePerformB + b() + +``` + +### Strict funcs + +The definition of `"strictFuncs"` was changed. +The old definition was roughly: "A store to a ref/ptr deref is forbidden unless it's coming from a `var T` parameter". +The new definition is: "A store to a ref/ptr deref is forbidden." + +This new definition is much easier to understand, the price is some expressitivity. The following code used to be +accepted: + +```nim + +{.experimental: "strictFuncs".} + +type Node = ref object + s: string + +func create(s: string): Node = + result = Node() + result.s = s # store to result[] + +``` + +Now it has to be rewritten to: + +```nim + +{.experimental: "strictFuncs".} + +type Node = ref object + s: string + +func create(s: string): Node = + result = Node(s: s) + +``` + +### Standard library + +Several standard library modules have been moved to nimble packages, use `nimble` or `atlas` to install them: + +- `std/punycode` => `punycode` +- `std/asyncftpclient` => `asyncftpclient` +- `std/smtp` => `smtp` +- `std/db_common` => `db_connector/db_common` +- `std/db_sqlite` => `db_connector/db_sqlite` +- `std/db_mysql` => `db_connector/db_mysql` +- `std/db_postgres` => `db_connector/db_postgres` +- `std/db_odbc` => `db_connector/db_odbc` +- `std/md5` => `checksums/md5` +- `std/sha1` => `checksums/sha1` +- `std/sums` => `sums` diff --git a/changelogs/changelog_2_0_0_details.md b/changelogs/changelog_2_0_0_details.md new file mode 100644 index 0000000000..24dc4edad0 --- /dev/null +++ b/changelogs/changelog_2_0_0_details.md @@ -0,0 +1,560 @@ +# v2.0.0 - 2023-08-01 + + +## Changes affecting backward compatibility + +- ORC is now the default memory management strategy. Use + `--mm:refc` for a transition period. + +- The `threads:on` option is now the default. + +- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response. It follows Apache's `HttpClient` (Java), `http` (go) and .NET `HttpWebResponse` (C#) behaviors. Previously it raised a `ValueError`. + +- `addr` is now available for all addressable locations, + `unsafeAddr` is now deprecated and an alias for `addr`. + +- Certain definitions from the default `system` module have been moved to + the following new modules: + + - `std/syncio` + - `std/assertions` + - `std/formatfloat` + - `std/objectdollar` + - `std/widestrs` + - `std/typedthreads` + - `std/sysatomics` + + In the future, these definitions will be removed from the `system` module, + and their respective modules will have to be imported to use them. + Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option + may be used. + +- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated + symbols in the `system` module: + - Aliases with an `Error` suffix to exception types that have a `Defect` suffix + (see [exceptions](https://nim-lang.github.io/Nim/exceptions.html)): + `ArithmeticError`, `DivByZeroError`, `OverflowError`, + `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`, + `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`, + `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`, + `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`, + `DeadThreadError`, `NilAccessError` + - `addQuitProc`, replaced by `exitprocs.addExitProc` + - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32` + - `TaintedString`, formerly a distinct alias to `string` + - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to + `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64` + +- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in + in the `system` module, which is replaced by [threading/channels](https://github.com/nim-lang/threading/blob/master/threading/channels.nim). Use the command `nimble install threading` and import `threading/channels`. + +- Enabling `-d:nimPreviewCstringConversion` causes `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` to not support conversion to `cstring` anymore. + +- Enabling `-d:nimPreviewProcConversion` causes `proc` to not support conversion to + `pointer` anymore. `cast` may be used instead. + +- The `gc:v2` option is removed. + +- The `mainmodule` and `m` options are removed. + +- Optional parameters in combination with `: body` syntax ([RFC #405](https://github.com/nim-lang/RFCs/issues/405)) + are now opt-in via `experimental:flexibleOptionalParams`. + +- Automatic dereferencing (experimental feature) is removed. + +- The `Math.trunc` polyfill for targeting Internet Explorer was + previously included in most JavaScript output files. + Now, it is only included with `-d:nimJsMathTruncPolyfill`. + If you are targeting Internet Explorer, you may choose to enable this option + or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). + Nim uses `Math.trunc` for the division and modulo operators for integers. + +- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and +`sink` for optimization purposes. + +- The experimental `nimPreviewDotLikeOps` switch is going to be removed or deprecated because it didn't fulfill its promises. + +- The `{.this.}` pragma, deprecated since 0.19, has been removed. +- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead. + ```nim + type Foo = distinct ptr int + + # Before: + var x: Foo = nil + # After: + var x: Foo = Foo(nil) + ``` +- Removed two type pragma syntaxes deprecated since 0.20, namely + `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`. Instead, + use `type Foo[T] {.final.} = object`. + +- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent + with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`. + This decision was made with the assumption that the old syntax was used rarely; + if your code used the old syntax, please be aware of this change. + +- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators + are no longer experimental. + +- `macros.getImpl` for `const` symbols now returns the full definition node + (as `nnkConstDef`) rather than the AST of the constant value. + +- Lock levels are deprecated, now a noop. + +- `strictEffects` are no longer experimental. + Use `legacy:laxEffects` to keep backward compatibility. + +- The `gorge`/`staticExec` calls will now return a descriptive message in the output + if the execution fails for whatever reason. To get back legacy behaviour, use `-d:nimLegacyGorgeErrors`. + +- Pointer to `cstring` conversions now trigger a `[PtrToCstringConv]` warning. + This warning will become an error in future versions! Use a `cast` operation + like `cast[cstring](x)` instead. + +- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`. + +- Redefining templates with the same signature was previously + allowed to support certain macro code. To do this explicitly, the + `{.redefine.}` pragma has been added. Note that this is only for templates. + Implicit redefinition of templates is now deprecated and will give an error in the future. + +- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead. + +- Several Standard libraries have been moved to nimble packages, use `nimble` to install them: + - `std/punycode` => `punycode` + - `std/asyncftpclient` => `asyncftpclient` + - `std/smtp` => `smtp` + - `std/db_common` => `db_connector/db_common` + - `std/db_sqlite` => `db_connector/db_sqlite` + - `std/db_mysql` => `db_connector/db_mysql` + - `std/db_postgres` => `db_connector/db_postgres` + - `std/db_odbc` => `db_connector/db_odbc` + - `std/md5` => `checksums/md5` + - `std/sha1` => `checksums/sha1` + - `std/sums` => `std/sums` + +- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of + `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`. + This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead. + +- When `--warning[BareExcept]:on` is enabled, if an `except` specifies no exception or any exception not inheriting from `Defect` or `CatchableError`, a `warnBareExcept` warning will be triggered. For example, the following code will emit a warning: + ```nim + try: + discard + except: # Warning: The bare except clause is deprecated; use `except CatchableError:` instead [BareExcept] + discard + ``` + +- The experimental `strictFuncs` feature now disallows a store to the heap via a `ref` or `ptr` indirection. + +- The underscore identifier (`_`) is now generally not added to scope when + used as the name of a definition. While this was already the case for + variables, it is now also the case for routine parameters, generic + parameters, routine declarations, type declarations, etc. This means that the following code now does not compile: + + ```nim + proc foo(_: int): int = _ + 1 + echo foo(1) + + proc foo[_](t: typedesc[_]): seq[_] = @[default(_)] + echo foo[int]() + + proc _() = echo "_" + _() + + type _ = int + let x: _ = 3 + ``` + + Whereas the following code now compiles: + + ```nim + proc foo(_, _: int): int = 123 + echo foo(1, 2) + + proc foo[_, _](): int = 123 + echo foo[int, bool]() + + proc foo[T, U](_: typedesc[T], _: typedesc[U]): (T, U) = (default(T), default(U)) + echo foo(int, bool) + + proc _() = echo "one" + proc _() = echo "two" + + type _ = int + type _ = float + ``` + +- Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages. + +- The JavaScript backend now uses [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt) + for 64-bit integer types (`int64` and `uint64`) by default. As this affects + JS code generation, code using these types to interface with the JS backend + may need to be updated. Note that `int` and `uint` are not affected. + + For compatibility with [platforms that do not support BigInt](https://caniuse.com/bigint) + and in the case of potential bugs with the new implementation, the + old behavior is currently still supported with the command line option + `--jsbigint64:off`. + +- The `proc` and `iterator` type classes now respectively only match + procs and iterators. Previously both type classes matched any of + procs or iterators. + + ```nim + proc prc(): int = + 123 + + iterator iter(): int {.closure.} = + yield 123 + + proc takesProc[T: proc](x: T) = discard + proc takesIter[T: iterator](x: T) = discard + + # always compiled: + takesProc(prc) + takesIter(iter) + # no longer compiles: + takesProc(iter) + takesIter(prc) + ``` + +- The `proc` and `iterator` type classes now accept a calling convention pragma + (i.e. `proc {.closure.}`) that must be shared by matching proc or iterator + types. Previously, pragmas were parsed but discarded if no parameter list + was given. + + This is represented in the AST by an `nnkProcTy`/`nnkIteratorTy` node with + an `nnkEmpty` node in the place of the `nnkFormalParams` node, and the pragma + node in the same place as in a concrete `proc` or `iterator` type node. This + state of the AST may be unexpected to existing code, both due to the + replacement of the `nnkFormalParams` node as well as having child nodes + unlike other type class AST. + +- Signed integer literals in `set` literals now default to a range type of + `0..255` instead of `0..65535` (the maximum size of sets). + +- `case` statements with `else` branches put before `elif`/`of` branches in macros + are rejected with "invalid order of case branches". + +- Destructors now default to `.raises: []` (i.e. destructors must not raise + unlisted exceptions) and explicitly raising destructors are implementation + defined behavior. + +- The very old, undocumented `deprecated` pragma statement syntax for + deprecated aliases is now a no-op. The regular deprecated pragma syntax is + generally sufficient instead. + + ```nim + # now does nothing: + {.deprecated: [OldName: NewName].} + + # instead use: + type OldName* {.deprecated: "use NewName instead".} = NewName + const oldName* {.deprecated: "use newName instead".} = newName + ``` + + `defined(nimalias)` can be used to check for versions when this syntax was + available; however since code that used this syntax is usually very old, + these deprecated aliases are likely not used anymore and it may make sense + to simply remove these statements. + +- `getProgramResult` and `setProgramResult` in `std/exitprocs` are no longer + declared when they are not available on the backend. Previously it would call + `doAssert false` at runtime despite the condition being checkable at compile-time. + +- Custom destructors now supports non-var parameters, e.g. ``proc `=destroy`[T: object](x: T)`` is valid. ``proc `=destroy`[T: object](x: var T)`` is deprecated. + +- Relative imports will not resolve to searched paths anymore, e.g. `import ./tables` now reports an error properly. + +## Standard library additions and changes + +[//]: # "Changes:" +- OpenSSL 3 is now supported. +- `macros.parseExpr` and `macros.parseStmt` now accept an optional + `filename` argument for more informative errors. +- The `colors` module is expanded with missing colors from the CSS color standard. + `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard. +- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). +- The `md5` module now works at compile time and in JavaScript. +- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef`, to support `const` tables. +- `strutils.find` now uses and defaults to `last = -1` for whole string searches, + making limiting it to just the first char (`last = 0`) valid. +- `strutils.split` and `strutils.rsplit` now return the source string as a single element for an empty separator. +- `random.rand` now works with `Ordinal`s. +- Undeprecated `os.isvalidfilename`. +- `std/oids` now uses `int64` to store time internally (before, it was int32). +- `std/uri.Uri` dollar (`$`) improved, precalculates the `string` result length from the `Uri`. +- `std/uri.Uri.isIpv6` is now exported. +- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2. + + +- `std/jsfetch.newFetchOptions` now has default values for all parameters. +- `std/jsformdata` now accepts the `Blob` data type. + +- `std/sharedlist` and `std/sharedtables` are now deprecated, see [RFC #433](https://github.com/nim-lang/RFCs/issues/433). + +- There is a new compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove the dependency on the Linux `getrandom` syscall. + + This compile flag only affects Linux builds and is necessary if either compiling on a Linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. + + On Linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since Linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall. + + When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be: + ```sh + $ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details + $ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8 + $ sh build.sh # per unix install instructions + $ bin/nim c koch # per unix install instructions + $ ./koch boot -d:release # per unix install instructions + $ ./koch tools -d:nimNoGetRandom # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall + ``` + + This is necessary to pass when building Nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for the stdlib (`sysrand` in particular). + +[//]: # "Additions:" +- Added ISO 8601 week date utilities in `times`: + - Added `IsoWeekRange`, a range type for weeks in a week-based year. + - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year. + - Added an `initDateTime` overload to create a `DateTime` from an ISO week date. + - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime. + - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year. +- Added new modules which were previously part of `std/os`: + - Added `std/oserrors` for OS error reporting. + - Added `std/envvars` for environment variables handling. + - Added `std/cmdline` for reading command line parameters. + - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`. +- Added `sep` parameter in `std/uri` to specify the query separator. +- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`. +- Added `complex.sgn` for obtaining the phase of complex numbers. +- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`, + `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`, + `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`, + `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`, + `toggleAttribute`, and `matches` to `std/dom`. +- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices). +- Added `capacity` for `string` and `seq` to return the current capacity, see [RFC #460](https://github.com/nim-lang/RFCs/issues/460). +- Added `openArray[char]` overloads for `std/parseutils` and `std/unicode`, allowing for more code reuse. +- Added a `safe` parameter to `base64.encodeMime`. +- Added `parseutils.parseSize` - inverse to `strutils.formatSize` - to parse human readable sizes. +- Added `minmax` to `sequtils`, as a more efficient `(min(_), max(_))` over sequences. +- `std/jscore` for the JavaScript target: + + Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) + and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask). + + Added `toDateString`, `toISOString`, `toJSON`, `toTimeString`, `toUTCString` converters for `DateTime`. +- Added `BackwardsIndex` overload for `CacheSeq`. +- Added support for nested `with` blocks in `std/with`. +- Added `ensureMove` to the system module. It ensures that the passed argument is moved, otherwise an error is given at the compile time. + + +[//]: # "Deprecations:" +- Deprecated `selfExe` for Nimscript. +- Deprecated `std/base64.encode` for collections of arbitrary integer element type. + Now only `byte` and `char` are supported. + +[//]: # "Removals:" +- Removed deprecated module `parseopt2`. +- Removed deprecated module `sharedstrings`. +- Removed deprecated module `dom_extensions`. +- Removed deprecated module `LockFreeHash`. +- Removed deprecated module `events`. +- Removed deprecated `oids.oidToString`. +- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. +- Removed deprecated `jsre.test` and `jsre.toString`. +- Removed deprecated `math.c_frexp`. +- Removed deprecated `` httpcore.`==` ``. +- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that take wrong argument types. +- Removed deprecated `osproc.poDemon`, symbol with typo. +- Removed deprecated `tables.rightSize`. + + +- Removed deprecated `posix.CLONE_STOPPED`. + + +## Language changes + +- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) now supports the definition of forbidden tags by the `.forbids` pragma + which can be used to disable certain effects in proc types. +- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental, + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. +- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed + for the right-hand side of type definitions in type sections. Previously + they would error with "invalid indentation". + +- Compile-time define changes: + - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. + In the command line, this is defined as `-d:a.b.c`. Older versions can + use backticks as in ``defined(`a.b.c`)`` to access such defines. + - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas) + now support a string argument for qualified define names. + + ```nim + # -d:package.FooBar=42 + const FooBar {.intdefine: "package.FooBar".}: int = 5 + echo FooBar # 42 + ``` + + This was added to help disambiguate similar define names for different packages. + In older versions, this could only be achieved with something like the following: + + ```nim + const FooBar = block: + const `package.FooBar` {.intdefine.}: int = 5 + `package.FooBar` + ``` + - A generic `define` pragma for constants has been added that interprets + the value of the define based on the type of the constant value. + See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-nimdefine-pragma) + for a list of supported types. + +- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes: + - Templates now accept macro pragmas. + - Macro pragmas for var/let/const sections have been redesigned in a way that works + similarly to routine macro pragmas. The new behavior is documented in the + [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas). + - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, + allowing multiple type definitions to be injected in place of the original type definition. + + ```nim + import macros + + macro multiply(amount: static int, s: untyped): untyped = + let name = $s[0].basename + result = newNimNode(nnkTypeSection) + for i in 1 .. amount: + result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2])) + + type + Foo = object + Bar {.multiply: 3.} = object + x, y, z: int + Baz = object + # becomes + type + Foo = object + Bar1 = object + x, y, z: int + Bar2 = object + x, y, z: int + Bar3 = object + x, y, z: int + Baz = object + ``` + +- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference) + has been implemented for a variety of basic cases. For example, code like the following now compiles: + + ```nim + let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")] + ``` + +- `cstring` is now accepted as a selector in `case` statements, removing the + need to convert to `string`. On the JS backend, this is translated directly + to a `switch` statement. + +- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters). +- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`. + +- IBM Z architecture and macOS m1 arm64 architecture are supported. + +- `=wasMoved` can now be overridden by users. + +- There is a new pragma called [quirky](https://nim-lang.github.io/Nim/manual_experimental.html#quirky-routines) that can be used to affect the code + generation of goto based exception handling. It can improve the produced code size but its effects can be subtle so use it with care. + +- Tuple unpacking for variables is now treated as syntax sugar that directly + expands into multiple assignments. Along with this, tuple unpacking for + variables can now be nested. + + ```nim + proc returnsNestedTuple(): (int, (int, int), int, int) = (4, (5, 7), 2, 3) + + let (x, (_, y), _, z) = returnsNestedTuple() + # roughly becomes + let + tmpTup1 = returnsNestedTuple() + x = tmpTup1[0] + tmpTup2 = tmpTup1[1] + y = tmpTup2[1] + z = tmpTup1[3] + ``` + + As a result `nnkVarTuple` nodes in variable sections will no longer be + reflected in `typed` AST. + +- C++ interoperability: + - New [`virtual`](https://nim-lang.github.io/Nim/manual_experimental.html#virtual-pragma) pragma added. + - Improvements to [`constructor`](https://nim-lang.github.io/Nim/manual_experimental.html#constructor-pragma) pragma. + +## Compiler changes + +- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the + reality better. (Nim moved away from all techniques based on "tracing".) + +- Defines the `gcRefc` symbol which allows writing specific code for the refc GC. + +- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, + without requiring `-d:nimVersion140` which is now a noop. + +- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package. + +- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and + related functions produced on the backend. This prevents conflicts with other Nim + static libraries. + +- When compiling for release, the flag `-fno-math-errno` is used for GCC. +- Removed deprecated `LineTooLong` hint. +- Line numbers and file names of source files work correctly inside templates for JavaScript targets. + +- Removed support for LCC (Local C), Pelles C, Digital Mars and Watcom compilers. + + +## Docgen + +- `Markdown` is now the default markup language of doc comments (instead + of the legacy `RstMarkdown` mode). In this release we begin to separate + RST and Markdown features to better follow specification of each + language, with the focus on Markdown development. + See also [the docs](https://nim-lang.github.io/Nim/markdown_rst.html). + + * Added a `{.doctype: Markdown | RST | RstMarkdown.}` pragma allowing to + select the markup language mode in the doc comments of the current `.nim` + file for processing by `nim doc`: + + 1. `Markdown` (default) is basically CommonMark (standard Markdown) + + some Pandoc Markdown features + some RST features that are missing + in our current implementation of CommonMark and Pandoc Markdown. + 2. `RST` closely follows the RST spec with few additional Nim features. + 3. `RstMarkdown` is a maximum mix of RST and Markdown features, which + is kept for the sake of compatibility and ease of migration. + + * Added separate `md2html` and `rst2html` commands for processing + standalone `.md` and `.rst` files respectively (and also `md2tex`/`rst2tex`). + +- Added Pandoc Markdown bracket syntax `[...]` for making anchor-less links. +- Docgen now supports concise syntax for referencing Nim symbols: + instead of specifying HTML anchors directly one can use original + Nim symbol declarations (adding the aforementioned link brackets + `[...]` around them). + * To use this feature across modules, a new `importdoc` directive was added. + Using this feature for referencing also helps to ensure that links + (inside one module or the whole project) are not broken. +- Added support for RST & Markdown quote blocks (blocks starting with `>`). +- Added a popular Markdown definition lists extension. +- Added Markdown indented code blocks (blocks indented by >= 4 spaces). +- Added syntax for additional parameters to Markdown code blocks: + + ```nim test="nim c $1" + ... + ``` + +## Tool changes + +- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs` before). Use `nimble develop --global` to create an old style link file in the special links directory documented at https://github.com/nim-lang/nimble#nimble-develop. +- nimgrep added the option `--inContext` (and `--notInContext`), which + allows to filter only matches with the context block containing a given pattern. +- nimgrep: names of options containing "include/exclude" are deprecated, + e.g. instead of `--includeFile` and `--excludeFile` we have + `--filename` and `--notFilename` respectively. + Also the semantics are now consistent for such positive/negative filters. +- koch now supports the `--skipIntegrityCheck` option. The command `koch --skipIntegrityCheck boot -d:release` always builds the compiler twice. diff --git a/changelogs/changelog_2_2_0.md b/changelogs/changelog_2_2_0.md new file mode 100644 index 0000000000..0a293d35fb --- /dev/null +++ b/changelogs/changelog_2_2_0.md @@ -0,0 +1,12 @@ +# v2.2.0 - 2023-mm-dd + +## Changes affecting backward compatibility + +## Standard library additions and changes + +## Language changes + +## Compiler changes + +## Tool changes + diff --git a/changelogs/changelog_X_XX_X.md b/changelogs/changelog_X_XX_X.md index 48520fd44c..524599b119 100644 --- a/changelogs/changelog_X_XX_X.md +++ b/changelogs/changelog_X_XX_X.md @@ -3,12 +3,15 @@ This is an example file. The changes should go to changelog.md! +## Changes affecting backward compatibility + +- `foo` now behaves differently, use `-d:nimLegacyFoo` for previous behavior. ## Standard library additions and changes - Added `example.exampleProc`. -- Changed `example.foo` to take additional `bar` parameter. +- Changed `example.foo` to take additional `bar` parameter. ## Language changes diff --git a/ci/action.nim b/ci/action.nim new file mode 100644 index 0000000000..ad0f4df0b6 --- /dev/null +++ b/ci/action.nim @@ -0,0 +1,26 @@ +import std/[strutils, os, osproc, parseutils, strformat] + + +proc main() = + var msg = "" + const cmd = "./koch boot --mm:orc -d:release" + + let (output, exitCode) = execCmdEx(cmd) + + doAssert exitCode == 0, output + + let start = rfind(output, "Hint: mm") + doAssert parseUntil(output, msg, "; proj", start) > 0, output + + let (commitHash, _) = execCmdEx("""git log --format="%H" -n 1""") + + let welcomeMessage = fmt"""Thanks for your hard work on this PR! +The lines below are statistics of the Nim compiler built from {commitHash} + +{msg} +""" + createDir "ci/nimcache" + writeFile "ci/nimcache/results.txt", welcomeMessage + +when isMainModule: + main() diff --git a/ci/build.bat b/ci/build.bat deleted file mode 100644 index cdce8d3d29..0000000000 --- a/ci/build.bat +++ /dev/null @@ -1,14 +0,0 @@ -REM Some debug info -echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%." -gcc -v - -git clone --depth 1 https://github.com/nim-lang/csources.git -cd csources -call build64.bat -cd .. -set PATH=%CD%\bin;%PATH% -nim -v -nim c koch -koch.exe boot -copy bin/nim bin/nimd -koch.exe boot -d:release diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100644 index 6321fffba8..0000000000 --- a/ci/build.sh +++ /dev/null @@ -1,15 +0,0 @@ -sh ci/deps.sh - -# Build from C sources. -git clone --depth 1 https://github.com/nim-lang/csources.git -cd csources -sh build.sh -cd .. -# Add Nim to the PATH -export PATH=$(pwd)/bin${PATH:+:$PATH} -# Bootstrap. -nim -v -nim c koch -./koch boot -cp bin/nim bin/nimd -./koch boot -d:release diff --git a/ci/build_autogen.bat b/ci/build_autogen.bat new file mode 100644 index 0000000000..0b6ca566e2 --- /dev/null +++ b/ci/build_autogen.bat @@ -0,0 +1,26 @@ +@echo off +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% +) + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + SET ARCH=64 + ) + CALL build.bat + cd .. + copy /y bin\nim.exe %nim_csources% +) diff --git a/ci/deps.bat b/ci/deps.bat deleted file mode 100644 index bda1fe14fb..0000000000 --- a/ci/deps.bat +++ /dev/null @@ -1,4 +0,0 @@ -nim e install_nimble.nims -nim e tests/test_nimscript.nims -nimble update -nimble install -y zip opengl sdl1 jester@#head niminst diff --git a/ci/deps.sh b/ci/deps.sh deleted file mode 100644 index f0f831a2a3..0000000000 --- a/ci/deps.sh +++ /dev/null @@ -1,16 +0,0 @@ -# Some debug info -echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS." - -# Packages -apt-get update -qq -apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs - -gcc -v - -export PATH=$(pwd)/bin${PATH:+:$PATH} - -# Nimble deps -nim e install_nimble.nims -nim e tests/test_nimscript.nims -nimble update -nimble install zip opengl sdl1 jester@#head niminst diff --git a/ci/funs.sh b/ci/funs.sh index de56ef1529..0263661983 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -1,8 +1,147 @@ -# utilities used in CI pipelines to avoid duplication. +# Utilities used in CI pipelines and tooling to avoid duplication. +# Avoid top-level statements. +# Prefer nim scripts whenever possible. +# functions starting with `_` are considered internal, less stable. echo_run () { # echo's a command before running it, which helps understanding logs echo "" - echo "$@" + echo "cmd: $@" # in azure we could also use this: echo '##[section]"$@"' "$@" } + +nimGetLastCommit() { + git log --no-merges -1 --pretty=format:"%s" +} + +nimIsCiSkip(){ + # D20210329T004830:here refs https://github.com/microsoft/azure-pipelines-agent/issues/2944 + # `--no-merges` is needed to avoid merge commits which occur for PR's. + # $(Build.SourceVersionMessage) is not helpful + # nor is `github.event.head_commit.message` for github actions. + # Note: `[skip ci]` is now handled automatically for github actions, see https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/ + commitMsg=$(nimGetLastCommit) + echo commitMsg: "$commitMsg" + if [[ $commitMsg == *"[skip ci]"* ]]; then + echo "skipci: true" + return 0 + else + echo "skipci: false" + return 1 + fi +} + +nimInternalInstallDepsWindows(){ + echo_run mkdir dist + echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z + echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip + echo_run 7z x dist/mingw64.7z -odist + echo_run 7z x dist/dlls.zip -obin +} + +nimInternalBuildKochAndRunCI(){ + echo_run nim c koch + if ! echo_run ./koch runCI; then + echo_run echo "runCI failed" + echo_run nim r tools/ci_testresults.nim + return 1 + fi +} + +nimDefineVars(){ + . config/build_config.txt + nim_csources=bin/nim_csources_$nim_csourcesHash +} + +_nimNumCpu(){ + # linux: $(nproc) + # FreeBSD | macOS: $(sysctl -n hw.ncpu) + # OpenBSD: $(sysctl -n hw.ncpuonline) + # windows: $NUMBER_OF_PROCESSORS ? + if env | grep -q '^NIMCORES='; then + echo $NIMCORES + else + echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1) + fi +} + +_nimBuildCsourcesIfNeeded(){ + # if some systems cannot use make or gmake, we could add support for calling `build.sh` + # but this is slower (not parallel jobs) and would require making build.sh + # understand the arguments passed to the makefile (e.g. `CC=gcc ucpu=amd64 uos=darwin`), + # instead of `--cpu amd64 --os darwin`. + unamestr=$(uname) + # uname values: https://en.wikipedia.org/wiki/Uname + if [ "$unamestr" = 'FreeBSD' ]; then + makeX=gmake + elif [ "$unamestr" = 'OpenBSD' ]; then + makeX=gmake + elif [ "$unamestr" = 'NetBSD' ]; then + makeX=gmake + elif [ "$unamestr" = 'CROSSOS' ]; then + makeX=gmake + elif [ "$unamestr" = 'SunOS' ]; then + makeX=gmake + else + makeX=make + fi + nCPU=$(_nimNumCpu) + echo_run which $makeX + # parallel jobs (5X faster on 16 cores: 10s instead of 50s) + echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU "$@" + # keep $nim_csources in case needed to investigate bootstrap issues + # without having to rebuild + echo_run cp bin/nim $nim_csources +} + +nimCiSystemInfo(){ + nimDefineVars + echo_run eval echo '$'nim_csources + echo_run pwd + echo_run date + echo_run uname -a + echo_run git log --no-merges -1 --pretty=oneline + echo_run eval echo '$'PATH + echo_run gcc -v + echo_run node -v + echo_run make -v +} + +nimCsourcesHash(){ + nimDefineVars + echo $nim_csourcesHash +} + +nimBuildCsourcesIfNeeded(){ + # goal: allow cachine each tagged version independently + # to avoid rebuilding, so that tools like `git bisect` + # can grab a cached past version without rebuilding. + nimDefineVars + ( + set -e + # avoid polluting caller scope with internal variable definitions. + if test -f "$nim_csources"; then + echo "$nim_csources exists." + else + if test -d "$nim_csourcesDir"; then + echo "$nim_csourcesDir exists." + else + # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g. + # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`. + echo_run git clone -q --depth 1 -b $nim_csourcesBranch \ + $nim_csourcesUrl "$nim_csourcesDir" + # old `git` versions don't support -C option, using `cd` explicitly: + echo_run cd "$nim_csourcesDir" + echo_run git checkout $nim_csourcesHash + echo_run cd "$OLDPWD" + # if needed we could also add: `git reset --hard $nim_csourcesHash` + fi + _nimBuildCsourcesIfNeeded "$@" + fi + + echo_run rm -f bin/nim + # fixes bug #17913, but it's unclear why it's needed, maybe specific to MacOS Big Sur 11.3 on M1 arch? + echo_run cp $nim_csources bin/nim + echo_run $nim_csources -v + ) +} diff --git a/ci/nsis_build.bat b/ci/nsis_build.bat deleted file mode 100644 index 4806810d7f..0000000000 --- a/ci/nsis_build.bat +++ /dev/null @@ -1,59 +0,0 @@ -REM - Run the full testsuite; testament\tester all - -REM - Uncomment the list of changes in news.txt -REM - write a news ticker entry -REM - Update the version - -REM - Generate the full docs; koch web0 -REM - Generate the installers; -REM - Update the version in system.nim -REM - Test the installers -REM - Tag the release -REM - Merge devel into master -REM - Update csources - -set NIMVER=%1 - -Rem Build -docs file: -koch web0 -cd web\upload -7z a -tzip docs-%NIMVER%.zip *.html -move /y docs-%NIMVER%.zip download -cd ..\.. - -Rem Build csources -koch csources -d:release || exit /b - -rem Grab C sources and nimsuggest -git clone --depth 1 https://github.com/nim-lang/csources.git - -set PATH=%CD%\bin;%PATH% - -ReM Build Win32 version: - -set PATH=C:\Users\araq\projects\mingw32\bin;%PATH% -cd csources -call build.bat -cd .. -ReM Rebuilding koch is necessary because it uses its pointer size to determine -ReM which mingw link to put in the NSIS installer. -nim c --out:koch_temp koch || exit /b -koch_temp boot -d:release || exit /b -koch_temp nsis -d:release || exit /b -koch_temp zip -d:release || exit /b -dir build -move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b -move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b - - -ReM Build Win64 version: -set PATH=C:\Users\araq\projects\mingw64\bin;%PATH% -cd csources -call build64.bat -cd .. -nim c --out:koch_temp koch || exit /b -koch_temp boot -d:release || exit /b -koch_temp nsis -d:release || exit /b -koch_temp zip -d:release || exit /b -move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b -move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b diff --git a/compiler.nimble b/compiler.nimble deleted file mode 100644 index f4da45519d..0000000000 --- a/compiler.nimble +++ /dev/null @@ -1,9 +0,0 @@ - -version = system.NimVersion -author = "Andreas Rumpf" -description = "Compiler package providing the compiler sources as a library." -license = "MIT" - -installDirs = @["compiler", "nimsuggest"] - -requires "nim >= 0.14.0" diff --git a/compiler/aliasanalysis.nim b/compiler/aliasanalysis.nim new file mode 100644 index 0000000000..e24c6d8e26 --- /dev/null +++ b/compiler/aliasanalysis.nim @@ -0,0 +1,128 @@ + +import ast + +import std / assertions + +const + PathKinds0* = {nkDotExpr, nkCheckedFieldExpr, + nkBracketExpr, nkDerefExpr, nkHiddenDeref, + nkAddr, nkHiddenAddr, + nkObjDownConv, nkObjUpConv} + PathKinds1* = {nkHiddenStdConv, nkHiddenSubConv} + +proc skipConvDfa*(n: PNode): PNode = + result = n + while true: + case result.kind + of nkObjDownConv, nkObjUpConv: + result = result[0] + of PathKinds1: + result = result[1] + else: break + +proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool = + var n = orig + while true: + case n.kind + of PathKinds0 - {nkHiddenDeref, nkDerefExpr}: + n = n[0] + of PathKinds1: + n = n[1] + of nkHiddenDeref, nkDerefExpr: + # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty + # pointer indirection. + # bug #14159, we cannot reason about sinkParam[].location as it can + # still be shared for tyRef. + n = n[0] + return n.kind == nkSym and n.sym.owner == owner and + (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned}) + else: break + # XXX Allow closure deref operations here if we know + # the owner controlled the closure allocation? + result = n.kind == nkSym and n.sym.owner == owner and + {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and + (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar) + # Note: There is a different move analyzer possible that checks for + # consume(param.key); param.key = newValue for all paths. Then code like + # + # let splited = split(move self.root, x) + # self.root = merge(splited.lower, splited.greater) + # + # could be written without the ``move self.root``. However, this would be + # wrong! Then the write barrier for the ``self.root`` assignment would + # free the old data and all is lost! Lesson: Don't be too smart, trust the + # lower level C++ optimizer to specialize this code. + +type AliasKind* = enum + yes, no, maybe + +proc aliases*(obj, field: PNode): AliasKind = + # obj -> field: + # x -> x: true + # x -> x.f: true + # x.f -> x: false + # x.f -> x.f: true + # x.f -> x.v: false + # x -> x[]: true + # x[] -> x: false + # x -> x[0]: true + # x[0] -> x: false + # x[0] -> x[0]: true + # x[0] -> x[1]: false + # x -> x[i]: true + # x[i] -> x: false + # x[i] -> x[i]: maybe; Further analysis could make this return true when i is a runtime-constant + # x[i] -> x[j]: maybe; also returns maybe if only one of i or j is a compiletime-constant + template collectImportantNodes(result, n) = + var result: seq[PNode] = @[] + var n = n + while true: + case n.kind + of PathKinds0 - {nkDotExpr, nkBracketExpr, nkDerefExpr, nkHiddenDeref}: + n = n[0] + of PathKinds1: + n = n[1] + of nkDotExpr, nkBracketExpr, nkDerefExpr, nkHiddenDeref: + result.add n + n = n[0] + of nkSym: + result.add n + break + else: return no + + collectImportantNodes(objImportantNodes, obj) + collectImportantNodes(fieldImportantNodes, field) + + # If field is less nested than obj, then it cannot be part of/aliased by obj + if fieldImportantNodes.len < objImportantNodes.len: return no + + result = yes + for i in 1..objImportantNodes.len: + # We compare the nodes leading to the location of obj and field + # with each other. + # We continue until they diverge, in which case we return no, or + # until we reach the location of obj, in which case we do not need + # to look further, since field must be part of/aliased by obj now. + # If we encounter an element access using an index which is a runtime value, + # we simply return maybe instead of yes; should further nodes not diverge. + let currFieldPath = fieldImportantNodes[^i] + let currObjPath = objImportantNodes[^i] + + if currFieldPath.kind != currObjPath.kind: + return no + + case currFieldPath.kind + of nkSym: + if currFieldPath.sym != currObjPath.sym: return no + of nkDotExpr: + if currFieldPath[1].sym != currObjPath[1].sym: return no + of nkDerefExpr, nkHiddenDeref: + discard + of nkBracketExpr: + if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals: + if currFieldPath[1].intVal != currObjPath[1].intVal: + return no + else: + result = maybe + else: assert false # unreachable + diff --git a/compiler/aliases.nim b/compiler/aliases.nim index 0006c9fe63..fa1167753f 100644 --- a/compiler/aliases.nim +++ b/compiler/aliases.nim @@ -10,7 +10,12 @@ ## Simple alias analysis for the HLO and the code generators. import - ast, astalgo, types, trees, intsets + ast, astalgo, types, trees + +import std/intsets + +when defined(nimPreviewSlimSystem): + import std/assertions type TAnalysisResult* = enum @@ -46,14 +51,16 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult = if compareTypes(a, b, dcEqIgnoreDistinct): return arYes case a.kind of tyObject: - if a[0] != nil: - result = isPartOfAux(a[0].skipTypes(skipPtrs), b, marker) + if a.baseClass != nil: + result = isPartOfAux(a.baseClass.skipTypes(skipPtrs), b, marker) if result == arNo: result = isPartOfAux(a.n, b, marker) of tyGenericInst, tyDistinct, tyAlias, tySink: - result = isPartOfAux(lastSon(a), b, marker) - of tyArray, tySet, tyTuple: - for i in 0..= 2 and b.len >= 2: @@ -141,7 +155,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = result = isPartOf(a[1], b[1]) of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr: result = isPartOf(a[0], b[0]) - else: discard + else: result = arNo # Calls return a new location, so a default of ``arNo`` is fine. else: # go down recursively; this is quite demanding: @@ -157,6 +171,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = of DerefKinds: # a* !<| b[] iff + result = arNo if isPartOf(a.typ, b.typ) != arNo: result = isPartOf(a, b[0]) if result == arNo: result = arMaybe @@ -178,7 +193,9 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = if isPartOf(a.typ, b.typ) != arNo: result = isPartOf(a[0], b) if result == arNo: result = arMaybe - else: discard + else: + result = arNo + else: result = arNo of nkObjConstr: result = arNo for i in 1.. 0: result = isPartOf(a, b[0]) - else: discard + else: + result = arNo + else: result = arNo diff --git a/compiler/ast.nim b/compiler/ast.nim index 50a2fb58c8..4de277ba9a 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,11 +10,19 @@ # abstract syntax tree + symbol table import - lineinfos, hashes, options, ropes, idents, int128 -from strutils import toLowerAscii + lineinfos, options, ropes, idents, int128, wordrecg + +import std/[tables, hashes] +from std/strutils import toLowerAscii + +when defined(nimPreviewSlimSystem): + import std/assertions export int128 +import nodekinds +export nodekinds + type TCallingConvention* = enum ccNimCall = "nimcall" # nimcall, also the default @@ -28,207 +36,12 @@ type ccThisCall = "thiscall" # thiscall (parameters are pushed right-to-left) ccClosure = "closure" # proc has a closure ccNoConvention = "noconv" # needed for generating proper C procs sometimes - -type - TNodeKind* = enum # order is extremely important, because ranges are used - # to check whether a node belongs to a certain class - nkNone, # unknown node kind: indicates an error - # Expressions: - # Atoms: - nkEmpty, # the node is empty - nkIdent, # node is an identifier - nkSym, # node is a symbol - nkType, # node is used for its typ field - - nkCharLit, # a character literal '' - nkIntLit, # an integer literal - nkInt8Lit, - nkInt16Lit, - nkInt32Lit, - nkInt64Lit, - nkUIntLit, # an unsigned integer literal - nkUInt8Lit, - nkUInt16Lit, - nkUInt32Lit, - nkUInt64Lit, - nkFloatLit, # a floating point literal - nkFloat32Lit, - nkFloat64Lit, - nkFloat128Lit, - nkStrLit, # a string literal "" - nkRStrLit, # a raw string literal r"" - nkTripleStrLit, # a triple string literal """ - nkNilLit, # the nil literal - # end of atoms - nkComesFrom, # "comes from" template/macro information for - # better stack trace generation - nkDotCall, # used to temporarily flag a nkCall node; - # this is used - # for transforming ``s.len`` to ``len(s)`` - - nkCommand, # a call like ``p 2, 4`` without parenthesis - nkCall, # a call like p(x, y) or an operation like +(a, b) - nkCallStrLit, # a call with a string literal - # x"abc" has two sons: nkIdent, nkRStrLit - # x"""abc""" has two sons: nkIdent, nkTripleStrLit - nkInfix, # a call like (a + b) - nkPrefix, # a call like !a - nkPostfix, # something like a! (also used for visibility) - nkHiddenCallConv, # an implicit type conversion via a type converter - - nkExprEqExpr, # a named parameter with equals: ''expr = expr'' - nkExprColonExpr, # a named parameter with colon: ''expr: expr'' - nkIdentDefs, # a definition like `a, b: typeDesc = expr` - # either typeDesc or expr may be nil; used in - # formal parameters, var statements, etc. - nkVarTuple, # a ``var (a, b) = expr`` construct - nkPar, # syntactic (); may be a tuple constructor - nkObjConstr, # object constructor: T(a: 1, b: 2) - nkCurly, # syntactic {} - nkCurlyExpr, # an expression like a{i} - nkBracket, # syntactic [] - nkBracketExpr, # an expression like a[i..j, k] - nkPragmaExpr, # an expression like a{.pragmas.} - nkRange, # an expression like i..j - nkDotExpr, # a.b - nkCheckedFieldExpr, # a.b, but b is a field that needs to be checked - nkDerefExpr, # a^ - nkIfExpr, # if as an expression - nkElifExpr, - nkElseExpr, - nkLambda, # lambda expression - nkDo, # lambda block appering as trailing proc param - nkAccQuoted, # `a` as a node - - nkTableConstr, # a table constructor {expr: expr} - nkBind, # ``bind expr`` node - nkClosedSymChoice, # symbol choice node; a list of nkSyms (closed) - nkOpenSymChoice, # symbol choice node; a list of nkSyms (open) - nkHiddenStdConv, # an implicit standard type conversion - nkHiddenSubConv, # an implicit type conversion from a subtype - # to a supertype - nkConv, # a type conversion - nkCast, # a type cast - nkStaticExpr, # a static expr - nkAddr, # a addr expression - nkHiddenAddr, # implicit address operator - nkHiddenDeref, # implicit ^ operator - nkObjDownConv, # down conversion between object types - nkObjUpConv, # up conversion between object types - nkChckRangeF, # range check for floats - nkChckRange64, # range check for 64 bit ints - nkChckRange, # range check for ints - nkStringToCString, # string to cstring - nkCStringToString, # cstring to string - # end of expressions - - nkAsgn, # a = b - nkFastAsgn, # internal node for a fast ``a = b`` - # (no string copy) - nkGenericParams, # generic parameters - nkFormalParams, # formal parameters - nkOfInherit, # inherited from symbol - - nkImportAs, # a 'as' b in an import statement - nkProcDef, # a proc - nkMethodDef, # a method - nkConverterDef, # a converter - nkMacroDef, # a macro - nkTemplateDef, # a template - nkIteratorDef, # an iterator - - nkOfBranch, # used inside case statements - # for (cond, action)-pairs - nkElifBranch, # used in if statements - nkExceptBranch, # an except section - nkElse, # an else part - nkAsmStmt, # an assembler block - nkPragma, # a pragma statement - nkPragmaBlock, # a pragma with a block - nkIfStmt, # an if statement - nkWhenStmt, # a when expression or statement - nkForStmt, # a for statement - nkParForStmt, # a parallel for statement - nkWhileStmt, # a while statement - nkCaseStmt, # a case statement - nkTypeSection, # a type section (consists of type definitions) - nkVarSection, # a var section - nkLetSection, # a let section - nkConstSection, # a const section - nkConstDef, # a const definition - nkTypeDef, # a type definition - nkYieldStmt, # the yield statement as a tree - nkDefer, # the 'defer' statement - nkTryStmt, # a try statement - nkFinally, # a finally section - nkRaiseStmt, # a raise statement - nkReturnStmt, # a return statement - nkBreakStmt, # a break statement - nkContinueStmt, # a continue statement - nkBlockStmt, # a block statement - nkStaticStmt, # a static statement - nkDiscardStmt, # a discard statement - nkStmtList, # a list of statements - nkImportStmt, # an import statement - nkImportExceptStmt, # an import x except a statement - nkExportStmt, # an export statement - nkExportExceptStmt, # an 'export except' statement - nkFromStmt, # a from * import statement - nkIncludeStmt, # an include statement - nkBindStmt, # a bind statement - nkMixinStmt, # a mixin statement - nkUsingStmt, # an using statement - nkCommentStmt, # a comment statement - nkStmtListExpr, # a statement list followed by an expr; this is used - # to allow powerful multi-line templates - nkBlockExpr, # a statement block ending in an expr; this is used - # to allow powerful multi-line templates that open a - # temporary scope - nkStmtListType, # a statement list ending in a type; for macros - nkBlockType, # a statement block ending in a type; for macros - # types as syntactic trees: - - nkWith, # distinct with `foo` - nkWithout, # distinct without `foo` - - nkTypeOfExpr, # type(1+2) - nkObjectTy, # object body - nkTupleTy, # tuple body - nkTupleClassTy, # tuple type class - nkTypeClassTy, # user-defined type class - nkStaticTy, # ``static[T]`` - nkRecList, # list of object parts - nkRecCase, # case section of object - nkRecWhen, # when section of object - nkRefTy, # ``ref T`` - nkPtrTy, # ``ptr T`` - nkVarTy, # ``var T`` - nkConstTy, # ``const T`` - nkMutableTy, # ``mutable T`` - nkDistinctTy, # distinct type - nkProcTy, # proc type - nkIteratorTy, # iterator type - nkSharedTy, # 'shared T' - # we use 'nkPostFix' for the 'not nil' addition - nkEnumTy, # enum body - nkEnumFieldDef, # `ident = expr` in an enumeration - nkArgList, # argument list - nkPattern, # a special pattern; used for matching - nkHiddenTryStmt, # a hidden try statement - nkClosure, # (prc, env)-pair (internally used for code gen) - nkGotoState, # used for the state machine (for iterators) - nkState, # give a label to a code section (for iterators) - nkBreakState, # special break statement for easier code generation - nkFuncDef, # a func - nkTupleConstr # a tuple constructor - nkModuleRef # for .rod file support: A (moduleId, itemId) pair - nkReplayAction # for .rod file support: A replay action - nkNilRodNode # for .rod file support: a 'nil' PNode + ccMember = "member" # proc is a (cpp) member TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # 46 flags! + TSymFlag* = enum # 51 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -256,7 +69,8 @@ type # *OR*: a proc is indirectly called (used as first class) sfCompilerProc, # proc is a compiler proc, that is a C proc that is # needed for the code generator - sfProcvar, # proc can be passed to a proc var + sfEscapes # param escapes + # currently unimplemented sfDiscriminant, # field is a discriminant in a record/object sfRequiresInit, # field must be initialized during construction sfDeprecated, # symbol is deprecated @@ -279,7 +93,7 @@ type sfNamedParamCall, # symbol needs named parameter call syntax in target # language; for interfacing with Objective C sfDiscardable, # returned value may be discarded implicitly - sfOverriden, # proc is overridden + sfOverridden, # proc is overridden sfCallsite # A flag for template symbols to tell the # compiler it should use line information from # the calling side of the macro, not from the @@ -295,9 +109,23 @@ type sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation sfNeverRaises # proc can never raise an exception, not even OverflowDefect # or out-of-memory + sfSystemRaisesDefect # proc in the system can raise defects sfUsedInFinallyOrExcept # symbol is used inside an 'except' or 'finally' sfSingleUsedTemp # For temporaries that we know will only be used once sfNoalias # 'noalias' annotation, means C's 'restrict' + # for templates and macros, means cannot be called + # as a lone symbol (cannot use alias syntax) + sfEffectsDelayed # an 'effectsDelayed' parameter + sfGeneratedType # A anonymous generic type that is generated by the compiler for + # objects that do not have generic parameters in case one of the + # object fields has one. + # + # This is disallowed but can cause the typechecking to go into + # an infinite loop, this flag is used as a sentinel to stop it. + sfVirtual # proc is a C++ virtual function + sfByCopy # param is marked as pass bycopy + sfMember # proc is a C++ member of a type + sfCodegenDecl # type, proc, global or proc param is marked as codegenDecl TSymFlags* = set[TSymFlag] @@ -323,13 +151,14 @@ const sfCompileToCpp* = sfInfixCall # compile the module as C++ code sfCompileToObjc* = sfNamedParamCall # compile the module as Objective-C code - sfExperimental* = sfOverriden # module uses the .experimental switch - sfGoto* = sfOverriden # var is used for 'goto' code generation + sfExperimental* = sfOverridden # module uses the .experimental switch + sfGoto* = sfOverridden # var is used for 'goto' code generation sfWrittenTo* = sfBorrow # param is assigned to - sfEscapes* = sfProcvar # param escapes + # currently unimplemented sfBase* = sfDiscriminant - sfIsSelf* = sfOverriden # param is 'self' sfCustomPragma* = sfRegister # symbol is custom pragma template + sfTemplateRedefinition* = sfExportc # symbol is a redefinition of an earlier template + sfCppMember* = { sfVirtual, sfMember, sfConstructor } # proc is a C++ member, meaning it will be attached to the type definition const # getting ready for the future expr/stmt merge @@ -342,7 +171,8 @@ const ensuresEffects* = 2 # 'ensures' annotation tagEffects* = 3 # user defined tag ('gc', 'time' etc.) pragmasEffects* = 4 # not an effect, but a slot for pragmas in proc type - effectListLen* = 5 # list of effects list + forbiddenEffects* = 5 # list of illegal effects + effectListLen* = 6 # list of effects list nkLastBlockStmts* = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt} # these must be last statements in a block @@ -378,7 +208,7 @@ type tySequence, tyProc, tyPointer, tyOpenArray, - tyString, tyCString, tyForward, + tyString, tyCstring, tyForward, tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers tyFloat, tyFloat32, tyFloat64, tyFloat128, tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64, @@ -412,9 +242,9 @@ type tyInferred # In the initial state `base` stores a type class constraining # the types that can be inferred. After a candidate type is - # selected, it's stored in `lastSon`. Between `base` and `lastSon` + # selected, it's stored in `last`. Between `base` and `last` # there may be 0, 2 or more types that were also considered as - # possible candidates in the inference process (i.e. lastSon will + # possible candidates in the inference process (i.e. last will # be updated to store a type best conforming to all candidates) tyAnd, tyOr, tyNot @@ -439,10 +269,12 @@ type tyVoid # now different from tyEmpty, hurray! + tyIterable static: # remind us when TTypeKind stops to fit in a single 64-bit word - assert TTypeKind.high.ord <= 63 + # assert TTypeKind.high.ord <= 63 + discard const tyPureObject* = tyTuple @@ -461,6 +293,8 @@ const # consider renaming as `tyAbstractVarRange` abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tyInferred, tySink, tyOwned} + abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, + tyInferred, tySink, tyOwned} # xxx what about tyStatic? type TTypeKinds* = set[TTypeKind] @@ -491,10 +325,13 @@ type # the flag is applied to proc default values and to calls nfExecuteOnReload # A top-level statement that will be executed during reloads nfLastRead # this node is a last read - nfFirstWrite# this node is a first write + nfFirstWrite # this node is a first write + nfHasComment # node has a comment + nfSkipFieldChecking # node skips field visable checking + nfOpenSym # node is a captured sym but can be overriden by local symbols TNodeFlags* = set[TNodeFlag] - TTypeFlag* = enum # keep below 32 for efficiency reasons (now: ~40) + TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 47) tfVarargs, # procedure has C styled varargs # tyArray type represeting a varargs list tfNoSideEffect, # procedure type does not allow side effects @@ -524,7 +361,7 @@ type tfIterator, # type is really an iterator, not a tyProc tfPartial, # type is declared as 'partial' tfNotNil, # type cannot be 'nil' - tfRequiresInit, # type constains a "not nil" constraint somewhere or + tfRequiresInit, # type contains a "not nil" constraint somewhere or # a `requiresInit` field, so the default zero init # is not appropriate tfNeedsFullInit, # object type marked with {.requiresInit.} @@ -561,6 +398,11 @@ type # (for importc types); type is fully specified, allowing to compute # sizeof, alignof, offsetof at CT tfExplicitCallConv + tfIsConstructor + tfEffectSystemWorkaround + tfIsOutParam + tfSendable + tfImplicitStatic TTypeFlags* = set[TTypeFlag] @@ -596,12 +438,13 @@ type # file (it is loaded on demand, which may # mean: never) skPackage, # symbol is a package (used for canonicalization) - skAlias # an alias (needs to be resolved immediately) TSymKinds* = set[TSymKind] const routineKinds* = {skProc, skFunc, skMethod, skIterator, skConverter, skMacro, skTemplate} + ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub} + routineKinds + tfUnion* = tfNoSideEffect tfGcSafe* = tfThread tfObjHasKids* = tfEnumHasHoles @@ -609,7 +452,7 @@ const skError* = skUnknown var - eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr, tfGcSafe, tfNoSideEffect} + eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr, tfGcSafe, tfNoSideEffect, tfIsOutParam} ## type flags that are essential for type equality. ## This is now a variable because for emulation of version:1.0 we ## might exclude {tfGcSafe, tfNoSideEffect}. @@ -644,7 +487,8 @@ type mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, + mCharToStr, mBoolToStr, + mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, mImplies, mIff, mExists, mForall, mOld, @@ -657,13 +501,13 @@ type mInSet, mRepr, mExit, mSetLengthStr, mSetLengthSeq, mIsPartOf, mAstToStr, mParallel, - mSwap, mIsNil, mArrToSeq, + mSwap, mIsNil, mArrToSeq, mOpenArrayToSeq, mNewString, mNewStringOfCap, mParseBiggestFloat, - mMove, mWasMoved, mDestroy, - mDefault, mUnown, mIsolate, mAccessEnv, mReset, + mMove, mEnsureMove, mWasMoved, mDup, mDestroy, mTrace, + mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, mRef, mPtr, mVar, mDistinct, mVoid, mTuple, - mOrdinal, + mOrdinal, mIterableType, mInt, mInt8, mInt16, mInt32, mInt64, mUInt, mUInt8, mUInt16, mUInt32, mUInt64, mFloat, mFloat32, mFloat64, mFloat128, @@ -680,19 +524,19 @@ type mNctPut, mNctLen, mNctGet, mNctHasNext, mNctNext, mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, - mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo, + mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetStrVal, mNLineInfo, mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNSizeOf, mNBindSym, mNCallSite, mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2, - mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples, + mNimvm, mIntDefine, mStrDefine, mBoolDefine, mGenericDefine, mRunnableExamples, mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf, - mSymIsInstantiationOf, mNodeId + mSymIsInstantiationOf, mNodeId, mPrivateAccess, mZeroDefault -# things that we can evaluate safely at compile time, even if not asked for it: const + # things that we can evaluate safely at compile time, even if not asked for it: ctfeWhitelist* = {mNone, mSucc, mPred, mInc, mDec, mOrd, mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq, @@ -712,19 +556,26 @@ const mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, + mCharToStr, mBoolToStr, + mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, mEqStr, mLeStr, mLtStr, mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem, - mInSet, mRepr} + mInSet, mRepr, mOpenArrayToSeq} + + generatedMagics* = {mNone, mIsolate, mFinished, mOpenArrayToSeq} + ## magics that are generated as normal procs in the backend type ItemId* = object module*: int32 item*: int32 +proc `$`*(x: ItemId): string = + "(module: " & $x.module & ", item: " & $x.item & ")" + proc `==`*(a, b: ItemId): bool {.inline.} = a.item == b.item and a.module == b.module @@ -735,10 +586,6 @@ proc hash*(x: ItemId): Hash = type - TIdObj* = object of RootObj - itemId*: ItemId - PIdObj* = ref TIdObj - PNode* = ref TNode TNodeSeq* = seq[PNode] PType* = ref TType @@ -762,7 +609,8 @@ type ident*: PIdent else: sons*: TNodeSeq - comment*: string + when defined(nimsuggest): + endInfo*: TLineInfo TStrTable* = object # a table[PIdent] of PSym counter*: int @@ -783,9 +631,6 @@ type locOther # location is something other TLocFlag* = enum lfIndirect, # backend introduced a pointer - lfFullExternalName, # only used when 'conf.cmd == cmdNimfix': Indicates - # that the symbol has been imported via 'importc: "fullname"' and - # no format string. lfNoDeepCopy, # no need for a deep copy lfNoDecl, # do not declare it in C lfDynamicLib, # link symbol to dynamic library @@ -820,7 +665,7 @@ type # keep in sync with PackedLib kind*: TLibKind generated*: bool # needed for the backends: - isOverriden*: bool + isOverridden*: bool name*: Rope path*: PNode # can be a string literal! @@ -834,20 +679,22 @@ type PInstantiation* = ref TInstantiation - TScope* = object + TScope* {.acyclic.} = object depthLevel*: int symbols*: TStrTable parent*: PScope + allowPrivateAccess*: seq[PSym] # # enable access to private fields PScope* = ref TScope PLib* = ref TLib - TSym* {.acyclic.} = object of TIdObj # Keep in sync with PackedSym + TSym* {.acyclic.} = object # Keep in sync with PackedSym + itemId*: ItemId # proc and type instantiations are cached in the generic symbol case kind*: TSymKind of routineKinds: #procInstCache*: seq[PInstantiation] - gcUnsafetyReason*: PSym # for better error messages wrt gcsafe + gcUnsafetyReason*: PSym # for better error messages regarding gcsafe transformedBody*: PNode # cached body after transf pass of skLet, skVar, skField, skForVar: guard*: PSym @@ -858,6 +705,9 @@ type typ*: PType name*: PIdent info*: TLineInfo + when defined(nimsuggest): + endInfo*: TLineInfo + hasUserSpecifiedType*: bool # used for determining whether to display inlay type hints owner*: PSym flags*: TSymFlags ast*: PNode # syntax tree of proc, iterator, etc.: @@ -879,7 +729,9 @@ type # for modules, an unique index corresponding # to the module's fileIdx # for variables a slot index for the evaluator - offset*: int # offset of record field + offset*: int32 # offset of record field + disamb*: int32 # disambiguation number; the basic idea is that + # `___` is unique loc*: TLoc annex*: PLib # additional fields (seldom used, so we use a # reference to another object to save space) @@ -887,33 +739,35 @@ type cname*: string # resolved C declaration name in importc decl, e.g.: # proc fun() {.importc: "$1aux".} => cname = funaux constraint*: PNode # additional constraints like 'lit|result'; also - # misused for the codegenDecl pragma in the hope + # misused for the codegenDecl and virtual pragmas in the hope # it won't cause problems # for skModule the string literal to output for # deprecated modules. + instantiatedFrom*: PSym # for instances, the generic symbol where it came from. when defined(nimsuggest): allUsages*: seq[TLineInfo] TTypeSeq* = seq[PType] - TLockLevel* = distinct int16 TTypeAttachedOp* = enum ## as usual, order is important here + attachedWasMoved, attachedDestructor, attachedAsgn, + attachedDup, attachedSink, attachedTrace, - attachedDispose, attachedDeepCopy - TType* {.acyclic.} = object of TIdObj # \ + TType* {.acyclic.} = object # \ # types are identical iff they have the # same id; there may be multiple copies of a type # in memory! # Keep in sync with PackedType + itemId*: ItemId kind*: TTypeKind # kind of type callConv*: TCallingConvention # for procs flags*: TTypeFlags # flags of the type - sons*: TTypeSeq # base types, etc. + sons: TTypeSeq # base types, etc. n*: PNode # node for types: # for range types a nkRange node # for record types a nkRecord node @@ -930,7 +784,6 @@ type # -1 means that the size is unkwown align*: int16 # the type's alignment requirements paddingAtEnd*: int16 # - lockLevel*: TLockLevel # lock level as required for deadlock checking loc*: TLoc typeInst*: PType # for generic instantiations the tyGenericInst that led to this # type. @@ -942,24 +795,6 @@ type TPairSeq* = seq[TPair] - TIdPair* = object - key*: PIdObj - val*: RootRef - - TIdPairSeq* = seq[TIdPair] - TIdTable* = object # the same as table[PIdent] of PObject - counter*: int - data*: TIdPairSeq - - TIdNodePair* = object - key*: PIdObj - val*: PNode - - TIdNodePairSeq* = seq[TIdNodePair] - TIdNodeTable* = object # the same as table[PIdObj] of PNode - counter*: int - data*: TIdNodePairSeq - TNodePair* = object h*: Hash # because it is expensive to compute! key*: PNode @@ -979,13 +814,47 @@ type TImplication* = enum impUnknown, impNo, impYes +template nodeId(n: PNode): int = cast[int](n) + +type Gconfig = object + # we put comments in a side channel to avoid increasing `sizeof(TNode)`, which + # reduces memory usage given that `PNode` is the most allocated type by far. + comments: Table[int, string] # nodeId => comment + useIc*: bool + +var gconfig {.threadvar.}: Gconfig + +proc setUseIc*(useIc: bool) = gconfig.useIc = useIc + +proc comment*(n: PNode): string = + if nfHasComment in n.flags and not gconfig.useIc: + # IC doesn't track comments, see `packed_ast`, so this could fail + result = gconfig.comments[n.nodeId] + else: + result = "" + +proc `comment=`*(n: PNode, a: string) = + let id = n.nodeId + if a.len > 0: + # if needed, we could periodically cleanup gconfig.comments when its size increases, + # to ensure only live nodes (and with nfHasComment) have an entry in gconfig.comments; + # for compiling compiler, the waste is very small: + # num calls to newNodeImpl: 14984160 (num of PNode allocations) + # size of gconfig.comments: 33585 + # num of nodes with comments that were deleted and hence wasted: 3081 + n.flags.incl nfHasComment + gconfig.comments[id] = a + elif nfHasComment in n.flags: + n.flags.excl nfHasComment + gconfig.comments.del(id) + # BUGFIX: a module is overloadable so that a proc can have the # same name as an imported module. This is necessary because of # the poor naming choices in the standard library. const OverloadableSyms* = {skProc, skFunc, skMethod, skIterator, - skConverter, skModule, skTemplate, skMacro} + skConverter, skModule, skTemplate, skMacro, skEnumField} GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody, tyGenericParam} @@ -999,23 +868,22 @@ const tyBool, tyChar, tyEnum, tyArray, tyObject, tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc, tyPointer, - tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128, + tyOpenArray, tyString, tyCstring, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64} IntegralTypes* = {tyBool, tyChar, tyEnum, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64} # weird name because it contains tyFloat ConstantDataTypes*: TTypeKinds = {tyArray, tySet, tyTuple, tySequence} - NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, + NilableTypes*: TTypeKinds = {tyPointer, tyCstring, tyRef, tyPtr, tyProc, tyError} # TODO PtrLikeKinds*: TTypeKinds = {tyPointer, tyPtr} # for VM - ExportableSymKinds* = {skVar, skConst, skProc, skFunc, skMethod, skType, - skIterator, - skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, nfIsRef, nfIsPtr, nfPreventCg, nfLL, nfFromTemplate, nfDefaultRefsParam, - nfExecuteOnReload, nfLastRead, nfFirstWrite} + nfExecuteOnReload, nfLastRead, + nfFirstWrite, nfSkipFieldChecking, + nfOpenSym} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 @@ -1028,8 +896,6 @@ const nfAllFieldsSet* = nfBase2 - nkCallKinds* = {nkCall, nkInfix, nkPrefix, nkPostfix, - nkCommand, nkCallStrLit, nkHiddenCallConv} nkIdentKinds* = {nkIdent, nkSym, nkAccQuoted, nkOpenSymChoice, nkClosedSymChoice} @@ -1051,28 +917,20 @@ const defaultSize = -1 defaultAlignment = -1 - defaultOffset = -1 + defaultOffset* = -1 - -proc getnimblePkg*(a: PSym): PSym = - result = a - while result != nil: - case result.kind - of skModule: - result = result.owner - assert result.kind == skPackage - of skPackage: - if result.owner == nil: - break - else: - result = result.owner - else: - assert false, $result.kind +proc getPIdent*(a: PNode): PIdent {.inline.} = + ## Returns underlying `PIdent` for `{nkSym, nkIdent}`, or `nil`. + case a.kind + of nkSym: a.sym.name + of nkIdent: a.ident + of nkOpenSymChoice, nkClosedSymChoice: a.sons[0].sym.name + else: nil const moduleShift = when defined(cpu32): 20 else: 24 -template id*(a: PIdObj): int = +template id*(a: PType | PSym): int = let x = a (x.itemId.module.int shl moduleShift) + x.itemId.item.int @@ -1082,15 +940,19 @@ type symId*: int32 typeId*: int32 sealed*: bool + disambTable*: CountTable[PIdent] const PackageModuleId* = -3'i32 proc idGeneratorFromModule*(m: PSym): IdGenerator = assert m.kind == skModule - result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0) + result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0, disambTable: initCountTable[PIdent]()) -proc nextSymId*(x: IdGenerator): ItemId {.inline.} = +proc idGeneratorForPackage*(nextIdWillBe: int32): IdGenerator = + result = IdGenerator(module: PackageModuleId, symId: nextIdWillBe - 1'i32, typeId: 0, disambTable: initCountTable[PIdent]()) + +proc nextSymId(x: IdGenerator): ItemId {.inline.} = assert(not x.sealed) inc x.symId result = ItemId(module: x.module, item: x.symId) @@ -1113,22 +975,14 @@ when false: assert dest.ItemId.item <= src.ItemId.item dest = src -proc getnimblePkgId*(a: PSym): int = - let b = a.getnimblePkg - result = if b == nil: -1 else: b.id - var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things -#var -# gMainPackageId*: int proc isCallExpr*(n: PNode): bool = result = n.kind in nkCallKinds proc discardSons*(father: PNode) -type Indexable = PNode | PType - -proc len*(n: Indexable): int {.inline.} = +proc len*(n: PNode): int {.inline.} = result = n.sons.len proc safeLen*(n: PNode): int {.inline.} = @@ -1142,25 +996,112 @@ proc safeArrLen*(n: PNode): int {.inline.} = elif n.kind in {nkNone..nkFloat128Lit}: result = 0 else: result = n.len -proc add*(father, son: Indexable) = +proc add*(father, son: PNode) = assert son != nil father.sons.add(son) -proc addAllowNil*(father, son: Indexable) {.inline.} = +proc addAllowNil*(father, son: PNode) {.inline.} = father.sons.add(son) -template `[]`*(n: Indexable, i: int): Indexable = n.sons[i] -template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x +template `[]`*(n: PNode, i: int): PNode = n.sons[i] +template `[]=`*(n: PNode, i: int; x: PNode) = n.sons[i] = x -template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int] -template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x +template `[]`*(n: PNode, i: BackwardsIndex): PNode = n[n.len - i.int] +template `[]=`*(n: PNode, i: BackwardsIndex; x: PNode) = n[n.len - i.int] = x + +proc add*(father, son: PType) = + assert son != nil + father.sons.add(son) + +proc addAllowNil*(father, son: PType) {.inline.} = + father.sons.add(son) + +template `[]`*(n: PType, i: int): PType = n.sons[i] +template `[]=`*(n: PType, i: int; x: PType) = n.sons[i] = x + +template `[]`*(n: PType, i: BackwardsIndex): PType = n[n.len - i.int] +template `[]=`*(n: PType, i: BackwardsIndex; x: PType) = n[n.len - i.int] = x + +proc getDeclPragma*(n: PNode): PNode = + ## return the `nkPragma` node for declaration `n`, or `nil` if no pragma was found. + ## Currently only supports routineDefs + {nkTypeDef}. + case n.kind + of routineDefs: + if n[pragmasPos].kind != nkEmpty: result = n[pragmasPos] + else: result = nil + of nkTypeDef: + #[ + type F3*{.deprecated: "x3".} = int + + TypeSection + TypeDef + PragmaExpr + Postfix + Ident "*" + Ident "F3" + Pragma + ExprColonExpr + Ident "deprecated" + StrLit "x3" + Empty + Ident "int" + ]# + if n[0].kind == nkPragmaExpr: + result = n[0][1] + else: + result = nil + else: + # support as needed for `nkIdentDefs` etc. + result = nil + if result != nil: + assert result.kind == nkPragma, $(result.kind, n.kind) + +proc extractPragma*(s: PSym): PNode = + ## gets the pragma node of routine/type/var/let/const symbol `s` + if s.kind in routineKinds: + result = s.ast[pragmasPos] + elif s.kind in {skType, skVar, skLet, skConst}: + if s.ast != nil and s.ast.len > 0: + if s.ast[0].kind == nkPragmaExpr and s.ast[0].len > 1: + # s.ast = nkTypedef / nkPragmaExpr / [nkSym, nkPragma] + result = s.ast[0][1] + else: + result = nil + else: + result = nil + else: + result = nil + assert result == nil or result.kind == nkPragma + +proc skipPragmaExpr*(n: PNode): PNode = + ## if pragma expr, give the node the pragmas are applied to, + ## otherwise give node itself + if n.kind == nkPragmaExpr: + result = n[0] + else: + result = n + +proc setInfoRecursive*(n: PNode, info: TLineInfo) = + ## set line info recursively + if n != nil: + for i in 0.. 0: newSeq(result.sons, children) - when defined(useNodeIds): - result.id = gNodeId - if result.id == nodeIdToDebug: - echo "KIND ", result.kind - writeStackTrace() - inc gNodeId + setIdMaybe() proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode = + ## new node with line info, type, and no children result = newNode(kind) result.info = info result.typ = typ +proc newNode*(kind: TNodeKind, info: TLineInfo): PNode = + ## new node with line info, no type, and no children + newNodeImpl(info) + setIdMaybe() + +proc newAtom*(ident: PIdent, info: TLineInfo): PNode = + result = newNode(nkIdent, info) + result.ident = ident + +proc newAtom*(kind: TNodeKind, intVal: BiggestInt, info: TLineInfo): PNode = + result = newNode(kind, info) + result.intVal = intVal + +proc newAtom*(kind: TNodeKind, floatVal: BiggestFloat, info: TLineInfo): PNode = + result = newNode(kind, info) + result.floatVal = floatVal + +proc newAtom*(kind: TNodeKind; strVal: sink string; info: TLineInfo): PNode = + result = newNode(kind, info) + result.strVal = strVal + +proc newTree*(kind: TNodeKind; info: TLineInfo; children: varargs[PNode]): PNode = + result = newNodeI(kind, info) + if children.len > 0: + result.info = children[0].info + result.sons = @children + proc newTree*(kind: TNodeKind; children: varargs[PNode]): PNode = result = newNode(kind) if children.len > 0: @@ -1212,7 +1178,7 @@ proc newTreeIT*(kind: TNodeKind; info: TLineInfo; typ: PType; children: varargs[ result.sons = @children template previouslyInferred*(t: PType): PType = - if t.sons.len > 1: t.lastSon else: nil + if t.sons.len > 1: t.last else: nil when false: import tables, strutils @@ -1223,27 +1189,24 @@ when false: echo k echo v -proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym, +proc newSym*(symKind: TSymKind, name: PIdent, idgen: IdGenerator; owner: PSym, info: TLineInfo; options: TOptions = {}): PSym = # generates a symbol and initializes the hash field too + assert not name.isNil + let id = nextSymId idgen result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id, - options: options, owner: owner, offset: defaultOffset) + options: options, owner: owner, offset: defaultOffset, + disamb: getOrDefault(idgen.disambTable, name).int32) + idgen.disambTable.inc name when false: - if id.item > 2141: - let s = getStackTrace() - const words = ["createTypeBoundOps", - "initOperators", - "generateInstance", - "semIdentDef", "addLocalDecl"] - for w in words: - if w in s: - x.inc w - return - x.inc "" + if id.module == 48 and id.item == 39: + writeStackTrace() + echo "kind ", symKind, " ", name.s + if owner != nil: echo owner.name.s proc astdef*(s: PSym): PNode = # get only the definition (initializer) portion of the ast - if s.ast != nil and s.ast.kind == nkIdentDefs: + if s.ast != nil and s.ast.kind in {nkIdentDefs, nkConstDef}: s.ast[2] else: s.ast @@ -1293,11 +1256,6 @@ proc copyStrTable*(dest: var TStrTable, src: TStrTable) = setLen(dest.data, src.data.len) for i in 0..high(src.data): dest.data[i] = src.data[i] -proc copyIdTable*(dest: var TIdTable, src: TIdTable) = - dest.counter = src.counter - newSeq(dest.data, src.data.len) - for i in 0..high(src.data): dest.data[i] = src.data[i] - proc copyObjectSet*(dest: var TObjectSet, src: TObjectSet) = dest.counter = src.counter setLen(dest.data, src.data.len) @@ -1335,7 +1293,42 @@ proc newIntNode*(kind: TNodeKind, intVal: Int128): PNode = result = newNode(kind) result.intVal = castToInt64(intVal) -proc lastSon*(n: Indexable): Indexable = n.sons[^1] +proc lastSon*(n: PNode): PNode {.inline.} = n.sons[^1] +template setLastSon*(n: PNode, s: PNode) = n.sons[^1] = s + +template firstSon*(n: PNode): PNode = n.sons[0] +template secondSon*(n: PNode): PNode = n.sons[1] + +template hasSon*(n: PNode): bool = n.len > 0 +template has2Sons*(n: PNode): bool = n.len > 1 + +proc replaceFirstSon*(n, newson: PNode) {.inline.} = + n.sons[0] = newson + +proc replaceSon*(n: PNode; i: int; newson: PNode) {.inline.} = + n.sons[i] = newson + +proc last*(n: PType): PType {.inline.} = n.sons[^1] + +proc elementType*(n: PType): PType {.inline.} = n.sons[^1] +proc skipModifier*(n: PType): PType {.inline.} = n.sons[^1] + +proc indexType*(n: PType): PType {.inline.} = n.sons[0] +proc baseClass*(n: PType): PType {.inline.} = n.sons[0] + +proc base*(t: PType): PType {.inline.} = + result = t.sons[0] + +proc returnType*(n: PType): PType {.inline.} = n.sons[0] +proc setReturnType*(n, r: PType) {.inline.} = n.sons[0] = r +proc setIndexType*(n, idx: PType) {.inline.} = n.sons[0] = idx + +proc firstParamType*(n: PType): PType {.inline.} = n.sons[1] +proc firstGenericParam*(n: PType): PType {.inline.} = n.sons[1] + +proc typeBodyImpl*(n: PType): PType {.inline.} = n.sons[^1] + +proc genericHead*(n: PType): PType {.inline.} = n.sons[0] proc skipTypes*(t: PType, kinds: TTypeKinds): PType = ## Used throughout the compiler code to test whether a type tree contains or @@ -1343,7 +1336,7 @@ proc skipTypes*(t: PType, kinds: TTypeKinds): PType = ## last child nodes of a type tree need to be searched. This is a really hot ## path within the compiler! result = t - while result.kind in kinds: result = lastSon(result) + while result.kind in kinds: result = last(result) proc newIntTypeNode*(intVal: BiggestInt, typ: PType): PNode = let kind = skipTypes(typ, abstractVarRange).kind @@ -1364,7 +1357,7 @@ proc newIntTypeNode*(intVal: BiggestInt, typ: PType): PNode = result = newNode(nkIntLit) of tyStatic: # that's a pre-existing bug, will fix in another PR result = newNode(nkIntLit) - else: doAssert false, $kind + else: raiseAssert $kind result.intVal = intVal result.typ = typ @@ -1393,16 +1386,8 @@ proc newProcNode*(kind: TNodeKind, info: TLineInfo, body: PNode, pragmas, exceptions, body] const - UnspecifiedLockLevel* = TLockLevel(-1'i16) - MaxLockLevel* = 1000'i16 - UnknownLockLevel* = TLockLevel(1001'i16) AttachedOpToStr*: array[TTypeAttachedOp, string] = [ - "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"] - -proc `$`*(x: TLockLevel): string = - if x.ord == UnspecifiedLockLevel.ord: result = "" - elif x.ord == UnknownLockLevel.ord: result = "" - else: result = $int16(x) + "=wasMoved", "=destroy", "=copy", "=dup", "=sink", "=trace", "=deepcopy"] proc `$`*(s: PSym): string = if s != nil: @@ -1410,27 +1395,139 @@ proc `$`*(s: PSym): string = else: result = "" -proc newType*(kind: TTypeKind, id: ItemId; owner: PSym): PType = +when false: + iterator items*(t: PType): PType = + for i in 0.. 0 + +proc hasElementType*(t: PType): bool {.inline.} = t.sons.len > 0 +proc isEmptyTupleType*(t: PType): bool {.inline.} = t.sons.len == 0 +proc isSingletonTupleType*(t: PType): bool {.inline.} = t.sons.len == 1 + +proc genericConstraint*(t: PType): PType {.inline.} = t.sons[0] + +iterator genericInstParams*(t: PType): (bool, PType) = + for i in 1..= b.sons.len: + yield (false, nil, nil) + else: + yield (true, a.sons[i], b.sons[i]) + +iterator genericBodyParams*(t: PType): (int, PType) = + for i in 0.. 1: + setLen(t.sons, 1) + proc assignType*(dest, src: PType) = dest.kind = src.kind dest.flags = src.flags @@ -1438,28 +1535,31 @@ proc assignType*(dest, src: PType) = dest.n = src.n dest.size = src.size dest.align = src.align - dest.lockLevel = src.lockLevel # this fixes 'type TLock = TSysLock': if src.sym != nil: if dest.sym != nil: - dest.sym.flags.incl src.sym.flags-{sfExported} + dest.sym.flags.incl src.sym.flags-{sfUsed, sfExported} if dest.sym.annex == nil: dest.sym.annex = src.sym.annex mergeLoc(dest.sym.loc, src.sym.loc) else: dest.sym = src.sym - newSons(dest, src.len) - for i in 0..` and ## returned. Otherwise ``typ`` is simply returned as-is. result = typ if typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject: - result = newType(tyRef, nextTypeId(idgen), typ.owner) - rawAddSon(result, typ) + result = newType(tyRef, idgen, typ.owner, typ) proc toObject*(typ: PType): PType = ## If ``typ`` is a tyRef then its immediate son is returned (which in many ## cases should be a ``tyObject``). ## Otherwise ``typ`` is simply returned as-is. let t = typ.skipTypes({tyAlias, tyGenericInst}) - if t.kind == tyRef: t.lastSon + if t.kind == tyRef: t.elementType else: typ +proc toObjectFromRefPtrGeneric*(typ: PType): PType = + #[ + See also `toObject`. + Finds the underlying `object`, even in cases like these: + type + B[T] = object f0: int + A1[T] = ref B[T] + A2[T] = ref object f1: int + A3 = ref object f2: int + A4 = object f3: int + ]# + result = typ + while true: + case result.kind + of tyGenericBody: result = result.last + of tyRef, tyPtr, tyGenericInst, tyGenericInvocation, tyAlias: result = result[0] + # automatic dereferencing is deep, refs #18298. + else: break + # result does not have to be object type + proc isImportedException*(t: PType; conf: ConfigRef): bool = assert t != nil @@ -1884,12 +1976,10 @@ proc isImportedException*(t: PType; conf: ConfigRef): bool = return false let base = t.skipTypes({tyAlias, tyPtr, tyDistinct, tyGenericInst}) - - if base.sym != nil and {sfCompileToCpp, sfImportc} * base.sym.flags != {}: - result = true + result = base.sym != nil and {sfCompileToCpp, sfImportc} * base.sym.flags != {} proc isInfixAs*(n: PNode): bool = - return n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s == "as" + return n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.id == ord(wAs) proc skipColon*(n: PNode): PNode = result = n @@ -1897,10 +1987,12 @@ proc skipColon*(n: PNode): PNode = result = n[1] proc findUnresolvedStatic*(n: PNode): PNode = - # n.typ == nil: see issue #14802 if n.kind == nkSym and n.typ != nil and n.typ.kind == tyStatic and n.typ.n == nil: return n - + if n.typ != nil and n.typ.kind == tyTypeDesc: + let t = skipTypes(n.typ, {tyTypeDesc}) + if t.kind == tyGenericParam and not t.genericParamHasConstraints: + return n for son in n: let n = son.findUnresolvedStatic if n != nil: return n @@ -1929,20 +2021,29 @@ template detailedInfo*(sym: PSym): string = proc isInlineIterator*(typ: PType): bool {.inline.} = typ.kind == tyProc and tfIterator in typ.flags and typ.callConv != ccClosure +proc isIterator*(typ: PType): bool {.inline.} = + typ.kind == tyProc and tfIterator in typ.flags + proc isClosureIterator*(typ: PType): bool {.inline.} = typ.kind == tyProc and tfIterator in typ.flags and typ.callConv == ccClosure proc isClosure*(typ: PType): bool {.inline.} = typ.kind == tyProc and typ.callConv == ccClosure +proc isNimcall*(s: PSym): bool {.inline.} = + s.typ.callConv == ccNimCall + +proc isExplicitCallConv*(s: PSym): bool {.inline.} = + tfExplicitCallConv in s.typ.flags + proc isSinkParam*(s: PSym): bool {.inline.} = s.kind == skParam and (s.typ.kind == tySink or tfHasOwned in s.typ.flags) proc isSinkType*(t: PType): bool {.inline.} = t.kind == tySink or tfHasOwned in t.flags -proc newProcType*(info: TLineInfo; id: ItemId; owner: PSym): PType = - result = newType(tyProc, id, owner) +proc newProcType*(info: TLineInfo; idgen: IdGenerator; owner: PSym): PType = + result = newType(tyProc, idgen, owner) result.n = newNodeI(nkFormalParams, info) rawAddSon(result, nil) # return type # result.n[0] used to be `nkType`, but now it's `nkEffectList` because @@ -1951,7 +2052,7 @@ proc newProcType*(info: TLineInfo; id: ItemId; owner: PSym): PType = result.n.add newNodeI(nkEffectList, info) proc addParam*(procType: PType; param: PSym) = - param.position = procType.len-1 + param.position = procType.sons.len-1 procType.n.add newSymNode(param) rawAddSon(procType, param.typ) @@ -1993,9 +2094,36 @@ proc toHumanStr*(kind: TTypeKind): string = ## strips leading `tk` result = toHumanStrImpl(kind, 2) -proc skipAddr*(n: PNode): PNode {.inline.} = +proc skipHiddenAddr*(n: PNode): PNode {.inline.} = (if n.kind == nkHiddenAddr: n[0] else: n) proc isNewStyleConcept*(n: PNode): bool {.inline.} = assert n.kind == nkTypeClassTy result = n[0].kind == nkEmpty + +proc isOutParam*(t: PType): bool {.inline.} = tfIsOutParam in t.flags + +const + nodesToIgnoreSet* = {nkNone..pred(nkSym), succ(nkSym)..nkNilLit, + nkTypeSection, nkProcDef, nkConverterDef, + nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, + nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, + nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, + nkTypeOfExpr, nkMixinStmt, nkBindStmt} + +proc isTrue*(n: PNode): bool = + n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or + n.kind == nkIntLit and n.intVal != 0 + +type + TypeMapping* = Table[ItemId, PType] + SymMapping* = Table[ItemId, PSym] + +template idTableGet*(tab: typed; key: PSym | PType): untyped = tab.getOrDefault(key.itemId) +template idTablePut*(tab: typed; key, val: PSym | PType) = tab[key.itemId] = val + +template initSymMapping*(): Table[ItemId, PSym] = initTable[ItemId, PSym]() +template initTypeMapping*(): Table[ItemId, PType] = initTable[ItemId, PType]() + +template resetIdTable*(tab: Table[ItemId, PSym]) = tab.clear() +template resetIdTable*(tab: Table[ItemId, PType]) = tab.clear() diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 300089d812..7a9892f78a 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -12,18 +12,18 @@ # the data structures here are used in various places of the compiler. import - ast, hashes, intsets, strutils, options, lineinfos, ropes, idents, rodutils, + ast, astyaml, options, lineinfos, idents, rodutils, msgs -proc hashNode*(p: RootRef): Hash -proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope - # Convert a tree into its YAML representation; this is used by the - # YAML code generator and it is invaluable for debugging purposes. - # If maxRecDepht <> -1 then it won't print the whole graph. -proc typeToYaml*(conf: ConfigRef; n: PType, indent: int = 0, maxRecDepth: int = - 1): Rope -proc symToYaml*(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1): Rope -proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): Rope +import std/[hashes, intsets] +import std/strutils except addf +export astyaml.treeToYaml, astyaml.typeToYaml, astyaml.symToYaml, astyaml.lineInfoToStr + +when defined(nimPreviewSlimSystem): + import std/assertions + +proc hashNode*(p: RootRef): Hash # these are for debugging only: They are not really deprecated, but I want # the warning so that release versions do not contain debugging statements: @@ -31,15 +31,6 @@ proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.} proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.} proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.} -proc typekinds*(t: PType) {.deprecated.} = - var t = t - var s = "" - while t != nil and t.len > 0: - s.add $t.kind - s.add " " - t = t.lastSon - echo s - template debug*(x: PSym|PType|PNode) {.deprecated.} = when compiles(c.config): debug(c.config, x) @@ -74,16 +65,6 @@ template mdbg*: bool {.deprecated.} = else: error() -# --------------------------- ident tables ---------------------------------- -proc idTableGet*(t: TIdTable, key: PIdObj): RootRef -proc idTableGet*(t: TIdTable, key: int): RootRef -proc idTablePut*(t: var TIdTable, key: PIdObj, val: RootRef) -proc idTableHasObjectAsKey*(t: TIdTable, key: PIdObj): bool - # checks if `t` contains the `key` (compared by the pointer value, not only - # `key`'s id) -proc idNodeTableGet*(t: TIdNodeTable, key: PIdObj): PNode -proc idNodeTablePut*(t: var TIdNodeTable, key: PIdObj, val: PNode) - # --------------------------------------------------------------------------- proc lookupInRecord*(n: PNode, field: PIdent): PSym @@ -104,7 +85,7 @@ type data*: TIIPairSeq -proc initIiTable*(x: var TIITable) +proc initIITable*(x: var TIITable) proc iiTableGet*(t: TIITable, key: int): int proc iiTablePut*(t: var TIITable, key, val: int) @@ -192,6 +173,7 @@ proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym = result = nil proc sameIgnoreBacktickGensymInfo(a, b: string): bool = + result = false if a[0] != b[0]: return false var alen = a.len - 1 while alen > 0 and a[alen] != '`': dec(alen) @@ -221,11 +203,11 @@ proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym = ## Named parameters are special because a named parameter can be ## gensym'ed and then they have '\`' suffix that we need to ## ignore, see compiler / evaltempl.nim, snippet: - ## - ## .. code-block:: nim - ## + ## ```nim ## result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id), ## if c.instLines: actual.info else: templ.info) + ## ``` + result = nil for i in 1.. counter) result = (length * 2 < counter * 3) or (length - counter < 4) -proc rspaces(x: int): Rope = - # returns x spaces - result = rope(spaces(x)) - -proc toYamlChar(c: char): string = - case c - of '\0'..'\x1F', '\x7F'..'\xFF': result = "\\u" & strutils.toHex(ord(c), 4) - of '\'', '\"', '\\': result = '\\' & c - else: result = $c - -proc makeYamlString*(s: string): Rope = - # We have to split long strings into many ropes. Otherwise - # this could trigger InternalError(111). See the ropes module for - # further information. - const MaxLineLength = 64 - result = nil - var res = "\"" - for i in 0.. 0: - result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)]) - result.addf("$N$1\"magic\": $2", [istr, makeYamlString($n.magic)]) - result.addf("$N$1\"ast\": $2", [istr, ast]) - result.addf("$N$1\"options\": $2", [istr, flagsToStr(n.options)]) - result.addf("$N$1\"position\": $2", [istr, rope(n.position)]) - result.addf("$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)]) - result.addf("$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)]) - if card(n.loc.flags) > 0: - result.addf("$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)]) - result.addf("$N$1\"r\": $2", [istr, n.loc.r]) - result.addf("$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)]) - result.addf("$N$1}", [rspaces(indent)]) - -proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int, - maxRecDepth: int): Rope = - var sonsRope: Rope - if n == nil: - sonsRope = rope("null") - elif containsOrIncl(marker, n.id): - sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope( - strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))] - else: - if n.len > 0: - sonsRope = rope("[") - for i in 0.. 0: sonsRope.add(",") - sonsRope.addf("$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n[i], - marker, indent + 4, maxRecDepth - 1)]) - sonsRope.addf("$N$1]", [rspaces(indent + 2)]) - else: - sonsRope = rope("null") - - let istr = rspaces(indent + 2) - result = rope("{") - result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)]) - result.addf("$N$1\"sym\": $2", [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)]) - result.addf("$N$1\"n\": $2", [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)]) - if card(n.flags) > 0: - result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)]) - result.addf("$N$1\"callconv\": $2", [istr, makeYamlString($n.callConv)]) - result.addf("$N$1\"size\": $2", [istr, rope(n.size)]) - result.addf("$N$1\"align\": $2", [istr, rope(n.align)]) - result.addf("$N$1\"sons\": $2", [istr, sonsRope]) - -proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int, - maxRecDepth: int): Rope = - if n == nil: - result = rope("null") - else: - var istr = rspaces(indent + 2) - result = "{$N$1\"kind\": $2" % [istr, makeYamlString($n.kind)] - if maxRecDepth != 0: - if conf != nil: - result.addf(",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)]) - case n.kind - of nkCharLit..nkInt64Lit: - result.addf(",$N$1\"intVal\": $2", [istr, rope(n.intVal)]) - of nkFloatLit, nkFloat32Lit, nkFloat64Lit: - result.addf(",$N$1\"floatVal\": $2", - [istr, rope(n.floatVal.toStrMaxPrecision)]) - of nkStrLit..nkTripleStrLit: - result.addf(",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)]) - of nkSym: - result.addf(",$N$1\"sym\": $2", - [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth)]) - of nkIdent: - if n.ident != nil: - result.addf(",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)]) - else: - result.addf(",$N$1\"ident\": null", [istr]) - else: - if n.len > 0: - result.addf(",$N$1\"sons\": [", [istr]) - for i in 0.. 0: result.add(",") - result.addf("$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n[i], - marker, indent + 4, maxRecDepth - 1)]) - result.addf("$N$1]", [istr]) - result.addf(",$N$1\"typ\": $2", - [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth)]) - result.addf("$N$1}", [rspaces(indent)]) - -proc treeToYaml(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope = - var marker = initIntSet() - result = treeToYamlAux(conf, n, marker, indent, maxRecDepth) - -proc typeToYaml(conf: ConfigRef; n: PType, indent: int = 0, maxRecDepth: int = - 1): Rope = - var marker = initIntSet() - result = typeToYamlAux(conf, n, marker, indent, maxRecDepth) - -proc symToYaml(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1): Rope = - var marker = initIntSet() - result = symToYamlAux(conf, n, marker, indent, maxRecDepth) - -import tables +import std/tables const backrefStyle = "\e[90m" const enumStyle = "\e[34m" @@ -564,14 +384,12 @@ proc value(this: var DebugPrinter; value: PType) = this.key "n" this.value value.n - if value.len > 0: - this.key "sons" - this.openBracket - for i in 0.. 0: this.comma + this.value a + this.closeBracket if value.n != nil: this.key "n" @@ -585,6 +403,9 @@ proc value(this: var DebugPrinter; value: PNode) = this.openCurly this.key "kind" this.value value.kind + if value.comment.len > 0: + this.key "comment" + this.value value.comment when defined(useNodeIds): this.key "id" this.value value.id @@ -637,30 +458,33 @@ proc value(this: var DebugPrinter; value: PNode) = proc debug(n: PSym; conf: ConfigRef) = - var this: DebugPrinter - this.visited = initTable[pointer, int]() - this.renderSymType = true - this.useColor = not defined(windows) + var this = DebugPrinter( + visited: initTable[pointer, int](), + renderSymType: true, + useColor: not defined(windows) + ) this.value(n) echo($this.res) proc debug(n: PType; conf: ConfigRef) = - var this: DebugPrinter - this.visited = initTable[pointer, int]() - this.renderSymType = true - this.useColor = not defined(windows) + var this = DebugPrinter( + visited: initTable[pointer, int](), + renderSymType: true, + useColor: not defined(windows) + ) this.value(n) echo($this.res) proc debug(n: PNode; conf: ConfigRef) = - var this: DebugPrinter - this.visited = initTable[pointer, int]() - #this.renderSymType = true - this.useColor = not defined(windows) + var this = DebugPrinter( + visited: initTable[pointer, int](), + renderSymType: false, + useColor: not defined(windows) + ) this.value(n) echo($this.res) -proc nextTry(h, maxHash: Hash): Hash = +proc nextTry(h, maxHash: Hash): Hash {.inline.} = result = ((5 * h) + 1) and maxHash # For any initial h in range(maxHash), repeating that maxHash times # generates each int in range(maxHash) exactly once (see any text on @@ -759,9 +583,9 @@ proc strTableAdd*(t: var TStrTable, n: PSym) = proc strTableInclReportConflict*(t: var TStrTable, n: PSym; onConflictKeepOld = false): PSym = - # returns true if n is already in the string table: - # It is essential that `n` is written nevertheless! - # This way the newest redefinition is picked by the semantic analyses! + # if `t` has a conflicting symbol (same identifier as `n`), return it + # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true` + # and a conflict was found. assert n.name != nil var h: Hash = n.name.h and high(t.data) var replaceSlot = -1 @@ -777,9 +601,10 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym; replaceSlot = h h = nextTry(h, high(t.data)) if replaceSlot >= 0: + result = t.data[replaceSlot] # found it if not onConflictKeepOld: t.data[replaceSlot] = n # overwrite it with newer definition! - return t.data[replaceSlot] # found it + return result # but return the old one elif mustRehash(t.data.len, t.counter): strTableEnlarge(t) strTableRawInsert(t.data, n) @@ -808,16 +633,21 @@ type name*: PIdent proc nextIdentIter*(ti: var TIdentIter, tab: TStrTable): PSym = + # hot spots var h = ti.h and high(tab.data) var start = h - result = tab.data[h] - while result != nil: - if result.name.id == ti.name.id: break + var p {.cursor.} = tab.data[h] + while p != nil: + if p.name.id == ti.name.id: break h = nextTry(h, high(tab.data)) if h == start: - result = nil + p = nil break - result = tab.data[h] + p = tab.data[h] + if p != nil: + result = p # increase the count + else: + result = nil ti.h = nextTry(h, high(tab.data)) proc initIdentIter*(ti: var TIdentIter, tab: TStrTable, s: PIdent): PSym = @@ -877,125 +707,12 @@ proc initTabIter*(ti: var TTabIter, tab: TStrTable): PSym = result = nextIter(ti, tab) iterator items*(tab: TStrTable): PSym = - var it: TTabIter + var it: TTabIter = default(TTabIter) var s = initTabIter(it, tab) while s != nil: yield s s = nextIter(it, tab) -proc hasEmptySlot(data: TIdPairSeq): bool = - for h in 0..high(data): - if data[h].key == nil: - return true - result = false - -proc idTableRawGet(t: TIdTable, key: int): int = - var h: Hash - h = key and high(t.data) # start with real hash value - while t.data[h].key != nil: - if t.data[h].key.id == key: - return h - h = nextTry(h, high(t.data)) - result = - 1 - -proc idTableHasObjectAsKey(t: TIdTable, key: PIdObj): bool = - var index = idTableRawGet(t, key.id) - if index >= 0: result = t.data[index].key == key - else: result = false - -proc idTableGet(t: TIdTable, key: PIdObj): RootRef = - var index = idTableRawGet(t, key.id) - if index >= 0: result = t.data[index].val - else: result = nil - -proc idTableGet(t: TIdTable, key: int): RootRef = - var index = idTableRawGet(t, key) - if index >= 0: result = t.data[index].val - else: result = nil - -iterator pairs*(t: TIdTable): tuple[key: int, value: RootRef] = - for i in 0..high(t.data): - if t.data[i].key != nil: - yield (t.data[i].key.id, t.data[i].val) - -proc idTableRawInsert(data: var TIdPairSeq, key: PIdObj, val: RootRef) = - var h: Hash - h = key.id and high(data) - while data[h].key != nil: - assert(data[h].key.id != key.id) - h = nextTry(h, high(data)) - assert(data[h].key == nil) - data[h].key = key - data[h].val = val - -proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) = - var - index: int - n: TIdPairSeq - index = idTableRawGet(t, key.id) - if index >= 0: - assert(t.data[index].key != nil) - t.data[index].val = val - else: - if mustRehash(t.data.len, t.counter): - newSeq(n, t.data.len * GrowthFactor) - for i in 0..high(t.data): - if t.data[i].key != nil: - idTableRawInsert(n, t.data[i].key, t.data[i].val) - assert(hasEmptySlot(n)) - swap(t.data, n) - idTableRawInsert(t.data, key, val) - inc(t.counter) - -iterator idTablePairs*(t: TIdTable): tuple[key: PIdObj, val: RootRef] = - for i in 0..high(t.data): - if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val) - -proc idNodeTableRawGet(t: TIdNodeTable, key: PIdObj): int = - var h: Hash - h = key.id and high(t.data) # start with real hash value - while t.data[h].key != nil: - if t.data[h].key.id == key.id: - return h - h = nextTry(h, high(t.data)) - result = - 1 - -proc idNodeTableGet(t: TIdNodeTable, key: PIdObj): PNode = - var index: int - index = idNodeTableRawGet(t, key) - if index >= 0: result = t.data[index].val - else: result = nil - -proc idNodeTableRawInsert(data: var TIdNodePairSeq, key: PIdObj, val: PNode) = - var h: Hash - h = key.id and high(data) - while data[h].key != nil: - assert(data[h].key.id != key.id) - h = nextTry(h, high(data)) - assert(data[h].key == nil) - data[h].key = key - data[h].val = val - -proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) = - var index = idNodeTableRawGet(t, key) - if index >= 0: - assert(t.data[index].key != nil) - t.data[index].val = val - else: - if mustRehash(t.data.len, t.counter): - var n: TIdNodePairSeq - newSeq(n, t.data.len * GrowthFactor) - for i in 0..high(t.data): - if t.data[i].key != nil: - idNodeTableRawInsert(n, t.data[i].key, t.data[i].val) - swap(t.data, n) - idNodeTableRawInsert(t.data, key, val) - inc(t.counter) - -iterator pairs*(t: TIdNodeTable): tuple[key: PIdObj, val: PNode] = - for i in 0..high(t.data): - if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val) - proc initIITable(x: var TIITable) = x.counter = 0 newSeq(x.data, StartSize) @@ -1041,15 +758,8 @@ proc iiTablePut(t: var TIITable, key, val: int) = iiTableRawInsert(t.data, key, val) inc(t.counter) -proc isAddrNode*(n: PNode): bool = - case n.kind - of nkAddr, nkHiddenAddr: true - of nkCallKinds: - if n[0].kind == nkSym and n[0].sym.magic == mAddr: true - else: false - else: false - proc listSymbolNames*(symbols: openArray[PSym]): string = + result = "" for sym in symbols: if result.len > 0: result.add ", " diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim new file mode 100644 index 0000000000..aeeff1fd0f --- /dev/null +++ b/compiler/astmsgs.nim @@ -0,0 +1,45 @@ +# this module avoids ast depending on msgs or vice versa +import std/strutils +import options, ast, msgs + +proc typSym*(t: PType): PSym = + result = t.sym + if result == nil and t.kind == tyGenericInst: # this might need to be refined + result = t.genericHead.sym + +proc addDeclaredLoc*(result: var string, conf: ConfigRef; sym: PSym) = + result.add " [$1 declared in $2]" % [sym.kind.toHumanStr, toFileLineCol(conf, sym.info)] + +proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; sym: PSym) = + if optDeclaredLocs in conf.globalOptions and sym != nil: + addDeclaredLoc(result, conf, sym) + +proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = + # xxx figure out how to resolve `tyGenericParam`, e.g. for + # proc fn[T](a: T, b: T) = discard + # fn(1.1, "a") + let typ = typ.skipTypes(abstractInst + {tyStatic, tySequence, tyArray, tySet, tyUserTypeClassInst, tyVar, tyRef, tyPtr} - {tyRange}) + result.add " [$1" % typ.kind.toHumanStr + if typ.sym != nil: + result.add " declared in " & toFileLineCol(conf, typ.sym.info) + result.add "]" + +proc addTypeNodeDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = + result.add " [$1" % typ.kind.toHumanStr + if typ.sym != nil: + result.add " declared in " & toFileLineCol(conf, typ.sym.info) + result.add "]" + +proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; typ: PType) = + if optDeclaredLocs in conf.globalOptions: addDeclaredLoc(result, conf, typ) + +template quoteExpr*(a: string): untyped = + ## can be used for quoting expressions in error msgs. + "'" & a & "'" + +proc genFieldDefect*(conf: ConfigRef, field: string, disc: PSym): string = + let obj = disc.owner.name.s # `types.typeToString` might be better, eg for generics + result = "field '$#' is not accessible for type '$#'" % [field, obj] + if optDeclaredLocs in conf.globalOptions: + result.add " [discriminant declared in $#]" % toFileLineCol(conf, disc.info) + result.add " using '$# = " % disc.name.s diff --git a/compiler/astyaml.nim b/compiler/astyaml.nim new file mode 100644 index 0000000000..c7e090cd3b --- /dev/null +++ b/compiler/astyaml.nim @@ -0,0 +1,154 @@ +# +# +# The Nim Compiler +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# AST YAML printing + +import "."/[ast, lineinfos, msgs, options, rodutils] +import std/[intsets, strutils] + +proc addYamlString*(res: var string; s: string) = + res.add "\"" + for c in s: + case c + of '\0' .. '\x1F', '\x7F' .. '\xFF': + res.add("\\u" & strutils.toHex(ord(c), 4)) + of '\"', '\\': + res.add '\\' & c + else: + res.add c + + res.add('\"') + +proc makeYamlString(s: string): string = + result = "" + result.addYamlString(s) + +proc flagsToStr[T](flags: set[T]): string = + if flags == {}: + result = "[]" + else: + result = "" + for x in items(flags): + if result != "": + result.add(", ") + result.addYamlString($x) + result = "[" & result & "]" + +proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): string = + result = "[" + result.addYamlString(toFilename(conf, info)) + result.addf ", $1, $2]", [toLinenumber(info), toColumn(info)] + +proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; indent, maxRecDepth: int) +proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; indent, maxRecDepth: int) +proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; indent, maxRecDepth: int) + +proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; indent: int; maxRecDepth: int) = + if n == nil: + res.add("null") + elif containsOrIncl(marker, n.id): + res.addYamlString(n.name.s) + else: + let istr = spaces(indent * 4) + + res.addf("kind: $1", [makeYamlString($n.kind)]) + res.addf("\n$1name: $2", [istr, makeYamlString(n.name.s)]) + res.addf("\n$1typ: ", [istr]) + res.typeToYamlAux(conf, n.typ, marker, indent + 1, maxRecDepth - 1) + if conf != nil: + # if we don't pass the config, we probably don't care about the line info + res.addf("\n$1info: $2", [istr, lineInfoToStr(conf, n.info)]) + if card(n.flags) > 0: + res.addf("\n$1flags: $2", [istr, flagsToStr(n.flags)]) + res.addf("\n$1magic: $2", [istr, makeYamlString($n.magic)]) + res.addf("\n$1ast: ", [istr]) + res.treeToYamlAux(conf, n.ast, marker, indent + 1, maxRecDepth - 1) + res.addf("\n$1options: $2", [istr, flagsToStr(n.options)]) + res.addf("\n$1position: $2", [istr, $n.position]) + res.addf("\n$1k: $2", [istr, makeYamlString($n.loc.k)]) + res.addf("\n$1storage: $2", [istr, makeYamlString($n.loc.storage)]) + if card(n.loc.flags) > 0: + res.addf("\n$1flags: $2", [istr, makeYamlString($n.loc.flags)]) + res.addf("\n$1r: $2", [istr, n.loc.r]) + res.addf("\n$1lode: $2", [istr]) + res.treeToYamlAux(conf, n.loc.lode, marker, indent + 1, maxRecDepth - 1) + +proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; indent: int; maxRecDepth: int) = + if n == nil: + res.add("null") + elif containsOrIncl(marker, n.id): + res.addf "\"$1 @$2\"" % [$n.kind, strutils.toHex(cast[uint](n), sizeof(n) * 2)] + else: + let istr = spaces(indent * 4) + res.addf("kind: $2", [istr, makeYamlString($n.kind)]) + res.addf("\n$1sym: ") + res.symToYamlAux(conf, n.sym, marker, indent + 1, maxRecDepth - 1) + res.addf("\n$1n: ") + res.treeToYamlAux(conf, n.n, marker, indent + 1, maxRecDepth - 1) + if card(n.flags) > 0: + res.addf("\n$1flags: $2", [istr, flagsToStr(n.flags)]) + res.addf("\n$1callconv: $2", [istr, makeYamlString($n.callConv)]) + res.addf("\n$1size: $2", [istr, $(n.size)]) + res.addf("\n$1align: $2", [istr, $(n.align)]) + if n.hasElementType: + res.addf("\n$1sons:") + for a in n.kids: + res.addf("\n - ") + res.typeToYamlAux(conf, a, marker, indent + 1, maxRecDepth - 1) + +proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; indent: int; + maxRecDepth: int) = + if n == nil: + res.add("null") + else: + var istr = spaces(indent * 4) + res.addf("kind: $1" % [makeYamlString($n.kind)]) + + if maxRecDepth != 0: + if conf != nil: + res.addf("\n$1info: $2", [istr, lineInfoToStr(conf, n.info)]) + case n.kind + of nkCharLit .. nkInt64Lit: + res.addf("\n$1intVal: $2", [istr, $(n.intVal)]) + of nkFloatLit, nkFloat32Lit, nkFloat64Lit: + res.addf("\n$1floatVal: $2", [istr, n.floatVal.toStrMaxPrecision]) + of nkStrLit .. nkTripleStrLit: + res.addf("\n$1strVal: $2", [istr, makeYamlString(n.strVal)]) + of nkSym: + res.addf("\n$1sym: ", [istr]) + res.symToYamlAux(conf, n.sym, marker, indent + 1, maxRecDepth) + of nkIdent: + if n.ident != nil: + res.addf("\n$1ident: $2", [istr, makeYamlString(n.ident.s)]) + else: + res.addf("\n$1ident: null", [istr]) + else: + if n.len > 0: + res.addf("\n$1sons: ", [istr]) + for i in 0 ..< n.len: + res.addf("\n$1 - ", [istr]) + res.treeToYamlAux(conf, n[i], marker, indent + 1, maxRecDepth - 1) + if n.typ != nil: + res.addf("\n$1typ: ", [istr]) + res.typeToYamlAux(conf, n.typ, marker, indent + 1, maxRecDepth) + +proc treeToYaml*(conf: ConfigRef; n: PNode; indent: int = 0; maxRecDepth: int = -1): string = + var marker = initIntSet() + result = newStringOfCap(1024) + result.treeToYamlAux(conf, n, marker, indent, maxRecDepth) + +proc typeToYaml*(conf: ConfigRef; n: PType; indent: int = 0; maxRecDepth: int = -1): string = + var marker = initIntSet() + result = newStringOfCap(1024) + result.typeToYamlAux(conf, n, marker, indent, maxRecDepth) + +proc symToYaml*(conf: ConfigRef; n: PSym; indent: int = 0; maxRecDepth: int = -1): string = + var marker = initIntSet() + result = newStringOfCap(1024) + result.symToYamlAux(conf, n, marker, indent, maxRecDepth) diff --git a/compiler/backendpragmas.nim b/compiler/backendpragmas.nim new file mode 100644 index 0000000000..b18644810a --- /dev/null +++ b/compiler/backendpragmas.nim @@ -0,0 +1,25 @@ +import pragmas, options, ast, trees + +proc pushBackendOption(optionsStack: var seq[TOptions], options: var TOptions) = + optionsStack.add options + +proc popBackendOption(optionsStack: var seq[TOptions], options: var TOptions) = + options = optionsStack[^1] + optionsStack.setLen(optionsStack.len-1) + +proc processPushBackendOption*(optionsStack: var seq[TOptions], options: var TOptions, + n: PNode, start: int) = + pushBackendOption(optionsStack, options) + for i in start..= 0) x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or (One shl elem.modElemSize) -proc bitSetExcl(x: var TBitSet, elem: BiggestInt) = +proc bitSetExcl*(x: var TBitSet, elem: BiggestInt) = x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and not(One shl elem.modElemSize) -proc bitSetInit(b: var TBitSet, length: int) = +proc bitSetInit*(b: var TBitSet, length: int) = newSeq(b, length) -proc bitSetUnion(x: var TBitSet, y: TBitSet) = +proc bitSetUnion*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] or y[i] -proc bitSetDiff(x: var TBitSet, y: TBitSet) = +proc bitSetDiff*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] and not y[i] -proc bitSetSymDiff(x: var TBitSet, y: TBitSet) = +proc bitSetSymDiff*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] xor y[i] -proc bitSetIntersect(x: var TBitSet, y: TBitSet) = +proc bitSetIntersect*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] and y[i] -proc bitSetEquals(x, y: TBitSet): bool = +proc bitSetEquals*(x, y: TBitSet): bool = for i in 0..high(x): if x[i] != y[i]: return false result = true -proc bitSetContains(x, y: TBitSet): bool = +proc bitSetContains*(x, y: TBitSet): bool = for i in 0..high(x): if (x[i] and not y[i]) != Zero: return false @@ -96,6 +86,12 @@ const populationCount: array[uint8, uint8] = block: arr -proc bitSetCard(x: TBitSet): BiggestInt = +proc bitSetCard*(x: TBitSet): BiggestInt = + result = 0 for it in x: result.inc int(populationCount[it]) + +proc bitSetToWord*(s: TBitSet; size: int): BiggestUInt = + result = 0 + for j in 0.. 0): message(p.config, le.info, warnObservableStores, $le) + # bug #19613 prevent dangerous aliasing too: + if dest != nil and dest != le: + for i in 1.. 0: @@ -209,41 +255,43 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope = for i in 0..Field0, $1->Field1" % [rdLoc(a)] else: - result = "$1.Field0, $1.Field1" % [rdLoc(a)] + result.add "$1.Field0, $1.Field1" % [rdLoc(a)] else: - result = "$1, $1Len_0" % [rdLoc(a)] + result.add "$1, $1Len_0" % [rdLoc(a)] of tyString, tySequence: let ntyp = skipTypes(n.typ, abstractInst) if formalType.skipTypes(abstractInst).kind in {tyVar} and ntyp.kind == tyString and optSeqDestructors in p.config.globalOptions: linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)]) if ntyp.kind in {tyVar} and not compileToCpp(p.module): - var t: TLoc - t.r = "(*$1)" % [a.rdLoc] - result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)] + var t = TLoc(r: "(*$1)" % [a.rdLoc]) + result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" % + [a.rdLoc, lenExpr(p, t), dataField(p), + dataFieldAccessor(p, "*" & a.rdLoc)] else: - result = "$1$3, $2" % [a.rdLoc, lenExpr(p, a), dataField(p)] + result.add "($4) ? ($1$3) : NIM_NIL, $2" % + [a.rdLoc, lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)] of tyArray: - result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))] + result.add "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))] of tyPtr, tyRef: - case lastSon(a.t).kind + case elementType(a.t).kind of tyString, tySequence: - var t: TLoc - t.r = "(*$1)" % [a.rdLoc] - result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)] + var t = TLoc(r: "(*$1)" % [a.rdLoc]) + result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" % + [a.rdLoc, lenExpr(p, t), dataField(p), + dataFieldAccessor(p, "*" & a.rdLoc)] of tyArray: - result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, lastSon(a.t)))] + result.add "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, elementType(a.t)))] else: internalError(p.config, "openArrayLoc: " & typeToString(a.t)) else: internalError(p.config, "openArrayLoc: " & typeToString(a.t)) @@ -252,56 +300,69 @@ proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc = # Bug https://github.com/status-im/nimbus-eth2/issues/1549 # Aliasing is preferred over stack overflows. # Also don't regress for non ARC-builds, too risky. - if needsTmp and a.lode.typ != nil and p.config.selectedGC in {gcArc, gcOrc} and + if needsTmp and a.lode.typ != nil and p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc} and getSize(p.config, a.lode.typ) < 1024: - getTemp(p, a.lode.typ, result, needsInit=false) + result = getTemp(p, a.lode.typ, needsInit=false) genAssignment(p, result, a, {}) else: result = a -proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} = - var a: TLoc - initLocExpr(p, n[0], a) - ropecg(p.module, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc]) +proc literalsNeedsTmp(p: BProc, a: TLoc): TLoc = + result = getTemp(p, a.lode.typ, needsInit=false) + genAssignment(p, result, a, {}) -proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rope = +proc genArgStringToCString(p: BProc, n: PNode; result: var Rope; needsTmp: bool) {.inline.} = + var a = initLocExpr(p, n[0]) + appcg(p.module, result, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc]) + +proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Rope; needsTmp = false) = var a: TLoc if n.kind == nkStringToCString: - result = genArgStringToCString(p, n, needsTmp) + genArgStringToCString(p, n, result, needsTmp) elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}: var n = if n.kind != nkHiddenAddr: n else: n[0] - result = openArrayLoc(p, param.typ, n) - elif ccgIntroducedPtr(p.config, param, call[0].typ[0]): - initLocExpr(p, n, a) - result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp)) + openArrayLoc(p, param.typ, n, result) + elif ccgIntroducedPtr(p.config, param, call[0].typ.returnType) and + (optByRef notin param.options or not p.module.compileToCpp): + a = initLocExpr(p, n) + if n.kind in {nkCharLit..nkNilLit}: + addAddrLoc(p.config, literalsNeedsTmp(p, a), result) + else: + addAddrLoc(p.config, withTmpIfNeeded(p, a, needsTmp), result) elif p.module.compileToCpp and param.typ.kind in {tyVar} and n.kind == nkHiddenAddr: - initLocExprSingleUse(p, n[0], a) + a = initLocExprSingleUse(p, n[0]) # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still # means '*T'. See posix.nim for lots of examples that do that in the wild. let callee = call[0] if callee.kind == nkSym and {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}: - result = addrLoc(p.config, a) + addAddrLoc(p.config, a, result) else: - result = rdLoc(a) + addRdLoc(a, result) else: - initLocExprSingleUse(p, n, a) - result = rdLoc(withTmpIfNeeded(p, a, needsTmp)) + a = initLocExprSingleUse(p, n) + if param.typ.kind in abstractPtrs: + let typ = skipTypes(param.typ, abstractPtrs) + if typ.sym != nil and sfImportc in typ.sym.flags: + a.r = "(($1) ($2))" % + [getTypeDesc(p.module, param.typ), rdCharLoc(a)] + addRdLoc(withTmpIfNeeded(p, a, needsTmp), result) #assert result != nil -proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope = +proc genArgNoParam(p: BProc, n: PNode; result: var Rope; needsTmp = false) = var a: TLoc if n.kind == nkStringToCString: - result = genArgStringToCString(p, n, needsTmp) + genArgStringToCString(p, n, result, needsTmp) else: - initLocExprSingleUse(p, n, a) - result = rdLoc(withTmpIfNeeded(p, a, needsTmp)) + a = initLocExprSingleUse(p, n) + addRdLoc(withTmpIfNeeded(p, a, needsTmp), result) -from dfa import aliases, AliasKind +import aliasanalysis proc potentialAlias(n: PNode, potentialWrites: seq[PNode]): bool = + result = false for p in potentialWrites: if p.aliases(n) != no or n.aliases(p) != no: return true @@ -310,64 +371,84 @@ proc skipTrivialIndirections(n: PNode): PNode = result = n while true: case result.kind - of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}: + of nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv: result = result[0] - of {nkHiddenStdConv, nkHiddenSubConv}: + of nkHiddenStdConv, nkHiddenSubConv: result = result[1] else: break -proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] = +proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) = case n.kind: - of nkLiterals, nkIdent: discard + of nkLiterals, nkIdent, nkFormalParams: discard of nkSym: if mutate: result.add n - of nkAsgn, nkFastAsgn: - result.add getPotentialWrites(n[0], true) - result.add getPotentialWrites(n[1], mutate) + of nkAsgn, nkFastAsgn, nkSinkAsgn: + getPotentialWrites(n[0], true, result) + getPotentialWrites(n[1], mutate, result) of nkAddr, nkHiddenAddr: - result.add getPotentialWrites(n[0], true) - of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr - for s in n.sons: - result.add getPotentialWrites(s, true) + getPotentialWrites(n[0], true, result) + of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr: + getPotentialWrites(n[0], mutate, result) + of nkCallKinds: + case n.getMagic: + of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem, + mAddr, mNew, mNewFinalize, mWasMoved, mDestroy: + getPotentialWrites(n[1], true, result) + for i in 2.. 1: pl.add(~", ") + if typ.returnType != nil: + if isInvalidReturnType(p.config, typ): + if ri.len > 1: pl.add(", ") # beware of 'result = p(result)'. We may need to allocate a temporary: - if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri): + if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri): # Great, we can use 'd': if d.k == locNone: - getTemp(p, typ[0], d, needsInit=true) + d = getTemp(p, typ.returnType, needsInit=true) elif d.k notin {locTemp} and not hasNoInit(ri): # reset before pass as 'result' var: discard "resetLoc(p, d)" pl.add(addrLoc(p.config, d)) genCallPattern() + if canRaise: raiseExit(p) else: - var tmp: TLoc - getTemp(p, typ[0], tmp, needsInit=true) + var tmp: TLoc = getTemp(p, typ.returnType, needsInit=true) pl.add(addrLoc(p.config, tmp)) genCallPattern() if canRaise: raiseExit(p) genAssignment(p, d, tmp, {}) # no need for deep copying elif isHarmlessStore(p, canRaise, d): - if d.k == locNone: getTemp(p, typ[0], d) + if d.k == locNone: d = getTemp(p, typ.returnType) assert(d.t != nil) # generate an assignment to d: - var list: TLoc - initLoc(list, locCall, d.lode, OnUnknown) + var list: TLoc = initLoc(locCall, d.lode, OnUnknown) if tfIterator in typ.flags: list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc] else: @@ -448,11 +526,9 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = genAssignment(p, d, list, {}) # no need for deep copying if canRaise: raiseExit(p) else: - var tmp: TLoc - getTemp(p, typ[0], tmp) + var tmp: TLoc = getTemp(p, typ.returnType) assert(d.t != nil) # generate an assignment to d: - var list: TLoc - initLoc(list, locCall, d.lode, OnUnknown) + var list: TLoc = initLoc(locCall, d.lode, OnUnknown) if tfIterator in typ.flags: list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc] else: @@ -464,24 +540,30 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = genCallPattern() if canRaise: raiseExit(p) -proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = - if i < typ.len: +proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope; + argsCounter: var int) = + if i < typ.n.len: # 'var T' is 'T&' in C++. This means we ignore the request of # any nkHiddenAddr when it's a 'var T'. let paramType = typ.n[i] assert(paramType.kind == nkSym) if paramType.typ.isCompileTimeOnly: - result = nil - elif typ[i].kind in {tyVar} and ri[i].kind == nkHiddenAddr: - result = genArgNoParam(p, ri[i][0]) + discard + elif paramType.typ.kind in {tyVar} and ri[i].kind == nkHiddenAddr: + if argsCounter > 0: result.add ", " + genArgNoParam(p, ri[i][0], result) + inc argsCounter else: - result = genArgNoParam(p, ri[i]) #, typ.n[i].sym) + if argsCounter > 0: result.add ", " + genArgNoParam(p, ri[i], result) #, typ.n[i].sym) + inc argsCounter else: if tfVarargs notin typ.flags: localError(p.config, ri.info, "wrong argument count") - result = nil else: - result = genArgNoParam(p, ri[i]) + if argsCounter > 0: result.add ", " + genArgNoParam(p, ri[i], result) + inc argsCounter discard """ Dot call syntax in C++ @@ -538,11 +620,11 @@ proc skipAddrDeref(node: PNode): PNode = else: result = node -proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = +proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope) = # for better or worse c2nim translates the 'this' argument to a 'var T'. # However manual wrappers may also use 'ptr T'. In any case we support both # for convenience. - internalAssert p.config, i < typ.len + internalAssert p.config, i < typ.n.len assert(typ.n[i].kind == nkSym) # if the parameter is lying (tyVar) and thus we required an additional deref, # skip the deref: @@ -552,75 +634,72 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = if t.kind in {tyVar}: let x = if ri.kind == nkHiddenAddr: ri[0] else: ri if x.typ.kind == tyPtr: - result = genArgNoParam(p, x) + genArgNoParam(p, x, result) result.add("->") elif x.kind in {nkHiddenDeref, nkDerefExpr} and x[0].typ.kind == tyPtr: - result = genArgNoParam(p, x[0]) + genArgNoParam(p, x[0], result) result.add("->") else: - result = genArgNoParam(p, x) + genArgNoParam(p, x, result) result.add(".") elif t.kind == tyPtr: if ri.kind in {nkAddr, nkHiddenAddr}: - result = genArgNoParam(p, ri[0]) + genArgNoParam(p, ri[0], result) result.add(".") else: - result = genArgNoParam(p, ri) + genArgNoParam(p, ri, result) result.add("->") else: ri = skipAddrDeref(ri) if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri[0] - result = genArgNoParam(p, ri) #, typ.n[i].sym) + genArgNoParam(p, ri, result) #, typ.n[i].sym) result.add(".") -proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope = +proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Rope) = var i = 0 var j = 1 while i < pat.len: case pat[i] of '@': - var first = true + var argsCounter = 0 for k in j.. 0: - if not first: - result.add(~", ") - first = false - result.add arg + genOtherArg(p, ri, k, typ, result, argsCounter) inc i of '#': if i+1 < pat.len and pat[i+1] in {'+', '@'}: let ri = ri[j] if ri.kind in nkCallKinds: let typ = skipTypes(ri[0].typ, abstractInst) - if pat[i+1] == '+': result.add genArgNoParam(p, ri[0]) - result.add(~"(") + if pat[i+1] == '+': genArgNoParam(p, ri[0], result) + result.add("(") if 1 < ri.len: - result.add genOtherArg(p, ri, 1, typ) + var argsCounterB = 0 + genOtherArg(p, ri, 1, typ, result, argsCounterB) for k in j+1.. 0 if pat.contains({'#', '(', '@', '\''}): - var pl = genPatternCall(p, ri, pat, typ) + var pl = newRopeAppender() + genPatternCall(p, ri, pat, typ, pl) # simpler version of 'fixupCall' that works with the pl+params combination: var typ = skipTypes(ri[0].typ, abstractInst) - if typ[0] != nil: + if typ.returnType != nil: if p.module.compileToCpp and lfSingleUse in d.flags: # do not generate spurious temporaries for C++! For C we're better off # with them to prevent undefined behaviour and because the codegen @@ -653,95 +731,87 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) = d.r = pl excl d.flags, lfSingleUse else: - if d.k == locNone: getTemp(p, typ[0], d) + if d.k == locNone: d = getTemp(p, typ.returnType) assert(d.t != nil) # generate an assignment to d: - var list: TLoc - initLoc(list, locCall, d.lode, OnUnknown) + var list: TLoc = initLoc(locCall, d.lode, OnUnknown) list.r = pl genAssignment(p, d, list, {}) # no need for deep copying else: - pl.add(~";$n") + pl.add(";\n") line(p, cpsStmts, pl) else: - var pl: Rope = nil - #var param = typ.n[1].sym + var pl = newRopeAppender() + var argsCounter = 0 if 1 < ri.len: - pl.add(genThisArg(p, ri, 1, typ)) + genThisArg(p, ri, 1, typ, pl) pl.add(op.r) - var params: Rope + var params = newRopeAppender() for i in 2.. 0 var start = 3 if ' ' in pat: start = 1 pl.add(op.r) if ri.len > 1: - pl.add(~": ") - pl.add(genArg(p, ri[1], typ.n[1].sym, ri)) + pl.add(": ") + genArg(p, ri[1], typ.n[1].sym, ri, pl) start = 2 else: if ri.len > 1: - pl.add(genArg(p, ri[1], typ.n[1].sym, ri)) - pl.add(~" ") + genArg(p, ri[1], typ.n[1].sym, ri, pl) + pl.add(" ") pl.add(op.r) if ri.len > 2: - pl.add(~": ") - pl.add(genArg(p, ri[2], typ.n[2].sym, ri)) + pl.add(": ") + genArg(p, ri[2], typ.n[2].sym, ri, pl) for i in start..= typ.len: + if i >= typ.n.len: internalError(p.config, ri.info, "varargs for objective C method?") assert(typ.n[i].kind == nkSym) var param = typ.n[i].sym - pl.add(~" ") + pl.add(" ") pl.add(param.name.s) - pl.add(~": ") - pl.add(genArg(p, ri[i], param, ri)) - if typ[0] != nil: - if isInvalidReturnType(p.config, typ[0]): - if ri.len > 1: pl.add(~" ") + pl.add(": ") + genArg(p, ri[i], param, ri, pl) + if typ.returnType != nil: + if isInvalidReturnType(p.config, typ): + if ri.len > 1: pl.add(" ") # beware of 'result = p(result)'. We always allocate a temporary: if d.k in {locTemp, locNone}: # We already got a temp. Great, special case it: - if d.k == locNone: getTemp(p, typ[0], d, needsInit=true) - pl.add(~"Result: ") + if d.k == locNone: d = getTemp(p, typ.returnType, needsInit=true) + pl.add("Result: ") pl.add(addrLoc(p.config, d)) - pl.add(~"];$n") + pl.add("];\n") line(p, cpsStmts, pl) else: - var tmp: TLoc - getTemp(p, typ[0], tmp, needsInit=true) + var tmp: TLoc = getTemp(p, typ.returnType, needsInit=true) pl.add(addrLoc(p.config, tmp)) - pl.add(~"];$n") + pl.add("];\n") line(p, cpsStmts, pl) genAssignment(p, d, tmp, {}) # no need for deep copying else: - pl.add(~"]") - if d.k == locNone: getTemp(p, typ[0], d) + pl.add("]") + if d.k == locNone: d = getTemp(p, typ.returnType) assert(d.t != nil) # generate an assignment to d: - var list: TLoc - initLoc(list, locCall, ri, OnUnknown) + var list: TLoc = initLoc(locCall, ri, OnUnknown) list.r = pl genAssignment(p, d, list, {}) # no need for deep copying else: - pl.add(~"];$n") + pl.add("];\n") line(p, cpsStmts, pl) proc notYetAlive(n: PNode): bool {.inline.} = @@ -779,6 +849,5 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = genNamedParamCall(p, ri, d) else: genPrefixCall(p, le, ri, d) - postStmtActions(p) proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index ea09b34001..18fac8c711 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -9,7 +9,7 @@ # included from cgen.nim -when defined(nimCompilerStackraceHints): +when defined(nimCompilerStacktraceHints): import std/stackframes proc getNullValueAuxT(p: BProc; orig, t: PType; obj, constOrNil: PNode, @@ -18,29 +18,31 @@ proc getNullValueAuxT(p: BProc; orig, t: PType; obj, constOrNil: PNode, # -------------------------- constant expressions ------------------------ -proc int64Literal(i: BiggestInt): Rope = +proc rdSetElemLoc(conf: ConfigRef; a: TLoc, typ: PType; result: var Rope) + +proc int64Literal(i: BiggestInt; result: var Rope) = if i > low(int64): - result = "IL64($1)" % [rope(i)] + result.add "IL64($1)" % [rope(i)] else: - result = ~"(IL64(-9223372036854775807) - IL64(1))" + result.add "(IL64(-9223372036854775807) - IL64(1))" -proc uint64Literal(i: uint64): Rope = rope($i & "ULL") +proc uint64Literal(i: uint64; result: var Rope) = result.add rope($i & "ULL") -proc intLiteral(i: BiggestInt): Rope = +proc intLiteral(i: BiggestInt; result: var Rope) = if i > low(int32) and i <= high(int32): - result = rope(i) + result.add rope(i) elif i == low(int32): # Nim has the same bug for the same reasons :-) - result = ~"(-2147483647 -1)" + result.add "(-2147483647 -1)" elif i > low(int64): - result = "IL64($1)" % [rope(i)] + result.add "IL64($1)" % [rope(i)] else: - result = ~"(IL64(-9223372036854775807) - IL64(1))" + result.add "(IL64(-9223372036854775807) - IL64(1))" -proc intLiteral(i: Int128): Rope = - intLiteral(toInt64(i)) +proc intLiteral(i: Int128; result: var Rope) = + intLiteral(toInt64(i), result) -proc genLiteral(p: BProc, n: PNode, ty: PType): Rope = +proc genLiteral(p: BProc, n: PNode, ty: PType; result: var Rope) = case n.kind of nkCharLit..nkUInt64Lit: var k: TTypeKind @@ -54,62 +56,63 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope = else: k = tyNil # don't go into the case variant that uses 'ty' case k of tyChar, tyNil: - result = intLiteral(n.intVal) + intLiteral(n.intVal, result) of tyBool: - if n.intVal != 0: result = ~"NIM_TRUE" - else: result = ~"NIM_FALSE" - of tyInt64: result = int64Literal(n.intVal) - of tyUInt64: result = uint64Literal(uint64(n.intVal)) + if n.intVal != 0: result.add "NIM_TRUE" + else: result.add "NIM_FALSE" + of tyInt64: int64Literal(n.intVal, result) + of tyUInt64: uint64Literal(uint64(n.intVal), result) else: - result = "(($1) $2)" % [getTypeDesc(p.module, - ty), intLiteral(n.intVal)] + result.add "((" + result.add getTypeDesc(p.module, ty) + result.add ")" + intLiteral(n.intVal, result) + result.add ")" of nkNilLit: let k = if ty == nil: tyPointer else: skipTypes(ty, abstractVarRange).kind if k == tyProc and skipTypes(ty, abstractVarRange).callConv == ccClosure: let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) - result = p.module.tmpBase & rope(id) + let tmpName = p.module.tmpBase & rope(id) if id == p.module.labels: # not found in cache: inc(p.module.labels) - p.module.s[cfsData].addf( + p.module.s[cfsStrData].addf( "static NIM_CONST $1 $2 = {NIM_NIL,NIM_NIL};$n", - [getTypeDesc(p.module, ty), result]) + [getTypeDesc(p.module, ty), tmpName]) + result.add tmpName elif k in {tyPointer, tyNil, tyProc}: - result = rope("NIM_NIL") + result.add rope("NIM_NIL") else: - result = "(($1) NIM_NIL)" % [getTypeDesc(p.module, ty)] + result.add "(($1) NIM_NIL)" % [getTypeDesc(p.module, ty)] of nkStrLit..nkTripleStrLit: let k = if ty == nil: tyString else: skipTypes(ty, abstractVarRange + {tyStatic, tyUserTypeClass, tyUserTypeClassInst}).kind case k of tyNil: - result = genNilStringLiteral(p.module, n.info) + genNilStringLiteral(p.module, n.info, result) of tyString: # with the new semantics for not 'nil' strings, we can map "" to nil and # save tons of allocations: if n.strVal.len == 0 and optSeqDestructors notin p.config.globalOptions: - result = genNilStringLiteral(p.module, n.info) + genNilStringLiteral(p.module, n.info, result) else: - result = genStringLiteral(p.module, n) + genStringLiteral(p.module, n, result) else: - result = makeCString(n.strVal) + result.add makeCString(n.strVal) of nkFloatLit, nkFloat64Lit: - result = rope(n.floatVal.toStrMaxPrecision) + if ty.kind == tyFloat32: + result.add rope(n.floatVal.float32.toStrMaxPrecision) + else: + result.add rope(n.floatVal.toStrMaxPrecision) of nkFloat32Lit: - result = rope(n.floatVal.toStrMaxPrecision("f")) + result.add rope(n.floatVal.float32.toStrMaxPrecision) else: internalError(p.config, n.info, "genLiteral(" & $n.kind & ')') - result = nil -proc genLiteral(p: BProc, n: PNode): Rope = - result = genLiteral(p, n, n.typ) +proc genLiteral(p: BProc, n: PNode; result: var Rope) = + genLiteral(p, n, n.typ, result) -proc bitSetToWord(s: TBitSet, size: int): BiggestUInt = - result = 0 - for j in 0.. 8: var res = "{\n" for i in 0.. 8: let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) - result = p.module.tmpBase & rope(id) + let tmpName = p.module.tmpBase & rope(id) if id == p.module.labels: # not found in cache: inc(p.module.labels) - p.module.s[cfsData].addf("static NIM_CONST $1 $2 = $3;$n", - [getTypeDesc(p.module, n.typ), result, genRawSetData(cs, size)]) + p.module.s[cfsStrData].addf("static NIM_CONST $1 $2 = ", + [getTypeDesc(p.module, n.typ), tmpName]) + genRawSetData(cs, size, p.module.s[cfsStrData]) + p.module.s[cfsStrData].addf(";$n", []) + result.add tmpName else: - result = genRawSetData(cs, size) + genRawSetData(cs, size, result) proc getStorageLoc(n: PNode): TStorageLoc = ## deadcode @@ -162,7 +168,9 @@ proc getStorageLoc(n: PNode): TStorageLoc = of tyVar, tyLent: result = OnUnknown of tyPtr: result = OnStack of tyRef: result = OnHeap - else: doAssert(false, "getStorageLoc") + else: + result = OnUnknown + doAssert(false, "getStorageLoc") of nkBracketExpr, nkDotExpr, nkObjDownConv, nkObjUpConv: result = getStorageLoc(n[0]) else: result = OnUnknown @@ -204,16 +212,20 @@ proc asgnComplexity(n: PNode): int = # 'case objects' are too difficult to inline their assignment operation: result = 100 of nkRecList: + result = 0 for t in items(n): result += asgnComplexity(t) - else: discard + else: result = 0 + else: + result = 0 proc optAsgnLoc(a: TLoc, t: PType, field: Rope): TLoc = - assert field != nil - result.k = locField - result.storage = a.storage - result.lode = lodeTyp t - result.r = rdLoc(a) & "." & field + assert field != "" + result = TLoc(k: locField, + storage: a.storage, + lode: lodeTyp t, + r: rdLoc(a) & "." & field + ) proc genOptAsgnTuple(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = let newflags = @@ -224,8 +236,7 @@ proc genOptAsgnTuple(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = else: flags let t = skipTypes(dest.t, abstractInst).getUniqueType() - for i in 0..Field1, $3);$n", + [addrLoc(p.config, dest), source, genTypeInfoV1(p.module, dest.t, dest.lode.info)]) of tySet: if mapSetType(p.config, ty) == ctArray: @@ -449,7 +466,7 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) = [rdLoc(dest), rdLoc(src), getSize(p.config, dest.t)]) else: linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)]) - of tyPointer, tyChar, tyBool, tyEnum, tyCString, + of tyPointer, tyChar, tyBool, tyEnum, tyCstring, tyInt..tyUInt64, tyRange, tyVar, tyLent: linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)]) else: internalError(p.config, "genDeepCopy: " & $ty.kind) @@ -462,10 +479,9 @@ proc putLocIntoDest(p: BProc, d: var TLoc, s: TLoc) = d = s # ``d`` is free, so fill it with ``s`` proc putDataIntoDest(p: BProc, d: var TLoc, n: PNode, r: Rope) = - var a: TLoc if d.k != locNone: + var a: TLoc = initLoc(locData, n, OnStatic) # need to generate an assignment here - initLoc(a, locData, n, OnStatic) a.r = r if lfNoDeepCopy in d.flags: genAssignment(p, d, a, {}) else: genAssignment(p, d, a, {needToCopy}) @@ -477,10 +493,9 @@ proc putDataIntoDest(p: BProc, d: var TLoc, n: PNode, r: Rope) = d.r = r proc putIntoDest(p: BProc, d: var TLoc, n: PNode, r: Rope; s=OnUnknown) = - var a: TLoc if d.k != locNone: # need to generate an assignment here - initLoc(a, locExpr, n, s) + var a: TLoc = initLoc(locExpr, n, s) a.r = r if lfNoDeepCopy in d.flags: genAssignment(p, d, a, {}) else: genAssignment(p, d, a, {needToCopy}) @@ -492,49 +507,42 @@ proc putIntoDest(p: BProc, d: var TLoc, n: PNode, r: Rope; s=OnUnknown) = d.r = r proc binaryStmt(p: BProc, e: PNode, d: var TLoc, op: string) = - var a, b: TLoc if d.k != locNone: internalError(p.config, e.info, "binaryStmt") - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) lineCg(p, cpsStmts, "$1 $2 $3;$n", [rdLoc(a), op, rdLoc(b)]) proc binaryStmtAddr(p: BProc, e: PNode, d: var TLoc, cpname: string) = - var a, b: TLoc if d.k != locNone: internalError(p.config, e.info, "binaryStmtAddr") - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) lineCg(p, cpsStmts, "#$1($2, $3);$n", [cpname, byRefLoc(p, a), rdLoc(b)]) template unaryStmt(p: BProc, e: PNode, d: var TLoc, frmt: string) = - var a: TLoc if d.k != locNone: internalError(p.config, e.info, "unaryStmt") - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) lineCg(p, cpsStmts, frmt, [rdLoc(a)]) template binaryExpr(p: BProc, e: PNode, d: var TLoc, frmt: string) = - var a, b: TLoc assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) putIntoDest(p, d, e, ropecg(p.module, frmt, [rdLoc(a), rdLoc(b)])) template binaryExprChar(p: BProc, e: PNode, d: var TLoc, frmt: string) = - var a, b: TLoc assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) putIntoDest(p, d, e, ropecg(p.module, frmt, [a.rdCharLoc, b.rdCharLoc])) template unaryExpr(p: BProc, e: PNode, d: var TLoc, frmt: string) = - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) putIntoDest(p, d, e, ropecg(p.module, frmt, [rdLoc(a)])) template unaryExprChar(p: BProc, e: PNode, d: var TLoc, frmt: string) = - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) putIntoDest(p, d, e, ropecg(p.module, frmt, [rdCharLoc(a)])) template binaryArithOverflowRaw(p: BProc, t: PType, a, b: TLoc; @@ -544,12 +552,21 @@ template binaryArithOverflowRaw(p: BProc, t: PType, a, b: TLoc; else: getTypeDesc(p.module, t) var result = getTempName(p.module) linefmt(p, cpsLocals, "$1 $2;$n", [storage, result]) - lineCg(p, cpsStmts, "if (#$2($3, $4, &$1)) { #raiseOverflow(); $5};$n", - [result, cpname, rdCharLoc(a), rdCharLoc(b), raiseInstr(p)]) + lineCg(p, cpsStmts, "if (#$2($3, $4, &$1)) { #raiseOverflow(); ", + [result, cpname, rdCharLoc(a), rdCharLoc(b)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "};$n", [] + if size < p.config.target.intSize or t.kind in {tyRange, tyEnum}: - linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3){ #raiseOverflow(); $4}$n", - [result, intLiteral(firstOrd(p.config, t)), intLiteral(lastOrd(p.config, t)), - raiseInstr(p)]) + var first = newRopeAppender() + intLiteral(firstOrd(p.config, t), first) + var last = newRopeAppender() + intLiteral(lastOrd(p.config, t), last) + linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3){ #raiseOverflow(); ", + [result, first, last]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + result proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = @@ -565,11 +582,10 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = "nimAddInt64", "nimSubInt64" ] opr: array[mAddI..mPred, string] = ["+", "-", "*", "/", "%", "+", "-"] - var a, b: TLoc assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) # skipping 'range' is correct here as we'll generate a proper range check # later via 'chckRange' let t = e.typ.skipTypes(abstractRange) @@ -579,24 +595,38 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = else: # we handle div by zero here so that we know that the compilerproc's # result is only for overflows. + var needsOverflowCheck = true if m in {mDivI, mModI}: - linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n", - [rdLoc(b), raiseInstr(p)]) - - let res = binaryArithOverflowRaw(p, t, a, b, - if t.kind == tyInt64: prc64[m] else: prc[m]) - putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res]) + var canBeZero = true + if e[2].kind in {nkIntLit..nkUInt64Lit}: + canBeZero = e[2].intVal == 0 + if e[2].kind in {nkIntLit..nkInt64Lit}: + needsOverflowCheck = e[2].intVal == -1 + if canBeZero: + linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); ", [rdLoc(b)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt(p, cpsStmts, "}$n", []) + if needsOverflowCheck: + let res = binaryArithOverflowRaw(p, t, a, b, + if t.kind == tyInt64: prc64[m] else: prc[m]) + putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res]) + else: + let res = "($1)(($2) $3 ($4))" % [getTypeDesc(p.module, e.typ), rdLoc(a), rope(opr[m]), rdLoc(b)] + putIntoDest(p, d, e, res) proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = - var - a: TLoc - t: PType + var t: PType assert(e[1].typ != nil) - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) t = skipTypes(e.typ, abstractRange) if optOverflowCheck in p.options: - linefmt(p, cpsStmts, "if ($1 == $2){ #raiseOverflow(); $3}$n", - [rdLoc(a), intLiteral(firstOrd(p.config, t)), raiseInstr(p)]) + var first = newRopeAppender() + intLiteral(firstOrd(p.config, t), first) + linefmt(p, cpsStmts, "if ($1 == $2){ #raiseOverflow(); ", + [rdLoc(a), first]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + case m of mUnaryMinusI: putIntoDest(p, d, e, "((NI$2)-($1))" % [rdLoc(a), rope(getSize(p.config, t) * 8)]) @@ -609,12 +639,11 @@ proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = var - a, b: TLoc - s, k: BiggestInt + s, k: BiggestInt = 0 assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) # BUGFIX: cannot use result-type here, as it may be a boolean s = max(getSize(p.config, a.t), getSize(p.config, b.t)) * 8 k = getSize(p.config, a.t) * 8 @@ -668,11 +697,10 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = assert(false, $op) proc genEqProc(p: BProc, e: PNode, d: var TLoc) = - var a, b: TLoc assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) if a.t.skipTypes(abstractInstOwned).callConv == ccClosure: putIntoDest(p, d, e, "($1.ClP_0 == $2.ClP_0 && $1.ClE_0 == $2.ClE_0)" % [rdLoc(a), rdLoc(b)]) @@ -688,10 +716,9 @@ proc genIsNil(p: BProc, e: PNode, d: var TLoc) = proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = var - a: TLoc t: PType assert(e[1].typ != nil) - initLocExpr(p, e[1], a) + var a = initLocExpr(p, e[1]) t = skipTypes(e.typ, abstractRange) template applyFormat(frmt: untyped) = @@ -717,7 +744,7 @@ proc isCppRef(p: BProc; typ: PType): bool {.inline.} = tfVarIsPtr notin skipTypes(typ, abstractInstOwned).flags proc genDeref(p: BProc, e: PNode, d: var TLoc) = - let mt = mapType(p.config, e[0].typ, mapTypeChooser(e[0])) + let mt = mapType(p.config, e[0].typ, mapTypeChooser(e[0]) == skParam) if mt in {ctArray, ctPtrToArray} and lfEnforceDeref notin d.flags: # XXX the amount of hacks for C's arrays is incredible, maybe we should # simply wrap them in a struct? --> Losing auto vectorization then? @@ -728,13 +755,13 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc) = var a: TLoc var typ = e[0].typ if typ.kind in {tyUserTypeClass, tyUserTypeClassInst} and typ.isResolvedUserTypeClass: - typ = typ.lastSon + typ = typ.last typ = typ.skipTypes(abstractInstOwned) if typ.kind in {tyVar} and tfVarIsPtr notin typ.flags and p.module.compileToCpp and e[0].kind == nkHiddenAddr: - initLocExprSingleUse(p, e[0][0], d) + d = initLocExprSingleUse(p, e[0][0]) return else: - initLocExprSingleUse(p, e[0], a) + a = initLocExprSingleUse(p, e[0]) if d.k == locNone: # dest = *a; <-- We do not know that 'dest' is on the heap! # It is completely wrong to set 'd.storage' here, unless it's not yet @@ -771,8 +798,7 @@ proc cowBracket(p: BProc; n: PNode) = if n.kind == nkBracketExpr and optSeqDestructors in p.config.globalOptions: let strCandidate = n[0] if strCandidate.typ.skipTypes(abstractInst).kind == tyString: - var a: TLoc - initLocExpr(p, strCandidate, a) + var a: TLoc = initLocExpr(p, strCandidate) linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)]) proc cow(p: BProc; n: PNode) {.inline.} = @@ -781,31 +807,28 @@ proc cow(p: BProc; n: PNode) {.inline.} = proc genAddr(p: BProc, e: PNode, d: var TLoc) = # careful 'addr(myptrToArray)' needs to get the ampersand: if e[0].typ.skipTypes(abstractInstOwned).kind in {tyRef, tyPtr}: - var a: TLoc - initLocExpr(p, e[0], a) + var a: TLoc = initLocExpr(p, e[0]) putIntoDest(p, d, e, "&" & a.r, a.storage) #Message(e.info, warnUser, "HERE NEW &") - elif mapType(p.config, e[0].typ, mapTypeChooser(e[0])) == ctArray or isCppRef(p, e.typ): + elif mapType(p.config, e[0].typ, mapTypeChooser(e[0]) == skParam) == ctArray or isCppRef(p, e.typ): expr(p, e[0], d) else: - var a: TLoc - initLocExpr(p, e[0], a) + var a: TLoc = initLocExpr(p, e[0]) putIntoDest(p, d, e, addrLoc(p.config, a), a.storage) template inheritLocation(d: var TLoc, a: TLoc) = if d.k == locNone: d.storage = a.storage -proc genRecordFieldAux(p: BProc, e: PNode, d, a: var TLoc) = - initLocExpr(p, e[0], a) +proc genRecordFieldAux(p: BProc, e: PNode, d: var TLoc, a: var TLoc) = + a = initLocExpr(p, e[0]) if e[1].kind != nkSym: internalError(p.config, e.info, "genRecordFieldAux") d.inheritLocation(a) discard getTypeDesc(p.module, a.t) # fill the record's fields.loc proc genTupleElem(p: BProc, e: PNode, d: var TLoc) = var - a: TLoc - i: int - initLocExpr(p, e[0], a) + i: int = 0 + var a: TLoc = initLocExpr(p, e[0]) let tupType = a.t.skipTypes(abstractInst+{tyVar}) assert tupType.kind == tyTuple d.inheritLocation(a) @@ -819,8 +842,9 @@ proc genTupleElem(p: BProc, e: PNode, d: var TLoc) = proc lookupFieldAgain(p: BProc, ty: PType; field: PSym; r: var Rope; resTyp: ptr PType = nil): PSym = + result = nil var ty = ty - assert r != nil + assert r != "" while ty != nil: ty = ty.skipTypes(skipPtrs) assert(ty.kind in {tyTuple, tyObject}) @@ -833,7 +857,13 @@ proc lookupFieldAgain(p: BProc, ty: PType; field: PSym; r: var Rope; if result == nil: internalError(p.config, field.info, "genCheckedRecordField") proc genRecordField(p: BProc, e: PNode, d: var TLoc) = - var a: TLoc + var a: TLoc = default(TLoc) + if p.module.compileToCpp and e.kind == nkDotExpr and e[1].kind == nkSym and e[1].typ.kind == tyPtr: + # special case for C++: we need to pull the type of the field as member and friends require the complete type. + let typ = e[1].typ.elementType + if typ.itemId in p.module.g.graph.memberProcsPerType: + discard getTypeDesc(p.module, typ) + genRecordFieldAux(p, e, d, a) var r = rdLoc(a) var f = e[1].sym @@ -841,15 +871,18 @@ proc genRecordField(p: BProc, e: PNode, d: var TLoc) = if ty.kind == tyTuple: # we found a unique tuple type which lacks field information # so we use Field$i - r.addf(".Field$1", [rope(f.position)]) + r.add ".Field" + r.add rope(f.position) putIntoDest(p, d, e, r, a.storage) else: - var rtyp: PType + var rtyp: PType = nil let field = lookupFieldAgain(p, ty, f, r, addr rtyp) - if field.loc.r == nil and rtyp != nil: fillObjectFields(p.module, rtyp) - if field.loc.r == nil: internalError(p.config, e.info, "genRecordField 3 " & typeToString(ty)) - r.addf(".$1", [field.loc.r]) + if field.loc.r == "" and rtyp != nil: fillObjectFields(p.module, rtyp) + if field.loc.r == "": internalError(p.config, e.info, "genRecordField 3 " & typeToString(ty)) + r.add "." + r.add field.loc.r putIntoDest(p, d, e, r, a.storage) + r.freeze proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) @@ -863,67 +896,114 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = if op.magic == mNot: it = it[1] let disc = it[2].skipConv assert(disc.kind == nkSym) - initLoc(test, locNone, it, OnStack) - initLocExpr(p, it[1], u) - initLoc(v, locExpr, disc, OnUnknown) - v.r = obj + test = initLoc(locNone, it, OnStack) + u = initLocExpr(p, it[1]) + v = initLoc(locExpr, disc, OnUnknown) + v.r = newRopeAppender() + v.r.add obj v.r.add(".") v.r.add(disc.sym.loc.r) genInExprAux(p, it, u, v, test) - let msg = genFieldDefect(field, disc.sym) - let strLit = genStringLiteral(p.module, newStrNode(nkStrLit, msg)) - if op.magic == mNot: - linefmt(p, cpsStmts, - "if ($1){ #raiseFieldError($2); $3}$n", - [rdLoc(test), strLit, raiseInstr(p)]) + var msg = "" + if optDeclaredLocs in p.config.globalOptions: + # xxx this should be controlled by a separate flag, and + # used for other similar defects so that location information is shown + # even without the expensive `--stacktrace`; binary size could be optimized + # by encoding the file names separately from `file(line:col)`, essentially + # passing around `TLineInfo` + the set of files in the project. + msg.add toFileLineCol(p.config, e.info) & " " + msg.add genFieldDefect(p.config, field.name.s, disc.sym) + var strLit = newRopeAppender() + genStringLiteral(p.module, newStrNode(nkStrLit, msg), strLit) + + ## discriminant check + template fun(code) = linefmt(p, cpsStmts, code, [rdLoc(test)]) + if op.magic == mNot: fun("if ($1) ") else: fun("if (!($1)) ") + + ## call raiseFieldError2 on failure + var discIndex = newRopeAppender() + rdSetElemLoc(p.config, v, u.t, discIndex) + if optTinyRtti in p.config.globalOptions: + let base = disc.typ.skipTypes(abstractInst+{tyRange}) + case base.kind + of tyEnum: + const code = "{ #raiseFieldErrorStr($1, $2); " + let toStrProc = getToStringProc(p.module.g.graph, base) + # XXX need to modify this logic for IC. + # need to analyze nkFieldCheckedExpr and marks procs "used" like range checks in dce + var toStr: TLoc = default(TLoc) + expr(p, newSymNode(toStrProc), toStr) + let enumStr = "$1($2)" % [rdLoc(toStr), rdLoc(v)] + linefmt(p, cpsStmts, code, [strLit, enumStr]) + else: + const code = "{ #raiseFieldError2($1, (NI)$2); " + linefmt(p, cpsStmts, code, [strLit, discIndex]) + else: - linefmt(p, cpsStmts, - "if (!($1)){ #raiseFieldError($2); $3}$n", - [rdLoc(test), strLit, raiseInstr(p)]) + # complication needed for signed types + let first = p.config.firstOrd(disc.sym.typ) + var firstLit = newRopeAppender() + int64Literal(cast[int](first), firstLit) + let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info) + const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$2) + (NI)$3, $4)); " + linefmt(p, cpsStmts, code, [strLit, discIndex, firstLit, discName]) + + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) = assert e[0].kind == nkDotExpr if optFieldCheck in p.options: - var a: TLoc + var a: TLoc = default(TLoc) genRecordFieldAux(p, e[0], d, a) let ty = skipTypes(a.t, abstractInst + tyUserTypeClasses) var r = rdLoc(a) let f = e[0][1].sym let field = lookupFieldAgain(p, ty, f, r) - if field.loc.r == nil: fillObjectFields(p.module, ty) - if field.loc.r == nil: + if field.loc.r == "": fillObjectFields(p.module, ty) + if field.loc.r == "": internalError(p.config, e.info, "genCheckedRecordField") # generate the checks: genFieldCheck(p, e, r, field) - r.add(ropecg(p.module, ".$1", [field.loc.r])) + r.add(".") + r.add field.loc.r putIntoDest(p, d, e[0], r, a.storage) + r.freeze else: genRecordField(p, e[0], d) proc genUncheckedArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = - var a, b: TLoc - initLocExpr(p, x, a) - initLocExpr(p, y, b) + var a = initLocExpr(p, x) + var b = initLocExpr(p, y) d.inheritLocation(a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = - var a, b: TLoc - initLocExpr(p, x, a) - initLocExpr(p, y, b) + var a = initLocExpr(p, x) + var b = initLocExpr(p, y) var ty = skipTypes(a.t, abstractVarRange + abstractPtrs + tyUserTypeClasses) - var first = intLiteral(firstOrd(p.config, ty)) + var first = newRopeAppender() + intLiteral(firstOrd(p.config, ty), first) # emit range check: if optBoundsCheck in p.options and ty.kind != tyUncheckedArray: if not isConstExpr(y): # semantic pass has already checked for const index expressions - if firstOrd(p.config, ty) == 0: + if firstOrd(p.config, ty) == 0 and lastOrd(p.config, ty) >= 0: if (firstOrd(p.config, b.t) < firstOrd(p.config, ty)) or (lastOrd(p.config, b.t) > lastOrd(p.config, ty)): - linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)){ #raiseIndexError2($1, $2); $3}$n", - [rdCharLoc(b), intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) + var last = newRopeAppender() + intLiteral(lastOrd(p.config, ty), last) + linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)){ #raiseIndexError2($1, $2); ", + [rdCharLoc(b), last]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] else: - linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3){ #raiseIndexError3($1, $2, $3); $4}$n", - [rdCharLoc(b), first, intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) + var last = newRopeAppender() + intLiteral(lastOrd(p.config, ty), last) + linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3){ #raiseIndexError3($1, $2, $3); ", + [rdCharLoc(b), first, last]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + else: let idx = getOrdValue(y) if idx < firstOrd(p.config, ty) or idx > lastOrd(p.config, ty): @@ -933,9 +1013,8 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = ropecg(p.module, "$1[($2)- $3]", [rdLoc(a), rdCharLoc(b), first]), a.storage) proc genCStringElem(p: BProc, n, x, y: PNode, d: var TLoc) = - var a, b: TLoc - initLocExpr(p, x, a) - initLocExpr(p, y, b) + var a = initLocExpr(p, x) + var b = initLocExpr(p, y) inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) @@ -947,57 +1026,78 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = if reifiedOpenArray(arr.lode): linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)($3.Field1) || (NU)($2) >= (NU)($3.Field1))){ #raiseIndexError(); $4}$n", - [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) + "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError4($1, $2, $3.Field1); ", + [rdLoc(a), rdLoc(b), rdLoc(arr)]) else: linefmt(p, cpsStmts, - "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))){ #raiseIndexError(); $4}$n", - [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) + "if ($2-$1 != -1 && ($1 < 0 || $1 >= $3Len_0 || $2 < 0 || $2 >= $3Len_0))" & + "{ #raiseIndexError4($1, $2, $3Len_0); ", + [rdLoc(a), rdLoc(b), rdLoc(arr)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + of tyArray: - let first = intLiteral(firstOrd(p.config, ty)) + var first = newRopeAppender() + intLiteral(firstOrd(p.config, ty), first) + var last = newRopeAppender() + intLiteral(lastOrd(p.config, ty), last) linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "($2-$1 < -1 || $1 < $3 || $1 > $4 || $2 < $3 || $2 > $4)){ #raiseIndexError(); $5}$n", - [rdCharLoc(a), rdCharLoc(b), first, intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) + "($2-$1 < -1 || $1 < $3 || $1 > $4 || $2 < $3 || $2 > $4)){ #raiseIndexError(); ", + [rdCharLoc(a), rdCharLoc(b), first, last]) + + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + of tySequence, tyString: linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)$3 || (NU)($2) >= (NU)$3)){ #raiseIndexError(); $4}$n", - [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)]) + "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError4($1, $2, $3); ", + [rdLoc(a), rdLoc(b), lenExpr(p, arr)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + else: discard proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = - var a, b: TLoc - initLocExpr(p, x, a) - initLocExpr(p, y, b) + var a = initLocExpr(p, x) + var b = initLocExpr(p, y) if not reifiedOpenArray(x): # emit range check: if optBoundsCheck in p.options: - linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n", - [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! + linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2Len_0){ #raiseIndexError2($1,$2Len_0-1); ", + [rdCharLoc(b), rdLoc(a)]) # BUGFIX: ``>=`` and not ``>``! + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) else: if optBoundsCheck in p.options: - linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2.Field1)){ #raiseIndexError2($1,$2.Field1-1); $3}$n", - [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! + linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2.Field1){ #raiseIndexError2($1,$2.Field1-1); ", + [rdCharLoc(b), rdLoc(a)]) # BUGFIX: ``>=`` and not ``>``! + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1.Field0[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = - var a, b: TLoc - initLocExpr(p, x, a) - initLocExpr(p, y, b) + var a = initLocExpr(p, x) + var b = initLocExpr(p, y) var ty = skipTypes(a.t, abstractVarRange) if ty.kind in {tyRef, tyPtr}: - ty = skipTypes(ty.lastSon, abstractVarRange) # emit range check: + ty = skipTypes(ty.elementType, abstractVarRange) + # emit range check: if optBoundsCheck in p.options: linefmt(p, cpsStmts, - "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n", - [rdLoc(b), lenExpr(p, a), raiseInstr(p)]) + "if ($1 < 0 || $1 >= $2){ #raiseIndexError2($1,$2-1); ", + [rdCharLoc(b), lenExpr(p, a)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + if d.k == locNone: d.storage = OnHeap if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}: a.r = ropecg(p.module, "(*$1)", [a.r]) @@ -1010,13 +1110,13 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = proc genBracketExpr(p: BProc; n: PNode; d: var TLoc) = var ty = skipTypes(n[0].typ, abstractVarRange + tyUserTypeClasses) - if ty.kind in {tyRef, tyPtr}: ty = skipTypes(ty.lastSon, abstractVarRange) + if ty.kind in {tyRef, tyPtr}: ty = skipTypes(ty.elementType, abstractVarRange) case ty.kind of tyUncheckedArray: genUncheckedArrayElem(p, n, n[0], n[1], d) of tyArray: genArrayElem(p, n, n[0], n[1], d) of tyOpenArray, tyVarargs: genOpenArrayElem(p, n, n[0], n[1], d) of tySequence, tyString: genSeqElem(p, n, n[0], n[1], d) - of tyCString: genCStringElem(p, n, n[0], n[1], d) + of tyCstring: genCStringElem(p, n, n[0], n[1], d) of tyTuple: genTupleElem(p, n, d) else: internalError(p.config, n.info, "expr(nkBracketExpr, " & $ty.kind & ')') discard getTypeDesc(p.module, n.typ) @@ -1035,8 +1135,7 @@ proc isSimpleExpr(n: PNode): bool = if n[i].kind notin {nkCommentStmt, nkEmpty}: return false result = isSimpleExpr(n.lastSon) else: - if n.isAtom: - result = true + result = n.isAtom proc genAndOr(p: BProc, e: PNode, d: var TLoc, m: TMagic) = # how to generate code? @@ -1061,11 +1160,10 @@ proc genAndOr(p: BProc, e: PNode, d: var TLoc, m: TMagic) = # a = tmp when false: #if isSimpleExpr(e) and p.module.compileToCpp: - var tmpA, tmpB: TLoc #getTemp(p, e.typ, tmpA) #getTemp(p, e.typ, tmpB) - initLocExprSingleUse(p, e[1], tmpA) - initLocExprSingleUse(p, e[2], tmpB) + var tmpA = initLocExprSingleUse(p, e[1]) + var tmpB = initLocExprSingleUse(p, e[2]) tmpB.k = locExpr if m == mOr: tmpB.r = "((" & rdLoc(tmpA) & ")||(" & rdLoc(tmpB) & "))" @@ -1078,8 +1176,7 @@ proc genAndOr(p: BProc, e: PNode, d: var TLoc, m: TMagic) = else: var L: TLabel - tmp: TLoc - getTemp(p, e.typ, tmp) # force it into a temp! + var tmp: TLoc = getTemp(p, e.typ) # force it into a temp! inc p.splitDecls expr(p, e[1], tmp) L = getLabel(p) @@ -1101,23 +1198,28 @@ proc genEcho(p: BProc, n: PNode) = internalAssert p.config, n.kind == nkBracket if p.config.target.targetOS == osGenode: # echo directly to the Genode LOG session - var args: Rope = nil + var args: Rope = "" var a: TLoc - for it in n.sons: + for i, it in n.sons: if it.skipConv.kind == nkNilLit: args.add(", \"\"") - else: - initLocExpr(p, it, a) - args.add(ropecg(p.module, ", Genode::Cstring($1->data, $1->len)", [rdLoc(a)])) + elif n.len != 0: + a = initLocExpr(p, it) + if i > 0: + args.add(", ") + case detectStrVersion(p.module) + of 2: + args.add(ropecg(p.module, "Genode::Cstring($1.p->data, $1.len)", [a.rdLoc])) + else: + args.add(ropecg(p.module, "Genode::Cstring($1->data, $1->len)", [a.rdLoc])) p.module.includeHeader("") p.module.includeHeader("") - linefmt(p, cpsStmts, """Genode::log(""$1);$n""", [args]) + linefmt(p, cpsStmts, """Genode::log($1);$n""", [args]) else: if n.len == 0: linefmt(p, cpsStmts, "#echoBinSafe(NIM_NIL, $1);$n", [n.len]) else: - var a: TLoc - initLocExpr(p, n, a) + var a: TLoc = initLocExpr(p, n) linefmt(p, cpsStmts, "#echoBinSafe($1, $2);$n", [a.rdLoc, n.len]) when false: p.module.includeHeader("") @@ -1136,7 +1238,7 @@ proc strLoc(p: BProc; d: TLoc): Rope = proc genStrConcat(p: BProc, e: PNode, d: var TLoc) = # - # s = 'Hello ' & name & ', how do you feel?' & 'z' + # s = "Hello " & name & ", how do you feel?" & 'z' # # # { @@ -1151,14 +1253,14 @@ proc genStrConcat(p: BProc, e: PNode, d: var TLoc) = # appendChar(tmp0, 'z'); # asgn(s, tmp0); # } - var a, tmp: TLoc - getTemp(p, e.typ, tmp) + var a: TLoc + var tmp: TLoc = getTemp(p, e.typ) var L = 0 - var appends: Rope = nil - var lens: Rope = nil + var appends: Rope = "" + var lens: Rope = "" for i in 0.. - # s &= 'Hello ' & name & ', how do you feel?' & 'z' + # s &= "Hello " & name & ", how do you feel?" & 'z' # // BUG: what if s is on the left side too? # # { @@ -1190,14 +1292,14 @@ proc genStrAppend(p: BProc, e: PNode, d: var TLoc) = # appendChar(s, 'z'); # } var - a, dest, call: TLoc - appends, lens: Rope + a, call: TLoc + appends, lens: Rope = "" assert(d.k == locNone) var L = 0 - initLocExpr(p, e[1], dest) + var dest = initLocExpr(p, e[1]) for i in 0.. # seq = (typeof seq) incrSeq(&seq->Sup, sizeof(x)); # seq->data[seq->len-1] = x; - var a, b, dest, tmpL, call: TLoc - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) let seqType = skipTypes(e[1].typ, {tyVar}) - initLoc(call, locCall, e, OnHeap) + var call = initLoc(locCall, e, OnHeap) if not p.module.compileToCpp: const seqAppendPattern = "($2) #incrSeqV3((TGenericSeq*)($1), $3)" call.r = ropecg(p.module, seqAppendPattern, [rdLoc(a), @@ -1244,35 +1345,25 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) = genRefAssign(p, a, call) #if bt != b.t: # echo "YES ", e.info, " new: ", typeToString(bt), " old: ", typeToString(b.t) - initLoc(dest, locExpr, e[2], OnHeap) - getIntTemp(p, tmpL) + var dest = initLoc(locExpr, e[2], OnHeap) + var tmpL = getIntTemp(p) lineCg(p, cpsStmts, "$1 = $2->$3++;$n", [tmpL.r, rdLoc(a), lenField(p)]) dest.r = ropecg(p.module, "$1$3[$2]", [rdLoc(a), tmpL.r, dataField(p)]) genAssignment(p, dest, b, {needToCopy}) gcUsage(p.config, e) -proc genReset(p: BProc, n: PNode) = - var a: TLoc - initLocExpr(p, n[1], a) - specializeReset(p, a) - when false: - linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n", - [addrLoc(p.config, a), - genTypeInfoV1(p.module, skipTypes(a.t, {tyVar}), n.info)]) - proc genDefault(p: BProc; n: PNode; d: var TLoc) = - if d.k == locNone: getTemp(p, n.typ, d, needsInit=true) + if d.k == locNone: d = getTemp(p, n.typ, needsInit=true) else: resetLoc(p, d) proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) = var sizeExpr = sizeExpr let typ = a.t - var b: TLoc - initLoc(b, locExpr, a.lode, OnHeap) + var b: TLoc = initLoc(locExpr, a.lode, OnHeap) let refType = typ.skipTypes(abstractInstOwned) assert refType.kind == tyRef - let bt = refType.lastSon - if sizeExpr.isNil: + let bt = refType.elementType + if sizeExpr == "": sizeExpr = "sizeof($1)" % [getTypeDesc(p.module, bt)] if optTinyRtti in p.config.globalOptions: @@ -1294,12 +1385,11 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) = localError(p.module.config, a.lode.info, "the destructor that is turned into a finalizer needs " & "to have the 'nimcall' calling convention") - var f: TLoc - initLocExpr(p, newSymNode(op), f) + var f: TLoc = initLocExpr(p, newSymNode(op)) p.module.s[cfsTypeInit3].addf("$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)]) if a.storage == OnHeap and usesWriteBarrier(p.config): - if canFormAcycle(a.t): + if canFormAcycle(p.module.g.graph, a.t): linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", [a.rdLoc]) else: linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", [a.rdLoc]) @@ -1320,23 +1410,20 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) = genObjectInit(p, cpsStmts, bt, a, constructRefObj) proc genNew(p: BProc, e: PNode) = - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) # 'genNew' also handles 'unsafeNew': if e.len == 3: - var se: TLoc - initLocExpr(p, e[2], se) + var se: TLoc = initLocExpr(p, e[2]) rawGenNew(p, a, se.rdLoc, needsInit = true) else: - rawGenNew(p, a, nil, needsInit = true) + rawGenNew(p, a, "", needsInit = true) gcUsage(p.config, e) proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope; lenIsZero: bool) = let seqtype = skipTypes(dest.t, abstractVarRange) - var call: TLoc - initLoc(call, locExpr, dest.lode, OnHeap) + var call: TLoc = initLoc(locExpr, dest.lode, OnHeap) if dest.storage == OnHeap and usesWriteBarrier(p.config): - if canFormAcycle(dest.t): + if canFormAcycle(p.module.g.graph, dest.t): linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", [dest.rdLoc]) else: linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", [dest.rdLoc]) @@ -1359,14 +1446,13 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope; lenIsZero: bool) = genAssignment(p, dest, call, {}) proc genNewSeq(p: BProc, e: PNode) = - var a, b: TLoc - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) if optSeqDestructors in p.config.globalOptions: let seqtype = skipTypes(e[1].typ, abstractVarRange) linefmt(p, cpsStmts, "$1.len = $2; $1.p = ($4*) #newSeqPayload($2, sizeof($3), NIM_ALIGNOF($3));$n", [a.rdLoc, b.rdLoc, - getTypeDesc(p.module, seqtype.lastSon), + getTypeDesc(p.module, seqtype.elementType), getSeqPayloadType(p.module, seqtype)]) else: let lenIsZero = e[2].kind == nkIntLit and e[2].intVal == 0 @@ -1375,15 +1461,15 @@ proc genNewSeq(p: BProc, e: PNode) = proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) = let seqtype = skipTypes(e.typ, abstractVarRange) - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) if optSeqDestructors in p.config.globalOptions: - if d.k == locNone: getTemp(p, e.typ, d, needsInit=false) - linefmt(p, cpsStmts, "$1.len = 0; $1.p = ($4*) #newSeqPayload($2, sizeof($3), NIM_ALIGNOF($3));$n", - [d.rdLoc, a.rdLoc, getTypeDesc(p.module, seqtype.lastSon), + if d.k == locNone: d = getTemp(p, e.typ, needsInit=false) + linefmt(p, cpsStmts, "$1.len = 0; $1.p = ($4*) #newSeqPayloadUninit($2, sizeof($3), NIM_ALIGNOF($3));$n", + [d.rdLoc, a.rdLoc, getTypeDesc(p.module, seqtype.elementType), getSeqPayloadType(p.module, seqtype), ]) else: + if d.k == locNone: d = getTemp(p, e.typ, needsInit=false) # bug #22560 putIntoDest(p, d, e, ropecg(p.module, "($1)#nimNewSeqOfCap($2, $3)", [ getTypeDesc(p.module, seqtype), @@ -1398,8 +1484,13 @@ proc rawConstExpr(p: BProc, n: PNode; d: var TLoc) = if id == p.module.labels: # expression not found in the cache: inc(p.module.labels) - p.module.s[cfsData].addf("static NIM_CONST $1 $2 = $3;$n", - [getTypeDesc(p.module, t), d.r, genBracedInit(p, n, isConst = true, t)]) + var data = "static NIM_CONST $1 $2 = " % [getTypeDesc(p.module, t), d.r] + # bug #23627; when generating const object fields, it's likely that + # we need to generate type infos for the object, which may be an object with + # custom hooks. We need to generate potential consts in the hooks first. + genBracedInit(p, n, isConst = true, t, data) + data.addf(";$n", []) + p.module.s[cfsData].add data proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = if d.k == locNone and n.len > ord(n.kind == nkObjConstr) and n.isDeepConstExpr: @@ -1408,8 +1499,31 @@ proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = else: result = false + +proc genFieldObjConstr(p: BProc; ty: PType; useTemp, isRef: bool; nField, val, check: PNode; d: var TLoc; r: Rope; info: TLineInfo) = + var tmp2 = TLoc(r: r) + let field = lookupFieldAgain(p, ty, nField.sym, tmp2.r) + if field.loc.r == "": fillObjectFields(p.module, ty) + if field.loc.r == "": internalError(p.config, info, "genFieldObjConstr") + if check != nil and optFieldCheck in p.options: + genFieldCheck(p, check, r, field) + tmp2.r.add(".") + tmp2.r.add(field.loc.r) + if useTemp: + tmp2.k = locTemp + tmp2.storage = if isRef: OnHeap else: OnStack + else: + tmp2.k = d.k + tmp2.storage = if isRef: OnHeap else: d.storage + tmp2.lode = val + if nField.typ.skipTypes(abstractVar).kind in {tyOpenArray, tyVarargs}: + var tmp3 = getTemp(p, val.typ) + expr(p, val, tmp3) + genOpenArrayConv(p, tmp2, tmp3, {}) + else: + expr(p, val, tmp2) + proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = - #echo renderTree e, " ", e.isDeepConstExpr # inheritance in C++ does not allow struct initialization so # we skip this step here: if not p.module.compileToCpp and optSeqDestructors notin p.config.globalOptions: @@ -1430,42 +1544,33 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = (d.k notin {locTemp,locLocalVar,locGlobalVar,locParam,locField}) or (isPartOf(d.lode, e) != arNo) - var tmp: TLoc + var tmp: TLoc = default(TLoc) var r: Rope + let needsZeroMem = p.config.selectedGC notin {gcArc, gcAtomicArc, gcOrc} or nfAllFieldsSet notin e.flags if useTemp: - getTemp(p, t, tmp) + tmp = getTemp(p, t) r = rdLoc(tmp) if isRef: - rawGenNew(p, tmp, nil, needsInit = nfAllFieldsSet notin e.flags) - t = t.lastSon.skipTypes(abstractInstOwned) + rawGenNew(p, tmp, "", needsInit = nfAllFieldsSet notin e.flags) + t = t.elementType.skipTypes(abstractInstOwned) r = "(*$1)" % [r] gcUsage(p.config, e) - else: + elif needsZeroMem: constructLoc(p, tmp) + else: + genObjectInit(p, cpsStmts, t, tmp, constructObj) else: - resetLoc(p, d) + if needsZeroMem: resetLoc(p, d) + else: genObjectInit(p, cpsStmts, d.t, d, if isRef: constructRefObj else: constructObj) r = rdLoc(d) discard getTypeDesc(p.module, t) let ty = getUniqueType(t) for i in 1..finalizer = (void*)$2;$n", [ti, rdLoc(f)]) b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", [ getTypeDesc(p.module, refType), - ti, getTypeDesc(p.module, skipTypes(refType.lastSon, abstractRange))]) + ti, getTypeDesc(p.module, skipTypes(refType.elementType, abstractRange))]) genAssignment(p, a, b, {}) # set the object type: - bt = skipTypes(refType.lastSon, abstractRange) + bt = skipTypes(refType.elementType, abstractRange) genObjectInit(p, cpsStmts, bt, a, constructRefObj) gcUsage(p.config, e) -proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo): Rope = +proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo; result: var Rope) = if optTinyRtti in p.config.globalOptions: - result = ropecg(p.module, "#isObj($1.m_type, $2)", - [a, genTypeInfo2Name(p.module, dest)]) + let token = $genDisplayElem(MD5Digest(hashType(dest, p.config))) + appcg(p.module, result, "#isObjDisplayCheck($#.m_type, $#, $#)", [a, getObjDepth(dest), token]) else: # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we # have to call it here first: let ti = genTypeInfoV1(p.module, dest, info) if tfFinal in dest.flags or (objHasKidsValid in p.module.flags and tfObjHasKids notin dest.flags): - result = "$1.m_type == $2" % [a, ti] + result.add "$1.m_type == $2" % [a, ti] else: - discard cgsym(p.module, "TNimType") + cgsym(p.module, "TNimType") inc p.module.labels let cache = "Nim_OfCheck_CACHE" & p.module.labels.rope p.module.s[cfsVars].addf("static TNimType* $#[2];$n", [cache]) - result = ropecg(p.module, "#isObjWithCache($#.m_type, $#, $#)", [a, ti, cache]) - when false: - # former version: - result = ropecg(p.module, "#isObj($1.m_type, $2)", - [a, genTypeInfoV1(p.module, dest, info)]) + appcg(p.module, result, "#isObjWithCache($#.m_type, $#, $#)", [a, ti, cache]) proc genOf(p: BProc, x: PNode, typ: PType, d: var TLoc) = - var a: TLoc - initLocExpr(p, x, a) + var a: TLoc = initLocExpr(p, x) var dest = skipTypes(typ, typedescPtrs) var r = rdLoc(a) - var nilCheck: Rope = nil + var nilCheck: Rope = "" var t = skipTypes(a.t, abstractInstOwned) while t.kind in {tyVar, tyLent, tyPtr, tyRef}: if t.kind notin {tyVar, tyLent}: nilCheck = r if t.kind notin {tyVar, tyLent} or not p.module.compileToCpp: r = ropecg(p.module, "(*$1)", [r]) - t = skipTypes(t.lastSon, typedescInst+{tyOwned}) + t = skipTypes(t.elementType, typedescInst+{tyOwned}) discard getTypeDesc(p.module, t) if not p.module.compileToCpp: - while t.kind == tyObject and t[0] != nil: - r.add(~".Sup") - t = skipTypes(t[0], skipPtrs) + while t.kind == tyObject and t.baseClass != nil: + r.add(".Sup") + t = skipTypes(t.baseClass, skipPtrs) if isObjLackingTypeField(t): globalError(p.config, x.info, "no 'of' operator available for pure objects") - if nilCheck != nil: - r = ropecg(p.module, "(($1) && ($2))", [nilCheck, genOfHelper(p, dest, r, x.info)]) + + var ro = newRopeAppender() + genOfHelper(p, dest, r, x.info, ro) + var ofExpr = newRopeAppender() + ofExpr.add "(" + if nilCheck != "": + ofExpr.add "(" + ofExpr.add nilCheck + ofExpr.add ") && (" + ofExpr.add ro + ofExpr.add "))" else: - r = ropecg(p.module, "($1)", [genOfHelper(p, dest, r, x.info)]) - putIntoDest(p, d, x, r, a.storage) + ofExpr.add ro + ofExpr.add ")" + + putIntoDest(p, d, x, ofExpr, a.storage) proc genOf(p: BProc, n: PNode, d: var TLoc) = genOf(p, n[1], n[2].typ, d) @@ -1623,8 +1739,7 @@ proc genOf(p: BProc, n: PNode, d: var TLoc) = proc genRepr(p: BProc, e: PNode, d: var TLoc) = if optTinyRtti in p.config.globalOptions: localError(p.config, e.info, "'repr' is not available for --newruntime") - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) var t = skipTypes(e[1].typ, abstractVarRange) case t.kind of tyInt..tyInt64, tyUInt..tyUInt64: @@ -1646,13 +1761,15 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = putIntoDest(p, d, e, ropecg(p.module, "#reprSet($1, $2)", [ addrLoc(p.config, a), genTypeInfoV1(p.module, t, e.info)]), a.storage) of tyOpenArray, tyVarargs: - var b: TLoc + var b: TLoc = default(TLoc) case skipTypes(a.t, abstractVarRange).kind of tyOpenArray, tyVarargs: putIntoDest(p, b, e, "$1, $1Len_0" % [rdLoc(a)], a.storage) of tyString, tySequence: putIntoDest(p, b, e, - "$1$3, $2" % [rdLoc(a), lenExpr(p, a), dataField(p)], a.storage) + "($4) ? ($1$3) : NIM_NIL, $2" % + [rdLoc(a), lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)], + a.storage) of tyArray: putIntoDest(p, b, e, "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))], a.storage) @@ -1660,7 +1777,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = putIntoDest(p, d, e, ropecg(p.module, "#reprOpenArray($1, $2)", [rdLoc(b), genTypeInfoV1(p.module, elemType(t), e.info)]), a.storage) - of tyCString, tyArray, tyRef, tyPtr, tyPointer, tyNil, tySequence: + of tyCstring, tyArray, tyRef, tyPtr, tyPointer, tyNil, tySequence: putIntoDest(p, d, e, ropecg(p.module, "#reprAny($1, $2)", [ rdLoc(a), genTypeInfoV1(p.module, t, e.info)]), a.storage) @@ -1672,25 +1789,26 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = a.storage) gcUsage(p.config, e) -proc rdMType(p: BProc; a: TLoc; nilCheck: var Rope; enforceV1 = false): Rope = - result = rdLoc(a) +proc rdMType(p: BProc; a: TLoc; nilCheck: var Rope; result: var Rope; enforceV1 = false) = + var derefs = rdLoc(a) var t = skipTypes(a.t, abstractInst) while t.kind in {tyVar, tyLent, tyPtr, tyRef}: - if t.kind notin {tyVar, tyLent}: nilCheck = result + if t.kind notin {tyVar, tyLent}: nilCheck = derefs if t.kind notin {tyVar, tyLent} or not p.module.compileToCpp: - result = "(*$1)" % [result] - t = skipTypes(t.lastSon, abstractInst) + derefs = "(*$1)" % [derefs] + t = skipTypes(t.elementType, abstractInst) + result.add derefs discard getTypeDesc(p.module, t) if not p.module.compileToCpp: - while t.kind == tyObject and t[0] != nil: + while t.kind == tyObject and t.baseClass != nil: result.add(".Sup") - t = skipTypes(t[0], skipPtrs) + t = skipTypes(t.baseClass, skipPtrs) result.add ".m_type" if optTinyRtti in p.config.globalOptions and enforceV1: result.add "->typeInfoV1" proc genGetTypeInfo(p: BProc, e: PNode, d: var TLoc) = - discard cgsym(p.module, "TNimType") + cgsym(p.module, "TNimType") let t = e[1].typ # ordinary static type information putIntoDest(p, d, e, genTypeInfoV1(p.module, t, e.info)) @@ -1701,18 +1819,26 @@ proc genGetTypeInfoV2(p: BProc, e: PNode, d: var TLoc) = # ordinary static type information putIntoDest(p, d, e, genTypeInfoV2(p.module, t, e.info)) else: - var a: TLoc - initLocExpr(p, e[1], a) - var nilCheck = Rope(nil) + var a: TLoc = initLocExpr(p, e[1]) + var nilCheck = "" # use the dynamic type stored at offset 0: - putIntoDest(p, d, e, rdMType(p, a, nilCheck)) + var rt = newRopeAppender() + rdMType(p, a, nilCheck, rt) + putIntoDest(p, d, e, rt) + +proc genAccessTypeField(p: BProc; e: PNode; d: var TLoc) = + var a: TLoc = initLocExpr(p, e[1]) + var nilCheck = "" + # use the dynamic type stored at offset 0: + var rt = newRopeAppender() + rdMType(p, a, nilCheck, rt) + putIntoDest(p, d, e, rt) template genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) = - var a: TLoc - initLocExpr(p, n[1], a) + var a: TLoc = initLocExpr(p, n[1]) a.r = ropecg(p.module, frmt, [rdLoc(a)]) a.flags.excl lfIndirect # this flag should not be propagated here (not just for HCR) - if d.k == locNone: getTemp(p, n.typ, d) + if d.k == locNone: d = getTemp(p, n.typ) genAssignment(p, d, a, {}) gcUsage(p.config, n) @@ -1725,34 +1851,43 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) = # Bug #9279, len(toOpenArray()) has to work: if a.kind in nkCallKinds and a[0].kind == nkSym and a[0].sym.magic == mSlice: # magic: pass slice to openArray: - var b, c: TLoc - initLocExpr(p, a[2], b) - initLocExpr(p, a[3], c) + var m = initLocExpr(p, a[1]) + var b = initLocExpr(p, a[2]) + var c = initLocExpr(p, a[3]) + if optBoundsCheck in p.options: + genBoundsCheck(p, m, b, c) if op == mHigh: - putIntoDest(p, d, e, ropecg(p.module, "($2)-($1)", [rdLoc(b), rdLoc(c)])) + putIntoDest(p, d, e, ropecg(p.module, "(($2)-($1))", [rdLoc(b), rdLoc(c)])) else: - putIntoDest(p, d, e, ropecg(p.module, "($2)-($1)+1", [rdLoc(b), rdLoc(c)])) + putIntoDest(p, d, e, ropecg(p.module, "(($2)-($1)+1)", [rdLoc(b), rdLoc(c)])) else: if not reifiedOpenArray(a): if op == mHigh: unaryExpr(p, e, d, "($1Len_0-1)") else: unaryExpr(p, e, d, "$1Len_0") else: - if op == mHigh: unaryExpr(p, e, d, "($1.Field1-1)") - else: unaryExpr(p, e, d, "$1.Field1") - of tyCString: - if op == mHigh: unaryExpr(p, e, d, "($1 ? (#nimCStrLen($1)-1) : -1)") - else: unaryExpr(p, e, d, "($1 ? #nimCStrLen($1) : 0)") + let isDeref = a.kind in {nkHiddenDeref, nkDerefExpr} + if op == mHigh: + if isDeref: + unaryExpr(p, e, d, "($1->Field1-1)") + else: + unaryExpr(p, e, d, "($1.Field1-1)") + else: + if isDeref: + unaryExpr(p, e, d, "$1->Field1") + else: + unaryExpr(p, e, d, "$1.Field1") + of tyCstring: + if op == mHigh: unaryExpr(p, e, d, "(#nimCStrLen($1)-1)") + else: unaryExpr(p, e, d, "#nimCStrLen($1)") of tyString: - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) var x = lenExpr(p, a) if op == mHigh: x = "($1-1)" % [x] putIntoDest(p, d, e, x) of tySequence: # we go through a temporary here because people write bullshit code. - var a, tmp: TLoc - initLocExpr(p, e[1], a) - getIntTemp(p, tmp) + var tmp: TLoc = getIntTemp(p) + var a = initLocExpr(p, e[1]) var x = lenExpr(p, a) if op == mHigh: x = "($1-1)" % [x] lineCg(p, cpsStmts, "$1 = $2;$n", [tmp.r, x]) @@ -1763,33 +1898,21 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) = else: putIntoDest(p, d, e, rope(lengthOrd(p.config, typ))) else: internalError(p.config, e.info, "genArrayLen()") -proc makePtrType(baseType: PType; idgen: IdGenerator): PType = - result = newType(tyPtr, nextTypeId idgen, baseType.owner) - addSonSkipIntLit(result, baseType, idgen) - -proc makeAddr(n: PNode; idgen: IdGenerator): PNode = - if n.kind == nkHiddenAddr: - result = n - else: - result = newTree(nkHiddenAddr, n) - result.typ = makePtrType(n.typ, idgen) - proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) = if optSeqDestructors in p.config.globalOptions: e[1] = makeAddr(e[1], p.module.idgen) genCall(p, e, d) return - var a, b, call: TLoc assert(d.k == locNone) var x = e[1] if x.kind in {nkAddr, nkHiddenAddr}: x = x[0] - initLocExpr(p, x, a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, x) + var b = initLocExpr(p, e[2]) let t = skipTypes(e[1].typ, {tyVar}) - initLoc(call, locCall, e, OnHeap) + var call = initLoc(locCall, e, OnHeap) if not p.module.compileToCpp: - const setLenPattern = "($3) #setLengthSeqV2(&($1)->Sup, $4, $2)" + const setLenPattern = "($3) #setLengthSeqV2(($1)?&($1)->Sup:NIM_NIL, $4, $2)" call.r = ropecg(p.module, setLenPattern, [ rdLoc(a), rdLoc(b), getTypeDesc(p.module, t), genTypeInfoV1(p.module, t.skipTypes(abstractInst), e.info)]) @@ -1807,12 +1930,11 @@ proc genSetLengthStr(p: BProc, e: PNode, d: var TLoc) = if optSeqDestructors in p.config.globalOptions: binaryStmtAddr(p, e, d, "setLengthStrV2") else: - var a, b, call: TLoc if d.k != locNone: internalError(p.config, e.info, "genSetLengthStr") - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) - initLoc(call, locCall, e, OnHeap) + var call = initLoc(locCall, e, OnHeap) call.r = ropecg(p.module, "#setLengthStr($1, $2)", [ rdLoc(a), rdLoc(b)]) genAssignment(p, a, call, {}) @@ -1825,22 +1947,24 @@ proc genSwap(p: BProc, e: PNode, d: var TLoc) = # b = temp cowBracket(p, e[1]) cowBracket(p, e[2]) - var a, b, tmp: TLoc - getTemp(p, skipTypes(e[1].typ, abstractVar), tmp) - initLocExpr(p, e[1], a) # eval a - initLocExpr(p, e[2], b) # eval b + var tmp: TLoc = getTemp(p, skipTypes(e[1].typ, abstractVar)) + var a = initLocExpr(p, e[1]) # eval a + var b = initLocExpr(p, e[2]) # eval b genAssignment(p, tmp, a, {}) genAssignment(p, a, b, {}) genAssignment(p, b, tmp, {}) -proc rdSetElemLoc(conf: ConfigRef; a: TLoc, typ: PType): Rope = +proc rdSetElemLoc(conf: ConfigRef; a: TLoc, typ: PType; result: var Rope) = # read a location of an set element; it may need a subtraction operation # before the set operation - result = rdCharLoc(a) + result.add "(" + result.add rdCharLoc(a) let setType = typ.skipTypes(abstractPtrs) assert(setType.kind == tySet) if firstOrd(conf, setType) != 0: - result = "($1- $2)" % [result, rope(firstOrd(conf, setType))] + result.add " - " + result.add rope(firstOrd(conf, setType)) + result.add ")" proc fewCmps(conf: ConfigRef; s: PNode): bool = # this function estimates whether it is better to emit code @@ -1854,7 +1978,9 @@ proc fewCmps(conf: ConfigRef; s: PNode): bool = result = s.len <= 8 # 8 seems to be a good value template binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) = - putIntoDest(p, d, e, frmt % [rdLoc(a), rdSetElemLoc(p.config, b, a.t)]) + var elem = newRopeAppender() + rdSetElemLoc(p.config, b, a.t, elem) + putIntoDest(p, d, e, frmt % [rdLoc(a), elem]) proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) = case int(getSize(p.config, skipTypes(e[1].typ, abstractVar))) @@ -1865,11 +1991,12 @@ proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) = else: binaryExprIn(p, e, a, b, d, "(($1[(NU)($2)>>3] &(1U<<((NU)($2)&7U)))!=0)") template binaryStmtInExcl(p: BProc, e: PNode, d: var TLoc, frmt: string) = - var a, b: TLoc assert(d.k == locNone) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) - lineF(p, cpsStmts, frmt, [rdLoc(a), rdSetElemLoc(p.config, b, a.t)]) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) + var elem = newRopeAppender() + rdSetElemLoc(p.config, b, a.t, elem) + lineF(p, cpsStmts, frmt, [rdLoc(a), elem]) proc genInOp(p: BProc, e: PNode, d: var TLoc) = var a, b, x, y: TLoc @@ -1882,19 +2009,19 @@ proc genInOp(p: BProc, e: PNode, d: var TLoc) = e[2][0] else: e[2] - initLocExpr(p, ea, a) - initLoc(b, locExpr, e, OnUnknown) + a = initLocExpr(p, ea) + b = initLoc(locExpr, e, OnUnknown) if e[1].len > 0: b.r = rope("(") for i in 0..= $2 && $1 <= $3", [rdCharLoc(a), rdCharLoc(x), rdCharLoc(y)]) else: - initLocExpr(p, it, x) + x = initLocExpr(p, it) b.r.addf("$1 == $2", [rdCharLoc(a), rdCharLoc(x)]) if i < e[1].len - 1: b.r.add(" || ") b.r.add(")") @@ -1905,8 +2032,8 @@ proc genInOp(p: BProc, e: PNode, d: var TLoc) = else: assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + a = initLocExpr(p, e[1]) + b = initLocExpr(p, e[2]) genInExprAux(p, e, a, b, d) proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = @@ -1922,7 +2049,8 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = "&", "|", "& ~"] - var a, b, i: TLoc + var a, b: TLoc + var i: TLoc var setType = skipTypes(e[1].typ, abstractVar) var size = int(getSize(p.config, setType)) case size @@ -1959,14 +2087,13 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mIncl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] |=(1U<<($2&7U));$n") of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] &= ~(1U<<($2&7U));$n") of mCard: - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [rdCharLoc(a), size])) of mLtSet, mLeSet: - getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) # our counter - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) - if d.k == locNone: getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyBool), d) + i = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt)) # our counter + a = initLocExpr(p, e[1]) + b = initLocExpr(p, e[2]) + if d.k == locNone: d = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyBool)) if op == mLtSet: linefmt(p, cpsStmts, lookupOpr[mLtSet], [rdLoc(i), size, rdLoc(d), rdLoc(a), rdLoc(b)]) @@ -1974,18 +2101,17 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = linefmt(p, cpsStmts, lookupOpr[mLeSet], [rdLoc(i), size, rdLoc(d), rdLoc(a), rdLoc(b)]) of mEqSet: - var a, b: TLoc assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) putIntoDest(p, d, e, ropecg(p.module, "(#nimCmpMem($1, $2, $3)==0)", [a.rdCharLoc, b.rdCharLoc, size])) of mMulSet, mPlusSet, mMinusSet: # we inline the simple for loop for better code generation: - getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) # our counter - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) - if d.k == locNone: getTemp(p, setType, d) + i = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt)) # our counter + a = initLocExpr(p, e[1]) + b = initLocExpr(p, e[2]) + if d.k == locNone: d = getTemp(p, setType) lineF(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) $n" & " $3[$1] = $4[$1] $6 $5[$1];$n", [ @@ -2002,8 +2128,7 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) = ValueTypes = {tyTuple, tyObject, tyArray, tyOpenArray, tyVarargs, tyUncheckedArray} # we use whatever C gives us. Except if we have a value-type, we need to go # through its address: - var a: TLoc - initLocExpr(p, e[1], a) + var a: TLoc = initLocExpr(p, e[1]) let etyp = skipTypes(e.typ, abstractRange+{tyOwned}) let srcTyp = skipTypes(e[1].typ, abstractRange) if etyp.kind in ValueTypes and lfIndirect notin a.flags: @@ -2023,6 +2148,11 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) = elif etyp.kind == tyBool and srcTyp.kind in IntegralTypes: putIntoDest(p, d, e, "(($1) != 0)" % [rdCharLoc(a)], a.storage) else: + if etyp.kind == tyPtr: + # generates the definition of structs for casts like cast[ptr object](addr x)[] + let internalType = etyp.skipTypes({tyPtr}) + if internalType.kind == tyObject: + discard getTypeDesc(p.module, internalType) putIntoDest(p, d, e, "(($1) ($2))" % [getTypeDesc(p.module, e.typ), rdCharLoc(a)], a.storage) @@ -2035,10 +2165,17 @@ proc genCast(p: BProc, e: PNode, d: var TLoc) = # 'cast' and some float type involved? --> use a union. inc(p.labels) var lbl = p.labels.rope - var tmp: TLoc + var tmp: TLoc = default(TLoc) tmp.r = "LOC$1.source" % [lbl] - linefmt(p, cpsLocals, "union { $1 source; $2 dest; } LOC$3;$n", - [getTypeDesc(p.module, e[1].typ), getTypeDesc(p.module, e.typ), lbl]) + let destsize = getSize(p.config, destt) + let srcsize = getSize(p.config, srct) + + if destsize > srcsize: + linefmt(p, cpsLocals, "union { $1 dest; $2 source; } LOC$3;$n #nimZeroMem(&LOC$3, sizeof(LOC$3));$n", + [getTypeDesc(p.module, e.typ), getTypeDesc(p.module, e[1].typ), lbl]) + else: + linefmt(p, cpsLocals, "union { $1 source; $2 dest; } LOC$3;$n", + [getTypeDesc(p.module, e[1].typ), getTypeDesc(p.module, e.typ), lbl]) tmp.k = locExpr tmp.lode = lodeTyp srct tmp.storage = OnStack @@ -2051,9 +2188,8 @@ proc genCast(p: BProc, e: PNode, d: var TLoc) = genSomeCast(p, e, d) proc genRangeChck(p: BProc, n: PNode, d: var TLoc) = - var a: TLoc + var a: TLoc = initLocExpr(p, n[0]) var dest = skipTypes(n.typ, abstractVar) - initLocExpr(p, n[0], a) if optRangeCheck notin p.options or (dest.kind in {tyUInt..tyUInt64} and checkUnsignedConversions notin p.config.legacyFeatures): discard "no need to generate a check because it was disabled" @@ -2062,26 +2198,39 @@ proc genRangeChck(p: BProc, n: PNode, d: var TLoc) = # emit range check: if n0t.kind in {tyUInt, tyUInt64}: - linefmt(p, cpsStmts, "if ($1 > ($6)($3)){ #raiseRangeErrorNoArgs(); $5}$n", - [rdCharLoc(a), genLiteral(p, n[1], dest), genLiteral(p, n[2], dest), - raiser, raiseInstr(p), getTypeDesc(p.module, n0t)]) + var first = newRopeAppender() + genLiteral(p, n[1], dest, first) + var last = newRopeAppender() + genLiteral(p, n[2], dest, last) + linefmt(p, cpsStmts, "if ($1 > ($5)($3)){ #raiseRangeErrorNoArgs(); ", + [rdCharLoc(a), first, last, + raiser, getTypeDesc(p.module, n0t)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + else: let raiser = case skipTypes(n.typ, abstractVarRange).kind of tyUInt..tyUInt64, tyChar: "raiseRangeErrorU" of tyFloat..tyFloat128: "raiseRangeErrorF" else: "raiseRangeErrorI" - discard cgsym(p.module, raiser) + cgsym(p.module, raiser) let boundaryCast = - if n0t.skipTypes(abstractVarRange).kind in {tyUInt, tyUInt32, tyUInt64} or - (n0t.sym != nil and sfSystemModule in n0t.sym.owner.flags and n0t.sym.name.s == "csize"): + if n0t.skipTypes(abstractVarRange).kind in {tyUInt, tyUInt32, tyUInt64}: "(NI64)" else: "" - linefmt(p, cpsStmts, "if ($6($1) < $2 || $6($1) > $3){ $4($1, $2, $3); $5}$n", - [rdCharLoc(a), genLiteral(p, n[1], dest), genLiteral(p, n[2], dest), - raiser, raiseInstr(p), boundaryCast]) + var first = newRopeAppender() + genLiteral(p, n[1], dest, first) + var last = newRopeAppender() + genLiteral(p, n[2], dest, last) + linefmt(p, cpsStmts, "if ($5($1) < $2 || $5($1) > $3){ $4($1, $2, $3); ", + [rdCharLoc(a), first, last, + raiser, boundaryCast]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + putIntoDest(p, d, n, "(($1) ($2))" % [getTypeDesc(p.module, dest), rdCharLoc(a)], a.storage) @@ -2093,19 +2242,23 @@ proc genConv(p: BProc, e: PNode, d: var TLoc) = genSomeCast(p, e, d) proc convStrToCStr(p: BProc, n: PNode, d: var TLoc) = - var a: TLoc - initLocExpr(p, n[0], a) + var a: TLoc = initLocExpr(p, n[0]) putIntoDest(p, d, n, ropecg(p.module, "#nimToCStringConv($1)", [rdLoc(a)]), # "($1 ? $1->data : (NCSTRING)\"\")" % [a.rdLoc], a.storage) proc convCStrToStr(p: BProc, n: PNode, d: var TLoc) = - var a: TLoc - initLocExpr(p, n[0], a) - putIntoDest(p, d, n, - ropecg(p.module, "#cstrToNimstr($1)", [rdLoc(a)]), - a.storage) + var a: TLoc = initLocExpr(p, n[0]) + if p.module.compileToCpp: + # fixes for const qualifier; bug #12703; bug #19588 + putIntoDest(p, d, n, + ropecg(p.module, "#cstrToNimstr((NCSTRING) $1)", [rdLoc(a)]), + a.storage) + else: + putIntoDest(p, d, n, + ropecg(p.module, "#cstrToNimstr($1)", [rdLoc(a)]), + a.storage) gcUsage(p.config, n) proc genStrEquals(p: BProc, e: PNode, d: var TLoc) = @@ -2113,11 +2266,11 @@ proc genStrEquals(p: BProc, e: PNode, d: var TLoc) = var a = e[1] var b = e[2] if a.kind in {nkStrLit..nkTripleStrLit} and a.strVal == "": - initLocExpr(p, e[2], x) + x = initLocExpr(p, e[2]) putIntoDest(p, d, e, ropecg(p.module, "($1 == 0)", [lenExpr(p, x)])) elif b.kind in {nkStrLit..nkTripleStrLit} and b.strVal == "": - initLocExpr(p, e[1], x) + x = initLocExpr(p, e[1]) putIntoDest(p, d, e, ropecg(p.module, "($1 == 0)", [lenExpr(p, x)])) else: @@ -2126,49 +2279,75 @@ proc genStrEquals(p: BProc, e: PNode, d: var TLoc) = proc binaryFloatArith(p: BProc, e: PNode, d: var TLoc, m: TMagic) = if {optNaNCheck, optInfCheck} * p.options != {}: const opr: array[mAddF64..mDivF64, string] = ["+", "-", "*", "/"] - var a, b: TLoc assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) putIntoDest(p, d, e, ropecg(p.module, "(($4)($2) $1 ($4)($3))", [opr[m], rdLoc(a), rdLoc(b), getSimpleTypeDesc(p.module, e[1].typ)])) if optNaNCheck in p.options: - linefmt(p, cpsStmts, "if ($1 != $1){ #raiseFloatInvalidOp(); $2}$n", [rdLoc(d), raiseInstr(p)]) + linefmt(p, cpsStmts, "if ($1 != $1){ #raiseFloatInvalidOp(); ", [rdLoc(d)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + if optInfCheck in p.options: - linefmt(p, cpsStmts, "if ($1 != 0.0 && $1*0.5 == $1) { #raiseFloatOverflow($1); $2}$n", [rdLoc(d), raiseInstr(p)]) + linefmt(p, cpsStmts, "if ($1 != 0.0 && $1*0.5 == $1) { #raiseFloatOverflow($1); ", [rdLoc(d)]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + else: binaryArith(p, e, d, m) -proc skipAddr(n: PNode): PNode = - result = if n.kind in {nkAddr, nkHiddenAddr}: n[0] else: n - proc genWasMoved(p: BProc; n: PNode) = var a: TLoc let n1 = n[1].skipAddr if p.withinBlockLeaveActions > 0 and notYetAlive(n1): discard else: - initLocExpr(p, n1, a) + a = initLocExpr(p, n1, {lfEnforceDeref}) resetLoc(p, a) #linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n", # [addrLoc(p.config, a), getTypeDesc(p.module, a.t)]) proc genMove(p: BProc; n: PNode; d: var TLoc) = - var a: TLoc - initLocExpr(p, n[1].skipAddr, a) + var a: TLoc = initLocExpr(p, n[1].skipAddr) if n.len == 4: # generated by liftdestructors: - var src: TLoc - initLocExpr(p, n[2], src) + var src: TLoc = initLocExpr(p, n[2]) linefmt(p, cpsStmts, "if ($1.p != $2.p) {", [rdLoc(a), rdLoc(src)]) genStmts(p, n[3]) linefmt(p, cpsStmts, "}$n$1.len = $2.len; $1.p = $2.p;$n", [rdLoc(a), rdLoc(src)]) else: - if d.k == locNone: getTemp(p, n.typ, d) - genAssignment(p, d, a, {}) - resetLoc(p, a) + if d.k == locNone: d = getTemp(p, n.typ) + if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}: + genAssignment(p, d, a, {}) + var op = getAttachedOp(p.module.g.graph, n.typ, attachedWasMoved) + if op == nil: + resetLoc(p, a) + else: + var b = initLocExpr(p, newSymNode(op)) + case skipTypes(a.t, abstractVar+{tyStatic}).kind + of tyOpenArray, tyVarargs: # todo fixme generated `wasMoved` hooks for + # openarrays, but it probably shouldn't? + var s: string + if reifiedOpenArray(a.lode): + if a.t.kind in {tyVar, tyLent}: + s = "$1->Field0, $1->Field1" % [rdLoc(a)] + else: + s = "$1.Field0, $1.Field1" % [rdLoc(a)] + else: + s = "$1, $1Len_0" % [rdLoc(a)] + linefmt(p, cpsStmts, "$1($2);$n", [rdLoc(b), s]) + else: + if p.module.compileToCpp: + linefmt(p, cpsStmts, "$1($2);$n", [rdLoc(b), rdLoc(a)]) + else: + linefmt(p, cpsStmts, "$1($2);$n", [rdLoc(b), byRefLoc(p, a)]) + else: + let flags = if not canMove(p, n[1], d): {needToCopy} else: {} + genAssignment(p, d, a, flags) + resetLoc(p, a) proc genDestroy(p: BProc; n: PNode) = if optSeqDestructors in p.config.globalOptions: @@ -2176,8 +2355,7 @@ proc genDestroy(p: BProc; n: PNode) = let t = arg.typ.skipTypes(abstractInst) case t.kind of tyString: - var a: TLoc - initLocExpr(p, arg, a) + var a: TLoc = initLocExpr(p, arg) if optThreads in p.config.globalOptions: linefmt(p, cpsStmts, "if ($1.p && !($1.p->cap & NIM_STRLIT_FLAG)) {$n" & " #deallocShared($1.p);$n" & @@ -2187,12 +2365,11 @@ proc genDestroy(p: BProc; n: PNode) = " #dealloc($1.p);$n" & "}$n", [rdLoc(a)]) of tySequence: - var a: TLoc - initLocExpr(p, arg, a) + var a: TLoc = initLocExpr(p, arg) linefmt(p, cpsStmts, "if ($1.p && !($1.p->cap & NIM_STRLIT_FLAG)) {$n" & " #alignedDealloc($1.p, NIM_ALIGNOF($2));$n" & "}$n", - [rdLoc(a), getTypeDesc(p.module, t.lastSon)]) + [rdLoc(a), getTypeDesc(p.module, t.elementType)]) else: discard "nothing to do" else: let t = n[1].typ.skipTypes(abstractVar) @@ -2203,10 +2380,9 @@ proc genDestroy(p: BProc; n: PNode) = proc genDispose(p: BProc; n: PNode) = when false: - let elemType = n[1].typ.skipTypes(abstractVar).lastSon + let elemType = n[1].typ.skipTypes(abstractVar).elementType - var a: TLoc - initLocExpr(p, n[1].skipAddr, a) + var a: TLoc = initLocExpr(p, n[1].skipAddr) if isFinal(elemType): if elemType.destructor != nil: @@ -2219,8 +2395,11 @@ proc genDispose(p: BProc; n: PNode) = lineCg(p, cpsStmts, ["#nimDestroyAndDispose($#)", rdLoc(a)]) proc genSlice(p: BProc; e: PNode; d: var TLoc) = - let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon) - if d.k == locNone: getTemp(p, e.typ, d) + let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.elementType, + prepareForMutation = e[1].kind == nkHiddenDeref and + e[1].typ.skipTypes(abstractInst).kind == tyString and + p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}) + if d.k == locNone: d = getTemp(p, e.typ) linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $3;$n", [rdLoc(d), x, y]) when false: localError(p.config, e.info, "invalid context for 'toOpenArray'; " & @@ -2255,11 +2434,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}: binaryStmt(p, e, d, opr[op]) else: - var a, b: TLoc assert(e[1].typ != nil) assert(e[2].typ != nil) - initLocExpr(p, e[1], a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) let ranged = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyDistinct}) let res = binaryArithOverflowRaw(p, ranged, a, b, @@ -2273,10 +2451,9 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = if optSeqDestructors in p.config.globalOptions: binaryStmtAddr(p, e, d, "nimAddCharV1") else: - var dest, b, call: TLoc - initLoc(call, locCall, e, OnHeap) - initLocExpr(p, e[1], dest) - initLocExpr(p, e[2], b) + var call = initLoc(locCall, e, OnHeap) + var dest = initLocExpr(p, e[1]) + var b = initLocExpr(p, e[2]) call.r = ropecg(p.module, "#addChar($1, $2)", [rdLoc(dest), rdLoc(b)]) genAssignment(p, dest, call, {}) of mAppendStrStr: genStrAppend(p, e, d) @@ -2290,14 +2467,16 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mLeStr: binaryExpr(p, e, d, "(#cmpStrings($1, $2) <= 0)") of mLtStr: binaryExpr(p, e, d, "(#cmpStrings($1, $2) < 0)") of mIsNil: genIsNil(p, e, d) - of mIntToStr: genDollar(p, e, d, "#nimIntToStr($1)") - of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)") of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)") of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)") - of mFloatToStr: genDollar(p, e, d, "#nimFloatToStr($1)") - of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)") + of mCStrToStr: + if p.module.compileToCpp: + # fixes for const qualifier; bug #12703; bug #19588 + genDollar(p, e, d, "#cstrToNimstr((NCSTRING) $1)") + else: + genDollar(p, e, d, "#cstrToNimstr($1)") of mStrToStr, mUnown: expr(p, e[1], d) - of mIsolate: genCall(p, e, d) + of generatedMagics: genCall(p, e, d) of mEnumToStr: if optTinyRtti in p.config.globalOptions: genEnumToStr(p, e, d) @@ -2307,20 +2486,24 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mNew: genNew(p, e) of mNewFinalize: if optTinyRtti in p.config.globalOptions: - var a: TLoc - initLocExpr(p, e[1], a) - rawGenNew(p, a, nil, needsInit = true) + var a: TLoc = initLocExpr(p, e[1]) + rawGenNew(p, a, "", needsInit = true) gcUsage(p.config, e) else: genNewFinalize(p, e) - of mNewSeq: genNewSeq(p, e) + of mNewSeq: + if optSeqDestructors in p.config.globalOptions: + e[1] = makeAddr(e[1], p.module.idgen) + genCall(p, e, d) + else: + genNewSeq(p, e) of mNewSeqOfCap: genNewSeqOfCap(p, e, d) of mSizeOf: let t = e[1].typ.skipTypes({tyTypeDesc}) - putIntoDest(p, d, e, "((NI)sizeof($1))" % [getTypeDesc(p.module, t, skVar)]) + putIntoDest(p, d, e, "((NI)sizeof($1))" % [getTypeDesc(p.module, t, dkVar)]) of mAlignOf: let t = e[1].typ.skipTypes({tyTypeDesc}) - putIntoDest(p, d, e, "((NI)NIM_ALIGNOF($1))" % [getTypeDesc(p.module, t, skVar)]) + putIntoDest(p, d, e, "((NI)NIM_ALIGNOF($1))" % [getTypeDesc(p.module, t, dkVar)]) of mOffsetOf: var dotExpr: PNode if e[1].kind == nkDotExpr: @@ -2328,9 +2511,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = elif e[1].kind == nkCheckedFieldExpr: dotExpr = e[1][0] else: + dotExpr = nil internalError(p.config, e.info, "unknown ast") let t = dotExpr[0].typ.skipTypes({tyTypeDesc}) - let tname = getTypeDesc(p.module, t, skVar) + let tname = getTypeDesc(p.module, t, dkVar) let member = if t.kind == tyTuple: "Field" & rope(dotExpr[1].sym.position) @@ -2340,8 +2524,12 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mOrd: genOrd(p, e, d) of mLengthArray, mHigh, mLengthStr, mLengthSeq, mLengthOpenArray: genArrayLen(p, e, d, op) - of mGCref: unaryStmt(p, e, d, "if ($1) { #nimGCref($1); }$n") - of mGCunref: unaryStmt(p, e, d, "if ($1) { #nimGCunref($1); }$n") + of mGCref: + # only a magic for the old GCs + unaryStmt(p, e, d, "if ($1) { #nimGCref($1); }$n") + of mGCunref: + # only a magic for the old GCs + unaryStmt(p, e, d, "if ($1) { #nimGCunref($1); }$n") of mSetLengthStr: genSetLengthStr(p, e, d) of mSetLengthSeq: genSetLengthSeq(p, e, d) of mIncl, mExcl, mCard, mLtSet, mLeSet, mEqSet, mMulSet, mPlusSet, mMinusSet, @@ -2367,46 +2555,50 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = let wasDeclared = containsOrIncl(p.module.declaredProtos, prc.id) # Make the function behind the magic get actually generated - this will # not lead to a forward declaration! The genCall will lead to one. - discard cgsym(p.module, $opr.loc.r) + cgsym(p.module, $opr.loc.r) # make sure we have pointer-initialising code for hot code reloading if not wasDeclared and p.hcrOn: p.module.s[cfsDynLibInit].addf("\t$1 = ($2) hcrGetProc($3, \"$1\");$n", [mangleDynLibProc(prc), getTypeDesc(p.module, prc.loc.t), getModuleDllPath(p.module, prc)]) genCall(p, e, d) - of mDefault: genDefault(p, e, d) - of mReset: genReset(p, e) + of mDefault, mZeroDefault: genDefault(p, e, d) of mEcho: genEcho(p, e[1].skipConv) of mArrToSeq: genArrToSeq(p, e, d) of mNLen..mNError, mSlurp..mQuoteAst: localError(p.config, e.info, strutils.`%`(errXMustBeCompileTime, e[0].sym.name.s)) of mSpawn: when defined(leanCompiler): - quit "compiler built without support for the 'spawn' statement" + p.config.quitOrRaise "compiler built without support for the 'spawn' statement" else: let n = spawn.wrapProcForSpawn(p.module.g.graph, p.module.idgen, p.module.module, e, e.typ, nil, nil) expr(p, n, d) of mParallel: when defined(leanCompiler): - quit "compiler built without support for the 'parallel' statement" + p.config.quitOrRaise "compiler built without support for the 'parallel' statement" else: let n = semparallel.liftParallel(p.module.g.graph, p.module.idgen, p.module.module, e) expr(p, n, d) of mDeepCopy: - if p.config.selectedGC in {gcArc, gcOrc} and optEnableDeepCopy notin p.config.globalOptions: + if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc} and optEnableDeepCopy notin p.config.globalOptions: localError(p.config, e.info, - "for --gc:arc|orc 'deepcopy' support has to be enabled with --deepcopy:on") + "for --mm:arc|atomicArc|orc 'deepcopy' support has to be enabled with --deepcopy:on") - var a, b: TLoc let x = if e[1].kind in {nkAddr, nkHiddenAddr}: e[1][0] else: e[1] - initLocExpr(p, x, a) - initLocExpr(p, e[2], b) + var a = initLocExpr(p, x) + var b = initLocExpr(p, e[2]) genDeepCopy(p, a, b) of mDotDot, mEqCString: genCall(p, e, d) of mWasMoved: genWasMoved(p, e) of mMove: genMove(p, e, d) of mDestroy: genDestroy(p, e) of mAccessEnv: unaryExpr(p, e, d, "$1.ClE_0") + of mAccessTypeField: genAccessTypeField(p, e, d) of mSlice: genSlice(p, e, d) + of mTrace: discard "no code to generate" + of mEnsureMove: + expr(p, e[1], d) + of mDup: + expr(p, e[1], d) else: when defined(debugMagics): echo p.prc.name.s, " ", p.prc.id, " ", p.prc.flags, " ", p.prc.ast[genericParamsPos].kind @@ -2418,60 +2610,90 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) = # nimZeroMem(tmp, sizeof(tmp)); inclRange(tmp, a, b); incl(tmp, c); # incl(tmp, d); incl(tmp, e); inclRange(tmp, f, g); var - a, b, idx: TLoc + a, b: TLoc + var idx: TLoc if nfAllConst in e.flags: - putIntoDest(p, d, e, genSetNode(p, e)) + var elem = newRopeAppender() + genSetNode(p, e, elem) + putIntoDest(p, d, e, elem) else: - if d.k == locNone: getTemp(p, e.typ, d) + if d.k == locNone: d = getTemp(p, e.typ) if getSize(p.config, e.typ) > 8: # big set: linefmt(p, cpsStmts, "#nimZeroMem($1, sizeof($2));$n", [rdLoc(d), getTypeDesc(p.module, e.typ)]) for it in e.sons: if it.kind == nkRange: - getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), idx) # our counter - initLocExpr(p, it[0], a) - initLocExpr(p, it[1], b) + idx = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt)) # our counter + a = initLocExpr(p, it[0]) + b = initLocExpr(p, it[1]) + var aa = newRopeAppender() + rdSetElemLoc(p.config, a, e.typ, aa) + var bb = newRopeAppender() + rdSetElemLoc(p.config, b, e.typ, bb) lineF(p, cpsStmts, "for ($1 = $3; $1 <= $4; $1++) $n" & "$2[(NU)($1)>>3] |=(1U<<((NU)($1)&7U));$n", [rdLoc(idx), rdLoc(d), - rdSetElemLoc(p.config, a, e.typ), rdSetElemLoc(p.config, b, e.typ)]) + aa, bb]) else: - initLocExpr(p, it, a) + a = initLocExpr(p, it) + var aa = newRopeAppender() + rdSetElemLoc(p.config, a, e.typ, aa) lineF(p, cpsStmts, "$1[(NU)($2)>>3] |=(1U<<((NU)($2)&7U));$n", - [rdLoc(d), rdSetElemLoc(p.config, a, e.typ)]) + [rdLoc(d), aa]) else: # small set var ts = "NU" & $(getSize(p.config, e.typ) * 8) lineF(p, cpsStmts, "$1 = 0;$n", [rdLoc(d)]) for it in e.sons: if it.kind == nkRange: - getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), idx) # our counter - initLocExpr(p, it[0], a) - initLocExpr(p, it[1], b) + idx = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt)) # our counter + a = initLocExpr(p, it[0]) + b = initLocExpr(p, it[1]) + var aa = newRopeAppender() + rdSetElemLoc(p.config, a, e.typ, aa) + var bb = newRopeAppender() + rdSetElemLoc(p.config, b, e.typ, bb) + lineF(p, cpsStmts, "for ($1 = $3; $1 <= $4; $1++) $n" & "$2 |=(($5)(1)<<(($1)%(sizeof($5)*8)));$n", [ - rdLoc(idx), rdLoc(d), rdSetElemLoc(p.config, a, e.typ), - rdSetElemLoc(p.config, b, e.typ), rope(ts)]) + rdLoc(idx), rdLoc(d), aa, bb, rope(ts)]) else: - initLocExpr(p, it, a) + a = initLocExpr(p, it) + var aa = newRopeAppender() + rdSetElemLoc(p.config, a, e.typ, aa) lineF(p, cpsStmts, "$1 |=(($3)(1)<<(($2)%(sizeof($3)*8)));$n", - [rdLoc(d), rdSetElemLoc(p.config, a, e.typ), rope(ts)]) + [rdLoc(d), aa, rope(ts)]) proc genTupleConstr(p: BProc, n: PNode, d: var TLoc) = var rec: TLoc if not handleConstExpr(p, n, d): let t = n.typ discard getTypeDesc(p.module, t) # so that any fields are initialized - if d.k == locNone: getTemp(p, t, d) + + var tmp: TLoc = default(TLoc) + # bug #16331 + let doesAlias = lhsDoesAlias(d.lode, n) + let dest = if doesAlias: addr(tmp) else: addr(d) + if doesAlias: + tmp = getTemp(p, n.typ) + elif d.k == locNone: + d = getTemp(p, n.typ) + for i in 0.. 0: exprOrStmt - if frameName != nil: + if frameName != "": p.s(cpsStmts).add deinitFrameNoDebug(p, frameName) proc genStmtListExpr(p: BProc, n: PNode, d: var TLoc) = @@ -2549,22 +2775,32 @@ proc genStmtList(p: BProc, n: PNode) = from parampatterns import isLValue proc upConv(p: BProc, n: PNode, d: var TLoc) = - var a: TLoc - initLocExpr(p, n[0], a) + var a: TLoc = initLocExpr(p, n[0]) let dest = skipTypes(n.typ, abstractPtrs) if optObjCheck in p.options and not isObjLackingTypeField(dest): - var nilCheck = Rope(nil) - let r = rdMType(p, a, nilCheck) - let checkFor = if optTinyRtti in p.config.globalOptions: - genTypeInfo2Name(p.module, dest) - else: - genTypeInfoV1(p.module, dest, n.info) - if nilCheck != nil: - linefmt(p, cpsStmts, "if ($1 && !#isObj($2, $3)){ #raiseObjectConversionError(); $4}$n", - [nilCheck, r, checkFor, raiseInstr(p)]) + var nilCheck = "" + var r = newRopeAppender() + rdMType(p, a, nilCheck, r) + if optTinyRtti in p.config.globalOptions: + let checkFor = $getObjDepth(dest) + let token = $genDisplayElem(MD5Digest(hashType(dest, p.config))) + if nilCheck != "": + linefmt(p, cpsStmts, "if ($1 && !#isObjDisplayCheck($2, $3, $4)){ #raiseObjectConversionError(); ", + [nilCheck, r, checkFor, token]) + else: + linefmt(p, cpsStmts, "if (!#isObjDisplayCheck($1, $2, $3)){ #raiseObjectConversionError(); ", + [r, checkFor, token]) else: - linefmt(p, cpsStmts, "if (!#isObj($1, $2)){ #raiseObjectConversionError(); $3}$n", - [r, checkFor, raiseInstr(p)]) + let checkFor = genTypeInfoV1(p.module, dest, n.info) + if nilCheck != "": + linefmt(p, cpsStmts, "if ($1 && !#isObj($2, $3)){ #raiseObjectConversionError(); ", + [nilCheck, r, checkFor]) + else: + linefmt(p, cpsStmts, "if (!#isObj($1, $2)){ #raiseObjectConversionError(); ", + [r, checkFor]) + raiseInstr(p, p.s(cpsStmts)) + linefmt p, cpsStmts, "}$n", [] + if n[0].typ.kind != tyObject: if n.isLValue: putIntoDest(p, d, n, @@ -2590,16 +2826,14 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) = # (see bug #837). However sometimes using a temporary is not correct: # init(TFigure(my)) # where it is passed to a 'var TFigure'. We test # this by ensuring the destination is also a pointer: - var a: TLoc - initLocExpr(p, arg, a) + var a: TLoc = initLocExpr(p, arg) putIntoDest(p, d, n, "(*(($1*) (&($2))))" % [getTypeDesc(p.module, n.typ), rdLoc(a)], a.storage) elif p.module.compileToCpp: # C++ implicitly downcasts for us expr(p, arg, d) else: - var a: TLoc - initLocExpr(p, arg, a) + var a: TLoc = initLocExpr(p, arg) var r = rdLoc(a) & (if isRef: "->Sup" else: ".Sup") for i in 2..abs(inheritanceDiff(dest, src)): r.add(".Sup") putIntoDest(p, d, n, if isRef: "&" & r else: r, a.storage) @@ -2613,8 +2847,10 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = if id == p.module.labels: # expression not found in the cache: inc(p.module.labels) - p.module.s[cfsData].addf("static NIM_CONST $1 $2 = $3;$n", - [getTypeDesc(p.module, t, skConst), tmp, genBracedInit(p, n, isConst = true, t)]) + p.module.s[cfsData].addf("static NIM_CONST $1 $2 = ", + [getTypeDesc(p.module, t, dkConst), tmp]) + genBracedInit(p, n, isConst = true, t, p.module.s[cfsData]) + p.module.s[cfsData].addf(";$n", []) if d.k == locNone: fillLoc(d, locData, n, tmp, OnStatic) @@ -2629,20 +2865,23 @@ proc genConstSetup(p: BProc; sym: PSym): bool = let m = p.module useHeader(m, sym) if sym.loc.k == locNone: - fillLoc(sym.loc, locData, sym.ast, mangleName(p.module, sym), OnStatic) + fillBackendName(p.module, sym) + fillLoc(sym.loc, locData, sym.astdef, OnStatic) if m.hcrOn: incl(sym.loc.flags, lfIndirect) result = lfNoDecl notin sym.loc.flags proc genConstHeader(m, q: BModule; p: BProc, sym: PSym) = - assert(sym.loc.r != nil) + if sym.loc.r == "": + if not genConstSetup(p, sym): return + assert(sym.loc.r != "", $sym.name.s & $sym.itemId) if m.hcrOn: - m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]); + m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, dkVar), sym.loc.r]); m.initProc.procSec(cpsLocals).addf( "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r, - getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(q, sym)]) + getTypeDesc(m, sym.loc.t, dkVar), getModuleDllPath(q, sym)]) else: let headerDecl = "extern NIM_CONST $1 $2;$n" % - [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r] + [getTypeDesc(m, sym.loc.t, dkVar), sym.loc.r] m.s[cfsData].add(headerDecl) if sfExportc in sym.flags and p.module.g.generatedHeader != nil: p.module.g.generatedHeader.s[cfsData].add(headerDecl) @@ -2650,12 +2889,15 @@ proc genConstHeader(m, q: BModule; p: BProc, sym: PSym) = proc genConstDefinition(q: BModule; p: BProc; sym: PSym) = # add a suffix for hcr - will later init the global pointer with this data let actualConstName = if q.hcrOn: sym.loc.r & "_const" else: sym.loc.r - q.s[cfsData].addf("N_LIB_PRIVATE NIM_CONST $1 $2 = $3;$n", - [getTypeDesc(q, sym.typ), actualConstName, - genBracedInit(q.initProc, sym.ast, isConst = true, sym.typ)]) + var data = newRopeAppender() + data.addf("N_LIB_PRIVATE NIM_CONST $1 $2 = ", + [getTypeDesc(q, sym.typ), actualConstName]) + genBracedInit(q.initProc, sym.astdef, isConst = true, sym.typ, data) + data.addf(";$n", []) + q.s[cfsData].add data if q.hcrOn: # generate the global pointer with the real name - q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(q, sym.loc.t, skVar), sym.loc.r]) + q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(q, sym.loc.t, dkVar), sym.loc.r]) # register it (but ignore the boolean result of hcrRegisterGlobal) q.initProc.procSec(cpsLocals).addf( "\thcrRegisterGlobal($1, \"$2\", sizeof($3), NULL, (void**)&$2);$n", @@ -2677,7 +2919,7 @@ proc genConstStmt(p: BProc, n: PNode) = genConstDefinition(m, p, sym) proc expr(p: BProc, n: PNode, d: var TLoc) = - when defined(nimCompilerStackraceHints): + when defined(nimCompilerStacktraceHints): setFrameMsg p.config$n.info & " " & $n.kind p.currLineInfo = n.info @@ -2704,15 +2946,17 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = genProcPrototype(p.module, sym) else: genProc(p.module, sym) - if sym.loc.r == nil or sym.loc.lode == nil: + if sym.loc.r == "" or sym.loc.lode == nil: internalError(p.config, n.info, "expr: proc not init " & sym.name.s) putLocIntoDest(p, d, sym.loc) of skConst: if isSimpleConst(sym.typ): - putIntoDest(p, d, n, genLiteral(p, sym.ast, sym.typ), OnStatic) + var lit = newRopeAppender() + genLiteral(p, sym.astdef, sym.typ, lit) + putIntoDest(p, d, n, lit, OnStatic) elif useAliveDataFromDce in p.module.flags: genConstHeader(p.module, p.module, p, sym) - assert((sym.loc.r != nil) and (sym.loc.t != nil)) + assert((sym.loc.r != "") and (sym.loc.t != nil)) putLocIntoDest(p, d, sym.loc) else: genComplexConst(p, sym, d) @@ -2727,7 +2971,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if sfCompileTime in sym.flags: genSingleVar(p, sym, n, astdef(sym)) - if sym.loc.r == nil or sym.loc.t == nil: + if sym.loc.r == "" or sym.loc.t == nil: #echo "FAILED FOR PRCO ", p.prc.name.s #echo renderTree(p.prc.ast, {renderIds}) internalError p.config, n.info, "expr: var not init " & sym.name.s & "_" & $sym.id @@ -2742,17 +2986,17 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of skTemp: when false: # this is more harmful than helpful. - if sym.loc.r == nil: + if sym.loc.r == "": # we now support undeclared 'skTemp' variables for easier # transformations in other parts of the compiler: assignLocalVar(p, n) - if sym.loc.r == nil or sym.loc.t == nil: + if sym.loc.r == "" or sym.loc.t == nil: #echo "FAILED FOR PRCO ", p.prc.name.s #echo renderTree(p.prc.ast, {renderIds}) internalError(p.config, n.info, "expr: temp not init " & sym.name.s & "_" & $sym.id) putLocIntoDest(p, d, sym.loc) of skParam: - if sym.loc.r == nil or sym.loc.t == nil: + if sym.loc.r == "" or sym.loc.t == nil: # echo "FAILED FOR PRCO ", p.prc.name.s # debug p.prc.typ.n # echo renderTree(p.prc.ast, {renderIds}) @@ -2761,19 +3005,24 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = else: internalError(p.config, n.info, "expr(" & $sym.kind & "); unknown symbol") of nkNilLit: if not isEmptyType(n.typ): - putIntoDest(p, d, n, genLiteral(p, n)) + var lit = newRopeAppender() + genLiteral(p, n, lit) + putIntoDest(p, d, n, lit) of nkStrLit..nkTripleStrLit: - putDataIntoDest(p, d, n, genLiteral(p, n)) - of nkIntLit..nkUInt64Lit, - nkFloatLit..nkFloat128Lit, nkCharLit: - putIntoDest(p, d, n, genLiteral(p, n)) + var lit = newRopeAppender() + genLiteral(p, n, lit) + putDataIntoDest(p, d, n, lit) + of nkIntLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkCharLit: + var lit = newRopeAppender() + genLiteral(p, n, lit) + putIntoDest(p, d, n, lit) of nkCall, nkHiddenCallConv, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit: genLineDir(p, n) # may be redundant, it is generated in fixupCall as well let op = n[0] if n.typ.isNil: # discard the value: - var a: TLoc + var a: TLoc = default(TLoc) if op.kind == nkSym and op.sym.magic != mNone: genMagicExpr(p, n, a, op.sym.magic) else: @@ -2786,7 +3035,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = genCall(p, n, d) of nkCurly: if isDeepConstExpr(n) and n.len != 0: - putIntoDest(p, d, n, genSetNode(p, n)) + var lit = newRopeAppender() + genSetNode(p, n, lit) + putIntoDest(p, d, n, lit) else: genSetConstr(p, n, d) of nkBracket: @@ -2806,7 +3057,16 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkObjConstr: genObjConstr(p, n, d) of nkCast: genCast(p, n, d) of nkHiddenStdConv, nkHiddenSubConv, nkConv: genConv(p, n, d) - of nkHiddenAddr, nkAddr: genAddr(p, n, d) + of nkHiddenAddr: + if n[0].kind == nkDerefExpr: + # addr ( deref ( x )) --> x + var x = n[0][0] + if n.typ.skipTypes(abstractVar).kind != tyOpenArray: + x.typ = n.typ + expr(p, x, d) + return + genAddr(p, n, d) + of nkAddr: genAddr(p, n, d) of nkBracketExpr: genBracketExpr(p, n, d) of nkDerefExpr, nkHiddenDeref: genDeref(p, n, d) of nkDotExpr: genRecordField(p, n, d) @@ -2826,7 +3086,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkLambdaKinds: var sym = n[namePos].sym genProc(p.module, sym) - if sym.loc.r == nil or sym.loc.lode == nil: + if sym.loc.r == "" or sym.loc.lode == nil: internalError(p.config, n.info, "expr: proc not init " & sym.name.s) putLocIntoDest(p, d, sym.loc) of nkClosure: genClosure(p, n, d) @@ -2846,7 +3106,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = cow(p, n[1]) if nfPreventCg notin n.flags: genAsgn(p, n, fastAsgn=false) - of nkFastAsgn: + of nkFastAsgn, nkSinkAsgn: cow(p, n[1]) if nfPreventCg notin n.flags: # transf is overly aggressive with 'nkFastAsgn', so we work around here. @@ -2856,8 +3116,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = let ex = n[0] if ex.kind != nkEmpty: genLineDir(p, n) - var a: TLoc - initLocExprSingleUse(p, ex, a) + var a: TLoc = initLocExprSingleUse(p, ex) line(p, cpsStmts, "(void)(" & a.r & ");\L") of nkAsmStmt: genAsmStmt(p, n) of nkTryStmt, nkHiddenTryStmt: @@ -2878,12 +3137,27 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt: discard of nkPragma: genPragma(p, n) - of nkPragmaBlock: expr(p, n.lastSon, d) + of nkPragmaBlock: + var inUncheckedAssignSection = 0 + let pragmaList = n[0] + for pi in pragmaList: + if whichPragma(pi) == wCast: + case whichPragma(pi[1]) + of wUncheckedAssign: + inUncheckedAssignSection = 1 + else: + discard + + inc p.inUncheckedAssignSection, inUncheckedAssignSection + expr(p, n.lastSon, d) + dec p.inUncheckedAssignSection, inUncheckedAssignSection + of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef: if n[genericParamsPos].kind == nkEmpty: var prc = n[namePos].sym if useAliveDataFromDce in p.module.flags: - if p.module.alive.contains(prc.itemId.item) and prc.magic in {mNone, mIsolate}: + if p.module.alive.contains(prc.itemId.item) and + prc.magic in generatedMagics: genProc(p.module, prc) elif prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags: if ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or @@ -2904,69 +3178,57 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkMixinStmt, nkBindStmt: discard else: internalError(p.config, n.info, "expr(" & $n.kind & "); unknown node kind") -proc genNamedConstExpr(p: BProc, n: PNode; isConst: bool): Rope = - if n.kind == nkExprColonExpr: result = genBracedInit(p, n[1], isConst, n[0].typ) - else: result = genBracedInit(p, n, isConst, n.typ) - -proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope = +proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Rope) = var t = skipTypes(typ, abstractRange+{tyOwned}-{tyTypeDesc}) case t.kind - of tyBool: result = rope"NIM_FALSE" - of tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64: result = rope"0" - of tyFloat..tyFloat128: result = rope"0.0" - of tyCString, tyVar, tyLent, tyPointer, tyPtr, tyUntyped, + of tyBool: result.add rope"NIM_FALSE" + of tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64: result.add rope"0" + of tyFloat..tyFloat128: result.add rope"0.0" + of tyCstring, tyVar, tyLent, tyPointer, tyPtr, tyUntyped, tyTyped, tyTypeDesc, tyStatic, tyRef, tyNil: - result = rope"NIM_NIL" + result.add rope"NIM_NIL" of tyString, tySequence: if optSeqDestructors in p.config.globalOptions: - result = rope"{0, NIM_NIL}" + result.add "{0, NIM_NIL}" else: - result = rope"NIM_NIL" + result.add "NIM_NIL" of tyProc: if t.callConv != ccClosure: - result = rope"NIM_NIL" + result.add "NIM_NIL" else: - result = rope"{NIM_NIL, NIM_NIL}" + result.add "{NIM_NIL, NIM_NIL}" of tyObject: var count = 0 result.add "{" getNullValueAuxT(p, t, t, t.n, nil, result, count, true, info) result.add "}" of tyTuple: - result = rope"{" - for i in 0.. 0: result.add ", " - result.add getDefaultValue(p, t[i], info) + getDefaultValue(p, a, info, result) result.add "}" of tyArray: - result = rope"{" - for i in 0.. 0: result.add ", " - result.add getDefaultValue(p, t.sons[1], info) + getDefaultValue(p, t.elementType, info, result) result.add "}" #result = rope"{}" of tyOpenArray, tyVarargs: - result = rope"{NIM_NIL, 0}" + result.add "{NIM_NIL, 0}" of tySet: - if mapSetType(p.config, t) == ctArray: result = rope"{}" - else: result = rope"0" + if mapSetType(p.config, t) == ctArray: result.add "{}" + else: result.add "0" else: globalError(p.config, info, "cannot create null element for: " & $t.kind) -proc caseObjDefaultBranch(obj: PNode; branch: Int128): int = - for i in 1 ..< obj.len: - for j in 0 .. obj[i].len - 2: - if obj[i][j].kind == nkRange: - let x = getOrdValue(obj[i][j][0]) - let y = getOrdValue(obj[i][j][1]) - if branch >= x and branch <= y: - return i - elif getOrdValue(obj[i][j]) == branch: - return i - if obj[i].len == 1: - # else branch - return i - assert(false, "unreachable") +proc isEmptyCaseObjectBranch(n: PNode): bool = + for it in n: + if it.kind == nkSym and not isEmptyType(it.sym.typ): return false + return true proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode, result: var Rope; count: var int; @@ -2996,7 +3258,7 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode, let b = lastSon(obj[selectedBranch]) # designated initilization is the only way to init non first element of unions # branches are allowed to have no members (b.len == 0), in this case they don't need initializer - if b.kind == nkRecList and b.len > 0: + if b.kind == nkRecList and not isEmptyCaseObjectBranch(b): result.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {" getNullValueAux(p, t, b, constOrNil, result, countB, isConst, info) result.add "}" @@ -3012,21 +3274,22 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode, if constOrNil != nil: for i in 1.. 0: result.add ",\n" - if it.kind == nkExprColonExpr: result.add genBracedInit(p, it[1], isConst, it[0].typ) - else: result.add genBracedInit(p, it, isConst, it.typ) + if it.kind == nkExprColonExpr: genBracedInit(p, it[1], isConst, it[0].typ, result) + else: genBracedInit(p, it, isConst, it.typ, result) result.add("}\n") -proc genConstTuple(p: BProc, n: PNode; isConst: bool; tup: PType): Rope = - result = rope("{") +proc genConstTuple(p: BProc, n: PNode; isConst: bool; tup: PType; result: var Rope) = + result.add "{" + if p.vccAndC and n.len == 0: + result.add "0" for i in 0.. 0: result.add ",\n" - if it.kind == nkExprColonExpr: result.add genBracedInit(p, it[1], isConst, tup[i]) - else: result.add genBracedInit(p, it, isConst, tup[i]) + if it.kind == nkExprColonExpr: genBracedInit(p, it[1], isConst, tup[i], result) + else: genBracedInit(p, it, isConst, tup[i], result) result.add("}\n") -proc genConstSeq(p: BProc, n: PNode, t: PType; isConst: bool): Rope = +proc genConstSeq(p: BProc, n: PNode, t: PType; isConst: bool; result: var Rope) = var data = "{{$1, $1 | NIM_STRLIT_FLAG}" % [n.len.rope] let base = t.skipTypes(abstractInst)[0] if n.len > 0: @@ -3081,44 +3348,45 @@ proc genConstSeq(p: BProc, n: PNode, t: PType; isConst: bool): Rope = data.add(", {") for i in 0.. 0: data.addf(",$n", []) - data.add genBracedInit(p, n[i], isConst, base) + genBracedInit(p, n[i], isConst, base, data) data.add("}") data.add("}") - result = getTempName(p.module) + let tmpName = getTempName(p.module) - appcg(p.module, cfsData, + appcg(p.module, cfsStrData, "static $5 struct {$n" & " #TGenericSeq Sup;$n" & " $1 data[$2];$n" & "} $3 = $4;$n", [ - getTypeDesc(p.module, base), n.len, result, data, + getTypeDesc(p.module, base), n.len, tmpName, data, if isConst: "NIM_CONST" else: ""]) - result = "(($1)&$2)" % [getTypeDesc(p.module, t), result] + result.add "(($1)&$2)" % [getTypeDesc(p.module, t), tmpName] -proc genConstSeqV2(p: BProc, n: PNode, t: PType; isConst: bool): Rope = +proc genConstSeqV2(p: BProc, n: PNode, t: PType; isConst: bool; result: var Rope) = let base = t.skipTypes(abstractInst)[0] - var data = rope"{" - for i in 0.. 0: data.addf(",$n", []) - data.add genBracedInit(p, n[i], isConst, base) - data.add("}") + var data = rope"" + if n.len > 0: + data.add(", {") + for i in 0.. 0: data.addf(",$n", []) + genBracedInit(p, n[i], isConst, base, data) + data.add("}") let payload = getTempName(p.module) - - appcg(p.module, cfsData, + appcg(p.module, cfsStrData, "static $5 struct {$n" & " NI cap; $1 data[$2];$n" & - "} $3 = {$2 | NIM_STRLIT_FLAG, $4};$n", [ + "} $3 = {$2 | NIM_STRLIT_FLAG$4};$n", [ getTypeDesc(p.module, base), n.len, payload, data, if isConst: "const" else: ""]) - result = "{$1, ($2*)&$3}" % [rope(n.len), getSeqPayloadType(p.module, t), payload] + result.add "{$1, ($2*)&$3}" % [rope(n.len), getSeqPayloadType(p.module, t), payload] -proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope = +proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; result: var Rope) = case n.kind of nkHiddenStdConv, nkHiddenSubConv: - result = genBracedInit(p, n[1], isConst, n.typ) + genBracedInit(p, n[1], isConst, n.typ, result) else: var ty = tyNone var typ: PType = nil @@ -3133,12 +3401,12 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope case ty of tySet: let cs = toBitSet(p.config, n) - result = genRawSetData(cs, int(getSize(p.config, n.typ))) + genRawSetData(cs, int(getSize(p.config, n.typ)), result) of tySequence: if optSeqDestructors in p.config.globalOptions: - result = genConstSeqV2(p, n, typ, isConst) + genConstSeqV2(p, n, typ, isConst, result) else: - result = genConstSeq(p, n, typ, isConst) + genConstSeq(p, n, typ, isConst, result) of tyProc: if typ.callConv == ccClosure and n.safeLen > 1 and n[1].kind == nkNilLit: # n.kind could be: nkClosure, nkTupleConstr and maybe others; `n.safeLen` @@ -3151,44 +3419,41 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope # leading to duplicate code like this: # "{NIM_NIL,NIM_NIL}, {NIM_NIL,NIM_NIL}" if n[0].kind == nkNilLit: - result = ~"{NIM_NIL,NIM_NIL}" + result.add "{NIM_NIL,NIM_NIL}" else: - var d: TLoc - initLocExpr(p, n[0], d) - result = "{(($1) $2),NIM_NIL}" % [getClosureType(p.module, typ, clHalfWithEnv), rdLoc(d)] + var d: TLoc = initLocExpr(p, n[0]) + result.add "{(($1) $2),NIM_NIL}" % [getClosureType(p.module, typ, clHalfWithEnv), rdLoc(d)] else: - var d: TLoc - initLocExpr(p, n, d) - result = rdLoc(d) + var d: TLoc = initLocExpr(p, n) + result.add rdLoc(d) of tyArray, tyVarargs: - result = genConstSimpleList(p, n, isConst) + genConstSimpleList(p, n, isConst, result) of tyTuple: - result = genConstTuple(p, n, isConst, typ) + genConstTuple(p, n, isConst, typ, result) of tyOpenArray: if n.kind != nkBracket: internalError(p.config, n.info, "const openArray expression is not an array construction") - let data = genConstSimpleList(p, n, isConst) + var data = newRopeAppender() + genConstSimpleList(p, n, isConst, data) let payload = getTempName(p.module) - let ctype = getTypeDesc(p.module, typ[0]) + let ctype = getTypeDesc(p.module, typ.elementType) let arrLen = n.len - appcg(p.module, cfsData, + appcg(p.module, cfsStrData, "static $5 $1 $3[$2] = $4;$n", [ ctype, arrLen, payload, data, if isConst: "const" else: ""]) - result = "{($1*)&$2, $3}" % [ctype, payload, rope arrLen] + result.add "{($1*)&$2, $3}" % [ctype, payload, rope arrLen] of tyObject: - result = genConstObjConstr(p, n, isConst) - of tyString, tyCString: + genConstObjConstr(p, n, isConst, result) + of tyString, tyCstring: if optSeqDestructors in p.config.globalOptions and n.kind != nkNilLit and ty == tyString: - result = genStringLiteralV2Const(p.module, n, isConst) + genStringLiteralV2Const(p.module, n, isConst, result) else: - var d: TLoc - initLocExpr(p, n, d) - result = rdLoc(d) + var d: TLoc = initLocExpr(p, n) + result.add rdLoc(d) else: - var d: TLoc - initLocExpr(p, n, d) - result = rdLoc(d) + var d: TLoc = initLocExpr(p, n) + result.add rdLoc(d) diff --git a/compiler/ccgliterals.nim b/compiler/ccgliterals.nim index ee56da5865..cbef6771f6 100644 --- a/compiler/ccgliterals.nim +++ b/compiler/ccgliterals.nim @@ -21,7 +21,7 @@ template detectVersion(field, corename) = if core == nil or core.kind != skConst: m.g.field = 1 else: - m.g.field = toInt(ast.getInt(core.ast)) + m.g.field = toInt(ast.getInt(core.astdef)) result = m.g.field proc detectStrVersion(m: BModule): int = @@ -32,82 +32,87 @@ proc detectSeqVersion(m: BModule): int = # ----- Version 1: GC'ed strings and seqs -------------------------------- -proc genStringLiteralDataOnlyV1(m: BModule, s: string): Rope = - discard cgsym(m, "TGenericSeq") - result = getTempName(m) - m.s[cfsData].addf("STRING_LITERAL($1, $2, $3);$n", - [result, makeCString(s), rope(s.len)]) +proc genStringLiteralDataOnlyV1(m: BModule, s: string; result: var Rope) = + cgsym(m, "TGenericSeq") + let tmp = getTempName(m) + result.add tmp + m.s[cfsStrData].addf("STRING_LITERAL($1, $2, $3);$n", + [tmp, makeCString(s), rope(s.len)]) -proc genStringLiteralV1(m: BModule; n: PNode): Rope = +proc genStringLiteralV1(m: BModule; n: PNode; result: var Rope) = if s.isNil: - result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", []) + appcg(m, result, "((#NimStringDesc*) NIM_NIL)", []) else: let id = nodeTableTestOrSet(m.dataCache, n, m.labels) if id == m.labels: # string literal not found in the cache: - result = ropecg(m, "((#NimStringDesc*) &$1)", - [genStringLiteralDataOnlyV1(m, n.strVal)]) + appcg(m, result, "((#NimStringDesc*) &", []) + genStringLiteralDataOnlyV1(m, n.strVal, result) + result.add ")" else: - result = ropecg(m, "((#NimStringDesc*) &$1$2)", + appcg(m, result, "((#NimStringDesc*) &$1$2)", [m.tmpBase, id]) # ------ Version 2: destructor based strings and seqs ----------------------- proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) = - m.s[cfsData].addf("static $4 struct {$n" & + m.s[cfsStrData].addf("static $4 struct {$n" & " NI cap; NIM_CHAR data[$2+1];$n" & "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n", [result, rope(s.len), makeCString(s), rope(if isConst: "const" else: "")]) -proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool): Rope = +proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool; result: var Rope) = let id = nodeTableTestOrSet(m.dataCache, n, m.labels) if id == m.labels: let pureLit = getTempName(m) genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst) - result = getTempName(m) - discard cgsym(m, "NimStrPayload") - discard cgsym(m, "NimStringV2") + let tmp = getTempName(m) + result.add tmp + cgsym(m, "NimStrPayload") + cgsym(m, "NimStringV2") # string literal not found in the cache: - m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n", - [result, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")]) + m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n", + [tmp, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")]) else: - result = getTempName(m) - m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n", - [result, rope(n.strVal.len), m.tmpBase & rope(id), + let tmp = getTempName(m) + result.add tmp + m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n", + [tmp, rope(n.strVal.len), m.tmpBase & rope(id), rope(if isConst: "const" else: "")]) -proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool): Rope = +proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Rope) = let id = nodeTableTestOrSet(m.dataCache, n, m.labels) var pureLit: Rope if id == m.labels: pureLit = getTempName(m) - discard cgsym(m, "NimStrPayload") - discard cgsym(m, "NimStringV2") + cgsym(m, "NimStrPayload") + cgsym(m, "NimStringV2") # string literal not found in the cache: genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst) else: pureLit = m.tmpBase & rope(id) - result = "{$1, (NimStrPayload*)&$2}" % [rope(n.strVal.len), pureLit] + result.addf "{$1, (NimStrPayload*)&$2}", [rope(n.strVal.len), pureLit] # ------ Version selector --------------------------------------------------- proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo; - isConst: bool): Rope = + isConst: bool; result: var Rope) = case detectStrVersion(m) - of 0, 1: result = genStringLiteralDataOnlyV1(m, s) + of 0, 1: genStringLiteralDataOnlyV1(m, s, result) of 2: - result = getTempName(m) - genStringLiteralDataOnlyV2(m, s, result, isConst) + let tmp = getTempName(m) + genStringLiteralDataOnlyV2(m, s, tmp, isConst) + result.add tmp else: localError(m.config, info, "cannot determine how to produce code for string literal") -proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope = - result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", []) +proc genNilStringLiteral(m: BModule; info: TLineInfo; result: var Rope) = + appcg(m, result, "((#NimStringDesc*) NIM_NIL)", []) -proc genStringLiteral(m: BModule; n: PNode): Rope = +proc genStringLiteral(m: BModule; n: PNode; result: var Rope) = case detectStrVersion(m) - of 0, 1: result = genStringLiteralV1(m, n) - of 2: result = genStringLiteralV2(m, n, isConst = true) + of 0, 1: genStringLiteralV1(m, n, result) + of 2: genStringLiteralV2(m, n, isConst = true, result) else: localError(m.config, n.info, "cannot determine how to produce code for string literal") diff --git a/compiler/ccgmerge_unused.nim b/compiler/ccgmerge_unused.nim index c7d19da7ac..a1413034fa 100644 --- a/compiler/ccgmerge_unused.nim +++ b/compiler/ccgmerge_unused.nim @@ -19,13 +19,11 @@ import const CFileSectionNames: array[TCFileSection, string] = [ - cfsMergeInfo: "", cfsHeaders: "NIM_merge_HEADERS", cfsFrameDefines: "NIM_merge_FRAME_DEFINES", cfsForwardTypes: "NIM_merge_FORWARD_TYPES", cfsTypes: "NIM_merge_TYPES", cfsSeqTypes: "NIM_merge_SEQ_TYPES", - cfsFieldInfo: "NIM_merge_FIELD_INFO", cfsTypeInfo: "NIM_merge_TYPE_INFO", cfsProcHeaders: "NIM_merge_PROC_HEADERS", cfsData: "NIM_merge_DATA", @@ -34,11 +32,8 @@ const cfsInitProc: "NIM_merge_INIT_PROC", cfsDatInitProc: "NIM_merge_DATINIT_PROC", cfsTypeInit1: "NIM_merge_TYPE_INIT1", - cfsTypeInit2: "NIM_merge_TYPE_INIT2", cfsTypeInit3: "NIM_merge_TYPE_INIT3", - cfsDebugInit: "NIM_merge_DEBUG_INIT", - cfsDynLibInit: "NIM_merge_DYNLIB_INIT", - cfsDynLibDeinit: "NIM_merge_DYNLIB_DEINIT", + cfsDynLibInit: "NIM_merge_DYNLIB_INIT" ] CProcSectionNames: array[TCProcSection, string] = [ cpsLocals: "NIM_merge_PROC_LOCALS", diff --git a/compiler/ccgreset.nim b/compiler/ccgreset.nim index ef1505f579..4af690e350 100644 --- a/compiler/ccgreset.nim +++ b/compiler/ccgreset.nim @@ -24,7 +24,7 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode; of nkRecCase: if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN") let disc = n[0].sym - if disc.loc.r == nil: fillObjectFields(p.module, typ) + if disc.loc.r == "": fillObjectFields(p.module, typ) if disc.loc.t == nil: internalError(p.config, n.info, "specializeResetN()") lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r]) @@ -42,7 +42,7 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode; of nkSym: let field = n.sym if field.typ.kind == tyVoid: return - if field.loc.r == nil: fillObjectFields(p.module, typ) + if field.loc.r == "": fillObjectFields(p.module, typ) if field.loc.t == nil: internalError(p.config, n.info, "specializeResetN()") specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t) @@ -54,25 +54,23 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) = case typ.kind of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred, tySink, tyOwned: - specializeResetT(p, accessor, lastSon(typ)) + specializeResetT(p, accessor, skipModifier(typ)) of tyArray: - let arraySize = lengthOrd(p.config, typ[0]) - var i: TLoc - getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) + let arraySize = lengthOrd(p.config, typ.indexType) + var i: TLoc = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt)) linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n", [i.r, arraySize]) - specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ[1]) + specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ.elementType) lineF(p, cpsStmts, "}$n", []) of tyObject: - for i in 0.. 2 # do not close and reopen blocks if this is a 'global' but inside of a block (if/while/block) @@ -84,23 +95,20 @@ proc genVarTuple(p: BProc, n: PNode) = startBlock(p) genLineDir(p, n) - initLocExpr(p, n[^1], tup) + var tup = initLocExpr(p, n[^1]) var t = tup.t.skipTypes(abstractInst) for i in 0.. 0: result.addf("FR_.len+=$1;$n", [b.frameLen.rope]) result.add(b.sections[cpsInit]) @@ -152,7 +160,7 @@ proc blockBody(b: var TBlock): Rope = proc endBlock(p: BProc, blockEnd: Rope) = let topBlock = p.blocks.len-1 # the block is merged into the parent block - p.blocks[topBlock-1].sections[cpsStmts].add(p.blocks[topBlock].blockBody) + p.blocks[topBlock].blockBody(p.blocks[topBlock-1].sections[cpsStmts]) setLen(p.blocks, topBlock) # this is done after the block is popped so $n is # properly indented when pretty printing is enabled @@ -161,10 +169,10 @@ proc endBlock(p: BProc, blockEnd: Rope) = proc endBlock(p: BProc) = let topBlock = p.blocks.len - 1 let frameLen = p.blocks[topBlock].frameLen - var blockEnd: Rope + var blockEnd: Rope = "" if frameLen > 0: blockEnd.addf("FR_.len-=$1;$n", [frameLen.rope]) - if p.blocks[topBlock].label != nil: + if p.blocks[topBlock].label.len != 0: blockEnd.addf("} $1: ;$n", [p.blocks[topBlock].label]) else: blockEnd.addf("}$n", []) @@ -236,8 +244,7 @@ proc genGotoState(p: BProc, n: PNode) = # switch (x.state) { # case 0: goto STATE0; # ... - var a: TLoc - initLocExpr(p, n[0], a) + var a: TLoc = initLocExpr(p, n[0]) lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)]) p.flags.incl beforeRetNeeded lineF(p, cpsStmts, "case -1:$n", []) @@ -256,13 +263,13 @@ proc genGotoState(p: BProc, n: PNode) = proc genBreakState(p: BProc, n: PNode, d: var TLoc) = var a: TLoc - initLoc(d, locExpr, n, OnUnknown) + d = initLoc(locExpr, n, OnUnknown) if n[0].kind == nkClosure: - initLocExpr(p, n[0][1], a) + a = initLocExpr(p, n[0][1]) d.r = "(((NI*) $1)[1] < 0)" % [rdLoc(a)] else: - initLocExpr(p, n[0], a) + a = initLocExpr(p, n[0]) # the environment is guaranteed to contain the 'state' field at offset 1: d.r = "((((NI*) $1.ClE_0)[1]) < 0)" % [rdLoc(a)] @@ -272,26 +279,59 @@ proc genGotoVar(p: BProc; value: PNode) = else: lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope]) -proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope +proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; result: var Rope) -proc potentialValueInit(p: BProc; v: PSym; value: PNode): Rope = +proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) = if lfDynamicLib in v.loc.flags or sfThread in v.flags or p.hcrOn: - result = nil + discard "nothing to do" elif sfGlobal in v.flags and value != nil and isDeepConstExpr(value, p.module.compileToCpp) and p.withinLoop == 0 and not containsGarbageCollectedRef(v.typ): #echo "New code produced for ", v.name.s, " ", p.config $ value.info - result = genBracedInit(p, value, isConst = false, v.typ) + genBracedInit(p, value, isConst = false, v.typ, result) + +proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): string = + result = "" + var argsCounter = 0 + let typ = skipTypes(call[0].typ, abstractInst) + assert(typ.kind == tyProc) + for i in 1.. 0: result.add "," + result.add genCppInitializer(p.module, p, call[i][0].sym.typ, didGenTemp) + else: + #We need to test for temp in globals, see: #23657 + let param = + if typ[i].kind in {tyVar} and call[i].kind == nkHiddenAddr: + call[i][0] + else: + call[i] + if param.kind != nkBracketExpr or param.typ.kind in + {tyRef, tyPtr, tyUncheckedArray, tyArray, tyOpenArray, + tyVarargs, tySequence, tyString, tyCstring, tyTuple}: + let tempLoc = initLocExprSingleUse(p, param) + didGenTemp = didGenTemp or tempLoc.k == locTemp + genOtherArg(p, call, i, typ, result, argsCounter) + +proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope, didGenTemp: var bool) = + let params = genCppParamsForCtor(p, call, didGenTemp) + if params.len == 0: + decl = runtimeFormat("$#;\n", [decl]) else: - result = nil + decl = runtimeFormat("$#($#);\n", [decl, params]) proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) = if sfGoto in v.flags: # translate 'var state {.goto.} = X' into 'goto LX': genGotoVar(p, value) return + let imm = isAssignedImmediately(p.config, value) + let isCppCtorCall = p.module.compileToCpp and imm and + value.kind in nkCallKinds and value[0].kind == nkSym and + v.typ.kind != tyPtr and sfConstructor in value[0].sym.flags var targetProc = p - var traverseProc: Rope - let valueAsRope = potentialValueInit(p, v, value) + var valueAsRope = "" + potentialValueInit(p, v, value, valueAsRope) if sfGlobal in v.flags: if v.flags * {sfImportc, sfExportc} == {sfImportc} and value.kind == nkEmpty and @@ -300,61 +340,63 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) = if sfPure in v.flags: # v.owner.kind != skModule: targetProc = p.module.preInitProc - assignGlobalVar(targetProc, vn, valueAsRope) + if isCppCtorCall and not containsHiddenPointer(v.typ): + var didGenTemp = false + callGlobalVarCppCtor(targetProc, v, vn, value, didGenTemp) + if didGenTemp: + message(p.config, vn.info, warnGlobalVarConstructorTemporary, vn.sym.name.s) + #We fail to call the constructor in the global scope so we do the call inside the main proc + assignGlobalVar(targetProc, vn, valueAsRope) + var loc = initLocExprSingleUse(targetProc, value) + genAssignment(targetProc, v.loc, loc, {}) + else: + assignGlobalVar(targetProc, vn, valueAsRope) + # XXX: be careful here. # Global variables should not be zeromem-ed within loops # (see bug #20). # That's why we are doing the construction inside the preInitProc. # genObjectInit relies on the C runtime's guarantees that # global variables will be initialized to zero. - if valueAsRope == nil: + if valueAsRope.len == 0: var loc = v.loc - # When the native TLS is unavailable, a global thread-local variable needs # one more layer of indirection in order to access the TLS block. # Only do this for complex types that may need a call to `objectInit` if sfThread in v.flags and emulatedThreadVars(p.config) and isComplexValueType(v.typ): - initLocExprSingleUse(p.module.preInitProc, vn, loc) + loc = initLocExprSingleUse(p.module.preInitProc, vn) genObjectInit(p.module.preInitProc, cpsInit, v.typ, loc, constructObj) # Alternative construction using default constructor (which may zeromem): # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc) if sfExportc in v.flags and p.module.g.generatedHeader != nil: genVarPrototype(p.module.g.generatedHeader, vn) - traverseProc = getTraverseProc(p, v) - if traverseProc != nil and not p.hcrOn: - registerTraverseProc(p, v, traverseProc) + registerTraverseProc(p, v) else: - let imm = isAssignedImmediately(p.config, value) if imm and p.module.compileToCpp and p.splitDecls == 0 and - not containsHiddenPointer(v.typ): + not containsHiddenPointer(v.typ) and + nimErrorFlagAccessed notin p.flags: # C++ really doesn't like things like 'Foo f; f = x' as that invokes a # parameterless constructor followed by an assignment operator. So we # generate better code here: 'Foo f = x;' genLineDir(p, vn) - let decl = localVarDecl(p, vn) + var decl = localVarDecl(p, vn) var tmp: TLoc - if value.kind in nkCallKinds and value[0].kind == nkSym and - sfConstructor in value[0].sym.flags: - var params: Rope - let typ = skipTypes(value[0].typ, abstractInst) - assert(typ.kind == tyProc) - for i in 1..= 0 and not p.blocks[idx].isLoop: dec idx if idx < 0 or not p.blocks[idx].isLoop: internalError(p.config, t.info, "no loop to break") - let label = assignLabel(p.blocks[idx]) + p.blocks[idx].label = "LA" & p.blocks[idx].id.rope blockLeaveActions(p, p.nestedTryStmts.len - p.blocks[idx].nestedTryStmts, p.inExceptBlockLen - p.blocks[idx].nestedExceptStmts) genLineDir(p, t) - lineF(p, cpsStmts, "goto $1;$n", [label]) + lineF(p, cpsStmts, "goto $1;$n", [p.blocks[idx].label]) proc raiseExit(p: BProc) = assert p.config.exc == excGoto @@ -714,6 +755,18 @@ proc raiseExit(p: BProc) = lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n", [p.nestedTryStmts[^1].label]) +proc raiseExitCleanup(p: BProc, destroy: string) = + assert p.config.exc == excGoto + if nimErrorFlagDisabled notin p.flags: + p.flags.incl nimErrorFlagAccessed + if p.nestedTryStmts.len == 0: + p.flags.incl beforeRetNeeded + # easy case, simply goto 'ret': + lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$1; goto BeforeRet_;}$n", [destroy]) + else: + lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) {$2; goto LA$1_;}$n", + [p.nestedTryStmts[^1].label, destroy]) + proc finallyActions(p: BProc) = if p.config.exc != excGoto and p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept: # if the current try stmt have a finally block, @@ -722,26 +775,23 @@ proc finallyActions(p: BProc) = if finallyBlock != nil: genSimpleBlock(p, finallyBlock[0]) -proc raiseInstr(p: BProc): Rope = +proc raiseInstr(p: BProc; result: var Rope) = if p.config.exc == excGoto: let L = p.nestedTryStmts.len if L == 0: p.flags.incl beforeRetNeeded # easy case, simply goto 'ret': - result = ropecg(p.module, "goto BeforeRet_;$n", []) + result.add ropecg(p.module, "goto BeforeRet_;$n", []) else: # raise inside an 'except' must go to the finally block, # raise outside an 'except' block must go to the 'except' list. - result = ropecg(p.module, "goto LA$1_;$n", + result.add ropecg(p.module, "goto LA$1_;$n", [p.nestedTryStmts[L-1].label]) # + ord(p.nestedTryStmts[L-1].inExcept)]) - else: - result = nil proc genRaiseStmt(p: BProc, t: PNode) = if t[0].kind != nkEmpty: - var a: TLoc - initLocExprSingleUse(p, t[0], a) + var a: TLoc = initLocExprSingleUse(p, t[0]) finallyActions(p) var e = rdLoc(a) discard getTypeDesc(p.module, t[0].typ) @@ -763,26 +813,20 @@ proc genRaiseStmt(p: BProc, t: PNode) = else: finallyActions(p) genLineDir(p, t) - # reraise the last exception: - if p.config.exc == excCpp: - line(p, cpsStmts, ~"throw;$n") - else: - linefmt(p, cpsStmts, "#reraiseException();$n", []) - let gotoInstr = raiseInstr(p) - if gotoInstr != nil: - line(p, cpsStmts, gotoInstr) + linefmt(p, cpsStmts, "#reraiseException();$n", []) + raiseInstr(p, p.s(cpsStmts)) template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc, rangeFormat, eqFormat: FormatStr, labl: TLabel) = var x, y: TLoc for i in 0.. 0: genIfForCaseUntil(p, n, d, rangeFormat = "if ($1 >= $2 && $1 <= $3) goto $4;$n", eqFormat = "if ($1 == $2) goto $3;$n", - splitPoint, a) else: nil + splitPoint, a) else: "" # generate switch part (might be empty): if splitPoint+1 < n.len: @@ -936,18 +1004,23 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) = hasDefault = true exprBlock(p, branch.lastSon, d) lineF(p, cpsStmts, "break;$n", []) - if (hasAssume in CC[p.config.cCompiler].props) and not hasDefault: - lineF(p, cpsStmts, "default: __assume(0);$n", []) + if not hasDefault: + if hasBuiltinUnreachable in CC[p.config.cCompiler].props: + lineF(p, cpsStmts, "default: __builtin_unreachable();$n", []) + elif hasAssume in CC[p.config.cCompiler].props: + lineF(p, cpsStmts, "default: __assume(0);$n", []) lineF(p, cpsStmts, "}$n", []) - if lend != nil: fixLabel(p, lend) + if lend != "": fixLabel(p, lend) proc genCase(p: BProc, t: PNode, d: var TLoc) = genLineDir(p, t) if not isEmptyType(t.typ) and d.k == locNone: - getTemp(p, t.typ, d) + d = getTemp(p, t.typ) case skipTypes(t[0].typ, abstractVarRange).kind of tyString: - genStringCase(p, t, d) + genStringCase(p, t, tyString, d) + of tyCstring: + genStringCase(p, t, tyCstring, d) of tyFloat..tyFloat128: genCaseGeneric(p, t, d, "if ($1 >= $2 && $1 <= $3) goto $4;$n", "if ($1 == $2) goto $3;$n") @@ -968,7 +1041,7 @@ proc genRestoreFrameAfterException(p: BProc) = proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = #[ code to generate: - std::exception_ptr error = nullptr; + std::exception_ptr error; try { body; } catch (Exception e) { @@ -993,13 +1066,13 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = p.module.includeHeader("") if not isEmptyType(t.typ) and d.k == locNone: - getTemp(p, t.typ, d) + d = getTemp(p, t.typ) genLineDir(p, t) inc(p.labels, 2) let etmp = p.labels - - p.procSec(cpsInit).add(ropecg(p.module, "\tstd::exception_ptr T$1_ = nullptr;", [etmp])) + #init on locals, fixes #23306 + lineCg(p, cpsLocals, "std::exception_ptr T$1_;$n", [etmp]) let fin = if t[^1].kind == nkFinally: t[^1] else: nil p.nestedTryStmts.add((fin, false, 0.Natural)) @@ -1033,12 +1106,11 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = if hasIf: lineF(p, cpsStmts, "else ", []) startBlock(p) # we handled the error: - linefmt(p, cpsStmts, "T$1_ = nullptr;$n", [etmp]) expr(p, t[i][0], d) linefmt(p, cpsStmts, "#popCurrentException();$n", []) endBlock(p) else: - var orExpr = Rope(nil) + var orExpr = newRopeAppender() var exvar = PNode(nil) for j in 0..$1, $2)", [memberName, checkFor]) + if optTinyRtti in p.config.globalOptions: + let checkFor = $getObjDepth(typeNode.typ) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ, p.config)))]) + else: + let checkFor = genTypeInfoV1(p.module, typeNode.typ, typeNode.info) + appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) - if orExpr != nil: + if orExpr.len != 0: if hasIf: startBlock(p, "else if ($1) {$n", [orExpr]) else: startBlock(p, "if ($1) {$n", [orExpr]) hasIf = true if exvar != nil: - fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnStack) + fillLocalName(p, exvar.sym) + fillLoc(exvar.sym.loc, locTemp, exvar, OnStack) linefmt(p, cpsStmts, "$1 $2 = T$3_;$n", [getTypeDesc(p.module, exvar.sym.typ), rdLoc(exvar.sym.loc), rope(etmp+1)]) # we handled the error: @@ -1094,7 +1168,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = if t[i].len == 1: # general except section: - startBlock(p, "catch (...) {", []) + startBlock(p, "catch (...) {$n", []) genExceptBranchBody(t[i][0]) endBlock(p) catchAllPresent = true @@ -1105,7 +1179,8 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = typeNode = t[i][j][1] if isImportedException(typeNode.typ, p.config): let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:` - fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnStack) + fillLocalName(p, exvar.sym) + fillLoc(exvar.sym.loc, locTemp, exvar, OnStack) startBlock(p, "catch ($1& $2) {$n", getTypeDesc(p.module, typeNode.typ), rdLoc(exvar.sym.loc)) genExceptBranchBody(t[i][^1]) # exception handler body will duplicated for every type endBlock(p) @@ -1119,7 +1194,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = # general finally block: if t.len > 0 and t[^1].kind == nkFinally: if not catchAllPresent: - startBlock(p, "catch (...) {", []) + startBlock(p, "catch (...) {$n", []) genRestoreFrameAfterException(p) linefmt(p, cpsStmts, "T$1_ = std::current_exception();$n", [etmp]) endBlock(p) @@ -1156,9 +1231,9 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) = expr(p, body, d) if not isEmptyType(t.typ) and d.k == locNone: - getTemp(p, t.typ, d) + d = getTemp(p, t.typ) genLineDir(p, t) - discard cgsym(p.module, "popCurrentExceptionEx") + cgsym(p.module, "popCurrentExceptionEx") let fin = if t[^1].kind == nkFinally: t[^1] else: nil p.nestedTryStmts.add((fin, false, 0.Natural)) startBlock(p, "try {$n") @@ -1184,7 +1259,8 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) = for j in 0..$1, $2)", [memberName, checkFor]) + if optTinyRtti in p.config.globalOptions: + let checkFor = $getObjDepth(t[i][j].typ) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", + [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))]) + else: + let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info) + appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) if i > 1: line(p, cpsStmts, "else ") startBlock(p, "if ($1) {$n", [orExpr]) @@ -1338,7 +1416,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = # propagateCurrentException(); # if not isEmptyType(t.typ) and d.k == locNone: - getTemp(p, t.typ, d) + d = getTemp(p, t.typ) let quirkyExceptions = p.config.exc == excQuirky or (t.kind == nkHiddenTryStmt and sfSystemModule in p.module.module.flags) if not quirkyExceptions: @@ -1346,8 +1424,8 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = else: p.flags.incl noSafePoints genLineDir(p, t) - discard cgsym(p.module, "Exception") - var safePoint: Rope + cgsym(p.module, "Exception") + var safePoint: Rope = "" if not quirkyExceptions: safePoint = getTempName(p.module) linefmt(p, cpsLocals, "#TSafePoint $1;$n", [safePoint]) @@ -1356,8 +1434,24 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimSigSetjmp"): linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", [safePoint]) + elif isDefined(p.config, "nimBuiltinSetjmp"): + linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimRawSetjmp"): - linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) + if isDefined(p.config, "mswindows"): + if isDefined(p.config, "vcc") or isDefined(p.config, "clangcl"): + # For the vcc compiler, use `setjmp()` with one argument. + # See https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setjmp?view=msvc-170 + linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint]) + else: + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) else: linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) lineCg(p, cpsStmts, "if ($1.status == 0) {$n", [safePoint]) @@ -1389,16 +1483,18 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = linefmt(p, cpsStmts, "#popCurrentException();$n", []) endBlock(p) else: - var orExpr: Rope = nil + var orExpr = newRopeAppender() for j in 0..$1, $2)", [memberName, checkFor]) + if optTinyRtti in p.config.globalOptions: + let checkFor = $getObjDepth(t[i][j].typ) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", + [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))]) + else: + let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info) + appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) if i > 1: line(p, cpsStmts, "else ") startBlock(p, "if ($1) {$n", [orExpr]) @@ -1424,35 +1520,33 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = if not quirkyExceptions: linefmt(p, cpsStmts, "if ($1.status != 0) #reraiseException();$n", [safePoint]) -proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope = +proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false; result: var Rope) = var res = "" - for it in t.sons: + let offset = + if isAsmStmt: 1 # first son is pragmas + else: 0 + + for i in offset..= 1 and n[0].kind in {nkStrLit..nkTripleStrLit}: let sec = n[0].strVal - if sec.startsWith("/*TYPESECTION*/"): result = cfsTypes + if sec.startsWith("/*TYPESECTION*/"): result = cfsForwardTypes # TODO WORKAROUND elif sec.startsWith("/*VARSECTION*/"): result = cfsVars elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders proc genEmit(p: BProc, t: PNode) = - var s = genAsmOrEmitStmt(p, t[1]) + var s = newRopeAppender() + genAsmOrEmitStmt(p, t[1], false, s) if p.prc == nil: # top level emit pragma? let section = determineSection(t[1]) @@ -1506,14 +1618,14 @@ proc genEmit(p: BProc, t: PNode) = line(p, cpsStmts, s) proc genPragma(p: BProc, n: PNode) = - for it in n.sons: + for i in 0.. 1: #we dont care about the return param + for i in 1.. error. - if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res): - res.add "_0" - result = res.rope - s.loc.r = result + s.loc.r = res.rope writeMangledName(m.ndi, s, m.config) -proc mangleLocalName(p: BProc; s: PSym): Rope = +proc fillLocalName(p: BProc; s: PSym) = assert s.kind in skLocalVars+{skTemp} #assert sfGlobal notin s.flags - result = s.loc.r - if result == nil: + if s.loc.r == "": var key = s.name.s.mangle - shallow(key) let counter = p.sigConflicts.getOrDefault(key) - result = key.rope + var result = key.rope if s.kind == skTemp: # speed up conflict search for temps (these are quite common): if counter != 0: result.add "_" & rope(counter+1) @@ -103,7 +129,6 @@ proc scopeMangledParam(p: BProc; param: PSym) = ## generate unique identifiers reliably (consider that ``var a = a`` is ## even an idiom in Nim). var key = param.name.s.mangle - shallow(key) p.sigConflicts.inc(key) const @@ -111,13 +136,12 @@ const tyDistinct, tyRange, tyStatic, tyAlias, tySink, tyInferred, tyOwned} -proc typeName(typ: PType): Rope = +proc typeName(typ: PType; result: var Rope) = let typ = typ.skipTypes(irrelevantForBackend) - result = - if typ.sym != nil and typ.kind in {tyObject, tyEnum}: - rope($typ.kind & '_' & typ.sym.name.s.mangle) - else: - rope($typ.kind) + result.add $typ.kind + if typ.sym != nil and typ.kind in {tyObject, tyEnum}: + result.add "_" + result.add typ.sym.name.s.mangle proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope = var t = typ @@ -126,18 +150,21 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope = return t.sym.loc.r if t.kind in irrelevantForBackend: - t = t.lastSon + t = t.skipModifier else: break - let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.lastSon else: typ - if typ.loc.r == nil: - typ.loc.r = typ.typeName & $sig + let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.elementType else: typ + if typ.loc.r == "": + typ.typeName(typ.loc.r) + typ.loc.r.add $sig else: when defined(debugSigHashes): # check consistency: - assert($typ.loc.r == $(typ.typeName & $sig)) + var tn = newRopeAppender() + typ.typeName(tn) + assert($typ.loc.r == $(tn & $sig)) result = typ.loc.r - if result == nil: internalError(m.config, "getTypeName: " & $typ.kind) + if result == "": internalError(m.config, "getTypeName: " & $typ.kind) proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind = case int(getSize(conf, typ)) @@ -147,7 +174,7 @@ proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind = of 8: result = ctInt64 else: result = ctArray -proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind = +proc mapType(conf: ConfigRef; typ: PType; isParam: bool): TCTypeKind = ## Maps a Nim type to a C type case typ.kind of tyNone, tyTyped: result = ctVoid @@ -156,16 +183,16 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind = of tyNil: result = ctPtr of tySet: result = mapSetType(conf, typ) of tyOpenArray, tyVarargs: - if kind == skParam: result = ctArray + if isParam: result = ctArray else: result = ctStruct of tyArray, tyUncheckedArray: result = ctArray of tyObject, tyTuple: result = ctStruct of tyUserTypeClasses: doAssert typ.isResolvedUserTypeClass - return mapType(conf, typ.lastSon, kind) + result = mapType(conf, typ.skipModifier, isParam) of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink, tyInferred, tyOwned: - result = mapType(conf, lastSon(typ), kind) + result = mapType(conf, skipModifier(typ), isParam) of tyEnum: if firstOrd(conf, typ) < 0: result = ctInt32 @@ -176,9 +203,9 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind = of 4: result = ctInt32 of 8: result = ctInt64 else: result = ctInt32 - of tyRange: result = mapType(conf, typ[0], kind) + of tyRange: result = mapType(conf, typ.elementType, isParam) of tyPtr, tyVar, tyLent, tyRef: - var base = skipTypes(typ.lastSon, typedescInst) + var base = skipTypes(typ.elementType, typedescInst) case base.kind of tyOpenArray, tyArray, tyVarargs, tyUncheckedArray: result = ctPtrToArray of tySet: @@ -189,18 +216,23 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind = of tySequence: result = ctNimSeq of tyProc: result = if typ.callConv != ccClosure: ctProc else: ctStruct of tyString: result = ctNimStr - of tyCString: result = ctCString + of tyCstring: result = ctCString of tyInt..tyUInt64: result = TCTypeKind(ord(typ.kind) - ord(tyInt) + ord(ctInt)) of tyStatic: - if typ.n != nil: result = mapType(conf, lastSon typ, kind) - else: doAssert(false, "mapType") - else: doAssert(false, "mapType") + if typ.n != nil: result = mapType(conf, typ.skipModifier, isParam) + else: + result = ctVoid + doAssert(false, "mapType: " & $typ.kind) + else: + result = ctVoid + doAssert(false, "mapType: " & $typ.kind) + proc mapReturnType(conf: ConfigRef; typ: PType): TCTypeKind = #if skipTypes(typ, typedescInst).kind == tyArray: result = ctPtr #else: - result = mapType(conf, typ, skResult) + result = mapType(conf, typ, false) proc isImportedType(t: PType): bool = result = t.sym != nil and sfImportc in t.sym.flags @@ -210,26 +242,42 @@ proc isImportedCppType(t: PType): bool = result = (t.sym != nil and sfInfixCall in t.sym.flags) or (x.sym != nil and sfInfixCall in x.sym.flags) -proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKind): Rope +proc isOrHasImportedCppType(typ: PType): bool = + searchTypeFor(typ.skipTypes({tyRef}), isImportedCppType) + +proc hasNoInit(t: PType): bool = + result = t.sym != nil and sfNoInit in t.sym.flags + +proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDescKind): Rope proc isObjLackingTypeField(typ: PType): bool {.inline.} = result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and - (typ[0] == nil) or isPureObject(typ)) + (typ.baseClass == nil) or isPureObject(typ)) -proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool = +proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool = # Arrays and sets cannot be returned by a C procedure, because C is # such a poor programming language. # We exclude records with refs too. This enhances efficiency and # is necessary for proper code generation of assignments. - if rettype == nil: result = true + var rettype = typ + var isAllowedCall = true + if isProc: + rettype = rettype[0] + isAllowedCall = typ.callConv in {ccClosure, ccInline, ccNimCall} + if rettype == nil or (isAllowedCall and + getSize(conf, rettype) > conf.target.floatSize*3): + result = true else: - case mapType(conf, rettype, skResult) + case mapType(conf, rettype, false) of ctArray: result = not (skipTypes(rettype, typedescInst).kind in {tyVar, tyLent, tyRef, tyPtr}) of ctStruct: let t = skipTypes(rettype, typedescInst) - if rettype.isImportedCppType or t.isImportedCppType: return false + if rettype.isImportedCppType or t.isImportedCppType or + (typ.callConv == ccCDecl and conf.selectedGC in {gcArc, gcAtomicArc, gcOrc}): + # prevents nrvo for cdecl procs; # bug #23401 + return false result = containsGarbageCollectedRef(t) or (t.kind == tyObject and not isObjLackingTypeField(t)) else: result = false @@ -239,7 +287,9 @@ const "N_STDCALL", "N_CDECL", "N_SAFECALL", "N_SYSCALL", # this is probably not correct for all platforms, # but one can #define it to what one wants - "N_INLINE", "N_NOINLINE", "N_FASTCALL", "N_THISCALL", "N_CLOSURE", "N_NOCONV"] + "N_INLINE", "N_NOINLINE", "N_FASTCALL", "N_THISCALL", "N_CLOSURE", "N_NOCONV", + "N_NOCONV" #ccMember is N_NOCONV + ] proc cacheGetType(tab: TypeCache; sig: SigHash): Rope = # returns nil if we need to declare this type @@ -247,7 +297,7 @@ proc cacheGetType(tab: TypeCache; sig: SigHash): Rope = # linear search is not necessary anymore: result = tab.getOrDefault(sig) -proc addAbiCheck(m: BModule, t: PType, name: Rope) = +proc addAbiCheck(m: BModule; t: PType, name: Rope) = if isDefined(m.config, "checkAbi") and (let size = getSize(m.config, t); size != szUnknownSize): var msg = "backend & Nim disagree on size for: " msg.addTypeHeader(m.config, t) @@ -256,38 +306,12 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) = m.s[cfsTypeInfo].addf("NIM_STATIC_ASSERT(sizeof($1) == $2, $3);$n", [name, rope(size), msg2.rope]) # see `testCodegenABICheck` for example error message it generates -proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool = - var pt = skipTypes(s.typ, typedescInst) - assert skResult != s.kind - if tfByRef in pt.flags: return true - elif tfByCopy in pt.flags: return false - case pt.kind - of tyObject: - if s.typ.sym != nil and sfForward in s.typ.sym.flags: - # forwarded objects are *always* passed by pointers for consistency! - result = true - elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3): - result = true # requested anyway - elif (tfFinal in pt.flags) and (pt[0] == nil): - result = false # no need, because no subtyping possible - else: - result = true # ordinary objects are always passed by reference, - # otherwise casting doesn't work - of tyTuple: - result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options) - else: - result = false - # first parameter and return type is 'lent T'? --> use pass by pointer - if s.position == 0 and retType != nil and retType.kind == tyLent: - result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or - pt.kind == tySet and mapSetType(conf, pt) == ctArray) - -proc fillResult(conf: ConfigRef; param: PNode) = - fillLoc(param.sym.loc, locParam, param, ~"Result", +proc fillResult(conf: ConfigRef; param: PNode, proctype: PType) = + fillLoc(param.sym.loc, locParam, param, "Result", OnStack) let t = param.sym.typ - if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, t): + if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, proctype): incl(param.sym.loc.flags, lfIndirect) param.sym.loc.storage = OnUnknown @@ -298,7 +322,7 @@ proc typeNameOrLiteral(m: BModule; t: PType, literal: string): Rope = else: result = rope(literal) -proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = +proc getSimpleTypeDesc(m: BModule; typ: PType): Rope = const NumericalTypeToStr: array[tyInt..tyUInt64, string] = [ "NI", "NI8", "NI16", "NI32", "NI64", @@ -310,51 +334,51 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = of tyString: case detectStrVersion(m) of 2: - discard cgsym(m, "NimStrPayload") - discard cgsym(m, "NimStringV2") + cgsym(m, "NimStrPayload") + cgsym(m, "NimStringV2") result = typeNameOrLiteral(m, typ, "NimStringV2") else: - discard cgsym(m, "NimStringDesc") + cgsym(m, "NimStringDesc") result = typeNameOrLiteral(m, typ, "NimStringDesc*") - of tyCString: result = typeNameOrLiteral(m, typ, "NCSTRING") + of tyCstring: result = typeNameOrLiteral(m, typ, "NCSTRING") of tyBool: result = typeNameOrLiteral(m, typ, "NIM_BOOL") of tyChar: result = typeNameOrLiteral(m, typ, "NIM_CHAR") of tyNil: result = typeNameOrLiteral(m, typ, "void*") of tyInt..tyUInt64: result = typeNameOrLiteral(m, typ, NumericalTypeToStr[typ.kind]) - of tyDistinct, tyRange, tyOrdinal: result = getSimpleTypeDesc(m, typ[0]) + of tyDistinct, tyRange, tyOrdinal: result = getSimpleTypeDesc(m, typ.skipModifier) of tyStatic: - if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ) - else: internalError(m.config, "tyStatic for getSimpleTypeDesc") + if typ.n != nil: result = getSimpleTypeDesc(m, skipModifier typ) + else: + result = "" + internalError(m.config, "tyStatic for getSimpleTypeDesc") of tyGenericInst, tyAlias, tySink, tyOwned: - result = getSimpleTypeDesc(m, lastSon typ) - else: result = nil + result = getSimpleTypeDesc(m, skipModifier typ) + else: result = "" - if result != nil and typ.isImportedType(): - let sig = hashType typ - if cacheGetType(m.typeCache, sig) == nil: + if result != "" and typ.isImportedType(): + let sig = hashType(typ, m.config) + if cacheGetType(m.typeCache, sig) == "": m.typeCache[sig] = result -proc pushType(m: BModule, typ: PType) = +proc pushType(m: BModule; typ: PType) = for i in 0..high(m.typeStack): # pointer equality is good enough here: if m.typeStack[i] == typ: return m.typeStack.add(typ) -proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope = +proc getTypePre(m: BModule; typ: PType; sig: SigHash): Rope = if typ == nil: result = rope("void") else: result = getSimpleTypeDesc(m, typ) - if result == nil: result = cacheGetType(m.typeCache, sig) + if result == "": result = cacheGetType(m.typeCache, sig) proc structOrUnion(t: PType): Rope = - let cachedUnion = rope("union") - let cachedStruct = rope("struct") let t = t.skipTypes({tyAlias, tySink}) - if tfUnion in t.flags: cachedUnion - else: cachedStruct + if tfUnion in t.flags: "union" + else: "struct" -proc addForwardStructFormat(m: BModule, structOrUnion: Rope, typename: Rope) = +proc addForwardStructFormat(m: BModule; structOrUnion: Rope, typename: Rope) = if m.compileToCpp: m.s[cfsForwardTypes].addf "$1 $2;$n", [structOrUnion, typename] else: @@ -364,11 +388,11 @@ proc seqStar(m: BModule): string = if optSeqDestructors in m.config.globalOptions: result = "" else: result = "*" -proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope = +proc getTypeForward(m: BModule; typ: PType; sig: SigHash): Rope = result = cacheGetType(m.forwTypeCache, sig) - if result != nil: return + if result != "": return result = getTypePre(m, typ, sig) - if result != nil: return + if result != "": return let concrete = typ.skipTypes(abstractInst) case concrete.kind of tySequence, tyTuple, tyObject: @@ -381,7 +405,7 @@ proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope = doAssert m.forwTypeCache[sig] == result else: internalError(m.config, "getTypeForward(" & $typ.kind & ')') -proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): Rope = +proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TypeDescKind): Rope = ## like getTypeDescAux but creates only a *weak* dependency. In other words ## we know we only need a pointer to it so we only generate a struct forward ## declaration: @@ -391,16 +415,16 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): R if isImportedCppType(etB) and t.kind == tyGenericInst: result = getTypeDescAux(m, t, check, kind) else: - result = getTypeForward(m, t, hashType(t)) + result = getTypeForward(m, t, hashType(t, m.config)) pushType(m, t) of tySequence: - let sig = hashType(t) + let sig = hashType(t, m.config) if optSeqDestructors in m.config.globalOptions: if skipTypes(etB[0], typedescInst).kind == tyEmpty: internalError(m.config, "cannot map the empty seq type to a C type") result = cacheGetType(m.forwTypeCache, sig) - if result == nil: + if result == "": result = getTypeName(m, t, sig) if not isImportedType(t): m.forwTypeCache[sig] = result @@ -408,38 +432,34 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): R let payload = result & "_Content" addForwardStructFormat(m, rope"struct", payload) - if cacheGetType(m.typeCache, sig) == nil: + if cacheGetType(m.typeCache, sig) == "": m.typeCache[sig] = result #echo "adding ", sig, " ", typeToString(t), " ", m.module.name.s appcg(m, m.s[cfsTypes], - "struct $1 {$N" & - " NI len; $1_Content* p;$N" & - "};$N", [result]) + "struct $1 {\n" & + " NI len; $1_Content* p;\n" & + "};\n", [result]) + pushType(m, t) else: result = getTypeForward(m, t, sig) & seqStar(m) - pushType(m, t) + pushType(m, t) else: result = getTypeDescAux(m, t, check, kind) proc getSeqPayloadType(m: BModule; t: PType): Rope = var check = initIntSet() - result = getTypeDescWeak(m, t, check, skParam) & "_Content" + result = getTypeDescWeak(m, t, check, dkParam) & "_Content" #result = getTypeForward(m, t, hashType(t)) & "_Content" proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) = - let sig = hashType(t) + let sig = hashType(t, m.config) let result = cacheGetType(m.typeCache, sig) - if result == nil: - discard getTypeDescAux(m, t, check, skVar) + if result == "": + discard getTypeDescAux(m, t, check, dkVar) else: - # little hack for now to prevent multiple definitions of the same - # Seq_Content: - appcg(m, m.s[cfsTypes], """$N -$3ifndef $2_Content_PP -$3define $2_Content_PP -struct $2_Content { NI cap; $1 data[SEQ_DECL_SIZE];}; -$3endif$N - """, [getTypeDescAux(m, t.skipTypes(abstractInst)[0], check, skVar), result, rope"#"]) + appcg(m, m.s[cfsTypes], """ +struct $2_Content { NI cap; $1 data[SEQ_DECL_SIZE]; }; +""", [getTypeDescAux(m, t.skipTypes(abstractInst)[0], check, dkVar), result]) proc paramStorageLoc(param: PSym): TStorageLoc = if param.typ.skipTypes({tyVar, tyLent, tyTypeDesc}).kind notin { @@ -448,37 +468,171 @@ proc paramStorageLoc(param: PSym): TStorageLoc = else: result = OnUnknown -proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, +macro unrollChars(x: static openArray[char], name, body: untyped) = + result = newStmtList() + for a in x: + result.add(newBlockStmt(newStmtList( + newConstStmt(name, newLit(a)), + copy body + ))) + +proc multiFormat*(frmt: var string, chars: static openArray[char], args: openArray[seq[string]]) = + var res: string + unrollChars(chars, c): + res = "" + let arg = args[find(chars, c)] + var i = 0 + var num = 0 + while i < frmt.len: + if frmt[i] == c: + inc(i) + case frmt[i] + of c: + res.add(c) + inc(i) + of '0'..'9': + var j = 0 + while true: + j = j * 10 + ord(frmt[i]) - ord('0') + inc(i) + if i >= frmt.len or frmt[i] notin {'0'..'9'}: break + num = j + if j > high(arg) + 1: + raiseAssert "invalid format string: " & frmt + else: + res.add(arg[j-1]) + else: + raiseAssert "invalid format string: " & frmt + var start = i + while i < frmt.len: + if frmt[i] != c: inc(i) + else: break + if i - 1 >= start: + res.add(substr(frmt, start, i - 1)) + frmt = res + +template cgDeclFrmt*(s: PSym): string = + s.constraint.strVal + +proc genMemberProcParams(m: BModule; prc: PSym, superCall, rettype, name, params: var string, check: var IntSet, declareEnvironment=true; - weakDep=false) = - params = nil - if t[0] == nil or isInvalidReturnType(m.config, t[0]): - rettype = ~"void" + weakDep=false;) = + let t = prc.typ + let isCtor = sfConstructor in prc.flags + if isCtor or (name[0] == '~' and sfMember in prc.flags): + # destructors can't have void + rettype = "" + elif t.returnType == nil or isInvalidReturnType(m.config, t): + rettype = "void" else: - rettype = getTypeDescAux(m, t[0], check, skResult) - for i in 1..raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name]) hasField = true else: - appcg(m, result, " {$n $1 Sup;$n", - [getTypeDescAux(m, typ[0].skipTypes(skipPtrs), check, skField)]) + appcg(m, result, " {$n $1 Sup;$n", [baseType]) hasField = true else: result.addf(" {$n", [name]) - let desc = getRecordFields(m, typ, check) - if desc == nil and not hasField: - result.addf("char dummy;$n", []) +proc getRecordDesc(m: BModule; typ: PType, name: Rope, + check: var IntSet): Rope = + # declare the record: + var hasField = false + var structOrUnion: string + if tfPacked in typ.flags: + if hasAttribute in CC[m.config.cCompiler].props: + structOrUnion = structOrUnion(typ) & " __attribute__((__packed__))" + else: + structOrUnion = "#pragma pack(push, 1)\L" & structOrUnion(typ) else: - result.add(desc) - result.add("};\L") + structOrUnion = structOrUnion(typ) + var baseType: string = "" + if typ.baseClass != nil: + baseType = getTypeDescAux(m, typ.baseClass.skipTypes(skipPtrs), check, dkField) + if typ.sym == nil or sfCodegenDecl notin typ.sym.flags: + result = structOrUnion & " " & name + result.add(getRecordDescAux(m, typ, name, baseType, check, hasField)) + let desc = getRecordFields(m, typ, check) + if not hasField and typ.itemId notin m.g.graph.memberProcsPerType: + if desc == "": + result.add("\tchar dummy;\n") + elif typ.n.len == 1 and typ.n[0].kind == nkSym: + let field = typ.n[0].sym + let fieldType = field.typ.skipTypes(abstractInst) + if fieldType.kind == tyUncheckedArray: + result.add("\tchar dummy;\n") + result.add(desc) + else: + result.add(desc) + result.add("};\L") + else: + let desc = getRecordFields(m, typ, check) + result = runtimeFormat(typ.sym.cgDeclFrmt, [name, desc, baseType]) if tfPacked in typ.flags and hasAttribute notin CC[m.config.cCompiler].props: result.add "#pragma pack(pop)\L" -proc getTupleDesc(m: BModule, typ: PType, name: Rope, +proc getTupleDesc(m: BModule; typ: PType, name: Rope, check: var IntSet): Rope = result = "$1 $2 {$n" % [structOrUnion(typ), name] - var desc: Rope = nil - for i in 0..= typ.len: - doAssert false, "invalid apostrophe type parameter index" + if idx >= typ.kidsLen: + raiseAssert "invalid apostrophe type parameter index" result = typ[idx] for i in 1..stars: - if result != nil and result.len > 0: - result = if result.kind == tyGenericInst: result[1] + if result != nil and result.kidsLen > 0: + result = if result.kind == tyGenericInst: result[FirstGenericParamAt] else: result.elemType -proc getOpenArrayDesc(m: BModule, t: PType, check: var IntSet; kind: TSymKind): Rope = - let sig = hashType(t) - if kind == skParam: - result = getTypeDescWeak(m, t[0], check, kind) & "*" +proc getOpenArrayDesc(m: BModule; t: PType, check: var IntSet; kind: TypeDescKind): Rope = + let sig = hashType(t, m.config) + if kind == dkParam: + result = getTypeDescWeak(m, t.elementType, check, kind) & "*" else: result = cacheGetType(m.typeCache, sig) - if result == nil: + if result == "": result = getTypeName(m, t, sig) m.typeCache[sig] = result - let elemType = getTypeDescWeak(m, t[0], check, kind) + let elemType = getTypeDescWeak(m, t.elementType, check, kind) m.s[cfsTypes].addf("typedef struct {$n$2* Field0;$nNI Field1;$n} $1;$n", [result, elemType]) -proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKind): Rope = +proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDescKind): Rope = # returns only the type's name - var t = origTyp.skipTypes(irrelevantForBackend-{tyOwned}) if containsOrIncl(check, t.id): if not (isImportedCppType(origTyp) or isImportedCppType(t)): @@ -711,23 +927,26 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin # C type generation into an analysis and a code generation phase somehow. if t.sym != nil: useHeader(m, t.sym) if t != origTyp and origTyp.sym != nil: useHeader(m, origTyp.sym) - let sig = hashType(origTyp) + let sig = hashType(origTyp, m.config) + result = "" # todo move `result = getTypePre(m, t, sig)` here ? defer: # defer is the simplest in this case if isImportedType(t) and not m.typeABICache.containsOrIncl(sig): addAbiCheck(m, t, result) result = getTypePre(m, t, sig) - if result != nil and t.kind != tyOpenArray: + if result != "" and t.kind != tyOpenArray: excl(check, t.id) + if kind == dkRefParam or kind == dkRefGenericParam and origTyp.kind == tyGenericInst: + result.add("&") return case t.kind of tyRef, tyPtr, tyVar, tyLent: var star = if t.kind in {tyVar} and tfVarIsPtr notin origTyp.flags and compileToCpp(m): "&" else: "*" - var et = origTyp.skipTypes(abstractInst).lastSon + var et = origTyp.skipTypes(abstractInst).elementType var etB = et.skipTypes(abstractInst) - if mapType(m.config, t, kind) == ctPtrToArray and (etB.kind != tyOpenArray or kind == skParam): + if mapType(m.config, t, kind == dkParam) == ctPtrToArray and (etB.kind != tyOpenArray or kind == dkParam): if etB.kind == tySet: et = getSysType(m.g.graph, unknownLineInfo, tyUInt8) else: @@ -740,7 +959,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result = getTypeDescAux(m, et, check, kind) & star else: # no restriction! We have a forward declaration for structs - let name = getTypeForward(m, et, hashType et) + let name = getTypeForward(m, et, hashType(et, m.config)) result = name & star m.typeCache[sig] = result of tySequence: @@ -749,7 +968,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin m.typeCache[sig] = result else: # no restriction! We have a forward declaration for structs - let name = getTypeForward(m, et, hashType et) + let name = getTypeForward(m, et, hashType(et, m.config)) result = name & seqStar(m) & star m.typeCache[sig] = result pushType(m, et) @@ -761,7 +980,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result = getOpenArrayDesc(m, t, check, kind) of tyEnum: result = cacheGetType(m.typeCache, sig) - if result == nil: + if result == "": result = getTypeName(m, origTyp, sig) if not (isImportedCppType(t) or (sfImportc in t.sym.flags and t.sym.magic == mNone)): @@ -791,7 +1010,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin of tyProc: result = getTypeName(m, origTyp, sig) m.typeCache[sig] = result - var rettype, desc: Rope + var rettype, desc: Rope = "" genProcParams(m, t, rettype, desc, check, true, true) if not isImportedType(t): if t.callConv != ccClosure: # procedure vars may need a closure! @@ -809,15 +1028,15 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin # we cannot use getTypeForward here because then t would be associated # with the name of the struct, not with the pointer to the struct: result = cacheGetType(m.forwTypeCache, sig) - if result == nil: + if result == "": result = getTypeName(m, origTyp, sig) if not isImportedType(t): addForwardStructFormat(m, structOrUnion(t), result) m.forwTypeCache[sig] = result - assert(cacheGetType(m.typeCache, sig) == nil) + assert(cacheGetType(m.typeCache, sig) == "") m.typeCache[sig] = result & seqStar(m) if not isImportedType(t): - if skipTypes(t[0], typedescInst).kind != tyEmpty: + if skipTypes(t.elementType, typedescInst).kind != tyEmpty: const cppSeq = "struct $2 : #TGenericSeq {$n" cSeq = "struct $2 {$n" & @@ -825,11 +1044,11 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin if m.compileToCpp: appcg(m, m.s[cfsSeqTypes], cppSeq & " $1 data[SEQ_DECL_SIZE];$n" & - "};$n", [getTypeDescAux(m, t[0], check, kind), result]) + "};$n", [getTypeDescAux(m, t.elementType, check, kind), result]) else: appcg(m, m.s[cfsSeqTypes], cSeq & " $1 data[SEQ_DECL_SIZE];$n" & - "};$n", [getTypeDescAux(m, t[0], check, kind), result]) + "};$n", [getTypeDescAux(m, t.elementType, check, kind), result]) else: result = rope("TGenericSeq") result.add(seqStar(m)) @@ -837,7 +1056,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result = getTypeName(m, origTyp, sig) m.typeCache[sig] = result if not isImportedType(t): - let foo = getTypeDescAux(m, t[0], check, kind) + let foo = getTypeDescAux(m, t.elementType, check, kind) m.s[cfsTypes].addf("typedef $1 $2[1];$n", [foo, result]) of tyArray: var n: BiggestInt = toInt64(lengthOrd(m.config, t)) @@ -845,44 +1064,46 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result = getTypeName(m, origTyp, sig) m.typeCache[sig] = result if not isImportedType(t): - let foo = getTypeDescAux(m, t[1], check, kind) + let e = getTypeDescAux(m, t.elementType, check, kind) m.s[cfsTypes].addf("typedef $1 $2[$3];$n", - [foo, result, rope(n)]) + [e, result, rope(n)]) of tyObject, tyTuple: - if isImportedCppType(t) and origTyp.kind == tyGenericInst: - let cppName = getTypeName(m, t, sig) + let tt = origTyp.skipTypes({tyDistinct}) + if isImportedCppType(t) and tt.kind == tyGenericInst: + let cppNameAsRope = getTypeName(m, t, sig) + let cppName = $cppNameAsRope var i = 0 var chunkStart = 0 template addResultType(ty: untyped) = if ty == nil or ty.kind == tyVoid: - result.add(~"void") + result.add("void") elif ty.kind == tyStatic: internalAssert m.config, ty.n != nil result.add ty.n.renderTree else: result.add getTypeDescAux(m, ty, check, kind) - while i < cppName.data.len: - if cppName.data[i] == '\'': + while i < cppName.len: + if cppName[i] == '\'': var chunkEnd = i-1 - var idx, stars: int - if scanCppGenericSlot(cppName.data, i, idx, stars): - result.add cppName.data.substr(chunkStart, chunkEnd) + var idx, stars: int = 0 + if scanCppGenericSlot(cppName, i, idx, stars): + result.add cppName.substr(chunkStart, chunkEnd) chunkStart = i - let typeInSlot = resolveStarsInCppType(origTyp, idx + 1, stars) + let typeInSlot = resolveStarsInCppType(tt, idx + 1, stars) addResultType(typeInSlot) else: inc i if chunkStart != 0: - result.add cppName.data.substr(chunkStart) + result.add cppName.substr(chunkStart) else: - result = cppName & "<" - for i in 1.. 1: result.add(" COMMA ") - addResultType(origTyp[i]) + result = cppNameAsRope & "<" + for needsComma, a in tt.genericInstParams: + if needsComma: result.add(" COMMA ") + addResultType(a) result.add("> ") # always call for sideeffects: assert t.kind != tyTuple @@ -890,13 +1111,13 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin # The resulting type will include commas and these won't play well # with the C macros for defining procs such as N_NIMCALL. We must # create a typedef for the type and use it in the proc signature: - let typedefName = ~"TY" & $sig + let typedefName = "TY" & $sig m.s[cfsTypes].addf("typedef $1 $2;$n", [result, typedefName]) m.typeCache[sig] = typedefName result = typedefName else: result = cacheGetType(m.forwTypeCache, sig) - if result == nil: + if result == "": result = getTypeName(m, origTyp, sig) m.forwTypeCache[sig] = result if not isImportedType(t): @@ -912,7 +1133,9 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin discard # addAbiCheck(m, t, result) # already handled elsewhere of tySet: # Don't use the imported name as it may be scoped: 'Foo::SomeKind' - result = $t.kind & '_' & t.lastSon.typeName & $t.lastSon.hashType + result = rope("tySet_") + t.elementType.typeName(result) + result.add $t.elementType.hashType(m.config) m.typeCache[sig] = result if not isImportedType(t): let s = int(getSize(m.config, t)) @@ -922,14 +1145,15 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin [result, rope(getSize(m.config, t))]) of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink, tyOwned, tyUserTypeClass, tyUserTypeClassInst, tyInferred: - result = getTypeDescAux(m, lastSon(t), check, kind) + result = getTypeDescAux(m, skipModifier(t), check, kind) else: internalError(m.config, "getTypeDescAux(" & $t.kind & ')') - result = nil + result = "" # fixes bug #145: excl(check, t.id) -proc getTypeDesc(m: BModule, typ: PType; kind = skParam): Rope = + +proc getTypeDesc(m: BModule; typ: PType; kind = dkParam): Rope = var check = initIntSet() result = getTypeDescAux(m, typ, check, kind) @@ -939,11 +1163,11 @@ type clHalfWithEnv, ## fn(args, void* env) type with trailing 'void* env' parameter clFull ## struct {fn(args, void* env), env} -proc getClosureType(m: BModule, t: PType, kind: TClosureTypeKind): Rope = +proc getClosureType(m: BModule; t: PType, kind: TClosureTypeKind): Rope = assert t.kind == tyProc var check = initIntSet() result = getTempName(m) - var rettype, desc: Rope + var rettype, desc: Rope = "" genProcParams(m, t, rettype, desc, check, declareEnvironment=kind != clHalf) if not isImportedType(t): if t.callConv != ccClosure or kind != clFull: @@ -963,59 +1187,128 @@ proc finishTypeDescriptions(m: BModule) = if optSeqDestructors in m.config.globalOptions and t.skipTypes(abstractInst).kind == tySequence: seqV2ContentType(m, t, check) else: - discard getTypeDescAux(m, t, check, skParam) + discard getTypeDescAux(m, t, check, dkParam) inc(i) m.typeStack.setLen 0 -template cgDeclFrmt*(s: PSym): string = - s.constraint.strVal - -proc isReloadable(m: BModule, prc: PSym): bool = +proc isReloadable(m: BModule; prc: PSym): bool = return m.hcrOn and sfNonReloadable notin prc.flags -proc isNonReloadable(m: BModule, prc: PSym): bool = +proc isNonReloadable(m: BModule; prc: PSym): bool = return m.hcrOn and sfNonReloadable in prc.flags -proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope = - var - rettype, params: Rope - # using static is needed for inline procs - if lfExportLib in prc.loc.flags: - if isHeaderFile in m.flags: - result.add "N_LIB_IMPORT " +proc parseVFunctionDecl(val: string; name, params, retType, superCall: var string; isFnConst, isOverride, isMemberVirtual, isStatic: var bool; isCtor: bool, isFunctor=false) = + var afterParams: string = "" + if scanf(val, "$*($*)$s$*", name, params, afterParams): + if name.strip() == "operator" and params == "": #isFunctor? + parseVFunctionDecl(afterParams, name, params, retType, superCall, isFnConst, isOverride, isMemberVirtual, isStatic, isCtor, true) + return + if name.find("static ") > -1: + isStatic = true + name = name.replace("static ", "") + isFnConst = afterParams.find("const") > -1 + isOverride = afterParams.find("override") > -1 + isMemberVirtual = name.find("virtual ") > -1 + if isMemberVirtual: + name = name.replace("virtual ", "") + if isFunctor: + name = "operator ()" + if isCtor: + discard scanf(afterParams, ":$s$*", superCall) else: - result.add "N_LIB_EXPORT " - elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc): - result.add "static " - elif sfImportc notin prc.flags: - result.add "N_LIB_PRIVATE " + discard scanf(afterParams, "->$s$* ", retType) + + params = "(" & params & ")" + +proc genMemberProcHeader(m: BModule; prc: PSym; result: var Rope; asPtr: bool = false, isFwdDecl: bool = false) = + assert sfCppMember * prc.flags != {} + let isCtor = sfConstructor in prc.flags var check = initIntSet() - fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown) - genProcParams(m, prc.typ, rettype, params, check) + fillBackendName(m, prc) + fillLoc(prc.loc, locProc, prc.ast[namePos], OnUnknown) + var memberOp = "#." #only virtual + var typ: PType + if isCtor: + typ = prc.typ.returnType + else: + typ = prc.typ.firstParamType + if typ.kind == tyPtr: + typ = typ.elementType + memberOp = "#->" + var typDesc = getTypeDescWeak(m, typ, check, dkParam) + let asPtrStr = rope(if asPtr: "_PTR" else: "") + var name, params, rettype, superCall: string = "" + var isFnConst, isOverride, isMemberVirtual, isStatic: bool = false + parseVFunctionDecl(prc.constraint.strVal, name, params, rettype, superCall, isFnConst, isOverride, isMemberVirtual, isStatic, isCtor) + genMemberProcParams(m, prc, superCall, rettype, name, params, check, true, false) + let isVirtual = sfVirtual in prc.flags or isMemberVirtual + var fnConst, override: string = "" + if isCtor: + name = typDesc + if isFnConst: + fnConst = " const" + if isFwdDecl: + if isStatic: + result.add "static " + if isVirtual: + rettype = "virtual " & rettype + if isOverride: + override = " override" + superCall = "" + else: + if not isCtor: + prc.loc.r = "$1$2(@)" % [memberOp, name] + elif superCall != "": + superCall = " : " & superCall + + name = "$1::$2" % [typDesc, name] + + result.add "N_LIB_PRIVATE " + result.addf("$1$2($3, $4)$5$6$7$8", + [rope(CallingConvToStr[prc.typ.callConv]), asPtrStr, rettype, name, + params, fnConst, override, superCall]) + +proc genProcHeader(m: BModule; prc: PSym; result: var Rope; asPtr: bool = false) = + # using static is needed for inline procs + var check = initIntSet() + fillBackendName(m, prc) + fillLoc(prc.loc, locProc, prc.ast[namePos], OnUnknown) + var rettype, params: Rope = "" + genProcParams(m, prc.typ, rettype, params, check, true, false) # handle the 2 options for hotcodereloading codegen - function pointer # (instead of forward declaration) or header for function body with "_actual" postfix let asPtrStr = rope(if asPtr: "_PTR" else: "") var name = prc.loc.r - if isReloadable(m, prc) and not asPtr: + if not asPtr and isReloadable(m, prc): name.add("_actual") # careful here! don't access ``prc.ast`` as that could reload large parts of # the object graph! - if prc.constraint.isNil: + if sfCodegenDecl notin prc.flags: + if lfExportLib in prc.loc.flags: + if isHeaderFile in m.flags: + result.add "N_LIB_IMPORT " + else: + result.add "N_LIB_EXPORT " + elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc): + result.add "static " + elif sfImportc notin prc.flags: + result.add "N_LIB_PRIVATE " result.addf("$1$2($3, $4)$5", [rope(CallingConvToStr[prc.typ.callConv]), asPtrStr, rettype, name, params]) else: let asPtrStr = if asPtr: (rope("(*") & name & ")") else: name - result = runtimeFormat(prc.cgDeclFrmt, [rettype, asPtrStr, params]) + result.add runtimeFormat(prc.cgDeclFrmt, [rettype, asPtrStr, params]) + # ------------------ type info generation ------------------------------------- -proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope +proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope proc getNimNode(m: BModule): Rope = result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)] inc(m.typeNodes) -proc tiNameForHcr(m: BModule, name: Rope): Rope = +proc tiNameForHcr(m: BModule; name: Rope): Rope = return if m.hcrOn: "(*".rope & name & ")" else: name proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; @@ -1033,7 +1326,7 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; if tfIncompleteStruct in typ.flags: size = rope"void*" else: - size = getTypeDesc(m, origType, skVar) + size = getTypeDesc(m, origType, dkVar) m.s[cfsTypeInit3].addf( "$1.size = sizeof($2);$n$1.align = NIM_ALIGNOF($2);$n$1.kind = $3;$n$1.base = $4;$n", [nameHcr, size, rope(nimtypeKind), base] @@ -1041,11 +1334,11 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; # compute type flags for GC optimization var flags = 0 if not containsGarbageCollectedRef(typ): flags = flags or 1 - if not canFormAcycle(typ): flags = flags or 2 - #else MessageOut("can contain a cycle: " & typeToString(typ)) + if not canFormAcycle(m.g.graph, typ): flags = flags or 2 + #else echo("can contain a cycle: " & typeToString(typ)) if flags != 0: m.s[cfsTypeInit3].addf("$1.flags = $2;$n", [nameHcr, rope(flags)]) - discard cgsym(m, "TNimType") + cgsym(m, "TNimType") if isDefined(m.config, "nimTypeNames"): var typename = typeToString(if origType.typeInst != nil: origType.typeInst else: origType, preferName) @@ -1053,22 +1346,22 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; typename = "anon ref object from " & m.config$origType.skipTypes(skipPtrs).sym.info m.s[cfsTypeInit3].addf("$1.name = $2;$n", [nameHcr, makeCString typename]) - discard cgsym(m, "nimTypeRoot") + cgsym(m, "nimTypeRoot") m.s[cfsTypeInit3].addf("$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n", [nameHcr]) if m.hcrOn: - m.s[cfsData].addf("static TNimType* $1;$n", [name]) + m.s[cfsStrData].addf("static TNimType* $1;$n", [name]) m.hcrCreateTypeInfosProc.addf("\thcrRegisterGlobal($2, \"$1\", sizeof(TNimType), NULL, (void**)&$1);$n", [name, getModuleDllPath(m, m.module)]) else: - m.s[cfsData].addf("N_LIB_PRIVATE TNimType $1;$n", [name]) + m.s[cfsStrData].addf("N_LIB_PRIVATE TNimType $1;$n", [name]) -proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope; +proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo) = var base: Rope - if typ.len > 0 and typ.lastSon != nil: - var x = typ.lastSon + if typ.hasElementType and typ.last != nil: + var x = typ.last if typ.kind == tyObject: x = x.skipTypes(skipPtrs) if typ.kind == tyPtr and x.kind == tyObject and incompleteType(x): base = rope("0") @@ -1078,23 +1371,23 @@ proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope; base = rope("0") genTypeInfoAuxBase(m, typ, origType, name, base, info) -proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope = +proc discriminatorTableName(m: BModule; objtype: PType, d: PSym): Rope = # bugfix: we need to search the type that contains the discriminator: var objtype = objtype.skipTypes(abstractPtrs) while lookupInRecord(objtype.n, d.name) == nil: objtype = objtype[0].skipTypes(abstractPtrs) if objtype.sym == nil: internalError(m.config, d.info, "anonymous obj with discriminator") - result = "NimDT_$1_$2" % [rope($hashType(objtype)), rope(d.name.s.mangle)] + result = "NimDT_$1_$2" % [rope($hashType(objtype, m.config)), rope(d.name.s.mangle)] proc rope(arg: Int128): Rope = rope($arg) -proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope = - discard cgsym(m, "TNimNode") +proc discriminatorTableDecl(m: BModule; objtype: PType, d: PSym): Rope = + cgsym(m, "TNimNode") var tmp = discriminatorTableName(m, objtype, d) result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(m.config, d.typ)+1)] -proc genTNimNodeArray(m: BModule, name: Rope, size: Rope) = +proc genTNimNodeArray(m: BModule; name: Rope, size: Rope) = if m.hcrOn: m.s[cfsData].addf("static TNimNode** $1;$n", [name]) m.hcrCreateTypeInfosProc.addf("\thcrRegisterGlobal($3, \"$1\", sizeof(TNimNode*) * $2, NULL, (void**)&$1);$n", @@ -1102,7 +1395,7 @@ proc genTNimNodeArray(m: BModule, name: Rope, size: Rope) = else: m.s[cfsTypeInit1].addf("static TNimNode* $1[$2];$n", [name, size]) -proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; +proc genObjectFields(m: BModule; typ, origType: PType, n: PNode, expr: Rope; info: TLineInfo) = case n.kind of nkRecList: @@ -1125,13 +1418,13 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; var tmp = discriminatorTableName(m, typ, field) var L = lengthOrd(m.config, field.typ) assert L > 0 - if field.loc.r == nil: fillObjectFields(m, typ) + if field.loc.r == "": fillObjectFields(m, typ) if field.loc.t == nil: internalError(m.config, n.info, "genObjectFields") m.s[cfsTypeInit3].addf("$1.kind = 3;$n" & "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" & "$1.name = $5;$n" & "$1.sons = &$6[0];$n" & - "$1.len = $7;$n", [expr, getTypeDesc(m, origType, skVar), field.loc.r, + "$1.len = $7;$n", [expr, getTypeDesc(m, origType, dkVar), field.loc.r, genTypeInfoV1(m, field.typ, info), makeCString(field.name.s), tmp, rope(L)]) @@ -1163,56 +1456,53 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; # Do not produce code for void types if isEmptyType(field.typ): return if field.bitsize == 0: - if field.loc.r == nil: fillObjectFields(m, typ) + if field.loc.r == "": fillObjectFields(m, typ) if field.loc.t == nil: internalError(m.config, n.info, "genObjectFields") m.s[cfsTypeInit3].addf("$1.kind = 1;$n" & "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" & - "$1.name = $5;$n", [expr, getTypeDesc(m, origType, skVar), + "$1.name = $5;$n", [expr, getTypeDesc(m, origType, dkVar), field.loc.r, genTypeInfoV1(m, field.typ, info), makeCString(field.name.s)]) else: internalError(m.config, n.info, "genObjectFields") -proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) = - if typ.kind == tyObject: - if incompleteType(typ): - localError(m.config, info, "request for RTTI generation for incomplete object: " & - typeToString(typ)) - genTypeInfoAux(m, typ, origType, name, info) - else: - genTypeInfoAuxBase(m, typ, origType, name, rope("0"), info) +proc genObjectInfo(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo) = + assert typ.kind == tyObject + if incompleteType(typ): + localError(m.config, info, "request for RTTI generation for incomplete object: " & + typeToString(typ)) + genTypeInfoAux(m, typ, origType, name, info) var tmp = getNimNode(m) if not isImportedType(typ): genObjectFields(m, typ, origType, typ.n, tmp, info) m.s[cfsTypeInit3].addf("$1.node = &$2;$n", [tiNameForHcr(m, name), tmp]) - var t = typ[0] + var t = typ.baseClass while t != nil: t = t.skipTypes(skipPtrs) t.flags.incl tfObjHasKids - t = t[0] + t = t.baseClass -proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) = +proc genTupleInfo(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo) = genTypeInfoAuxBase(m, typ, typ, name, rope("0"), info) var expr = getNimNode(m) - if typ.len > 0: - var tmp = getTempName(m) & "_" & $typ.len - genTNimNodeArray(m, tmp, rope(typ.len)) - for i in 0.. 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions: + for i in countdown(depth, 1): + result.add seqs[i] & ", " + result.add seqs[0] + result.add "}" + +proc genVTable(seqs: seq[PSym]): string = + result = "{" + for i in 0.. 0: result.add ", " + result.add "(void *) " & seqs[i].loc.r + result.add "}" + +proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) = + cgsym(m, "TNimTypeV2") + m.s[cfsStrData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name]) + + var flags = 0 + if not canFormAcycle(m.g.graph, t): flags = flags or 1 + + var typeEntry = newRopeAppender() + addf(typeEntry, "$1.destructor = (void*)", [name]) + genHook(m, t, info, attachedDestructor, typeEntry) + + addf(typeEntry, "; $1.traceImpl = (void*)", [name]) + genHook(m, t, info, attachedTrace, typeEntry) + + let objDepth = if t.kind == tyObject: getObjDepth(t) else: -1 + + if t.kind in {tyObject, tyDistinct} and incompleteType(t): + localError(m.config, info, "request for RTTI generation for incomplete object: " & + typeToString(t)) + + if isDefined(m.config, "nimTypeNames"): + var typeName: Rope + if t.kind in {tyObject, tyDistinct}: + typeName = genTypeInfo2Name(m, t) + else: + typeName = rope("NIM_NIL") + addf(typeEntry, "; $1.name = $2", [name, typeName]) + addf(typeEntry, "; $1.size = sizeof($2); $1.align = (NI16) NIM_ALIGNOF($2); $1.depth = $3; $1.flags = $4;", + [name, getTypeDesc(m, t), rope(objDepth), rope(flags)]) + + if objDepth >= 0: + let objDisplay = genDisplay(m, t, objDepth) + let objDisplayStore = getTempName(m) + m.s[cfsVars].addf("static $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), dkVar), objDisplayStore, rope(objDepth+1), objDisplay]) + addf(typeEntry, "$1.display = $2;$n", [name, rope(objDisplayStore)]) + + let dispatchMethods = toSeq(getMethodsPerType(m.g.graph, t)) + if dispatchMethods.len > 0: + let vTablePointerName = getTempName(m) + m.s[cfsVars].addf("static void* $1[$2] = $3;$n", [vTablePointerName, rope(dispatchMethods.len), genVTable(dispatchMethods)]) + for i in dispatchMethods: + genProcPrototype(m, i) + addf(typeEntry, "$1.vTable = $2;$n", [name, vTablePointerName]) + + m.s[cfsTypeInit3].add typeEntry + + if t.kind == tyObject and t.baseClass != nil and optEnableDeepCopy in m.config.globalOptions: discard genTypeInfoV1(m, t, info) -proc moduleOpenForCodegen(m: BModule; module: int32): bool {.inline.} = - result = module < m.g.modules.len and m.g.modules[module] != nil +proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) = + cgsym(m, "TNimTypeV2") + m.s[cfsStrData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name]) -proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = + var flags = 0 + if not canFormAcycle(m.g.graph, t): flags = flags or 1 + + var typeEntry = newRopeAppender() + addf(typeEntry, "N_LIB_PRIVATE TNimTypeV2 $1 = {", [name]) + add(typeEntry, ".destructor = (void*)") + genHook(m, t, info, attachedDestructor, typeEntry) + + let objDepth = if t.kind == tyObject: getObjDepth(t) else: -1 + + if t.kind in {tyObject, tyDistinct} and incompleteType(t): + localError(m.config, info, "request for RTTI generation for incomplete object: " & + typeToString(t)) + + addf(typeEntry, ", .size = sizeof($1), .align = (NI16) NIM_ALIGNOF($1), .depth = $2", + [getTypeDesc(m, t), rope(objDepth)]) + + if objDepth >= 0: + let objDisplay = genDisplay(m, t, objDepth) + let objDisplayStore = getTempName(m) + m.s[cfsVars].addf("static NIM_CONST $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), dkVar), objDisplayStore, rope(objDepth+1), objDisplay]) + addf(typeEntry, ", .display = $1", [rope(objDisplayStore)]) + if isDefined(m.config, "nimTypeNames"): + var typeName: Rope + if t.kind in {tyObject, tyDistinct}: + typeName = genTypeInfo2Name(m, t) + else: + typeName = rope("NIM_NIL") + addf(typeEntry, ", .name = $1", [typeName]) + add(typeEntry, ", .traceImpl = (void*)") + genHook(m, t, info, attachedTrace, typeEntry) + + let dispatchMethods = toSeq(getMethodsPerType(m.g.graph, t)) + if dispatchMethods.len > 0: + addf(typeEntry, ", .flags = $1", [rope(flags)]) + for i in dispatchMethods: + genProcPrototype(m, i) + addf(typeEntry, ", .vTable = $1};$n", [genVTable(dispatchMethods)]) + m.s[cfsVars].add typeEntry + else: + addf(typeEntry, ", .flags = $1};$n", [rope(flags)]) + m.s[cfsVars].add typeEntry + + if t.kind == tyObject and t.baseClass != nil and optEnableDeepCopy in m.config.globalOptions: + discard genTypeInfoV1(m, t, info) + +proc genTypeInfoV2(m: BModule; t: PType; info: TLineInfo): Rope = let origType = t # distinct types can have their own destructors var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses - {tyDistinct}) let prefixTI = if m.hcrOn: "(" else: "(&" - let sig = hashType(origType) + let sig = hashType(origType, m.config) result = m.typeInfoMarkerV2.getOrDefault(sig) - if result != nil: + if result != "": return prefixTI.rope & result & ")".rope let marker = m.g.typeInfoMarkerV2.getOrDefault(sig) - if marker.str != nil: - discard cgsym(m, "TNimTypeV2") + if marker.str != "": + cgsym(m, "TNimTypeV2") declareNimType(m, "TNimTypeV2", marker.str, marker.owner) # also store in local type section: m.typeInfoMarkerV2[sig] = marker.str @@ -1384,24 +1832,26 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = m.typeInfoMarkerV2[sig] = result let owner = t.skipTypes(typedescPtrs).itemId.module - if owner != m.module.position and moduleOpenForCodegen(m, owner): + if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner): # make sure the type info is created in the owner module - assert m.g.modules[owner] != nil discard genTypeInfoV2(m.g.modules[owner], origType, info) # reference the type info as extern here - discard cgsym(m, "TNimTypeV2") + cgsym(m, "TNimTypeV2") declareNimType(m, "TNimTypeV2", result, owner) return prefixTI.rope & result & ")".rope m.g.typeInfoMarkerV2[sig] = (str: result, owner: owner) - genTypeInfoV2Impl(m, t, origType, result, info) + if m.compileToCpp or m.hcrOn: + genTypeInfoV2OldImpl(m, t, origType, result, info) + else: + genTypeInfoV2Impl(m, t, origType, result, info) result = prefixTI.rope & result & ")".rope proc openArrayToTuple(m: BModule; t: PType): PType = - result = newType(tyTuple, nextTypeId m.idgen, t.owner) - let p = newType(tyPtr, nextTypeId m.idgen, t.owner) - let a = newType(tyUncheckedArray, nextTypeId m.idgen, t.owner) - a.add t.lastSon + result = newType(tyTuple, m.idgen, t.owner) + let p = newType(tyPtr, m.idgen, t.owner) + let a = newType(tyUncheckedArray, m.idgen, t.owner) + a.add t.elementType p.add a result.add p result.add getSysType(m.g.graph, t.owner.info, tyInt) @@ -1411,8 +1861,7 @@ proc typeToC(t: PType): string = ## to be unique. let s = typeToString(t) result = newStringOfCap(s.len) - for i in 0.. 0 and i < name.len-1 and name[i+1] in Digits: - discard - else: - result.add(c) - of '$': special "dollar" - of '%': special "percent" - of '&': special "amp" - of '^': special "roof" - of '!': special "emark" - of '?': special "qmark" - of '*': special "star" - of '+': special "plus" - of '-': special "minus" - of '/': special "slash" - of '\\': special "backslash" - of '=': special "eq" - of '<': special "lt" - of '>': special "gt" - of '~': special "tilde" - of ':': special "colon" - of '.': special "dot" - of '@': special "at" - of '|': special "bar" +proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind = + case int(getSize(conf, typ)) + of 1: result = ctInt8 + of 2: result = ctInt16 + of 4: result = ctInt32 + of 8: result = ctInt64 + else: result = ctArray + +proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool = + var pt = skipTypes(s.typ, typedescInst) + assert skResult != s.kind + + #note precedence: params override types + if optByRef in s.options: return true + elif sfByCopy in s.flags: return false + elif tfByRef in pt.flags: return true + elif tfByCopy in pt.flags: return false + case pt.kind + of tyObject: + if s.typ.sym != nil and sfForward in s.typ.sym.flags: + # forwarded objects are *always* passed by pointers for consistency! + result = true + elif s.typ.kind == tySink and conf.selectedGC notin {gcArc, gcAtomicArc, gcOrc, gcHooks}: + # bug #23354: + result = false + elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3): + result = true # requested anyway + elif (tfFinal in pt.flags) and (pt[0] == nil): + result = false # no need, because no subtyping possible else: - result.add("X" & toHex(ord(c), 2)) - requiresUnderscore = true - if requiresUnderscore: - result.add "_" + result = true # ordinary objects are always passed by reference, + # otherwise casting doesn't work + of tyTuple: + result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options) + else: + result = false + # first parameter and return type is 'lent T'? --> use pass by pointer + if s.position == 0 and retType != nil and retType.kind == tyLent: + result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or + pt.kind == tySet and mapSetType(conf, pt) == ctArray) + +proc encodeName*(name: string): string = + result = mangle(name) + result = $result.len & result + +proc makeUnique(m: BModule; s: PSym, name: string = ""): string = + result = if name == "": s.name.s else: name + result.add "__" + result.add m.g.graph.ifaces[s.itemId.module].uniqueName + result.add "_u" + result.add $s.itemId.item + +proc encodeSym*(m: BModule; s: PSym; makeUnique: bool = false): string = + #Module::Type + var name = s.name.s + if makeUnique: + name = makeUnique(m, s, name) + "N" & encodeName(s.skipGenericOwner.name.s) & encodeName(name) & "E" + +proc encodeType*(m: BModule; t: PType): string = + result = "" + var kindName = ($t.kind)[2..^1] + kindName[0] = toLower($kindName[0])[0] + case t.kind + of tyObject, tyEnum, tyDistinct, tyUserTypeClass, tyGenericParam: + result = encodeSym(m, t.sym) + of tyGenericInst, tyUserTypeClassInst, tyGenericBody: + result = encodeName(t[0].sym.name.s) + result.add "I" + for i in 1..= start: - strLit.add(substr(frmt, start, i - 1)) + result.add newCall(formatValue, resVar, newCall(ident"cgsymValue", m, ident)) + elif frmt[i] == '#' and frmt[i+1] == '#': + inc(i, 2) + strLit.add("#") + else: + strLit.add(frmt[i]) + inc(i) flushStrLit() result.add newCall(ident"rope", resVar) -proc indentLine(p: BProc, r: Rope): Rope = - result = r - for i in 0..= 0 if optLineDir in conf.options and line > 0: - r.addf("$N#line $2 $1$N", - [rope(makeSingleLineCString(filename)), rope(line)]) + if fileIdx == InvalidFileIdx: + r.add(rope("\n#line " & $line & " \"generated_not_to_break_here\"\n")) + else: + r.add(rope("\n#line " & $line & " FX_" & $fileIdx.int32 & "\n")) + +proc genCLineDir(r: var Rope, fileIdx: FileIndex, line: int; p: BProc; info: TLineInfo; lastFileIndex: FileIndex) = + assert line >= 0 + if optLineDir in p.config.options and line > 0: + if fileIdx == InvalidFileIdx: + r.add(rope("\n#line " & $line & " \"generated_not_to_break_here\"\n")) + else: + r.add(rope("\n#line " & $line & " FX_" & $fileIdx.int32 & "\n")) proc genCLineDir(r: var Rope, info: TLineInfo; conf: ConfigRef) = - genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf) + if optLineDir in conf.options: + genCLineDir(r, info.fileIndex, info.safeLineNm, conf) proc freshLineInfo(p: BProc; info: TLineInfo): bool = if p.lastLineInfo.line != info.line or @@ -260,26 +302,34 @@ proc freshLineInfo(p: BProc; info: TLineInfo): bool = p.lastLineInfo.line = info.line p.lastLineInfo.fileIndex = info.fileIndex result = true + else: + result = false + +proc genCLineDir(r: var Rope, p: BProc, info: TLineInfo; conf: ConfigRef) = + if optLineDir in conf.options: + let lastFileIndex = p.lastLineInfo.fileIndex + if freshLineInfo(p, info): + genCLineDir(r, info.fileIndex, info.safeLineNm, p, info, lastFileIndex) proc genLineDir(p: BProc, t: PNode) = + if p == p.module.preInitProc: return let line = t.info.safeLineNm if optEmbedOrigSrc in p.config.globalOptions: - p.s(cpsStmts).add(~"//" & sourceLine(p.config, t.info) & "\L") - genCLineDir(p.s(cpsStmts), toFullPath(p.config, t.info), line, p.config) + p.s(cpsStmts).add("//" & sourceLine(p.config, t.info) & "\L") + let lastFileIndex = p.lastLineInfo.fileIndex + let freshLine = freshLineInfo(p, t.info) + if freshLine: + genCLineDir(p.s(cpsStmts), t.info.fileIndex, line, p, t.info, lastFileIndex) if ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and (p.prc == nil or sfPure notin p.prc.flags) and t.info.fileIndex != InvalidFileIdx: - if freshLineInfo(p, t.info): - linefmt(p, cpsStmts, "nimln_($1, $2);$n", - [line, quotedFilename(p.config, t.info)]) - -proc postStmtActions(p: BProc) {.inline.} = - p.s(cpsStmts).add(p.module.injectStmt) + if freshLine: + line(p, cpsStmts, genPostprocessDir("nimln", $line, $t.info.fileIndex.int32)) proc accessThreadLocalVar(p: BProc, s: PSym) proc emulatedThreadVars(conf: ConfigRef): bool {.inline.} proc genProc(m: BModule, prc: PSym) -proc raiseInstr(p: BProc): Rope +proc raiseInstr(p: BProc; result: var Rope) template compileToCpp(m: BModule): untyped = m.config.backend == backendCpp or sfCompileToCpp in m.module.flags @@ -290,10 +340,18 @@ proc getTempName(m: BModule): Rope = proc rdLoc(a: TLoc): Rope = # 'read' location (deref if indirect) - result = a.r - if lfIndirect in a.flags: result = "(*$1)" % [result] + if lfIndirect in a.flags: + result = "(*" & a.r & ")" + else: + result = a.r -proc lenField(p: BProc): Rope = +proc addRdLoc(a: TLoc; result: var Rope) = + if lfIndirect in a.flags: + result.add "(*" & a.r & ")" + else: + result.add a.r + +proc lenField(p: BProc): Rope {.inline.} = result = rope(if p.module.compileToCpp: "len" else: "Sup.len") proc lenExpr(p: BProc; a: TLoc): Rope = @@ -302,12 +360,20 @@ proc lenExpr(p: BProc; a: TLoc): Rope = else: result = "($1 ? $1->$2 : 0)" % [rdLoc(a), lenField(p)] +proc dataFieldAccessor(p: BProc, sym: Rope): Rope = + if optSeqDestructors in p.config.globalOptions: + result = "(" & sym & ").p" + else: + result = sym + proc dataField(p: BProc): Rope = if optSeqDestructors in p.config.globalOptions: result = rope".p->data" else: result = rope"->data" +proc genProcPrototype(m: BModule, sym: PSym) + include ccgliterals include ccgtypes @@ -318,16 +384,24 @@ template mapTypeChooser(n: PNode): TSymKind = template mapTypeChooser(a: TLoc): TSymKind = mapTypeChooser(a.lode) +proc addAddrLoc(conf: ConfigRef; a: TLoc; result: var Rope) = + if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a) == skParam) != ctArray: + result.add "(&" & a.r & ")" + else: + result.add a.r + proc addrLoc(conf: ConfigRef; a: TLoc): Rope = - result = a.r - if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a)) != ctArray: - result = "(&" & result & ")" + if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a) == skParam) != ctArray: + result = "(&" & a.r & ")" + else: + result = a.r proc byRefLoc(p: BProc; a: TLoc): Rope = - result = a.r - if lfIndirect notin a.flags and mapType(p.config, a.t, mapTypeChooser(a)) != ctArray and not + if lfIndirect notin a.flags and mapType(p.config, a.t, mapTypeChooser(a) == skParam) != ctArray and not p.module.compileToCpp: - result = "(&" & result & ")" + result = "(&" & a.r & ")" + else: + result = a.r proc rdCharLoc(a: TLoc): Rope = # read a location that may need a char-cast: @@ -338,6 +412,7 @@ proc rdCharLoc(a: TLoc): Rope = type TAssignmentFlag = enum needToCopy + needTempForOpenArray TAssignmentFlags = set[TAssignmentFlag] proc genObjConstr(p: BProc, e: PNode, d: var TLoc) @@ -369,13 +444,13 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc, linefmt(p, section, "$1.m_type = $2;$n", [r, genTypeInfoV1(p.module, t, a.lode.info)]) of frEmbedded: if optTinyRtti in p.config.globalOptions: - var tmp: TLoc + var tmp: TLoc = default(TLoc) if mode == constructRefObj: let objType = t.skipTypes(abstractInst+{tyRef}) rawConstExpr(p, newNodeIT(nkType, a.lode.info, objType), tmp) linefmt(p, cpsStmts, "#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n", - [rdLoc(a), rdLoc(tmp), getTypeDesc(p.module, objType, mapTypeChooser(a))]) + [rdLoc(a), rdLoc(tmp), getTypeDesc(p.module, objType, descKindFromSymKind mapTypeChooser(a))]) else: rawConstExpr(p, newNodeIT(nkType, a.lode.info, t), tmp) genAssignment(p, a, tmp, {}) @@ -408,7 +483,7 @@ proc resetLoc(p: BProc, loc: var TLoc) = let typ = skipTypes(loc.t, abstractVarRange) if isImportedCppType(typ): return if optSeqDestructors in p.config.globalOptions and typ.kind in {tyString, tySequence}: - assert rdLoc(loc) != nil + assert loc.r != "" let atyp = skipTypes(loc.t, abstractInst) if atyp.kind in {tyVar, tyLent}: @@ -417,8 +492,7 @@ proc resetLoc(p: BProc, loc: var TLoc) = linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)]) elif not isComplexValueType(typ): if containsGcRef: - var nilLoc: TLoc - initLoc(nilLoc, locTemp, loc.lode, OnStack) + var nilLoc: TLoc = initLoc(locTemp, loc.lode, OnStack) nilLoc.r = rope("NIM_NIL") genRefAssign(p, loc, nilLoc) else: @@ -435,9 +509,17 @@ proc resetLoc(p: BProc, loc: var TLoc) = else: # array passed as argument decayed into pointer, bug #7332 # so we use getTypeDesc here rather than rdLoc(loc) - linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n", - [addrLoc(p.config, loc), - getTypeDesc(p.module, loc.t, mapTypeChooser(loc))]) + let tyDesc = getTypeDesc(p.module, loc.t, descKindFromSymKind mapTypeChooser(loc)) + if p.module.compileToCpp and isOrHasImportedCppType(typ): + if lfIndirect in loc.flags: + #C++ cant be just zeroed. We need to call the ctors + var tmp = getTemp(p, loc.t) + linefmt(p, cpsStmts,"#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n", + [addrLoc(p.config, loc), addrLoc(p.config, tmp), tyDesc]) + else: + linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n", + [addrLoc(p.config, loc), tyDesc]) + # XXX: We can be extra clever here and call memset only # on the bytes following the m_type field? genObjectInit(p, cpsStmts, loc.t, loc, constructObj) @@ -447,15 +529,20 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}: linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)]) elif not isComplexValueType(typ): - linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), - getTypeDesc(p.module, typ, mapTypeChooser(loc))]) + if containsGarbageCollectedRef(loc.t): + var nilLoc: TLoc = initLoc(locTemp, loc.lode, OnStack) + nilLoc.r = rope("NIM_NIL") + genRefAssign(p, loc, nilLoc) + else: + linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), + getTypeDesc(p.module, typ, descKindFromSymKind mapTypeChooser(loc))]) else: - if not isTemp or containsGarbageCollectedRef(loc.t): + if (not isTemp or containsGarbageCollectedRef(loc.t)) and not hasNoInit(loc.t): # don't use nimZeroMem for temporary values for performance if we can # avoid it: - if not isImportedCppType(typ): + if not isOrHasImportedCppType(typ): linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n", - [addrLoc(p.config, loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))]) + [addrLoc(p.config, loc), getTypeDesc(p.module, typ, descKindFromSymKind mapTypeChooser(loc))]) genObjectInit(p, cpsStmts, loc.t, loc, constructObj) proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) = @@ -470,14 +557,16 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) = if not immediateAsgn: constructLoc(p, v.loc) -proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = +proc getTemp(p: BProc, t: PType, needsInit=false): TLoc = inc(p.labels) - result.r = "T" & rope(p.labels) & "_" - linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, skVar), result.r]) - result.k = locTemp - result.lode = lodeTyp t - result.storage = OnStack - result.flags = {} + result = TLoc(r: "T" & rope(p.labels) & "_", k: locTemp, lode: lodeTyp t, + storage: OnStack, flags: {}) + if p.module.compileToCpp and isOrHasImportedCppType(t): + var didGenTemp = false + linefmt(p, cpsLocals, "$1 $2$3;$n", [getTypeDesc(p.module, t, dkVar), result.r, + genCppInitializer(p.module, p, t, didGenTemp)]) + else: + linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, dkVar), result.r]) constructLoc(p, result, not needsInit) when false: # XXX Introduce a compiler switch in order to detect these easily. @@ -488,33 +577,33 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = echo "ENORMOUS TEMPORARY! ", p.config $ p.lastLineInfo writeStackTrace() -proc getTempCpp(p: BProc, t: PType, result: var TLoc; value: Rope) = +proc getTempCpp(p: BProc, t: PType, value: Rope): TLoc = inc(p.labels) - result.r = "T" & rope(p.labels) & "_" - linefmt(p, cpsStmts, "$1 $2 = $3;$n", [getTypeDesc(p.module, t, skVar), result.r, value]) - result.k = locTemp - result.lode = lodeTyp t - result.storage = OnStack - result.flags = {} + result = TLoc(r: "T" & rope(p.labels) & "_", k: locTemp, lode: lodeTyp t, + storage: OnStack, flags: {}) + linefmt(p, cpsStmts, "auto $1 = $2;$n", [result.r, value]) -proc getIntTemp(p: BProc, result: var TLoc) = +proc getIntTemp(p: BProc): TLoc = inc(p.labels) - result.r = "T" & rope(p.labels) & "_" + result = TLoc(r: "T" & rope(p.labels) & "_", k: locTemp, + storage: OnStack, lode: lodeTyp getSysType(p.module.g.graph, unknownLineInfo, tyInt), + flags: {}) linefmt(p, cpsLocals, "NI $1;$n", [result.r]) - result.k = locTemp - result.storage = OnStack - result.lode = lodeTyp getSysType(p.module.g.graph, unknownLineInfo, tyInt) - result.flags = {} proc localVarDecl(p: BProc; n: PNode): Rope = + result = "" let s = n.sym if s.loc.k == locNone: - fillLoc(s.loc, locLocalVar, n, mangleLocalName(p, s), OnStack) + fillLocalName(p, s) + fillLoc(s.loc, locLocalVar, n, OnStack) if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy) if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0: result.addf("NIM_ALIGN($1) ", [rope(s.alignment)]) - result.add getTypeDesc(p.module, s.typ, skVar) - if s.constraint.isNil: + + genCLineDir(result, p, n.info, p.config) + + result.add getTypeDesc(p.module, s.typ, dkVar) + if sfCodegenDecl notin s.flags: if sfRegister in s.flags: result.add(" register") #elif skipTypes(s.typ, abstractInst).kind in GcTypeKinds: # decl.add(" GC_GUARD") @@ -529,8 +618,12 @@ proc assignLocalVar(p: BProc, n: PNode) = #assert(s.loc.k == locNone) # not yet assigned # this need not be fulfilled for inline procs; they are regenerated # for each module that uses them! - let nl = if optLineDir in p.config.options: "" else: "\L" - let decl = localVarDecl(p, n) & ";" & nl + let nl = if optLineDir in p.config.options: "" else: "\n" + var decl = localVarDecl(p, n) + if p.module.compileToCpp and isOrHasImportedCppType(n.typ): + var didGenTemp = false + decl.add genCppInitializer(p.module, p, n.typ, didGenTemp) + decl.add ";" & nl line(p, cpsLocals, decl) include ccgthreadvars @@ -543,10 +636,34 @@ proc treatGlobalDifferentlyForHCR(m: BModule, s: PSym): bool = # and s.owner.kind == skModule # owner isn't always a module (global pragma on local var) # and s.loc.k == locGlobalVar # loc isn't always initialized when this proc is used +proc genGlobalVarDecl(p: BProc, n: PNode; td, value: Rope; decl: var Rope) = + let s = n.sym + if sfCodegenDecl notin s.flags: + if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0: + decl.addf "NIM_ALIGN($1) ", [rope(s.alignment)] + if p.hcrOn: decl.add("static ") + elif sfImportc in s.flags: decl.add("extern ") + elif lfExportLib in s.loc.flags: decl.add("N_LIB_EXPORT_VAR ") + else: decl.add("N_LIB_PRIVATE ") + if s.kind == skLet and value != "": decl.add("NIM_CONST ") + decl.add(td) + if p.hcrOn: decl.add("*") + if sfRegister in s.flags: decl.add(" register") + if sfVolatile in s.flags: decl.add(" volatile") + if sfNoalias in s.flags: decl.add(" NIM_NOALIAS") + else: + if value != "": + decl = runtimeFormat(s.cgDeclFrmt & " = $#;$n", [td, s.loc.r, value]) + else: + decl = runtimeFormat(s.cgDeclFrmt & ";$n", [td, s.loc.r]) + +proc genCppVarForCtor(p: BProc; call: PNode; decl: var Rope; didGenTemp: var bool) + proc assignGlobalVar(p: BProc, n: PNode; value: Rope) = let s = n.sym if s.loc.k == locNone: - fillLoc(s.loc, locGlobalVar, n, mangleName(p.module, s), OnHeap) + fillBackendName(p.module, s) + fillLoc(s.loc, locGlobalVar, n, OnHeap) if treatGlobalDifferentlyForHCR(p.module, s): incl(s.loc.flags, lfIndirect) if lfDynamicLib in s.loc.flags: @@ -555,7 +672,7 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) = varInDynamicLib(q, s) else: s.loc.r = mangleDynLibProc(s) - if value != nil: + if value != "": internalError(p.config, n.info, ".dynlib variables cannot have a value") return useHeader(p.module, s) @@ -563,71 +680,86 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) = if not containsOrIncl(p.module.declaredThings, s.id): if sfThread in s.flags: declareThreadVar(p.module, s, sfImportc in s.flags) - if value != nil: + if value != "": internalError(p.config, n.info, ".threadvar variables cannot have a value") else: - var decl: Rope = nil - var td = getTypeDesc(p.module, s.loc.t, skVar) + var decl: Rope = "" + let td = getTypeDesc(p.module, s.loc.t, dkVar) + genGlobalVarDecl(p, n, td, value, decl) if s.constraint.isNil: - if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0: - decl.addf "NIM_ALIGN($1) ", [rope(s.alignment)] - if p.hcrOn: decl.add("static ") - elif sfImportc in s.flags: decl.add("extern ") - elif lfExportLib in s.loc.flags: decl.add("N_LIB_EXPORT_VAR ") - else: decl.add("N_LIB_PRIVATE ") - if s.kind == skLet and value != nil: decl.add("NIM_CONST ") - decl.add(td) - if p.hcrOn: decl.add("*") - if sfRegister in s.flags: decl.add(" register") - if sfVolatile in s.flags: decl.add(" volatile") - if sfNoalias in s.flags: decl.add(" NIM_NOALIAS") - if value != nil: - decl.addf(" $1 = $2;$n", [s.loc.r, value]) + if value != "": + if p.module.compileToCpp and value.startsWith "{{}": + # TODO: taking this branch, re"\{\{\}(,\s\{\})*\}" might be emitted, resulting in + # either warnings (GCC 12.2+) or errors (Clang 15, MSVC 19.3+) of C++11+ compilers **when + # explicit constructors are around** due to overload resolution rules in place [^0][^1][^2] + # *Workaround* here: have C++'s static initialization mechanism do the default init work, + # for us lacking a deeper knowledge of an imported object's constructors' ex-/implicitness + # (so far) *and yet* trying to achieve default initialization. + # Still, generating {}s in genConstObjConstr() just to omit them here is faaaar from ideal; + # need to figure out a better way, possibly by keeping around more data about the + # imported objects' contructors? + # + # [^0]: https://en.cppreference.com/w/cpp/language/aggregate_initialization + # [^1]: https://cplusplus.github.io/CWG/issues/1518.html + # [^2]: https://eel.is/c++draft/over.match.ctor + decl.addf(" $1;$n", [s.loc.r]) + else: + decl.addf(" $1 = $2;$n", [s.loc.r, value]) else: decl.addf(" $1;$n", [s.loc.r]) - else: - if value != nil: - decl = runtimeFormat(s.cgDeclFrmt & " = $#;$n", [td, s.loc.r, value]) - else: - decl = runtimeFormat(s.cgDeclFrmt & ";$n", [td, s.loc.r]) + p.module.s[cfsVars].add(decl) - if p.withinLoop > 0 and value == nil: + if p.withinLoop > 0 and value == "": # fixes tests/run/tzeroarray: resetLoc(p, s.loc) +proc callGlobalVarCppCtor(p: BProc; v: PSym; vn, value: PNode; didGenTemp: var bool) = + let s = vn.sym + fillBackendName(p.module, s) + fillLoc(s.loc, locGlobalVar, vn, OnHeap) + var decl: Rope = "" + let td = getTypeDesc(p.module, vn.sym.typ, dkVar) + genGlobalVarDecl(p, vn, td, "", decl) + decl.add " " & $s.loc.r + genCppVarForCtor(p, value, decl, didGenTemp) + if didGenTemp: return # generated in the caller + p.module.s[cfsVars].add decl + proc assignParam(p: BProc, s: PSym, retType: PType) = - assert(s.loc.r != nil) + assert(s.loc.r != "") scopeMangledParam(p, s) proc fillProcLoc(m: BModule; n: PNode) = let sym = n.sym if sym.loc.k == locNone: - fillLoc(sym.loc, locProc, n, mangleName(m, sym), OnStack) + fillBackendName(m, sym) + fillLoc(sym.loc, locProc, n, OnStack) proc getLabel(p: BProc): TLabel = inc(p.labels) result = "LA" & rope(p.labels) & "_" proc fixLabel(p: BProc, labl: TLabel) = - lineF(p, cpsStmts, "$1: ;$n", [labl]) + p.s(cpsStmts).add("$1: ;$n" % [labl]) proc genVarPrototype(m: BModule, n: PNode) proc requestConstImpl(p: BProc, sym: PSym) proc genStmts(p: BProc, t: PNode) proc expr(p: BProc, n: PNode, d: var TLoc) -proc genProcPrototype(m: BModule, sym: PSym) -proc putLocIntoDest(p: BProc, d: var TLoc, s: TLoc) -proc intLiteral(i: BiggestInt): Rope -proc genLiteral(p: BProc, n: PNode): Rope -proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope -proc raiseExit(p: BProc) -proc initLocExpr(p: BProc, e: PNode, result: var TLoc) = - initLoc(result, locNone, e, OnUnknown) +proc putLocIntoDest(p: BProc, d: var TLoc, s: TLoc) +proc intLiteral(i: BiggestInt; result: var Rope) +proc genLiteral(p: BProc, n: PNode; result: var Rope) +proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope; argsCounter: var int) +proc raiseExit(p: BProc) +proc raiseExitCleanup(p: BProc, destroy: string) + +proc initLocExpr(p: BProc, e: PNode, flags: TLocFlags = {}): TLoc = + result = initLoc(locNone, e, OnUnknown, flags) expr(p, e, result) -proc initLocExprSingleUse(p: BProc, e: PNode, result: var TLoc) = - initLoc(result, locNone, e, OnUnknown) +proc initLocExprSingleUse(p: BProc, e: PNode): TLoc = + result = initLoc(locNone, e, OnUnknown) if e.kind in nkCallKinds and (e[0].kind != nkSym or e[0].sym.magic == mNone): # We cannot check for tfNoSideEffect here because of mutable parameters. discard "bug #8202; enforce evaluation order for nested calls for C++ too" @@ -643,25 +775,29 @@ include ccgcalls, "ccgstmts.nim" proc initFrame(p: BProc, procname, filename: Rope): Rope = const frameDefines = """ - $1 define nimfr_(proc, file) \ - TFrame FR_; \ - FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = 0; #nimFrame(&FR_); +$1define nimfr_(proc, file) \ + TFrame FR_; \ + FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = 0; #nimFrame(&FR_); - $1 define nimfrs_(proc, file, slots, length) \ - struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR_; \ - FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = length; #nimFrame((TFrame*)&FR_); + $1define nimfrs_(proc, file, slots, length) \ + struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename;NI len;VarSlot s[slots];} FR_; \ + FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = length; #nimFrame((TFrame*)&FR_); - $1 define nimln_(n, file) \ - FR_.line = n; FR_.filename = file; - """ + $1define nimln_(n) \ + FR_.line = n; + + $1define nimlf_(n, file) \ + FR_.line = n; FR_.filename = file; + +""" if p.module.s[cfsFrameDefines].len == 0: appcg(p.module, p.module.s[cfsFrameDefines], frameDefines, ["#"]) - discard cgsym(p.module, "nimFrame") + cgsym(p.module, "nimFrame") result = ropecg(p.module, "\tnimfr_($1, $2);$n", [procname, filename]) proc initFrameNoDebug(p: BProc; frame, procname, filename: Rope; line: int): Rope = - discard cgsym(p.module, "nimFrame") + cgsym(p.module, "nimFrame") p.blocks[0].sections[cpsLocals].addf("TFrame $1;$n", [frame]) result = ropecg(p.module, "\t$1.procname = $2; $1.filename = $3; " & " $1.line = $4; $1.len = -1; nimFrame(&$1);$n", @@ -688,33 +824,36 @@ proc loadDynamicLib(m: BModule, lib: PLib) = if not lib.generated: lib.generated = true var tmp = getTempName(m) - assert(lib.name == nil) + assert(lib.name == "") lib.name = tmp # BUGFIX: cgsym has awful side-effects m.s[cfsVars].addf("static void* $1;$n", [tmp]) if lib.path.kind in {nkStrLit..nkTripleStrLit}: var s: TStringSeq = @[] libCandidates(lib.path.strVal, s) rawMessage(m.config, hintDependency, lib.path.strVal) - var loadlib: Rope = nil + var loadlib: Rope = "" for i in 0..high(s): inc(m.labels) if i > 0: loadlib.add("||") let n = newStrNode(nkStrLit, s[i]) n.info = lib.path.info - appcg(m, loadlib, "($1 = #nimLoadLibrary($2))$n", - [tmp, genStringLiteral(m, n)]) + appcg(m, loadlib, "($1 = #nimLoadLibrary(", [tmp]) + genStringLiteral(m, n, loadlib) + loadlib.addf "))$n", [] appcg(m, m.s[cfsDynLibInit], - "if (!($1)) #nimLoadLibraryError($2);$n", - [loadlib, genStringLiteral(m, lib.path)]) + "if (!($1)) #nimLoadLibraryError(", + [loadlib]) + genStringLiteral(m, lib.path, m.s[cfsDynLibInit]) + m.s[cfsDynLibInit].addf ");$n", [] + else: var p = newProc(nil, m) p.options.excl optStackTrace p.flags.incl nimErrorFlagDisabled - var dest: TLoc - initLoc(dest, locTemp, lib.path, OnStack) + var dest: TLoc = initLoc(locTemp, lib.path, OnStack) dest.r = getTempName(m) appcg(m, m.s[cfsDynLibInit],"$1 $2;$n", - [getTypeDesc(m, lib.path.typ, skVar), rdLoc(dest)]) + [getTypeDesc(m, lib.path.typ, dkVar), rdLoc(dest)]) expr(p, lib.path, dest) m.s[cfsVars].add(p.s(cpsLocals)) @@ -724,7 +863,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) = "if (!($1 = #nimLoadLibrary($2))) #nimLoadLibraryError($2);$n", [tmp, rdLoc(dest)]) - if lib.name == nil: internalError(m.config, "loadDynamicLib") + if lib.name == "": internalError(m.config, "loadDynamicLib") proc mangleDynLibProc(sym: PSym): Rope = # we have to build this as a single rope in order not to trip the @@ -746,15 +885,14 @@ proc symInDynamicLib(m: BModule, sym: PSym) = inc(m.labels, 2) if isCall: let n = lib.path - var a: TLoc - initLocExpr(m.initProc, n[0], a) + var a: TLoc = initLocExpr(m.initProc, n[0]) var params = rdLoc(a) & "(" for i in 1.. 0 and n[0].kind == nkEmpty + of nkSym: + if n.sym.kind == skResult: + result = true + else: + for i in 0.. 0: if n[0].kind == nkEmpty and result != InitSkippable: @@ -929,7 +1092,7 @@ proc allPathsAsgnResult(n: PNode): InitResultEnum = # initialized. This avoids cases like #9286 where this heuristic lead to # wrong code being generated. result = InitRequired - else: result = allPathsAsgnResult(n[0]) + else: result = allPathsAsgnResult(p, n[0]) of nkIfStmt, nkIfExpr: var exhaustive = false result = InitSkippable @@ -946,7 +1109,7 @@ proc allPathsAsgnResult(n: PNode): InitResultEnum = if containsResult(n[0]): return InitRequired result = InitSkippable var exhaustive = skipTypes(n[0].typ, - abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString} + abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString, tyCstring} for i in 1..= prc.ast.len: internalError(m.config, prc.info, "proc has no result symbol") let resNode = prc.ast[resultPos] let res = resNode.sym # get result symbol - if not isInvalidReturnType(m.config, prc.typ[0]): + if not isInvalidReturnType(m.config, prc.typ) and sfConstructor notin prc.flags: if sfNoInit in prc.flags: incl(res.flags, sfNoInit) if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(procBody); val != nil): var decl = localVarDecl(p, resNode) - var a: TLoc - initLocExprSingleUse(p, val, a) + var a: TLoc = initLocExprSingleUse(p, val) linefmt(p, cpsStmts, "$1 = $2;$n", [decl, rdLoc(a)]) else: # declare the result symbol: assignLocalVar(p, resNode) - assert(res.loc.r != nil) - initLocalVar(p, res, immediateAsgn=false) + assert(res.loc.r != "") + if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc} and + allPathsAsgnResult(p, procBody) == InitSkippable: + # In an ideal world the codegen could rely on injectdestructors doing its job properly + # and then the analysis step would not be required. + discard "result init optimized out" + else: + initLocalVar(p, res, immediateAsgn=false) returnStmt = ropecg(p.module, "\treturn $1;$n", [rdLoc(res.loc)]) + elif sfConstructor in prc.flags: + resNode.sym.loc.flags.incl lfIndirect + fillLoc(resNode.sym.loc, locParam, resNode, "this", OnHeap) + prc.loc.r = getTypeDesc(m, resNode.sym.loc.t, dkVar) else: - fillResult(p.config, resNode) - assignParam(p, res, prc.typ[0]) + fillResult(p.config, resNode, prc.typ) + assignParam(p, res, prc.typ.returnType) # We simplify 'unsureAsgn(result, nil); unsureAsgn(result, x)' # to 'unsureAsgn(result, x)' # Sketch why this is correct: If 'result' points to a stack location @@ -1045,7 +1237,7 @@ proc genProcAux(m: BModule, prc: PSym) = # global is either 'nil' or points to valid memory and so the RC operation # succeeds without touching not-initialized memory. if sfNoInit in prc.flags: discard - elif allPathsAsgnResult(procBody) == InitSkippable: discard + elif allPathsAsgnResult(p, procBody) == InitSkippable: discard else: resetLoc(p, res.loc) if skipTypes(res.typ, abstractInst).kind == tyArray: @@ -1055,17 +1247,19 @@ proc genProcAux(m: BModule, prc: PSym) = for i in 1.. 0: m.s[cfsVars].addf "NIM_ALIGN($1) ", [rope(sym.alignment)] m.s[cfsVars].add(if m.hcrOn: "static " else: "extern ") - m.s[cfsVars].add(getTypeDesc(m, sym.loc.t, skVar)) + m.s[cfsVars].add(getTypeDesc(m, sym.loc.t, dkVar)) if m.hcrOn: m.s[cfsVars].add("*") if lfDynamicLib in sym.loc.flags: m.s[cfsVars].add("*") if sfRegister in sym.flags: m.s[cfsVars].add(" register") @@ -1255,7 +1451,7 @@ proc genVarPrototype(m: BModule, n: PNode) = m.s[cfsVars].addf(" $1;$n", [sym.loc.r]) if m.hcrOn: m.initProc.procSec(cpsLocals).addf( "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r, - getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(m, sym)]) + getTypeDesc(m, sym.loc.t, dkVar), getModuleDllPath(m, sym)]) proc addNimDefines(result: var Rope; conf: ConfigRef) {.inline.} = result.addf("#define NIM_INTBITS $1\L", [ @@ -1285,23 +1481,29 @@ proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope = if conf.hcrOn: result.add("#define NIM_HOT_CODE_RELOADING\L") addNimDefines(result, conf) -proc getSomeNameForModule(m: PSym): Rope = - assert m.kind == skModule - assert m.owner.kind == skPackage - if {sfSystemModule, sfMainModule} * m.flags == {}: - result = m.owner.name.s.mangle.rope - result.add "_" - result.add m.name.s.mangle +proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope = + ## Returns a mangled module name. + result = "" + result.add mangleModuleName(conf, filename).mangle + +proc getSomeNameForModule(m: BModule): Rope = + ## Returns a mangled module name. + assert m.module.kind == skModule + assert m.module.owner.kind == skPackage + result = "" + result.add mangleModuleName(m.g.config, m.filename).mangle proc getSomeInitName(m: BModule, suffix: string): Rope = if not m.hcrOn: - result = getSomeNameForModule(m.module) + result = getSomeNameForModule(m) + else: + result = "" result.add suffix proc getInitName(m: BModule): Rope = if sfMainModule in m.module.flags: # generate constant name for main module, for "easy" debugging. - result = rope"NimMainModule" + result = rope(m.config.nimMainPrefix) & rope"NimMainModule" else: result = getSomeInitName(m, "Init000") @@ -1314,67 +1516,84 @@ proc genMainProc(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propagate ## into other modules, only simple rope manipulations are allowed - - var preMainCode: Rope + var preMainCode: Rope = "" if m.hcrOn: proc loadLib(handle: string, name: string): Rope = + result = "" let prc = magicsys.getCompilerProc(m.g.graph, name) assert prc != nil let n = newStrNode(nkStrLit, prc.annex.path.strVal) n.info = prc.annex.path.info + var strLit = newRopeAppender() + genStringLiteral(m, n, strLit) appcg(m, result, "\tif (!($1 = #nimLoadLibrary($2)))$N" & "\t\t#nimLoadLibraryError($2);$N", - [handle, genStringLiteral(m, n)]) + [handle, strLit]) preMainCode.add(loadLib("hcr_handle", "hcrGetProc")) - preMainCode.add("\tvoid* rtl_handle;\L") - preMainCode.add(loadLib("rtl_handle", "nimGC_setStackBottom")) - preMainCode.add(hcrGetProcLoadCode(m, "nimGC_setStackBottom", "nimrtl_", "rtl_handle", "nimGetProcAddr")) - preMainCode.add("\tinner = PreMain;\L") - preMainCode.add("\tinitStackBottomWith_actual((void *)&inner);\L") - preMainCode.add("\t(*inner)();\L") + if m.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}: + preMainCode.add("\t$1PreMain();\L" % [rope m.config.nimMainPrefix]) + else: + preMainCode.add("\tvoid* rtl_handle;\L") + preMainCode.add(loadLib("rtl_handle", "nimGC_setStackBottom")) + preMainCode.add(hcrGetProcLoadCode(m, "nimGC_setStackBottom", "nimrtl_", "rtl_handle", "nimGetProcAddr")) + preMainCode.add("\tinner = $1PreMain;\L" % [rope m.config.nimMainPrefix]) + preMainCode.add("\tinitStackBottomWith_actual((void *)&inner);\L") + preMainCode.add("\t(*inner)();\L") else: - preMainCode.add("\tPreMain();\L") + preMainCode.add("\t$1PreMain();\L" % [rope m.config.nimMainPrefix]) + + var posixCmdLine: Rope = "" + if optNoMain notin m.config.globalOptions: + posixCmdLine.add "N_LIB_PRIVATE int cmdCount;\L" + posixCmdLine.add "N_LIB_PRIVATE char** cmdLine;\L" + posixCmdLine.add "N_LIB_PRIVATE char** gEnv;\L" const - # not a big deal if we always compile these 3 global vars... makes the HCR code easier - PosixCmdLine = - "N_LIB_PRIVATE int cmdCount;$N" & - "N_LIB_PRIVATE char** cmdLine;$N" & - "N_LIB_PRIVATE char** gEnv;$N" - # The use of a volatile function pointer to call Pre/NimMainInner # prevents inlining of the NimMainInner function and dependent # functions, which might otherwise merge their stack frames. + PreMainBody = "$N" & - "N_LIB_PRIVATE void PreMainInner(void) {$N" & + "N_LIB_PRIVATE void $3PreMainInner(void) {$N" & "$2" & "}$N$N" & - PosixCmdLine & - "N_LIB_PRIVATE void PreMain(void) {$N" & + "$4" & + "N_LIB_PRIVATE void $3PreMain(void) {$N" & + "##if $5$N" & # 1 for volatile call, 0 for non-volatile "\tvoid (*volatile inner)(void);$N" & - "\tinner = PreMainInner;$N" & + "\tinner = $3PreMainInner;$N" & "$1" & "\t(*inner)();$N" & + "##else$N" & + "$1" & + "\t$3PreMainInner();$N" & + "##endif$N" & "}$N$N" MainProcs = - "\tNimMain();$N" + "\t$^NimMain();$N" MainProcsWithResult = MainProcs & ("\treturn $1nim_program_result;$N") - NimMainInner = "N_LIB_PRIVATE N_CDECL(void, NimMainInner)(void) {$N" & + NimMainInner = "N_LIB_PRIVATE N_CDECL(void, $5NimMainInner)(void) {$N" & "$1" & "}$N$N" NimMainProc = - "N_CDECL(void, NimMain)(void) {$N" & - "\tvoid (*volatile inner)(void);$N" & - "$4" & - "\tinner = NimMainInner;$N" & - "$2" & - "\t(*inner)();$N" & + "N_CDECL(void, $5NimMain)(void) {$N" & + "##if $6$N" & # 1 for volatile call, 0 for non-volatile + "\tvoid (*volatile inner)(void);$N" & + "$4" & + "\tinner = $5NimMainInner;$N" & + "$2" & + "\t(*inner)();$N" & + "##else$N" & + "$4" & + "$2" & + "\t$5NimMainInner();$N" & + "##endif$N" & "}$N$N" NimMainBody = NimMainInner & NimMainProc @@ -1405,7 +1624,7 @@ proc genMainProc(m: BModule) = WinCDllMain = "BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdreason, $N" & " LPVOID lpvReserved) {$N" & - "\tif(fwdreason == DLL_PROCESS_ATTACH) {$N" & MainProcs & "}$N" & + "\tif (fwdreason == DLL_PROCESS_ATTACH) {$N" & MainProcs & "\t}$N" & "\treturn 1;$N}$N$N" PosixNimDllMain = WinNimDllMain @@ -1417,7 +1636,7 @@ proc genMainProc(m: BModule) = GenodeNimMain = "extern Genode::Env *nim_runtime_env;$N" & - "extern void nim_component_construct(Genode::Env*);$N$N" & + "extern \"C\" void nim_component_construct(Genode::Env*);$N$N" & NimMainBody ComponentConstruct = @@ -1439,67 +1658,91 @@ proc genMainProc(m: BModule) = m.includeHeader("") let initStackBottomCall = - if m.config.target.targetOS == osStandalone or m.config.selectedGC == gcNone: "".rope + if m.config.target.targetOS == osStandalone or m.config.selectedGC in {gcNone, gcArc, gcAtomicArc, gcOrc}: "".rope else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", []) inc(m.labels) - appcg(m, m.s[cfsProcs], PreMainBody, [m.g.mainDatInit, m.g.otherModsInit]) + + let isVolatile = if m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc}: "1" else: "0" + appcg(m, m.s[cfsProcs], PreMainBody, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine, isVolatile]) if m.config.target.targetOS == osWindows and m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: if optGenGuiApp in m.config.globalOptions: const nimMain = WinNimMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix, isVolatile]) else: const nimMain = WinNimDllMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix, isVolatile]) elif m.config.target.targetOS == osGenode: const nimMain = GenodeNimMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix, isVolatile]) elif optGenDynLib in m.config.globalOptions: const nimMain = PosixNimDllMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) - elif m.config.target.targetOS == osStandalone: - const nimMain = NimMainBody - appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix, isVolatile]) else: const nimMain = NimMainBody appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) - + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix, isVolatile]) if optNoMain notin m.config.globalOptions: if m.config.cppCustomNamespace.len > 0: - m.s[cfsProcs].add closeNamespaceNim() & "using namespace " & m.config.cppCustomNamespace & ";\L" + closeNamespaceNim(m.s[cfsProcs]) + m.s[cfsProcs].add "using namespace " & m.config.cppCustomNamespace & ";\L" if m.config.target.targetOS == osWindows and m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: if optGenGuiApp in m.config.globalOptions: const otherMain = WinCMain - appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""]) + appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix]) else: const otherMain = WinCDllMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif m.config.target.targetOS == osGenode: const otherMain = ComponentConstruct - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif optGenDynLib in m.config.globalOptions: const otherMain = PosixCDllMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif m.config.target.targetOS == osStandalone: const otherMain = StandaloneCMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) else: const otherMain = PosixCMain - appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""]) - + appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix]) if m.config.cppCustomNamespace.len > 0: - m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace) + openNamespaceNim(m.config.cppCustomNamespace, m.s[cfsProcs]) + +proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) = + ## Called from the IC backend. + if HasDatInitProc in flags: + let datInit = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "DatInit000" + g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) + g.mainDatInit.addf("\t$1();$N", [datInit]) + if HasModuleInitProc in flags: + let init = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "Init000" + g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) + let initCall = "\t$1();$N" % [init] + if sfMainModule in m.flags: + g.mainModInit.add(initCall) + elif sfSystemModule in m.flags: + g.mainDatInit.add(initCall) # systemInit must called right after systemDatInit if any + else: + g.otherModsInit.add(initCall) + +proc whichInitProcs*(m: BModule): set[ModuleBackendFlag] = + # called from IC. + result = {} + if m.hcrOn or m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: + result.incl HasModuleInitProc + for i in cfsTypeInit1..cfsDynLibInit: + if m.s[i].len != 0: + result.incl HasDatInitProc + break proc registerModuleToMain(g: BModuleList; m: BModule) = let @@ -1518,7 +1761,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = hcrModuleMeta.addf("\t\"\"};$n", []) hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", []) hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n", - [($sigHash(m.module)).rope]) + [($sigHash(m.module, m.config)).rope]) if sfMainModule in m.module.flags: g.mainModProcs.add(hcrModuleMeta) g.mainModProcs.addf("static void* hcr_handle;$N", []) @@ -1559,7 +1802,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = if sfSystemModule in m.module.flags: if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone: g.mainDatInit.add(ropecg(m, "\t#initThreadVarsEmulation();$N", [])) - if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}: + if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc}: g.mainDatInit.add(ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", [])) if m.s[cfsInitProc].len > 0: @@ -1584,7 +1827,7 @@ proc genDatInitCode(m: BModule) = # we don't want to break into such init code - could happen if a line # directive from a function written by the user spills after itself - genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) + genCLineDir(prc, InvalidFileIdx, 999999, m.config) for i in cfsTypeInit1..cfsDynLibInit: if m.s[i].len != 0: @@ -1595,6 +1838,7 @@ proc genDatInitCode(m: BModule) = if moduleDatInitRequired: m.s[cfsDatInitProc].add(prc) + #rememberFlag(m.g.graph, m.module, HasDatInitProc) # Very similar to the contents of symInDynamicLib - basically only the # things needed for the hot code reloading runtime procs to be loaded @@ -1609,10 +1853,10 @@ proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): R prc.typ.sym = nil if not containsOrIncl(m.declaredThings, prc.id): - m.s[cfsVars].addf("static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t, skVar)]) + m.s[cfsVars].addf("static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t, dkVar)]) result = "\t$1 = ($2) $3($4, $5);$n" % - [tmp, getTypeDesc(m, prc.typ, skVar), getProcFunc.rope, handle.rope, makeCString(prefix & sym)] + [tmp, getTypeDesc(m, prc.typ, dkVar), getProcFunc.rope, handle.rope, makeCString(prefix & sym)] proc genInitCode(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, @@ -1624,7 +1868,7 @@ proc genInitCode(m: BModule) = [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), initname] # we don't want to break into such init code - could happen if a line # directive from a function written by the user spills after itself - genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) + genCLineDir(prc, InvalidFileIdx, 999999, m.config) if m.typeNodes > 0: if m.hcrOn: appcg(m, m.s[cfsTypeInit1], "\t#TNimNode* $1;$N", [m.typeNodesName]) @@ -1654,7 +1898,7 @@ proc genInitCode(m: BModule) = # Give this small function its own scope prc.addf("{$N", []) # Keep a bogus frame in case the code needs one - prc.add(~"\tTFrame FR_; FR_.len = 0;$N") + prc.add("\tTFrame FR_; FR_.len = 0;\n") writeSection(preInitProc, cpsLocals) writeSection(preInitProc, cpsInit, m.hcrOn) @@ -1680,20 +1924,23 @@ proc genInitCode(m: BModule) = var procname = makeCString(m.module.name.s) prc.add(initFrame(m.initProc, procname, quotedFilename(m.config, m.module.info))) else: - prc.add(~"\tTFrame FR_; FR_.len = 0;$N") + prc.add("\tTFrame FR_; FR_.len = 0;\n") writeSection(initProc, cpsInit, m.hcrOn) writeSection(initProc, cpsStmts) if beforeRetNeeded in m.initProc.flags: - prc.add(~"\tBeforeRet_: ;$n") + prc.add("\tBeforeRet_: ;\n") - if sfMainModule in m.module.flags and m.config.exc == excGoto: + if m.config.exc == excGoto: if getCompilerProc(m.g.graph, "nimTestErrorFlag") != nil: m.appcg(prc, "\t#nimTestErrorFlag();$n", []) if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: prc.add(deinitFrame(m.initProc)) + elif m.config.exc == excGoto: + if getCompilerProc(m.g.graph, "nimTestErrorFlag") != nil: + m.appcg(prc, "\t#nimTestErrorFlag();$n", []) prc.addf("}$N", []) @@ -1717,7 +1964,7 @@ proc genInitCode(m: BModule) = m.s[cfsInitProc].addf("}$N$N", []) for i, el in pairs(m.extensionLoaders): - if el != nil: + if el != "": let ex = "NIM_EXTERNC N_NIMCALL(void, nimLoadProcs$1)(void) {$2}$N$N" % [(i.ord - '0'.ord).rope, el] moduleInitRequired = true @@ -1725,6 +1972,7 @@ proc genInitCode(m: BModule) = if moduleInitRequired or sfMainModule in m.module.flags: m.s[cfsInitProc].add(prc) + #rememberFlag(m.g.graph, m.module, HasModuleInitProc) genDatInitCode(m) @@ -1735,6 +1983,40 @@ proc genInitCode(m: BModule) = registerModuleToMain(m.g, m) +proc postprocessCode(conf: ConfigRef, r: var Rope) = + # find the first directive + var f = r.find(postprocessDirStart) + if f == -1: + return + + var + nimlnDirLastF = "" + + var res: Rope = r.substr(0, f - 1) + while f != -1: + var + e = r.find(postprocessDirEnd, f + 1) + dir = r.substr(f + 1, e - 1).split(postprocessDirSep) + case dir[0] + of "nimln": + if dir[2] == nimlnDirLastF: + res.add("nimln_(" & dir[1] & ");") + else: + res.add("nimlf_(" & dir[1] & ", " & quotedFilename(conf, dir[2].parseInt.FileIndex) & ");") + nimlnDirLastF = dir[2] + else: + raiseAssert "unexpected postprocess directive" + + # find the next directive + f = r.find(postprocessDirStart, e + 1) + # copy the code until the next directive + if f != -1: + res.add(r.substr(e + 1, f - 1)) + else: + res.add(r.substr(e + 1)) + + r = res + proc genModule(m: BModule, cfile: Cfile): Rope = var moduleIsEmpty = true @@ -1744,11 +2026,9 @@ proc genModule(m: BModule, cfile: Cfile): Rope = generateHeaders(m) result.add(m.s[cfsHeaders]) if m.config.cppCustomNamespace.len > 0: - result.add openNamespaceNim(m.config.cppCustomNamespace) + openNamespaceNim(m.config.cppCustomNamespace, result) if m.s[cfsFrameDefines].len > 0: result.add(m.s[cfsFrameDefines]) - else: - result.add("#define nimfr_(x, y)\n#define nimln_(x, y)\n") for i in cfsForwardTypes..cfsProcs: if m.s[i].len > 0: @@ -1763,10 +2043,18 @@ proc genModule(m: BModule, cfile: Cfile): Rope = result.add(m.s[cfsDatInitProc]) if m.config.cppCustomNamespace.len > 0: - result.add closeNamespaceNim() + closeNamespaceNim(result) + + if optLineDir in m.config.options: + var srcFileDefs = "" + for fi in 0..m.config.m.fileInfos.high: + srcFileDefs.add("#define FX_" & $fi & " " & makeSingleLineCString(toFullPath(m.config, fi.FileIndex)) & "\n") + result = srcFileDefs & result if moduleIsEmpty: - result = nil + result = "" + + postprocessCode(m.config, result) proc initProcOptions(m: BModule): TOptions = let opts = m.config.options @@ -1787,11 +2075,12 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule result.typeInfoMarker = initTable[SigHash, Rope]() result.sigConflicts = initCountTable[SigHash]() result.initProc = newProc(nil, result) + for i in low(result.s)..high(result.s): result.s[i] = newRopeAppender() result.initProc.options = initProcOptions(result) result.preInitProc = newProc(nil, result) result.preInitProc.flags.incl nimErrorFlagDisabled result.preInitProc.labels = 100_000 # little hack so that unique temporaries are generated - initNodeTable(result.dataCache) + result.dataCache = initNodeTable() result.typeStack = @[] result.typeNodesName = getTempName(result) result.nimTypesName = getTempName(result) @@ -1820,10 +2109,7 @@ template injectG() {.dirty.} = graph.backend = newModuleList(graph) let g = BModuleList(graph.backend) -when not defined(nimHasSinkInference): - {.pragma: nosinks.} - -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = injectG() result = newModule(g, module, graph.config) result.idgen = idgen @@ -1844,13 +2130,14 @@ proc writeHeader(m: BModule) = generateThreadLocalStorage(m) for i in cfsHeaders..cfsProcs: result.add(m.s[i]) - if m.config.cppCustomNamespace.len > 0 and i == cfsHeaders: result.add openNamespaceNim(m.config.cppCustomNamespace) + if m.config.cppCustomNamespace.len > 0 and i == cfsHeaders: + openNamespaceNim(m.config.cppCustomNamespace, result) result.add(m.s[cfsInitProc]) if optGenDynLib in m.config.globalOptions: result.add("N_LIB_IMPORT ") - result.addf("N_CDECL(void, NimMain)(void);$n", []) - if m.config.cppCustomNamespace.len > 0: result.add closeNamespaceNim() + result.addf("N_CDECL(void, $1NimMain)(void);$n", [rope m.config.nimMainPrefix]) + if m.config.cppCustomNamespace.len > 0: closeNamespaceNim(result) result.addf("#endif /* $1 */$n", [guard]) if not writeRope(result, m.filename): rawMessage(m.config, errCannotOpenFile, m.filename.string) @@ -1860,7 +2147,7 @@ proc getCFile(m: BModule): AbsoluteFile = if m.compileToCpp: ".nim.cpp" elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m" else: ".nim.c" - result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext) + result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext) when false: proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext = @@ -1890,7 +2177,7 @@ proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) = proc genTopLevelStmt*(m: BModule; n: PNode) = ## Also called from `ic/cbackend.nim`. - if passes.skipCodegen(m.config, n): return + if pipelineutils.skipCodegen(m.config, n): return m.initProc.options = initProcOptions(m) #softRnl = if optLineDir in m.config.options: noRnl else: rnl # XXX replicate this logic! @@ -1903,12 +2190,6 @@ proc genTopLevelStmt*(m: BModule; n: PNode) = else: genProcBody(m.initProc, transformedN) -proc myProcess(b: PPassContext, n: PNode): PNode = - result = n - if b != nil: - var m = BModule(b) - genTopLevelStmt(m, n) - proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = if optForceFullMake notin m.config.globalOptions: if not moduleHasChanged(m.g.graph, m.module): @@ -1954,7 +2235,7 @@ proc writeModule(m: BModule, pending: bool) = var cf = Cfile(nimname: m.module.name.s, cname: cfile, obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {}) var code = genModule(m, cf) - if code != nil or m.config.symbolFiles != disabledSf: + if code != "" or m.config.symbolFiles != disabledSf: when hasTinyCBackend: if m.config.cmd == cmdTcc: tccgen.compileCCode($code, m.config) @@ -1974,18 +2255,41 @@ proc updateCachedModule(m: BModule) = cf.flags = {CfileFlag.Cached} addFileToCompile(m.config, cf) +proc generateLibraryDestroyGlobals(graph: ModuleGraph; m: BModule; body: PNode; isDynlib: bool): PSym = + let procname = getIdent(graph.cache, "NimDestroyGlobals") + result = newSym(skProc, procname, m.idgen, m.module.owner, m.module.info) + result.typ = newProcType(m.module.info, m.idgen, m.module.owner) + result.typ.callConv = ccCDecl + incl result.flags, sfExportc + result.loc.r = "NimDestroyGlobals" + if isDynlib: + incl(result.loc.flags, lfExportLib) + + let theProc = newNodeI(nkProcDef, m.module.info, bodyPos+1) + for i in 0.. 0 if we are within a loop splitDecls*: int # > 0 if we are in some context for C++ that @@ -99,6 +97,7 @@ type withinTryWithExcept*: int # required for goto based exception handling withinBlockLeaveActions*: int # complex to explain sigConflicts*: CountTable[string] + inUncheckedAssignSection*: int TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] @@ -138,6 +137,7 @@ type # unconditionally... # nimtvDeps is VERY hard to cache because it's # not a list of IDs nor can it be made to be one. + mangledPrcs*: HashSet[string] TCGen = object of PPassContext # represents a C source file s*: TCFileSections # sections of the C file @@ -151,7 +151,7 @@ type typeABICache*: HashSet[SigHash] # cache for ABI checks; reusing typeCache # would be ideal but for some reason enums # don't seem to get cached so it'd generate - # 1 ABI check per occurence in code + # 1 ABI check per occurrence in code forwTypeCache*: TypeCache # cache for forward declarations of types declaredThings*: IntSet # things we have declared in this .c file declaredProtos*: IntSet # prototypes we have declared in this .c file @@ -170,13 +170,13 @@ type labels*: Natural # for generating unique module-scope names extensionLoaders*: array['0'..'9', Rope] # special procs for the # OpenGL wrapper - injectStmt*: Rope sigConflicts*: CountTable[SigHash] g*: BModuleList ndi*: NdiFile template config*(m: BModule): ConfigRef = m.g.config template config*(p: BProc): ConfigRef = p.module.g.config +template vccAndC*(p: BProc): bool = p.module.config.cCompiler == ccVcc and p.module.config.backend == backendC proc includeHeader*(this: BModule; header: string) = if not this.headerFiles.contains header: @@ -190,16 +190,23 @@ proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} = # top level proc sections result = p.blocks[0].sections[s] +proc initBlock*(): TBlock = + result = TBlock() + for i in low(result.sections)..high(result.sections): + result.sections[i] = newRopeAppender() + proc newProc*(prc: PSym, module: BModule): BProc = - new(result) - result.prc = prc - result.module = module - result.options = if prc != nil: prc.options - else: module.config.options - newSeq(result.blocks, 1) - result.nestedTryStmts = @[] - result.finallySafePoints = @[] - result.sigConflicts = initCountTable[string]() + result = BProc( + prc: prc, + module: module, + optionsStack: if module.initProc != nil: module.initProc.optionsStack + else: @[], + options: if prc != nil: prc.options + else: module.config.options, + blocks: @[initBlock()], + sigConflicts: initCountTable[string]()) + if optQuirky in result.options: + result.flags = {nimErrorFlagDisabled} proc newModuleList*(g: ModuleGraph): BModuleList = BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: int32]](), diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 484bc9d976..b132c60493 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -10,8 +10,15 @@ ## This module implements code generation for methods. import - intsets, options, ast, msgs, idents, renderer, types, magicsys, - sempass2, strutils, modulegraphs, lineinfos + options, ast, msgs, idents, renderer, types, magicsys, + sempass2, modulegraphs, lineinfos, astalgo + +import std/intsets + +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/[tables] proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode = var dest = skipTypes(d, abstractPtrs) @@ -40,12 +47,15 @@ proc getDispatcher*(s: PSym): PSym = if dispatcherPos < s.ast.len: result = s.ast[dispatcherPos].sym doAssert sfDispatcher in result.flags + else: + result = nil proc methodCall*(n: PNode; conf: ConfigRef): PNode = result = n # replace ordinary method by dispatcher method: let disp = getDispatcher(result[0].sym) if disp != nil: + result[0].typ = disp.typ result[0].sym = disp # change the arguments to up/downcasts to fit the dispatcher's parameters: for i in 1.. resultPos: - disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextSymId(idgen)) + disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, idgen) else: # We've encountered a method prototype without a filled-in # resultPos slot. We put a placeholder in there that will @@ -143,25 +157,16 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) = disp.ast[resultPos].kind == nkEmpty: disp.ast[resultPos] = copyTree(meth.ast[resultPos]) - # The following code works only with lock levels, so we disable - # it when they're not available. - when declared(TLockLevel): - proc `<`(a, b: TLockLevel): bool {.borrow.} - proc `==`(a, b: TLockLevel): bool {.borrow.} - if disp.typ.lockLevel == UnspecifiedLockLevel: - disp.typ.lockLevel = meth.typ.lockLevel - elif meth.typ.lockLevel != UnspecifiedLockLevel and - meth.typ.lockLevel != disp.typ.lockLevel: - message(conf, meth.info, warnLockLevel, - "method has lock level $1, but another method has $2" % - [$meth.typ.lockLevel, $disp.typ.lockLevel]) - # XXX The following code silences a duplicate warning in - # checkMethodeffects() in sempass2.nim for now. - if disp.typ.lockLevel < meth.typ.lockLevel: - disp.typ.lockLevel = meth.typ.lockLevel - proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) = - var witness: PSym + var witness: PSym = nil + if s.typ.firstParamType.owner.getModule != s.getModule and vtables in g.config.features and not + g.config.isDefined("nimInternalNonVtablesTesting"): + localError(g.config, s.info, errGenerated, "method `" & s.name.s & + "` can be defined only in the same module with its type (" & s.typ.firstParamType.typeToString() & ")") + if sfImportc in s.flags: + localError(g.config, s.info, errGenerated, "method `" & s.name.s & + "` is not allowed to have 'importc' pragmas") + for i in 0.. 0: # echo "hi" # :state = 1 # Next state @@ -26,12 +27,14 @@ # else: # :state = 2 # Next state # break :stateLoop # Proceed to the next state -# STATE1: +# of 1: # dec a # :state = 0 # Next state # break :stateLoop # Proceed to the next state -# STATE2: +# of 2: # :state = -1 # End of execution +# else: +# return # The transformation should play well with lambdalifting, however depending # on situation, it can be called either before or after lambdalifting @@ -104,12 +107,13 @@ # Is transformed to (yields are left in place for example simplicity, # in reality the code is subdivided even more, as described above): # -# STATE0: # Try +# case :state +# of 0: # Try # yield 0 # raise ... # :state = 2 # What would happen should we not raise # break :stateLoop -# STATE1: # Except +# of 1: # Except # yield 1 # :tmpResult = 3 # Return # :unrollFinally = true # Return @@ -117,21 +121,31 @@ # break :stateLoop # :state = 2 # What would happen should we not return # break :stateLoop -# STATE2: # Finally +# of 2: # Finally # yield 2 # if :unrollFinally: # This node is created by `newEndFinallyNode` # if :curExc.isNil: -# return :tmpResult +# if nearestFinally == 0: +# return :tmpResult +# else: +# :state = nearestFinally # bubble up # else: # closureIterSetupExc(nil) # raise # state = -1 # Goto next state. In this case we just exit # break :stateLoop +# else: +# return import ast, msgs, idents, renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos, - tables, options + options + +import std/tables + +when defined(nimPreviewSlimSystem): + import std/assertions type Ctx = object @@ -142,7 +156,7 @@ type unrollFinallySym: PSym # Indicates that we're unrolling finally states (either exception happened or premature return) curExcSym: PSym # Current exception - states: seq[PNode] # The resulting states. Every state is an nkState node. + states: seq[tuple[label: int, body: PNode]] # The resulting states. blockLevel: int # Temp used to transform break and continue stmts stateLoopLabel: PSym # Label to break on, when jumping between states. exitStateIdx: int # index of the last state @@ -158,6 +172,7 @@ type const nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt, nkCommentStmt, nkMixinStmt, nkBindStmt} + procDefs + emptyStateLabel = -1 proc newStateAccess(ctx: var Ctx): PNode = if ctx.stateVarSym.isNil: @@ -177,8 +192,9 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode = ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt))) proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym = - result = newSym(skVar, getIdent(ctx.g.cache, name), nextSymId(ctx.idgen), ctx.fn, ctx.fn.info) + result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.idgen, ctx.fn, ctx.fn.info) result.typ = typ + result.flags.incl sfNoInit assert(not typ.isNil) if not ctx.stateVarSym.isNil: @@ -190,7 +206,7 @@ proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym = else: let envParam = getEnvParam(ctx.fn) # let obj = envParam.typ.lastSon - result = addUniqueField(envParam.typ.lastSon, result, ctx.g.cache, ctx.idgen) + result = addUniqueField(envParam.typ.elementType, result, ctx.g.cache, ctx.idgen) proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode = if ctx.stateVarSym.isNil: @@ -200,7 +216,7 @@ proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode = proc newTmpResultAccess(ctx: var Ctx): PNode = if ctx.tmpResultSym.isNil: - ctx.tmpResultSym = ctx.newEnvVar(":tmpResult", ctx.fn.typ[0]) + ctx.tmpResultSym = ctx.newEnvVar(":tmpResult", ctx.fn.typ.returnType) ctx.newEnvVarAccess(ctx.tmpResultSym) proc newUnrollFinallyAccess(ctx: var Ctx, info: TLineInfo): PNode = @@ -220,10 +236,7 @@ proc newState(ctx: var Ctx, n, gotoOut: PNode): int = result = ctx.states.len let resLit = ctx.g.newIntLit(n.info, result) - let s = newNodeI(nkState, n.info) - s.add(resLit) - s.add(n) - ctx.states.add(s) + ctx.states.add((result, n)) ctx.exceptionTable.add(ctx.curExcHandlingState) if not gotoOut.isNil: @@ -252,10 +265,11 @@ proc hasYields(n: PNode): bool = of nkYieldStmt: result = true of nkSkip: - discard + result = false else: - for c in n: - if c.hasYields: + result = false + for i in ord(n.kind == nkCast).. 1: - var cond: PNode + var cond: PNode = nil for i in 0.. ", e + when false: + echo "TRANSFORM TO STATES: " + echo renderTree(result) + + echo "exception table:" + for i, e in ctx.exceptionTable: + echo i, " -> ", e diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim index c6a0f200ae..e512486395 100644 --- a/compiler/cmdlinehelper.nim +++ b/compiler/cmdlinehelper.nim @@ -7,11 +7,13 @@ # distribution, for details about the copyright. # -## Helpers for binaries that use compiler passes, e.g.: nim, nimsuggest, nimfix +## Helpers for binaries that use compiler passes, e.g.: nim, nimsuggest import options, idents, nimconf, extccomp, commands, msgs, - lineinfos, modulegraphs, condsyms, os, pathutils, parseopt + lineinfos, modulegraphs, condsyms, pathutils + +import std/[os, parseopt] proc prependCurDir*(f: AbsoluteFile): AbsoluteFile = when defined(unix): @@ -44,14 +46,7 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) = elif self.supportsStdinFile and conf.projectName == "-": handleStdinInput(conf) elif conf.projectName != "": - try: - conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName) - except OSError: - conf.projectFull = AbsoluteFile conf.projectName - let p = splitFile(conf.projectFull) - let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir - conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir) - conf.projectName = p.name + setFromProjectName(conf, conf.projectName) else: conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir()) @@ -62,6 +57,11 @@ proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: Confi if conf.cmd == cmdNimscript: incl(conf.globalOptions, optWasNimscript) loadConfigs(DefaultConfig, cache, conf, graph.idgen) # load all config files + # restores `conf.notes` after loading config files + # because it has overwrites the notes when compiling the system module which + # is a foreign module compared to the project + if conf.cmd in cmdBackends: + conf.notes = conf.mainPackageNotes if not self.suggestMode: let scriptFile = conf.projectFull.changeFileExt("nims") @@ -71,7 +71,8 @@ proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: Confi if conf.cmd == cmdNimscript: return false # now process command line arguments again, because some options in the # command line can overwrite the config file's settings - extccomp.initVars(conf) + if conf.backend != backendJs: # bug #19059 + extccomp.initVars(conf) self.processCmdLine(passCmd2, "", conf) if conf.cmd == cmdNone: rawMessage(conf, errGenerated, "command missing") diff --git a/compiler/commands.nim b/compiler/commands.nim index f36a4f5156..176b730447 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -9,7 +9,6 @@ # This module handles the parsing of command line arguments. - # We do this here before the 'import' statement so 'defined' does not get # confused with 'TGCMode.gcMarkAndSweep' etc. template bootSwitch(name, expr, userString) = @@ -25,18 +24,20 @@ bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep") bootSwitch(usedGoGC, defined(gogc), "--gc:go") bootSwitch(usedNoGC, defined(nogc), "--gc:none") +import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs] import - os, msgs, options, nversion, condsyms, strutils, extccomp, platform, - wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos, - pathutils, strtabs + msgs, options, nversion, condsyms, extccomp, platform, + wordrecg, nimblecmd, lineinfos, pathutils -from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect +import std/pathnorm + +from ast import setUseIc, eqTypeFlags, tfGcSafe, tfNoSideEffect + +when defined(nimPreviewSlimSystem): + import std/assertions # but some have deps to imported modules. Yay. bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc") -bootSwitch(usedNativeStacktrace, - defined(nativeStackTrace) and nativeStackTraceSupported, - "-d:nativeStackTrace") bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI") type @@ -101,7 +102,7 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) = msgWriteln(conf, "git hash: " & gitHash, {msgStdout}) msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger & - usedTinyC & useLinenoise & usedNativeStacktrace & + usedTinyC & useLinenoise & usedFFI & usedBoehm & usedMarkAndSweep & usedGoGC & usedNoGC, {msgStdout}) msgQuit(0) @@ -117,7 +118,7 @@ const errInvalidCmdLineOption = "invalid command line option: '$1'" errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found" errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found" - errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found" + errOffHintsError = "'off', 'hint', 'error' or 'usages' expected, but '$1' found" proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) = if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-") @@ -141,10 +142,19 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC elif switch[i] == '[': arg = substr(switch, i) else: invalidCmdLineOption(conf, pass, switch, info) +template switchOn(arg: string): bool = + # xxx use `switchOn` wherever appropriate + case arg.normalize + of "", "on": true + of "off": false + else: + localError(conf, info, errOnOrOffExpectedButXFound % arg) + false + proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = case arg.normalize - of "","on": conf.options.incl op + of "", "on": conf.options.incl op of "off": conf.options.excl op else: localError(conf, info, errOnOrOffExpectedButXFound % arg) @@ -176,7 +186,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, info: TLineInfo; orig: string; conf: ConfigRef) = var id = "" # arg = key or [key] or key:val or [key]:val; with val=on|off var i = 0 - var n = hintMin + var notes: set[TMsgKind] = {} var isBracket = false if i < arg.len and arg[i] == '[': isBracket = true @@ -191,37 +201,42 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, if i == arg.len: discard elif i < arg.len and (arg[i] in {':', '='}): inc(i) else: invalidCmdLineOption(conf, pass, orig, info) - # unfortunately, hintUser and warningUser clash - if state in {wHint, wHintAsError}: - let x = findStr(hintMin, hintMax, id, errUnknown) - if x != errUnknown: n = TNoteKind(x) - else: localError(conf, info, "unknown hint: " & id) - else: - let x = findStr(warnMin, warnMax, id, errUnknown) - if x != errUnknown: n = TNoteKind(x) - else: localError(conf, info, "unknown warning: " & id) + let isSomeHint = state in {wHint, wHintAsError} + template findNote(noteMin, noteMax, name) = + # unfortunately, hintUser and warningUser clash, otherwise implementation would simplify a bit + let x = findStr(noteMin, noteMax, id, errUnknown) + if x != errUnknown: notes = {TNoteKind(x)} + else: + if isSomeHint: + message(conf, info, hintUnknownHint, id) + else: + localError(conf, info, "unknown $#: $#" % [name, id]) + case id.normalize + of "all": # other note groups would be easy to support via additional cases + notes = if isSomeHint: {hintMin..hintMax} else: {warnMin..warnMax} + elif isSomeHint: findNote(hintMin, hintMax, "hint") + else: findNote(warnMin, warnMax, "warning") var val = substr(arg, i).normalize if val == "": val = "on" if val notin ["on", "off"]: + # xxx in future work we should also allow users to have control over `foreignPackageNotes` + # so that they can enable `hints|warnings|warningAsErrors` for all the code they depend on. localError(conf, info, errOnOrOffExpectedButXFound % arg) - elif n notin conf.cmdlineNotes or pass == passCmd1: - if pass == passCmd1: incl(conf.cmdlineNotes, n) - incl(conf.modifiedyNotes, n) - case val - of "on": - if state in {wWarningAsError, wHintAsError}: - incl(conf.warningAsErrors, n) # xxx rename warningAsErrors to noteAsErrors - else: - incl(conf.notes, n) - incl(conf.mainPackageNotes, n) - of "off": - if state in {wWarningAsError, wHintAsError}: - excl(conf.warningAsErrors, n) - else: - excl(conf.notes, n) - excl(conf.mainPackageNotes, n) - excl(conf.foreignPackageNotes, n) + else: + let isOn = val == "on" + if isOn and id.normalize == "all": + localError(conf, info, "only 'all:off' is supported") + for n in notes: + if n notin conf.cmdlineNotes or pass == passCmd1: + if pass == passCmd1: incl(conf.cmdlineNotes, n) + incl(conf.modifiedyNotes, n) + if state in {wWarningAsError, wHintAsError}: + conf.warningAsErrors[n] = isOn # xxx rename warningAsErrors to noteAsErrors + else: + conf.notes[n] = isOn + conf.mainPackageNotes[n] = isOn + if not isOn: excl(conf.foreignPackageNotes, n) proc processCompile(conf: ConfigRef; filename: string) = var found = findFile(conf, filename) @@ -229,10 +244,10 @@ proc processCompile(conf: ConfigRef; filename: string) = extccomp.addExternalFileToCompile(conf, found) const - errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found" + errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'atomicArc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found" errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found" - errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found" - errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found" + errGuiConsoleOrLibExpectedButXFound = "'gui', 'console', 'lib' or 'staticlib' expected, but '$1' found" + errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found" template warningOptionNoop(switch: string) = warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch) @@ -242,7 +257,7 @@ template deprecatedAlias(oldName, newName: string) = proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool = case switch.normalize - of "gc": + of "gc", "mm": case arg.normalize of "boehm": result = conf.selectedGC == gcBoehm of "refc": result = conf.selectedGC == gcRefc @@ -253,14 +268,18 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo of "go": result = conf.selectedGC == gcGo of "none": result = conf.selectedGC == gcNone of "stack", "regions": result = conf.selectedGC == gcRegions - of "v2", "generational": warningOptionNoop(arg) - else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) + of "atomicarc": result = conf.selectedGC == gcAtomicArc + else: + result = false + localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) of "opt": case arg.normalize of "speed": result = contains(conf.options, optOptimizeSpeed) of "size": result = contains(conf.options, optOptimizeSize) of "none": result = conf.options * {optOptimizeSpeed, optOptimizeSize} == {} - else: localError(conf, info, errNoneSpeedOrSizeExpectedButXFound % arg) + else: + result = false + localError(conf, info, errNoneSpeedOrSizeExpectedButXFound % arg) of "verbosity": result = $conf.verbosity == arg of "app": case arg.normalize @@ -270,7 +289,9 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo not contains(conf.globalOptions, optGenGuiApp) of "staticlib": result = contains(conf.globalOptions, optGenStaticLib) and not contains(conf.globalOptions, optGenGuiApp) - else: localError(conf, info, errGuiConsoleOrLibExpectedButXFound % arg) + else: + result = false + localError(conf, info, errGuiConsoleOrLibExpectedButXFound % arg) of "dynliboverride": result = isDynlibOverride(conf, arg) of "exceptions": @@ -279,8 +300,12 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo of "setjmp": result = conf.exc == excSetjmp of "quirky": result = conf.exc == excQuirky of "goto": result = conf.exc == excGoto - else: localError(conf, info, errInvalidExceptionSystem % arg) - else: invalidCmdLineOption(conf, passCmd1, switch, info) + else: + result = false + localError(conf, info, errInvalidExceptionSystem % arg) + else: + result = false + invalidCmdLineOption(conf, passCmd1, switch, info) proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool = case switch.normalize @@ -318,6 +343,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool of "run", "r": result = contains(conf.globalOptions, optRun) of "symbolfiles": result = conf.symbolFiles != disabledSf of "genscript": result = contains(conf.globalOptions, optGenScript) + of "gencdeps": result = contains(conf.globalOptions, optGenCDeps) of "threads": result = contains(conf.globalOptions, optThreads) of "tlsemulation": result = contains(conf.globalOptions, optTlsEmulation) of "implicitstatic": result = contains(conf.options, optImplicitStatic) @@ -325,8 +351,14 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool if switch.normalize == "patterns": deprecatedAlias(switch, "trmacros") result = contains(conf.options, optTrMacros) of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace) - of "nilseqs", "nilchecks", "taintmode": warningOptionNoop(switch) - else: invalidCmdLineOption(conf, passCmd1, switch, info) + of "nilseqs", "nilchecks", "taintmode": + warningOptionNoop(switch) + result = false + of "panics": result = contains(conf.globalOptions, optPanics) + of "jsbigint64": result = contains(conf.globalOptions, optJsBigInt64) + else: + result = false + invalidCmdLineOption(conf, passCmd1, switch, info) proc processPath(conf: ConfigRef; path: string, info: TLineInfo, notRelativeToProj = false): AbsoluteDir = @@ -359,31 +391,53 @@ proc processCfgPath(conf: ConfigRef; path: string, info: TLineInfo): AbsoluteDir const errInvalidNumber = "$1 is not a valid number" +proc makeAbsolute(s: string): AbsoluteFile = + if isAbsolute(s): + AbsoluteFile pathnorm.normalizePath(s) + else: + AbsoluteFile pathnorm.normalizePath(os.getCurrentDir() / s) + +proc setTrackingInfo(conf: ConfigRef; dirty, file, line, column: string, + info: TLineInfo) = + ## set tracking info, common code for track, trackDirty, & ideTrack + var ln: int = 0 + var col: int = 0 + if parseUtils.parseInt(line, ln) <= 0: + localError(conf, info, errInvalidNumber % line) + if parseUtils.parseInt(column, col) <= 0: + localError(conf, info, errInvalidNumber % column) + + let a = makeAbsolute(file) + if dirty == "": + conf.m.trackPos = newLineInfo(conf, a, ln, col) + else: + let dirtyOriginalIdx = fileInfoIdx(conf, a) + if dirtyOriginalIdx.int32 >= 0: + msgs.setDirtyFile(conf, dirtyOriginalIdx, makeAbsolute(dirty)) + conf.m.trackPos = newLineInfo(dirtyOriginalIdx, ln, col) + proc trackDirty(conf: ConfigRef; arg: string, info: TLineInfo) = var a = arg.split(',') if a.len != 4: localError(conf, info, "DIRTY_BUFFER,ORIGINAL_FILE,LINE,COLUMN expected") - var line, column: int - if parseUtils.parseInt(a[2], line) <= 0: - localError(conf, info, errInvalidNumber % a[1]) - if parseUtils.parseInt(a[3], column) <= 0: - localError(conf, info, errInvalidNumber % a[2]) - - let dirtyOriginalIdx = fileInfoIdx(conf, AbsoluteFile a[1]) - if dirtyOriginalIdx.int32 >= 0: - msgs.setDirtyFile(conf, dirtyOriginalIdx, AbsoluteFile a[0]) - - conf.m.trackPos = newLineInfo(dirtyOriginalIdx, line, column) + setTrackingInfo(conf, a[0], a[1], a[2], a[3], info) proc track(conf: ConfigRef; arg: string, info: TLineInfo) = var a = arg.split(',') if a.len != 3: localError(conf, info, "FILE,LINE,COLUMN expected") - var line, column: int - if parseUtils.parseInt(a[1], line) <= 0: - localError(conf, info, errInvalidNumber % a[1]) - if parseUtils.parseInt(a[2], column) <= 0: - localError(conf, info, errInvalidNumber % a[2]) - conf.m.trackPos = newLineInfo(conf, AbsoluteFile a[0], line, column) + setTrackingInfo(conf, "", a[0], a[1], a[2], info) + +proc trackIde(conf: ConfigRef; cmd: IdeCmd, arg: string, info: TLineInfo) = + ## set the tracking info related to an ide cmd, supports optional dirty file + var a = arg.split(',') + case a.len + of 4: + setTrackingInfo(conf, a[0], a[1], a[2], a[3], info) + of 3: + setTrackingInfo(conf, "", a[0], a[1], a[2], info) + else: + localError(conf, info, "[DIRTY_BUFFER,]ORIGINAL_FILE,LINE,COLUMN expected") + conf.ideCmd = cmd proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = if pass in {passCmd2, passPP}: @@ -408,16 +462,21 @@ proc handleCmdInput*(conf: ConfigRef) = proc parseCommand*(command: string): Command = case command.normalize of "c", "cc", "compile", "compiletoc": cmdCompileToC + of "nir": cmdCompileToNir of "cpp", "compiletocpp": cmdCompileToCpp of "objc", "compiletooc": cmdCompileToOC of "js", "compiletojs": cmdCompileToJS of "r": cmdCrun + of "m": cmdM of "run": cmdTcc of "check": cmdCheck of "e": cmdNimscript of "doc0": cmdDoc0 - of "doc2", "doc": cmdDoc2 + of "doc2", "doc": cmdDoc + of "doc2tex": cmdDoc2tex of "rst2html": cmdRst2html + of "md2tex": cmdMd2tex + of "md2html": cmdMd2html of "rst2tex": cmdRst2tex of "jsondoc0": cmdJsondoc0 of "jsondoc2", "jsondoc": cmdJsondoc @@ -441,6 +500,7 @@ proc setCmd*(conf: ConfigRef, cmd: Command) = of cmdCompileToCpp: conf.backend = backendCpp of cmdCompileToOC: conf.backend = backendObjc of cmdCompileToJS: conf.backend = backendJs + of cmdCompileToNir: conf.backend = backendNir else: discard proc setCommandEarly*(conf: ConfigRef, command: string) = @@ -449,15 +509,121 @@ proc setCommandEarly*(conf: ConfigRef, command: string) = # command early customizations # must be handled here to honor subsequent `--hint:x:on|off` case conf.cmd - of cmdRst2html, cmdRst2tex: # xxx see whether to add others: cmdGendepend, etc. + of cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex: + # xxx see whether to add others: cmdGendepend, etc. conf.foreignPackageNotes = {hintSuccessX} else: conf.foreignPackageNotes = foreignPackageNotesDefault +proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) = + # Keep this syncronized with the default config/nim.cfg! + if cmpIgnoreStyle(key, "nimQuirky") == 0: + conf.exc = excQuirky + elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0: + if pass in {passCmd1, passPP}: + conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize} + conf.globalOptions.excl {optExcessiveStackTrace, optCDebug} + conf.options.incl optOptimizeSpeed + if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0: + if pass in {passCmd1, passPP}: + conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, + optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} + conf.globalOptions.excl {optCDebug} + +proc initOrcDefines*(conf: ConfigRef) = + conf.selectedGC = gcOrc + defineSymbol(conf.symbols, "gcorc") + defineSymbol(conf.symbols, "gcdestructors") + incl conf.globalOptions, optSeqDestructors + incl conf.globalOptions, optTinyRtti + defineSymbol(conf.symbols, "nimSeqsV2") + defineSymbol(conf.symbols, "nimV2") + if conf.exc == excNone and conf.backend != backendCpp: + conf.exc = excGoto + +proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef) = + defineSymbol(conf.symbols, "gcdestructors") + incl conf.globalOptions, optSeqDestructors + incl conf.globalOptions, optTinyRtti + if pass in {passCmd2, passPP}: + defineSymbol(conf.symbols, "nimSeqsV2") + defineSymbol(conf.symbols, "nimV2") + if conf.exc == excNone and conf.backend != backendCpp: + conf.exc = excGoto + +proc unregisterArcOrc*(conf: ConfigRef) = + undefSymbol(conf.symbols, "gcdestructors") + undefSymbol(conf.symbols, "gcarc") + undefSymbol(conf.symbols, "gcorc") + undefSymbol(conf.symbols, "gcatomicarc") + undefSymbol(conf.symbols, "nimSeqsV2") + undefSymbol(conf.symbols, "nimV2") + excl conf.globalOptions, optSeqDestructors + excl conf.globalOptions, optTinyRtti + +proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass, + info: TLineInfo; conf: ConfigRef) = + if conf.backend == backendJs: return # for: bug #16033 + expectArg(conf, switch, arg, pass, info) + if pass in {passCmd2, passPP}: + case arg.normalize + of "boehm": + unregisterArcOrc(conf) + conf.selectedGC = gcBoehm + defineSymbol(conf.symbols, "boehmgc") + incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS + of "refc": + unregisterArcOrc(conf) + defineSymbol(conf.symbols, "gcrefc") + conf.selectedGC = gcRefc + of "markandsweep": + unregisterArcOrc(conf) + conf.selectedGC = gcMarkAndSweep + defineSymbol(conf.symbols, "gcmarkandsweep") + of "destructors", "arc": + conf.selectedGC = gcArc + defineSymbol(conf.symbols, "gcarc") + registerArcOrc(pass, conf) + of "orc": + conf.selectedGC = gcOrc + defineSymbol(conf.symbols, "gcorc") + registerArcOrc(pass, conf) + of "atomicarc": + conf.selectedGC = gcAtomicArc + defineSymbol(conf.symbols, "gcatomicarc") + registerArcOrc(pass, conf) + of "hooks": + conf.selectedGC = gcHooks + defineSymbol(conf.symbols, "gchooks") + incl conf.globalOptions, optSeqDestructors + processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info) + if pass in {passCmd2, passPP}: + defineSymbol(conf.symbols, "nimSeqsV2") + of "go": + unregisterArcOrc(conf) + conf.selectedGC = gcGo + defineSymbol(conf.symbols, "gogc") + of "none": + unregisterArcOrc(conf) + conf.selectedGC = gcNone + defineSymbol(conf.symbols, "nogc") + of "stack", "regions": + unregisterArcOrc(conf) + conf.selectedGC = gcRegions + defineSymbol(conf.symbols, "gcregions") + else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) + +proc pathRelativeToConfig(arg: string, pass: TCmdLinePass, conf: ConfigRef): string = + if pass == passPP and not isAbsolute(arg): + assert isAbsolute(conf.currentConfigDir), "something is wrong with currentConfigDir" + result = conf.currentConfigDir / arg + else: + result = arg + proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = - var - key, val: string + var key = "" + var val = "" case switch.normalize of "eval": expectArg(conf, switch, arg, pass, info) @@ -472,17 +638,17 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; for path in nimbleSubs(conf, arg): addPath(conf, if pass == passPP: processCfgPath(conf, path, info) else: processPath(conf, path, info), info) - of "nimblepath", "babelpath": - if switch.normalize == "babelpath": deprecatedAlias(switch, "nimblepath") + of "nimblepath": if pass in {passCmd2, passPP} and optNoNimblePath notin conf.globalOptions: expectArg(conf, switch, arg, pass, info) var path = processPath(conf, arg, info, notRelativeToProj=true) let nimbleDir = AbsoluteDir getEnv("NIMBLE_DIR") if not nimbleDir.isEmpty and pass == passPP: + path = nimbleDir / RelativeDir"pkgs2" + nimblePath(conf, path, info) path = nimbleDir / RelativeDir"pkgs" nimblePath(conf, path, info) - of "nonimblepath", "nobabelpath": - if switch.normalize == "nobabelpath": deprecatedAlias(switch, "nonimblepath") + of "nonimblepath": expectNoArg(conf, switch, arg, pass, info) disableNimblePath(conf) of "clearnimblepath": @@ -495,7 +661,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.lazyPaths.keepItIf(it != path) of "nimcache": expectArg(conf, switch, arg, pass, info) - conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true) + var arg = arg + # refs bug #18674, otherwise `--os:windows` messes up with `--nimcache` set + # in config nims files, e.g. via: `import os; switch("nimcache", "/tmp/somedir")` + if conf.target.targetOS == osWindows and DirSep == '/': arg = arg.replace('\\', '/') + conf.nimcacheDir = processPath(conf, pathRelativeToConfig(arg, pass, conf), info, notRelativeToProj=true) of "out", "o": expectArg(conf, switch, arg, pass, info) let f = splitFile(processPath(conf, arg, info, notRelativeToProj=true).string) @@ -514,18 +684,21 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "backend", "b": let backend = parseEnum(arg.normalize, TBackend.default) if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg) + if backend == backendJs: # bug #21209 + conf.globalOptions.excl {optThreadAnalysis, optThreads} + if optRun in conf.globalOptions: + # for now, -r uses nodejs, so define nodejs + defineSymbol(conf.symbols, "nodejs") conf.backend = backend of "doccmd": conf.docCmd = arg of "define", "d": expectArg(conf, switch, arg, pass, info) if {':', '='} in arg: splitSwitch(conf, arg, key, val, pass, info) - if cmpIgnoreStyle(key, "nimQuirky") == 0: - conf.exc = excQuirky + specialDefine(conf, key, pass) defineSymbol(conf.symbols, key, val) else: - if cmpIgnoreStyle(arg, "nimQuirky") == 0: - conf.exc = excQuirky + specialDefine(conf, arg, pass) defineSymbol(conf.symbols, arg) of "undef", "u": expectArg(conf, switch, arg, pass, info) @@ -552,59 +725,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "project": processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info) of "gc": - if conf.backend == backendJs: return # for: bug #16033 - expectArg(conf, switch, arg, pass, info) - if pass in {passCmd2, passPP}: - case arg.normalize - of "boehm": - conf.selectedGC = gcBoehm - defineSymbol(conf.symbols, "boehmgc") - incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS - of "refc": - conf.selectedGC = gcRefc - of "markandsweep": - conf.selectedGC = gcMarkAndSweep - defineSymbol(conf.symbols, "gcmarkandsweep") - of "destructors", "arc": - conf.selectedGC = gcArc - defineSymbol(conf.symbols, "gcdestructors") - defineSymbol(conf.symbols, "gcarc") - incl conf.globalOptions, optSeqDestructors - incl conf.globalOptions, optTinyRtti - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - defineSymbol(conf.symbols, "nimV2") - if conf.exc == excNone and conf.backend != backendCpp: - conf.exc = excGoto - of "orc": - conf.selectedGC = gcOrc - defineSymbol(conf.symbols, "gcdestructors") - defineSymbol(conf.symbols, "gcorc") - incl conf.globalOptions, optSeqDestructors - incl conf.globalOptions, optTinyRtti - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - defineSymbol(conf.symbols, "nimV2") - if conf.exc == excNone and conf.backend != backendCpp: - conf.exc = excGoto - of "hooks": - conf.selectedGC = gcHooks - defineSymbol(conf.symbols, "gchooks") - incl conf.globalOptions, optSeqDestructors - processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info) - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - of "go": - conf.selectedGC = gcGo - defineSymbol(conf.symbols, "gogc") - of "none": - conf.selectedGC = gcNone - defineSymbol(conf.symbols, "nogc") - of "stack", "regions": - conf.selectedGC = gcRegions - defineSymbol(conf.symbols, "gcregions") - of "v2": warningOptionNoop(arg) - else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) + warningDeprecated(conf, info, "`gc:option` is deprecated; use `mm:option` instead") + processMemoryManagementOption(switch, arg, pass, info, conf) + of "mm": + processMemoryManagementOption(switch, arg, pass, info, conf) of "warnings", "w": if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf) of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf) @@ -672,10 +796,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info) of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info) of "threads": - if conf.backend == backendJs: discard + if conf.backend == backendJs or conf.cmd == cmdNimscript: discard else: processOnOffSwitchG(conf, {optThreads}, arg, pass, info) #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe) - of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info) + of "tlsemulation": + processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info) + if optTlsEmulation in conf.globalOptions: + conf.legacyFeatures.incl emitGenerics of "implicitstatic": processOnOffSwitch(conf, {optImplicitStatic}, arg, pass, info) of "patterns", "trmacros": @@ -712,6 +839,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; defineSymbol(conf.symbols, "dll") of "staticlib": incl(conf.globalOptions, optGenStaticLib) + incl(conf.globalOptions, optNoMain) excl(conf.globalOptions, optGenGuiApp) defineSymbol(conf.symbols, "library") defineSymbol(conf.symbols, "staticlib") @@ -731,12 +859,20 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "clib": expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: - conf.cLinkedLibs.add processPath(conf, arg, info).string + conf.cLinkedLibs.add arg of "header": if conf != nil: conf.headerFile = arg incl(conf.globalOptions, optGenIndex) + of "nimbasepattern": + if conf != nil: conf.nimbasePattern = arg of "index": - processOnOffSwitchG(conf, {optGenIndex}, arg, pass, info) + case arg.normalize + of "", "on": conf.globalOptions.incl {optGenIndex} + of "only": conf.globalOptions.incl {optGenIndexOnly, optGenIndex} + of "off": conf.globalOptions.excl {optGenIndex, optGenIndexOnly} + else: localError(conf, info, errOnOrOffExpectedButXFound % arg) + of "noimportdoc": + processOnOffSwitchG(conf, {optNoImportdoc}, arg, pass, info) of "import": expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: @@ -769,21 +905,28 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; setTarget(conf.target, conf.target.targetOS, cpu) of "run", "r": processOnOffSwitchG(conf, {optRun}, arg, pass, info) + if conf.backend == backendJs: + # for now, -r uses nodejs, so define nodejs + defineSymbol(conf.symbols, "nodejs") of "maxloopiterationsvm": expectArg(conf, switch, arg, pass, info) - conf.maxLoopIterationsVM = parseInt(arg) + var value: int = 10_000_000 + discard parseSaturatedNatural(arg, value) + if not value > 0: localError(conf, info, "maxLoopIterationsVM must be a positive integer greater than zero") + conf.maxLoopIterationsVM = value of "errormax": expectArg(conf, switch, arg, pass, info) # Note: `nim check` (etc) can overwrite this. # `0` is meaningless, give it a useful meaning as in clang's -ferror-limit # If user doesn't set this flag and the code doesn't either, it'd # have the same effect as errorMax = 1 - let ret = parseInt(arg) - conf.errorMax = if ret == 0: high(int) else: ret + var value: int = 0 + discard parseSaturatedNatural(arg, value) + conf.errorMax = if value == 0: high(int) else: value of "verbosity": expectArg(conf, switch, arg, pass, info) let verbosity = parseInt(arg) - if verbosity notin {0..3}: + if verbosity notin 0..3: localError(conf, info, "invalid verbosity level: '$1'" % arg) conf.verbosity = verbosity var verb = NotesVerbosity[conf.verbosity] @@ -793,7 +936,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.mainPackageNotes = conf.notes of "parallelbuild": expectArg(conf, switch, arg, pass, info) - conf.numberOfProcessors = parseInt(arg) + var value: int = 0 + discard parseSaturatedNatural(arg, value) + conf.numberOfProcessors = value of "version", "v": expectNoArg(conf, switch, arg, pass, info) writeVersionInfo(conf, pass) @@ -818,6 +963,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "v2": conf.symbolFiles = v2Sf of "stress": conf.symbolFiles = stressTest else: localError(conf, info, "invalid option for --incremental: " & arg) + setUseIc(conf.symbolFiles != disabledSf) of "skipcfg": processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info) of "skipprojcfg": @@ -830,6 +976,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if switch.normalize == "gendeps": deprecatedAlias(switch, "genscript") processOnOffSwitchG(conf, {optGenScript}, arg, pass, info) processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info) + of "gencdeps": + processOnOffSwitchG(conf, {optGenCDeps}, arg, pass, info) of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info) of "lib": expectArg(conf, switch, arg, pass, info) @@ -839,8 +987,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; splitSwitch(conf, arg, key, val, pass, info) os.putEnv(key, val) of "cc": - expectArg(conf, switch, arg, pass, info) - setCC(conf, arg, info) + if conf.backend != backendJs: # bug #19330 + expectArg(conf, switch, arg, pass, info) + setCC(conf, arg, info) of "track": expectArg(conf, switch, arg, pass, info) track(conf, arg, info) @@ -851,18 +1000,40 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) conf.ideCmd = ideSug of "def": - expectNoArg(conf, switch, arg, pass, info) - conf.ideCmd = ideDef + expectArg(conf, switch, arg, pass, info) + trackIde(conf, ideDef, arg, info) of "context": expectNoArg(conf, switch, arg, pass, info) conf.ideCmd = ideCon of "usages": - expectNoArg(conf, switch, arg, pass, info) - conf.ideCmd = ideUse + expectArg(conf, switch, arg, pass, info) + trackIde(conf, ideUse, arg, info) + of "defusages": + expectArg(conf, switch, arg, pass, info) + trackIde(conf, ideDus, arg, info) of "stdout": processOnOffSwitchG(conf, {optStdout}, arg, pass, info) + of "filenames": + case arg.normalize + of "abs": conf.filenameOption = foAbs + of "canonical": conf.filenameOption = foCanonical + of "legacyrelproj": conf.filenameOption = foLegacyRelProj + else: localError(conf, info, "expected: abs|canonical|legacyRelProj, got: $1" % arg) + of "processing": + incl(conf.notes, hintProcessing) + incl(conf.mainPackageNotes, hintProcessing) + case arg.normalize + of "dots": conf.hintProcessingDots = true + of "filenames": conf.hintProcessingDots = false + of "off": + excl(conf.notes, hintProcessing) + excl(conf.mainPackageNotes, hintProcessing) + else: localError(conf, info, "expected: dots|filenames|off, got: $1" % arg) + of "unitsep": + conf.unitSep = if switchOn(arg): "\31" else: "" of "listfullpaths": - processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info) + # xxx in future work, use `warningDeprecated` + conf.filenameOption = if switchOn(arg): foAbs else: foCanonical of "spellsuggest": if arg.len == 0: conf.spellSuggestMax = spellSuggestSecretSauce elif arg == "auto": conf.spellSuggestMax = spellSuggestSecretSauce @@ -890,6 +1061,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) conf.exc = low(ExceptionSystem) defineSymbol(conf.symbols, "noCppExceptions") + of "shownonexports": + expectNoArg(conf, switch, arg, pass, info) + showNonExportedFields(conf) of "exceptions": case arg.normalize of "cpp": conf.exc = excCpp @@ -924,6 +1098,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError} of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint} - {optStyleError} of "error": conf.globalOptions = conf.globalOptions + {optStyleError} + of "usages": conf.globalOptions.incl optStyleUsages else: localError(conf, info, errOffHintsError % arg) of "showallmismatches": processOnOffSwitchG(conf, {optShowAllMismatches}, arg, pass, info) @@ -943,25 +1118,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "expandarc": expectArg(conf, switch, arg, pass, info) conf.arcToExpand[arg] = "T" - of "useversion": - expectArg(conf, switch, arg, pass, info) - case arg - of "1.0": - defineSymbol(conf.symbols, "NimMajor", "1") - defineSymbol(conf.symbols, "NimMinor", "0") - # old behaviors go here: - defineSymbol(conf.symbols, "nimOldRelativePathBehavior") - undefSymbol(conf.symbols, "nimDoesntTrackDefects") - ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect} - conf.globalOptions.incl optNimV1Emulation - of "1.2": - defineSymbol(conf.symbols, "NimMajor", "1") - defineSymbol(conf.symbols, "NimMinor", "2") - conf.globalOptions.incl optNimV12Emulation - else: - localError(conf, info, "unknown Nim version; currently supported values are: `1.0`, `1.2`") - # always be compatible with 1.x.100: - defineSymbol(conf.symbols, "NimPatch", "100") of "benchmarkvm": processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info) of "profilevm": @@ -975,6 +1131,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optPanics}, arg, pass, info) if optPanics in conf.globalOptions: defineSymbol(conf.symbols, "nimPanics") + of "jsbigint64": + processOnOffSwitchG(conf, {optJsBigInt64}, arg, pass, info) of "sourcemap": # xxx document in --fullhelp conf.globalOptions.incl optSourcemap conf.options.incl optLineDir @@ -982,13 +1140,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info) of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -) handleStdinInput(conf) - of "nilseqs", "nilchecks", "mainmodule", "m", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch) + of "nilseqs", "nilchecks", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch) + of "nimmainprefix": conf.nimMainPrefix = arg else: if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg) else: invalidCmdLineOption(conf, pass, switch, info) proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) = - var cmd, arg: string + var cmd = "" + var arg = "" splitSwitch(config, switch, cmd, arg, pass, gCmdLineInfo) processSwitch(cmd, arg, pass, gCmdLineInfo, config) @@ -1015,13 +1175,20 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; config.projectName = unixToNativePath(p.key) config.arguments = cmdLineRest(p) result = true - elif pass != passCmd2: setCommandEarly(config, p.key) + elif pass != passCmd2: + setCommandEarly(config, p.key) + result = false + else: result = false else: if pass == passCmd1: config.commandArgs.add p.key if argsCount == 1: + if p.key.endsWith(".nims"): + incl(config.globalOptions, optWasNimscript) # support UNIX style filenames everywhere for portable build scripts: if config.projectName.len == 0: config.projectName = unixToNativePath(p.key) config.arguments = cmdLineRest(p) result = true + else: + result = false inc argsCount diff --git a/compiler/compiler.nimble b/compiler/compiler.nimble new file mode 100644 index 0000000000..ef3f343e11 --- /dev/null +++ b/compiler/compiler.nimble @@ -0,0 +1,28 @@ +include "../lib/system/compilation.nim" +version = $NimMajor & "." & $NimMinor & "." & $NimPatch +author = "Andreas Rumpf" +description = "Compiler package providing the compiler sources as a library." +license = "MIT" +skipDirs = @["."] +installDirs = @["compiler"] + +import os + +var compilerDir = "" + +before install: + rmDir("compiler") + + let + files = listFiles(".") + dirs = listDirs(".") + + mkDir("compiler") + + for f in files: + cpFile(f, "compiler" / f) + + for d in dirs: + cpDir(d, "compiler" / d) + +requires "nim" diff --git a/compiler/concepts.nim b/compiler/concepts.nim index 54579f73f6..d48bacdc52 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -11,10 +11,12 @@ ## for details. Note this is a first implementation and only the "Concept matching" ## section has been implemented. -import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, - types, intsets +import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types -from magicsys import addSonSkipIntLit +import std/intsets + +when defined(nimPreviewSlimSystem): + import std/assertions const logBindings = false @@ -23,30 +25,18 @@ const ## -------------------------------------- proc declareSelf(c: PContext; info: TLineInfo) = - ## adds the magical 'Self' symbols to the current scope. + ## Adds the magical 'Self' symbols to the current scope. let ow = getCurrOwner(c) - let s = newSym(skType, getIdent(c.cache, "Self"), nextSymId(c.idgen), ow, info) - s.typ = newType(tyTypeDesc, nextTypeId(c.idgen), ow) + let s = newSym(skType, getIdent(c.cache, "Self"), c.idgen, ow, info) + s.typ = newType(tyTypeDesc, c.idgen, ow) s.typ.flags.incl {tfUnresolved, tfPacked} - s.typ.add newType(tyEmpty, nextTypeId(c.idgen), ow) + s.typ.add newType(tyEmpty, c.idgen, ow) addDecl(c, s, info) -proc isSelf*(t: PType): bool {.inline.} = - ## is this the magical 'Self' type? - t.kind == tyTypeDesc and tfPacked in t.flags - -proc makeTypeDesc*(c: PContext, typ: PType): PType = - if typ.kind == tyTypeDesc and not isSelf(typ): - result = typ - else: - result = newTypeS(tyTypeDesc, c) - incl result.flags, tfCheckedForDestructor - result.addSonSkipIntLit(typ, c.idgen) - proc semConceptDecl(c: PContext; n: PNode): PNode = ## Recursive helper for semantic checking for the concept declaration. - ## Currently we only support lists of statements containing 'proc' - ## declarations and the like. + ## Currently we only support (possibly empty) lists of statements + ## containing 'proc' declarations and the like. case n.kind of nkStmtList, nkStmtListExpr: result = shallowCopy(n) @@ -59,8 +49,10 @@ proc semConceptDecl(c: PContext; n: PNode): PNode = for i in 0.. 0 and f[0].kind != tyNone: + if f.hasElementType and f.elementType.kind != tyNone: # also check the generic's constraints: let oldLen = m.inferred.len - result = matchType(c, f[0], a, m) + result = matchType(c, f.elementType, a, m) m.inferred.setLen oldLen if result: when logBindings: echo "A adding ", f, " ", ak m.inferred.add((f, ak)) - elif m.magic == mArrGet and ak.kind in {tyArray, tyOpenArray, tySequence, tyVarargs, tyCString, tyString}: + elif m.magic == mArrGet and ak.kind in {tyArray, tyOpenArray, tySequence, tyVarargs, tyCstring, tyString}: when logBindings: echo "B adding ", f, " ", lastSon ak - m.inferred.add((f, lastSon ak)) + m.inferred.add((f, last ak)) result = true else: when logBindings: echo "C adding ", f, " ", ak @@ -152,25 +148,27 @@ proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool = result = matchType(c, old, ak, m) if m.magic == mArrPut and ak.kind == tyGenericParam: result = true + else: + result = false #echo "B for ", result, " to ", typeToString(a), " to ", typeToString(m.potentialImplementation) of tyVar, tySink, tyLent, tyOwned: # modifiers in the concept must be there in the actual implementation # too but not vice versa. if a.kind == f.kind: - result = matchType(c, f.sons[0], a.sons[0], m) + result = matchType(c, f.elementType, a.elementType, m) elif m.magic == mArrPut: - result = matchType(c, f.sons[0], a, m) + result = matchType(c, f.elementType, a, m) else: result = false of tyEnum, tyObject, tyDistinct: result = sameType(f, a) - of tyEmpty, tyString, tyCString, tyPointer, tyNil, tyUntyped, tyTyped, tyVoid: + of tyEmpty, tyString, tyCstring, tyPointer, tyNil, tyUntyped, tyTyped, tyVoid: result = a.skipTypes(ignorableForArgType).kind == f.kind of tyBool, tyChar, tyInt..tyUInt64: let ak = a.skipTypes(ignorableForArgType) result = ak.kind == f.kind or ak.kind == tyOrdinal or - (ak.kind == tyGenericParam and ak.len > 0 and ak[0].kind == tyOrdinal) + (ak.kind == tyGenericParam and ak.hasElementType and ak.elementType.kind == tyOrdinal) of tyConcept: let oldLen = m.inferred.len let oldPotentialImplementation = m.potentialImplementation @@ -181,9 +179,11 @@ proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool = m.inferred.setLen oldLen of tyArray, tyTuple, tyVarargs, tyOpenArray, tyRange, tySequence, tyRef, tyPtr, tyGenericInst: + # ^ XXX Rewrite this logic, it's more complex than it needs to be. + result = false let ak = a.skipTypes(ignorableForArgType - {f.kind}) - if ak.kind == f.kind and f.len == ak.len: - for i in 0..= a.len + result = covered >= a.kidsLen if not result: m.inferred.setLen oldLen else: - for i in 0.. 0.20.0 - defineSymbol("nimNoZeroExtendMagic") - defineSymbol("nimMacrosGetNodeId") for f in Feature: defineSymbol("nimHas" & $f) @@ -98,31 +110,18 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimFixedOwned") defineSymbol("nimHasStyleChecks") - defineSymbol("nimToOpenArrayCString") - defineSymbol("nimHasUsed") - defineSymbol("nimFixedForwardGeneric") - defineSymbol("nimnomagic64") - defineSymbol("nimNewShiftOps") - defineSymbol("nimHasCursor") - defineSymbol("nimAlignPragma") - defineSymbol("nimHasExceptionsQuery") - defineSymbol("nimHasIsNamedTuple") - defineSymbol("nimHashOrdinalFixed") when defined(nimHasLibFFI): # Renaming as we can't conflate input vs output define flags; e.g. this # will report the right thing regardless of whether user adds # `-d:nimHasLibFFI` in his user config. - defineSymbol("nimHasLibFFIEnabled") + defineSymbol("nimHasLibFFIEnabled") # deadcode - defineSymbol("nimHasSinkInference") - defineSymbol("nimNewIntegerOps") - defineSymbol("nimHasInvariant") - defineSymbol("nimHasStacktraceMsgs") + defineSymbol("nimHasStacktraceMsgs") # deadcode defineSymbol("nimDoesntTrackDefects") - defineSymbol("nimHasLentIterators") - defineSymbol("nimHasDeclaredMagic") - defineSymbol("nimHasStacktracesModule") + defineSymbol("nimHasLentIterators") # deadcode + defineSymbol("nimHasDeclaredMagic") # deadcode + defineSymbol("nimHasStacktracesModule") # deadcode defineSymbol("nimHasEffectTraitsModule") defineSymbol("nimHasCastPragmaBlocks") defineSymbol("nimHasDeclaredLocs") @@ -130,3 +129,40 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasWarningAsError") defineSymbol("nimHasHintAsError") defineSymbol("nimHasSpellSuggest") + defineSymbol("nimHasCustomLiterals") + defineSymbol("nimHasUnifiedTuple") + defineSymbol("nimHasIterable") + defineSymbol("nimHasTypeofVoid") # deadcode + defineSymbol("nimHasDragonBox") # deadcode + defineSymbol("nimHasHintAll") + defineSymbol("nimHasTrace") + defineSymbol("nimHasEffectsOf") + + defineSymbol("nimHasEnforceNoRaises") + defineSymbol("nimHasTopDownInference") + defineSymbol("nimHasTemplateRedefinitionPragma") + defineSymbol("nimHasCstringCase") + defineSymbol("nimHasCallsitePragma") + + defineSymbol("nimHasWarnCastSizes") # deadcode + defineSymbol("nimHasOutParams") + defineSymbol("nimHasSystemRaisesDefect") + defineSymbol("nimHasWarnUnnamedBreak") + defineSymbol("nimHasGenericDefine") + defineSymbol("nimHasDefineAliases") + defineSymbol("nimHasWarnBareExcept") + defineSymbol("nimHasDup") + defineSymbol("nimHasChecksums") + defineSymbol("nimHasSendable") + defineSymbol("nimAllowNonVarDestructor") + defineSymbol("nimHasQuirky") + defineSymbol("nimHasEnsureMove") + defineSymbol("nimHasNoReturnError") + + defineSymbol("nimUseStrictDefs") + defineSymbol("nimHasNolineTooLong") + + defineSymbol("nimHasCastExtendedVm") + defineSymbol("nimHasWarnStdPrefix") + + defineSymbol("nimHasVtables") diff --git a/compiler/debugutils.nim b/compiler/debugutils.nim new file mode 100644 index 0000000000..adbb0517fb --- /dev/null +++ b/compiler/debugutils.nim @@ -0,0 +1,72 @@ +##[ +Utilities to help with debugging nim compiler. + +Experimental API, subject to change. +]## + +#[ +## example +useful debugging flags: +--stacktrace -d:debug -d:nimDebugUtils + nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim + +## future work +* expose and improve astalgo.debug, replacing it by std/prettyprints, + refs https://github.com/nim-lang/RFCs/issues/385 +]# + +import options +import std/wrapnils +export wrapnils + # allows using things like: `?.n.sym.typ.len` + +import std/stackframes +export stackframes + # allows using things like: `setFrameMsg c.config$n.info & " " & $n.kind` + # which doesn't log, but augments stacktrace with side channel information + +var conf0: ConfigRef + +proc onNewConfigRef*(conf: ConfigRef) {.inline.} = + ## Caches `conf`, which can be retrieved with `getConfigRef`. + ## This avoids having to forward `conf` all the way down the call chain to + ## procs that need it during a debugging session. + conf0 = conf + +proc getConfigRef*(): ConfigRef = + ## nil, if -d:nimDebugUtils wasn't specified + result = conf0 + +proc isCompilerDebug*(): bool = + ##[ + Provides a simple way for user code to enable/disable logging in the compiler + in a granular way. This can then be used in the compiler as follows: + ```nim + if isCompilerDebug(): + echo ?.n.sym.typ.len + ``` + ]## + runnableExamples: + proc main = + echo 2 + {.define(nimCompilerDebug).} + echo 3.5 # code section in which `isCompilerDebug` will be true + {.undef(nimCompilerDebug).} + echo 'x' + conf0.isDefined("nimCompilerDebug") + +proc enteringDebugSection*() {.exportc, dynlib.} = + ## Provides a way for native debuggers to enable breakpoints, watchpoints, etc + ## when code of interest is being compiled. + ## + ## Set your debugger to break on entering `nimCompilerIsEnteringDebugSection` + ## and then execute a desired command. + discard + +proc exitingDebugSection*() {.exportc, dynlib.} = + ## Provides a way for native debuggers to disable breakpoints, watchpoints, etc + ## when code of interest is no longer being compiled. + ## + ## Set your debugger to break on entering `exitingDebugSection` + ## and then execute a desired command. + discard diff --git a/compiler/depends.nim b/compiler/depends.nim index 7225b6b478..638f1eb51a 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -9,10 +9,17 @@ # This module implements a dependency file generator. -import - options, ast, ropes, idents, passes, modulepaths, pathutils +import options, ast, ropes, pathutils, msgs, lineinfos + +import modulegraphs + +import std/[os, parseutils] +import std/strutils except addf +import std/private/globs + +when defined(nimPreviewSlimSystem): + import std/assertions -from modulegraphs import ModuleGraph, PPassContext type TGen = object of PPassContext @@ -25,21 +32,67 @@ type dotGraph: Rope proc addDependencyAux(b: Backend; importing, imported: string) = - b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)]) + b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)]) # s1 -> s2_4[label="[0-9]"]; -proc addDotDependency(c: PPassContext, n: PNode): PNode = +proc toNimblePath(s: string, isStdlib: bool): string = + const stdPrefix = "std/" + const pkgPrefix = "pkg/" + if isStdlib: + let sub = "lib/" + var start = s.find(sub) + if start < 0: + raiseAssert "unreachable" + else: + start += sub.len + let base = s[start..^1] + + if base.startsWith("system") or base.startsWith("std"): + result = base + else: + for dir in stdlibDirs: + if base.startsWith(dir): + return stdPrefix & base.splitFile.name + + result = stdPrefix & base + else: + var sub = getEnv("NIMBLE_DIR") + if sub.len == 0: + sub = ".nimble/pkgs/" + else: + sub.add "/pkgs/" + var start = s.find(sub) + if start < 0: + sub[^1] = '2' + sub.add '/' + start = s.find(sub) # /pkgs2 + if start < 0: + return s + + start += sub.len + start += skipUntil(s, '/', start) + start += 1 + result = pkgPrefix & s[start..^1] + +proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) = + doAssert n.kind == nkSym, $n.kind + + let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex)) + let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex)) + let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module)) + let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym)) + addDependencyAux(b, parent, child) + +proc addDotDependency*(c: PPassContext, n: PNode): PNode = result = n let g = PGen(c) let b = Backend(g.graph.backend) case n.kind of nkImportStmt: for i in 0.. we need to 'join' every pushed 'fork' here - - -This works and then our abstract interpretation needs to deal with 'fork' -differently. It really causes a split in execution. Two threads are -"spawned" and both need to reach the 'join L' instruction. Afterwards -the abstract interpretations are joined and execution resumes single -threaded. - - -Abstract Interpretation ------------------------ - -proc interpret(pc, state, comesFrom): state = - result = state - # we need an explicit 'create' instruction (an explicit heap), in order - # to deal with 'var x = create(); var y = x; var z = y; destroy(z)' - while true: - case pc - of fork: - let a = interpret(pc+1, result, pc) - let b = interpret(forkTarget, result, pc) - result = a ++ b # ++ is a union operation - inc pc - of join: - if joinTarget == comesFrom: return result - else: inc pc - of use X: - if not result.contains(x): - error "variable not initialized " & x - inc pc - of def X: - if not result.contains(x): - result.incl X - else: - error "overwrite of variable causes memory leak " & x - inc pc - of destroy X: - result.excl X - -This is correct but still can lead to false positives: - -proc p(cond: bool) = - if cond: - new(x) - otherThings() - if cond: - destroy x - -Is not a leak. We should find a way to model *data* flow, not just -control flow. One solution is to rewrite the 'if' without a fork -instruction. The unstructured aspect can now be easily dealt with -the 'goto' and 'join' instructions. - -proc p(cond: bool) = - L0: fork Lend - new(x) - # do not 'join' here! - - Lend: - otherThings() - join L0 # SKIP THIS FOR new(x) SOMEHOW - destroy x - join L0 # but here. - - - -But if we follow 'goto Louter' we will never come to the join point. -We restore the bindings after popping pc from the stack then there -"no" problem?! - - -while cond: - prelude() - if not condB: break - postlude() - ----> -var setFlag = true -while cond and not setFlag: - prelude() - if not condB: - setFlag = true # BUT: Dependency - if not setFlag: # HERE - postlude() - ----> -var setFlag = true -while cond and not setFlag: - prelude() - if not condB: - postlude() - setFlag = true - - -------------------------------------------------- - -while cond: - prelude() - if more: - if not condB: break - stuffHere() - postlude() - ---> -var setFlag = true -while cond and not setFlag: - prelude() - if more: - if not condB: - setFlag = false - else: - stuffHere() - postlude() - else: - postlude() - -This is getting complicated. Instead we keep the whole 'join' idea but -duplicate the 'join' instructions on breaks and return exits! - -]# + c.code.add Instr(kind: goto, dest: 0) proc genLabel(c: Con): TPosition = TPosition(c.code.len) @@ -272,8 +103,8 @@ template checkedDistance(dist): int = doAssert low(int) div 2 + 1 < dist and dist < high(int) div 2 dist -proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) = - c.code.add Instr(n: n, kind: goto, dest: checkedDistance(p.int - c.code.len)) +proc jmpBack(c: var Con, p = TPosition(0)) = + c.code.add Instr(kind: loop, dest: checkedDistance(p.int - c.code.len)) proc patch(c: var Con, p: TPosition) = # patch with current index @@ -282,13 +113,13 @@ proc patch(c: var Con, p: TPosition) = proc gen(c: var Con; n: PNode) proc popBlock(c: var Con; oldLen: int) = - var exits: seq[TPosition] - exits.add c.gotoI(newNode(nkEmpty)) + var exits: seq[TPosition] = @[] + exits.add c.gotoI() for f in c.blocks[oldLen].breakFixups: c.patch(f[0]) for finale in f[1]: c.gen(finale) - exits.add c.gotoI(newNode(nkEmpty)) + exits.add c.gotoI() for e in exits: c.patch e c.blocks.setLen(oldLen) @@ -299,90 +130,29 @@ template withBlock(labl: PSym; body: untyped) = body popBlock(c, oldLen) -proc isTrue(n: PNode): bool = - n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or - n.kind == nkIntLit and n.intVal != 0 - -when true: - proc genWhile(c: var Con; n: PNode) = - # We unroll every loop 3 times. We emulate 0, 1, 2 iterations - # through the loop. We need to prove this is correct for our - # purposes. But Herb Sutter claims it is. (Proof by authority.) - #[ - while cond: - body - - Becomes: - - block: - if cond: - body - if cond: - body - if cond: - body - - We still need to ensure 'break' resolves properly, so an AST to AST - translation is impossible. - - So the code to generate is: - - cond - fork L4 # F1 - body - cond - fork L5 # F2 - body - cond - fork L6 # F3 - body - L6: - join F3 - L5: - join F2 - L4: - join F1 - ]# - if isTrue(n[0]): - # 'while true' is an idiom in Nim and so we produce - # better code for it: - withBlock(nil): - for i in 0..2: - c.gen(n[1]) - else: - withBlock(nil): - var endings: array[3, TPosition] - for i in 0..2: - c.gen(n[0]) - endings[i] = c.forkI(n) - c.gen(n[1]) - for i in countdown(endings.high, 0): - c.patch(endings[i]) - -else: - proc genWhile(c: var Con; n: PNode) = - # lab1: - # cond, tmp - # fork tmp, lab2 - # body - # jmp lab1 - # lab2: - let lab1 = c.genLabel - withBlock(nil): - if isTrue(n[0]): - c.gen(n[1]) - c.jmpBack(n, lab1) - else: - c.gen(n[0]) - forkT(n): - c.gen(n[1]) - c.jmpBack(n, lab1) - -template forkT(n, body) = - let lab1 = c.forkI(n) +template forkT(body) = + let lab1 = c.forkI() body c.patch(lab1) +proc genWhile(c: var Con; n: PNode) = + # lab1: + # cond, tmp + # fork tmp, lab2 + # body + # jmp lab1 + # lab2: + let lab1 = c.genLabel + withBlock(nil): + if isTrue(n[0]): + c.gen(n[1]) + c.jmpBack(lab1) + else: + c.gen(n[0]) + forkT: + c.gen(n[1]) + c.jmpBack(lab1) + proc genIf(c: var Con, n: PNode) = #[ @@ -411,34 +181,32 @@ proc genIf(c: var Con, n: PNode) = goto Lend3 L3: D - goto Lend3 # not eliminated to simplify the join generation - Lend3: - join F3 - Lend2: - join F2 - Lend: - join F1 - ]# var endings: seq[TPosition] = @[] + let oldInteresting = c.interestingInstructions + let oldLen = c.code.len + for i in 0.. 0: #Ok, we are in a try, lets see which (if any) try's we break out from: + var trailingFinales: seq[PNode] = @[] + if c.inTryStmt > 0: + # Ok, we are in a try, lets see which (if any) try's we break out from: for b in countdown(c.blocks.high, i): if c.blocks[b].isTryBlock: trailingFinales.add c.blocks[b].finale @@ -493,6 +262,7 @@ proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) = c.blocks[i].breakFixups.add (lab1, trailingFinales) proc genBreak(c: var Con; n: PNode) = + inc c.interestingInstructions if n[0].kind == nkSym: for i in countdown(c.blocks.high, 0): if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym: @@ -523,9 +293,9 @@ proc genTry(c: var Con; n: PNode) = for i in 1.. 0: for i in countdown(c.blocks.high, 0): if c.blocks[i].isTryBlock: genBreakOrRaiseAux(c, i, n) return - assert false #Unreachable + assert false # Unreachable else: - genNoReturn(c, n) + genNoReturn(c) proc genImplicitReturn(c: var Con) = if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len: gen(c, c.owner.ast[resultPos]) proc genReturn(c: var Con; n: PNode) = + inc c.interestingInstructions if n[0].kind != nkEmpty: gen(c, n[0]) else: @@ -561,124 +333,6 @@ proc genReturn(c: var Con; n: PNode) = const InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp} - PathKinds0 = {nkDotExpr, nkCheckedFieldExpr, - nkBracketExpr, nkDerefExpr, nkHiddenDeref, - nkAddr, nkHiddenAddr, - nkObjDownConv, nkObjUpConv} - PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv} - -proc skipConvDfa*(n: PNode): PNode = - result = n - while true: - case result.kind - of nkObjDownConv, nkObjUpConv: - result = result[0] - of PathKinds1: - result = result[1] - else: break - -type AliasKind* = enum - yes, no, maybe - -proc aliases*(obj, field: PNode): AliasKind = - # obj -> field: - # x -> x: true - # x -> x.f: true - # x.f -> x: false - # x.f -> x.f: true - # x.f -> x.v: false - # x -> x[0]: true - # x[0] -> x: false - # x[0] -> x[0]: true - # x[0] -> x[1]: false - # x -> x[i]: true - # x[i] -> x: false - # x[i] -> x[i]: maybe; Further analysis could make this return true when i is a runtime-constant - # x[i] -> x[j]: maybe; also returns maybe if only one of i or j is a compiletime-constant - template collectImportantNodes(result, n) = - var result: seq[PNode] - var n = n - while true: - case n.kind - of PathKinds0 - {nkDotExpr, nkCheckedFieldExpr, nkBracketExpr}: - n = n[0] - of PathKinds1: - n = n[1] - of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr: - result.add n - n = n[0] - of nkSym: - result.add n; break - else: return no - - collectImportantNodes(objImportantNodes, obj) - collectImportantNodes(fieldImportantNodes, field) - - # If field is less nested than obj, then it cannot be part of/aliased by obj - if fieldImportantNodes.len < objImportantNodes.len: return no - - result = yes - for i in 1..objImportantNodes.len: - # We compare the nodes leading to the location of obj and field - # with each other. - # We continue until they diverge, in which case we return no, or - # until we reach the location of obj, in which case we do not need - # to look further, since field must be part of/aliased by obj now. - # If we encounter an element access using an index which is a runtime value, - # we simply return maybe instead of yes; should further nodes not diverge. - let currFieldPath = fieldImportantNodes[^i] - let currObjPath = objImportantNodes[^i] - - if currFieldPath.kind != currObjPath.kind: - return no - - case currFieldPath.kind - of nkSym: - if currFieldPath.sym != currObjPath.sym: return no - of nkDotExpr: - if currFieldPath[1].sym != currObjPath[1].sym: return no - of nkCheckedFieldExpr: - if currFieldPath[0][1].sym != currObjPath[0][1].sym: return no - of nkBracketExpr: - if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals: - if currFieldPath[1].intVal != currObjPath[1].intVal: - return no - else: - result = maybe - else: assert false # unreachable - -proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool = - var n = orig - while true: - case n.kind - of PathKinds0 - {nkHiddenDeref, nkDerefExpr}: - n = n[0] - of PathKinds1: - n = n[1] - of nkHiddenDeref, nkDerefExpr: - # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty - # pointer indirection. - # bug #14159, we cannot reason about sinkParam[].location as it can - # still be shared for tyRef. - n = n[0] - return n.kind == nkSym and n.sym.owner == owner and - (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned}) - else: break - # XXX Allow closure deref operations here if we know - # the owner controlled the closure allocation? - result = n.kind == nkSym and n.sym.owner == owner and - {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and - (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar) - # Note: There is a different move analyzer possible that checks for - # consume(param.key); param.key = newValue for all paths. Then code like - # - # let splited = split(move self.root, x) - # self.root = merge(splited.lower, splited.greater) - # - # could be written without the ``move self.root``. However, this would be - # wrong! Then the write barrier for the ``self.root`` assignment would - # free the old data and all is lost! Lesson: Don't be too smart, trust the - # lower level C++ optimizer to specialize this code. proc skipTrivials(c: var Con, n: PNode): PNode = result = n @@ -696,16 +350,20 @@ proc skipTrivials(c: var Con, n: PNode): PNode = proc genUse(c: var Con; orig: PNode) = let n = c.skipTrivials(orig) - if n.kind == nkSym and n.sym.kind in InterestingSyms: - c.code.add Instr(n: orig, kind: use) - elif n.kind in nkCallKinds: + if n.kind == nkSym: + if n.sym.kind in InterestingSyms and n.sym == c.root: + c.code.add Instr(kind: use, n: orig) + inc c.interestingInstructions + else: gen(c, n) proc genDef(c: var Con; orig: PNode) = let n = c.skipTrivials(orig) if n.kind == nkSym and n.sym.kind in InterestingSyms: - c.code.add Instr(n: orig, kind: def) + if n.sym == c.root: + c.code.add Instr(kind: def, n: orig) + inc c.interestingInstructions proc genCall(c: var Con; n: PNode) = gen(c, n[0]) @@ -713,18 +371,17 @@ proc genCall(c: var Con; n: PNode) = if t != nil: t = t.skipTypes(abstractInst) for i in 1.. 0 and canRaiseConservative(n[0]): + inc c.interestingInstructions # we generate the instruction sequence: # fork lab1 # goto exceptionHandler (except or finally) # lab1: - # join F1 - forkT(n): + forkT: for i in countdown(c.blocks.high, 0): if c.blocks[i].isTryBlock: genBreakOrRaiseAux(c, i, n) @@ -762,12 +419,18 @@ proc gen(c: var Con; n: PNode) = else: genCall(c, n) if sfNoReturn in n[0].sym.flags: - genNoReturn(c, n) + genNoReturn(c) else: genCall(c, n) of nkCharLit..nkNilLit: discard - of nkAsgn, nkFastAsgn: + of nkAsgn, nkFastAsgn, nkSinkAsgn: gen(c, n[1]) + + if n[0].kind in PathKinds0: + let a = c.skipTrivials(n[0]) + if a.kind in nkCallKinds: + gen(c, a) + # watch out: 'obj[i].f2 = value' sets 'f2' but # "uses" 'i'. But we are only talking about builtin array indexing so # it doesn't matter and 'x = 34' is NOT a usage of 'x'. @@ -794,16 +457,35 @@ proc gen(c: var Con; n: PNode) = of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, PathKinds1: gen(c, n[1]) of nkVarSection, nkLetSection: genVarSection(c, n) - of nkDefer: doAssert false, "dfa construction pass requires the elimination of 'defer'" + of nkDefer: raiseAssert "dfa construction pass requires the elimination of 'defer'" else: discard -proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph = +when false: + proc optimizeJumps(c: var ControlFlowGraph) = + for i in 0.. pc: + pc = newPc + else: + break + c[i].dest = pc - i + of loop, def, use: discard + +proc constructCfg*(s: PSym; body: PNode; root: PSym): ControlFlowGraph = ## constructs a control flow graph for ``body``. - var c = Con(code: @[], blocks: @[], owner: s) + var c = Con(code: @[], blocks: @[], owner: s, root: root) withBlock(s): gen(c, body) - genImplicitReturn(c) - when defined(gcArc) or defined(gcOrc): + if root.kind == skResult: + genImplicitReturn(c) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): result = c.code # will move else: shallowCopy(result, c.code) + when false: + optimizeJumps result diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 57d5e7b013..b4c4baa2b6 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -7,19 +7,28 @@ # distribution, for details about the copyright. # -# This is the documentation generator. Cross-references are generated -# by knowing how the anchors are going to be named. +## This is the Nim documentation generator. Cross-references are generated +## by knowing how the anchors are going to be named. +## +## .. importdoc:: ../docgen.md +## +## For corresponding users' documentation see [Nim DocGen Tools Guide]. import - ast, strutils, strtabs, options, msgs, os, ropes, idents, - wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast, - packages/docutils/rst, packages/docutils/rstgen, - json, xmltree, trees, types, - typesrenderer, astalgo, lineinfos, intsets, - pathutils, trees, tables, nimpaths, renderverbatim, osproc + ast, options, msgs, idents, + wordrecg, syntaxes, renderer, lexer, + packages/docutils/[rst, rstidx, rstgen, dochelpers], + trees, types, + typesrenderer, astalgo, lineinfos, + pathutils, nimpaths, renderverbatim, packages +import packages/docutils/rstast except FileIndex, TLineInfo -from uri import encodeUrl -from std/private/globs import nativeToUnixPath +import std/[os, strutils, strtabs, algorithm, json, osproc, tables, intsets, xmltree, sequtils] +from std/uri import encodeUrl +from nodejs import findNodeJs + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] const @@ -28,25 +37,66 @@ const DocColOffset = "## ".len # assuming that a space was added after ## type - TSections = array[TSymKind, Rope] + ItemFragment = object ## A fragment from each item will be eventually + ## constructed by converting `rst` fields to strings. + case isRst: bool + of true: + rst: PRstNode + of false: ## contains ready markup e.g. from runnableExamples + str: string + ItemPre = seq[ItemFragment] ## A pre-processed item. + Item = object ## Any item in documentation, e.g. symbol + ## entry. Configuration variable ``doc.item`` + ## is used for its HTML rendering. + descRst: ItemPre ## Description of the item (may contain + ## runnableExamples). + substitutions: seq[string] ## Variable names in `doc.item`... + sortName: string ## The string used for sorting in output + info: rstast.TLineInfo ## place where symbol was defined (for messages) + anchor: string ## e.g. HTML anchor + name: string ## short name of the symbol, not unique + ## (includes backticks ` if present) + detailedName: string ## longer name like `proc search(x: int): int` + ModSection = object ## Section like Procs, Types, etc. + secItems: Table[string, seq[Item]] + ## Map basic name -> pre-processed items. + finalMarkup: string ## The items, after RST pass 2 and rendering. + ModSections = array[TSymKind, ModSection] + TocItem = object ## HTML TOC item + content: string + sortName: string + TocSectionsFinal = array[TSymKind, string] ExampleGroup = ref object ## a group of runnableExamples with same rdoccmd rdoccmd: string ## from 1st arg in `runnableExamples(rdoccmd): body` docCmd: string ## from user config, e.g. --doccmd:-d:foo code: string ## contains imports; each import contains `body` index: int ## group index + JsonItem = object # pre-processed item: `rst` should be finalized + json: JsonNode + rst: PRstNode + rstField: string TDocumentor = object of rstgen.RstGenerator - modDesc: Rope # module description + modDescPre: ItemPre # module description, not finalized + modDescFinal: string # module description, after RST pass 2 and rendering module: PSym - modDeprecationMsg: Rope - toc, toc2, section: TSections - tocTable: array[TSymKind, Table[string, Rope]] + modDeprecationMsg: string + section: ModSections # entries of ``.nim`` file (for `proc`s, etc) + tocSimple: array[TSymKind, seq[TocItem]] + # TOC entries for non-overloadable symbols (e.g. types, constants)... + tocTable: array[TSymKind, Table[string, seq[TocItem]]] + # ...otherwise (e.g. procs) + toc2: TocSectionsFinal # TOC `content`, which is probably wrapped + # in `doc.section.toc2` + toc: TocSectionsFinal # final TOC (wrapped in `doc.section.toc`) indexValFilename: string analytics: string # Google Analytics javascript, "" if doesn't exist seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. - jArray: JsonNode + jEntriesPre: seq[JsonItem] # pre-processed RST + JSON content + jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable - isPureRst: bool + sharedState: PRstSharedState + standaloneDoc: bool # is markup (.rst/.md) document? conf*: ConfigRef cache*: IdentCache exampleCounter: int @@ -55,15 +105,77 @@ type thisDir*: AbsoluteDir exampleGroups: OrderedTable[string, ExampleGroup] wroteSupportFiles*: bool + nimToRstFid: Table[lineinfos.FileIndex, rstast.FileIndex] + ## map Nim FileIndex -> RST one, it's needed because we keep them separate PDoc* = ref TDocumentor ## Alias to type less. +proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst) +proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str) + +proc addRstFileIndex(d: PDoc, fileIndex: lineinfos.FileIndex): rstast.FileIndex = + let invalid = rstast.FileIndex(-1) + result = d.nimToRstFid.getOrDefault(fileIndex, default = invalid) + if result == invalid: + let fname = toFullPath(d.conf, fileIndex) + result = addFilename(d.sharedState, fname) + d.nimToRstFid[fileIndex] = result + +proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex = + addRstFileIndex(d, info.fileIndex) + +proc cmpDecimalsIgnoreCase(a, b: string): int = + ## For sorting with correct handling of cases like 'uint8' and 'uint16'. + ## Also handles leading zeros well (however note that leading zeros are + ## significant when lengths of numbers mismatch, e.g. 'bar08' > 'bar8' !). + runnableExamples: + doAssert cmpDecimalsIgnoreCase("uint8", "uint16") < 0 + doAssert cmpDecimalsIgnoreCase("val00032", "val16suffix") > 0 + doAssert cmpDecimalsIgnoreCase("val16suffix", "val16") > 0 + doAssert cmpDecimalsIgnoreCase("val_08_32", "val_08_8") > 0 + doAssert cmpDecimalsIgnoreCase("val_07_32", "val_08_8") < 0 + doAssert cmpDecimalsIgnoreCase("ab8", "ab08") < 0 + doAssert cmpDecimalsIgnoreCase("ab8de", "ab08c") < 0 # sanity check + let aLen = a.len + let bLen = b.len + var + iA = 0 + iB = 0 + while iA < aLen and iB < bLen: + if isDigit(a[iA]) and isDigit(b[iB]): + var + limitA = iA # index after the last (least significant) digit + limitB = iB + while limitA < aLen and isDigit(a[limitA]): inc limitA + while limitB < bLen and isDigit(b[limitB]): inc limitB + var pos = max(limitA-iA, limitB-iA) + while pos > 0: + if limitA-pos < iA: # digit in `a` is 0 effectively + result = ord('0') - ord(b[limitB-pos]) + elif limitB-pos < iB: # digit in `b` is 0 effectively + result = ord(a[limitA-pos]) - ord('0') + else: + result = ord(a[limitA-pos]) - ord(b[limitB-pos]) + if result != 0: return + dec pos + result = (limitA - iA) - (limitB - iB) # consider 'bar08' > 'bar8' + if result != 0: return + iA = limitA + iB = limitB + else: + result = ord(toLowerAscii(a[iA])) - ord(toLowerAscii(b[iB])) + if result != 0: return + inc iA + inc iB + result = (aLen - iA) - (bLen - iB) + proc prettyString(a: object): string = # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906 + result = "" for k, v in fieldPairs(a): result.add k & ": " & $v & "\n" -proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): RelativeFile = +proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = ## returns a relative file that will be appended to outDir let file2 = $file template bail() = @@ -88,7 +200,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re if result.isEmpty: bail() elif conf.docRoot.len > 0: # we're (currently) requiring `isAbsolute` to avoid confusion when passing - # a relative path (would it be relative wrt $PWD or to projectfile) + # a relative path (would it be relative with regard to $PWD or to projectfile) conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot conf.globalAssert conf.docRoot.dirExists, arg=conf.docRoot # needed because `canonicalizePath` called on `file` @@ -97,10 +209,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re bail() if isAbsolute(result.string): result = file.string.splitPath()[1].RelativeFile - if isTitle: - result = result.string.nativeToUnixPath.RelativeFile - else: - result = result.string.replace("..", dotdotMangle).RelativeFile + result = result.string.replace("..", dotdotMangle).RelativeFile doAssert not result.isEmpty doAssert not isAbsolute(result.string) @@ -108,12 +217,16 @@ proc whichType(d: PDoc; n: PNode): PSym = if n.kind == nkSym: if d.types.strTableContains(n.sym): result = n.sym + else: + result = nil else: + result = nil for i in 0.. 0: check(0) for i in 2.. @@ -196,18 +364,22 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, """ % [conf.configVars.getOrDefault"doc.googleAnalytics"] + elif conf.configVars.hasKey("doc.plausibleAnalytics"): + result.analytics = """ + + """ % [conf.configVars.getOrDefault"doc.plausibleAnalytics"] else: result.analytics = "" result.seenSymbols = newStringTable(modeCaseInsensitive) result.id = 100 - result.jArray = newJArray() - initStrTable result.types + result.jEntriesFinal = newJArray() + result.types = initStrTable() result.onTestSnippet = - proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) = + proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) {.gcsafe.} = if conf.docCmd == docCmdSkip: return inc(gen.id) - var d = TDocumentor(gen) + var d = (ptr TDocumentor)(addr gen) var outp: AbsoluteFile if filename.len == 0: let nameOnly = splitFile(d.filename).name @@ -223,7 +395,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, # Make sure the destination directory exists createDir(outp.splitFile.dir) # Include the current file if we're parsing a nim file - let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")] + let importStmt = if d.standaloneDoc: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")] writeFile(outp, importStmt & content) proc interpSnippetCmd(cmd: string): string = @@ -245,11 +417,12 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, if gotten != status: rawMessage(conf, errGenerated, "snippet failed: cmd: '$1' status: $2 expected: $3 output: $4" % [cmd, $gotten, $status, output]) result.emitted = initIntSet() - result.destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, false).string + result.destFile = destFile result.thisDir = result.destFile.AbsoluteFile.splitFile.dir -template dispA(conf: ConfigRef; dest: var Rope, xml, tex: string, args: openArray[Rope]) = - if conf.cmd != cmdRst2tex: dest.addf(xml, args) +template dispA(conf: ConfigRef; dest: var string, xml, tex: string, + args: openArray[string]) = + if not conf.isLatexCmd: dest.addf(xml, args) else: dest.addf(tex, args) proc getVarIdx(varnames: openArray[string], id: string): int = @@ -258,84 +431,25 @@ proc getVarIdx(varnames: openArray[string], id: string): int = return i result = -1 -proc ropeFormatNamedVars(conf: ConfigRef; frmt: FormatStr, - varnames: openArray[string], - varvalues: openArray[Rope]): Rope = - var i = 0 - result = nil - var num = 0 - while i < frmt.len: - if frmt[i] == '$': - inc(i) # skip '$' - case frmt[i] - of '#': - result.add(varvalues[num]) - inc(num) - inc(i) - of '$': - result.add("$") - inc(i) - of '0'..'9': - var j = 0 - while true: - j = (j * 10) + ord(frmt[i]) - ord('0') - inc(i) - if (i > frmt.len + 0 - 1) or not (frmt[i] in {'0'..'9'}): break - if j > high(varvalues) + 1: - rawMessage(conf, errGenerated, "Invalid format string; too many $s: " & frmt) - num = j - result.add(varvalues[j - 1]) - of 'A'..'Z', 'a'..'z', '\x80'..'\xFF': - var id = "" - while true: - id.add(frmt[i]) - inc(i) - if not (frmt[i] in {'A'..'Z', '_', 'a'..'z', '\x80'..'\xFF'}): break - var idx = getVarIdx(varnames, id) - if idx >= 0: result.add(varvalues[idx]) - else: rawMessage(conf, errGenerated, "unknown substition variable: " & id) - of '{': - var id = "" - inc(i) - while i < frmt.len and frmt[i] != '}': - id.add(frmt[i]) - inc(i) - if i >= frmt.len: - rawMessage(conf, errGenerated, "expected closing '}'") - else: - inc(i) # skip } - # search for the variable: - let idx = getVarIdx(varnames, id) - if idx >= 0: result.add(varvalues[idx]) - else: rawMessage(conf, errGenerated, "unknown substition variable: " & id) - else: - result.add("$") - var start = i - while i < frmt.len: - if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: result.add(substr(frmt, start, i - 1)) - -proc genComment(d: PDoc, n: PNode): string = - result = "" +proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: - let comment = n.comment - when false: - # RFC: to preseve newlines in comments, this would work: - comment = comment.replace("\n", "\n\n") - renderRstToOut(d[], - parseRst(comment, toFullPath(d.conf, n.info), - toLinenumber(n.info), - toColumn(n.info) + DocColOffset, - (var dummy: bool; dummy), d.options, d.conf), - result) + d.sharedState.currFileIdx = addRstFileIndex(d, n.info) + try: + result = parseRst(n.comment, + toLinenumber(n.info), + toColumn(n.info) + DocColOffset, + d.conf, d.sharedState) + except ERecoverableError: + result = newRstNode(rnLiteralBlock, @[newRstLeaf(n.comment)]) + else: + result = nil -proc genRecCommentAux(d: PDoc, n: PNode): Rope = +proc genRecCommentAux(d: PDoc, n: PNode): PRstNode = if n == nil: return nil - result = genComment(d, n).rope + result = genComment(d, n) if result == nil: - if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef, - nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}: + if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef, nkTypeClassTy, + nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkSinkAsgn, nkHiddenStdConv}: # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}: for i in 0.. 0: return -proc belongsToPackage(conf: ConfigRef; module: PSym): bool = - result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId - proc externalDep(d: PDoc; module: PSym): string = if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0: let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position) @@ -378,15 +490,15 @@ proc externalDep(d: PDoc; module: PSym): string = else: result = extractFilename toFullPath(d.conf, FileIndex module.position) -proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}; - procLink: Rope) = - var r: TSrcGen +proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var string; + renderFlags: TRenderFlags = {}; + procLink: string) = + var r: TSrcGen = initTokRender(n, renderFlags) var literal = "" - initTokRender(r, n, renderFlags) var kind = tkEof var tokenPos = 0 var procTokenPos = 0 - template escLit(): untyped = rope(esc(d.target, literal)) + template escLit(): untyped = esc(d.target, literal) while true: getNextTok(r, kind, literal) inc tokenPos @@ -400,11 +512,11 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe if kind in {tkProc, tkMethod, tkIterator, tkMacro, tkTemplate, tkFunc, tkConverter}: procTokenPos = tokenPos dispA(d.conf, result, "$1", "\\spanKeyword{$1}", - [rope(literal)]) + [literal]) of tkOpr: dispA(d.conf, result, "$1", "\\spanOperator{$1}", [escLit]) - of tkStrLit..tkTripleStrLit: + of tkStrLit..tkTripleStrLit, tkCustomLit: dispA(d.conf, result, "$1", "\\spanStringLit{$1}", [escLit]) of tkCharLit: @@ -419,37 +531,37 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe of tkSymbol: let s = getTokSym(r) # -2 because of the whitespace in between: - if procTokenPos == tokenPos-2 and procLink != nil: + if procTokenPos == tokenPos-2 and procLink != "": dispA(d.conf, result, "$1", "\\spanIdentifier{$1}", [escLit, procLink]) elif s != nil and s.kind in {skType, skVar, skLet, skConst} and sfExported in s.flags and s.owner != nil and - belongsToPackage(d.conf, s.owner) and d.target == outHtml: + belongsToProjectPackage(d.conf, s.owner) and d.target == outHtml: let external = externalDep(d, s.owner) result.addf "$3", - [rope changeFileExt(external, "html"), rope literal, + [changeFileExt(external, "html"), literal, escLit] else: dispA(d.conf, result, "$1", "\\spanIdentifier{$1}", [escLit]) of tkSpaces, tkInvalid: result.add(literal) - of tkCurlyDotLe: + of tkHideableStart: template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit]) if renderRunnableExamples in renderFlags: fun "$1" - else: fun: "" & # This span is required for the JS to work properly - """{...} + else: + # 1st span is required for the JS to work properly + fun """ + +... - -$1 -""".replace("\n", "") # Must remove newlines because wrapped in a 
-    of tkCurlyDotRi:
+""".replace("\n", "")  # Must remove newlines because wrapped in a 
+    of tkHideableEnd:
       template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit])
       if renderRunnableExamples in renderFlags: fun "$1"
-      else: fun """
-
-$1
-""".replace("\n", "")
+      else: fun ""
+    of tkCurlyDotLe: dispA(d.conf, result, "$1", "\\spanOther{$1}", [escLit])
+    of tkCurlyDotRi: dispA(d.conf, result, "$1", "\\spanOther{$1}", [escLit])
     of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
        tkBracketDotLe, tkBracketDotRi, tkParDotLe,
        tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
@@ -474,26 +586,37 @@ proc runAllExamples(d: PDoc) =
     # most useful semantics is that `docCmd` comes after `rdoccmd`, so that we can (temporarily) override
     # via command line
     # D20210224T221756:here
-    let cmd = "$nim $backend -r --lib:$libpath --warning:UnusedImport:off --path:$path --nimcache:$nimcache $rdoccmd $docCmd $file" % [
+    var pathArgs = "--path:$path" % [ "path", quoteShell(d.conf.projectPath) ]
+    for p in d.conf.searchPaths:
+      pathArgs = "$args --path:$path" % [ "args", pathArgs, "path", quoteShell(p) ]
+    let cmd = "$nim $backend -r --lib:$libpath --warning:UnusedImport:off $pathArgs --nimcache:$nimcache $rdoccmd $docCmd $file" % [
       "nim", quoteShell(os.getAppFilename()),
       "backend", $d.conf.backend,
-      "path", quoteShell(d.conf.projectPath),
+      "pathArgs", pathArgs,
       "libpath", quoteShell(d.conf.libpath),
       "nimcache", quoteShell(outputDir),
       "file", quoteShell(outp),
       "rdoccmd", group.rdoccmd,
       "docCmd", group.docCmd,
     ]
-    if os.execShellCmd(cmd) != 0:
-      quit "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, group[].prettyString, cmd]
+    if d.conf.backend == backendJs and findNodeJs() == "":
+      discard "ignore JS runnableExample"
+    elif os.execShellCmd(cmd) != 0:
+      d.conf.quitOrRaise "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, group[].prettyString, cmd]
     else:
       # keep generated source file `outp` to allow inspection.
       rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
       # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
 
-proc quoted(a: string): string = result.addQuoted(a)
+proc quoted(a: string): string =
+  result = ""
+  result.addQuoted(a)
 
-proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
+proc toInstantiationInfo(conf: ConfigRef, info: TLineInfo): (string, int, int) =
+  # xxx expose in compiler/lineinfos.nim
+  (conf.toMsgFilename(info), info.line.int, info.col.int + ColOffset)
+
+proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, code: string] =
   ## returns `rdoccmd` and source code for this runnableExamples
   var rdoccmd = ""
   if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
@@ -514,13 +637,14 @@ proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
   let outputDir = d.exampleOutputDir
   createDir(outputDir)
   inc d.exampleCounter
-  let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" & ("_examples$1.nim" % $d.exampleCounter)))
+  let outp = outputDir / RelativeFile("$#_examples_$#.nim" % [d.filename.extractFilename.changeFileExt"", $d.exampleCounter])
 
   if useRenderModule:
     var docComment = newTree(nkCommentStmt)
     docComment.comment = comment
     var runnableExamples = newTree(nkStmtList,
         docComment,
+        newTree(nkImportStmt, newStrNode(nkStrLit, "std/assertions")),
         newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
     runnableExamples.info = n.info
     for a in n.lastSon: runnableExamples.add a
@@ -532,19 +656,36 @@ proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
     renderModule(runnableExamples, outp.string, conf = d.conf)
 
   else:
-    let code2 = """
+    var code2 = code
+    if code.len > 0 and "codeReordering" notin code:
+      let codeIndent = extractRunnableExamplesSource(d.conf, n, indent = 2)
+      # hacky but simplest solution, until we devise a way to make `{.line.}`
+      # work without introducing a scope
+      code2 = """
+{.line: $#.}:
+$#
+""" % [$toInstantiationInfo(d.conf, n.info), codeIndent]
+    code2 = """
 #[
-$1
+$#
 ]#
-import $2
-$3
-""" % [comment, d.filename.quoted, code]
+import std/assertions
+import $#
+$#
+""" % [comment, d.filename.quoted, code2]
     writeFile(outp.string, code2)
 
   if rdoccmd notin d.exampleGroups:
     d.exampleGroups[rdoccmd] = ExampleGroup(rdoccmd: rdoccmd, docCmd: d.conf.docCmd, index: d.exampleGroups.len)
   d.exampleGroups[rdoccmd].code.add "import $1\n" % outp.string.quoted
-  result = (rdoccmd, code)
+
+  var codeShown: string
+  if topLevel: # refs https://github.com/nim-lang/RFCs/issues/352
+    let title = canonicalImport(d.conf, AbsoluteFile d.filename)
+    codeShown = "import $#\n$#" % [title, code]
+  else:
+    codeShown = code
+  result = (rdoccmd, codeShown)
   when false:
     proc extractImports(n: PNode; result: PNode) =
       if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
@@ -564,7 +705,9 @@ type RunnableState = enum
   rsRunnable
   rsDone
 
-proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState): RunnableState =
+proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var ItemPre,
+                                state: RunnableState, topLevel: bool):
+                               RunnableState =
   ##[
   Simple state machine to tell whether we render runnableExamples and doc comments.
   This is to ensure that we can interleave runnableExamples and doc comments freely;
@@ -588,20 +731,25 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: Runnab
       return rsComment
   of nkCallKinds:
     if isRunnableExamples(n[0]) and
-        n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
-      let (rdoccmd, code) = prepareExample(d, n)
-      var msg = "Example:"
-      if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
-      dispA(d.conf, dest, "\n
$1
\n",
-          "\n\\textbf{$1}\n", [msg.rope])
-      inc d.listingCounter
-      let id = $d.listingCounter
-      dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
-      var dest2 = ""
-      renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
-      dest.add dest2
-      dest.add(d.config.getOrDefault"doc.listing_end" % id)
-      return rsRunnable
+        n.len >= 2 and n.lastSon.kind == nkStmtList:
+      if state in {rsStart, rsComment, rsRunnable}:
+        let (rdoccmd, code) = prepareExample(d, n, topLevel)
+        var msg = "Example:"
+        if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
+        var s: string = ""
+        dispA(d.conf, s, "\n
$1
\n",
+            "\n\n\\textbf{$1}\n", [msg])
+        dest.add s
+        inc d.listingCounter
+        let id = $d.listingCounter
+        dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
+        var dest2 = ""
+        renderNimCode(dest2, code, d.target)
+        dest.add dest2
+        dest.add(d.config.getOrDefault"doc.listing_end" % id)
+        return rsRunnable
+      else:
+        localError(d.conf, n.info, errUser, "runnableExamples must appear before the first non-comment statement")
   else: discard
   return rsDone
     # change this to `rsStart` if you want to keep generating doc comments
@@ -637,22 +785,22 @@ proc getRoutineBody(n: PNode): PNode =
     doAssert result.len == 2
     result = result[1]
 
-proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
+proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var ItemPre) =
   var n = n
   var state = rsStart
-  template fn(n2) =
-    state = getAllRunnableExamplesImpl(d, n2, dest, state)
-  dest.add genComment(d, n).rope
+  template fn(n2, topLevel) =
+    state = getAllRunnableExamplesImpl(d, n2, dest, state, topLevel)
+  dest.add genComment(d, n)
   case n.kind
   of routineDefs:
     n = n.getRoutineBody
     case n.kind
-    of nkCommentStmt, nkCallKinds: fn(n)
+    of nkCommentStmt, nkCallKinds: fn(n, topLevel = false)
     else:
       for i in 0.. 0:
-    result.delete(pos, result.len - 1)
+    result.setLen(pos - 1)
     result.add("…")
   if pos < maxDocstringChars:
     return
@@ -794,31 +937,32 @@ proc docstringSummary(rstText: string): string =
   pos = result.find({'.', ',', ':'})
   let last = result.len - 1
   if pos > 0 and pos < last:
-    result.delete(pos, last)
+    result.setLen(pos - 1)
     result.add("…")
 
-proc genDeprecationMsg(d: PDoc, n: PNode): Rope =
+proc genDeprecationMsg(d: PDoc, n: PNode): string =
   ## Given a nkPragma wDeprecated node output a well-formatted section
   if n == nil: return
 
   case n.safeLen:
   of 0: # Deprecated w/o any message
-    result = ropeFormatNamedVars(d.conf,
-      getConfigVar(d.conf, "doc.deprecationmsg"), ["label", "message"],
-      [~"Deprecated", nil])
+    result = getConfigVar(d.conf, "doc.deprecationmsg") % [
+       "label" , "Deprecated", "message", ""]
   of 2: # Deprecated w/ a message
     if n[1].kind in {nkStrLit..nkTripleStrLit}:
-      result = ropeFormatNamedVars(d.conf,
-        getConfigVar(d.conf, "doc.deprecationmsg"), ["label", "message"],
-        [~"Deprecated:", rope(xmltree.escape(n[1].strVal))])
+      result = getConfigVar(d.conf, "doc.deprecationmsg") % [
+          "label", "Deprecated:", "message", xmltree.escape(n[1].strVal)]
+    else:
+      result = ""
   else:
-    doAssert false
+    raiseAssert "unreachable"
 
 type DocFlags = enum
   kDefault
   kForceExport
 
-proc genSeeSrcRope(d: PDoc, path: string, line: int): Rope =
+proc genSeeSrc(d: PDoc, path: string, line: int): string =
+  result = ""
   let docItemSeeSrc = getConfigVar(d.conf, "doc.item.seesrc")
   if docItemSeeSrc.len > 0:
     let path = relativeTo(AbsoluteFile path, AbsoluteDir getCurrentDir(), '/')
@@ -834,119 +978,214 @@ proc genSeeSrcRope(d: PDoc, path: string, line: int): Rope =
         else: "version-$1-$2" % [$NimMajor, $NimMinor]
       let commit = getConfigVar(d.conf, "git.commit", defaultBranch)
       let develBranch = getConfigVar(d.conf, "git.devel", "devel")
-      dispA(d.conf, result, "$1", "", [ropeFormatNamedVars(d.conf, docItemSeeSrc,
-          ["path", "line", "url", "commit", "devel"], [rope path.string,
-          rope($line), rope gitUrl, rope commit, rope develBranch])])
+      dispA(d.conf, result, "$1", "", [docItemSeeSrc % [
+          "path", path.string, "line", $line, "url", gitUrl,
+          "commit", commit, "devel", develBranch]])
 
-proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
+proc symbolPriority(k: TSymKind): int =
+  result = case k
+    of skMacro: -3
+    of skTemplate: -2
+    of skIterator: -1
+    else: 0  # including skProc which have higher priority
+    # documentation itself has even higher priority 1
+
+proc getTypeKind(n: PNode): string =
+  case n[2].kind
+  of nkEnumTy: "enum"
+  of nkObjectTy: "object"
+  of nkTupleTy: "tuple"
+  else: ""
+
+proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
+  ## Converts symbol info (names/types/parameters) in `n` into format
+  ## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``.
+  result = LangSymbol(name: baseName.nimIdentNormalize,
+      symKind: k.toHumanStr
+  )
+  if k in routineKinds:
+    var
+      paramTypes: seq[string] = @[]
+    renderParamTypes(paramTypes, n[paramsPos], toNormalize=true)
+    let paramNames = renderParamNames(n[paramsPos], toNormalize=true)
+    # In some rare cases (system.typeof) parameter type is not set for default:
+    doAssert paramTypes.len <= paramNames.len
+    for i in 0 ..< paramNames.len:
+      if i < paramTypes.len:
+        result.parameters.add (paramNames[i], paramTypes[i])
+      else:
+        result.parameters.add (paramNames[i], "")
+    result.parametersProvided = true
+
+    result.outType = renderOutType(n[paramsPos], toNormalize=true)
+
+  if k in {skProc, skFunc, skType, skIterator}:
+    # Obtain `result.generics`
+    # Use `n[miscPos]` since n[genericParamsPos] does not contain constraints
+    var genNode: PNode = nil
+    if k == skType:
+      genNode = n[1]  # FIXME: what is index 1?
+    else:
+      if n[miscPos].kind != nkEmpty:
+        genNode = n[miscPos][1]   # FIXME: what is index 1?
+    if genNode != nil:
+      var literal = ""
+      var r: TSrcGen = initTokRender(genNode, {renderNoBody, renderNoComments,
+        renderNoPragmas, renderNoProcDefs, renderExpandUsing, renderNoPostfix})
+      var kind = tkEof
+      while true:
+        getNextTok(r, kind, literal)
+        if kind == tkEof:
+          break
+        if kind != tkSpaces:
+          result.generics.add(literal.nimIdentNormalize)
+
+  if k == skType: result.symTypeKind = getTypeKind(n)
+
+proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonExports: bool = false) =
   if (docFlags != kForceExport) and not isVisible(d, nameNode): return
   let
-    name = getName(d, nameNode)
-    nameRope = name.rope
+    name = getName(nameNode)
+    nameEsc = esc(d.target, name)
   var plainDocstring = getPlainDocstring(n) # call here before genRecComment!
-  var result: Rope = nil
+  var result = ""
   var literal, plainName = ""
   var kind = tkEof
-  var comm: Rope = nil
+  var comm: ItemPre = default(ItemPre)
   if n.kind in routineDefs:
     getAllRunnableExamples(d, n, comm)
   else:
     comm.add genRecComment(d, n)
 
-  var r: TSrcGen
   # Obtain the plain rendered string for hyperlink titles.
-  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
-    renderNoPragmas, renderNoProcDefs})
+  var r: TSrcGen = initTokRender(n, {renderNoBody, renderNoComments, renderDocComments,
+    renderNoPragmas, renderNoProcDefs, renderExpandUsing, renderNoPostfix})
   while true:
     getNextTok(r, kind, literal)
     if kind == tkEof:
       break
     plainName.add(literal)
 
-  var pragmaNode: PNode = nil
-  if n.isCallable and n[pragmasPos].kind != nkEmpty:
-    pragmaNode = findPragma(n[pragmasPos], wDeprecated)
+  var pragmaNode = getDeclPragma(n)
+  if pragmaNode != nil: pragmaNode = findPragma(pragmaNode, wDeprecated)
 
   inc(d.id)
   let
-    plainNameRope = rope(xmltree.escape(plainName.strip))
+    plainNameEsc = esc(d.target, plainName.strip)
+    typeDescr =
+      if k == skType and getTypeKind(n) != "": getTypeKind(n)
+      else: k.toHumanStr
+    detailedName = typeDescr & " " & (
+        if k in routineKinds: plainName else: name)
+    uniqueName = if k in routineKinds: plainNameEsc else: nameEsc
+    sortName = if k in routineKinds: plainName.strip else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
-    plainSymbolRope = rope(cleanPlainSymbol)
-    plainSymbolEncRope = rope(encodeUrl(cleanPlainSymbol))
-    itemIDRope = rope(d.id)
+    plainSymbolEnc = encodeUrl(cleanPlainSymbol, usePlus = false)
     symbolOrId = d.newUniquePlainSymbol(complexSymbol)
-    symbolOrIdRope = symbolOrId.rope
-    symbolOrIdEncRope = encodeUrl(symbolOrId).rope
-    deprecationMsgRope = genDeprecationMsg(d, pragmaNode)
+    symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false)
+    deprecationMsg = genDeprecationMsg(d, pragmaNode)
+    rstLangSymbol = toLangSymbol(k, n, cleanPlainSymbol)
+    symNameNode =
+      if nameNode.kind == nkPostfix: nameNode[1]
+      else: nameNode
 
-  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
-    renderDocComments, renderSyms}, symbolOrIdEncRope)
+  # we generate anchors automatically for subsequent use in doc comments
+  let lineinfo = rstast.TLineInfo(
+      line: nameNode.info.line, col: nameNode.info.col,
+      fileIndex: addRstFileIndex(d, nameNode.info))
+  addAnchorNim(d.sharedState, external = false, refn = symbolOrId,
+               tooltip = detailedName, langSym = rstLangSymbol,
+               priority = symbolPriority(k), info = lineinfo,
+               module = addRstFileIndex(d, FileIndex d.module.position))
 
-  let seeSrcRope = genSeeSrcRope(d, toFullPath(d.conf, n.info), n.info.line.int)
-  d.section[k].add(ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item"),
-    ["name", "header", "desc", "itemID", "header_plain", "itemSym",
-      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc", "deprecationMsg"],
-    [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope,
-      symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope,
-      deprecationMsgRope]))
+  var renderFlags = {renderNoBody, renderNoComments, renderDocComments,
+    renderSyms, renderExpandUsing, renderNoPostfix}
+  if nonExports:
+    renderFlags.incl renderNonExportedFields
+  nodeToHighlightedHtml(d, n, result, renderFlags, symbolOrIdEnc)
+
+  let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
+
+  d.section[k].secItems.mgetOrPut(cleanPlainSymbol, newSeq[Item]()).add Item(
+    descRst: comm,
+    sortName: sortName,
+    info: lineinfo,
+    anchor: symbolOrId,
+    detailedName: detailedName,
+    name: name,
+    substitutions: @[
+     "uniqueName", uniqueName,
+     "header", result, "itemID", $d.id,
+     "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
+     "itemSymEnc", plainSymbolEnc,
+     "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
+     "deprecationMsg", deprecationMsg])
 
   let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/').changeFileExt(HtmlExt).string
 
-  var attype: Rope
-  if k in routineKinds and nameNode.kind == nkSym:
+  var attype = ""
+  if k in routineKinds and symNameNode.kind == nkSym:
     let att = attachToType(d, nameNode.sym)
     if att != nil:
-      attype = rope esc(d.target, att.name.s)
-  elif k == skType and nameNode.kind == nkSym and nameNode.sym.typ.kind in {tyEnum, tyBool}:
-    let etyp = nameNode.sym.typ
+      attype = esc(d.target, att.name.s)
+  elif k == skType and symNameNode.kind == nkSym and
+      symNameNode.sym.typ.kind in {tyEnum, tyBool}:
+    let etyp = symNameNode.sym.typ
     for e in etyp.n:
       if e.sym.kind != skEnumField: continue
       let plain = renderPlainSymbolName(e)
       let symbolOrId = d.newUniquePlainSymbol(plain)
-      setIndexTerm(d[], external, symbolOrId, plain, nameNode.sym.name.s & '.' & plain,
-        xmltree.escape(getPlainDocstring(e).docstringSummary))
+      setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId,
+                   term = plain, linkTitle = symNameNode.sym.name.s & '.' & plain,
+                   linkDesc = xmltree.escape(getPlainDocstring(e).docstringSummary),
+                   line = n.info.line.int)
 
-  d.toc[k].add(ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item.toc"),
-    ["name", "header_plain", "itemSymOrIDEnc"],
-    [nameRope, plainNameRope, symbolOrIdEncRope]))
+  d.tocSimple[k].add TocItem(
+    sortName: sortName,
+    content: getConfigVar(d.conf, "doc.item.toc") % [
+      "name", name, "header_plain", plainNameEsc,
+      "itemSymOrIDEnc", symbolOrIdEnc])
 
-  d.tocTable[k].mgetOrPut(cleanPlainSymbol, nil).add(ropeFormatNamedVars(
-    d.conf, getConfigVar(d.conf, "doc.item.tocTable"),
-    ["name", "header_plain", "itemSymOrID", "itemSymOrIDEnc"],
-    [nameRope, plainNameRope, rope(symbolOrId.replace(",", ",")), symbolOrIdEncRope]))
+  d.tocTable[k].mgetOrPut(cleanPlainSymbol, newSeq[TocItem]()).add TocItem(
+    sortName: sortName,
+    content: getConfigVar(d.conf, "doc.item.tocTable") % [
+      "name", name, "header_plain", plainNameEsc,
+      "itemSymOrID", symbolOrId.replace(",", ","),
+      "itemSymOrIDEnc", symbolOrIdEnc])
 
-  # Ironically for types the complexSymbol is *cleaner* than the plainName
-  # because it doesn't include object fields or documentation comments. So we
-  # use the plain one for callable elements, and the complex for the rest.
-  var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": "
-  if n.isCallable: linkTitle.add(xmltree.escape(plainName.strip))
-  else: linkTitle.add(xmltree.escape(complexSymbol.strip))
+  setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId, term = name,
+               linkTitle = detailedName,
+               linkDesc = xmltree.escape(plainDocstring.docstringSummary),
+               line = n.info.line.int)
+  if k == skType and symNameNode.kind == nkSym:
+    d.types.strTableAdd symNameNode.sym
 
-  setIndexTerm(d[], external, symbolOrId, name, linkTitle,
-    xmltree.escape(plainDocstring.docstringSummary))
-  if k == skType and nameNode.kind == nkSym:
-    d.types.strTableAdd nameNode.sym
-
-proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
+proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind, nonExports = false): JsonItem =
   if not isVisible(d, nameNode): return
   var
-    name = getName(d, nameNode)
-    comm = $genRecComment(d, n)
+    name = getNameEsc(d, nameNode)
+    comm = genRecComment(d, n)
     r: TSrcGen
-  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
-  result = %{ "name": %name, "type": %($k), "line": %n.info.line.int,
-                 "col": %n.info.col}
-  if comm.len > 0:
-    result["description"] = %comm
+    renderFlags = {renderNoBody, renderNoComments, renderDocComments,
+      renderExpandUsing, renderNoPostfix}
+  if nonExports:
+    renderFlags.incl renderNonExportedFields
+  r = initTokRender(n, renderFlags)
+  result = JsonItem(json: %{ "name": %name, "type": %($k), "line": %n.info.line.int,
+                   "col": %n.info.col}
+  )
+  if comm != nil:
+    result.rst = comm
+    result.rstField = "description"
   if r.buf.len > 0:
-    result["code"] = %r.buf
+    result.json["code"] = %r.buf
   if k in routineKinds:
-    result["signature"] = newJObject()
+    result.json["signature"] = newJObject()
     if n[paramsPos][0].kind != nkEmpty:
-      result["signature"]["return"] = %($n[paramsPos][0])
+      result.json["signature"]["return"] = %($n[paramsPos][0])
     if n[paramsPos].len > 1:
-      result["signature"]["arguments"] = newJArray()
+      result.json["signature"]["arguments"] = newJArray()
     for paramIdx in 1 ..< n[paramsPos].len:
       for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2:
         let
@@ -954,22 +1193,61 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
           paramType = $n[paramsPos][paramIdx][^2]
         if n[paramsPos][paramIdx][^1].kind != nkEmpty:
           let paramDefault = $n[paramsPos][paramIdx][^1]
-          result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault}
+          result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault}
         else:
-          result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType}
+          result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType}
     if n[pragmasPos].kind != nkEmpty:
-      result["signature"]["pragmas"] = newJArray()
+      result.json["signature"]["pragmas"] = newJArray()
       for pragma in n[pragmasPos]:
-        result["signature"]["pragmas"].add %($pragma)
+        result.json["signature"]["pragmas"].add %($pragma)
     if n[genericParamsPos].kind != nkEmpty:
-      result["signature"]["genericParams"] = newJArray()
+      result.json["signature"]["genericParams"] = newJArray()
       for genericParam in n[genericParamsPos]:
         var param = %{"name": %($genericParam)}
-        if genericParam.sym.typ.sons.len > 0:
+        if genericParam.sym.typ.len > 0:
           param["types"] = newJArray()
-        for kind in genericParam.sym.typ.sons:
-          param["types"].add %($kind)
-        result["signature"]["genericParams"].add param
+        param["types"].add %($genericParam.sym.typ.elementType)
+        result.json["signature"]["genericParams"].add param
+  if optGenIndex in d.conf.globalOptions:
+    genItem(d, n, nameNode, k, kForceExport)
+
+proc setDoctype(d: PDoc, n: PNode) =
+  ## Processes `{.doctype.}` pragma changing Markdown/RST parsing options.
+  if n == nil:
+    return
+  if n.len != 2:
+    localError(d.conf, n.info, errUser,
+      "doctype pragma takes exactly 1 argument"
+    )
+    return
+  var dt = ""
+  case n[1].kind
+  of nkStrLit:
+    dt = toLowerAscii(n[1].strVal)
+  of nkIdent:
+    dt = toLowerAscii(n[1].ident.s)
+  else:
+    localError(d.conf, n.info, errUser,
+      "unknown argument type $1 provided to doctype" % [$n[1].kind]
+    )
+    return
+  case dt
+  of "markdown":
+    d.sharedState.options.incl roSupportMarkdown
+    d.sharedState.options.incl roPreferMarkdown
+  of "rstmarkdown":
+    d.sharedState.options.incl roSupportMarkdown
+    d.sharedState.options.excl roPreferMarkdown
+  of "rst":
+    d.sharedState.options.excl roSupportMarkdown
+    d.sharedState.options.excl roPreferMarkdown
+  else:
+    localError(d.conf, n.info, errUser,
+      (
+        "unknown doctype value \"$1\", should be from " &
+        "\"RST\", \"Markdown\", \"RstMarkdown\""
+      ) % [dt]
+    )
 
 proc checkForFalse(n: PNode): bool =
   result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
@@ -986,42 +1264,44 @@ proc traceDeps(d: PDoc, it: PNode) =
     for x in it[2]:
       a[2] = x
       traceDeps(d, a)
-  elif it.kind == nkSym and belongsToPackage(d.conf, it.sym):
+  elif it.kind == nkSym and belongsToProjectPackage(d.conf, it.sym):
     let external = externalDep(d, it.sym)
-    if d.section[k] != nil: d.section[k].add(", ")
-    dispA(d.conf, d.section[k],
+    if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
+    dispA(d.conf, d.section[k].finalMarkup,
           "$1",
-          "$1", [rope esc(d.target, external.prettyLink),
-          rope changeFileExt(external, "html")])
+          "$1", [esc(d.target, external.prettyLink),
+                 changeFileExt(external, "html")])
 
 proc exportSym(d: PDoc; s: PSym) =
   const k = exportSection
-  if s.kind == skModule and belongsToPackage(d.conf, s):
+  if s.kind == skModule and belongsToProjectPackage(d.conf, s):
     let external = externalDep(d, s)
-    if d.section[k] != nil: d.section[k].add(", ")
-    dispA(d.conf, d.section[k],
+    if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
+    dispA(d.conf, d.section[k].finalMarkup,
           "$1",
-          "$1", [rope esc(d.target, external.prettyLink),
-          rope changeFileExt(external, "html")])
+          "$1", [esc(d.target, external.prettyLink),
+                 changeFileExt(external, "html")])
   elif s.kind != skModule and s.owner != nil:
     let module = originatingModule(s)
-    if belongsToPackage(d.conf, module):
+    if belongsToProjectPackage(d.conf, module):
       let
         complexSymbol = complexName(s.kind, s.ast, s.name.s)
-        symbolOrIdRope = rope(d.newUniquePlainSymbol(complexSymbol))
+        symbolOrId = d.newUniquePlainSymbol(complexSymbol)
         external = externalDep(d, module)
-      if d.section[k] != nil: d.section[k].add(", ")
+      if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
       # XXX proper anchor generation here
-      dispA(d.conf, d.section[k],
+      dispA(d.conf, d.section[k].finalMarkup,
             "$1",
-            "$1", [rope esc(d.target, s.name.s),
-            rope changeFileExt(external, "html"),
-            symbolOrIdRope])
+            "$1", [esc(d.target, s.name.s),
+                   changeFileExt(external, "html"),
+                   symbolOrId])
 
 proc documentNewEffect(cache: IdentCache; n: PNode): PNode =
   let s = n[namePos].sym
   if tfReturnsNew in s.typ.flags:
     result = newIdentNode(getIdent(cache, "new"), n.info)
+  else:
+    result = nil
 
 proc documentEffect(cache: IdentCache; n, x: PNode, effectType: TSpecialWord, idx: int): PNode =
   let spec = effectSpec(x, effectType)
@@ -1031,10 +1311,11 @@ proc documentEffect(cache: IdentCache; n, x: PNode, effectType: TSpecialWord, id
     let actual = s.typ.n[0]
     if actual.len != effectListLen: return
     let real = actual[idx]
-
+    if real == nil: return
+    let realLen = real.len
     # warning: hack ahead:
-    var effects = newNodeI(nkBracket, n.info, real.len)
-    for i in 0.. 0:
     result = newTreeI(nkExprColonExpr, n.info,
       newIdentNode(getIdent(cache, pragmaName), n.info), effects)
+  else:
+    result = nil
 
 proc documentRaises*(cache: IdentCache; n: PNode) =
   if n[namePos].kind != nkSym: return
@@ -1065,8 +1350,9 @@ proc documentRaises*(cache: IdentCache; n: PNode) =
   let p3 = documentWriteEffect(cache, n, sfWrittenTo, "writes")
   let p4 = documentNewEffect(cache, n)
   let p5 = documentWriteEffect(cache, n, sfEscapes, "escapes")
+  let p6 = documentEffect(cache, n, pragmas, wForbids, forbiddenEffects)
 
-  if p1 != nil or p2 != nil or p3 != nil or p4 != nil or p5 != nil:
+  if p1 != nil or p2 != nil or p3 != nil or p4 != nil or p5 != nil or p6 != nil:
     if pragmas.kind == nkEmpty:
       n[pragmasPos] = newNodeI(nkPragma, n.info)
     if p1 != nil: n[pragmasPos].add p1
@@ -1074,15 +1360,22 @@ proc documentRaises*(cache: IdentCache; n: PNode) =
     if p3 != nil: n[pragmasPos].add p3
     if p4 != nil: n[pragmasPos].add p4
     if p5 != nil: n[pragmasPos].add p5
+    if p6 != nil: n[pragmasPos].add p6
 
-proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
+proc generateDoc*(d: PDoc, n, orig: PNode, config: ConfigRef, docFlags: DocFlags = kDefault) =
+  ## Goes through nim nodes recursively and collects doc comments.
+  ## Main function for `doc`:option: command,
+  ## which is implemented in ``docgen2.nim``.
   template genItemAux(skind) =
     genItem(d, n, n[namePos], skind, docFlags)
+  let showNonExports = optShowNonExportedFields in config.globalOptions
   case n.kind
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)
     d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode))
-  of nkCommentStmt: d.modDesc.add(genComment(d, n))
+    let doctypeNode = findPragma(n, wDoctype)
+    setDoctype(d, doctypeNode)
+  of nkCommentStmt: d.modDescPre.add(genComment(d, n))
   of nkProcDef, nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
     genItemAux(skProc)
@@ -1102,40 +1395,185 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
       if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         genItem(d, n[i], n[i][0],
-                succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags)
+                succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags, showNonExports)
   of nkStmtList:
-    for i in 0.. 1:
+          let refn = overloadGroupName(plainName, k)
+          let tooltip = "$1 ($2 overloads)" % [
+                      k.toHumanStr & " " & plainName, $overloadChoices.len]
+          let name = nimIdentBackticksNormalize(plainName)
+          # save overload group to ``.idx``
+          let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/').
+                         changeFileExt(HtmlExt).string
+          setIndexTerm(d[], ieNimGroup, htmlFile = external, id = refn,
+                       term = name, linkTitle = k.toHumanStr,
+                       linkDesc = "", line = overloadChoices[0].info.line.int)
+          if optGenIndexOnly in d.conf.globalOptions: continue
+          addAnchorNim(d.sharedState, external=false, refn, tooltip,
+                       LangSymbol(symKind: k.toHumanStr,
+                                  name: name,
+                                  isGroup: true),
+                       priority = symbolPriority(k),
+                       # select index `0` just to have any meaningful warning:
+                       info = overloadChoices[0].info,
+                       module = addRstFileIndex(d, FileIndex d.module.position))
+
+  if optGenIndexOnly in d.conf.globalOptions:
+    return
+
+  # Finalize fragments of ``.nim`` or ``.rst`` file
+  proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) =
+    for f in fragments:
+      case f.isRst:
+      of true:
+        var resolved = resolveSubs(d.sharedState, f.rst)
+        renderRstToOut(d[], resolved, result)
+      of false: result &= f.str
+  proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
+  for k in TSymKind:
+    # add symbols to section for each `k`, while optionally wrapping
+    # overloadable items with the same basic name by ``doc.item2``
+    let overloadableNames = toSeq(keys(d.section[k].secItems))
+    for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase):
+      var overloadChoices = d.section[k].secItems[plainName]
+      overloadChoices.sort(cmp)
+      var nameContent = ""
+      for item in overloadChoices:
+        var itemDesc: string = ""
+        renderItemPre(d, item.descRst, itemDesc)
+        nameContent.add(
+          getConfigVar(d.conf, "doc.item") % (
+              item.substitutions & @[
+                "desc", itemDesc,
+                "name", item.name,
+                "itemSymOrID", item.anchor]))
+      if k in routineKinds:
+        let plainNameEsc1 = esc(d.target, plainName.strip)
+        let plainNameEsc2 = esc(d.target, plainName.strip, escMode=emUrl)
+        d.section[k].finalMarkup.add(
+          getConfigVar(d.conf, "doc.item2") % (
+            @["header_plain", plainNameEsc1,
+              "overloadGroupName", overloadGroupName(plainNameEsc2, k),
+              "content", nameContent]))
+      else:
+        d.section[k].finalMarkup.add(nameContent)
+    d.section[k].secItems.clear
+  renderItemPre(d, d.modDescPre, d.modDescFinal)
+  d.modDescPre.setLen 0
+
+  # Finalize fragments of ``.json`` file
+  for i, entry in d.jEntriesPre:
+    if entry.rst != nil:
+      let resolved = resolveSubs(d.sharedState, entry.rst)
+      var str: string = ""
+      renderRstToOut(d[], resolved, str)
+      entry.json[entry.rstField] = %str
+      d.jEntriesPre[i].rst = nil
+
+    d.jEntriesFinal.add entry.json # generates docs
+
+  setIndexTitle(d, useMetaTitle = d.standaloneDoc)
+  completePass2(d.sharedState)
+
+proc add(d: PDoc; j: JsonItem) =
+  if j.json != nil or j.rst != nil: d.jEntriesPre.add j
+
+proc generateJson*(d: PDoc, n: PNode, config: ConfigRef, includeComments: bool = true) =
   case n.kind
+  of nkPragma:
+    let doctypeNode = findPragma(n, wDoctype)
+    setDoctype(d, doctypeNode)
   of nkCommentStmt:
     if includeComments:
-      d.add %*{"comment": genComment(d, n)}
+      d.add JsonItem(rst: genComment(d, n), rstField: "comment",
+                     json: %Table[string, string]())
     else:
-      d.modDesc.add(genComment(d, n))
+      d.modDescPre.add(genComment(d, n))
   of nkProcDef, nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
     d.add genJsonItem(d, n, n[namePos], skProc)
@@ -1157,20 +1595,20 @@ proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
       if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         d.add genJsonItem(d, n[i], n[i][0],
-                succ(skType, ord(n.kind)-ord(nkTypeSection)))
+                succ(skType, ord(n.kind)-ord(nkTypeSection)), optShowNonExportedFields in config.globalOptions)
   of nkStmtList:
     for i in 0..$1",
-        "\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle].rope])
+        "\\\\\\vspace{0.5em}\\large $1", [esc(d.target, d.meta[metaSubtitle])])
 
   var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
-  let bodyname = if d.hasToc and not d.isPureRst:
+  let bodyname = if d.hasToc and not d.standaloneDoc and not d.conf.isLatexCmd:
                    groupsection.setLen 0
                    "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"
                  else: "doc.body_no_toc"
-  let seeSrcRope = genSeeSrcRope(d, d.filename, 1)
-  content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title", "subtitle",
-      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg", "theindexhref", "body_toc_groupsection", "seeSrc"],
-      [title.rope, subtitle, toc, d.modDesc, rope(getDateStr()),
-      rope(getClockStr()), code, d.modDeprecationMsg, relLink(d.conf.outDir, d.destFile.AbsoluteFile, theindexFname.RelativeFile), groupsection.rope, seeSrcRope])
+  let seeSrc = genSeeSrc(d, d.filename, 1)
+  content = getConfigVar(d.conf, bodyname) % [
+      "title", title, "subtitle", subtitle,
+      "tableofcontents", toc, "moduledesc", d.modDescFinal, "date", getDateStr(),
+      "time", getClockStr(), "content", code,
+      "deprecationMsg", d.modDeprecationMsg,
+      "theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
+                              theindexFname.RelativeFile),
+      "body_toc_groupsection", groupsection, "seeSrc", seeSrc]
   if optCompileOnly notin d.conf.globalOptions:
     # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
-    code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
-        "nimdoccss", "dochackjs",  "title", "subtitle", "tableofcontents", "moduledesc", "date", "time",
-        "content", "author", "version", "analytics", "deprecationMsg"],
-        [relLink(d.conf.outDir, d.destFile.AbsoluteFile, nimdocOutCss.RelativeFile),
-        relLink(d.conf.outDir, d.destFile.AbsoluteFile, docHackJsFname.RelativeFile),
-        title.rope, subtitle, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
-        content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
+    code = getConfigVar(d.conf, "doc.file") % [
+        "nimdoccss", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
+                             nimdocOutCss.RelativeFile),
+        "dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
+                             docHackJsFname.RelativeFile),
+        "title", title, "subtitle", subtitle, "tableofcontents", toc,
+        "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(),
+        "content", content, "author", d.meta[metaAuthor],
+        "version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics,
+        "deprecationMsg", d.modDeprecationMsg, "nimVersion", $NimMajor & "." & $NimMinor & "." & $NimPatch]
   else:
     code = content
   result = code
 
+proc indexFile(d: PDoc): AbsoluteFile =
+  let dir = d.conf.outDir
+  result = dir / changeFileExt(presentationPath(d.conf,
+                                                AbsoluteFile d.filename),
+                               IndexExt)
+  let (finalDir, _, _) = result.string.splitFile
+  createDir(finalDir)
+
 proc generateIndex*(d: PDoc) =
   if optGenIndex in d.conf.globalOptions:
-    let dir = d.conf.outDir
-    createDir(dir)
-    let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt)
+    let dest = indexFile(d)
     writeIndexFile(d[], dest.string)
 
 proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
@@ -1305,22 +1773,31 @@ proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
         d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
 
 proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
+  if optGenIndexOnly in d.conf.globalOptions:
+    d.conf.outFile = indexFile(d).relativeTo(d.conf.outDir)  # just for display
+    return
   runAllExamples(d)
   var content = genOutFile(d, groupedToc)
   if optStdout in d.conf.globalOptions:
-    writeRope(stdout, content)
+    write(stdout, content)
   else:
     template outfile: untyped = d.destFile.AbsoluteFile
     #let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt)
     let dir = outfile.splitFile.dir
     createDir(dir)
     updateOutfile(d, outfile)
-    if not writeRope(content, outfile):
+    try:
+      writeFile(outfile, content)
+    except IOError:
       rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
         outfile.string)
-    elif not d.wroteSupportFiles: # nimdoc.css + dochack.js
+    if not d.wroteSupportFiles: # nimdoc.css + dochack.js
       let nimr = $d.conf.getPrefixDir()
-      copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
+      case d.target
+      of outHtml:
+        copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
+      of outLatex:
+        copyFile(docCls.interp(nimr = nimr), $d.conf.outDir / nimdocOutCls)
       if optGenIndex in d.conf.globalOptions:
         let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename())
         copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart)
@@ -1328,17 +1805,19 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
 
 proc writeOutputJson*(d: PDoc, useWarning = false) =
   runAllExamples(d)
-  var modDesc: string
-  for desc in d.modDesc:
+  var modDesc: string = ""
+  for desc in d.modDescFinal:
     modDesc &= desc
   let content = %*{"orig": d.filename,
     "nimble": getPackageName(d.conf, d.filename),
     "moduleDescription": modDesc,
-    "entries": d.jArray}
+    "entries": d.jEntriesFinal}
   if optStdout in d.conf.globalOptions:
-    write(stdout, $content)
+    writeLine(stdout, $content)
   else:
-    var f: File
+    let dir = d.destFile.splitFile.dir
+    createDir(dir)
+    var f: File = default(File)
     if open(f, d.destFile, fmWrite):
       write(f, $content)
       close(f)
@@ -1356,94 +1835,120 @@ proc handleDocOutputOptions*(conf: ConfigRef) =
     conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
 
 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
+  ## implementation of deprecated ``doc0`` command (without semantic checking)
   handleDocOutputOptions conf
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
-  var d = newDocumentor(conf.projectFull, cache, conf)
-  d.hasToc = true
-  generateDoc(d, ast, ast)
+  var d = newDocumentor(conf.projectFull, cache, conf, hasToc = true)
+  generateDoc(d, ast, ast, conf)
+  finishGenerateDoc(d)
   writeOutput(d)
   generateIndex(d)
 
 proc commandRstAux(cache: IdentCache, conf: ConfigRef;
-                   filename: AbsoluteFile, outExt: string) =
+                   filename: AbsoluteFile, outExt: string,
+                   preferMarkdown: bool) =
   var filen = addFileExt(filename, "txt")
-  var d = newDocumentor(filen, cache, conf, outExt)
+  var d = newDocumentor(filen, cache, conf, outExt, standaloneDoc = true,
+                        preferMarkdown = preferMarkdown, hasToc = false)
+  try:
+    let rst = parseRst(readFile(filen.string),
+                      line=LineRstInit, column=ColRstInit,
+                      conf, d.sharedState)
+    d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
+    finishGenerateDoc(d)
+    writeOutput(d)
+    generateIndex(d)
+  except ERecoverableError:
+    discard "already reported the error"
 
-  d.isPureRst = true
-  var rst = parseRst(readFile(filen.string), filen.string,
-                     line=LineRstInit, column=ColRstInit,
-                     d.hasToc, {roSupportRawDirective, roSupportMarkdown}, conf)
-  var modDesc = newStringOfCap(30_000)
-  renderRstToOut(d[], rst, modDesc)
-  d.modDesc = rope(modDesc)
-  writeOutput(d)
-  generateIndex(d)
+proc commandRst2Html*(cache: IdentCache, conf: ConfigRef,
+                      preferMarkdown=false) =
+  commandRstAux(cache, conf, conf.projectFull, HtmlExt, preferMarkdown)
 
-proc commandRst2Html*(cache: IdentCache, conf: ConfigRef) =
-  commandRstAux(cache, conf, conf.projectFull, HtmlExt)
-
-proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) =
-  commandRstAux(cache, conf, conf.projectFull, TexExt)
+proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef,
+                     preferMarkdown=false) =
+  commandRstAux(cache, conf, conf.projectFull, TexExt, preferMarkdown)
 
 proc commandJson*(cache: IdentCache, conf: ConfigRef) =
+  ## implementation of a deprecated jsondoc0 command
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
-  var d = newDocumentor(conf.projectFull, cache, conf)
+  var d = newDocumentor(conf.projectFull, cache, conf, hasToc = true)
   d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
-                          status: int; content: string) =
+                          status: int; content: string) {.gcsafe.} =
     localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
                warnUser, "the ':test:' attribute is not supported by this backend")
-  d.hasToc = true
-  generateJson(d, ast)
-  let json = d.jArray
-  let content = rope(pretty(json))
+  generateJson(d, ast, conf)
+  finishGenerateDoc(d)
+  let json = d.jEntriesFinal
+  let content = pretty(json)
 
   if optStdout in d.conf.globalOptions:
-    writeRope(stdout, content)
+    write(stdout, content)
   else:
     #echo getOutFile(gProjectFull, JsonExt)
     let filename = getOutFile(conf, RelativeFile conf.projectName, JsonExt)
-    if not writeRope(content, filename):
+    try:
+      writeFile(filename, content)
+    except IOError:
       rawMessage(conf, errCannotOpenFile, filename.string)
 
 proc commandTags*(cache: IdentCache, conf: ConfigRef) =
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
-  var d = newDocumentor(conf.projectFull, cache, conf)
+  var d = newDocumentor(conf.projectFull, cache, conf, hasToc = true)
   d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
-                          status: int; content: string) =
+                          status: int; content: string) {.gcsafe.} =
     localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
                warnUser, "the ':test:' attribute is not supported by this backend")
-  d.hasToc = true
   var
-    content: Rope
+    content = ""
   generateTags(d, ast, content)
 
   if optStdout in d.conf.globalOptions:
-    writeRope(stdout, content)
+    write(stdout, content)
   else:
     #echo getOutFile(gProjectFull, TagsExt)
     let filename = getOutFile(conf, RelativeFile conf.projectName, TagsExt)
-    if not writeRope(content, filename):
+    try:
+      writeFile(filename, content)
+    except IOError:
       rawMessage(conf, errCannotOpenFile, filename.string)
 
 proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
-  var content = mergeIndexes(dir).rope
+  if optGenIndexOnly in conf.globalOptions:
+    return
+  var content = mergeIndexes(dir)
 
   var outFile = outFile
   if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
   let filename = getOutFile(conf, outFile, HtmlExt)
 
-  let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
-      "nimdoccss", "dochackjs",
-      "title", "subtitle", "tableofcontents", "moduledesc", "date", "time",
-      "content", "author", "version", "analytics"],
-      [relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
-      relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
-      rope"Index", rope"", nil, nil, rope(getDateStr()),
-      rope(getClockStr()), content, nil, nil, nil])
+  let code = getConfigVar(conf, "doc.file") % [
+      "nimdoccss", relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
+      "dochackjs", relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
+      "title", "Index",
+      "subtitle", "", "tableofcontents", "", "moduledesc", "",
+      "date", getDateStr(), "time", getClockStr(),
+      "content", content, "author", "", "version", "", "analytics", "", "nimVersion", $NimMajor & "." & $NimMinor & "." & $NimPatch]
   # no analytics because context is not available
 
-  if not writeRope(code, filename):
+  try:
+    writeFile(filename, code)
+  except IOError:
+    rawMessage(conf, errCannotOpenFile, filename.string)
+
+proc commandBuildIndexJson*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
+  var (modules, symbols, docs) = readIndexDir(dir)
+  let documents = toSeq(keys(Table[IndexEntry, seq[IndexEntry]](docs)))
+  let body = %*({"documents": documents, "modules": modules, "symbols": symbols})
+
+  var outFile = outFile
+  if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
+  let filename = getOutFile(conf, outFile, JsonExt)
+
+  try:
+    writeFile(filename, $body)
+  except IOError:
     rawMessage(conf, errCannotOpenFile, filename.string)
diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim
index cfbb33156f..7fb11a3bd7 100644
--- a/compiler/docgen2.nim
+++ b/compiler/docgen2.nim
@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, passes, docgen, lineinfos, pathutils
+  options, ast, msgs, docgen, lineinfos, pathutils, packages
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -23,7 +23,7 @@ type
   PGen = ref TGen
 
 proc shouldProcess(g: PGen): bool =
-  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
+  (optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
@@ -31,31 +31,34 @@ template closeImpl(body: untyped) {.dirty.} =
   let useWarning = sfMainModule notin g.module.flags
   let groupedToc = true
   if shouldProcess(g):
+    finishGenerateDoc(g.doc)
     body
     try:
       generateIndex(g.doc)
     except IOError:
       discard
 
-proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeDoc*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+  result = nil
   closeImpl:
     writeOutput(g.doc, useWarning, groupedToc)
 
-proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeJson*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+  result = nil
   closeImpl:
     writeOutputJson(g.doc, useWarning)
 
-proc processNode(c: PPassContext, n: PNode): PNode =
+proc processNode*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateDoc(g.doc, n, n)
+    generateDoc(g.doc, n, n, g.config)
 
-proc processNodeJson(c: PPassContext, n: PNode): PNode =
+proc processNodeJson*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateJson(g.doc, n, false)
+    generateJson(g.doc, n, g.config, false)
 
 template myOpenImpl(ext: untyped) {.dirty.} =
   var g: PGen
@@ -63,20 +66,15 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.module = module
   g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
-      graph.cache, graph.config, ext, module)
-  d.hasToc = true
+      graph.cache, graph.config, ext, module, hasToc = true)
   g.doc = d
   result = g
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openHtml*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(HtmlExt)
 
-proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openTex*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+  myOpenImpl(TexExt)
+
+proc openJson*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(JsonExt)
-
-const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
-                                  close = closeJson)
-
-proc finishDoc2Pass*(project: string) =
-  discard
diff --git a/compiler/enumtostr.nim b/compiler/enumtostr.nim
index 9bfa7001a9..dc516d2e52 100644
--- a/compiler/enumtostr.nim
+++ b/compiler/enumtostr.nim
@@ -1,16 +1,20 @@
 
 import ast, idents, lineinfos, modulegraphs, magicsys
 
-proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info)
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+
+proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
+  result = newSym(skProc, getIdent(g.cache, "$"), idgen, t.owner, info)
+
+  let dest = newSym(skParam, getIdent(g.cache, "e"), idgen, result, info)
   dest.typ = t
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), idgen, result, info)
   res.typ = getSysType(g, info, tyString)
 
-  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
+  result.typ = newType(tyProc, idgen, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)
@@ -26,7 +30,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGener
     assert(t.n[i].kind == nkSym)
     var field = t.n[i].sym
     let val = if field.ast == nil: field.name.s else: field.ast.strVal
-    caseStmt.add newTree(nkOfBranch, newSymNode(field),
+    caseStmt.add newTree(nkOfBranch, newIntTypeNode(field.position, t),
       newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newStrNode(val, info))))
     #newIntTypeNode(nkIntLit, field.position, t)
 
@@ -52,26 +56,27 @@ proc searchObjCaseImpl(obj: PNode; field: PSym): PNode =
     if obj.kind == nkRecCase and obj[0].kind == nkSym and obj[0].sym == field:
       result = obj
     else:
+      result = nil
       for x in obj:
         result = searchObjCaseImpl(x, field)
         if result != nil: break
 
 proc searchObjCase(t: PType; field: PSym): PNode =
   result = searchObjCaseImpl(t.n, field)
-  if result == nil and t.len > 0:
-    result = searchObjCase(t[0].skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
+  if result == nil and t.baseClass != nil:
+    result = searchObjCase(t.baseClass.skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
   doAssert result != nil
 
 proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextSymId idgen, t.owner, info)
+  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), idgen, t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), idgen, result, info)
   dest.typ = field.typ
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), idgen, result, info)
   res.typ = getSysType(g, info, tyUInt8)
 
-  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
+  result.typ = newType(tyProc, idgen, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)
diff --git a/compiler/errorhandling.nim b/compiler/errorhandling.nim
new file mode 100644
index 0000000000..2cde9e3fb0
--- /dev/null
+++ b/compiler/errorhandling.nim
@@ -0,0 +1,85 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module contains support code for new-styled error
+## handling via an `nkError` node kind.
+
+import ast, renderer, options, types
+import std/strutils
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
+type
+  ErrorKind* = enum ## expand as you need.
+    RawTypeMismatchError
+    ExpressionCannotBeCalled
+    CustomError
+    WrongNumberOfArguments
+    AmbiguousCall
+
+proc errorSubNode*(n: PNode): PNode =
+  case n.kind
+  of nkEmpty..nkNilLit:
+    result = nil
+  of nkError:
+    result = n
+  else:
+    result = nil
+    for i in 0.. 1
+  let wrongNode = n[0]
+  case ErrorKind(n[1].intVal)
+  of RawTypeMismatchError:
+    result = "type mismatch"
+  of ExpressionCannotBeCalled:
+    result = "expression '$1' cannot be called" % wrongNode[0].renderTree
+  of CustomError:
+    result = n[2].strVal
+  of WrongNumberOfArguments:
+    result = "wrong number of arguments"
+  of AmbiguousCall:
+    let a = n[2].sym
+    let b = n[3].sym
+    var args = "("
+    for i in 1.. 1: args.add(", ")
+      args.add(typeToString(wrongNode[i].typ))
+    args.add(")")
+    result = "ambiguous call; both $1 and $2 match for: $3" % [
+      getProcHeader(config, a),
+      getProcHeader(config, b),
+      args]
diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim
index 0d471cf98f..9871c81af6 100644
--- a/compiler/evalffi.nim
+++ b/compiler/evalffi.nim
@@ -9,9 +9,11 @@
 
 ## This file implements the FFI part of the evaluator for Nim code.
 
-import ast, types, options, tables, dynlib, msgs, lineinfos
-from os import getAppFilename
-import pkg/libffi
+import ast, types, options, msgs, lineinfos
+from std/os import getAppFilename
+import libffi/libffi
+
+import std/[tables, dynlib]
 
 when defined(windows):
   const libcDll = "msvcrt.dll"
@@ -37,9 +39,10 @@ else:
   var gExeHandle = loadLib()
 
 proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo): pointer =
+  result = nil
   if dll in cache:
     return cache[dll]
-  var libs: seq[string]
+  var libs: seq[string] = @[]
   libCandidates(dll, libs)
   for c in libs:
     result = loadLib(c)
@@ -61,23 +64,23 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
     let lib = sym.annex
     if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}:
       globalError(conf, sym.info, "dynlib needs to be a string lit")
-    var theAddr: pointer
+    var theAddr: pointer = nil
     if (lib.isNil or lib.kind == libHeader) and not gExeHandle.isNil:
       libPathMsg = "current exe: " & getAppFilename() & " nor libc: " & libcDll
       # first try this exe itself:
-      theAddr = gExeHandle.symAddr(name)
+      theAddr = gExeHandle.symAddr(name.cstring)
       # then try libc:
       if theAddr.isNil:
         let dllhandle = getDll(conf, gDllCache, libcDll, sym.info)
-        theAddr = dllhandle.symAddr(name)
+        theAddr = dllhandle.symAddr(name.cstring)
     elif not lib.isNil:
       let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
       libPathMsg = dll
       let dllhandle = getDll(conf, gDllCache, dll, sym.info)
-      theAddr = dllhandle.symAddr(name)
+      theAddr = dllhandle.symAddr(name.cstring)
     if theAddr.isNil: globalError(conf, sym.info,
       "cannot import symbol: " & name & " from " & libPathMsg)
-    result.intVal = cast[ByteAddress](theAddr)
+    result.intVal = cast[int](theAddr)
 
 proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
   if t == nil: return addr libffi.type_void
@@ -92,11 +95,11 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
     else: result = nil
   of tyFloat, tyFloat64: result = addr libffi.type_double
   of tyFloat32: result = addr libffi.type_float
-  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyUntyped,
+  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCstring, tySequence, tyString, tyUntyped,
      tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
-    result = mapType(conf, t[0])
+    result = mapType(conf, t.skipModifier)
   else:
     result = nil
   # too risky:
@@ -108,12 +111,13 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI
   of ccStdCall: result = when defined(windows) and defined(x86): STDCALL else: DEFAULT_ABI
   of ccCDecl: result = DEFAULT_ABI
   else:
+    result = default(TABI)
     globalError(conf, info, "cannot map calling convention to FFI")
 
-template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
-template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
+template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[]
+template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v
 template `+!`(x, y: untyped): untyped =
-  cast[pointer](cast[ByteAddress](x) + y)
+  cast[pointer](cast[int](x) + y)
 
 proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
   ## computes the size of the blob
@@ -122,16 +126,18 @@ proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
-      result = sizeof(pointer) + packSize(conf, v[0], typ.lastSon)
+      result = sizeof(pointer) + packSize(conf, v[0], typ.elementType)
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = packSize(conf, v, typ[0])
+    result = packSize(conf, v, typ.skipModifier)
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
     # we use the real length here instead
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     elif v.len != 0:
-      result = v.len * packSize(conf, v[0], typ[1])
+      result = v.len * packSize(conf, v[0], typ.elementType)
+    else:
+      result = 0
   else:
     result = getSize(conf, typ).int
 
@@ -140,6 +146,7 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer)
 proc getField(conf: ConfigRef, n: PNode; position: int): PSym =
   case n.kind
   of nkRecList:
+    result = nil
     for i in 0..= x and branch <= y:
+          return i
+      elif getOrdValue(obj[i][j]) == branch:
+        return i
+    if obj[i].len == 1:
+      # else branch
+      return i
+  return 1
+
+template newZero(t: PType; info: TLineInfo; k = nkIntLit): PNode = newNodeIT(k, info, t)
+
+proc expandDefault*(t: PType; info: TLineInfo): PNode
+
+proc expandField(s: PSym; info: TLineInfo): PNode =
+  result = newNodeIT(nkExprColonExpr, info, s.typ)
+  result.add newSymNode(s)
+  result.add expandDefault(s.typ, info)
+
+proc expandDefaultN(n: PNode; info: TLineInfo; res: PNode) =
+  case n.kind
+  of nkRecList:
+    for i in 0.. 0:
+    flushDot(conf)
+    stderr.writeLine(cmd)
+
 proc nameToCC*(name: string): TSystemCC =
   ## Returns the kind of compiler referred to by `name`, or ccNone
   ## if the name doesn't refer to any known compiler.
@@ -306,7 +319,7 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   var fullSuffix = suffix
   case conf.backend
   of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix
-  of backendC: discard
+  of backendC, backendNir: discard
   of backendInvalid:
     # during parsing of cfg files; we don't know the backend yet, no point in
     # guessing wrong thing
@@ -318,7 +331,9 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
                      platform.OS[conf.target.targetOS].name & '.' &
                      CC[c].name & fullSuffix
     result = getConfigVar(conf, fullCCname)
-    if result.len == 0:
+    if existsConfigVar(conf, fullCCname):
+      result = getConfigVar(conf, fullCCname)
+    else:
       # not overridden for this cross compilation setting?
       result = getConfigVar(conf, CC[c].name & fullSuffix)
   else:
@@ -362,6 +377,7 @@ proc initVars*(conf: ConfigRef) =
 
 proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
                         createSubDir: bool = true): AbsoluteFile =
+  ## Generate the absolute file path to the generated modules.
   result = completeGeneratedFilePath(conf, cfile, createSubDir)
 
 proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
@@ -372,7 +388,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
   conf.toCompile.add(cf)
 
 proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
-  let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
+  let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string
   var value = conf.cfileSpecificOptions.getOrDefault(key)
   if strutils.find(value, option, 0) < 0:
     addOpt(value, option)
@@ -425,7 +441,13 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} =
   # really: Cross compilation from Linux to Linux for example is entirely
   # reasonable.
   # `optGenMapping` is included here for niminst.
-  result = conf.globalOptions * {optGenScript, optGenMapping} != {}
+  # We use absolute paths for vcc / cl, see issue #19883.
+  let options =
+    if conf.cCompiler == ccVcc:
+      {optGenMapping}
+    else:
+      {optGenScript, optGenMapping}
+  result = conf.globalOptions * options != {}
 
 proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string =
   result = conf.compileOptions
@@ -465,6 +487,10 @@ proc vccplatform(conf: ConfigRef): string =
         of cpuArm: " --platform:arm"
         of cpuAmd64: " --platform:amd64"
         else: ""
+    else:
+      result = ""
+  else:
+    result = ""
 
 proc getLinkOptions(conf: ConfigRef): string =
   result = conf.linkOptions & " " & conf.linkOptionsCmd & " "
@@ -478,7 +504,10 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
            (conf.target.hostOS == osWindows)
 
 proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
-  conf.backend == backendCpp and not cfile.string.endsWith(".c")
+  # List of possible file extensions taken from gcc
+  for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]:
+    if cfile.string.endsWith(ext): return true
+  false
 
 proc envFlags(conf: ConfigRef): string =
   result = if conf.backend == backendCpp:
@@ -486,14 +515,14 @@ proc envFlags(conf: ConfigRef): string =
           else:
             getEnv("CFLAGS")
 
-proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
+proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string =
   if compiler == ccEnv:
-    result = if useCpp(conf, cfile):
+    result = if isCpp:
               getEnv("CXX")
             else:
               getEnv("CC")
   else:
-    result = if useCpp(conf, cfile):
+    result = if isCpp:
               CC[compiler].cppCompiler
             else:
               CC[compiler].compilerExe
@@ -507,47 +536,36 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
     result = false # assume an old or crappy GCC
     var exe = getConfigVar(conf, conf.cCompiler, ".exe")
     if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
-    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    # NOTE: should we need the full version, use -dumpfullversion
+    let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1)
     if exitCode == 0:
-      var i = 0
-      var j = 0
-      # the version is the last part of the first line:
-      while i < s.len and s[i] != '\n':
-        if s[i] in {' ', '\t'}: j = i+1
-        inc i
-      if j > 0:
-        var major = 0
-        while j < s.len and s[j] in {'0'..'9'}:
-          major = major * 10 + (ord(s[j]) - ord('0'))
-          inc j
-        if i < s.len and s[j] == '.': inc j
-        while j < s.len and s[j] in {'0'..'9'}:
-          inc j
-        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
-          # we found a third version number, chances are high
-          # we really parsed the version:
-          result = major >= 5
+      var major: int = 0
+      discard parseInt(s, major)
+      result = major >= 5
   else:
     result = conf.cCompiler == ccCLang
 
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
-           else: getCompilerExe(conf, compiler, AbsoluteFile"")
+           else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp)
 
 proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
                          isMainFile = false; produceOutput = false): string =
-  let c = conf.cCompiler
+  let
+    c = conf.cCompiler
+    isCpp = useCpp(conf, cfile.cname)
   # We produce files like module.nim.cpp, so the absolute Nim filename is not
   # cfile.name but `cfile.cname.changeFileExt("")`:
   var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
-  if useCpp(conf, cfile.cname):
+  if isCpp:
     # needs to be prepended so that --passc:-std=c++17 can override default.
     # we could avoid allocation by making cFileSpecificOptions inplace
     options = CC[c].cppXsupport & ' ' & options
+    # If any C++ file was compiled, we need to use C++ driver for linking as well
+    incl conf.globalOptions, optMixedMode
 
   var exe = getConfigVar(conf, c, ".exe")
-  if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
+  if exe.len == 0: exe = getCompilerExe(conf, c, isCpp)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
   if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
@@ -567,7 +585,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
 
     compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    compilePattern = getCompilerExe(conf, c, cfile.cname)
+    compilePattern = exe
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
 
@@ -609,7 +627,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
           "for the selected C compiler: " & CC[conf.cCompiler].name)
 
   result.add(' ')
-  result.addf(CC[c].compileTmpl, [
+  strutils.addf(result, CC[c].compileTmpl, [
     "dfile", dfile,
     "file", cfsh, "objfile", quoteShell(objfile),
     "options", options, "include", includeCmd,
@@ -629,9 +647,9 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
   if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good
 
-  let hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
+  let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1")
   let currentHash = footprint(conf, cfile)
-  var f: File
+  var f: File = default(File)
   if open(f, hashFile.string, fmRead):
     let oldHash = parseSecureHash(f.readLine())
     close(f)
@@ -644,8 +662,10 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
       close(f)
 
 proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
+  # we want to generate the hash file unconditionally
+  let extFileChanged = externalFileChanged(conf, c)
   if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
-      not externalFileChanged(conf, c):
+      not extFileChanged:
     c.flags.incl CfileFlag.Cached
   else:
     # make sure Nim keeps recompiling the external file on reruns
@@ -660,11 +680,13 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
   addExternalFileToCompile(conf, c)
 
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
-                objfiles: string, isDllBuild: bool): string =
+                objfiles: string, isDllBuild: bool, removeStaticFile: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    removeFile output # fixes: bug #16947
+    if removeStaticFile:
+      removeFile output # fixes: bug #16947
     result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
-                                            "objfiles", objfiles]
+                                            "objfiles", objfiles,
+                                            "vccplatform", vccplatform(conf)]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
     if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
@@ -697,7 +719,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
         "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles,
         "exefile", exefile, "nim", getPrefixDir(conf).string, "lib", conf.libpath.string])
     result.add ' '
-    result.addf(linkTmpl, ["builddll", builddll,
+    strutils.addf(result, linkTmpl, ["builddll", builddll,
         "mapfile", mapfile,
         "buildgui", buildgui, "options", linkOptions,
         "objfiles", objfiles, "exefile", exefile,
@@ -745,8 +767,9 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
   if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc:
     result.add " /Zi /FS /Od"
 
-template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string =
-  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions)
+template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string,
+                    removeStaticFile = false): string =
+  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile)
 
 template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) =
   try:
@@ -761,6 +784,7 @@ template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body
     raise
 
 proc getExtraCmds(conf: ConfigRef; output: AbsoluteFile): seq[string] =
+  result = @[]
   when defined(macosx):
     if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions:
       # if needed, add an option to skip or override location
@@ -816,10 +840,22 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
   else:
     writeFile(linkerArgs, args)
   try:
-    execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
+    when defined(macosx):
+      execLinkCmd(conf, "xargs " & cmd.substr(0, last) & " < " & linkerArgs)
+    else:
+      execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
   finally:
     removeFile(linkerArgs)
 
+proc linkViaShellScript(conf: ConfigRef; cmd: string) =
+  let linkerScript = conf.projectName & "_" & "linkerScript.sh"
+  writeFile(linkerScript, cmd)
+  let shell = getEnv("SHELL")
+  try:
+    execLinkCmd(conf, shell & " " & linkerScript)
+  finally:
+    removeFile(linkerScript)
+
 proc getObjFilePath(conf: ConfigRef, f: Cfile): string =
   if noAbsolutePaths(conf): f.obj.extractFilename
   else: f.obj.string
@@ -831,28 +867,39 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
   result = conf.getNimcacheDir / RelativeFile(targetName)
 
 proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
+  result = ""
   if conf.hasHint(hintCC):
     if optListCmd in conf.globalOptions or conf.verbosity > 1:
-      result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
+      result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd)
     else:
-      result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
+      result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name)
+
+proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) =
+  # Prevent linkcmd from exceeding the maximum command line length.
+  # Windows's command line limit is about 8K (8191 characters) so C compilers on
+  # Windows support a feature where the command line can be passed via ``@linkcmd``
+  # to them.
+  const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000
+  if linkCmd.len > MaxCmdLen:
+    when defined(macosx):
+      linkViaShellScript(conf, linkCmd)
+    else:
+      linkViaResponseFile(conf, linkCmd)
+  else:
+    execLinkCmd(conf, linkCmd)
 
 proc callCCompiler*(conf: ConfigRef) =
   var
-    linkCmd: string
+    linkCmd: string = ""
     extraCmds: seq[string]
   if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}:
     return # speed up that call if only compiling and no script shall be
            # generated
   #var c = cCompiler
-  var script: Rope = nil
-  var cmds: TStringSeq
-  var prettyCmds: TStringSeq
-  let prettyCb = proc (idx: int) =
-    if prettyCmds[idx].len > 0:
-      flushDot(conf)
-      # xxx should probably use stderr like other compiler messages, not stdout
-      echo prettyCmds[idx]
+  var script: Rope = ""
+  var cmds: TStringSeq = default(TStringSeq)
+  var prettyCmds: TStringSeq = default(TStringSeq)
+  let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
 
   for idx, it in conf.toCompile:
     # call the C compiler for the .c file:
@@ -890,7 +937,7 @@ proc callCCompiler*(conf: ConfigRef) =
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB: MSVC flag is used
         if isVSCompatible(conf):
@@ -912,18 +959,12 @@ proc callCCompiler*(conf: ConfigRef) =
         objfiles.add(' ')
         objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
-                       else: AbsoluteFile(conf.projectName)
-      linkCmd = getLinkCmd(conf, mainOutput, objfiles)
+                       else: AbsoluteFile(conf.outFile)
+
+      linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true)
       extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
-        const MaxCmdLen = when defined(windows): 8_000 else: 32_000
-        if linkCmd.len > MaxCmdLen:
-          # Windows's command line limit is about 8K (don't laugh...) so C compilers on
-          # Windows support a feature where the command line can be passed via ``@linkcmd``
-          # to them.
-          linkViaResponseFile(conf, linkCmd)
-        else:
-          execLinkCmd(conf, linkCmd)
+        preventLinkCmdMaxCmdLen(conf, linkCmd)
         for cmd in extraCmds:
           execExternalProgram(conf, cmd, hintExecuting)
   else:
@@ -933,210 +974,108 @@ proc callCCompiler*(conf: ConfigRef) =
     script.add("\n")
     generateScript(conf, script)
 
-
 template hashNimExe(): string = $secureHashFile(os.getAppFilename())
 
-proc writeJsonBuildInstructions*(conf: ConfigRef) =
-  template lit(x: string) = f.write x
-  template str(x: string) =
-    buf.setLen 0
-    escapeJson(x, buf)
-    f.write buf
+proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile =
+  # `outFile` is better than `projectName`, as it allows having different json
+  # files for a given source file compiled with different options; it also
+  # works out of the box with `hashMainCompilationParams`.
+  result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json")
 
-  proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) =
-    var comma = false
-    for i, it in clist:
-      if CfileFlag.Cached in it.flags: continue
-      let compileCmd = getCompileCFileCmd(conf, it)
-      if comma: lit ",\L" else: comma = true
-      lit "["
-      str it.cname.string
-      lit ", "
-      str compileCmd
-      lit "]"
+const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes
+type BuildCache = object
+  cacheVersion: string
+  outputFile: string
+  compile: seq[(string, string)]
+  link: seq[string]
+  linkcmd: string
+  extraCmds: seq[string]
+  configFiles: seq[string] # the hash shouldn't be needed
+  stdinInput: bool
+  projectIsCmd: bool
+  cmdInput: string
+  currentDir: string
+  cmdline: string
+  depfiles: seq[(string, string)]
+  nimexe: string
 
-  proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList;
-                 llist: seq[string]) =
-    var pastStart = false
-    for it in llist:
-      let objfile = if noAbsolutePaths(conf): it.extractFilename
-                    else: it
-      let objstr = addFileExt(objfile, CC[conf.cCompiler].objExt)
-      objfiles.add(' ')
-      objfiles.add(objstr)
-      if pastStart: lit ",\L"
-      str objstr
-      pastStart = true
-
-    for it in clist:
-      let objstr = quoteShell(it.obj)
-      objfiles.add(' ')
-      objfiles.add(objstr)
-      if pastStart: lit ",\L"
-      str objstr
-      pastStart = true
-    lit "\L"
-
-  proc depfiles(conf: ConfigRef; f: File; buf: var string) =
-    var i = 0
+proc writeJsonBuildInstructions*(conf: ConfigRef; deps: StringTableRef) =
+  var linkFiles = collect(for it in conf.externalToLink:
+    var it = it
+    if conf.noAbsolutePaths: it = it.extractFilename
+    it.addFileExt(CC[conf.cCompiler].objExt))
+  for it in conf.toCompile: linkFiles.add it.obj.string
+  var bcache = BuildCache(
+    cacheVersion: cacheVersion,
+    outputFile: conf.absOutFile.string,
+    compile: collect(for i, it in conf.toCompile:
+      if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))),
+    link: linkFiles,
+    linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand),
+    extraCmds: getExtraCmds(conf, conf.absOutFile),
+    stdinInput: conf.projectIsStdin,
+    projectIsCmd: conf.projectIsCmd,
+    cmdInput: conf.cmdInput,
+    configFiles: conf.configFiles.mapIt(it.string),
+    currentDir: getCurrentDir())
+  if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
+    bcache.cmdline = conf.commandLine
     for it in conf.m.fileInfos:
       let path = it.fullPath.string
       if isAbsolute(path): # TODO: else?
-        if i > 0: lit "],\L"
-        lit "["
-        str path
-        lit ", "
-        str $secureHashFile(path)
-        inc i
-    lit "]\L"
+        if path in deps:
+          bcache.depfiles.add (path, deps[path])
+        else: # backup for configs etc.
+          bcache.depfiles.add (path, $secureHashFile(path))
 
+    bcache.nimexe = hashNimExe()
+  conf.jsonBuildFile = conf.jsonBuildInstructionsFile
+  conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty)
 
-  var buf = newStringOfCap(50)
-
-  let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
-  conf.jsonBuildFile = jsonFile
-  let output = conf.absOutFile
-
-  var f: File
-  if open(f, jsonFile.string, fmWrite):
-    lit "{\L"
-    lit "\"outputFile\": "
-    str $output
-
-    lit ",\L\"compile\":[\L"
-    cfiles(conf, f, buf, conf.toCompile, false)
-    lit "],\L\"link\":[\L"
-    var objfiles = ""
-    # XXX add every file here that is to link
-    linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink)
-
-    lit "],\L\"linkcmd\": "
-    str getLinkCmd(conf, output, objfiles)
-
-    lit ",\L\"extraCmds\": "
-    lit $(%* getExtraCmds(conf, conf.absOutFile))
-
-    lit ",\L\"stdinInput\": "
-    lit $(%* conf.projectIsStdin)
-    lit ",\L\"projectIsCmd\": "
-    lit $(%* conf.projectIsCmd)
-    lit ",\L\"cmdInput\": "
-    lit $(%* conf.cmdInput)
-    lit ",\L\"currentDir\": "
-    lit $(%* getCurrentDir())
-
-    if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
-      lit ",\L\"cmdline\": "
-      str conf.commandLine
-      lit ",\L\"depfiles\":[\L"
-      depfiles(conf, f, buf)
-      lit "],\L\"nimexe\": \L"
-      str hashNimExe()
-      lit "\L"
-
-    lit "\L}\L"
-    close(f)
-
-proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool =
-  let jsonFile = toGeneratedFile(conf, projectfile, "json")
-  if not fileExists(jsonFile): return true
-  if not fileExists(conf.absOutFile): return true
+proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool =
   result = false
-  try:
-    let data = json.parseFile(jsonFile.string)
-    for key in "depfiles cmdline stdinInput currentDir".split:
-      if not data.hasKey(key): return true
-    if getCurrentDir() != data["currentDir"].getStr:
-      # fixes bug #16271
-      # Note that simply comparing `expandFilename(projectFile)` would
-      # not be sufficient in case other flags depend implicitly on `getCurrentDir`,
-      # and would require much more care. Simply re-compiling is safer for now.
-      # A better strategy for future work would be to cache (with an LRU cache)
-      # the N most recent unique build instructions, as done with `rdmd`,
-      # which is both robust and avoids recompilation when switching back and forth
-      # between projects, see https://github.com/timotheecour/Nim/issues/199
-      return true
-    let oldCmdLine = data["cmdline"].getStr
-    if conf.commandLine != oldCmdLine:
-      return true
-    if hashNimExe() != data["nimexe"].getStr:
-      return true
-    let stdinInput = data["stdinInput"].getBool
-    let projectIsCmd = data["projectIsCmd"].getBool
-    if conf.projectIsStdin or stdinInput:
-      # could optimize by returning false if stdin input was the same,
-      # but I'm not sure how to get full stding input
-      return true
-
-    if conf.projectIsCmd or projectIsCmd:
-      if not (conf.projectIsCmd and projectIsCmd): return true
-      if not data.hasKey("cmdInput"): return true
-      let cmdInput = data["cmdInput"].getStr
-      if cmdInput != conf.cmdInput: return true
-
-    let depfilesPairs = data["depfiles"]
-    doAssert depfilesPairs.kind == JArray
-    for p in depfilesPairs:
-      doAssert p.kind == JArray
-      # >= 2 for forwards compatibility with potential later .json files:
-      doAssert p.len >= 2
-      let depFilename = p[0].getStr
-      let oldHashValue = p[1].getStr
-      let newHashValue = $secureHashFile(depFilename)
-      if oldHashValue != newHashValue:
-        return true
+  if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true
+  var bcache: BuildCache = default(BuildCache)
+  try: bcache.fromJson(jsonFile.string.parseFile)
   except IOError, OSError, ValueError:
-    echo "Warning: JSON processing failed: ", getCurrentExceptionMsg()
-    result = true
+    stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string
+    return true
+  if bcache.currentDir != getCurrentDir() or # fixes bug #16271
+     bcache.configFiles != conf.configFiles.mapIt(it.string) or
+     bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or
+     bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or
+     bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true
+  if bcache.stdinInput or conf.projectIsStdin: return true
+    # xxx optimize by returning false if stdin input was the same
+  for (file, hash) in bcache.depfiles:
+    if $secureHashFile(file) != hash: return true
 
-proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
-  let jsonFile = toGeneratedFile(conf, projectfile, "json")
-  try:
-    let data = json.parseFile(jsonFile.string)
-
-    let output = data["outputFile"].getStr
-    createDir output.parentDir
-    let outputCurrent = $conf.absOutFile
-    if output != outputCurrent:
-      # previously, any specified output file would be silently ignored;
-      # simply copying won't work in some cases, for example with `extraCmds`,
-      # so we just make it an error, user should use same command for jsonscript
-      # as was used with --compileOnly.
-      globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string])
-
-    let toCompile = data["compile"]
-    doAssert toCompile.kind == JArray
-    var cmds: TStringSeq
-    var prettyCmds: TStringSeq
-    let prettyCb = proc (idx: int) =
-      if prettyCmds[idx].len > 0: echo prettyCmds[idx]
-
-    for c in toCompile:
-      doAssert c.kind == JArray
-      doAssert c.len >= 2
-
-      cmds.add(c[1].getStr)
-      prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr)
-
-    execCmdsInParallel(conf, cmds, prettyCb)
-
-    let linkCmd = data["linkcmd"]
-    doAssert linkCmd.kind == JString
-    execLinkCmd(conf, linkCmd.getStr)
-    if data.hasKey("extraCmds"):
-      let extraCmds = data["extraCmds"]
-      doAssert extraCmds.kind == JArray
-      for cmd in extraCmds:
-        doAssert cmd.kind == JString, $cmd.kind
-        let cmd2 = cmd.getStr
-        execExternalProgram(conf, cmd2, hintExecuting)
-
-  except:
+proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
+  var bcache: BuildCache = default(BuildCache)
+  try: bcache.fromJson(jsonFile.string.parseFile)
+  except ValueError, KeyError, JsonKindError:
     let e = getCurrentException()
-    quit "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() &
-         "error evaluating JSON file: " & jsonFile.string
+    conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" %
+      [e.msg, e.getStackTrace(), jsonFile.string]
+  let output = bcache.outputFile
+  createDir output.parentDir
+  let outputCurrent = $conf.absOutFile
+  if output != outputCurrent or bcache.cacheVersion != cacheVersion:
+    globalError(conf, gCmdLineInfo,
+      "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " %
+      [outputCurrent, output, jsonFile.string])
+  var cmds: TStringSeq = default(TStringSeq)
+  var prettyCmds: TStringSeq= default(TStringSeq)
+  let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
+  for (name, cmd) in bcache.compile:
+    cmds.add cmd
+    prettyCmds.add displayProgressCC(conf, name, cmd)
+  execCmdsInParallel(conf, cmds, prettyCb)
+  preventLinkCmdMaxCmdLen(conf, bcache.linkcmd)
+  for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting)
 
 proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
+  result = ""
   for it in list:
     result.addf("--file:r\"$1\"$N", [rope(it.cname.string)])
 
diff --git a/compiler/filter_tmpl.nim b/compiler/filter_tmpl.nim
index 6165ff2f32..921a94b314 100644
--- a/compiler/filter_tmpl.nim
+++ b/compiler/filter_tmpl.nim
@@ -10,9 +10,11 @@
 # This module implements Nim's standard template filter.
 
 import
-  llstream, strutils, ast, msgs, options,
+  llstream, ast, msgs, options,
   filters, lineinfos, pathutils
 
+import std/strutils
+
 type
   TParseState = enum
     psDirective, psTempl
@@ -22,7 +24,7 @@ type
     info: TLineInfo
     indent, emitPar: int
     x: string                # the current input line
-    outp: PLLStream          # the output will be parsed by pnimsyn
+    outp: PLLStream          # the output will be parsed by parser
     subsChar, nimDirective: char
     emit, conc, toStr: string
     curly, bracket, par: int
@@ -201,17 +203,15 @@ proc parseLine(p: var TTmplParser) =
 
 proc filterTmpl*(conf: ConfigRef, stdin: PLLStream, filename: AbsoluteFile,
                  call: PNode): PLLStream =
-  var p: TTmplParser
-  p.config = conf
-  p.info = newLineInfo(conf, filename, 0, 0)
-  p.outp = llStreamOpen("")
-  p.inp = stdin
-  p.subsChar = charArg(conf, call, "subschar", 1, '$')
-  p.nimDirective = charArg(conf, call, "metachar", 2, '#')
-  p.emit = strArg(conf, call, "emit", 3, "result.add")
-  p.conc = strArg(conf, call, "conc", 4, " & ")
-  p.toStr = strArg(conf, call, "tostring", 5, "$")
-  p.x = newStringOfCap(120)
+  var p = TTmplParser(config: conf, info: newLineInfo(conf, filename, 0, 0),
+                outp: llStreamOpen(""), inp: stdin,
+                subsChar: charArg(conf, call, "subschar", 1, '$'),
+                nimDirective: charArg(conf, call, "metachar", 2, '#'),
+                emit: strArg(conf, call, "emit", 3, "result.add"),
+                conc: strArg(conf, call, "conc", 4, " & "),
+                toStr: strArg(conf, call, "tostring", 5, "$"),
+                x: newStringOfCap(120)
+                )
   # do not process the first line which contains the directive:
   if llStreamReadLine(p.inp, p.x):
     inc p.info.line
diff --git a/compiler/filters.nim b/compiler/filters.nim
index 8151c0b938..3cd56e3be5 100644
--- a/compiler/filters.nim
+++ b/compiler/filters.nim
@@ -10,9 +10,11 @@
 # This module implements Nim's simple filters and helpers for filters.
 
 import
-  llstream, idents, strutils, ast, msgs, options,
+  llstream, idents, ast, msgs, options,
   renderer, pathutils
 
+import std/strutils
+
 proc invalidPragma(conf: ConfigRef; n: PNode) =
   localError(conf, n.info,
       "'$1' not allowed here" % renderTree(n, {renderNoComments}))
@@ -29,23 +31,30 @@ proc getArg(conf: ConfigRef; n: PNode, name: string, pos: int): PNode =
       return n[i]
 
 proc charArg*(conf: ConfigRef; n: PNode, name: string, pos: int, default: char): char =
+
   var x = getArg(conf, n, name, pos)
   if x == nil: result = default
   elif x.kind == nkCharLit: result = chr(int(x.intVal))
-  else: invalidPragma(conf, n)
+  else:
+    result = default(char)
+    invalidPragma(conf, n)
 
 proc strArg*(conf: ConfigRef; n: PNode, name: string, pos: int, default: string): string =
   var x = getArg(conf, n, name, pos)
   if x == nil: result = default
   elif x.kind in {nkStrLit..nkTripleStrLit}: result = x.strVal
-  else: invalidPragma(conf, n)
+  else:
+    result = ""
+    invalidPragma(conf, n)
 
 proc boolArg*(conf: ConfigRef; n: PNode, name: string, pos: int, default: bool): bool =
   var x = getArg(conf, n, name, pos)
   if x == nil: result = default
   elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "true") == 0: result = true
   elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "false") == 0: result = false
-  else: invalidPragma(conf, n)
+  else:
+    result = false
+    invalidPragma(conf, n)
 
 proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, call: PNode): PLLStream =
   var pattern = strArg(conf, call, "startswith", 1, "")
diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim
index d4aeb6a77e..da911c84cf 100644
--- a/compiler/gorgeimpl.nim
+++ b/compiler/gorgeimpl.nim
@@ -9,8 +9,14 @@
 
 ## Module that implements ``gorge`` for the compiler.
 
-import msgs, std / sha1, os, osproc, streams, options,
-  lineinfos, pathutils
+import msgs, options, lineinfos, pathutils
+
+import std/[os, osproc, streams]
+
+when defined(nimPreviewSlimSystem):
+  import std/syncio
+
+import ../dist/checksums/src/checksums/sha1
 
 proc readOutput(p: Process): (string, int) =
   result[0] = ""
@@ -24,10 +30,11 @@ proc readOutput(p: Process): (string, int) =
 
 proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (string, int) =
   let workingDir = parentDir(toFullPath(conf, info))
+  result = ("", 0)
   if cache.len > 0:
     let h = secureHash(cmd & "\t" & input & "\t" & cache)
     let filename = toGeneratedFile(conf, AbsoluteFile("gorge_" & $h), "txt").string
-    var f: File
+    var f: File = default(File)
     if optForceFullMake notin conf.globalOptions and open(f, filename):
       result = (f.readAll, 0)
       f.close
@@ -46,7 +53,11 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
       if result[1] == 0:
         writeFile(filename, result[0])
     except IOError, OSError:
-      if not readSuccessful: result = ("", -1)
+      if not readSuccessful:
+        when defined(nimLegacyGorgeErrors):
+          result = ("", -1)
+        else:
+          result = ("Error running startProcess: " & getCurrentExceptionMsg(), -1)
   else:
     try:
       var p = startProcess(cmd, workingDir,
@@ -57,4 +68,7 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
       result = p.readOutput
       p.close()
     except IOError, OSError:
-      result = ("", -1)
+      when defined(nimLegacyGorgeErrors):
+        result = ("", -1)
+      else:
+        result = ("Error running startProcess: " & getCurrentExceptionMsg(), -1)
diff --git a/compiler/guards.nim b/compiler/guards.nim
index 0281421276..bbb2398674 100644
--- a/compiler/guards.nim
+++ b/compiler/guards.nim
@@ -12,6 +12,9 @@
 import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
   saturate, modulegraphs, options, lineinfos, int128
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 const
   someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
     mEqStr, mEqSet, mEqCString}
@@ -48,6 +51,10 @@ proc isLet(n: PNode): bool =
     elif n.sym.kind == skParam and skipTypes(n.sym.typ,
                                              abstractInst).kind notin {tyVar}:
       result = true
+    else:
+      result = false
+  else:
+    result = false
 
 proc isVar(n: PNode): bool =
   n.kind == nkSym and n.sym.kind in {skResult, skVar} and
@@ -65,9 +72,12 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv:
       n = n[0]
-    of nkDerefExpr, nkHiddenDeref:
+    of nkDerefExpr:
       n = n[0]
       inc derefs
+    of nkHiddenDeref:
+      n = n[0]
+      if not isApprox: inc derefs
     of nkBracketExpr:
       if isConstExpr(n[1]) or isLet(n[1]) or isConstExpr(n[1].skipConv):
         n = n[0]
@@ -130,6 +140,8 @@ proc neg(n: PNode; o: Operators): PNode =
       result = a
     elif b != nil:
       result = b
+    else:
+      result = nil
   else:
     # leave  not (a == 4)  as it is
     result = newNodeI(nkCall, n.info, 2)
@@ -197,7 +209,7 @@ proc highBound*(conf: ConfigRef; x: PNode; o: Operators): PNode =
              nkIntLit.newIntNode(lastOrd(conf, typ))
            elif typ.kind == tySequence and x.kind == nkSym and
                x.sym.kind == skConst:
-             nkIntLit.newIntNode(x.sym.ast.len-1)
+             nkIntLit.newIntNode(x.sym.astdef.len-1)
            else:
              o.opAdd.buildCall(o.opLen.buildCall(x), minusOne())
   result.info = x.info
@@ -324,6 +336,8 @@ proc usefulFact(n: PNode; o: Operators): PNode =
         result = n
       elif n[1].getMagic in someLen or n[2].getMagic in someLen:
         result = n
+      else:
+        result = nil
   of someLe+someLt:
     if isLetLocation(n[1], true) or isLetLocation(n[2], true):
       # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
@@ -331,12 +345,18 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     elif n[1].getMagic in someLen or n[2].getMagic in someLen:
       # XXX Rethink this whole idea of 'usefulFact' for semparallel
       result = n
+    else:
+      result = nil
   of mIsNil:
     if isLetLocation(n[1], false) or isVar(n[1]):
       result = n
+    else:
+      result = nil
   of someIn:
     if isLetLocation(n[1], true):
       result = n
+    else:
+      result = nil
   of mAnd:
     let
       a = usefulFact(n[1], o)
@@ -350,10 +370,14 @@ proc usefulFact(n: PNode; o: Operators): PNode =
       result = a
     elif b != nil:
       result = b
+    else:
+      result = nil
   of mNot:
     let a = usefulFact(n[1], o)
     if a != nil:
       result = a.neg(o)
+    else:
+      result = nil
   of mOr:
     # 'or' sucks! (p.isNil or q.isNil) --> hard to do anything
     # with that knowledge...
@@ -370,6 +394,8 @@ proc usefulFact(n: PNode; o: Operators): PNode =
       result[1] = a
       result[2] = b
       result = result.neg(o)
+    else:
+      result = nil
   elif n.kind == nkSym and n.sym.kind == skLet:
     # consider:
     #   let a = 2 < x
@@ -378,8 +404,12 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     # We make can easily replace 'a' by '2 < x' here:
     if n.sym.astdef != nil:
       result = usefulFact(n.sym.astdef, o)
+    else:
+      result = nil
   elif n.kind == nkStmtListExpr:
     result = usefulFact(n.lastSon, o)
+  else:
+    result = nil
 
 type
   TModel* = object
@@ -441,8 +471,15 @@ proc sameTree*(a, b: PNode): bool =
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    for i in 0..n.safeLen-1:
-      if hasSubTree(n[i], x): return true
+    case n.kind
+    of nkEmpty..nkNilLit:
+      result = n.sameTree(x)
+    of nkFormalParams:
+      result = false
+    else:
+      result = false
+      for i in 0.. unknown!
       if sameTree(fact[2], eq[val]): result = impYes
       elif valuesUnequal(fact[2], eq[val]): result = impNo
+      else:
+        result = impUnknown
     elif sameTree(fact[2], eq[loc]):
       if sameTree(fact[1], eq[val]): result = impYes
       elif valuesUnequal(fact[1], eq[val]): result = impNo
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
   of mInSet:
     # remember: mInSet is 'contains' so the set comes first!
     if sameTree(fact[2], eq[loc]) and isValue(eq[val]):
       if inSet(fact[1], eq[val]): result = impYes
       else: result = impNo
-  of mNot, mOr, mAnd: assert(false, "impliesEq")
-  else: discard
+    else:
+      result = impUnknown
+  of mNot, mOr, mAnd:
+    result = impUnknown
+    assert(false, "impliesEq")
+  else: result = impUnknown
 
 proc leImpliesIn(x, c, aSet: PNode): TImplication =
   if c.kind in {nkCharLit..nkUInt64Lit}:
@@ -500,13 +549,19 @@ proc leImpliesIn(x, c, aSet: PNode): TImplication =
     var value = newIntNode(c.kind, firstOrd(nil, x.typ))
     # don't iterate too often:
     if c.intVal - value.intVal < 1000:
-      var i, pos, neg: int
+      var i, pos, neg: int = 0
       while value.intVal <= c.intVal:
         if inSet(aSet, value): inc pos
         else: inc neg
         inc i; inc value.intVal
       if pos == i: result = impYes
       elif neg == i: result = impNo
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
+  else:
+    result = impUnknown
 
 proc geImpliesIn(x, c, aSet: PNode): TImplication =
   if c.kind in {nkCharLit..nkUInt64Lit}:
@@ -517,17 +572,23 @@ proc geImpliesIn(x, c, aSet: PNode): TImplication =
     let max = lastOrd(nil, x.typ)
     # don't iterate too often:
     if max - getInt(value) < toInt128(1000):
-      var i, pos, neg: int
+      var i, pos, neg: int = 0
       while value.intVal <= max:
         if inSet(aSet, value): inc pos
         else: inc neg
         inc i; inc value.intVal
       if pos == i: result = impYes
       elif neg == i: result = impNo
+      else: result = impUnknown
+    else:
+      result = impUnknown
+  else:
+    result = impUnknown
 
 proc compareSets(a, b: PNode): TImplication =
   if equalSets(nil, a, b): result = impYes
   elif intersectSets(nil, a, b).len == 0: result = impNo
+  else: result = impUnknown
 
 proc impliesIn(fact, loc, aSet: PNode): TImplication =
   case fact[0].sym.magic
@@ -538,22 +599,32 @@ proc impliesIn(fact, loc, aSet: PNode): TImplication =
     elif sameTree(fact[2], loc):
       if inSet(aSet, fact[1]): result = impYes
       else: result = impNo
+    else:
+      result = impUnknown
   of mInSet:
     if sameTree(fact[2], loc):
       result = compareSets(fact[1], aSet)
+    else:
+      result = impUnknown
   of someLe:
     if sameTree(fact[1], loc):
       result = leImpliesIn(fact[1], fact[2], aSet)
     elif sameTree(fact[2], loc):
       result = geImpliesIn(fact[2], fact[1], aSet)
+    else:
+      result = impUnknown
   of someLt:
     if sameTree(fact[1], loc):
       result = leImpliesIn(fact[1], fact[2].pred, aSet)
     elif sameTree(fact[2], loc):
       # 4 < x  -->  3 <= x
       result = geImpliesIn(fact[2], fact[1].pred, aSet)
-  of mNot, mOr, mAnd: assert(false, "impliesIn")
-  else: discard
+    else:
+      result = impUnknown
+  of mNot, mOr, mAnd:
+    result = impUnknown
+    assert(false, "impliesIn")
+  else: result = impUnknown
 
 proc valueIsNil(n: PNode): TImplication =
   if n.kind == nkNilLit: impYes
@@ -565,13 +636,19 @@ proc impliesIsNil(fact, eq: PNode): TImplication =
   of mIsNil:
     if sameTree(fact[1], eq[1]):
       result = impYes
+    else:
+      result = impUnknown
   of someEq:
     if sameTree(fact[1], eq[1]):
       result = valueIsNil(fact[2].skipConv)
     elif sameTree(fact[2], eq[1]):
       result = valueIsNil(fact[1].skipConv)
-  of mNot, mOr, mAnd: assert(false, "impliesIsNil")
-  else: discard
+    else:
+      result = impUnknown
+  of mNot, mOr, mAnd:
+    result = impUnknown
+    assert(false, "impliesIsNil")
+  else: result = impUnknown
 
 proc impliesGe(fact, x, c: PNode): TImplication =
   assert isLocation(x)
@@ -582,32 +659,57 @@ proc impliesGe(fact, x, c: PNode): TImplication =
         # fact:  x = 4;  question x >= 56? --> true iff 4 >= 56
         if leValue(c, fact[2]): result = impYes
         else: result = impNo
+      else:
+        result = impUnknown
     elif sameTree(fact[2], x):
       if isValue(fact[1]) and isValue(c):
         if leValue(c, fact[1]): result = impYes
         else: result = impNo
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
   of someLt:
     if sameTree(fact[1], x):
       if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question N <= x? --> false iff N <= 4
         if leValue(fact[2], c): result = impNo
+        else: result = impUnknown
         # fact:  x < 4;  question 2 <= x? --> we don't know
+      else:
+        result = impUnknown
     elif sameTree(fact[2], x):
       # fact: 3 < x; question: N-1 < x ?  --> true iff N-1 <= 3
       if isValue(fact[1]) and isValue(c):
         if leValue(c.pred, fact[1]): result = impYes
+        else: result = impUnknown
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
   of someLe:
     if sameTree(fact[1], x):
       if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x >= 56? --> false iff 4 <= 56
         if leValue(fact[2], c): result = impNo
         # fact:  x <= 4;  question x >= 2? --> we don't know
+        else:
+          result = impUnknown
+      else:
+        result = impUnknown
     elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x >= 2 ?  --> true iff 2 <= 3
       if isValue(fact[1]) and isValue(c):
         if leValue(c, fact[1]): result = impYes
-  of mNot, mOr, mAnd: assert(false, "impliesGe")
-  else: discard
+        else: result = impUnknown
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
+  of mNot, mOr, mAnd:
+    result = impUnknown
+    assert(false, "impliesGe")
+  else: result = impUnknown
 
 proc impliesLe(fact, x, c: PNode): TImplication =
   if not isLocation(x):
@@ -622,35 +724,59 @@ proc impliesLe(fact, x, c: PNode): TImplication =
         # fact:  x = 4;  question x <= 56? --> true iff 4 <= 56
         if leValue(fact[2], c): result = impYes
         else: result = impNo
+      else:
+        result = impUnknown
     elif sameTree(fact[2], x):
       if isValue(fact[1]) and isValue(c):
         if leValue(fact[1], c): result = impYes
         else: result = impNo
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
   of someLt:
     if sameTree(fact[1], x):
       if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question x <= N? --> true iff N-1 <= 4
         if leValue(fact[2], c.pred): result = impYes
+        else:
+          result = impUnknown
         # fact:  x < 4;  question x <= 2? --> we don't know
+      else:
+        result = impUnknown
     elif sameTree(fact[2], x):
       # fact: 3 < x; question: x <= 1 ?  --> false iff 1 <= 3
       if isValue(fact[1]) and isValue(c):
         if leValue(c, fact[1]): result = impNo
-
+        else: result = impUnknown
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
   of someLe:
     if sameTree(fact[1], x):
       if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x <= 56? --> true iff 4 <= 56
         if leValue(fact[2], c): result = impYes
+        else: result = impUnknown
         # fact:  x <= 4;  question x <= 2? --> we don't know
+      else:
+        result = impUnknown
 
     elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x <= 2 ?  --> false iff 2 < 3
       if isValue(fact[1]) and isValue(c):
         if leValue(c, fact[1].pred): result = impNo
+        else:result = impUnknown
+      else:
+        result = impUnknown
+    else:
+      result = impUnknown
 
-  of mNot, mOr, mAnd: assert(false, "impliesLe")
-  else: discard
+  of mNot, mOr, mAnd:
+    result = impUnknown
+    assert(false, "impliesLe")
+  else: result = impUnknown
 
 proc impliesLt(fact, x, c: PNode): TImplication =
   # x < 3  same as x <= 2:
@@ -662,6 +788,8 @@ proc impliesLt(fact, x, c: PNode): TImplication =
     let q = x.pred
     if q != x:
       result = impliesLe(fact, q, c)
+    else:
+      result = impUnknown
 
 proc `~`(x: TImplication): TImplication =
   case x
@@ -713,6 +841,7 @@ proc factImplies(fact, prop: PNode): TImplication =
 
 proc doesImply*(facts: TModel, prop: PNode): TImplication =
   assert prop.kind in nkCallKinds
+  result = impUnknown
   for f in facts.s:
     # facts can be invalidated, in which case they are 'nil':
     if not f.isNil:
@@ -741,7 +870,7 @@ template isSub(x): untyped = x.getMagic in someSub
 template isVal(x): untyped = x.kind in {nkCharLit..nkUInt64Lit}
 template isIntVal(x, y): untyped = x.intVal == y
 
-import macros
+import std/macros
 
 macro `=~`(x: PNode, pat: untyped): bool =
   proc m(x, pat, conds: NimNode) =
@@ -888,6 +1017,7 @@ proc applyReplacements(n: PNode; rep: TReplacements): PNode =
 
 proc pleViaModelRec(m: var TModel; a, b: PNode): TImplication =
   # now check for inferrable facts: a <= b and b <= c  implies a <= c
+  result = impUnknown
   for i in 0..m.s.high:
     let fact = m.s[i]
     if fact != nil and fact.getMagic in someLe:
@@ -933,7 +1063,7 @@ proc pleViaModel(model: TModel; aa, bb: PNode): TImplication =
       let b = fact[2]
       if a.kind == nkSym: replacements.add((a,b))
       else: replacements.add((b,a))
-  var m: TModel
+  var m = TModel()
   var a = aa
   var b = bb
   if replacements.len > 0:
@@ -968,8 +1098,8 @@ proc addFactLt*(m: var TModel; a, b: PNode) =
   addFactLe(m, a, bb)
 
 proc settype(n: PNode): PType =
-  result = newType(tySet, ItemId(module: -1, item: -1), n.typ.owner)
-  var idgen: IdGenerator
+  var idgen = idGeneratorForPackage(-1'i32)
+  result = newType(tySet, idgen, n.typ.owner)
   addSonSkipIntLit(result, n.typ, idgen)
 
 proc buildOf(it, loc: PNode; o: Operators): PNode =
@@ -1045,8 +1175,12 @@ proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
     assert check.getMagic == mNot
     result = buildProperFieldCheck(access, check[1], o).neg(o)
 
-proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
+proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef; produceError: bool) =
   for i in 1..= 0 and idx < t.vals.len
+
 proc enlarge[T](t: var BiTable[T]) =
   var n: seq[LitId]
   newSeq(n, len(t.keys) * 2)
@@ -86,13 +94,13 @@ proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId =
   t.vals.add v
 
 
-proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} =
-  let idx = idToIdx LitId
+proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} =
+  let idx = idToIdx litId
   assert idx < t.vals.len
   result = t.vals[idx]
 
-proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} =
-  let idx = idToIdx LitId
+proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} =
+  let idx = idToIdx litId
   assert idx < t.vals.len
   result = t.vals[idx]
 
@@ -111,6 +119,12 @@ proc load*[T](f: var RodFile; t: var BiTable[T]) =
   loadSeq(f, t.vals)
   loadSeq(f, t.keys)
 
+proc sizeOnDisc*(t: BiTable[string]): int =
+  result = 4
+  for x in t.vals:
+    result += x.len + 4
+  result += t.keys.len * sizeof(LitId)
+
 when isMainModule:
 
   var t: BiTable[string]
diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim
index 88b2a94772..83f1b4cc75 100644
--- a/compiler/ic/cbackend.nim
+++ b/compiler/ic/cbackend.nim
@@ -18,10 +18,13 @@
 ## also doing cross-module dependency tracking and DCE that we don't need
 ## anymore. DCE is now done as prepass over the entire packed module graph.
 
-import std/[packedsets, algorithm]
-  # std/intsets would give `UnusedImport`, pending https://github.com/nim-lang/Nim/issues/14246
+import std/[packedsets, algorithm, tables]
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen,
-  pathutils, extccomp, msgs]
+  pathutils, extccomp, msgs, modulepaths]
 
 import packed_ast, ic, dce, rodfiles
 
@@ -30,12 +33,16 @@ proc unpackTree(g: ModuleGraph; thisModule: int;
   var decoder = initPackedDecoder(g.config, g.cache)
   result = loadNodes(decoder, g.packed, thisModule, tree, n)
 
-proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) =
+proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) =
   if g.backend == nil:
     g.backend = cgendata.newModuleList(g)
-
+  assert g.backend != nil
   var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config)
   bmod.idgen = idgenFromLoadedModule(m)
+
+proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) =
+  var bmod = BModuleList(g.backend).modules[m.module.position]
+  assert bmod != nil
   bmod.flags.incl useAliveDataFromDce
   bmod.alive = move alive[m.module.position]
 
@@ -44,6 +51,14 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive
     cgen.genTopLevelStmt(bmod, n)
 
   finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
+  for disp in getDispatchers(g):
+    genProcAux(bmod, disp)
+  m.fromDisk.backendFlags = cgen.whichInitProcs(bmod)
+
+proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) =
+  for x in mitems(m.fromDisk.emittedTypeInfo):
+    #echo "found type ", x, " for file ", int(origin)
+    g.emittedTypeInfo[x] = origin
 
 proc addFileToLink(config: ConfigRef; m: PSym) =
   let filename = AbsoluteFile toFullPath(config, m.position.FileIndex)
@@ -51,7 +66,8 @@ proc addFileToLink(config: ConfigRef; m: PSym) =
       if config.backend == backendCpp: ".nim.cpp"
       elif config.backend == backendObjc: ".nim.m"
       else: ".nim.c"
-  let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext)
+  let cfile = changeFileExt(completeCfilePath(config,
+                            mangleModuleName(config, filename).AbsoluteFile), ext)
   let objFile = completeCfilePath(config, toObjFile(config, cfile))
   if fileExists(objFile):
     var cf = Cfile(nimname: m.name.s, cname: cfile,
@@ -59,26 +75,69 @@ proc addFileToLink(config: ConfigRef; m: PSym) =
                    flags: {CfileFlag.Cached})
     addFileToCompile(config, cf)
 
-proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
+when defined(debugDce):
+  import os, std/packedsets
+
+proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) =
+  var f = rodfiles.create(asymFile.string)
+  f.storeHeader()
+  f.storeSection aliveSymsSection
+  f.storeSeq(s)
+  close f
+
+template prepare {.dirty.} =
   let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms")
   var s = newSeqOfCap[int32](alive[position].len)
   for a in items(alive[position]): s.add int32(a)
   sort(s)
+
+proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) =
+  prepare()
+  storeAliveSymsImpl(asymFile, s)
+
+proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
+  prepare()
   var f2 = rodfiles.open(asymFile.string)
   f2.loadHeader()
   f2.loadSection aliveSymsSection
-  var oldData: seq[int32]
+  var oldData: seq[int32] = @[]
   f2.loadSeq(oldData)
   f2.close
   if f2.err == ok and oldData == s:
     result = false
   else:
+    when defined(debugDce):
+      let oldAsSet = toPackedSet[int32](oldData)
+      let newAsSet = toPackedSet[int32](s)
+      echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err
+      echo "in old but not in new ", oldAsSet.difference(newAsSet), " number of entries in old ", oldAsSet.len
+      echo "in new but not in old ", newAsSet.difference(oldAsSet), " number of entries in new ", newAsSet.len
+      #if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0:
+      #  echo "command failed"
     result = true
-    var f = rodfiles.create(asymFile.string)
-    f.storeHeader()
-    f.storeSection aliveSymsSection
-    f.storeSeq(s)
-    close f
+    storeAliveSymsImpl(asymFile, s)
+
+proc genPackedModule(g: ModuleGraph, i: int; alive: var AliveSyms) =
+  # case statement here to enforce exhaustive checks.
+  case g.packed[i].status
+  of undefined:
+    discard "nothing to do"
+  of loading, stored:
+    assert false
+  of storing, outdated:
+    storeAliveSyms(g.config, g.packed[i].module.position, alive)
+    generateCodeForModule(g, g.packed[i], alive)
+    closeRodFile(g, g.packed[i].module)
+  of loaded:
+    if g.packed[i].loadedButAliveSetChanged:
+      generateCodeForModule(g, g.packed[i], alive)
+    else:
+      addFileToLink(g.config, g.packed[i].module)
+      replayTypeInfo(g, g.packed[i], FileIndex(i))
+
+      if g.backend == nil:
+        g.backend = cgendata.newModuleList(g)
+      registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags)
 
 proc generateCode*(g: ModuleGraph) =
   ## The single entry point, generate C(++) code for the entire
@@ -86,21 +145,36 @@ proc generateCode*(g: ModuleGraph) =
   resetForBackend(g)
   var alive = computeAliveSyms(g.packed, g.config)
 
-  for i in 0..high(g.packed):
+  when false:
+    for i in 0..= 0:
+    genPackedModule(g, mainModuleIdx, alive)
diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim
index 0918fc3799..6eb36431ea 100644
--- a/compiler/ic/dce.nim
+++ b/compiler/ic/dce.nim
@@ -9,7 +9,11 @@
 
 ## Dead code elimination (=DCE) for IC.
 
-import std / [intsets, tables]
+import std/[intsets, tables]
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 import ".." / [ast, options, lineinfos, types]
 
 import packed_ast, ic, bitabs
@@ -27,7 +31,7 @@ type
 proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool =
   ## "Exported to C" procs are special (these are marked with '.exportc') because these
   ## must not be optimized away!
-  let symPtr = addr g[c.thisModule].fromDisk.sh.syms[symId]
+  let symPtr = unsafeAddr g[c.thisModule].fromDisk.syms[symId]
   let flags = symPtr.flags
   # due to a bug/limitation in the lambda lifting, unused inner procs
   # are not transformed correctly; issue (#411). However, the whole purpose here
@@ -36,10 +40,14 @@ proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): boo
     if ({sfExportc, sfCompilerProc} * flags != {}) or
         (symPtr.kind == skMethod):
       result = true
+    else:
+      result = false
       # XXX: This used to be a condition to:
       #  (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
     if sfCompilerProc in flags:
-      c.compilerProcs[g[c.thisModule].fromDisk.sh.strings[symPtr.name]] = (c.thisModule, symId)
+      c.compilerProcs[g[c.thisModule].fromDisk.strings[symPtr.name]] = (c.thisModule, symId)
+  else:
+    result = false
 
 template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty
 
@@ -47,17 +55,17 @@ proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: i
   ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
   ## be analysed.
   if not c.alive[module].containsOrIncl(item):
-    var body = g[module].fromDisk.sh.syms[item].ast
+    var body = g[module].fromDisk.syms[item].ast
     if body != emptyNodeId:
-      let opt = g[module].fromDisk.sh.syms[item].options
-      if g[module].fromDisk.sh.syms[item].kind in routineKinds:
+      let opt = g[module].fromDisk.syms[item].options
+      if g[module].fromDisk.syms[item].kind in routineKinds:
         body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos)
       c.stack.add((module, opt, NodePos(body)))
 
     when false:
-      let nid = g[module].fromDisk.sh.syms[item].name
+      let nid = g[module].fromDisk.syms[item].name
       if nid != LitId(0):
-        let name = g[module].fromDisk.sh.strings[nid]
+        let name = g[module].fromDisk.strings[nid]
         if name in ["nimFrame", "callDepthLimitReached"]:
           echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item
 
@@ -66,12 +74,12 @@ proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string
   followLater(c, g, module, item)
 
 proc loadTypeKind(t: PackedItemId; c: AliveContext; g: PackedModuleGraph; toSkip: set[TTypeKind]): TTypeKind =
-  template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.sh.types[t.item].kind
+  template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.types[t.item].kind
 
   var t2 = translateId(t, g, c.thisModule, c.decoder.config)
   result = t2.kind
   while result in toSkip:
-    t2 = translateId(g[t2.module].fromDisk.sh.types[t2.item].types[^1], g, t2.module, c.decoder.config)
+    t2 = translateId(g[t2.module].fromDisk.types[t2.item].types[^1], g, t2.module, c.decoder.config)
     result = t2.kind
 
 proc rangeCheckAnalysis(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) =
@@ -101,13 +109,13 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N
     discard "ignore non-sym atoms"
   of nkSym:
     # This symbol is alive and everything its body references.
-    followLater(c, g, c.thisModule, n.operand)
+    followLater(c, g, c.thisModule, tree[n].soperand)
   of nkModuleRef:
     let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
+    assert n1.kind == nkNone
+    assert n2.kind == nkNone
     let m = n1.litId
-    let item = n2.operand
+    let item = tree[n2].soperand
     let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int
     followLater(c, g, otherModule, item)
   of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr,
@@ -123,7 +131,7 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N
     rangeCheckAnalysis(c, g, tree, n)
   of nkProcDef, nkConverterDef, nkMethodDef, nkFuncDef, nkIteratorDef:
     if n.firstSon.kind == nkSym and isNotGeneric(n):
-      let item = n.firstSon.operand
+      let item = tree[n.firstSon].soperand
       if isExportedToC(c, g, item):
         # This symbol is alive and everything its body references.
         followLater(c, g, c.thisModule, item)
@@ -145,7 +153,7 @@ proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
   var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf),
                        thisModule: -1, alive: newSeq[IntSet](g.len),
                        options: conf.options)
-  for i in countdown(high(g), 0):
+  for i in countdown(len(g)-1, 0):
     if g[i].status != undefined:
       c.thisModule = i
       for p in allNodes(g[i].fromDisk.topLevel):
diff --git a/compiler/ic/design.rst b/compiler/ic/design.rst
index d8e1315b1d..b096e3103a 100644
--- a/compiler/ic/design.rst
+++ b/compiler/ic/design.rst
@@ -7,12 +7,8 @@ The frontend produces a set of `.rod` files. Every `.nim` module
 produces its own `.rod` file.
 
 - The IR must be a faithful representation of the AST in memory.
-- The backend can do its own caching but doesn't have to.
-- We know by comparing 'nim check compiler/nim' against 'nim c compiler/nim'
-  that 2/3 of the compiler's runtime is spent in the frontend. Hence we
-  implement IC for the frontend first and only later for the backend. The
-  backend will recompile everything until we implement its own caching
-  mechanisms.
+- The backend can do its own caching but doesn't have to. In the
+  current implementation the backend also caches its results.
 
 Advantage of the "set of files" vs the previous global database:
 - By construction, we either read from the `.rod` file or from the
diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim
index 99a68e0f03..c9f546c76a 100644
--- a/compiler/ic/ic.nim
+++ b/compiler/ic/ic.nim
@@ -7,12 +7,19 @@
 #    distribution, for details about the copyright.
 #
 
-import std / [hashes, tables, intsets, sha1]
+import std/[hashes, tables, intsets, monotimes]
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
-  pathutils, condsyms]
+  pathutils, condsyms, packages, modulepaths]
 #import ".." / [renderer, astalgo]
-from std / os import removeFile, isAbsolute
+from std/os import removeFile, isAbsolute
+
+import ../../dist/checksums/src/checksums/sha1
+
+import ".." / nir / nirlineinfos
+
+when defined(nimPreviewSlimSystem):
+  import std/[syncio, assertions, formatfloat]
 
 type
   PackedConfig* = object
@@ -22,27 +29,43 @@ type
     options: TOptions
     globalOptions: TGlobalOptions
 
+  ModuleBackendFlag* = enum
+    HasDatInitProc
+    HasModuleInitProc
+
   PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file
     definedSymbols: string
-    includes: seq[(LitId, string)] # first entry is the module filename itself
+    moduleFlags: TSymFlags
+    includes*: seq[(LitId, string)] # first entry is the module filename itself
     imports: seq[LitId] # the modules this module depends on
-    toReplay: PackedTree # pragmas and VM specific state to replay.
+    toReplay*: PackedTree # pragmas and VM specific state to replay.
     topLevel*: PackedTree  # top level statements
     bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
     #producedGenerics*: Table[GenericKey, SymId]
     exports*: seq[(LitId, int32)]
-    reexports*: seq[(LitId, PackedItemId)]
+    hidden: seq[(LitId, int32)]
+    reexports: seq[(LitId, PackedItemId)]
     compilerProcs*: seq[(LitId, int32)]
     converters*, methods*, trmacros*, pureEnums*: seq[int32]
-    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
 
     typeInstCache*: seq[(PackedItemId, PackedItemId)]
     procInstCache*: seq[PackedInstantiation]
-    attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
-    methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
+    attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)]
+    methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)]
     enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
+    methodsPerType*: seq[(PackedItemId, PackedItemId)]
+    dispatchers*: seq[PackedItemId]
+
+    emittedTypeInfo*: seq[string]
+    backendFlags*: set[ModuleBackendFlag]
+
+    syms*: seq[PackedSym]
+    types*: seq[PackedType]
+    strings*: BiTable[string] # we could share these between modules.
+    numbers*: BiTable[BiggestInt] # we also store floats in here so
+                                  # that we can assure that every bit is kept
+    man*: LineInfoManager
 
-    sh*: Shared
     cfg: PackedConfig
 
   PackedEncoder* = object
@@ -57,8 +80,51 @@ type
     symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
     config*: ConfigRef
 
+proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int;
+               result: var string) =
+  if result.len > 0 and result[^1] notin {' ', '\n'}:
+    result.add ' '
+
+  result.add $tree[pos].kind
+  case tree[pos].kind
+  of nkEmpty, nkNilLit, nkType: discard
+  of nkIdent, nkStrLit..nkTripleStrLit:
+    result.add " "
+    result.add m.strings[LitId tree[pos].uoperand]
+  of nkSym:
+    result.add " "
+    result.add m.strings[m.syms[tree[pos].soperand].name]
+  of directIntLit:
+    result.add " "
+    result.addInt tree[pos].soperand
+  of externSIntLit:
+    result.add " "
+    result.addInt m.numbers[LitId tree[pos].uoperand]
+  of externUIntLit:
+    result.add " "
+    result.addInt cast[uint64](m.numbers[LitId tree[pos].uoperand])
+  of nkFloatLit..nkFloat128Lit:
+    result.add " "
+    result.addFloat cast[BiggestFloat](m.numbers[LitId tree[pos].uoperand])
+  else:
+    result.add "(\n"
+    for i in 1..(nesting+1)*2: result.add ' '
+    for child in sonsReadonly(tree, pos):
+      toString(tree, child, m, nesting + 1, result)
+    result.add "\n"
+    for i in 1..nesting*2: result.add ' '
+    result.add ")"
+    #for i in 1..nesting*2: result.add ' '
+
+proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string =
+  result = ""
+  toString(tree, n, m, 0, result)
+
+proc debug*(tree: PackedTree; m: PackedModule) =
+  stdout.write toString(tree, NodePos 0, m)
+
 proc isActive*(e: PackedEncoder): bool = e.config != nil
-proc disable*(e: var PackedEncoder) = e.config = nil
+proc disable(e: var PackedEncoder) = e.config = nil
 
 template primConfigFields(fn: untyped) {.dirty.} =
   fn backend
@@ -81,14 +147,27 @@ proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef
   #primConfigFields rem
   m.cfg = pc
 
+const
+  debugConfigDiff = defined(debugConfigDiff)
+
+when debugConfigDiff:
+  import hashes, tables, intsets, sha1, strutils, sets
+
 proc configIdentical(m: PackedModule; config: ConfigRef): bool =
   result = m.definedSymbols == definedSymbolsAsString(config)
-  #if not result:
-  #  echo "A ", m.definedSymbols, " ", definedSymbolsAsString(config)
+  when debugConfigDiff:
+    if not result:
+      var wordsA = m.definedSymbols.split(Whitespace).toHashSet()
+      var wordsB = definedSymbolsAsString(config).split(Whitespace).toHashSet()
+      for c in wordsA - wordsB:
+        echo "in A but not in B ", c
+      for c in wordsB - wordsA:
+        echo "in B but not in A ", c
   template eq(x) =
     result = result and m.cfg.x == config.x
-    #if not result:
-    #  echo "B ", m.cfg.x, " ", config.x
+    when debugConfigDiff:
+      if m.cfg.x != config.x:
+        echo "B ", m.cfg.x, " ", config.x
   primConfigFields eq
 
 proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) =
@@ -112,14 +191,14 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
     result = c.filenames.getOrDefault(x)
     if result == LitId(0):
       let p = msgs.toFullPath(c.config, x)
-      result = getOrIncl(m.sh.strings, p)
+      result = getOrIncl(m.strings, p)
       c.filenames[x] = result
     c.lastFile = x
     c.lastLit = result
-    assert result != LitId(0)
+  assert result != LitId(0)
 
 proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
-  result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
+  result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x])
 
 proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
   for it in mitems(m.includes):
@@ -129,12 +208,14 @@ proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
 
 proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) =
   ## setup a context for serializing to packed ast
-  m.sh = Shared()
   c.thisModule = moduleSym.itemId.module
   c.config = config
+  m.moduleFlags = moduleSym.flags
   m.bodies = newTreeFrom(m.topLevel)
   m.toReplay = newTreeFrom(m.topLevel)
 
+  c.lastFile = FileIndex(-10)
+
   let thisNimFile = FileIndex c.thisModule
   var h = msgs.getHash(config, thisNimFile)
   if h.len == 0:
@@ -153,11 +234,20 @@ proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex)
 proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) =
   m.imports.add toLitId(f, c, m)
 
+proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+  assert s.kind != skUnknown
+  let nameId = getOrIncl(m.strings, s.name.s)
+  m.hidden.add((nameId, s.itemId.item))
+  assert s.itemId.module == c.thisModule
+
 proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  assert s.kind != skUnknown
+  assert s.itemId.module == c.thisModule
+  let nameId = getOrIncl(m.strings, s.name.s)
   m.exports.add((nameId, s.itemId.item))
 
 proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+  assert c.thisModule == s.itemId.module
   m.converters.add(s.itemId.item)
 
 proc addTrmacro*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
@@ -171,12 +261,14 @@ proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
   m.methods.add s.itemId.item
 
 proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  assert s.kind != skUnknown
+  if s.kind == skModule: return
+  let nameId = getOrIncl(m.strings, s.name.s)
   m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m),
                                         item: s.itemId.item)))
 
 proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  let nameId = getOrIncl(m.strings, s.name.s)
   m.compilerProcs.add((nameId, s.itemId.item))
 
 proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule)
@@ -195,14 +287,15 @@ proc flush(c: var PackedEncoder; m: var PackedModule) =
 
 proc toLitId(x: string; m: var PackedModule): LitId =
   ## store a string as a literal
-  result = getOrIncl(m.sh.strings, x)
+  result = getOrIncl(m.strings, x)
 
 proc toLitId(x: BiggestInt; m: var PackedModule): LitId =
   ## store an integer as a literal
-  result = getOrIncl(m.sh.integers, x)
+  result = getOrIncl(m.numbers, x)
 
 proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo =
-  PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m))
+  pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32)
+  #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m))
 
 proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} =
   ## given a symbol, produce an ItemId with the correct properties
@@ -244,81 +337,55 @@ proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): Packed
   # we only write one tree into m.bodies after the other.
   if t.isNil: return nilItemId
 
-  if t.uniqueId.module != c.thisModule:
-    # XXX Assert here that it already was serialized in the foreign module!
-    # it is a foreign type:
-    assert t.uniqueId.module >= 0
-    assert t.uniqueId.item > 0
-    return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
-  assert t.itemId.module >= 0
+  assert t.uniqueId.module >= 0
   assert t.uniqueId.item > 0
-  result = PackedItemId(module: toLitId(t.itemId.module.FileIndex, c, m), item: t.uniqueId.item)
-  addMissing(c, t)
+  result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
+  if t.uniqueId.module == c.thisModule:
+    # the type belongs to this module, so serialize it here, eventually.
+    addMissing(c, t)
 
 proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId =
   if s.isNil: return nilItemId
   assert s.itemId.module >= 0
-  if s.itemId.module != c.thisModule:
-    # XXX Assert here that it already was serialized in the foreign module!
-    # it is a foreign symbol:
-    assert s.itemId.module >= 0
-    return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
-  assert s.itemId.module >= 0
+  assert s.itemId.item >= 0
   result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
-  addMissing(c, s)
+  if s.itemId.module == c.thisModule:
+    # the sym belongs to this module, so serialize it here, eventually.
+    addMissing(c, s)
 
 proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId =
   ## serialize a ptype
   if t.isNil: return nilItemId
 
-  if t.uniqueId.module != c.thisModule:
-    # XXX Assert here that it already was serialized in the foreign module!
-    # it is a foreign type:
-    assert t.uniqueId.module >= 0
-    assert t.uniqueId.item > 0
-    return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
+  assert t.uniqueId.module >= 0
+  assert t.uniqueId.item > 0
+  result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
 
-  if not c.typeMarker.containsOrIncl(t.uniqueId.item):
-    if t.uniqueId.item >= m.sh.types.len:
-      setLen m.sh.types, t.uniqueId.item+1
+  if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item):
+    if t.uniqueId.item >= m.types.len:
+      setLen m.types, t.uniqueId.item+1
 
     var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
       size: t.size, align: t.align, nonUniqueId: t.itemId.item,
-      paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel)
+      paddingAtEnd: t.paddingAtEnd)
     storeNode(p, t, n)
-
-    when false:
-      for op, s in pairs t.attachedOps:
-        c.addMissing s
-        p.attachedOps[op] = s.safeItemId(c, m)
-
     p.typeInst = t.typeInst.storeType(c, m)
-    for kid in items t.sons:
+    for kid in kids t:
       p.types.add kid.storeType(c, m)
-
-    when false:
-      for i, s in items t.methods:
-        c.addMissing s
-        p.methods.add (i, s.safeItemId(c, m))
     c.addMissing t.sym
     p.sym = t.sym.safeItemId(c, m)
     c.addMissing t.owner
     p.owner = t.owner.safeItemId(c, m)
 
     # fill the reserved slot, nothing else:
-    m.sh.types[t.uniqueId.item] = p
-
-  assert t.itemId.module >= 0
-  assert t.uniqueId.item > 0
-  result = PackedItemId(module: toLitId(t.itemId.module.FileIndex, c, m), item: t.uniqueId.item)
+    m.types[t.uniqueId.item] = p
 
 proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib =
   ## the plib hangs off the psym via the .annex field
   if l.isNil: return
-  result.kind = l.kind
-  result.generated = l.generated
-  result.isOverriden = l.isOverriden
-  result.name = toLitId($l.name, m)
+  result = PackedLib(kind: l.kind, generated: l.generated,
+    isOverridden: l.isOverridden, name: toLitId($l.name, m)
+  )
   storeNode(result, l, path)
 
 proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId =
@@ -326,21 +393,16 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
   if s.isNil: return nilItemId
 
   assert s.itemId.module >= 0
+  result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
 
-  if s.itemId.module != c.thisModule:
-    # XXX Assert here that it already was serialized in the foreign module!
-    # it is a foreign symbol:
-    assert s.itemId.module >= 0
-    return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
-
-  if not c.symMarker.containsOrIncl(s.itemId.item):
-    if s.itemId.item >= m.sh.syms.len:
-      setLen m.sh.syms, s.itemId.item+1
+  if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item):
+    if s.itemId.item >= m.syms.len:
+      setLen m.syms, s.itemId.item+1
 
     assert sfForward notin s.flags
 
     var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
-      position: s.position, offset: s.offset, options: s.options,
+      position: s.position, offset: s.offset, disamb: s.disamb, options: s.options,
       name: s.name.s.toLitId(m))
 
     storeNode(p, s, ast)
@@ -352,7 +414,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
       p.bitsize = s.bitsize
       p.alignment = s.alignment
 
-    p.externalName = toLitId(if s.loc.r.isNil: "" else: $s.loc.r, m)
+    p.externalName = toLitId(s.loc.r, m)
     p.locFlags = s.loc.flags
     c.addMissing s.typ
     p.typ = s.typ.storeType(c, m)
@@ -361,63 +423,67 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
     p.annex = toPackedLib(s.annex, c, m)
     when hasFFI:
       p.cname = toLitId(s.cname, m)
+    p.instantiatedFrom = s.instantiatedFrom.safeItemId(c, m)
 
     # fill the reserved slot, nothing else:
-    m.sh.syms[s.itemId.item] = p
-
-  assert s.itemId.module >= 0
-  result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
+    m.syms[s.itemId.item] = p
 
 proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
   ## add a remote symbol reference to the tree
   let info = n.info.toPackedInfo(c, m)
-  ir.nodes.add PackedNode(kind: nkModuleRef, operand: 3.int32, # spans 3 nodes in total
-                          typeId: storeTypeLater(n.typ, c, m), info: info)
-  ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
-                          operand: toLitId(n.sym.itemId.module.FileIndex, c, m).int32)
-  ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
-                          operand: n.sym.itemId.item)
+  if n.typ != n.sym.typ:
+    ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total
+               info = info, flags = n.flags,
+               typeId = storeTypeLater(n.typ, c, m))
+  else:
+    ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total
+              info = info, flags = n.flags)
+  ir.addNode(kind = nkNone, info = info,
+             operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32)
+  ir.addNode(kind = nkNone, info = info,
+             operand = n.sym.itemId.item)
 
 proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
   ## serialize a node into the tree
   if n == nil:
-    ir.nodes.add PackedNode(kind: nkNilRodNode, flags: {}, operand: 1)
+    ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo)
     return
   let info = toPackedInfo(n.info, c, m)
   case n.kind
   of nkNone, nkEmpty, nkNilLit, nkType:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, operand: 0,
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags, operand = 0,
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkIdent:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.strings, n.ident.s),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+                operand = int32 getOrIncl(m.strings, n.ident.s),
+                typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkSym:
     if n.sym.itemId.module == c.thisModule:
       # it is a symbol that belongs to the module we're currently
       # packing:
       let id = n.sym.storeSymLater(c, m).item
-      ir.nodes.add PackedNode(kind: nkSym, flags: n.flags, operand: id,
-                              typeId: storeTypeLater(n.typ, c, m), info: info)
+      if n.typ != n.sym.typ:
+        ir.addNode(kind = nkSym, flags = n.flags, operand = id,
+                   info = info,
+                   typeId = storeTypeLater(n.typ, c, m))
+      else:
+        ir.addNode(kind = nkSym, flags = n.flags, operand = id,
+                   info = info)
     else:
       # store it as an external module reference:
       addModuleRef(n, ir, c, m)
-  of directIntLit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32(n.intVal),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
   of externIntLit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.integers, n.intVal),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+               operand = int32 getOrIncl(m.numbers, n.intVal),
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkStrLit..nkTripleStrLit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.strings, n.strVal),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+               operand = int32 getOrIncl(m.strings, n.strVal),
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkFloatLit..nkFloat128Lit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.floats, n.floatVal),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+               operand = int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)),
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   else:
     let patchPos = ir.prepare(n.kind, n.flags,
                               storeTypeLater(n.typ, c, m), info)
@@ -442,8 +508,8 @@ proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var
       # do not serialize the body of the proc, it's unnecessary since
       # n[0].sym.ast has the sem'checked variant of it which is what
       # everybody should use instead.
-      ir.nodes.add PackedNode(kind: nkEmpty, flags: {}, operand: 0,
-                              typeId: nilItemId, info: info)
+      ir.addNode(kind = nkEmpty, flags = {}, operand = 0,
+                 typeId = nilItemId, info = info)
   ir.patch patchPos
 
 proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) =
@@ -462,6 +528,9 @@ proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var Pac
   of nkStmtList, nkStmtListExpr:
     for it in n:
       toPackedNodeIgnoreProcDefs(it, encoder, m)
+  of nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
+     nkFromStmt, nkIncludeStmt:
+    discard "nothing to do"
   else:
     toPackedNode(n, m.topLevel, encoder, m)
 
@@ -478,6 +547,15 @@ proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var Packe
   toPackedProcDef(s.ast, m.topLevel, encoder, m)
   #flush encoder, m
 
+proc storeAttachedProcDef*(t: PType; op: TTypeAttachedOp; s: PSym,
+                           encoder: var PackedEncoder; m: var PackedModule) =
+  assert s.kind in routineKinds
+  assert isActive(encoder)
+  let tid = storeTypeLater(t, encoder, m)
+  let sid = storeSymLater(s, encoder, m)
+  m.attachedOps.add (tid, op, sid)
+  toPackedGeneratedProcDef(s, encoder, m)
+
 proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
   var t = newSeq[PackedItemId](i.concreteTypes.len)
   for j in 0..high(i.concreteTypes):
@@ -487,6 +565,9 @@ proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i:
                                           concreteTypes: t)
   toPackedGeneratedProcDef(i.sym, c, m)
 
+proc storeExpansion*(c: var PackedEncoder; m: var PackedModule; info: TLineInfo; s: PSym) =
+  toPackedNode(newSymNode(s, info), m.bodies, c, m)
+
 proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
   case err
   of cannotOpen:
@@ -494,16 +575,35 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
   of includeFileChanged:
     rawMessage(config, warnFileChanged, filename.string)
   else:
-    echo "Error: ", $err, " loading file: ", filename.string
+    rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err)
+    #echo "Error: ", $err, " loading file: ", filename.string
+
+proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile =
+  result = changeFileExt(completeGeneratedFilePath(conf,
+    mangleModuleName(conf, f).AbsoluteFile), ext)
+
+const
+  BenchIC* = false
+
+when BenchIC:
+  var gloadBodies: MonoTime
+
+  template bench(x, body) =
+    let start = getMonoTime()
+    body
+    x = x + (getMonoTime() - start)
+
+else:
+  template bench(x, body) = body
 
 proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef;
                   ignoreConfig = false): RodFileError =
-  m.sh = Shared()
   var f = rodfiles.open(filename.string)
   f.loadHeader()
   f.loadSection configSection
 
   f.loadPrim m.definedSymbols
+  f.loadPrim m.moduleFlags
   f.loadPrim m.cfg
 
   if f.err == ok and not configIdentical(m, config) and not ignoreConfig:
@@ -517,41 +617,51 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
     f.loadSection section
     f.load data
 
-  loadTabSection stringsSection, m.sh.strings
+  loadTabSection stringsSection, m.strings
 
   loadSeqSection checkSumsSection, m.includes
-  if not includesIdentical(m, config):
+  if config.cmd != cmdM and not includesIdentical(m, config):
     f.err = includeFileChanged
 
   loadSeqSection depsSection, m.imports
 
-  loadTabSection integersSection, m.sh.integers
-  loadTabSection floatsSection, m.sh.floats
+  bench gloadBodies:
 
-  loadSeqSection exportsSection, m.exports
+    loadTabSection numbersSection, m.numbers
 
-  loadSeqSection reexportsSection, m.reexports
+    loadSeqSection exportsSection, m.exports
+    loadSeqSection hiddenSection, m.hidden
+    loadSeqSection reexportsSection, m.reexports
 
-  loadSeqSection compilerProcsSection, m.compilerProcs
+    loadSeqSection compilerProcsSection, m.compilerProcs
 
-  loadSeqSection trmacrosSection, m.trmacros
+    loadSeqSection trmacrosSection, m.trmacros
 
-  loadSeqSection convertersSection, m.converters
-  loadSeqSection methodsSection, m.methods
-  loadSeqSection pureEnumsSection, m.pureEnums
-  loadSeqSection macroUsagesSection, m.macroUsages
+    loadSeqSection convertersSection, m.converters
+    loadSeqSection methodsSection, m.methods
+    loadSeqSection pureEnumsSection, m.pureEnums
 
-  loadSeqSection toReplaySection, m.toReplay.nodes
-  loadSeqSection topLevelSection, m.topLevel.nodes
-  loadSeqSection bodiesSection, m.bodies.nodes
-  loadSeqSection symsSection, m.sh.syms
-  loadSeqSection typesSection, m.sh.types
+    loadTabSection toReplaySection, m.toReplay
+    loadTabSection topLevelSection, m.topLevel
 
-  loadSeqSection typeInstCacheSection, m.typeInstCache
-  loadSeqSection procInstCacheSection, m.procInstCache
-  loadSeqSection attachedOpsSection, m.attachedOps
-  loadSeqSection methodsPerTypeSection, m.methodsPerType
-  loadSeqSection enumToStringProcsSection, m.enumToStringProcs
+    loadTabSection bodiesSection, m.bodies
+    loadSeqSection symsSection, m.syms
+    loadSeqSection typesSection, m.types
+
+    loadSeqSection typeInstCacheSection, m.typeInstCache
+    loadSeqSection procInstCacheSection, m.procInstCache
+    loadSeqSection attachedOpsSection, m.attachedOps
+    loadSeqSection methodsPerGenericTypeSection, m.methodsPerGenericType
+    loadSeqSection enumToStringProcsSection, m.enumToStringProcs
+    loadSeqSection methodsPerTypeSection, m.methodsPerType
+    loadSeqSection dispatchersSection, m.dispatchers
+    loadSeqSection typeInfoSection, m.emittedTypeInfo
+
+    f.loadSection backendFlagsSection
+    f.loadPrim m.backendFlags
+
+    f.loadSection sideChannelSection
+  f.load m.man
 
   close(f)
   result = f.err
@@ -570,6 +680,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   f.storeHeader()
   f.storeSection configSection
   f.storePrim m.definedSymbols
+  f.storePrim m.moduleFlags
   f.storePrim m.cfg
 
   template storeSeqSection(section, data) {.dirty.} =
@@ -580,17 +691,16 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
     f.storeSection section
     f.store data
 
-  storeTabSection stringsSection, m.sh.strings
+  storeTabSection stringsSection, m.strings
 
   storeSeqSection checkSumsSection, m.includes
 
   storeSeqSection depsSection, m.imports
 
-  storeTabSection integersSection, m.sh.integers
-  storeTabSection floatsSection, m.sh.floats
+  storeTabSection numbersSection, m.numbers
 
   storeSeqSection exportsSection, m.exports
-
+  storeSeqSection hiddenSection, m.hidden
   storeSeqSection reexportsSection, m.reexports
 
   storeSeqSection compilerProcsSection, m.compilerProcs
@@ -599,21 +709,29 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection convertersSection, m.converters
   storeSeqSection methodsSection, m.methods
   storeSeqSection pureEnumsSection, m.pureEnums
-  storeSeqSection macroUsagesSection, m.macroUsages
 
-  storeSeqSection toReplaySection, m.toReplay.nodes
-  storeSeqSection topLevelSection, m.topLevel.nodes
+  storeTabSection toReplaySection, m.toReplay
+  storeTabSection topLevelSection, m.topLevel
 
-  storeSeqSection bodiesSection, m.bodies.nodes
-  storeSeqSection symsSection, m.sh.syms
+  storeTabSection bodiesSection, m.bodies
+  storeSeqSection symsSection, m.syms
 
-  storeSeqSection typesSection, m.sh.types
+  storeSeqSection typesSection, m.types
 
   storeSeqSection typeInstCacheSection, m.typeInstCache
   storeSeqSection procInstCacheSection, m.procInstCache
   storeSeqSection attachedOpsSection, m.attachedOps
-  storeSeqSection methodsPerTypeSection, m.methodsPerType
+  storeSeqSection methodsPerGenericTypeSection, m.methodsPerGenericType
   storeSeqSection enumToStringProcsSection, m.enumToStringProcs
+  storeSeqSection methodsPerTypeSection, m.methodsPerType
+  storeSeqSection dispatchersSection, m.dispatchers
+  storeSeqSection typeInfoSection, m.emittedTypeInfo
+
+  f.storeSection backendFlagsSection
+  f.storePrim m.backendFlags
+
+  f.storeSection sideChannelSection
+  f.store m.man
 
   close(f)
   encoder.disable()
@@ -642,24 +760,43 @@ type
     storing,  # state is strictly for stress-testing purposes
     loading,
     loaded,
-    outdated
+    outdated,
+    stored    # store is complete, no further additions possible
 
   LoadedModule* = object
     status*: ModuleStatus
-    symsInit, typesInit: bool
+    symsInit, typesInit, loadedButAliveSetChanged*: bool
     fromDisk*: PackedModule
     syms: seq[PSym] # indexed by itemId
     types: seq[PType]
     module*: PSym # the one true module symbol.
-    iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too
+    iface, ifaceHidden: Table[PIdent, seq[PackedItemId]]
+      # PackedItemId so that it works with reexported symbols too
+      # ifaceHidden includes private symbols
 
-  PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
+type
+  PackedModuleGraph* = object
+    pm*: seq[LoadedModule] # indexed by FileIndex
+    when BenchIC:
+      depAnalysis: MonoTime
+      loadBody: MonoTime
+      loadSym, loadType, loadBodies: MonoTime
+
+when BenchIC:
+  proc echoTimes*(m: PackedModuleGraph) =
+    echo "analysis: ", m.depAnalysis, " loadBody: ", m.loadBody, " loadSym: ",
+      m.loadSym, " loadType: ", m.loadType, " all bodies: ", gloadBodies
+
+template `[]`*(m: PackedModuleGraph; i: int): LoadedModule = m.pm[i]
+template len*(m: PackedModuleGraph): int = m.pm.len
 
 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType
 proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym
 
 proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: int; f: LitId): FileIndex =
-  if c.lastLit == f and c.lastModule == thisModule:
+  if f == LitId(0):
+    result = InvalidFileIdx
+  elif c.lastLit == f and c.lastModule == thisModule:
     result = c.lastFile
   else:
     result = toFileIndex(f, g[thisModule].fromDisk, c.config)
@@ -669,9 +806,10 @@ proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule:
 
 proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
                        x: PackedLineInfo): TLineInfo =
-  assert g[thisModule].status in {loaded, storing}
-  result = TLineInfo(line: x.line, col: x.col,
-            fileIndex: toFileIndexCached(c, g, thisModule, x.file))
+  assert g[thisModule].status in {loaded, storing, stored}
+  let (fileId, line, col) = unpack(g[thisModule].fromDisk.man, x)
+  result = TLineInfo(line: line.uint16, col: col.int16,
+            fileIndex: toFileIndexCached(c, g, thisModule, fileId))
 
 proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
                 tree: PackedTree; n: NodePos): PNode =
@@ -685,26 +823,28 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
   result.flags = n.flags
 
   case k
-  of nkEmpty, nkNilLit, nkType:
+  of nkNone, nkEmpty, nkNilLit, nkType:
     discard
   of nkIdent:
-    result.ident = getIdent(c.cache, g[thisModule].fromDisk.sh.strings[n.litId])
+    result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId])
   of nkSym:
-    result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
-  of directIntLit:
-    result.intVal = tree.nodes[n.int].operand
+    result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree[n].soperand))
+    if result.typ == nil and nfOpenSym notin result.flags:
+      result.typ = result.sym.typ
   of externIntLit:
-    result.intVal = g[thisModule].fromDisk.sh.integers[n.litId]
+    result.intVal = g[thisModule].fromDisk.numbers[n.litId]
   of nkStrLit..nkTripleStrLit:
-    result.strVal = g[thisModule].fromDisk.sh.strings[n.litId]
+    result.strVal = g[thisModule].fromDisk.strings[n.litId]
   of nkFloatLit..nkFloat128Lit:
-    result.floatVal = g[thisModule].fromDisk.sh.floats[n.litId]
+    result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId])
   of nkModuleRef:
     let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
+    assert n1.kind == nkNone
+    assert n2.kind == nkNone
     transitionNoneToSym(result)
-    result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand))
+    result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree[n2].soperand))
+    if result.typ == nil and nfOpenSym notin result.flags:
+      result.typ = result.sym.typ
   else:
     for n0 in sonsReadonly(tree, n):
       result.addAllowNil loadNodes(c, g, thisModule, tree, n0)
@@ -736,6 +876,7 @@ proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph; thisModule:
 
 proc loadProcBody(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
                   tree: PackedTree; n: NodePos): PNode =
+  result = nil
   var i = 0
   for n0 in sonsReadonly(tree, n):
     if i == bodyPos:
@@ -753,8 +894,10 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     kind: s.kind, magic: s.magic, flags: s.flags,
     info: translateLineInfo(c, g, si, s.info),
     options: s.options,
-    position: s.position,
-    name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name])
+    position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position,
+    offset: if s.kind in routineKinds: defaultOffset else: s.offset,
+    disamb: s.disamb,
+    name: getIdent(c.cache, g[si].fromDisk.strings[s.name])
   )
 
 template loadAstBody(p, field) =
@@ -771,8 +914,8 @@ proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
   if l.name.int == 0:
     result = nil
   else:
-    result = PLib(generated: l.generated, isOverriden: l.isOverriden,
-                  kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name])
+    result = PLib(generated: l.generated, isOverridden: l.isOverridden,
+                  kind: l.kind, name: rope g[si].fromDisk.strings[l.name])
     loadAstBody(l, path)
 
 proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
@@ -785,37 +928,53 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     loadAstBody(s, ast)
   result.annex = loadLib(c, g, si, item, s.annex)
   when hasFFI:
-    result.cname = g[si].fromDisk.sh.strings[s.cname]
+    result.cname = g[si].fromDisk.strings[s.cname]
 
   if s.kind in {skLet, skVar, skField, skForVar}:
     result.guard = loadSym(c, g, si, s.guard)
     result.bitsize = s.bitsize
     result.alignment = s.alignment
   result.owner = loadSym(c, g, si, s.owner)
-  let externalName = g[si].fromDisk.sh.strings[s.externalName]
+  let externalName = g[si].fromDisk.strings[s.externalName]
   if externalName != "":
     result.loc.r = rope externalName
   result.loc.flags = s.locFlags
+  result.instantiatedFrom = loadSym(c, g, si, s.instantiatedFrom)
+
+proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
+                    fileIdx: FileIndex; cachedModules: var seq[FileIndex]): bool
+proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
+                       fileIdx: FileIndex; m: var LoadedModule)
 
 proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym =
   if s == nilItemId:
     result = nil
   else:
     let si = moduleIndex(c, g, thisModule, s)
-    assert g[si].status in {loaded, storing}
+    if si >= g.len:
+      g.pm.setLen(si+1)
+
+    if g[si].status == undefined and c.config.cmd == cmdM:
+      var cachedModules: seq[FileIndex] = @[]
+      discard needsRecompile(g, c.config, c.cache, FileIndex(si), cachedModules)
+      for m in cachedModules:
+        loadToReplayNodes(g, c.config, c.cache, m, g[int m])
+
+    assert g[si].status in {loaded, storing, stored}
     if not g[si].symsInit:
       g[si].symsInit = true
-      setLen g[si].syms, g[si].fromDisk.sh.syms.len
+      setLen g[si].syms, g[si].fromDisk.syms.len
 
     if g[si].syms[s.item] == nil:
-      if g[si].fromDisk.sh.syms[s.item].kind != skModule:
-        result = symHeaderFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item)
+      if g[si].fromDisk.syms[s.item].kind != skModule:
+        result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item)
         # store it here early on, so that recursions work properly:
         g[si].syms[s.item] = result
-        symBodyFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item, result)
+        symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result)
       else:
         result = g[si].module
         assert result != nil
+        g[si].syms[s.item] = result
 
     else:
       result = g[si].syms[s.item]
@@ -824,7 +983,7 @@ proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                           t: PackedType; si, item: int32): PType =
   result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind,
                 flags: t.flags, size: t.size, align: t.align,
-                paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel,
+                paddingAtEnd: t.paddingAtEnd,
                 uniqueId: ItemId(module: si, item: item),
                 callConv: t.callConv)
 
@@ -836,8 +995,10 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     for op, item in pairs t.attachedOps:
       result.attachedOps[op] = loadSym(c, g, si, item)
   result.typeInst = loadType(c, g, si, t.typeInst)
+  var sons = newSeq[PType]()
   for son in items t.types:
-    result.sons.add loadType(c, g, si, son)
+    sons.add loadType(c, g, si, son)
+  result.setSons(sons)
   loadAstBody(t, n)
   when false:
     for gen, id in items t.methods:
@@ -848,42 +1009,43 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
     result = nil
   else:
     let si = moduleIndex(c, g, thisModule, t)
-    assert g[si].status in {loaded, storing}
+    assert g[si].status in {loaded, storing, stored}
     assert t.item > 0
 
     if not g[si].typesInit:
       g[si].typesInit = true
-      setLen g[si].types, g[si].fromDisk.sh.types.len
+      setLen g[si].types, g[si].fromDisk.types.len
 
     if g[si].types[t.item] == nil:
-      result = typeHeaderFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item)
+      result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item)
       # store it here early on, so that recursions work properly:
       g[si].types[t.item] = result
-      typeBodyFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item, result)
+      typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result)
+      #assert result.itemId.item == t.item, $(result.itemId.item, t.item)
+      assert result.itemId.item > 0, $(result.itemId.item, t.item)
     else:
       result = g[si].types[t.item]
-    assert result.itemId.item > 0
-
-proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
-  let filename = AbsoluteFile toFullPath(config, fileIdx)
-  let name = getIdent(cache, splitFile(filename).name)
-  let info = newLineInfo(fileIdx, 1, 1)
-  let
-    pck = getPackageName(config, filename.string)
-    pck2 = if pck.len > 0: pck else: "unknown"
-    pack = getIdent(cache, pck2)
-  result = newSym(skPackage, getIdent(cache, pck2),
-    ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info)
+      assert result.itemId.item > 0, "2"
 
 proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
   m.iface = initTable[PIdent, seq[PackedItemId]]()
-  for e in m.fromDisk.exports:
+  m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]()
+  template impl(iface, e) =
     let nameLit = e[0]
-    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1]))
-  for re in m.fromDisk.reexports:
-    let nameLit = re[0]
-    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1])
+    let e2 =
+      when e[1] is PackedItemId: e[1]
+      else: PackedItemId(module: LitId(0), item: e[1])
+    iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2)
+
+  for e in m.fromDisk.exports:
+    m.iface.impl(e)
+    m.ifaceHidden.impl(e)
+  for e in m.fromDisk.reexports:
+    m.iface.impl(e)
+    m.ifaceHidden.impl(e)
+  for e in m.fromDisk.hidden:
+    m.ifaceHidden.impl(e)
 
   let filename = AbsoluteFile toFullPath(conf, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
@@ -892,9 +1054,8 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
                   name: getIdent(cache, splitFile(filename).name),
                   info: newLineInfo(fileIdx, 1, 1),
                   position: int(fileIdx))
-  m.module.owner = newPackage(conf, cache, fileIdx)
-  if fileIdx == conf.projectMainIdx2:
-    m.module.flags.incl sfMainModule
+  m.module.owner = getPackage(conf, cache, fileIdx)
+  m.module.flags = m.fromDisk.moduleFlags
 
 proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
@@ -914,30 +1075,37 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache
   # Does the file belong to the fileIdx need to be recompiled?
   let m = int(fileIdx)
   if m >= g.len:
-    g.setLen(m+1)
+    g.pm.setLen(m+1)
 
   case g[m].status
   of undefined:
     g[m].status = loading
     let fullpath = msgs.toFullPath(conf, fileIdx)
     let rod = toRodFile(conf, AbsoluteFile fullpath)
-    let err = loadRodFile(rod, g[m].fromDisk, conf)
+    let err = loadRodFile(rod, g[m].fromDisk, conf, ignoreConfig = conf.cmd == cmdM)
     if err == ok:
-      result = optForceFullMake in conf.globalOptions
-      # check its dependencies:
-      for dep in g[m].fromDisk.imports:
-        let fid = toFileIndex(dep, g[m].fromDisk, conf)
-        # Warning: we need to traverse the full graph, so
-        # do **not use break here**!
-        if needsRecompile(g, conf, cache, fid, cachedModules):
-          result = true
-
-      if not result:
+      if conf.cmd == cmdM:
         setupLookupTables(g, conf, cache, fileIdx, g[m])
         cachedModules.add fileIdx
         g[m].status = loaded
+        result = false
       else:
-        g[m] = LoadedModule(status: outdated, module: g[m].module)
+        result = optForceFullMake in conf.globalOptions
+        # check its dependencies:
+        let imp = g[m].fromDisk.imports
+        for dep in imp:
+          let fid = toFileIndex(dep, g[m].fromDisk, conf)
+          # Warning: we need to traverse the full graph, so
+          # do **not use break here**!
+          if needsRecompile(g, conf, cache, fid, cachedModules):
+            result = true
+
+        if not result:
+          setupLookupTables(g, conf, cache, fileIdx, g[m])
+          cachedModules.add fileIdx
+          g[m].status = loaded
+        else:
+          g.pm[m] = LoadedModule(status: outdated, module: g[m].module)
     else:
       loadError(err, rod, conf)
       g[m].status = outdated
@@ -946,18 +1114,19 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache
   of loading, loaded:
     # For loading: Assume no recompile is required.
     result = false
-  of outdated, storing:
+  of outdated, storing, stored:
     result = true
 
 proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                         fileIdx: FileIndex; cachedModules: var seq[FileIndex]): PSym =
   ## Returns 'nil' if the module needs to be recompiled.
-  if needsRecompile(g, conf, cache, fileIdx, cachedModules):
-    result = nil
-  else:
-    result = g[int fileIdx].module
-    assert result != nil
-    assert result.position == int(fileIdx)
+  bench g.depAnalysis:
+    if needsRecompile(g, conf, cache, fileIdx, cachedModules):
+      result = nil
+    else:
+      result = g[int fileIdx].module
+      assert result != nil
+      assert result.position == int(fileIdx)
   for m in cachedModules:
     loadToReplayNodes(g, conf, cache, m, g[int m])
 
@@ -971,46 +1140,49 @@ template setupDecoder() {.dirty.} =
 
 proc loadProcBody*(config: ConfigRef, cache: IdentCache;
                    g: var PackedModuleGraph; s: PSym): PNode =
-  let mId = s.itemId.module
-  var decoder = PackedDecoder(
-    lastModule: int32(-1),
-    lastLit: LitId(0),
-    lastFile: FileIndex(-1),
-    config: config,
-    cache: cache)
-  let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast
-  assert pos != emptyNodeId
-  result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos)
+  bench g.loadBody:
+    let mId = s.itemId.module
+    var decoder = PackedDecoder(
+      lastModule: int32(-1),
+      lastLit: LitId(0),
+      lastFile: FileIndex(-1),
+      config: config,
+      cache: cache)
+    let pos = g[mId].fromDisk.syms[s.itemId.item].ast
+    assert pos != emptyNodeId
+    result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos)
 
 proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
                      g: var PackedModuleGraph; module: int; id: PackedItemId): PType =
-  if id.item < g[module].types.len:
-    result = g[module].types[id.item]
-  else:
-    result = nil
-  if result == nil:
-    var decoder = PackedDecoder(
-      lastModule: int32(-1),
-      lastLit: LitId(0),
-      lastFile: FileIndex(-1),
-      config: config,
-      cache: cache)
-    result = loadType(decoder, g, module, id)
+  bench g.loadType:
+    if id.item < g[module].types.len:
+      result = g[module].types[id.item]
+    else:
+      result = nil
+    if result == nil:
+      var decoder = PackedDecoder(
+        lastModule: int32(-1),
+        lastLit: LitId(0),
+        lastFile: FileIndex(-1),
+        config: config,
+        cache: cache)
+      result = loadType(decoder, g, module, id)
 
 proc loadSymFromId*(config: ConfigRef, cache: IdentCache;
                     g: var PackedModuleGraph; module: int; id: PackedItemId): PSym =
-  if id.item < g[module].syms.len:
-    result = g[module].syms[id.item]
-  else:
-    result = nil
-  if result == nil:
-    var decoder = PackedDecoder(
-      lastModule: int32(-1),
-      lastLit: LitId(0),
-      lastFile: FileIndex(-1),
-      config: config,
-      cache: cache)
-    result = loadSym(decoder, g, module, id)
+  bench g.loadSym:
+    if id.item < g[module].syms.len:
+      result = g[module].syms[id.item]
+    else:
+      result = nil
+    if result == nil:
+      var decoder = PackedDecoder(
+        lastModule: int32(-1),
+        lastLit: LitId(0),
+        lastFile: FileIndex(-1),
+        config: config,
+        cache: cache)
+      result = loadSym(decoder, g, module, id)
 
 proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId =
   if id.module == LitId(0):
@@ -1020,15 +1192,15 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi
 
 proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) =
   var bugs = 0
-  for i in 1 .. high(m.sh.syms):
-    if m.sh.syms[i].kind == skUnknown:
+  for i in 1 .. high(m.syms):
+    if m.syms[i].kind == skUnknown:
       echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId))
       inc bugs
   assert bugs == 0
   when false:
     var nones = 0
-    for i in 1 .. high(m.sh.types):
-      inc nones, m.sh.types[i].kind == tyNone
+    for i in 1 .. high(m.types):
+      inc nones, m.types[i].kind == tyNone
     assert nones < 1
 
 proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
@@ -1050,24 +1222,31 @@ type
     values: seq[PackedItemId]
     i, module: int
 
+template interfSelect(a: LoadedModule, importHidden: bool): auto =
+  var ret = a.iface.addr
+  if importHidden: ret = a.ifaceHidden.addr
+  ret[]
+
 proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache;
                   g: var PackedModuleGraph; module: FileIndex;
-                  name: PIdent): PSym =
+                  name: PIdent, importHidden: bool): PSym =
   it.decoder = PackedDecoder(
     lastModule: int32(-1),
     lastLit: LitId(0),
     lastFile: FileIndex(-1),
     config: config,
     cache: cache)
-  it.values = g[int module].iface.getOrDefault(name)
+  it.values = g[int module].interfSelect(importHidden).getOrDefault(name)
   it.i = 0
   it.module = int(module)
   if it.i < it.values.len:
     result = loadSym(it.decoder, g, int(module), it.values[it.i])
     inc it.i
+  else:
+    result = nil
 
 proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
-                         g: var PackedModuleGraph; module: FileIndex): PSym =
+                         g: var PackedModuleGraph; module: FileIndex; importHidden: bool): PSym =
   it.decoder = PackedDecoder(
     lastModule: int32(-1),
     lastLit: LitId(0),
@@ -1076,23 +1255,27 @@ proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
     cache: cache)
   it.values = @[]
   it.module = int(module)
-  for v in g[int module].iface.values:
+  for v in g[int module].interfSelect(importHidden).values:
     it.values.add v
   it.i = 0
   if it.i < it.values.len:
     result = loadSym(it.decoder, g, int(module), it.values[it.i])
     inc it.i
+  else:
+    result = nil
 
 proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym =
   if it.i < it.values.len:
     result = loadSym(it.decoder, g, it.module, it.values[it.i])
     inc it.i
+  else:
+    result = nil
 
 iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
                            g: var PackedModuleGraph; module: FileIndex;
-                           name: PIdent): PSym =
+                           name: PIdent, importHidden: bool): PSym =
   setupDecoder()
-  let values = g[int module].iface.getOrDefault(name)
+  let values = g[int module].interfSelect(importHidden).getOrDefault(name)
   for pid in values:
     let s = loadSym(decoder, g, int(module), pid)
     assert s != nil
@@ -1100,48 +1283,72 @@ iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
 
 proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
                       g: var PackedModuleGraph; module: FileIndex;
-                      name: PIdent): PSym =
+                      name: PIdent, importHidden: bool): PSym =
   setupDecoder()
-  let values = g[int module].iface.getOrDefault(name)
+  let values = g[int module].interfSelect(importHidden).getOrDefault(name)
   result = loadSym(decoder, g, int(module), values[0])
 
 proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
-  IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
-              typeId: int32 m.fromDisk.sh.types.len)
+  IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len,
+              typeId: int32 m.fromDisk.types.len)
 
 proc searchForCompilerproc*(m: LoadedModule; name: string): int32 =
   # slow, linear search, but the results are cached:
   for it in items(m.fromDisk.compilerProcs):
-    if m.fromDisk.sh.strings[it[0]] == name:
+    if m.fromDisk.strings[it[0]] == name:
       return it[1]
   return -1
 
 # ------------------------- .rod file viewer ---------------------------------
 
 proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
-  var m: PackedModule
+  var m: PackedModule = PackedModule()
   let err = loadRodFile(rodfile, m, config, ignoreConfig=true)
   if err != ok:
-    echo "Error: could not load: ", rodfile.string, " reason: ", err
-    quit 1
+    config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err
 
-  when true:
+  when false:
     echo "exports:"
     for ex in m.exports:
-      echo "  ", m.sh.strings[ex[0]], " local ID: ", ex[1]
-      assert ex[0] == m.sh.syms[ex[1]].name
+      echo "  ", m.strings[ex[0]], " local ID: ", ex[1]
+      assert ex[0] == m.syms[ex[1]].name
       # ex[1] int32
 
     echo "reexports:"
     for ex in m.reexports:
-      echo "  ", m.sh.strings[ex[0]]
+      echo "  ", m.strings[ex[0]]
     #  reexports*: seq[(LitId, PackedItemId)]
 
-  echo "all symbols"
-  for i in 0..high(m.sh.syms):
-    echo "  ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i
+    echo "hidden: " & $m.hidden.len
+    for ex in m.hidden:
+      echo "  ", m.strings[ex[0]], " local ID: ", ex[1]
 
-  echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len,
-    " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
-    " strings: ", m.sh.strings.len, " integers: ", m.sh.integers.len,
-    " floats: ", m.sh.floats.len
+  when false:
+    echo "all symbols"
+    for i in 0..high(m.syms):
+      if m.syms[i].name != LitId(0):
+        echo "  ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind
+      else:
+        echo "   local ID: ", i, " kind ", m.syms[i].kind
+
+  echo "symbols: ", m.syms.len, " types: ", m.types.len,
+    " top level nodes: ", m.topLevel.len, " other nodes: ", m.bodies.len,
+    " strings: ", m.strings.len, " numbers: ", m.numbers.len
+
+  echo "SIZES:"
+  echo "symbols: ", m.syms.len * sizeof(PackedSym), " types: ", m.types.len * sizeof(PackedType),
+    " top level nodes: ", m.topLevel.len * sizeof(PackedNode),
+    " other nodes: ", m.bodies.len * sizeof(PackedNode),
+    " strings: ", sizeOnDisc(m.strings)
+  when false:
+    var tt = 0
+    var fc = 0
+    for x in m.topLevel:
+      if x.kind == nkSym or x.typeId == nilItemId: inc tt
+      if x.flags == {}: inc fc
+    for x in m.bodies:
+      if x.kind == nkSym or x.typeId == nilItemId: inc tt
+      if x.flags == {}: inc fc
+    let total = float(m.topLevel.len + m.bodies.len)
+    echo "nodes with nil type: ", tt, " in % ", tt.float / total
+    echo "nodes with empty flags: ", fc.float / total
diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim
new file mode 100644
index 0000000000..d78e568479
--- /dev/null
+++ b/compiler/ic/integrity.nim
@@ -0,0 +1,155 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Integrity checking for a set of .rod files.
+## The set must cover a complete Nim project.
+
+import std/sets
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
+import ".." / [ast, modulegraphs]
+import packed_ast, bitabs, ic
+
+type
+  CheckedContext = object
+    g: ModuleGraph
+    thisModule: int32
+    checkedSyms: HashSet[ItemId]
+    checkedTypes: HashSet[ItemId]
+
+proc checkType(c: var CheckedContext; typeId: PackedItemId)
+proc checkForeignSym(c: var CheckedContext; symId: PackedItemId)
+proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos)
+
+proc checkTypeObj(c: var CheckedContext; typ: PackedType) =
+  for child in typ.types:
+    checkType(c, child)
+  if typ.n != emptyNodeId:
+    checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos typ.n)
+  if typ.sym != nilItemId:
+    checkForeignSym(c, typ.sym)
+  if typ.owner != nilItemId:
+    checkForeignSym(c, typ.owner)
+  checkType(c, typ.typeInst)
+
+proc checkType(c: var CheckedContext; typeId: PackedItemId) =
+  if typeId == nilItemId: return
+  let itemId = translateId(typeId, c.g.packed, c.thisModule, c.g.config)
+  if not c.checkedTypes.containsOrIncl(itemId):
+    let oldThisModule = c.thisModule
+    c.thisModule = itemId.module
+    checkTypeObj c, c.g.packed[itemId.module].fromDisk.types[itemId.item]
+    c.thisModule = oldThisModule
+
+proc checkSym(c: var CheckedContext; s: PackedSym) =
+  if s.name != LitId(0):
+    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId s.name
+  checkType c, s.typ
+  if s.ast != emptyNodeId:
+    checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast)
+  if s.owner != nilItemId:
+    checkForeignSym(c, s.owner)
+
+proc checkLocalSym(c: var CheckedContext; item: int32) =
+  let itemId = ItemId(module: c.thisModule, item: item)
+  if not c.checkedSyms.containsOrIncl(itemId):
+    checkSym c, c.g.packed[c.thisModule].fromDisk.syms[item]
+
+proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) =
+  let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config)
+  if not c.checkedSyms.containsOrIncl(itemId):
+    let oldThisModule = c.thisModule
+    c.thisModule = itemId.module
+    checkSym c, c.g.packed[itemId.module].fromDisk.syms[itemId.item]
+    c.thisModule = oldThisModule
+
+proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) =
+  let t = findType(tree, n)
+  if t != nilItemId:
+    checkType(c, t)
+  case n.kind
+  of nkEmpty, nkNilLit, nkType, nkNilRodNode:
+    discard
+  of nkIdent:
+    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
+  of nkSym:
+    checkLocalSym(c, tree[n].soperand)
+  of directIntLit:
+    discard
+  of externIntLit, nkFloatLit..nkFloat128Lit:
+    assert c.g.packed[c.thisModule].fromDisk.numbers.hasLitId n.litId
+  of nkStrLit..nkTripleStrLit:
+    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
+  of nkModuleRef:
+    let (n1, n2) = sons2(tree, n)
+    assert n1.kind == nkNone
+    assert n2.kind == nkNone
+    checkForeignSym(c, PackedItemId(module: n1.litId, item: tree[n2].soperand))
+  else:
+    for n0 in sonsReadonly(tree, n):
+      checkNode(c, tree, n0)
+
+proc checkTree(c: var CheckedContext; t: PackedTree) =
+  for p in allNodes(t): checkNode(c, t, p)
+
+proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) =
+  for symId in symIds:
+    assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len
+
+proc checkModule(c: var CheckedContext; m: PackedModule) =
+  # We check that:
+  # - Every symbol references existing types and symbols.
+  # - Every tree node references existing types and symbols.
+  for i in 0..high(m.syms):
+    checkLocalSym c, int32(i)
+
+  checkTree c, m.toReplay
+  checkTree c, m.topLevel
+
+  for e in m.exports:
+    assert e[1] >= 0 and e[1] < m.syms.len
+    assert e[0] == m.syms[e[1]].name
+
+  for e in m.compilerProcs:
+    assert e[1] >= 0 and e[1] < m.syms.len
+    assert e[0] == m.syms[e[1]].name
+
+  checkLocalSymIds c, m, m.converters
+  checkLocalSymIds c, m, m.methods
+  checkLocalSymIds c, m, m.trmacros
+  checkLocalSymIds c, m, m.pureEnums
+  #[
+    To do: Check all these fields:
+
+    reexports*: seq[(LitId, PackedItemId)]
+    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
+
+    typeInstCache*: seq[(PackedItemId, PackedItemId)]
+    procInstCache*: seq[PackedInstantiation]
+    attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
+    methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)]
+    enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
+    methodsPerType*: seq[(PackedItemId, PackedItemId)]
+    dispatchers*: seq[PackedItemId]
+  ]#
+
+proc checkIntegrity*(g: ModuleGraph) =
+  var c = CheckedContext(g: g)
+  for i in 0..= currentCol and col < currentCol+tokenLen:
+      result = true
+    else:
+      result = false
+  else:
+    result = false
+
+proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool =
+  result = s.name != LitId(0) and
+    isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len)
+
+proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool =
+  let name = c.g.packed[s.module].fromDisk.syms[s.item].name
+  result = name != LitId(0) and
+    isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len)
+
+const
+  EmptyItemId = ItemId(module: -1'i32, item: -1'i32)
+
+proc search(c: var NavContext; tree: PackedTree): ItemId =
+  # We use the linear representation here directly:
+  for i in 0..= 0 and tree[n].kind in declarativeNodes
+
+proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) =
+  let (fileId, line, col) = unpack(c.g.packed[c.thisModule].fromDisk.man, info)
+  var m = ""
+  var file = c.g.packed[c.thisModule].fromDisk.strings[fileId]
+  if c.outputSep == ' ':
+    file = os.extractFilename file
+  toLocation(m, file, line, col + ColOffset)
+  if not c.alreadyEmitted.containsOrIncl(m):
+    msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m
+
+proc list(c: var NavContext; tree: PackedTree; sym: ItemId) =
+  for i in 0.. nkNilLit
+  let k = tree.nodes[pos].kind
+  assert k > nkNilLit
   let distance = int32(tree.nodes.len - pos)
-  tree.nodes[pos].operand = distance
+  assert distance > 0
+  tree.nodes[pos].x = toX(k, cast[uint32](distance))
 
 proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
 
-proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} =
-  tree.nodes[i]
+proc `[]`*(tree: PackedTree; i: NodePos): lent PackedNode {.inline.} =
+  tree.nodes[i.int]
+
+template rawSpan(n: PackedNode): int = int(uoperand(n))
 
 proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
   if tree.nodes[pos].kind > nkNilLit:
-    assert tree.nodes[pos].operand > 0
-    inc pos, tree.nodes[pos].operand
+    assert tree.nodes[pos].uoperand > 0
+    inc pos, tree.nodes[pos].rawSpan
   else:
     inc pos
 
 iterator sonsReadonly*(tree: PackedTree; n: NodePos): NodePos =
   var pos = n.int
   assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
+  let last = pos + tree.nodes[pos].rawSpan
   inc pos
   while pos < last:
     yield NodePos pos
@@ -234,7 +204,7 @@ iterator isons*(dest: var PackedTree; tree: PackedTree;
 iterator sonsFrom1*(tree: PackedTree; n: NodePos): NodePos =
   var pos = n.int
   assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
+  let last = pos + tree.nodes[pos].rawSpan
   inc pos
   if pos < last:
     nextChild tree, pos
@@ -248,7 +218,7 @@ iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos =
     inc count
   var pos = n.int
   assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
+  let last = pos + tree.nodes[pos].rawSpan
   inc pos
   while pos < last and count > 2:
     yield NodePos pos
@@ -258,9 +228,9 @@ iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos =
 proc parentImpl(tree: PackedTree; n: NodePos): NodePos =
   # finding the parent of a node is rather easy:
   var pos = n.int - 1
-  while pos >= 0 and isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int):
+  while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].rawSpan - 1 < n.int)):
     dec pos
-  assert pos >= 0, "node has no parent"
+  #assert pos >= 0, "node has no parent"
   result = NodePos(pos)
 
 template parent*(n: NodePos): NodePos = parentImpl(tree, n)
@@ -284,20 +254,32 @@ proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} =
 proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} =
   tree.nodes[n.int].kind
 proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} =
-  LitId tree.nodes[n.int].operand
+  LitId tree.nodes[n.int].uoperand
 proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} =
   tree.nodes[n.int].info
 
-template typ*(n: NodePos): PackedItemId =
-  tree.nodes[n.int].typeId
-template flags*(n: NodePos): TNodeFlags =
-  tree.nodes[n.int].flags
+proc findType*(tree: PackedTree; n: NodePos): PackedItemId =
+  for x in tree.withTypes:
+    if x[0] == int32(n): return x[1]
+    if x[0] > int32(n): return nilItemId
+  return nilItemId
 
-template operand*(n: NodePos): int32 =
-  tree.nodes[n.int].operand
+proc findFlags*(tree: PackedTree; n: NodePos): TNodeFlags =
+  for x in tree.withFlags:
+    if x[0] == int32(n): return x[1]
+    if x[0] > int32(n): return {}
+  return {}
+
+template typ*(n: NodePos): PackedItemId =
+  tree.findType(n)
+template flags*(n: NodePos): TNodeFlags =
+  tree.findFlags(n)
+
+template uoperand*(n: NodePos): uint32 =
+  tree.nodes[n.int].uoperand
 
 proc span*(tree: PackedTree; pos: int): int {.inline.} =
-  if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
+  if isAtom(tree, pos): 1 else: tree.nodes[pos].rawSpan
 
 proc sons2*(tree: PackedTree; n: NodePos): (NodePos, NodePos) =
   assert(not isAtom(tree, n.int))
@@ -313,6 +295,7 @@ proc sons3*(tree: PackedTree; n: NodePos): (NodePos, NodePos, NodePos) =
   result = (NodePos a, NodePos b, NodePos c)
 
 proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos =
+  result = default(NodePos)
   if tree.nodes[n.int].kind > nkNilLit:
     var count = 0
     for child in sonsReadonly(tree, n):
@@ -326,105 +309,28 @@ when false:
 
 template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind
 template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info
-template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].operand
+template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].uoperand
 
-template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
+template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].soperand
 
 proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
 
-when false:
-  proc strLit*(tree: PackedTree; n: NodePos): lent string =
-    assert n.kind == nkStrLit
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-
-  proc strVal*(tree: PackedTree; n: NodePos): string =
-    assert n.kind == nkStrLit
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-    #result = cookedStrLit(raw)
-
-  proc filenameVal*(tree: PackedTree; n: NodePos): string =
-    case n.kind
-    of nkStrLit:
-      result = strVal(tree, n)
-    of nkIdent:
-      result = tree.sh.strings[n.litId]
-    of nkSym:
-      result = tree.sh.strings[tree.sh.syms[int n.symId].name]
-    else:
-      result = ""
-
-  proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
-    assert n.kind == nkIdent
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-
 const
   externIntLit* = {nkCharLit,
     nkIntLit,
     nkInt8Lit,
     nkInt16Lit,
+    nkInt32Lit,
     nkInt64Lit,
     nkUIntLit,
     nkUInt8Lit,
     nkUInt16Lit,
     nkUInt32Lit,
-    nkUInt64Lit} # nkInt32Lit is missing by design!
+    nkUInt64Lit}
 
-  externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt64Lit}
+  externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit}
   externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
-  directIntLit* = nkInt32Lit
-
-proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int;
-               result: var string) =
-  let pos = n.int
-  if result.len > 0 and result[^1] notin {' ', '\n'}:
-    result.add ' '
-
-  result.add $tree[pos].kind
-  case tree.nodes[pos].kind
-  of nkNone, nkEmpty, nkNilLit, nkType: discard
-  of nkIdent, nkStrLit..nkTripleStrLit:
-    result.add " "
-    result.add sh.strings[LitId tree.nodes[pos].operand]
-  of nkSym:
-    result.add " "
-    result.add sh.strings[sh.syms[tree.nodes[pos].operand].name]
-  of directIntLit:
-    result.add " "
-    result.addInt tree.nodes[pos].operand
-  of externSIntLit:
-    result.add " "
-    result.addInt sh.integers[LitId tree.nodes[pos].operand]
-  of externUIntLit:
-    result.add " "
-    result.add $cast[uint64](sh.integers[LitId tree.nodes[pos].operand])
-  else:
-    result.add "(\n"
-    for i in 1..(nesting+1)*2: result.add ' '
-    for child in sonsReadonly(tree, n):
-      toString(tree, child, sh, nesting + 1, result)
-    result.add "\n"
-    for i in 1..nesting*2: result.add ' '
-    result.add ")"
-    #for i in 1..nesting*2: result.add ' '
-
-
-proc toString*(tree: PackedTree; n: NodePos; sh: Shared): string =
-  result = ""
-  toString(tree, n, sh, 0, result)
-
-proc debug*(tree: PackedTree; sh: Shared) =
-  stdout.write toString(tree, NodePos 0, sh)
-
-when false:
-  proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
-    if n.kind == nkIdent:
-      result = n.litId
-    elif n.kind == nkSym:
-      result = tree.sh.syms[int n.symId].name
-    else:
-      result = LitId(0)
-
-  template identId*(n: NodePos): LitId = identIdImpl(tree, n)
+  directIntLit* = nkNone
 
 template copyInto*(dest, n, body) =
   let patchPos = prepare(dest, tree, n)
@@ -436,28 +342,8 @@ template copyIntoKind*(dest, kind, info, body) =
   body
   patch dest, patchPos
 
-when false:
-  proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
-    let litId = tree.sh.strings.getKeyId(pragma)
-    if litId == LitId(0):
-      return false
-    assert n.kind == nkPragma
-    for ch0 in sonsReadonly(tree, n):
-      if ch0.kind == nkExprColonExpr:
-        if ch0.firstSon.identId == litId:
-          return true
-      elif ch0.identId == litId:
-        return true
-
 proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len
 
-when false:
-  proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) =
-    let patchPos = prepare(dest, nkError, n.info)
-    dest.add nkStrLit, msg, n.info
-    copyTree(dest, tree, n)
-    patch dest, patchPos
-
 iterator allNodes*(tree: PackedTree): NodePos =
   var p = 0
   while p < tree.len:
@@ -467,3 +353,13 @@ iterator allNodes*(tree: PackedTree): NodePos =
 
 proc toPackedItemId*(item: int32): PackedItemId {.inline.} =
   PackedItemId(module: LitId(0), item: item)
+
+proc load*(f: var RodFile; t: var PackedTree) =
+  loadSeq f, t.nodes
+  loadSeq f, t.withFlags
+  loadSeq f, t.withTypes
+
+proc store*(f: var RodFile; t: PackedTree) =
+  storeSeq f, t.nodes
+  storeSeq f, t.withFlags
+  storeSeq f, t.withTypes
diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim
index 61aa0e697f..b244ec885c 100644
--- a/compiler/ic/replayer.nim
+++ b/compiler/ic/replayer.nim
@@ -14,7 +14,10 @@
 import ".." / [ast, modulegraphs, trees, extccomp, btrees,
   msgs, lineinfos, pathutils, options, cgmeth]
 
-import tables
+import std/tables
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 import packed_ast, ic, bitabs
 
@@ -86,6 +89,31 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) =
       else:
         internalAssert g.config, false
 
+proc replayBackendProcs*(g: ModuleGraph; module: int) =
+  for it in mitems(g.packed[module].fromDisk.attachedOps):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let op = it[1]
+    let tmp = translateId(it[2], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[2])
+    g.attachedOps[op][key] = LazySym(id: symId, sym: nil)
+
+  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let tmp = translateId(it[1], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[1])
+    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
+
+  for it in mitems(g.packed[module].fromDisk.methodsPerType):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let tmp = translateId(it[1], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[1])
+    g.methodsPerType.mgetOrPut(key, @[]).add LazySym(id: symId, sym: nil)
+
+  for it in mitems(g.packed[module].fromDisk.dispatchers):
+    let tmp = translateId(it, g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it)
+    g.dispatchers.add LazySym(id: symId, sym: nil)
+
 proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
   ## We remember the generic instantiations a module performed
   ## in order to to avoid the code bloat that generic code tends
@@ -110,18 +138,14 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
       module: module, sym: FullId(module: sym.module, packed: it.sym),
       concreteTypes: concreteTypes, inst: nil)
 
-  for it in mitems(g.packed[module].fromDisk.methodsPerType):
+  for it in mitems(g.packed[module].fromDisk.methodsPerGenericType):
     let key = translateId(it[0], g.packed, module, g.config)
     let col = it[1]
     let tmp = translateId(it[2], g.packed, module, g.config)
     let symId = FullId(module: tmp.module, packed: it[2])
-    g.methodsPerType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil))
+    g.methodsPerGenericType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil))
 
-  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let tmp = translateId(it[1], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[1])
-    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
+  replayBackendProcs(g, module)
 
   for it in mitems(g.packed[module].fromDisk.methods):
     let sym = loadSymFromId(g.config, g.cache, g.packed, module,
diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim
index fa0a7c7342..5eef3874a2 100644
--- a/compiler/ic/rodfiles.nim
+++ b/compiler/ic/rodfiles.nim
@@ -7,7 +7,66 @@
 #    distribution, for details about the copyright.
 #
 
-from typetraits import supportsCopyMem
+## Low level binary format used by the compiler to store and load various AST
+## and related data.
+##
+## NB: this is incredibly low level and if you're interested in how the
+##     compiler works and less a storage format, you're probably looking for
+##     the `ic` or `packed_ast` modules to understand the logical format.
+
+from std/typetraits import supportsCopyMem
+
+when defined(nimPreviewSlimSystem):
+  import std/[syncio, assertions]
+
+## Overview
+## ========
+## `RodFile` represents a Rod File (versioned binary format), and the
+## associated data for common interactions such as IO and error tracking
+## (`RodFileError`). The file format broken up into sections (`RodSection`)
+## and preceded by a header (see: `cookie`). The precise layout, section
+## ordering and data following the section are determined by the user. See
+## `ic.loadRodFile`.
+##
+## A basic but "wrong" example of the lifecycle:
+## ---------------------------------------------
+## 1. `create` or `open`        - create a new one or open an existing
+## 2. `storeHeader`             - header info
+## 3. `storePrim` or `storeSeq` - save your stuff
+## 4. `close`                   - and we're done
+##
+## Now read the bits below to understand what's missing.
+##
+## ### Issues with the Example
+## Missing Sections:
+## This is a low level API, so headers and sections need to be stored and
+## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &
+## `loadSection`, respectively.
+##
+## No Error Handling:
+## The API is centered around IO and prone to error, each operation checks or
+## sets the `RodFile.err` field. A user of this API needs to handle these
+## appropriately.
+##
+## API Notes
+## =========
+##
+## Valid inputs for Rod files
+## --------------------------
+## ASTs, hopes, dreams, and anything as long as it and any children it may have
+## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are:
+## * string
+## * objects & tuples (fields are recursed)
+## * sequences AKA `seq[T]`
+##
+## Note on error handling style
+## ----------------------------
+## A flag based approach is used where operations no-op in case of a
+## preexisting error and set the flag if they encounter one.
+##
+## Misc
+## ----
+## * 'Prim' is short for 'primitive', as in a non-sequence type
 
 type
   RodSection* = enum
@@ -16,16 +75,15 @@ type
     stringsSection
     checkSumsSection
     depsSection
-    integersSection
-    floatsSection
+    numbersSection
     exportsSection
+    hiddenSection
     reexportsSection
     compilerProcsSection
     trmacrosSection
     convertersSection
     methodsSection
     pureEnumsSection
-    macroUsagesSection
     toReplaySection
     topLevelSection
     bodiesSection
@@ -34,9 +92,16 @@ type
     typeInstCacheSection
     procInstCacheSection
     attachedOpsSection
-    methodsPerTypeSection
+    methodsPerGenericTypeSection
     enumToStringProcsSection
+    methodsPerTypeSection
+    dispatchersSection
+    typeInfoSection  # required by the backend
+    backendFlagsSection
     aliveSymsSection # beware, this is stored in a `.alivesyms` file.
+    sideChannelSection
+    namespaceSection
+    symnamesSection
 
   RodFileError* = enum
     ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch,
@@ -49,8 +114,8 @@ type
                        # better than exceptions.
 
 const
-  RodVersion = 1
-  cookie = [byte(0), byte('R'), byte('O'), byte('D'),
+  RodVersion = 2
+  defaultCookie = [byte(0), byte('R'), byte('O'), byte('D'),
             byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)]
 
 proc setError(f: var RodFile; err: RodFileError) {.inline.} =
@@ -58,6 +123,8 @@ proc setError(f: var RodFile; err: RodFileError) {.inline.} =
   #raise newException(IOError, "IO error")
 
 proc storePrim*(f: var RodFile; s: string) =
+  ## Stores a string.
+  ## The len is prefixed to allow for later retreival.
   if f.err != ok: return
   if s.len >= high(int32):
     setError f, tooBig
@@ -71,6 +138,9 @@ proc storePrim*(f: var RodFile; s: string) =
         setError f, ioFailure
 
 proc storePrim*[T](f: var RodFile; x: T) =
+  ## Stores a non-sequence/string `T`.
+  ## If `T` doesn't support `copyMem` and is an object or tuple then the fields
+  ## are written -- the user from context will need to know which `T` to load.
   if f.err != ok: return
   when supportsCopyMem(T):
     if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
@@ -88,6 +158,7 @@ proc storePrim*[T](f: var RodFile; x: T) =
     {.error: "unsupported type for 'storePrim'".}
 
 proc storeSeq*[T](f: var RodFile; s: seq[T]) =
+  ## Stores a sequence of `T`s, with the len as a prefix for later retrieval.
   if f.err != ok: return
   if s.len >= high(int32):
     setError f, tooBig
@@ -100,6 +171,7 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) =
       storePrim(f, s[i])
 
 proc loadPrim*(f: var RodFile; s: var string) =
+  ## Read a string, the length was stored as a prefix
   if f.err != ok: return
   var lenPrefix = int32(0)
   if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
@@ -111,6 +183,7 @@ proc loadPrim*(f: var RodFile; s: var string) =
         setError f, ioFailure
 
 proc loadPrim*[T](f: var RodFile; x: var T) =
+  ## Load a non-sequence/string `T`.
   if f.err != ok: return
   when supportsCopyMem(T):
     if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
@@ -128,6 +201,7 @@ proc loadPrim*[T](f: var RodFile; x: var T) =
     {.error: "unsupported type for 'loadPrim'".}
 
 proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
+  ## `T` must be compatible with `copyMem`, see `loadPrim`
   if f.err != ok: return
   var lenPrefix = int32(0)
   if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
@@ -137,38 +211,46 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
     for i in 0.. 0:
+      (result[0][^1], result[1]) = splitPragmas(c, result[0][^1])
+
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
+  let (n, kws) = splitPragmas(c, n)
+  if kws.len > 0:
+    globalError(c.config, n.info, "unexpected pragma")
+
   let ident = lookups.considerQuotedIdent(c, n)
   let s = someSym(c.graph, fromMod, ident)
   if s == nil:
@@ -119,7 +144,7 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
     # for an enumeration we have to add all identifiers
     if multiImport:
       # for a overloadable syms add all overloaded routines
-      var it: ModuleIter
+      var it: ModuleIter = default(ModuleIter)
       var e = initModuleIter(it, c.graph, fromMod, s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
@@ -163,19 +188,24 @@ proc addImport(c: PContext; im: sink ImportedModule) =
   c.imports.add im
 
 template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} =
-  for it in c.graph.ifaces[fromMod.position].converters:
+  for it in mitems c.graph.ifaces[fromMod.position].converters:
     if filter:
-      addConverter(c, it)
-  for it in c.graph.ifaces[fromMod.position].patterns:
+      loadPackedSym(c.graph, it)
+      if sfExported in it.sym.flags:
+        addConverter(c, it)
+  for it in mitems c.graph.ifaces[fromMod.position].patterns:
     if filter:
-      addPattern(c, it)
-  for it in c.graph.ifaces[fromMod.position].pureEnums:
+      loadPackedSym(c.graph, it)
+      if sfExported in it.sym.flags:
+        addPattern(c, it)
+  for it in mitems c.graph.ifaces[fromMod.position].pureEnums:
     if filter:
+      loadPackedSym(c.graph, it)
       importPureEnumFields(c, it.sym, it.sym.typ)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
   c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
-  addUnnamedIt(c, fromMod, it.sym.id notin exceptSet)
+  addUnnamedIt(c, fromMod, it.sym.name.id notin exceptSet)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   c.addImport ImportedModule(m: fromMod, mode: importAll)
@@ -201,18 +231,47 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im
     for i in 0..n.safeLen-1:
       importForwarded(c, n[i], exceptSet, fromMod, importSet)
 
-proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
+proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym =
   result = realModule
-  c.unusedImports.add((realModule, n.info))
+  template createModuleAliasImpl(ident): untyped =
+    createModuleAlias(realModule, c.idgen, ident, n.info, c.config.options)
   if n.kind != nkImportAs: discard
   elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
   elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, nextSymId c.idgen, n[1].ident, realModule.info,
-                               c.config.options)
+    result = createModuleAliasImpl(n[1].ident)
+  if result == realModule:
+    # avoids modifying `realModule`, see D20201209T194412 for `import {.all.}`
+    result = createModuleAliasImpl(realModule.name)
+  if importHidden:
+    result.options.incl optImportHidden
+  c.unusedImports.add((result, n.info))
+  c.importModuleMap[result.id] = realModule.id
+  c.importModuleLookup.mgetOrPut(result.name.id, @[]).addUnique realModule.id
 
-proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
+proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] =
+  result = (nil, false)
+  var ret = default(typeof(result))
+  proc processPragma(n2: PNode): PNode =
+    let (result2, kws) = splitPragmas(c, n2)
+    result = result2
+    for ai in kws:
+      case ai
+      of wImportHidden: ret.importHidden = true
+      else: globalError(c.config, n.info, "invalid pragma, expected: " & ${wImportHidden})
+
+  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
+    ret.node = newNodeI(nkImportAs, n.info)
+    ret.node.add n[1].processPragma
+    ret.node.add n[2]
+  else:
+    ret.node = n.processPragma
+  return ret
+
+proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
+  let transf = transformImportAs(c, n)
+  n = transf.node
   let f = checkModuleName(c.config, n)
   if f != InvalidFileIdx:
     addImportFileDep(c, f)
@@ -228,64 +287,80 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
                 toFullPath(c.config, c.graph.importStack[i+1])
       c.recursiveDep = err
 
+    var realModule: PSym
     discard pushOptionEntry(c)
-    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
+    realModule = c.graph.importModuleCallback(c.graph, c.module, f)
+    result = importModuleAs(c, n, realModule, transf.importHidden)
     popOptionEntry(c)
 
     #echo "set back to ", L
     c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
-    # test: modules/import_in_config)
-    when true:
-      if result.info.fileIndex == c.module.info.fileIndex and
-          result.info.fileIndex == n.info.fileIndex:
-        localError(c.config, n.info, "A module cannot import itself")
-    if sfDeprecated in result.flags:
-      if result.constraint != nil:
-        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
+    # test: modules/import_in_config) # xxx is that still true?
+    if realModule == c.module:
+      localError(c.config, n.info, "module '$1' cannot import itself" % realModule.name.s)
+    if sfDeprecated in realModule.flags:
+      var prefix = ""
+      if realModule.constraint != nil: prefix = realModule.constraint.strVal & "; "
+      message(c.config, n.info, warnDeprecated, prefix & realModule.name.s & " is deprecated")
+    let moduleName = getModuleName(c.config, n)
+    if belongsToStdlib(c.graph, result) and not startsWith(moduleName, stdPrefix) and
+        not startsWith(moduleName, "system/") and not startsWith(moduleName, "packages/"):
+      message(c.config, n.info, warnStdPrefix, realModule.name.s)
+
+    proc suggestMod(n: PNode; s: PSym) =
+      if n.kind == nkImportAs:
+        suggestMod(n[0], realModule)
+      elif n.kind == nkInfix:
+        suggestMod(n[2], s)
       else:
-        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
-    suggestSym(c.graph, n.info, result, c.graph.usageSym, false)
+        suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
+    suggestMod(n, result)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
-
-proc transformImportAs(c: PContext; n: PNode): PNode =
-  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
-    result = newNodeI(nkImportAs, n.info)
-    result.add n[1]
-    result.add n[2]
   else:
-    result = n
+    result = nil
+
+proc afterImport(c: PContext, m: PSym) =
+  if isCachedModule(c.graph, m): return
+  # fixes bug #17510, for re-exported symbols
+  let realModuleId = c.importModuleMap[m.id]
+  for s in allSyms(c.graph, m):
+    if s.owner.id != realModuleId:
+      c.exportIndirections.incl((m.id, s.id))
 
 proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
-  let it = transformImportAs(c, it)
+  var it = it
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
     importAllSymbols(c, m)
     #importForwarded(c, m.ast, emptySet, m)
+    afterImport(c, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   for i in 0.. 0:
+  var shouldDebug = false
+
+template dbg(body) =
+  when toDebug.len > 0:
+    if shouldDebug:
+      body
 
 proc hasDestructor(c: Con; t: PType): bool {.inline.} =
   result = ast.hasDestructor(t)
   when toDebug.len > 0:
     # for more effective debugging
-    if not result and c.graph.config.selectedGC in {gcArc, gcOrc}:
+    if not result and c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
       assert(not containsGarbageCollectedRef(t))
 
-template dbg(body) =
-  when toDebug.len > 0:
-    if c.owner.name.s == toDebug or toDebug == "always":
-      body
-
 proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode =
-  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextSymId c.idgen, c.owner, info)
+  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), c.idgen, c.owner, info)
   sym.typ = typ
   s.vars.add(sym)
   result = newSymNode(sym)
 
-proc nestedScope(parent: var Scope): Scope =
-  Scope(vars: @[], wasMoved: @[], final: @[], needsTry: false, parent: addr(parent))
+proc nestedScope(parent: var Scope; body: PNode): Scope =
+  Scope(vars: @[], locals: @[], wasMoved: @[], final: @[], body: body, needsTry: false, parent: addr(parent))
 
-proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode
-proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope; isDecl = false): PNode
+proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSingleUsedTemp}; inReturn = false): PNode
 
-import sets, hashes, tables
+type
+  MoveOrCopyFlag = enum
+    IsDecl, IsExplicitSink, IsReturn
 
-proc hash(n: PNode): Hash = hash(cast[pointer](n))
+proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope; flags: set[MoveOrCopyFlag] = {}): PNode
 
-type AliasCache = Table[(PNode, PNode), AliasKind]
-proc aliasesCached(cache: var AliasCache, obj, field: PNode): AliasKind =
-  let key = (obj, field)
-  if not cache.hasKey(key):
-    cache[key] = aliases(obj, field)
-  cache[key]
+when false:
+  var
+    perfCounters: array[InstrKind, int]
 
-proc collectLastReads(cfg: ControlFlowGraph; cache: var AliasCache, lastReads, potLastReads: var IntSet; pc: var int, until: int) =
-  template aliasesCached(obj, field: PNode): untyped =
-    aliasesCached(cache, obj, field)
-  while pc < until:
-    case cfg[pc].kind
-    of def:
-      let potLastReadsCopy = potLastReads
-      for r in potLastReadsCopy:
-        if cfg[pc].n.aliasesCached(cfg[r].n) == yes:
-          # the path leads to a redefinition of 's' --> sink 's'.
-          lastReads.incl r
-          potLastReads.excl r
-        elif cfg[r].n.aliasesCached(cfg[pc].n) != no:
-          # only partially writes to 's' --> can't sink 's', so this def reads 's'
-          # or maybe writes to 's' --> can't sink 's'
-          cfg[r].n.comment = '\n' & $pc
-          potLastReads.excl r
+  proc showCounters*() =
+    for i in low(InstrKind)..high(InstrKind):
+      echo "INSTR ", i, " ", perfCounters[i]
 
-      inc pc
-    of use:
-      let potLastReadsCopy = potLastReads
-      for r in potLastReadsCopy:
-        if cfg[pc].n.aliasesCached(cfg[r].n) != no or cfg[r].n.aliasesCached(cfg[pc].n) != no:
-          cfg[r].n.comment = '\n' & $pc
-          potLastReads.excl r
+proc isLastReadImpl(n: PNode; c: var Con; scope: var Scope): bool =
+  let root = parampatterns.exprRoot(n, allowCalls=false)
+  if root == nil: return false
 
-      potLastReads.incl pc
+  var s = addr(scope)
+  while s != nil:
+    if s.locals.contains(root): break
+    s = s.parent
 
-      inc pc
-    of goto:
-      pc += cfg[pc].dest
-    of fork:
-      var variantA = pc + 1
-      var variantB = pc + cfg[pc].dest
-      var potLastReadsA, potLastReadsB = potLastReads
-      var lastReadsA, lastReadsB: IntSet
-      while variantA != variantB and max(variantA, variantB) < cfg.len and min(variantA, variantB) < until:
-        if variantA < variantB:
-          collectLastReads(cfg, cache, lastReadsA, potLastReadsA, variantA, min(variantB, until))
-        else:
-          collectLastReads(cfg, cache, lastReadsB, potLastReadsB, variantB, min(variantA, until))
+  c.g = constructCfg(c.owner, if s != nil: s.body else: c.body, root)
+  dbg:
+    echo "\n### ", c.owner.name.s, ":\nCFG:"
+    echoCfg(c.g)
+    #echo c.body
 
-      # Add those last reads that were turned into last reads on both branches
-      lastReads.incl lastReadsA * lastReadsB
-      # Add those last reads that were turned into last reads on only one branch,
-      # but where the read operation itself also belongs to only that branch
-      lastReads.incl (lastReadsA + lastReadsB) - potLastReads
+  var j = 0
+  while j < c.g.len:
+    if c.g[j].kind == use and c.g[j].n == n: break
+    inc j
+  c.otherUsage = unknownLineInfo
+  if j < c.g.len:
+    var pcs = @[j+1]
+    var marked = initIntSet()
+    result = true
+    while pcs.len > 0:
+      var pc = pcs.pop()
+      if not marked.contains(pc):
+        let oldPc = pc
+        while pc < c.g.len:
+          dbg:
+            echo "EXEC ", c.g[pc].kind, " ", pc, " ", n
+          when false:
+            inc perfCounters[c.g[pc].kind]
+          case c.g[pc].kind
+          of loop:
+            let back = pc + c.g[pc].dest
+            if not marked.containsOrIncl(back):
+              pc = back
+            else:
+              break
+          of goto:
+            pc = pc + c.g[pc].dest
+          of fork:
+            if not marked.contains(pc+1):
+              pcs.add pc + 1
+            pc = pc + c.g[pc].dest
+          of use:
+            if c.g[pc].n.aliases(n) != no or n.aliases(c.g[pc].n) != no:
+              c.otherUsage = c.g[pc].n.info
+              return false
+            inc pc
+          of def:
+            if c.g[pc].n.aliases(n) == yes:
+              # the path leads to a redefinition of 's' --> sink 's'.
+              break
+            elif n.aliases(c.g[pc].n) != no:
+              # only partially writes to 's' --> can't sink 's', so this def reads 's'
+              # or maybe writes to 's' --> can't sink 's'
+              c.otherUsage = c.g[pc].n.info
+              return false
+            inc pc
+        marked.incl oldPc
+  else:
+    result = false
 
-      let oldPotLastReads = potLastReads
-      potLastReads = initIntSet()
+proc isLastRead(n: PNode; c: var Con; s: var Scope): bool =
+  # bug #23354; an object type could have a non-trival assignements when it is passed to a sink parameter
+  if not hasDestructor(c, n.typ) and (n.typ.kind != tyObject or isTrival(getAttachedOp(c.graph, n.typ, attachedAsgn))): return true
 
-      potLastReads.incl potLastReadsA + potLastReadsB
-
-      # Remove potential last reads that were invalidated in a branch,
-      # but don't remove those which were turned into last reads on that branch
-      potLastReads.excl ((oldPotLastReads - potLastReadsA) - lastReadsA)
-      potLastReads.excl ((oldPotLastReads - potLastReadsB) - lastReadsB)
-
-      pc = min(variantA, variantB)
-
-proc collectFirstWrites(cfg: ControlFlowGraph; alreadySeen: var HashSet[PNode]; pc: var int, until: int) =
-  while pc < until:
-    case cfg[pc].kind
-    of def:
-      var alreadySeenThisNode = false
-      for s in alreadySeen:
-        if cfg[pc].n.aliases(s) != no or s.aliases(cfg[pc].n) != no:
-          alreadySeenThisNode = true; break
-      if alreadySeenThisNode: cfg[pc].n.flags.excl nfFirstWrite
-      else: cfg[pc].n.flags.incl nfFirstWrite
-
-      alreadySeen.incl cfg[pc].n
-
-      inc pc
-    of use:
-      alreadySeen.incl cfg[pc].n
-
-      inc pc
-    of goto:
-      pc += cfg[pc].dest
-    of fork:
-      var variantA = pc + 1
-      var variantB = pc + cfg[pc].dest
-      var alreadySeenA, alreadySeenB = alreadySeen
-      while variantA != variantB and max(variantA, variantB) < cfg.len and min(variantA, variantB) < until:
-        if variantA < variantB:
-          collectFirstWrites(cfg, alreadySeenA, variantA, min(variantB, until))
-        else:
-          collectFirstWrites(cfg, alreadySeenB, variantB, min(variantA, until))
-
-      alreadySeen.incl alreadySeenA + alreadySeenB
-
-      pc = min(variantA, variantB)
-
-proc isLastRead(n: PNode; c: var Con): bool =
-  let m = dfa.skipConvDfa(n)
-  (m.kind == nkSym and sfSingleUsedTemp in m.sym.flags) or nfLastRead in m.flags
+  let m = skipConvDfa(n)
+  result = (m.kind == nkSym and sfSingleUsedTemp in m.sym.flags) or
+      isLastReadImpl(n, c, s)
 
 proc isFirstWrite(n: PNode; c: var Con): bool =
-  let m = dfa.skipConvDfa(n)
-  nfFirstWrite in m.flags
-
-proc initialized(code: ControlFlowGraph; pc: int,
-                 init, uninit: var IntSet; until: int): int =
-  ## Computes the set of definitely initialized variables across all code paths
-  ## as an IntSet of IDs.
-  var pc = pc
-  while pc < code.len:
-    case code[pc].kind
-    of goto:
-      pc += code[pc].dest
-    of fork:
-      var initA = initIntSet()
-      var initB = initIntSet()
-      var variantA = pc + 1
-      var variantB = pc + code[pc].dest
-      while variantA != variantB:
-        if max(variantA, variantB) > until:
-          break
-        if variantA < variantB:
-          variantA = initialized(code, variantA, initA, uninit, min(variantB, until))
-        else:
-          variantB = initialized(code, variantB, initB, uninit, min(variantA, until))
-      pc = min(variantA, variantB)
-      # we add vars if they are in both branches:
-      for v in initA:
-        if v in initB:
-          init.incl v
-    of use:
-      let v = code[pc].n.sym
-      if v.kind != skParam and v.id notin init:
-        # attempt to read an uninit'ed variable
-        uninit.incl v.id
-      inc pc
-    of def:
-      let v = code[pc].n.sym
-      init.incl v.id
-      inc pc
-  return pc
+  let m = skipConvDfa(n)
+  result = nfFirstWrite in m.flags
 
 proc isCursor(n: PNode): bool =
   case n.kind
@@ -236,41 +188,52 @@ proc isCursor(n: PNode): bool =
 template isUnpackedTuple(n: PNode): bool =
   ## we move out all elements of unpacked tuples,
   ## hence unpacked tuples themselves don't need to be destroyed
-  (n.kind == nkSym and n.sym.kind == skTemp and n.sym.typ.kind == tyTuple)
+  ## except it's already a cursor
+  (n.kind == nkSym and n.sym.kind == skTemp and
+   n.sym.typ.kind == tyTuple and sfCursor notin n.sym.flags)
 
-from strutils import parseInt
-
-proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
+proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string; inferredFromCopy = false) =
   var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
-  if (opname == "=" or opname == "=copy") and ri != nil:
+  if inferredFromCopy:
+    m.add ", which is inferred from unavailable '=copy'"
+
+  if (opname == "=" or opname == "=copy" or opname == "=dup") and ri != nil:
     m.add "; requires a copy because it's not the last read of '"
     m.add renderTree(ri)
     m.add '\''
-    if ri.comment.startsWith('\n'):
+    if c.otherUsage != unknownLineInfo:
+       # ri.comment.startsWith('\n'):
       m.add "; another read is done here: "
-      m.add c.graph.config $ c.g[parseInt(ri.comment[1..^1])].n.info
+      m.add c.graph.config $ c.otherUsage
+      #m.add c.graph.config $ c.g[parseInt(ri.comment[1..^1])].n.info
     elif ri.kind == nkSym and ri.sym.kind == skParam and not isSinkType(ri.sym.typ):
       m.add "; try to make "
       m.add renderTree(ri)
       m.add " a 'sink' parameter"
   m.add "; routine: "
   m.add c.owner.name.s
+  #m.add "\n\n"
+  #m.add renderTree(c.body, {renderIds})
   localError(c.graph.config, ri.info, errGenerated, m)
 
 proc makePtrType(c: var Con, baseType: PType): PType =
-  result = newType(tyPtr, nextTypeId c.idgen, c.owner)
+  result = newType(tyPtr, c.idgen, c.owner)
   addSonSkipIntLit(result, baseType, c.idgen)
 
 proc genOp(c: var Con; op: PSym; dest: PNode): PNode =
-  let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
-  addrExp.add(dest)
+  var addrExp: PNode
+  if op.typ != nil and op.typ.signatureLen > 1 and op.typ.firstParamType.kind != tyVar:
+    addrExp = dest
+  else:
+    addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
+    addrExp.add(dest)
   result = newTree(nkCall, newSymNode(op), addrExp)
 
 proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
   var op = getAttachedOp(c.graph, t, kind)
   if op == nil or op.ast.isGenericRoutine:
     # give up and find the canonical type instead:
-    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
+    let h = sighashes.hashType(t, c.graph.config, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.graph.canonTypes.getOrDefault(h)
     if canon != nil:
       op = getAttachedOp(c.graph, canon, kind)
@@ -301,10 +264,20 @@ proc canBeMoved(c: Con; t: PType): bool {.inline.} =
 proc isNoInit(dest: PNode): bool {.inline.} =
   result = dest.kind == nkSym and sfNoInit in dest.sym.flags
 
-proc genSink(c: var Con; dest, ri: PNode, isDecl = false): PNode =
-  if (c.inLoopCond == 0 and (isUnpackedTuple(dest) or isDecl or
+proc deepAliases(dest, ri: PNode): bool =
+  case ri.kind
+  of nkCallKinds, nkStmtListExpr, nkBracket, nkTupleConstr, nkObjConstr,
+     nkCast, nkConv, nkObjUpConv, nkObjDownConv:
+    for r in ri:
+      if deepAliases(dest, r): return true
+    return false
+  else:
+    return aliases(dest, ri) != no
+
+proc genSink(c: var Con; s: var Scope; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNode =
+  if (c.inLoopCond == 0 and (isUnpackedTuple(dest) or IsDecl in flags or
       (isAnalysableFieldAccess(dest, c.owner) and isFirstWrite(dest, c)))) or
-      isNoInit(dest):
+      isNoInit(dest) or IsReturn in flags:
     # optimize sink call into a bitwise memcopy
     result = newTree(nkFastAsgn, dest, ri)
   else:
@@ -315,12 +288,19 @@ proc genSink(c: var Con; dest, ri: PNode, isDecl = false): PNode =
     else:
       # the default is to use combination of `=destroy(dest)` and
       # and copyMem(dest, source). This is efficient.
-      result = newTree(nkStmtList, c.genDestroy(dest), newTree(nkFastAsgn, dest, ri))
+      if deepAliases(dest, ri):
+        # consider: x = x + y, it is wrong to destroy the destination first!
+        # tmp to support self assignments
+        let tmp = c.getTemp(s, dest.typ, dest.info)
+        result = newTree(nkStmtList, newTree(nkFastAsgn, tmp, dest), newTree(nkFastAsgn, dest, ri),
+                         c.genDestroy(tmp))
+      else:
+        result = newTree(nkStmtList, c.genDestroy(dest), newTree(nkFastAsgn, dest, ri))
 
 proc isCriticalLink(dest: PNode): bool {.inline.} =
   #[
   Lins's idea that only "critical" links can introduce a cycle. This is
-  critical for the performance gurantees that we strive for: If you
+  critical for the performance guarantees that we strive for: If you
   traverse a data structure, no tracing will be performed at all.
   ORC is about this promise: The GC only touches the memory that the
   mutator touches too.
@@ -339,14 +319,14 @@ proc isCriticalLink(dest: PNode): bool {.inline.} =
 
 proc finishCopy(c: var Con; result, dest: PNode; isFromSink: bool) =
   if c.graph.config.selectedGC == gcOrc:
-    let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
-    if cyclicType(t):
+    let t = dest.typ.skipTypes(tyUserTypeClasses + {tyGenericInst, tyAlias, tySink, tyDistinct})
+    if cyclicType(c.graph, t):
       result.add boolLit(c.graph, result.info, isFromSink or isCriticalLink(dest))
 
 proc genMarkCyclic(c: var Con; result, dest: PNode) =
   if c.graph.config.selectedGC == gcOrc:
     let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
-    if cyclicType(t):
+    if cyclicType(c.graph, t):
       if t.kind == tyRef:
         result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, dest)
       else:
@@ -354,16 +334,25 @@ proc genMarkCyclic(c: var Con; result, dest: PNode) =
         xenv.typ = getSysType(c.graph, dest.info, tyPointer)
         result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, xenv)
 
-proc genCopyNoCheck(c: var Con; dest, ri: PNode): PNode =
+proc genCopyNoCheck(c: var Con; dest, ri: PNode; a: TTypeAttachedOp): PNode =
   let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-  result = c.genOp(t, attachedAsgn, dest, ri)
+  result = c.genOp(t, a, dest, ri)
+  assert ri.typ != nil
 
-proc genCopy(c: var Con; dest, ri: PNode): PNode =
+proc genCopy(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag]): PNode =
+  if c.inEnsureMove > 0:
+    localError(c.graph.config, ri.info, errFailedMove, "cannot move '" & $ri &
+                                                      "', which introduces an implicit copy")
   let t = dest.typ
   if tfHasOwned in t.flags and ri.kind != nkNilLit:
     # try to improve the error message here:
-    c.checkForErrorPragma(t, ri, "=copy")
-  result = c.genCopyNoCheck(dest, ri)
+    if IsExplicitSink in flags:
+      c.checkForErrorPragma(t, ri, "=sink")
+    else:
+      c.checkForErrorPragma(t, ri, "=copy")
+  let a = if IsExplicitSink in flags: attachedSink else: attachedAsgn
+  result = c.genCopyNoCheck(dest, ri, a)
+  assert ri.typ != nil
 
 proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode =
   # discriminator is ordinal value that doesn't need sink destroy
@@ -382,30 +371,38 @@ proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode =
 
   if hasDestructor(c, objType):
     if getAttachedOp(c.graph, objType, attachedDestructor) != nil and
-        sfOverriden in getAttachedOp(c.graph, objType, attachedDestructor).flags:
+        sfOverridden in getAttachedOp(c.graph, objType, attachedDestructor).flags:
       localError(c.graph.config, n.info, errGenerated, """Assignment to discriminant for objects with user defined destructor is not supported, object must have default destructor.
 It is best to factor out piece of object that needs custom destructor into separate object or not use discriminator assignment""")
       result.add newTree(nkFastAsgn, le, tmp)
       return
 
     # generate: if le != tmp: `=destroy`(le)
-    let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info, c.idgen)
-    let cond = newNodeIT(nkInfix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
-    cond.add newSymNode(getMagicEqSymForType(c.graph, le.typ, n.info))
-    cond.add le
-    cond.add tmp
-    let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
-    notExpr.add newSymNode(createMagic(c.graph, c.idgen, "not", mNot))
-    notExpr.add cond
-    result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, c.genOp(branchDestructor, le)))
+    if c.inUncheckedAssignSection != 0:
+      let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info, c.idgen)
+      let cond = newNodeIT(nkInfix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+      cond.add newSymNode(getMagicEqSymForType(c.graph, le.typ, n.info))
+      cond.add le
+      cond.add tmp
+      let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+      notExpr.add newSymNode(createMagic(c.graph, c.idgen, "not", mNot))
+      notExpr.add cond
+      result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, c.genOp(branchDestructor, le)))
   result.add newTree(nkFastAsgn, le, tmp)
 
 proc genWasMoved(c: var Con, n: PNode): PNode =
-  result = newNodeI(nkCall, n.info)
-  result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved)))
-  result.add copyTree(n) #mWasMoved does not take the address
-  #if n.kind != nkSym:
-  #  message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")")
+  let typ = n.typ.skipTypes({tyGenericInst, tyAlias, tySink})
+  let op = getAttachedOp(c.graph, n.typ, attachedWasMoved)
+  if op != nil:
+    if sfError in op.flags:
+      c.checkForErrorPragma(n.typ, n, "=wasMoved")
+    result = genOp(c, op, n)
+  else:
+    result = newNodeI(nkCall, n.info)
+    result.add(newSymNode(createMagic(c.graph, c.idgen, "`=wasMoved`", mWasMoved)))
+    result.add copyTree(n) #mWasMoved does not take the address
+    #if n.kind != nkSym:
+    #  message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")")
 
 proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
   result = newNodeI(nkCall, info)
@@ -414,13 +411,16 @@ proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
 
 proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
   # generate: (let tmp = v; reset(v); tmp)
-  if not hasDestructor(c, n.typ):
-    assert n.kind != nkSym or not hasDestructor(c, n.sym.typ)
+  if (not hasDestructor(c, n.typ)) and c.inEnsureMove == 0:
+    assert n.kind != nkSym or not hasDestructor(c, n.sym.typ) or
+          (n.typ.kind == tyPtr and n.sym.typ.kind == tyRef)
+      # bug #23505; transformed by `transf`: addr (deref ref) -> ptr
+      # we know it's really a pointer; so here we assign it directly
     result = copyTree(n)
   else:
     result = newNodeIT(nkStmtListExpr, n.info, n.typ)
 
-    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), nextSymId c.idgen, c.owner, n.info)
+    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), c.idgen, c.owner, n.info)
     temp.typ = n.typ
     var v = newNodeI(nkLetSection, n.info)
     let tempAsNode = newSymNode(temp)
@@ -433,7 +433,8 @@ proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
 
     result.add v
     let nn = skipConv(n)
-    c.genMarkCyclic(result, nn)
+    if hasDestructor(c, n.typ):
+      c.genMarkCyclic(result, nn)
     let wasMovedCall = c.genWasMoved(nn)
     result.add wasMovedCall
     result.add tempAsNode
@@ -441,24 +442,50 @@ proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
 proc isCapturedVar(n: PNode): bool =
   let root = getRoot(n)
   if root != nil: result = root.name.s[0] == ':'
+  else: result = false
 
 proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-  let tmp = c.getTemp(s, n.typ, n.info)
-  if hasDestructor(c, n.typ):
-    result.add c.genWasMoved(tmp)
-    var m = c.genCopy(tmp, n)
-    m.add p(n, c, s, normal)
-    c.finishCopy(m, n, isFromSink = true)
-    result.add m
-    if isLValue(n) and not isCapturedVar(n) and n.typ.skipTypes(abstractInst).kind != tyRef and c.inSpawn == 0:
+  let nTyp = n.typ.skipTypes(tyUserTypeClasses)
+  let tmp = c.getTemp(s, nTyp, n.info)
+  if hasDestructor(c, nTyp):
+    let typ = nTyp.skipTypes({tyGenericInst, tyAlias, tySink})
+    let op = getAttachedOp(c.graph, typ, attachedDup)
+    if op != nil and tfHasOwned notin typ.flags:
+      if sfError in op.flags:
+        c.checkForErrorPragma(nTyp, n, "=dup")
+      else:
+        let copyOp = getAttachedOp(c.graph, typ, attachedAsgn)
+        if copyOp != nil and sfError in copyOp.flags and
+           sfOverridden notin op.flags:
+          c.checkForErrorPragma(nTyp, n, "=dup", inferredFromCopy = true)
+
+      let src = p(n, c, s, normal)
+      var newCall = newTreeIT(nkCall, src.info, src.typ,
+            newSymNode(op),
+            src)
+      c.finishCopy(newCall, n, isFromSink = true)
+      result.add newTreeI(nkFastAsgn,
+          src.info, tmp,
+          newCall
+      )
+    else:
+      result.add c.genWasMoved(tmp)
+      var m = c.genCopy(tmp, n, {})
+      m.add p(n, c, s, normal)
+      c.finishCopy(m, n, isFromSink = true)
+      result.add m
+    if isLValue(n) and not isCapturedVar(n) and nTyp.skipTypes(abstractInst).kind != tyRef and c.inSpawn == 0:
       message(c.graph.config, n.info, hintPerformance,
         ("passing '$1' to a sink parameter introduces an implicit copy; " &
         "if possible, rearrange your program's control flow to prevent it") % $n)
+    if c.inEnsureMove > 0:
+      localError(c.graph.config, n.info, errFailedMove,
+        ("cannot move '$1', passing '$1' to a sink parameter introduces an implicit copy") % $n)
   else:
-    if c.graph.config.selectedGC in {gcArc, gcOrc}:
-      assert(not containsManagedMemory(n.typ))
-    if n.typ.skipTypes(abstractInst).kind in {tyOpenArray, tyVarargs}:
+    if c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
+      assert(not containsManagedMemory(nTyp))
+    if nTyp.skipTypes(abstractInst).kind in {tyOpenArray, tyVarargs}:
       localError(c.graph.config, n.info, "cannot create an implicit openArray copy to be passed to a sink parameter")
     result.add newTree(nkAsgn, tmp, p(n, c, s, normal))
   # Since we know somebody will take over the produced copy, there is
@@ -467,7 +494,7 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
 
 proc isDangerousSeq(t: PType): bool {.inline.} =
   let t = t.skipTypes(abstractInst)
-  result = t.kind == tySequence and tfHasOwned notin t[0].flags
+  result = t.kind == tySequence and tfHasOwned notin t.elementType.flags
 
 proc containsConstSeq(n: PNode): bool =
   if n.kind == nkBracket and n.len > 0 and n.typ != nil and isDangerousSeq(n.typ):
@@ -492,14 +519,14 @@ proc ensureDestruction(arg, orig: PNode; c: var Con; s: var Scope): PNode =
     # This was already done in the sink parameter handling logic.
     result = newNodeIT(nkStmtListExpr, arg.info, arg.typ)
     let tmp = c.getTemp(s, arg.typ, arg.info)
-    result.add c.genSink(tmp, arg, isDecl = true)
+    result.add c.genSink(s, tmp, arg, {IsDecl})
     result.add tmp
     s.final.add c.genDestroy(tmp)
   else:
     result = arg
 
 proc cycleCheck(n: PNode; c: var Con) =
-  if c.graph.config.selectedGC != gcArc: return
+  if c.graph.config.selectedGC notin {gcArc, gcAtomicArc}: return
   var value = n[1]
   if value.kind == nkClosure:
     value = value[1]
@@ -537,7 +564,7 @@ proc pVarTopLevel(v: PNode; c: var Con; s: var Scope; res: PNode) =
       res.add newTree(nkFastAsgn, v, genDefaultCall(v.typ, c, v.info))
   elif sfThread notin v.sym.flags and sfCursor notin v.sym.flags:
     # do not destroy thread vars for now at all for consistency.
-    if sfGlobal in v.sym.flags and s.parent == nil: #XXX: Rethink this logic (see tarcmisc.test2)
+    if {sfGlobal, sfPure} <= v.sym.flags or sfGlobal in v.sym.flags and s.parent == nil:
       c.graph.globalDestructors.add c.genDestroy(v)
     else:
       s.final.add c.genDestroy(v)
@@ -564,16 +591,19 @@ proc processScope(c: var Con; s: var Scope; ret: PNode): PNode =
 
   if s.parent != nil: s.parent[].needsTry = s.parent[].needsTry or s.needsTry
 
-template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: untyped): PNode =
+template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: untyped, tmpFlags: TSymFlags): PNode =
   assert not ret.typ.isEmptyType
-  var result = newNodeI(nkStmtListExpr, ret.info)
+  var result = newNodeIT(nkStmtListExpr, ret.info, ret.typ)
   # There is a possibility to do this check: s.wasMoved.len > 0 or s.final.len > 0
   # later and use it to eliminate the temporary when theres no need for it, but its
   # tricky because you would have to intercept moveOrCopy at a certain point
   let tmp = c.getTemp(s.parent[], ret.typ, ret.info)
-  tmp.sym.flags.incl sfSingleUsedTemp
-  let cpy = if hasDestructor(c, ret.typ):
-              moveOrCopy(tmp, ret, c, s, isDecl = true)
+  tmp.sym.flags = tmpFlags
+  let cpy = if hasDestructor(c, ret.typ) and
+                ret.typ.kind notin {tyOpenArray, tyVarargs}:
+                # bug #23247 we don't own the data, so it's harmful to destroy it
+              s.parent[].final.add c.genDestroy(tmp)
+              moveOrCopy(tmp, ret, c, s, {IsDecl})
             else:
               newTree(nkFastAsgn, tmp, p(ret, c, s, normal))
 
@@ -597,7 +627,8 @@ template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: unt
 
   result
 
-template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
+template handleNestedTempl(n, processCall: untyped, willProduceStmt = false,
+                           tmpFlags = {sfSingleUsedTemp}) =
   template maybeVoid(child, s): untyped =
     if isEmptyType(child.typ): p(child, c, s, normal)
     else: processCall(child, s)
@@ -621,11 +652,11 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
       var branch = shallowCopy(it)
       for j in 0 ..< it.len-1:
         branch[j] = copyTree(it[j])
-      var ofScope = nestedScope(s)
+      var ofScope = nestedScope(s, it.lastSon)
       branch[^1] = if it[^1].typ.isEmptyType or willProduceStmt:
                      processScope(c, ofScope, maybeVoid(it[^1], ofScope))
                    else:
-                     processScopeExpr(c, ofScope, it[^1], processCall)
+                     processScopeExpr(c, ofScope, it[^1], processCall, tmpFlags)
       result.add branch
 
   of nkWhileStmt:
@@ -634,7 +665,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
     result = copyNode(n)
     result.add p(n[0], c, s, normal)
     dec c.inLoopCond
-    var bodyScope = nestedScope(s)
+    var bodyScope = nestedScope(s, n[1])
     let bodyResult = p(n[1], c, bodyScope, normal)
     result.add processScope(c, bodyScope, bodyResult)
     dec c.inLoop
@@ -646,7 +677,7 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
     for i in 0.. 0:
-              result.add moveOrCopy(v, genDefaultCall(v.typ, c, v.info), c, s, isDecl = v.kind == nkSym)
+              let skipInit = v.kind == nkDotExpr and # Closure var
+                             sfNoInit in v[1].sym.flags
+              if not skipInit:
+                result.add moveOrCopy(v, genDefaultCall(v.typ, c, v.info), c, s, if v.kind == nkSym: {IsDecl} else: {})
         else: # keep the var but transform 'ri':
           var v = copyNode(n)
           var itCopy = copyNode(it)
           for j in 0.. 0 and isDangerousSeq(ri.typ):
-        result = c.genCopy(dest, ri)
+        inc c.inEnsureMove, isEnsureMove
+        result = c.genCopy(dest, ri, flags)
+        dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
       else:
-        result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+        result = c.genSink(s, dest, p(ri, c, s, consumed), flags)
     of nkObjConstr, nkTupleConstr, nkClosure, nkCharLit..nkNilLit:
-      result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+      result = c.genSink(s, dest, p(ri, c, s, consumed), flags)
     of nkSym:
-      if isSinkParam(ri.sym) and isLastRead(ri, c):
+      if isSinkParam(ri.sym) and isLastRead(ri, c, s):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
+        let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
-      elif ri.sym.kind != skParam and ri.sym.owner == c.owner and
-          isLastRead(ri, c) and canBeMoved(c, dest.typ) and not isCursor(ri):
+      elif ri.sym.kind != skParam and
+          isAnalysableFieldAccess(ri, c.owner) and
+          isLastRead(ri, c, s) and canBeMoved(c, dest.typ):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
+        let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       else:
-        result = c.genCopy(dest, ri)
+        inc c.inEnsureMove, isEnsureMove
+        result = c.genCopy(dest, ri, flags)
+        dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
-    of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv:
-      result = c.genSink(dest, p(ri, c, s, sinkArg), isDecl)
+    of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv, nkCast:
+      result = c.genSink(s, dest, p(ri, c, s, sinkArg), flags)
     of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
-      template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl)
+      template process(child, s): untyped = moveOrCopy(dest, child, c, s, flags)
       # We know the result will be a stmt so we use that fact to optimize
       handleNestedTempl(ri, process, willProduceStmt = true)
     of nkRaiseStmt:
       result = pRaiseStmt(ri, c, s)
     else:
-      if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c) and
+      if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c, s) and
           canBeMoved(c, dest.typ):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
+        let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       else:
-        result = c.genCopy(dest, ri)
+        inc c.inEnsureMove, isEnsureMove
+        result = c.genCopy(dest, ri, flags)
+        dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
 
-proc computeUninit(c: var Con) =
-  if not c.uninitComputed:
-    c.uninitComputed = true
-    c.uninit = initIntSet()
-    var init = initIntSet()
-    discard initialized(c.g, pc = 0, init, c.uninit, int.high)
+when false:
+  proc computeUninit(c: var Con) =
+    if not c.uninitComputed:
+      c.uninitComputed = true
+      c.uninit = initIntSet()
+      var init = initIntSet()
+      discard initialized(c.g, pc = 0, init, c.uninit, int.high)
 
-proc injectDefaultCalls(n: PNode, c: var Con) =
-  case n.kind
-  of nkVarSection, nkLetSection:
-    for it in n:
-      if it.kind == nkIdentDefs and it[^1].kind == nkEmpty:
-        computeUninit(c)
-        for j in 0.. 0:
+    shouldDebug = toDebug == owner.name.s or toDebug == "always"
   if sfGeneratedOp in owner.flags or (owner.kind == skIterator and isInlineIterator(owner.typ)):
     return n
-  var c = Con(owner: owner, graph: g, g: constructCfg(owner, n), idgen: idgen)
-  dbg:
-    echo "\n### ", owner.name.s, ":\nCFG:"
-    echoCfg(c.g)
-    echo n
+  var c = Con(owner: owner, graph: g, idgen: idgen, body: n, otherUsage: unknownLineInfo)
 
   if optCursorInference in g.config.options:
     computeCursors(owner, n, g)
 
-  block:
-    var cache = initTable[(PNode, PNode), AliasKind]()
-    var lastReads, potLastReads: IntSet
-    var pc = 0
-    collectLastReads(c.g, cache, lastReads, potLastReads, pc, c.g.len)
-    lastReads.incl potLastReads
-    var lastReadTable: Table[PNode, seq[int]]
-    for position, node in c.g:
-      if node.kind == use:
-        lastReadTable.mgetOrPut(node.n, @[]).add position
-    for node, positions in lastReadTable:
-      var allPositionsLastRead = true
-      for p in positions:
-        if p notin lastReads: allPositionsLastRead = false; break
-      if allPositionsLastRead:
-        node.flags.incl nfLastRead
-
-    var alreadySeen: HashSet[PNode]
-    pc = 0
-    collectFirstWrites(c.g, alreadySeen, pc, c.g.len)
-
-  var scope: Scope
+  var scope = Scope(body: n)
   let body = p(n, c, scope, normal)
 
   if owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter}:
diff --git a/compiler/installer.ini b/compiler/installer.ini
index 3f1630a921..54a35dbeea 100644
--- a/compiler/installer.ini
+++ b/compiler/installer.ini
@@ -6,13 +6,14 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64;loongarch64
   macosx: i386;amd64;powerpc64;arm64
   solaris: i386;amd64;sparc;sparc64
   freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el
-  netbsd: i386;amd64
+  netbsd: i386;amd64;arm64
   openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
+  crossos: amd64
   haiku: i386;amd64
   android: i386;arm;arm64
   nintendoswitch: arm64
@@ -64,11 +65,13 @@ Files: "compiler"
 Files: "doc"
 Files: "doc/html"
 Files: "tools"
-Files: "tools/nim-gdb.py"
+Files: "tools/debug/nim-gdb.py"
 Files: "nimpretty"
 Files: "testament"
 Files: "nimsuggest"
 Files: "nimsuggest/tests/*.nim"
+Files: "changelogs/*.md"
+Files: "ci/funs.sh"
 
 [Lib]
 Files: "lib"
@@ -76,6 +79,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/checksums"
 
 Files: "tests"
 
@@ -89,6 +93,7 @@ Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
 Files: "bin/testament.exe"
 Files: "bin/nim-gdb.bat"
+Files: "bin/atlas.exe"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -119,7 +124,7 @@ Files: "bin/nim"
 InstallScript: "yes"
 UninstallScript: "yes"
 Files: "bin/nim-gdb"
-Files: "bin/nim-gdb.bash"
+Files: "build_all.sh"
 
 
 [InnoSetup]
@@ -141,5 +146,5 @@ shortDesc: "The Nim Compiler"
 licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
-pkgName: "compiler"
-pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"
+pkgName: "nim"
+pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css;doc/nimdoc.cls"
diff --git a/compiler/int128.nim b/compiler/int128.nim
index 8d3cd71131..74e581cd51 100644
--- a/compiler/int128.nim
+++ b/compiler/int128.nim
@@ -5,6 +5,9 @@
 
 from std/math import trunc
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   Int128* = object
     udata: array[4, uint32]
@@ -30,39 +33,35 @@ template high*(t: typedesc[Int128]): Int128 = Max
 proc `$`*(a: Int128): string
 
 proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
-  when T is bool: result.sdata(0) = int32(arg)
-  elif T is SomeUnsignedInt:
-    when sizeof(arg) <= 4:
-      result.udata[0] = uint32(arg)
+  {.noSideEffect.}:
+    result = Zero
+    when T is bool: result.sdata(0) = int32(arg)
+    elif T is SomeUnsignedInt:
+      when sizeof(arg) <= 4:
+        result.udata[0] = uint32(arg)
+      else:
+        result.udata[0] = uint32(arg and T(0xffffffff))
+        result.udata[1] = uint32(arg shr 32)
+    elif sizeof(arg) <= 4:
+      result.sdata(0) = int32(arg)
+      if arg < 0: # sign extend
+        result.sdata(1) = -1
+        result.sdata(2) = -1
+        result.sdata(3) = -1
     else:
-      result.udata[0] = uint32(arg and T(0xffffffff))
-      result.udata[1] = uint32(arg shr 32)
-  elif sizeof(arg) <= 4:
-    result.sdata(0) = int32(arg)
-    if arg < 0: # sign extend
-      result.sdata(1) = -1
-      result.sdata(2) = -1
-      result.sdata(3) = -1
-  else:
-    let tmp = int64(arg)
-    result.udata[0] = uint32(tmp and 0xffffffff)
-    result.sdata(1) = int32(tmp shr 32)
-    if arg < 0: # sign extend
-      result.sdata(2) = -1
-      result.sdata(3) = -1
+      let tmp = int64(arg)
+      result.udata[0] = uint32(tmp and 0xffffffff)
+      result.sdata(1) = int32(tmp shr 32)
+      if arg < 0: # sign extend
+        result.sdata(2) = -1
+        result.sdata(3) = -1
 
 template isNegative(arg: Int128): bool =
   arg.sdata(3) < 0
 
-template isNegative(arg: int32): bool =
-  arg < 0
-
 proc bitconcat(a, b: uint32): uint64 =
   (uint64(a) shl 32) or uint64(b)
 
-proc bitsplit(a: uint64): (uint32, uint32) =
-  (cast[uint32](a shr 32), cast[uint32](a))
-
 proc toInt64*(arg: Int128): int64 =
   if isNegative(arg):
     assert(arg.sdata(3) == -1, "out of range")
@@ -173,6 +172,7 @@ proc addToHex*(result: var string; arg: Int128) =
     i -= 1
 
 proc toHex*(arg: Int128): string =
+  result = ""
   result.addToHex(arg)
 
 proc inc*(a: var Int128, y: uint32 = 1) =
@@ -208,37 +208,36 @@ proc `==`*(a, b: Int128): bool =
   if a.udata[3] != b.udata[3]: return false
   return true
 
-proc inplaceBitnot(a: var Int128) =
-  a.udata[0] = not a.udata[0]
-  a.udata[1] = not a.udata[1]
-  a.udata[2] = not a.udata[2]
-  a.udata[3] = not a.udata[3]
-
 proc bitnot*(a: Int128): Int128 =
+  result = Zero
   result.udata[0] = not a.udata[0]
   result.udata[1] = not a.udata[1]
   result.udata[2] = not a.udata[2]
   result.udata[3] = not a.udata[3]
 
 proc bitand*(a, b: Int128): Int128 =
+  result = Zero
   result.udata[0] = a.udata[0] and b.udata[0]
   result.udata[1] = a.udata[1] and b.udata[1]
   result.udata[2] = a.udata[2] and b.udata[2]
   result.udata[3] = a.udata[3] and b.udata[3]
 
 proc bitor*(a, b: Int128): Int128 =
+  result = Zero
   result.udata[0] = a.udata[0] or b.udata[0]
   result.udata[1] = a.udata[1] or b.udata[1]
   result.udata[2] = a.udata[2] or b.udata[2]
   result.udata[3] = a.udata[3] or b.udata[3]
 
 proc bitxor*(a, b: Int128): Int128 =
+  result = Zero
   result.udata[0] = a.udata[0] xor b.udata[0]
   result.udata[1] = a.udata[1] xor b.udata[1]
   result.udata[2] = a.udata[2] xor b.udata[2]
   result.udata[3] = a.udata[3] xor b.udata[3]
 
 proc `shr`*(a: Int128, b: int): Int128 =
+  result = Zero
   let b = b and 127
   if b < 32:
     result.sdata(3) = a.sdata(3) shr b
@@ -265,6 +264,7 @@ proc `shr`*(a: Int128, b: int): Int128 =
     result.sdata(0) = a.sdata(3) shr (b and 31)
 
 proc `shl`*(a: Int128, b: int): Int128 =
+  result = Zero
   let b = b and 127
   if b < 32:
     result.udata[0] = a.udata[0] shl b
@@ -288,6 +288,7 @@ proc `shl`*(a: Int128, b: int): Int128 =
     result.udata[3] = a.udata[0] shl (b and 31)
 
 proc `+`*(a, b: Int128): Int128 =
+  result = Zero
   let tmp0 = uint64(a.udata[0]) + uint64(b.udata[0])
   result.udata[0] = cast[uint32](tmp0)
   let tmp1 = uint64(a.udata[1]) + uint64(b.udata[1]) + (tmp0 shr 32)
@@ -320,6 +321,7 @@ proc abs(a: int32): int =
   if a < 0: -a else: a
 
 proc `*`(a: Int128, b: uint32): Int128 =
+  result = Zero
   let tmp0 = uint64(a.udata[0]) * uint64(b)
   let tmp1 = uint64(a.udata[1]) * uint64(b)
   let tmp2 = uint64(a.udata[2]) * uint64(b)
@@ -338,10 +340,11 @@ proc `*`*(a: Int128, b: int32): Int128 =
   if b < 0:
     result = -result
 
-proc `*=`*(a: var Int128, b: int32): Int128 =
-  result = result * b
+proc `*=`(a: var Int128, b: int32) =
+  a = a * b
 
 proc makeInt128(high, low: uint64): Int128 =
+  result = Zero
   result.udata[0] = cast[uint32](low)
   result.udata[1] = cast[uint32](low shr 32)
   result.udata[2] = cast[uint32](high)
@@ -354,23 +357,10 @@ proc low64(a: Int128): uint64 =
   bitconcat(a.udata[1], a.udata[0])
 
 proc `*`*(lhs, rhs: Int128): Int128 =
-  let
-    a = cast[uint64](lhs.udata[0])
-    b = cast[uint64](lhs.udata[1])
-    c = cast[uint64](lhs.udata[2])
-    d = cast[uint64](lhs.udata[3])
-
-    e = cast[uint64](rhs.udata[0])
-    f = cast[uint64](rhs.udata[1])
-    g = cast[uint64](rhs.udata[2])
-    h = cast[uint64](rhs.udata[3])
-
-
-  let a32 = cast[uint64](lhs.udata[1])
-  let a00 = cast[uint64](lhs.udata[0])
-  let b32 = cast[uint64](rhs.udata[1])
-  let b00 = cast[uint64](rhs.udata[0])
-
+  let a32 = uint64(lhs.udata[1])
+  let a00 = uint64(lhs.udata[0])
+  let b32 = uint64(rhs.udata[1])
+  let b00 = uint64(rhs.udata[0])
   result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
   result += toInt128(a32 * b00) shl 32
   result += toInt128(a00 * b32) shl 32
@@ -381,6 +371,7 @@ proc `*=`*(a: var Int128, b: Int128) =
 import std/bitops
 
 proc fastLog2*(a: Int128): int =
+  result = 0
   if a.udata[3] != 0:
     return 96 + fastLog2(a.udata[3])
   if a.udata[2] != 0:
@@ -392,6 +383,8 @@ proc fastLog2*(a: Int128): int =
 
 proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
   assert(divisor != Zero)
+  result = (Zero, Zero)
+
   let isNegativeA = isNegative(dividend)
   let isNegativeB = isNegative(divisor)
 
@@ -441,17 +434,17 @@ proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
     result.remainder = dividend
 
 proc `div`*(a, b: Int128): Int128 =
-  let (a, b) = divMod(a, b)
+  let (a, _) = divMod(a, b)
   return a
 
 proc `mod`*(a, b: Int128): Int128 =
-  let (a, b) = divMod(a, b)
+  let (_, b) = divMod(a, b)
   return b
 
 proc addInt128*(result: var string; value: Int128) =
   let initialSize = result.len
   if value == Zero:
-    result.add "0"
+    result.add '0'
   elif value == low(Int128):
     result.add "-170141183460469231731687303715884105728"
   else:
@@ -472,6 +465,8 @@ proc addInt128*(result: var string; value: Int128) =
       j -= 1
 
 proc `$`*(a: Int128): string =
+  # "-170141183460469231731687303715884105728".len == 41
+  result = newStringOfCap(41)
   result.addInt128(a)
 
 proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
@@ -556,24 +551,28 @@ proc toInt128*(arg: float64): Int128 =
     return res
 
 proc maskUInt64*(arg: Int128): Int128 {.noinit, inline.} =
+  result = Zero
   result.udata[0] = arg.udata[0]
   result.udata[1] = arg.udata[1]
   result.udata[2] = 0
   result.udata[3] = 0
 
 proc maskUInt32*(arg: Int128): Int128 {.noinit, inline.} =
+  result = Zero
   result.udata[0] = arg.udata[0]
   result.udata[1] = 0
   result.udata[2] = 0
   result.udata[3] = 0
 
 proc maskUInt16*(arg: Int128): Int128 {.noinit, inline.} =
+  result = Zero
   result.udata[0] = arg.udata[0] and 0xffff
   result.udata[1] = 0
   result.udata[2] = 0
   result.udata[3] = 0
 
 proc maskUInt8*(arg: Int128): Int128 {.noinit, inline.} =
+  result = Zero
   result.udata[0] = arg.udata[0] and 0xff
   result.udata[1] = 0
   result.udata[2] = 0
@@ -590,4 +589,4 @@ proc maskBytes*(arg: Int128, numbytes: int): Int128 {.noinit.} =
   of 8:
     return maskUInt64(arg)
   else:
-    assert(false, "masking only implemented for 1, 2, 4 and 8 bytes")
+    raiseAssert "masking only implemented for 1, 2, 4 and 8 bytes"
diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim
index 01f0a002a5..17fbde29ee 100644
--- a/compiler/isolation_check.nim
+++ b/compiler/isolation_check.nim
@@ -11,13 +11,19 @@
 ## https://github.com/nim-lang/RFCs/issues/244 for more details.
 
 import
-  ast, types, renderer, intsets
+  ast, types, renderer
+
+import std/intsets
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 proc canAlias(arg, ret: PType; marker: var IntSet): bool
 
 proc canAliasN(arg: PType; n: PNode; marker: var IntSet): bool =
   case n.kind
   of nkRecList:
+    result = false
     for i in 0.. 0 and ret[0] != nil:
-        result = canAlias(arg, ret[0], marker)
+      if not result and ret.baseClass != nil:
+        result = canAlias(arg, ret.baseClass, marker)
     else:
       result = true
   of tyTuple:
-    for i in 0.. can stay expression based
-  if n.kind in nkCallKinds+{nkBracketExpr, nkDotExpr, nkPar, nkTupleConstr} or
-      (n.kind in {nkObjConstr, nkBracket, nkCurly}):
+  case n.kind
+  of nkCallKinds, nkBracketExpr, nkDotExpr, nkPar, nkTupleConstr,
+    nkObjConstr, nkBracket, nkCurly,
+    nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr,
+    nkConv, nkHiddenStdConv, nkHiddenSubConv:
     for c in n:
       if not p.isSimpleExpr(c): return false
     result = true
-  elif n.isAtom:
-    result = true
+  of nkStmtListExpr:
+    for i in 0..>> 0"
-  else: rope""
+  of 1: "& 0xff"
+  of 2: "& 0xffff"
+  of 4: ">>> 0"
+  else: ""
 
-
-template unsignedTrimmer(size: BiggestInt): Rope =
-  size.unsignedTrimmerJS
+proc signedTrimmer(size: BiggestInt): string =
+  # sign extension is done by shifting to the left and then back to the right
+  "<< $1 >> $1" % [$(32 - size * 8)]
 
 proc binaryUintExpr(p: PProc, n: PNode, r: var TCompRes, op: string,
                     reassign: static[bool] = false) =
-  var x, y: TCompRes
+  var x, y: TCompRes = default(TCompRes)
   gen(p, n[1], x)
   gen(p, n[2], y)
-  let trimmer = unsignedTrimmer(n[1].typ.skipTypes(abstractRange).size)
+  let size = n[1].typ.skipTypes(abstractRange).size
   when reassign:
     let (a, tmp) = maybeMakeTempAssignable(p, n[1], x)
-    r.res = "$1 = (($5 $2 $3) $4)" % [a, rope op, y.rdLoc, trimmer, tmp]
+    if size == 8 and optJsBigInt64 in p.config.globalOptions:
+      r.res = "$1 = BigInt.asUintN(64, ($4 $2 $3))" % [a, rope op, y.rdLoc, tmp]
+    else:
+      let trimmer = unsignedTrimmer(size)
+      r.res = "$1 = (($5 $2 $3) $4)" % [a, rope op, y.rdLoc, trimmer, tmp]
   else:
-    r.res = "(($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
+    if size == 8 and optJsBigInt64 in p.config.globalOptions:
+      r.res = "BigInt.asUintN(64, ($1 $2 $3))" % [x.rdLoc, rope op, y.rdLoc]
+    else:
+      let trimmer = unsignedTrimmer(size)
+      r.res = "(($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
   r.kind = resExpr
 
 template ternaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
@@ -572,8 +601,8 @@ template unaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
 
 proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   var
-    x, y: TCompRes
-    xLoc,yLoc: Rope
+    x, y: TCompRes = default(TCompRes)
+    xLoc, yLoc: Rope = ""
   let i = ord(optOverflowCheck notin p.options)
   useMagic(p, jsMagics[op][i])
   if n.len > 2:
@@ -590,32 +619,133 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   template applyFormat(frmtA, frmtB) =
     if i == 0: applyFormat(frmtA) else: applyFormat(frmtB)
 
-  case op:
-  of mAddI: applyFormat("addInt($1, $2)", "($1 + $2)")
-  of mSubI: applyFormat("subInt($1, $2)", "($1 - $2)")
-  of mMulI: applyFormat("mulInt($1, $2)", "($1 * $2)")
-  of mDivI: applyFormat("divInt($1, $2)", "Math.trunc($1 / $2)")
-  of mModI: applyFormat("modInt($1, $2)", "Math.trunc($1 % $2)")
-  of mSucc: applyFormat("addInt($1, $2)", "($1 + $2)")
-  of mPred: applyFormat("subInt($1, $2)", "($1 - $2)")
+  template bitwiseExpr(op: string) =
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.kind in {tyUInt, tyUInt32}:
+      r.res = "(($1 $2 $3) >>> 0)" % [xLoc, op, yLoc]
+    else:
+      r.res = "($1 $2 $3)" % [xLoc, op, yLoc]
+
+  case op
+  of mAddI:
+    if i == 0:
+      if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        useMagic(p, "addInt64")
+        applyFormat("addInt64($1, $2)")
+      else:
+        applyFormat("addInt($1, $2)")
+    else:
+      applyFormat("($1 + $2)")
+  of mSubI:
+    if i == 0:
+      if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        useMagic(p, "subInt64")
+        applyFormat("subInt64($1, $2)")
+      else:
+        applyFormat("subInt($1, $2)")
+    else:
+      applyFormat("($1 - $2)")
+  of mMulI:
+    if i == 0:
+      if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        useMagic(p, "mulInt64")
+        applyFormat("mulInt64($1, $2)")
+      else:
+        applyFormat("mulInt($1, $2)")
+    else:
+      applyFormat("($1 * $2)")
+  of mDivI:
+    if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+      useMagic(p, "divInt64")
+      applyFormat("divInt64($1, $2)", "$1 / $2")
+    else:
+      applyFormat("divInt($1, $2)", "Math.trunc($1 / $2)")
+  of mModI:
+    if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+      useMagic(p, "modInt64")
+      applyFormat("modInt64($1, $2)", "$1 % $2")
+    else:
+      applyFormat("modInt($1, $2)", "Math.trunc($1 % $2)")
+  of mSucc:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    case typ.kind
+    of tyUInt..tyUInt32:
+      binaryUintExpr(p, n, r, "+")
+    of tyUInt64:
+      if optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asUintN(64, $1 + BigInt($2))")
+      else: binaryUintExpr(p, n, r, "+")
+    elif typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+      if optOverflowCheck notin p.options:
+        applyFormat("BigInt.asIntN(64, $1 + BigInt($2))")
+      else: binaryExpr(p, n, r, "addInt64", "addInt64($1, BigInt($2))")
+    else:
+      if optOverflowCheck notin p.options: applyFormat("$1 + $2")
+      else: binaryExpr(p, n, r, "addInt", "addInt($1, $2)")
+  of mPred:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    case typ.kind
+    of tyUInt..tyUInt32:
+      binaryUintExpr(p, n, r, "-")
+    of tyUInt64:
+      if optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asUintN(64, $1 - BigInt($2))")
+      else: binaryUintExpr(p, n, r, "-")
+    elif typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+      if optOverflowCheck notin p.options:
+        applyFormat("BigInt.asIntN(64, $1 - BigInt($2))")
+      else: binaryExpr(p, n, r, "subInt64", "subInt64($1, BigInt($2))")
+    else:
+      if optOverflowCheck notin p.options: applyFormat("$1 - $2")
+      else: binaryExpr(p, n, r, "subInt", "subInt($1, $2)")
   of mAddF64: applyFormat("($1 + $2)", "($1 + $2)")
   of mSubF64: applyFormat("($1 - $2)", "($1 - $2)")
   of mMulF64: applyFormat("($1 * $2)", "($1 * $2)")
   of mDivF64: applyFormat("($1 / $2)", "($1 / $2)")
-  of mShrI: applyFormat("", "")
+  of mShrI:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+      applyFormat("BigInt.asIntN(64, BigInt.asUintN(64, $1) >> BigInt($2))")
+    elif typ.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions:
+      applyFormat("($1 >> BigInt($2))")
+    else:
+      if typ.kind in {tyInt..tyInt32}:
+        let trimmerU = unsignedTrimmer(typ.size)
+        let trimmerS = signedTrimmer(typ.size)
+        r.res = "((($1 $2) >>> $3) $4)" % [xLoc, trimmerU, yLoc, trimmerS]
+      else:
+        applyFormat("($1 >>> $2)")
   of mShlI:
-    if n[1].typ.size <= 4:
-      applyFormat("($1 << $2)", "($1 << $2)")
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.size == 8:
+      if typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asIntN(64, $1 << BigInt($2))")
+      elif typ.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asUintN(64, $1 << BigInt($2))")
+      else:
+        applyFormat("($1 * Math.pow(2, $2))")
     else:
-      applyFormat("($1 * Math.pow(2,$2))", "($1 * Math.pow(2,$2))")
+      if typ.kind in {tyUInt..tyUInt32}:
+        let trimmer = unsignedTrimmer(typ.size)
+        r.res = "(($1 << $2) $3)" % [xLoc, yLoc, trimmer]
+      else:
+        let trimmer = signedTrimmer(typ.size)
+        r.res = "(($1 << $2) $3)" % [xLoc, yLoc, trimmer]
   of mAshrI:
-    if n[1].typ.size <= 4:
-      applyFormat("($1 >> $2)", "($1 >> $2)")
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.size == 8:
+      if optJsBigInt64 in p.config.globalOptions:
+        applyFormat("($1 >> BigInt($2))")
+      else:
+        applyFormat("Math.floor($1 / Math.pow(2, $2))")
     else:
-      applyFormat("Math.floor($1 / Math.pow(2,$2))", "Math.floor($1 / Math.pow(2,$2))")
-  of mBitandI: applyFormat("($1 & $2)", "($1 & $2)")
-  of mBitorI: applyFormat("($1 | $2)", "($1 | $2)")
-  of mBitxorI: applyFormat("($1 ^ $2)", "($1 ^ $2)")
+      if typ.kind in {tyUInt..tyUInt32}:
+        applyFormat("($1 >>> $2)")
+      else:
+        applyFormat("($1 >> $2)")
+  of mBitandI: bitwiseExpr("&")
+  of mBitorI: bitwiseExpr("|")
+  of mBitxorI: bitwiseExpr("^")
   of mMinI: applyFormat("nimMin($1, $2)", "nimMin($1, $2)")
   of mMaxI: applyFormat("nimMax($1, $2)", "nimMax($1, $2)")
   of mAddU: applyFormat("", "")
@@ -648,21 +778,31 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mEqProc: applyFormat("($1 == $2)", "($1 == $2)")
   of mUnaryMinusI: applyFormat("negInt($1)", "-($1)")
   of mUnaryMinusI64: applyFormat("negInt64($1)", "-($1)")
-  of mAbsI: applyFormat("absInt($1)", "Math.abs($1)")
+  of mAbsI:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+      useMagic(p, "absInt64")
+      applyFormat("absInt64($1)", "absInt64($1)")
+    else:
+      applyFormat("absInt($1)", "Math.abs($1)")
   of mNot: applyFormat("!($1)", "!($1)")
   of mUnaryPlusI: applyFormat("+($1)", "+($1)")
-  of mBitnotI: applyFormat("~($1)", "~($1)")
+  of mBitnotI:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.kind in {tyUInt..tyUInt64}:
+      if typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asUintN(64, ~($1))")
+      else:
+        let trimmer = unsignedTrimmer(typ.size)
+        r.res = "(~($1) $2)" % [xLoc, trimmer]
+    else:
+      applyFormat("~($1)")
   of mUnaryPlusF64: applyFormat("+($1)", "+($1)")
   of mUnaryMinusF64: applyFormat("-($1)", "-($1)")
   of mCharToStr: applyFormat("nimCharToStr($1)", "nimCharToStr($1)")
   of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)")
-  of mIntToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
-  of mInt64ToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
-  of mFloatToStr:
-    useMagic(p, "nimFloatToString")
-    applyFormat "cstrToNimstr(nimFloatToString($1))"
   of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)")
-  of mStrToStr, mUnown, mIsolate: applyFormat("$1", "$1")
+  of mStrToStr, mUnown, mIsolate, mFinished: applyFormat("$1", "$1")
   else:
     assert false, $op
 
@@ -673,29 +813,29 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mMulU: binaryUintExpr(p, n, r, "*")
   of mDivU:
     binaryUintExpr(p, n, r, "/")
-    if n[1].typ.skipTypes(abstractRange).size == 8:
+    if optJsBigInt64 notin p.config.globalOptions and
+        n[1].typ.skipTypes(abstractRange).size == 8:
+      # bigint / already truncates
       r.res = "Math.trunc($1)" % [r.res]
   of mDivI:
     arithAux(p, n, r, op)
   of mModI:
     arithAux(p, n, r, op)
-  of mShrI:
-    var x, y: TCompRes
-    gen(p, n[1], x)
-    gen(p, n[2], y)
-    let trimmer = unsignedTrimmer(n[1].typ.skipTypes(abstractRange).size)
-    r.res = "(($1 $2) >>> $3)" % [x.rdLoc, trimmer, y.rdLoc]
-  of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr,
-      mCStrToStr, mStrToStr, mEnumToStr:
+  of mCharToStr, mBoolToStr, mCStrToStr, mStrToStr, mEnumToStr:
     arithAux(p, n, r, op)
   of mEqRef:
     if mapType(n[1].typ) != etyBaseIndex:
       arithAux(p, n, r, op)
     else:
-      var x, y: TCompRes
+      var x, y: TCompRes = default(TCompRes)
       gen(p, n[1], x)
       gen(p, n[2], y)
       r.res = "($# == $# && $# == $#)" % [x.address, y.address, x.res, y.res]
+  of mEqProc:
+    if skipTypes(n[1].typ, abstractInst).callConv == ccClosure:
+      binaryExpr(p, n, r, "cmpClosures", "cmpClosures($1, $2)")
+    else:
+      arithAux(p, n, r, op)
   else:
     arithAux(p, n, r, op)
   r.kind = resExpr
@@ -705,20 +845,27 @@ proc hasFrameInfo(p: PProc): bool =
       ((p.prc == nil) or not (sfPure in p.prc.flags))
 
 proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope =
-  ropes.`%`("// line $2 \"$1\"$n",
-         [rope(toFullPath(config, info)), rope(line)])
+  "/* line $2:$3 \"$1\" */$n" % [
+    rope(toFullPath(config, info)), rope(line), rope(info.toColumn)
+  ]
 
 proc genLineDir(p: PProc, n: PNode) =
   let line = toLinenumber(n.info)
   if line < 0:
     return
+  if optEmbedOrigSrc in p.config.globalOptions:
+    lineF(p, "//$1$n", [sourceLine(p.config, n.info)])
   if optLineDir in p.options or optLineDir in p.config.options:
     lineF(p, "$1", [lineDir(p.config, n.info, line)])
   if hasFrameInfo(p):
     lineF(p, "F.line = $1;$n", [rope(line)])
+    let currentFileName = toFilename(p.config, n.info)
+    if p.previousFileName != currentFileName:
+      lineF(p, "F.filename = $1;$n", [makeJSString(currentFileName)])
+      p.previousFileName = currentFileName
 
 proc genWhileStmt(p: PProc, n: PNode) =
-  var cond: TCompRes
+  var cond: TCompRes = default(TCompRes)
   internalAssert p.config, isEmptyType(n.typ)
   genLineDir(p, n)
   inc(p.unique)
@@ -726,9 +873,9 @@ proc genWhileStmt(p: PProc, n: PNode) =
   p.blocks[^1].id = -p.unique
   p.blocks[^1].isLoop = true
   let labl = p.unique.rope
-  lineF(p, "L$1: while (true) {$n", [labl])
+  lineF(p, "Label$1: while (true) {$n", [labl])
   p.nested: gen(p, n[0], cond)
-  lineF(p, "if (!$1) break L$2;$n",
+  lineF(p, "if (!$1) break Label$2;$n",
        [cond.res, labl])
   p.nested: genStmt(p, n[1])
   lineF(p, "}$n", [labl])
@@ -741,7 +888,7 @@ proc moveInto(p: PProc, src: var TCompRes, dest: TCompRes) =
     else:
       lineF(p, "$1;$n", [src.rdLoc])
     src.kind = resNone
-    src.res = nil
+    src.res = ""
 
 proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   # code to generate:
@@ -751,8 +898,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   #  try {
   #    stmts;
   #    --excHandler;
-  #  } catch (EXC) {
-  #    var prevJSError = lastJSError; lastJSError = EXC;
+  #  } catch (EXCEPTION) {
+  #    var prevJSError = lastJSError; lastJSError = EXCEPTION;
   #    framePtr = tmpFramePtr;
   #    --excHandler;
   #    if (e.typ && e.typ == NTI433 || e.typ == NTI2321) {
@@ -777,18 +924,16 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   if catchBranchesExist:
     p.body.add("++excHandler;\L")
   var tmpFramePtr = rope"F"
-  if optStackTrace notin p.options:
-    tmpFramePtr = p.getTemp(true)
-    line(p, tmpFramePtr & " = framePtr;\L")
   lineF(p, "try {$n", [])
-  var a: TCompRes
+  var a: TCompRes = default(TCompRes)
   gen(p, n[0], a)
   moveInto(p, a, r)
   var generalCatchBranchExists = false
   if catchBranchesExist:
-    p.body.addf("--excHandler;$n} catch (EXC) {$n var prevJSError = lastJSError;$n" &
-        " lastJSError = EXC;$n --excHandler;$n", [])
-    line(p, "framePtr = $1;$n" % [tmpFramePtr])
+    p.body.addf("--excHandler;$n} catch (EXCEPTION) {$n var prevJSError = lastJSError;$n" &
+        " lastJSError = EXCEPTION;$n --excHandler;$n", [])
+    if hasFrameInfo(p):
+      line(p, "framePtr = $1;$n" % [tmpFramePtr])
   while i < n.len and n[i].kind == nkExceptBranch:
     if n[i].len == 1:
       # general except section:
@@ -798,7 +943,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
       moveInto(p, a, r)
       if i > 1: lineF(p, "}$n", [])
     else:
-      var orExpr: Rope = nil
+      var orExpr: Rope = ""
       var excAlias: PNode = nil
 
       useMagic(p, "isObj")
@@ -811,13 +956,14 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
           excAlias = it[2]
           # If this is a ``except exc as sym`` branch there must be no following
           # nodes
-          doAssert orExpr == nil
+          doAssert orExpr == ""
         elif it.kind == nkType:
           throwObj = it
         else:
+          throwObj = nil
           internalError(p.config, n.info, "genTryStmt")
 
-        if orExpr != nil: orExpr.add("||")
+        if orExpr != "": orExpr.add("||")
         # Generate the correct type checking code depending on whether this is a
         # NIM-native or a JS-native exception
         # if isJsObject(throwObj.typ):
@@ -847,14 +993,15 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
       line(p, "}\L")
     lineF(p, "lastJSError = prevJSError;$n")
   line(p, "} finally {\L")
-  line(p, "framePtr = $1;$n" % [tmpFramePtr])
+  if hasFrameInfo(p):
+    line(p, "framePtr = $1;$n" % [tmpFramePtr])
   if i < n.len and n[i].kind == nkFinally:
     genStmt(p, n[i][0])
   line(p, "}\L")
 
 proc genRaiseStmt(p: PProc, n: PNode) =
   if n[0].kind != nkEmpty:
-    var a: TCompRes
+    var a: TCompRes = default(TCompRes)
     gen(p, n[0], a)
     let typ = skipTypes(n[0].typ, abstractPtrs)
     genLineDir(p, n)
@@ -868,14 +1015,18 @@ proc genRaiseStmt(p: PProc, n: PNode) =
 
 proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
   var
-    cond, stmt: TCompRes
-    totalRange = 0
+    a, b, cond, stmt: TCompRes = default(TCompRes)
   genLineDir(p, n)
   gen(p, n[0], cond)
-  let stringSwitch = skipTypes(n[0].typ, abstractVar).kind == tyString
-  if stringSwitch:
+  let typeKind = skipTypes(n[0].typ, abstractVar+{tyRange}).kind
+  var transferRange = false
+  let anyString = typeKind in {tyString, tyCstring}
+  case typeKind
+  of tyString:
     useMagic(p, "toJSStr")
     lineF(p, "switch (toJSStr($1)) {$n", [cond.rdLoc])
+  of tyFloat..tyFloat128, tyInt..tyInt64, tyUInt..tyUInt64:
+    transferRange = true
   else:
     lineF(p, "switch ($1) {$n", [cond.rdLoc])
   if not isEmptyType(n.typ):
@@ -883,41 +1034,71 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
     r.res = getTemp(p)
   for i in 1.. 65535:
-            localError(p.config, n.info,
-                       "Your case statement contains too many branches, consider using if/else instead!")
-          while v.intVal <= e[1].intVal:
-            gen(p, v, cond)
-            lineF(p, "case $1:$n", [cond.rdLoc])
-            inc(v.intVal)
+          if transferRange:
+            gen(p, e[0], a)
+            gen(p, e[1], b)
+            if j != itLen - 2:
+              lineF(p, "$1 >= $2 && $1 <= $3 || $n", [cond.rdLoc, a.rdLoc, b.rdLoc])
+            else:
+              lineF(p, "$1 >= $2 && $1 <= $3", [cond.rdLoc, a.rdLoc, b.rdLoc])
+          else:
+            var v = copyNode(e[0])
+            while v.intVal <= e[1].intVal:
+              gen(p, v, cond)
+              lineF(p, "case $1:$n", [cond.rdLoc])
+              inc(v.intVal)
         else:
-          if stringSwitch:
+          if anyString:
             case e.kind
             of nkStrLit..nkTripleStrLit: lineF(p, "case $1:$n",
                 [makeJSString(e.strVal, false)])
+            of nkNilLit: lineF(p, "case null:$n", [])
             else: internalError(p.config, e.info, "jsgen.genCaseStmt: 2")
           else:
-            gen(p, e, cond)
-            lineF(p, "case $1:$n", [cond.rdLoc])
+            if transferRange:
+              gen(p, e, a)
+              if j != itLen - 2:
+                lineF(p, "$1 == $2 || $n", [cond.rdLoc, a.rdLoc])
+              else:
+                lineF(p, "$1 == $2", [cond.rdLoc, a.rdLoc])
+            else:
+              gen(p, e, a)
+              lineF(p, "case $1:$n", [a.rdLoc])
+      if transferRange:
+        lineF(p, "){", [])
       p.nested:
         gen(p, lastSon(it), stmt)
         moveInto(p, stmt, r)
-        lineF(p, "break;$n", [])
+        if transferRange:
+          lineF(p, "}$n", [])
+        else:
+          lineF(p, "break;$n", [])
     of nkElse:
-      lineF(p, "default: $n", [])
+      if transferRange:
+         lineF(p, "else{$n", [])
+      else:
+        lineF(p, "default: $n", [])
       p.nested:
         gen(p, it[0], stmt)
         moveInto(p, stmt, r)
-        lineF(p, "break;$n", [])
+        if transferRange:
+           lineF(p, "}$n", [])
+        else:
+          lineF(p, "break;$n", [])
     else: internalError(p.config, it.info, "jsgen.genCaseStmt")
-  lineF(p, "}$n", [])
+  if not transferRange:
+    lineF(p, "}$n", [])
 
 proc genBlock(p: PProc, n: PNode, r: var TCompRes) =
   inc(p.unique)
@@ -929,12 +1110,12 @@ proc genBlock(p: PProc, n: PNode, r: var TCompRes) =
     sym.loc.k = locOther
     sym.position = idx+1
   let labl = p.unique
-  lineF(p, "L$1: do {$n", [labl.rope])
+  lineF(p, "Label$1: {$n", [labl.rope])
   setLen(p.blocks, idx + 1)
   p.blocks[idx].id = - p.unique # negative because it isn't used yet
   gen(p, n[1], r)
   setLen(p.blocks, idx)
-  lineF(p, "} while(false);$n", [labl.rope])
+  lineF(p, "};$n", [labl.rope])
 
 proc genBreakStmt(p: PProc, n: PNode) =
   var idx: int
@@ -952,12 +1133,16 @@ proc genBreakStmt(p: PProc, n: PNode) =
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(p.config, n.info, "no loop to break")
   p.blocks[idx].id = abs(p.blocks[idx].id) # label is used
-  lineF(p, "break L$1;$n", [rope(p.blocks[idx].id)])
+  lineF(p, "break Label$1;$n", [rope(p.blocks[idx].id)])
 
-proc genAsmOrEmitStmt(p: PProc, n: PNode) =
+proc genAsmOrEmitStmt(p: PProc, n: PNode; isAsmStmt = false) =
   genLineDir(p, n)
-  p.body.add p.indentLine(nil)
-  for i in 0..= 2 and x[0].typ.skipTypes(abstractInst).kind == tyCString:
+  if x.kind == nkBracketExpr and x.len >= 2 and x[0].typ.skipTypes(abstractInst).kind == tyCstring:
     localError(p.config, x.info, "cstring doesn't support `[]=` operator")
 
   gen(p, x, a)
@@ -1067,14 +1261,14 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
     xtyp = etySeq
   case xtyp
   of etySeq:
-    if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
       lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
-    if x.typ.kind in {tyVar} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar, tyLent, tyOpenArray, tyVarargs} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
@@ -1093,16 +1287,24 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
       elif b.typ == etyBaseIndex:
         lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res])
       elif b.typ == etyNone:
-        internalAssert p.config, b.address == nil
+        internalAssert p.config, b.address == ""
         lineF(p, "$# = [$#, 0];$n", [a.address, b.res])
       elif x.typ.kind == tyVar and y.typ.kind == tyPtr:
         lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res])
         lineF(p, "$1 = $2;$n", [a.address, b.res])
         lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
+      elif a.typ == etyBaseIndex:
+        # array indexing may not map to var type
+        if b.address != "":
+          lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
+        else:
+          lineF(p, "$1 = $2;$n", [a.address, b.res])
       else:
         internalError(p.config, x.info, $("genAsgn", b.typ, a.typ))
-    else:
+    elif b.address != "":
       lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
+    else:
+      lineF(p, "$1 = $2;$n", [a.address, b.res])
   else:
     lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
 
@@ -1121,28 +1323,21 @@ proc genFastAsgn(p: PProc, n: PNode) =
   genAsgnAux(p, n[0], n[1], noCopyNeeded=noCopy)
 
 proc genSwap(p: PProc, n: PNode) =
-  var a, b: TCompRes
-  gen(p, n[1], a)
-  gen(p, n[2], b)
-  var tmp = p.getTemp(false)
-  if mapType(p, skipTypes(n[1].typ, abstractVar)) == etyBaseIndex:
-    let tmp2 = p.getTemp(false)
-    if a.typ != etyBaseIndex or b.typ != etyBaseIndex:
-      internalError(p.config, n.info, "genSwap")
-    lineF(p, "var $1 = $2; $2 = $3; $3 = $1;$n",
-             [tmp, a.address, b.address])
-    tmp = tmp2
-  lineF(p, "var $1 = $2; $2 = $3; $3 = $1;",
-           [tmp, a.res, b.res])
+  let stmtList = lowerSwap(p.module.graph, n, p.module.idgen, if p.prc != nil: p.prc else: p.module.module)
+  assert stmtList.kind == nkStmtList
+  for i in 0.. 0
     if pat.contains({'#', '(', '@'}):
       var typ = skipTypes(n[0].typ, abstractInst)
@@ -1602,13 +1801,13 @@ proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) =
   if n.len != 1:
     gen(p, n[1], r)
     if r.typ == etyBaseIndex:
-      if r.address == nil:
+      if r.address == "":
         globalError(p.config, n.info, "cannot invoke with infix syntax")
       r.res = "$1[$2]" % [r.address, r.res]
-      r.address = nil
+      r.address = ""
       r.typ = etyNone
     r.res.add(".")
-  var op: TCompRes
+  var op: TCompRes = default(TCompRes)
   gen(p, n[0], op)
   r.res.add(op.res)
   genArgs(p, n, r, 2)
@@ -1670,33 +1869,56 @@ proc createObjInitList(p: PProc, typ: PType, excludedFieldIDs: IntSet, output: v
   while t != nil:
     t = t.skipTypes(skipPtrs)
     createRecordVarAux(p, t.n, excludedFieldIDs, output)
-    t = t[0]
+    t = t.baseClass
 
-proc arrayTypeForElemType(typ: PType): string =
-  # XXX This should also support tyEnum and tyBool
+proc arrayTypeForElemType(conf: ConfigRef; typ: PType): string =
+  let typ = typ.skipTypes(abstractRange)
   case typ.kind
   of tyInt, tyInt32: "Int32Array"
   of tyInt16: "Int16Array"
   of tyInt8: "Int8Array"
+  of tyInt64:
+    if optJsBigInt64 in conf.globalOptions:
+      "BigInt64Array"
+    else:
+      ""
   of tyUInt, tyUInt32: "Uint32Array"
   of tyUInt16: "Uint16Array"
-  of tyUInt8: "Uint8Array"
+  of tyUInt8, tyChar, tyBool: "Uint8Array"
+  of tyUInt64:
+    if optJsBigInt64 in conf.globalOptions:
+      "BigUint64Array"
+    else:
+      ""
   of tyFloat32: "Float32Array"
   of tyFloat64, tyFloat: "Float64Array"
+  of tyEnum:
+    case typ.size
+    of 1: "Uint8Array"
+    of 2: "Uint16Array"
+    of 4: "Uint32Array"
+    else: ""
   else: ""
 
 proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   var t = skipTypes(typ, abstractInst)
   case t.kind
-  of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar:
+  of tyInt8..tyInt32, tyUInt8..tyUInt32, tyEnum, tyChar:
+    result = putToSeq("0", indirect)
+  of tyInt, tyUInt:
     if $t.sym.loc.r == "bigint":
       result = putToSeq("0n", indirect)
     else:
       result = putToSeq("0", indirect)
+  of tyInt64, tyUInt64:
+    if optJsBigInt64 in p.config.globalOptions:
+      result = putToSeq("0n", indirect)
+    else:
+      result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
-  of tyRange, tyGenericInst, tyAlias, tySink, tyOwned:
-    result = createVar(p, lastSon(typ), indirect)
+  of tyRange, tyGenericInst, tyAlias, tySink, tyOwned, tyLent:
+    result = createVar(p, skipModifier(typ), indirect)
   of tySet:
     result = putToSeq("{}", indirect)
   of tyBool:
@@ -1706,7 +1928,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyArray:
     let length = toInt(lengthOrd(p.config, t))
     let e = elemType(t)
-    let jsTyp = arrayTypeForElemType(e)
+    let jsTyp = arrayTypeForElemType(p.config, e)
     if jsTyp.len > 0:
       result = "new $1($2)" % [rope(jsTyp), rope(length)]
     elif length > 32:
@@ -1733,34 +1955,34 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     result.add("}")
     if indirect: result = "[$1]" % [result]
   of tyObject:
-    var initList: Rope
+    var initList: Rope = ""
     createObjInitList(p, t, initIntSet(), initList)
     result = ("({$1})") % [initList]
     if indirect: result = "[$1]" % [result]
-  of tyVar, tyPtr, tyLent, tyRef, tyPointer:
+  of tyVar, tyPtr, tyRef, tyPointer:
     if mapType(p, t) == etyBaseIndex:
       result = putToSeq("[null, 0]", indirect)
     else:
       result = putToSeq("null", indirect)
   of tySequence, tyString:
     result = putToSeq("[]", indirect)
-  of tyCString, tyProc:
+  of tyCstring, tyProc, tyOpenArray:
     result = putToSeq("null", indirect)
   of tyStatic:
     if t.n != nil:
-      result = createVar(p, lastSon t, indirect)
+      result = createVar(p, skipModifier t, indirect)
     else:
       internalError(p.config, "createVar: " & $t.kind)
-      result = nil
+      result = ""
   else:
     internalError(p.config, "createVar: " & $t.kind)
-    result = nil
+    result = ""
 
-template returnType: untyped = ~""
+template returnType: untyped = ""
 
 proc genVarInit(p: PProc, v: PSym, n: PNode) =
   var
-    a: TCompRes
+    a: TCompRes = default(TCompRes)
     s: Rope
     varCode: string
     varName = mangleName(p.module, v)
@@ -1775,7 +1997,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       inc p.extraIndent
     elif useGlobalPragmas:
       lineF(p, "if (globalThis.$1 === undefined) {$n", varName)
-      varCode = $varName
+      varCode = "globalThis." & $varName
       inc p.extraIndent
     else:
       varCode = "var $2"
@@ -1796,7 +2018,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     gen(p, n, a)
     case mapType(p, v.typ)
     of etyObject, etySeq:
-      if needsNoCopy(p, n):
+      if v.typ.kind in {tyOpenArray, tyVarargs} or needsNoCopy(p, n):
         s = a.res
       else:
         useMagic(p, "nimCopy")
@@ -1833,6 +2055,17 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     dec p.extraIndent
     lineF(p, "}$n")
 
+proc genClosureVar(p: PProc, n: PNode) =
+  # assert n[2].kind != nkEmpty
+  # TODO: fixme transform `var env.x` into `var env.x = default()` after
+  # the order of transf and lambdalifting is fixed
+  if n[2].kind != nkEmpty:
+    genAsgnAux(p, n[0], n[2], false)
+  else:
+    var a: TCompRes = default(TCompRes)
+    gen(p, n[0], a)
+    line(p, runtimeFormat("$1 = $2;$n", [rdLoc(a), createVar(p, n[0].typ, false)]))
+
 proc genVarStmt(p: PProc, n: PNode) =
   for i in 0.. 0:
     # generate typed array
     # for example Nim generates `new Uint8Array([1, 2, 3])` for `[byte(1), 2, 3]`
     # TODO use `set` or loop to initialize typed array which improves performances in some situations
-    var a: TCompRes
+    var a: TCompRes = default(TCompRes)
     r.res = "new $1([" % [rope(jsTyp)]
     r.kind = resExpr
     for i in 0 ..< n.len:
@@ -2219,7 +2493,7 @@ proc genArrayConstr(p: PProc, n: PNode, r: var TCompRes) =
     genJSArrayConstr(p, n, r)
 
 proc genTupleConstr(p: PProc, n: PNode, r: var TCompRes) =
-  var a: TCompRes
+  var a: TCompRes = default(TCompRes)
   r.res = rope("{")
   r.kind = resExpr
   for i in 0.. 1: initList.add(", ")
     var it = n[i]
@@ -2249,8 +2524,8 @@ proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) =
     let val = it[1]
     gen(p, val, a)
     var f = it[0].sym
-    if f.loc.r == nil: f.loc.r = mangleName(p.module, f)
-    fieldIDs.incl(f.id)
+    if f.loc.r == "": f.loc.r = mangleName(p.module, f)
+    fieldIDs.incl(lookupFieldAgain(n.typ.skipTypes({tyDistinct}), f).id)
 
     let typ = val.typ.skipTypes(abstractInst)
     if a.typ == etyBaseIndex:
@@ -2282,7 +2557,29 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
     r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
   elif toInt:
-    r.res = "(($1)|0)" % [r.res]
+    if src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "Number($1)" % [r.res]
+    else:
+      r.res = "(($1) | 0)" % [r.res]
+  elif dest.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+    if fromInt or fromUint or src.kind in {tyBool, tyChar, tyEnum}:
+      r.res = "BigInt($1)" % [r.res]
+    elif src.kind in {tyFloat..tyFloat64}:
+      r.res = "BigInt(Math.trunc($1))" % [r.res]
+    elif src.kind == tyUInt64:
+      r.res = "BigInt.asIntN(64, $1)" % [r.res]
+  elif dest.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions:
+    if fromUint or src.kind in {tyBool, tyChar, tyEnum}:
+      r.res = "BigInt($1)" % [r.res]
+    elif fromInt: # could be negative
+      r.res = "BigInt.asUintN(64, BigInt($1))" % [r.res]
+    elif src.kind in {tyFloat..tyFloat64}:
+      r.res = "BigInt.asUintN(64, BigInt(Math.trunc($1)))" % [r.res]
+    elif src.kind == tyInt64:
+      r.res = "BigInt.asUintN(64, $1)" % [r.res]
+  elif toUint or dest.kind in tyFloat..tyFloat64:
+    if src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "Number($1)" % [r.res]
   else:
     # TODO: What types must we handle here?
     discard
@@ -2291,12 +2588,23 @@ proc upConv(p: PProc, n: PNode, r: var TCompRes) =
   gen(p, n[0], r)        # XXX
 
 proc genRangeChck(p: PProc, n: PNode, r: var TCompRes, magic: string) =
-  var a, b: TCompRes
+  var a, b: TCompRes = default(TCompRes)
   gen(p, n[0], r)
-  if optRangeCheck notin p.options or (skipTypes(n.typ, abstractVar).kind in {tyUInt..tyUInt64} and
-      checkUnsignedConversions notin p.config.legacyFeatures):
-    discard "XXX maybe emit masking instructions here"
+  let src = skipTypes(n[0].typ, abstractVarRange)
+  let dest = skipTypes(n.typ, abstractVarRange)
+  if optRangeCheck notin p.options:
+    return
+  elif dest.kind in {tyUInt..tyUInt64} and checkUnsignedConversions notin p.config.legacyFeatures:
+    if src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "BigInt.asUintN($1, $2)" % [$(dest.size * 8), r.res]
+    else:
+      r.res = "BigInt.asUintN($1, BigInt($2))" % [$(dest.size * 8), r.res]
+    if not (dest.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions):
+      r.res = "Number($1)" % [r.res]
   else:
+    if src.kind in {tyInt64, tyUInt64} and dest.kind notin {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      # we do a range check anyway, so it's ok if the number gets rounded
+      r.res = "Number($1)" % [r.res]
     gen(p, n[1], a)
     gen(p, n[2], b)
     useMagic(p, "chckRange")
@@ -2310,7 +2618,7 @@ proc convStrToCStr(p: PProc, n: PNode, r: var TCompRes) =
     gen(p, n[0][0], r)
   else:
     gen(p, n[0], r)
-    if r.res == nil: internalError(p.config, n.info, "convStrToCStr")
+    if r.res == "": internalError(p.config, n.info, "convStrToCStr")
     useMagic(p, "toJSStr")
     r.res = "toJSStr($1)" % [r.res]
     r.kind = resExpr
@@ -2322,7 +2630,7 @@ proc convCStrToStr(p: PProc, n: PNode, r: var TCompRes) =
     gen(p, n[0][0], r)
   else:
     gen(p, n[0], r)
-    if r.res == nil: internalError(p.config, n.info, "convCStrToStr")
+    if r.res == "": internalError(p.config, n.info, "convCStrToStr")
     useMagic(p, "cstrToNimstr")
     r.res = "cstrToNimstr($1)" % [r.res]
     r.kind = resExpr
@@ -2338,7 +2646,7 @@ proc genReturnStmt(p: PProc, n: PNode) =
 
 proc frameCreate(p: PProc; procname, filename: Rope): Rope =
   const frameFmt =
-    "var F={procname:$1,prev:framePtr,filename:$2,line:0};$n"
+    "var F = {procname: $1, prev: framePtr, filename: $2, line: 0};$n"
 
   result = p.indentLine(frameFmt % [procname, filename])
   result.add p.indentLine(ropes.`%`("framePtr = F;$n", []))
@@ -2352,11 +2660,11 @@ proc genProcBody(p: PProc, prc: PSym): Rope =
               makeJSString(prc.owner.name.s & '.' & prc.name.s),
               makeJSString(toFilenameOption(p.config, prc.info.fileIndex, foStacktrace)))
   else:
-    result = nil
+    result = ""
   if p.beforeRetNeeded:
-    result.add p.indentLine(~"BeforeRet: do {$n")
+    result.add p.indentLine("BeforeRet: {\n")
     result.add p.body
-    result.add p.indentLine(~"} while (false);$n")
+    result.add p.indentLine("};\n")
   else:
     result.add(p.body)
   if prc.typ.callConv == ccSysCall:
@@ -2366,29 +2674,32 @@ proc genProcBody(p: PProc, prc: PSym): Rope =
     result.add(frameDestroy(p))
 
 proc optionalLine(p: Rope): Rope =
-  if p == nil:
-    return nil
+  if p == "":
+    return ""
   else:
     return p & "\L"
 
 proc genProc(oldProc: PProc, prc: PSym): Rope =
+  ## Generate a JS procedure ('function').
+  result = ""
   var
     resultSym: PSym
-    a: TCompRes
+    a: TCompRes = default(TCompRes)
   #if gVerbosity >= 3:
   #  echo "BEGIN generating code for: " & prc.name.s
   var p = newProc(oldProc.g, oldProc.module, prc.ast, prc.options)
-  p.up = oldProc
-  var returnStmt: Rope = nil
-  var resultAsgn: Rope = nil
+  var returnStmt: Rope = ""
+  var resultAsgn: Rope = ""
   var name = mangleName(p.module, prc)
-  let header = generateHeader(p, prc.typ)
-  if prc.typ[0] != nil and sfPure notin prc.flags:
+  let header = generateHeader(p, prc)
+  if prc.typ.returnType != nil and sfPure notin prc.flags:
     resultSym = prc.ast[resultPos].sym
     let mname = mangleName(p.module, resultSym)
-    if not isIndirect(resultSym) and
+    # otherwise uses "fat pointers"
+    let useRawPointer = not isIndirect(resultSym) and
       resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and
-        mapType(p, resultSym.typ) == etyBaseIndex:
+        mapType(p, resultSym.typ) == etyBaseIndex
+    if useRawPointer:
       resultAsgn = p.indentLine(("var $# = null;$n") % [mname])
       resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname])
     else:
@@ -2400,7 +2711,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
     else:
       returnStmt = "return $#;$n" % [a.res]
 
-  var transformedBody = transformBody(p.module.graph, p.module.idgen, prc, cache = false)
+  var transformedBody = transformBody(p.module.graph, p.module.idgen, prc, {})
   if sfInjectDestructors in prc.flags:
     transformedBody = injectDestructorCalls(p.module.graph, p.module.idgen, prc, transformedBody)
 
@@ -2423,7 +2734,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
               optionalLine(p.indentLine(returnStmt))])
   else:
     # if optLineDir in p.config.options:
-      # result.add(~"\L")
+      # result.add("\L")
 
     if p.config.hcrOn:
       # Here, we introduce thunks that create the equivalent of a jump table
@@ -2432,10 +2743,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
       # references will end up calling the reloaded code.
       var thunkName = name
       name = name & "IMLP"
-      result.add("function $#() { return $#.apply(this, arguments); }$n" %
+      result.add("\Lfunction $#() { return $#.apply(this, arguments); }$n" %
                  [thunkName, name])
 
-    def = "function $#($#) {$n$#$#$#$#$#" %
+    def = "\Lfunction $#($#) {$n$#$#$#$#$#" %
             [ name,
               header,
               optionalLine(p.globals),
@@ -2446,20 +2757,25 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
 
   dec p.extraIndent
   result.add p.indentLine(def)
-  result.add p.indentLine(~"}$n")
+  result.add p.indentLine("}\n")
 
   #if gVerbosity >= 3:
   #  echo "END   generated code for: " & prc.name.s
 
 proc genStmt(p: PProc, n: PNode) =
-  var r: TCompRes
+  var r: TCompRes = default(TCompRes)
   gen(p, n, r)
-  if r.res != nil: lineF(p, "$#;$n", [r.res])
+  if r.res != "": lineF(p, "$#;$n", [r.res])
 
 proc genPragma(p: PProc, n: PNode) =
-  for it in n.sons:
+  for i in 0.. 0: s.add(", \L")
         s.add(genObjectFields(p, typ, n[i]))
@@ -44,13 +44,13 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
     s = genTypeInfo(p, field.typ)
     for i in 1.. 0: s.add(", \L")
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
@@ -102,7 +102,7 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
   p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
 
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
-  var s: Rope = nil
+  var s: Rope = ""
   for i in 0.. which cannot be captured as it would violate memory" &
-       " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") %
+       " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases." &
+       " Consider using a  which can be captured.") %
       [s.name.s, typeToString(s.typ), g.config$s.info])
-  elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags):
+  elif not (owner.typ.isClosure or owner.isNimcall and not owner.isExplicitCallConv or isEnv):
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
       [s.name.s, owner.name.s, $owner.typ.callConv])
   incl(owner.typ.flags, tfCapturesEnv)
-  owner.typ.callConv = ccClosure
+  if not isEnv:
+    owner.typ.callConv = ccClosure
 
 type
   DetectionPass = object
     processed, capturedVars: IntSet
     ownerToType: Table[int, PType]
     somethingToDo: bool
+    inTypeOf: bool
     graph: ModuleGraph
     idgen: IdGenerator
 
 proc initDetectionPass(g: ModuleGraph; fn: PSym; idgen: IdGenerator): DetectionPass =
-  result.processed = initIntSet()
-  result.capturedVars = initIntSet()
-  result.ownerToType = initTable[int, PType]()
-  result.processed.incl(fn.id)
-  result.graph = g
-  result.idgen = idgen
+  result = DetectionPass(processed: toIntSet([fn.id]),
+    capturedVars: initIntSet(), ownerToType: initTable[int, PType](),
+    graph: g, idgen: idgen
+  )
 
 discard """
 proc outer =
@@ -330,7 +335,7 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
                         info: TLineInfo): PType =
   result = c.ownerToType.getOrDefault(owner.id)
   if result.isNil:
-    result = newType(tyRef, nextTypeId(c.idgen), owner)
+    result = newType(tyRef, c.idgen, owner)
     let obj = createEnvObj(c.graph, c.idgen, owner, info)
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
@@ -338,7 +343,7 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
 proc asOwnedRef(c: var DetectionPass; t: PType): PType =
   if optOwnedRefs in c.graph.config.globalOptions:
     assert t.kind == tyRef
-    result = newType(tyOwned, nextTypeId(c.idgen), t.owner)
+    result = newType(tyOwned, c.idgen, t.owner)
     result.flags.incl tfHasOwned
     result.rawAddSon t
   else:
@@ -347,7 +352,7 @@ proc asOwnedRef(c: var DetectionPass; t: PType): PType =
 proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
                           info: TLineInfo): PType =
   var r = c.getEnvTypeForOwner(owner, info)
-  result = newType(tyPtr, nextTypeId(c.idgen), owner)
+  result = newType(tyPtr, c.idgen, owner)
   rawAddSon(result, r.skipTypes({tyOwned, tyRef, tyPtr}))
 
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
@@ -375,7 +380,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
       if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
         localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
-    let result = newSym(skField, upIdent, nextSymId(c.idgen), obj.owner, obj.owner.info)
+    let result = newSym(skField, upIdent, c.idgen, obj.owner, obj.owner.info)
     result.typ = fieldType
     when false:
       if c.graph.config.selectedGC == gcDestructors:
@@ -408,12 +413,15 @@ Consider:
 
 """
 
+proc isTypeOf(n: PNode): bool =
+  n.kind == nkSym and n.sym.magic in {mTypeOf, mType}
+
 proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) =
   var cp = getEnvParam(fn)
   let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner
   let t = c.getEnvTypeForOwner(owner, info)
   if cp == nil:
-    cp = newSym(skParam, getIdent(c.graph.cache, paramName), nextSymId(c.idgen), fn, fn.info)
+    cp = newSym(skParam, getIdent(c.graph.cache, paramName), c.idgen, fn, fn.info)
     incl(cp.flags, sfFromGeneric)
     cp.typ = t
     addHiddenParam(fn, cp)
@@ -436,24 +444,28 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, c.idgen, s, cache = true)
+        let body = transformBody(c.graph, c.idgen, s, {useCache})
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
+    let innerClosure = innerProc and s.typ.callConv == ccClosure and not s.isIterator
+    let interested = interestingVar(s)
     if ow == owner:
       if owner.isIterator:
         c.somethingToDo = true
         addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
-          if not c.capturedVars.containsOrIncl(s.id):
+          if not c.capturedVars.contains(s.id):
+            if not c.inTypeOf: c.capturedVars.incl(s.id)
             let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
             #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef, tyPtr})
 
             if s.name.id == getIdent(c.graph.cache, ":state").id:
+              obj.n[0].sym.flags.incl sfNoInit
               obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
             else:
-              addField(obj, s, c.graph.cache, c.idgen)
+              discard addField(obj, s, c.graph.cache, c.idgen)
     # direct or indirect dependency:
-    elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
+    elif innerClosure or interested:
       discard """
         proc outer() =
           var x: int
@@ -470,10 +482,12 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       addClosureParam(c, owner, n.info)
       #echo "capturing ", n.info
       # variable 's' is actually captured:
-      if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
-        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
-        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
-        addField(obj, s, c.graph.cache, c.idgen)
+      if interestingVar(s):
+        if not c.capturedVars.contains(s.id):
+          if not c.inTypeOf: c.capturedVars.incl(s.id)
+          let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
+          #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
+          discard addField(obj, s, c.graph.cache, c.idgen)
       # create required upFields:
       var w = owner.skipGenericOwner
       if isInnerProc(w) or owner.isIterator:
@@ -505,9 +519,14 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       detectCapturedVars(n[namePos], owner, c)
   of nkReturnStmt:
     detectCapturedVars(n[0], owner, c)
+  of nkIdentDefs:
+    detectCapturedVars(n[^1], owner, c)
   else:
+    if n.isCallExpr and n[0].isTypeOf:
+      c.inTypeOf = true
     for i in 0.. 0 and not em.endsInWhite and
+    if tsLeading in tok.spacing and not em.endsInWhite and
         (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
     wr(em, $tok.tokType, ltSomeParLe)
@@ -528,7 +533,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     wr(em, $tok.tokType, ltOther)
     if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
-    if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
+    if em.inquote or (tok.spacing == {} and
         tok.ident.s notin ["<", ">", "<=", ">=", "==", "!="]):
       # bug #9504: remember to not spacify a keyword:
       lastTokWasTerse = true
@@ -538,7 +543,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
       if not em.endsInWhite: wrSpace(em)
       wr(em, tok.ident.s, ltOpr)
       template isUnary(tok): bool =
-        tok.strongSpaceB == 0 and tok.strongSpaceA > 0
+        tok.spacing == {tsLeading}
 
       if not isUnary(tok):
         rememberSplit(splitBinary)
@@ -551,11 +556,15 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     if not preventComment:
       emitComment(em, tok, dontIndent = false)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
-    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
-    em.lineSpan = countNewlines(lit)
-    if em.lineSpan > 0: calcCol(em, lit)
-    wr em, lit, ltLit
+    if not em.inquote:
+      let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+      if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
+      em.lineSpan = countNewlines(lit)
+      if em.lineSpan > 0: calcCol(em, lit)
+      wr em, lit, ltLit
+    else:
+      if endsInAlpha(em): wrSpace(em)
+      wr em, tok.literal, ltLit
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal
diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index 729ba34352..ad5dd560c0 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -7,25 +7,30 @@
 #    distribution, for details about the copyright.
 #
 
-# This scanner is handwritten for efficiency. I used an elegant buffering
+# This lexer is handwritten for efficiency. I used an elegant buffering
 # scheme which I have not seen anywhere else:
 # We guarantee that a whole line is in the buffer. Thus only when scanning
 # the \n or \r character we have to check whether we need to read in the next
 # chunk. (\n or \r already need special handling for incrementing the line
-# counter; choosing both \n and \r allows the scanner to properly read Unix,
+# counter; choosing both \n and \r allows the lexer to properly read Unix,
 # DOS or Macintosh text files, even when it is not the native format.
 
 import
-  hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
-  wordrecg, lineinfos, pathutils, parseutils
+  options, msgs, platform, idents, nimlexbase, llstream,
+  wordrecg, lineinfos, pathutils
+
+import std/[hashes, parseutils, strutils]
+
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, formatfloat]
 
 const
-  MaxLineLength* = 80         # lines longer than this lead to a warning
   numChars*: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'}
   SymChars*: set[char] = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
   SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
   OpChars*: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
     '|', '=', '%', '&', '$', '@', '~', ':'}
+  UnaryMinusWhitelist = {' ', '\t', '\n', '\r', ',', ';', '(', '[', '{'}
 
 # don't forget to update the 'highlite' module if these charsets should change
 
@@ -51,26 +56,27 @@ type
     tkVar = "var", tkWhen = "when", tkWhile = "while", tkXor = "xor",
     tkYield = "yield", # end of keywords
 
-    tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit", 
+    tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit",
     tkInt32Lit = "tkInt32Lit", tkInt64Lit = "tkInt64Lit",
-    tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit", 
+    tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit",
     tkUInt32Lit = "tkUInt32Lit", tkUInt64Lit = "tkUInt64Lit",
     tkFloatLit = "tkFloatLit", tkFloat32Lit = "tkFloat32Lit",
     tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
     tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit",
-    tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", 
-    
+    tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit",
+    tkCustomLit = "tkCustomLit",
+
     tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
     tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}",
     tkBracketDotLe = "[.", tkBracketDotRi = ".]",
     tkCurlyDotLe = "{.", tkCurlyDotRi = ".}",
     tkParDotLe = "(.", tkParDotRi = ".)",
     tkComma = ",", tkSemiColon = ";",
-    tkColon = ":", tkColonColon = "::", tkEquals = "=", 
+    tkColon = ":", tkColonColon = "::", tkEquals = "=",
     tkDot = ".", tkDotDot = "..", tkBracketLeColon = "[:",
     tkOpr, tkComment, tkAccent = "`",
     # these are fake tokens used by renderer.nim
-    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
+    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr, tkHideableStart, tkHideableEnd
 
   TokTypes* = set[TokType]
 
@@ -88,19 +94,21 @@ type
                               # so that it is the correct default value
     base2, base8, base16
 
-  Token* = object             # a Nim token
-    tokType*: TokType         # the type of the token
-    indent*: int              # the indentation; != -1 if the token has been
-                              # preceded with indentation
-    ident*: PIdent            # the parsed identifier
-    iNumber*: BiggestInt      # the parsed integer literal
-    fNumber*: BiggestFloat    # the parsed floating point literal
-    base*: NumericalBase      # the numerical base; only valid for int
-                              # or float literals
-    strongSpaceA*: int8       # leading spaces of an operator
-    strongSpaceB*: int8       # trailing spaces of an operator
-    literal*: string          # the parsed (string) literal; and
-                              # documentation comments are here too
+  TokenSpacing* = enum
+    tsLeading, tsTrailing, tsEof
+
+  Token* = object                # a Nim token
+    tokType*: TokType            # the type of the token
+    base*: NumericalBase         # the numerical base; only valid for int
+                                 # or float literals
+    spacing*: set[TokenSpacing]  # spaces around token
+    indent*: int                 # the indentation; != -1 if the token has been
+                                 # preceded with indentation
+    ident*: PIdent               # the parsed identifier
+    iNumber*: BiggestInt         # the parsed integer literal
+    fNumber*: BiggestFloat       # the parsed floating point literal
+    literal*: string             # the parsed (string) literal; and
+                                 # documentation comments are here too
     line*, col*: int
     when defined(nimpretty):
       offsetA*, offsetB*: int # used for pretty printing so that literals
@@ -114,11 +122,12 @@ type
                               # this is needed because scanning comments
                               # needs so much look-ahead
     currLineIndent*: int
-    strongSpaces*, allowTabs*: bool
     errorHandler*: ErrorHandler
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
+      tokenEnd*: TLineInfo
+      previousTokenEnd*: TLineInfo
     config*: ConfigRef
 
 proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
@@ -140,9 +149,11 @@ proc isNimIdentifier*(s: string): bool =
     var i = 1
     while i < sLen:
       if s[i] == '_': inc(i)
-      if i < sLen and s[i] notin SymChars: return
+      if i < sLen and s[i] notin SymChars: return false
       inc(i)
     result = true
+  else:
+    result = false
 
 proc `$`*(tok: Token): string =
   case tok.tokType
@@ -161,34 +172,9 @@ proc prettyTok*(tok: Token): string =
   else: $tok
 
 proc printTok*(conf: ConfigRef; tok: Token) =
+  # xxx factor with toLocation
   msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" & $tok.tokType & " " & $tok)
 
-proc initToken*(L: var Token) =
-  L.tokType = tkInvalid
-  L.iNumber = 0
-  L.indent = 0
-  L.strongSpaceA = 0
-  L.literal = ""
-  L.fNumber = 0.0
-  L.base = base10
-  L.ident = nil
-  when defined(nimpretty):
-    L.commentOffsetA = 0
-    L.commentOffsetB = 0
-
-proc fillToken(L: var Token) =
-  L.tokType = tkInvalid
-  L.iNumber = 0
-  L.indent = 0
-  L.strongSpaceA = 0
-  setLen(L.literal, 0)
-  L.fNumber = 0.0
-  L.base = base10
-  L.ident = nil
-  when defined(nimpretty):
-    L.commentOffsetA = 0
-    L.commentOffsetB = 0
-
 proc openLexer*(lex: var Lexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
@@ -312,11 +298,9 @@ proc getNumber(L: var Lexer, result: var Token) =
 
   proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) =
     # Used to get slightly human friendlier err messages.
-    const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
-      'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
+    const literalishChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '.', '\''}
     var msgPos = L.bufpos
-    var t: Token
-    t.literal = ""
+    var t = Token(literal: "")
     L.bufpos = startpos # Use L.bufpos as pos because of matchChars
     matchChars(L, t, literalishChars)
     # We must verify +/- specifically so that we're not past the literal
@@ -325,15 +309,14 @@ proc getNumber(L: var Lexer, result: var Token) =
       t.literal.add(L.buf[L.bufpos])
       inc(L.bufpos)
       matchChars(L, t, literalishChars)
-    if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
-      inc(L.bufpos)
+    if L.buf[L.bufpos] in literalishChars:
       t.literal.add(L.buf[L.bufpos])
+      inc(L.bufpos)
       matchChars(L, t, {'0'..'9'})
     L.bufpos = msgPos
     lexMessage(L, msgKind, msg % t.literal)
 
   var
-    startpos, endpos: int
     xi: BiggestInt
     isBase10 = true
     numDigits = 0
@@ -345,8 +328,17 @@ proc getNumber(L: var Lexer, result: var Token) =
   result.tokType = tkIntLit   # int literal until we know better
   result.literal = ""
   result.base = base10
-  startpos = L.bufpos
-  tokenBegin(result, startpos)
+  tokenBegin(result, L.bufpos)
+
+  var isPositive = true
+  if L.buf[L.bufpos] == '-':
+    eatChar(L, result)
+    isPositive = false
+
+  let startpos = L.bufpos
+
+  template setNumber(field, value) =
+    field = (if isPositive: value else: -value)
 
   # First stage: find out base, make verifications, build token literal string
   # {'c', 'C'} is added for deprecation reasons to provide a clear error message
@@ -386,200 +378,184 @@ proc getNumber(L: var Lexer, result: var Token) =
       discard matchUnderscoreChars(L, result, {'0'..'9'})
     if L.buf[L.bufpos] in {'e', 'E'}:
       result.tokType = tkFloatLit
-      eatChar(L, result, 'e')
+      eatChar(L, result)
       if L.buf[L.bufpos] in {'+', '-'}:
         eatChar(L, result)
       discard matchUnderscoreChars(L, result, {'0'..'9'})
-  endpos = L.bufpos
+  let endpos = L.bufpos
 
   # Second stage, find out if there's a datatype suffix and handle it
   var postPos = endpos
+
   if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
+    let errPos = postPos
+    var customLitPossible = false
     if L.buf[postPos] == '\'':
       inc(postPos)
+      customLitPossible = true
 
-    case L.buf[postPos]
-    of 'f', 'F':
-      inc(postPos)
-      if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
-        result.tokType = tkFloat32Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
-        result.tokType = tkFloat64Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '1') and
-           (L.buf[postPos + 1] == '2') and
-           (L.buf[postPos + 2] == '8'):
-        result.tokType = tkFloat128Lit
-        inc(postPos, 3)
-      else:   # "f" alone defaults to float32
-        result.tokType = tkFloat32Lit
-    of 'd', 'D':  # ad hoc convenience shortcut for f64
-      inc(postPos)
-      result.tokType = tkFloat64Lit
-    of 'i', 'I':
-      inc(postPos)
-      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
-        result.tokType = tkInt64Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
-        result.tokType = tkInt32Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
-        result.tokType = tkInt16Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '8'):
-        result.tokType = tkInt8Lit
-        inc(postPos)
+    if L.buf[postPos] in SymChars:
+      var suffix = newStringOfCap(10)
+      while true:
+        suffix.add L.buf[postPos]
+        inc postPos
+        if L.buf[postPos] notin SymChars+{'_'}: break
+      let suffixAsLower = suffix.toLowerAscii
+      case suffixAsLower
+      of "f", "f32": result.tokType = tkFloat32Lit
+      of "d", "f64": result.tokType = tkFloat64Lit
+      of "f128": result.tokType = tkFloat128Lit
+      of "i8": result.tokType = tkInt8Lit
+      of "i16": result.tokType = tkInt16Lit
+      of "i32": result.tokType = tkInt32Lit
+      of "i64": result.tokType = tkInt64Lit
+      of "u": result.tokType = tkUIntLit
+      of "u8": result.tokType = tkUInt8Lit
+      of "u16": result.tokType = tkUInt16Lit
+      of "u32": result.tokType = tkUInt32Lit
+      of "u64": result.tokType = tkUInt64Lit
+      elif customLitPossible:
+        # remember the position of the `'` so that the parser doesn't
+        # have to reparse the custom literal:
+        result.iNumber = len(result.literal)
+        result.literal.add '\''
+        result.literal.add suffix
+        result.tokType = tkCustomLit
       else:
-        lexMessageLitNum(L, "invalid number: '$1'", startpos)
-    of 'u', 'U':
-      inc(postPos)
-      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
-        result.tokType = tkUInt64Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
-        result.tokType = tkUInt32Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
-        result.tokType = tkUInt16Lit
-        inc(postPos, 2)
-      elif (L.buf[postPos] == '8'):
-        result.tokType = tkUInt8Lit
-        inc(postPos)
-      else:
-        result.tokType = tkUIntLit
+        lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
     else:
-      lexMessageLitNum(L, "invalid number: '$1'", startpos)
+      lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
 
   # Is there still a literalish char awaiting? Then it's an error!
   if  L.buf[postPos] in literalishChars or
      (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
     lexMessageLitNum(L, "invalid number: '$1'", startpos)
 
-  # Third stage, extract actual number
-  L.bufpos = startpos            # restore position
-  var pos: int = startpos
-  try:
-    if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars):
-      inc(pos, 2)
-      xi = 0                  # it is a base prefix
+  if result.tokType != tkCustomLit:
+    # Third stage, extract actual number
+    L.bufpos = startpos            # restore position
+    var pos = startpos
+    try:
+      if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars):
+        inc(pos, 2)
+        xi = 0                  # it is a base prefix
 
-      case L.buf[pos - 1]
-      of 'b', 'B':
-        result.base = base2
-        while pos < endpos:
-          if L.buf[pos] != '_':
-            xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
-          inc(pos)
-      # 'c', 'C' is deprecated
-      of 'o', 'c', 'C':
-        result.base = base8
-        while pos < endpos:
-          if L.buf[pos] != '_':
-            xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
-          inc(pos)
-      of 'x', 'X':
-        result.base = base16
-        while pos < endpos:
-          case L.buf[pos]
-          of '_':
+        case L.buf[pos - 1]
+        of 'b', 'B':
+          result.base = base2
+          while pos < endpos:
+            if L.buf[pos] != '_':
+              xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
             inc(pos)
-          of '0'..'9':
-            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
+        # 'c', 'C' is deprecated (a warning is issued elsewhere)
+        of 'o', 'c', 'C':
+          result.base = base8
+          while pos < endpos:
+            if L.buf[pos] != '_':
+              xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
             inc(pos)
-          of 'a'..'f':
-            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
-            inc(pos)
-          of 'A'..'F':
-            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
-            inc(pos)
-          else:
-            break
+        of 'x', 'X':
+          result.base = base16
+          while pos < endpos:
+            case L.buf[pos]
+            of '_':
+              inc(pos)
+            of '0'..'9':
+              xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
+              inc(pos)
+            of 'a'..'f':
+              xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
+              inc(pos)
+            of 'A'..'F':
+              xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
+              inc(pos)
+            else:
+              break
+        else:
+          internalError(L.config, getLineInfo(L), "getNumber")
+
+        case result.tokType
+        of tkIntLit, tkInt64Lit: setNumber result.iNumber, xi
+        of tkInt8Lit: setNumber result.iNumber, ashr(xi shl 56, 56)
+        of tkInt16Lit: setNumber result.iNumber, ashr(xi shl 48, 48)
+        of tkInt32Lit: setNumber result.iNumber, ashr(xi shl 32, 32)
+        of tkUIntLit, tkUInt64Lit: setNumber result.iNumber, xi
+        of tkUInt8Lit: setNumber result.iNumber, xi and 0xff
+        of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff
+        of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff
+        of tkFloat32Lit:
+          setNumber result.fNumber, (cast[ptr float32](addr(xi)))[]
+          # note: this code is endian neutral!
+          # XXX: Test this on big endian machine!
+        of tkFloat64Lit, tkFloatLit:
+          setNumber result.fNumber, (cast[ptr float64](addr(xi)))[]
+        else: internalError(L.config, getLineInfo(L), "getNumber")
+
+        # Bounds checks. Non decimal literals are allowed to overflow the range of
+        # the datatype as long as their pattern don't overflow _bitwise_, hence
+        # below checks of signed sizes against uint*.high is deliberate:
+        # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
+        if result.tokType notin floatTypes:
+          let outOfRange =
+            case result.tokType
+            of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi
+            of tkInt8Lit:  (xi > BiggestInt(uint8.high))
+            of tkInt16Lit: (xi > BiggestInt(uint16.high))
+            of tkInt32Lit: (xi > BiggestInt(uint32.high))
+            else: false
+
+          if outOfRange:
+            #echo "out of range num: ", result.iNumber, " vs ", xi
+            lexMessageLitNum(L, "number out of range: '$1'", startpos)
+
       else:
-        internalError(L.config, getLineInfo(L), "getNumber")
+        case result.tokType
+        of floatTypes:
+          result.fNumber = parseFloat(result.literal)
+        of tkUInt64Lit, tkUIntLit:
+          var iNumber: uint64 = uint64(0)
+          var len: int = 0
+          try:
+            len = parseBiggestUInt(result.literal, iNumber)
+          except ValueError:
+            raise newException(OverflowDefect, "number out of range: " & result.literal)
+          if len != result.literal.len:
+            raise newException(ValueError, "invalid integer: " & result.literal)
+          result.iNumber = cast[int64](iNumber)
+        else:
+          var iNumber: int64 = int64(0)
+          var len: int = 0
+          try:
+            len = parseBiggestInt(result.literal, iNumber)
+          except ValueError:
+            raise newException(OverflowDefect, "number out of range: " & result.literal)
+          if len != result.literal.len:
+            raise newException(ValueError, "invalid integer: " & result.literal)
+          result.iNumber = iNumber
 
-      case result.tokType
-      of tkIntLit, tkInt64Lit: result.iNumber = xi
-      of tkInt8Lit: result.iNumber = ashr(xi shl 56, 56)
-      of tkInt16Lit: result.iNumber = ashr(xi shl 48, 48)
-      of tkInt32Lit: result.iNumber = ashr(xi shl 32, 32)
-      of tkUIntLit, tkUInt64Lit: result.iNumber = xi
-      of tkUInt8Lit: result.iNumber = xi and 0xff
-      of tkUInt16Lit: result.iNumber = xi and 0xffff
-      of tkUInt32Lit: result.iNumber = xi and 0xffffffff
-      of tkFloat32Lit:
-        result.fNumber = (cast[PFloat32](addr(xi)))[]
-        # note: this code is endian neutral!
-        # XXX: Test this on big endian machine!
-      of tkFloat64Lit, tkFloatLit:
-        result.fNumber = (cast[PFloat64](addr(xi)))[]
-      else: internalError(L.config, getLineInfo(L), "getNumber")
-
-      # Bounds checks. Non decimal literals are allowed to overflow the range of
-      # the datatype as long as their pattern don't overflow _bitwise_, hence
-      # below checks of signed sizes against uint*.high is deliberate:
-      # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
-      if result.tokType notin floatTypes:
-        let outOfRange = case result.tokType:
-        of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi
-        of tkInt8Lit: (xi > BiggestInt(uint8.high))
-        of tkInt16Lit: (xi > BiggestInt(uint16.high))
-        of tkInt32Lit: (xi > BiggestInt(uint32.high))
-        else: false
+        # Explicit bounds checks.
+        let outOfRange =
+          case result.tokType
+          of tkInt8Lit: result.iNumber > int8.high or result.iNumber < int8.low
+          of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) or result.iNumber < 0
+          of tkInt16Lit: result.iNumber > int16.high or result.iNumber < int16.low
+          of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) or result.iNumber < 0
+          of tkInt32Lit: result.iNumber > int32.high or result.iNumber < int32.low
+          of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) or result.iNumber < 0
+          else: false
 
         if outOfRange:
-          #echo "out of range num: ", result.iNumber, " vs ", xi
           lexMessageLitNum(L, "number out of range: '$1'", startpos)
 
-    else:
-      case result.tokType
-      of floatTypes:
-        result.fNumber = parseFloat(result.literal)
-      of tkUInt64Lit, tkUIntLit:
-        var iNumber: uint64
-        var len: int
-        try:
-          len = parseBiggestUInt(result.literal, iNumber)
-        except ValueError:
-          raise newException(OverflowDefect, "number out of range: " & $result.literal)
-        if len != result.literal.len:
-          raise newException(ValueError, "invalid integer: " & $result.literal)
-        result.iNumber = cast[int64](iNumber)
-      else:
-        var iNumber: int64
-        var len: int
-        try:
-          len = parseBiggestInt(result.literal, iNumber)
-        except ValueError:
-          raise newException(OverflowDefect, "number out of range: " & $result.literal)
-        if len != result.literal.len:
-          raise newException(ValueError, "invalid integer: " & $result.literal)
-        result.iNumber = iNumber
+      # Promote int literal to int64? Not always necessary, but more consistent
+      if result.tokType == tkIntLit:
+        if result.iNumber > high(int32) or result.iNumber < low(int32):
+          result.tokType = tkInt64Lit
 
-      # Explicit bounds checks. Only T.high needs to be considered
-      # since result.iNumber can't be negative.
-      let outOfRange =
-        case result.tokType
-        of tkInt8Lit: result.iNumber > int8.high
-        of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high)
-        of tkInt16Lit: result.iNumber > int16.high
-        of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high)
-        of tkInt32Lit: result.iNumber > int32.high
-        of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high)
-        else: false
-
-      if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos)
-
-    # Promote int literal to int64? Not always necessary, but more consistent
-    if result.tokType == tkIntLit:
-      if result.iNumber > high(int32):
-        result.tokType = tkInt64Lit
-
-  except ValueError:
-    lexMessageLitNum(L, "invalid number: '$1'", startpos)
-  except OverflowDefect, RangeDefect:
-    lexMessageLitNum(L, "number out of range: '$1'", startpos)
+    except ValueError:
+      lexMessageLitNum(L, "invalid number: '$1'", startpos)
+    except OverflowDefect, RangeDefect:
+      lexMessageLitNum(L, "number out of range: '$1'", startpos)
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
@@ -735,10 +711,6 @@ proc handleCRLF(L: var Lexer, pos: int): int =
   template registerLine =
     let col = L.getColNumber(pos)
 
-    when not defined(nimpretty):
-      if col > MaxLineLength:
-        lexMessagePos(L, hintLineTooLong, pos)
-
   case L.buf[pos]
   of CR:
     registerLine()
@@ -798,7 +770,7 @@ proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
     if mode != normal: tok.tokType = tkRStrLit
     else: tok.tokType = tkStrLit
     while true:
-      var c = L.buf[pos]
+      let c = L.buf[pos]
       if c == '\"':
         if mode != normal and L.buf[pos+1] == '\"':
           inc(pos, 2)
@@ -820,10 +792,11 @@ proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
         inc(pos)
     L.bufpos = pos
 
-proc getCharacter(L: var Lexer, tok: var Token) =
+proc getCharacter(L: var Lexer; tok: var Token) =
   tokenBegin(tok, L.bufpos)
+  let startPos = L.bufpos
   inc(L.bufpos)               # skip '
-  var c = L.buf[L.bufpos]
+  let c = L.buf[L.bufpos]
   case c
   of '\0'..pred(' '), '\'':
     lexMessage(L, errGenerated, "invalid character literal")
@@ -832,10 +805,59 @@ proc getCharacter(L: var Lexer, tok: var Token) =
   else:
     tok.literal = $c
     inc(L.bufpos)
-  if L.buf[L.bufpos] != '\'':
-    lexMessage(L, errGenerated, "missing closing ' for character literal")
-  tokenEndIgnore(tok, L.bufpos)
-  inc(L.bufpos)               # skip '
+  if L.buf[L.bufpos] == '\'':
+    tokenEndIgnore(tok, L.bufpos)
+    inc(L.bufpos)               # skip '
+  else:
+    if startPos > 0 and L.buf[startPos-1] == '`':
+      tok.literal = "'"
+      L.bufpos = startPos+1
+    else:
+      lexMessage(L, errGenerated, "missing closing ' for character literal")
+    tokenEndIgnore(tok, L.bufpos)
+
+const
+  UnicodeOperatorStartChars = {'\226', '\194', '\195'}
+    # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔")
+    # all start with one of these.
+
+type
+  UnicodeOprPred = enum
+    Mul, Add
+
+proc unicodeOprLen(buf: cstring; pos: int): (int8, UnicodeOprPred) =
+  template m(len): untyped = (int8(len), Mul)
+  template a(len): untyped = (int8(len), Add)
+  result = 0.m
+  case buf[pos]
+  of '\226':
+    if buf[pos+1] == '\136':
+      if buf[pos+2] == '\152': result = 3.m # ∘
+      elif buf[pos+2] == '\153': result = 3.m # ∙
+      elif buf[pos+2] == '\167': result = 3.m # ∧
+      elif buf[pos+2] == '\168': result = 3.a # ∨
+      elif buf[pos+2] == '\169': result = 3.m # ∩
+      elif buf[pos+2] == '\170': result = 3.a # ∪
+    elif buf[pos+1] == '\138':
+      if buf[pos+2] == '\147': result = 3.m # ⊓
+      elif buf[pos+2] == '\148': result = 3.a # ⊔
+      elif buf[pos+2] == '\149': result = 3.a # ⊕
+      elif buf[pos+2] == '\150': result = 3.a # ⊖
+      elif buf[pos+2] == '\151': result = 3.m # ⊗
+      elif buf[pos+2] == '\152': result = 3.m # ⊘
+      elif buf[pos+2] == '\153': result = 3.m # ⊙
+      elif buf[pos+2] == '\155': result = 3.m # ⊛
+      elif buf[pos+2] == '\158': result = 3.a # ⊞
+      elif buf[pos+2] == '\159': result = 3.a # ⊟
+      elif buf[pos+2] == '\160': result = 3.m # ⊠
+      elif buf[pos+2] == '\161': result = 3.m # ⊡
+    elif buf[pos+1] == '\152' and buf[pos+2] == '\133': result = 3.m # ★
+  of '\194':
+    if buf[pos+1] == '\177': result = 2.a # ±
+  of '\195':
+    if buf[pos+1] == '\151': result = 2.m # ×
+  else:
+    discard
 
 proc getSymbol(L: var Lexer, tok: var Token) =
   var h: Hash = 0
@@ -845,7 +867,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
   while true:
     var c = L.buf[pos]
     case c
-    of 'a'..'z', '0'..'9', '\x80'..'\xFF':
+    of 'a'..'z', '0'..'9':
       h = h !& ord(c)
       inc(pos)
     of 'A'..'Z':
@@ -859,10 +881,16 @@ proc getSymbol(L: var Lexer, tok: var Token) =
         break
       inc(pos)
       suspicious = true
+    of '\x80'..'\xFF':
+      if c in UnicodeOperatorStartChars and unicodeOprLen(L.buf, pos)[0] != 0:
+        break
+      else:
+        h = h !& ord(c)
+        inc(pos)
     else: break
   tokenEnd(tok, pos-1)
   h = !$h
-  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
   if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
@@ -876,7 +904,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
 proc endOperator(L: var Lexer, tok: var Token, pos: int,
                  hash: Hash) {.inline.} =
   var h = !$hash
-  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
   else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon))
   L.bufpos = pos
@@ -886,43 +914,66 @@ proc getOperator(L: var Lexer, tok: var Token) =
   tokenBegin(tok, pos)
   var h: Hash = 0
   while true:
-    var c = L.buf[pos]
-    if c notin OpChars: break
-    h = h !& ord(c)
-    inc(pos)
+    let c = L.buf[pos]
+    if c in OpChars:
+      h = h !& ord(c)
+      inc(pos)
+    elif c in UnicodeOperatorStartChars:
+      let oprLen = unicodeOprLen(L.buf, pos)[0]
+      if oprLen == 0: break
+      for i in 0.. 1 and tok.ident.s[^1] == '>' and
-      tok.ident.s[^2] in {'-', '~', '='}: return 1
+      tok.ident.s[^2] in {'-', '~', '='}: return 0
 
     template considerAsgn(value: untyped) =
       result = if tok.ident.s[^1] == '=': 1 else: value
 
     case relevantChar
     of '$', '^': considerAsgn(10)
-    of '*', '%', '/', '\\': considerAsgn(9)
+    of '*', '%', '/', '\\': considerAsgn(MulPred)
     of '~': result = 8
-    of '+', '-', '|': considerAsgn(8)
+    of '+', '-', '|': considerAsgn(PlusPred)
     of '&': considerAsgn(7)
     of '=', '<', '>', '!': result = 5
     of '.': considerAsgn(6)
     of '?': result = 2
+    of UnicodeOperatorStartChars:
+      if tok.ident.s[^1] == '=':
+        result = 1
+      else:
+        let (len, pred) = unicodeOprLen(cstring(tok.ident.s), 0)
+        if len != 0:
+          result = if pred == Mul: MulPred else: PlusPred
+        else:
+          result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
   of tkDotDot: result = 6
@@ -931,22 +982,6 @@ proc getPrecedence*(tok: Token): int =
   of tkOr, tkXor, tkPtr, tkRef: result = 3
   else: return -10
 
-proc newlineFollows*(L: Lexer): bool =
-  var pos = L.bufpos
-  while true:
-    case L.buf[pos]
-    of ' ', '\t':
-      inc(pos)
-    of CR, LF:
-      result = true
-      break
-    of '#':
-      inc(pos)
-      if L.buf[pos] == '#': inc(pos)
-      if L.buf[pos] != '[': return true
-    else:
-      break
-
 proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
                           isDoc: bool) =
   var pos = start
@@ -998,7 +1033,6 @@ proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
       when defined(nimpretty): tok.literal.add "\L"
       if isDoc:
         when not defined(nimpretty): tok.literal.add "\n"
-        inc tok.iNumber
         var c = toStrip
         while L.buf[pos] == ' ' and c > 0:
           inc pos
@@ -1017,8 +1051,6 @@ proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
 proc scanComment(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tok.tokType = tkComment
-  # iNumber contains the number of '\n' in the token
-  tok.iNumber = 0
   assert L.buf[pos+1] == '#'
   when defined(nimpretty):
     tok.commentOffsetA = L.offsetBase + pos
@@ -1041,9 +1073,7 @@ proc scanComment(L: var Lexer, tok: var Token) =
         toStrip = 0
       else:  # found first non-whitespace character
         stripInit = true
-    var lastBackslash = -1
     while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
-      if L.buf[pos] == '\\': lastBackslash = pos+1
       tok.literal.add(L.buf[pos])
       inc(pos)
     tokenEndIgnore(tok, pos)
@@ -1061,7 +1091,6 @@ proc scanComment(L: var Lexer, tok: var Token) =
         while L.buf[pos] == ' ' and c > 0:
           inc pos
           dec c
-        inc tok.iNumber
     else:
       if L.buf[pos] > ' ':
         L.indentAhead = indent
@@ -1074,7 +1103,7 @@ proc scanComment(L: var Lexer, tok: var Token) =
 proc skip(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
-  tok.strongSpaceA = 0
+  tok.spacing.excl(tsLeading)
   when defined(nimpretty):
     var hasComment = false
     var commentIndent = L.currLineIndent
@@ -1085,9 +1114,9 @@ proc skip(L: var Lexer, tok: var Token) =
     case L.buf[pos]
     of ' ':
       inc(pos)
-      inc(tok.strongSpaceA)
+      tok.spacing.incl(tsLeading)
     of '\t':
-      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
+      lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
       inc(pos)
     of CR, LF:
       tokenEndPrevious(tok, pos)
@@ -1107,7 +1136,7 @@ proc skip(L: var Lexer, tok: var Token) =
           pos = L.bufpos
         else:
           break
-      tok.strongSpaceA = 0
+      tok.spacing.excl(tsLeading)
       when defined(nimpretty):
         if L.buf[pos] == '#' and tok.line < 0: commentIndent = indent
       if L.buf[pos] > ' ' and (L.buf[pos] != '#' or L.buf[pos+1] == '#'):
@@ -1146,12 +1175,16 @@ proc skip(L: var Lexer, tok: var Token) =
 proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
+      L.previousTokenEnd.line = L.tokenEnd.line
+      L.previousTokenEnd.col = L.tokenEnd.col
+      L.tokenEnd.line = tok.line.uint16
+      L.tokenEnd.col = getColNumber(L, L.bufpos).int16
       # we attach the cursor to the last *strong* token
       if tok.tokType notin weakTokens:
         L.previousToken.line = tok.line.uint16
         L.previousToken.col = tok.col.int16
 
-  fillToken(tok)
+  reset(tok)
   if L.indentAhead >= 0:
     tok.indent = L.indentAhead
     L.currLineIndent = L.indentAhead
@@ -1163,13 +1196,18 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
     if tok.tokType == tkComment:
       L.indentAhead = L.currLineIndent
       return
-  var c = L.buf[L.bufpos]
+  let c = L.buf[L.bufpos]
   tok.line = L.lineNumber
   tok.col = getColNumber(L, L.bufpos)
-  if c in SymStartChars - {'r', 'R'}:
+  if c in SymStartChars - {'r', 'R'} - UnicodeOperatorStartChars:
     getSymbol(L, tok)
   else:
     case c
+    of UnicodeOperatorStartChars:
+      if unicodeOprLen(L.buf, L.bufpos)[0] != 0:
+        getOperator(L, tok)
+      else:
+        getSymbol(L, tok)
     of '#':
       scanComment(L, tok)
     of '*':
@@ -1277,7 +1315,28 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
       getNumber(L, tok)
       let c = L.buf[L.bufpos]
       if c in SymChars+{'_'}:
-        lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
+        if c in UnicodeOperatorStartChars and
+            unicodeOprLen(L.buf, L.bufpos)[0] != 0:
+          discard
+        else:
+          lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
+    of '-':
+      if L.buf[L.bufpos+1] in {'0'..'9'} and
+          (L.bufpos-1 == 0 or L.buf[L.bufpos-1] in UnaryMinusWhitelist):
+        # x)-23 # binary minus
+        # ,-23  # unary minus
+        # \n-78 # unary minus? Yes.
+        # =-3   # parsed as `=-` anyway
+        getNumber(L, tok)
+        let c = L.buf[L.bufpos]
+        if c in SymChars+{'_'}:
+          if c in UnicodeOperatorStartChars and
+              unicodeOprLen(L.buf, L.bufpos)[0] != 0:
+            discard
+          else:
+            lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
+      else:
+        getOperator(L, tok)
     else:
       if c in OpChars:
         getOperator(L, tok)
@@ -1293,9 +1352,9 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
 
 proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
                      cache: IdentCache; config: ConfigRef): int =
-  var lex: Lexer
-  var tok: Token
-  initToken(tok)
+  result = 0
+  var lex: Lexer = default(Lexer)
+  var tok: Token = default(Token)
   openLexer(lex, fileIdx, inputstream, cache, config)
   var prevToken = tkEof
   while tok.tokType != tkEof:
@@ -1308,11 +1367,11 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
 
 proc getPrecedence*(ident: PIdent): int =
   ## assumes ident is binary operator already
-  var tok: Token
-  initToken(tok)
-  tok.ident = ident
-  tok.tokType =
-    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
-      TokType(tok.ident.id + ord(tkSymbol))
-    else: tkOpr
+  let
+    tokType =
+      if ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
+        TokType(ident.id + ord(tkSymbol))
+      else: tkOpr
+    tok = Token(ident: ident, tokType: tokType)
+
   getPrecedence(tok)
diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim
index 980e77e4be..f62927cec7 100644
--- a/compiler/liftdestructors.nim
+++ b/compiler/liftdestructors.nim
@@ -8,13 +8,17 @@
 #
 
 ## This module implements lifting for type-bound operations
-## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
+## (`=sink`, `=copy`, `=destroy`, `=deepCopy`, `=wasMoved`, `=dup`).
 
 import modulegraphs, lineinfos, idents, ast, renderer, semdata,
-  sighashes, lowerings, options, types, msgs, magicsys, tables
+  sighashes, lowerings, options, types, msgs, magicsys, ccgutils
 
+import std/tables
 from trees import isCaseObj
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 type
   TLiftCtx = object
     g: ModuleGraph
@@ -31,11 +35,12 @@ type
 
 template destructor*(t: PType): PSym = getAttachedOp(c.g, t, attachedDestructor)
 template assignment*(t: PType): PSym = getAttachedOp(c.g, t, attachedAsgn)
+template dup*(t: PType): PSym = getAttachedOp(c.g, t, attachedDup)
 template asink*(t: PType): PSym = getAttachedOp(c.g, t, attachedSink)
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
 proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo; idgen: IdGenerator): PSym
+              info: TLineInfo; idgen: IdGenerator; isDistinct = false): PSym
 
 proc createTypeBoundOps*(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo;
                          idgen: IdGenerator)
@@ -46,15 +51,15 @@ proc at(a, i: PNode, elemType: PType): PNode =
   result[1] = i
   result.typ = elemType
 
-proc destructorOverriden(g: ModuleGraph; t: PType): bool =
+proc destructorOverridden(g: ModuleGraph; t: PType): bool =
   let op = getAttachedOp(g, t, attachedDestructor)
-  op != nil and sfOverriden in op.flags
+  op != nil and sfOverridden in op.flags
 
 proc fillBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  for i in 0.. 0 and t[0] != nil:
-    fillBody(c, skipTypes(t[0], abstractPtrs), body, x, y)
+  if t.baseClass != nil:
+    let obj = newNodeIT(nkHiddenSubConv, c.info, t.baseClass)
+    obj.add newNodeI(nkEmpty, c.info)
+    obj.add x
+    fillBody(c, skipTypes(t.baseClass, abstractPtrs), body, obj, y)
   fillBodyObj(c, t.n, body, x, y, enforceDefaultOp = false)
 
 proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
   var hasCase = isCaseObj(t.n)
   var obj = t
-  while obj.len > 0 and obj[0] != nil:
-    obj = skipTypes(obj[0], abstractPtrs)
+  while obj.baseClass != nil:
+    obj = skipTypes(obj.baseClass, abstractPtrs)
     hasCase = hasCase or isCaseObj(obj.n)
 
   if hasCase and c.kind in {attachedAsgn, attachedDeepCopy}:
@@ -230,7 +250,16 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     # for every field (dependent on dest.kind):
     #   `=` dest.field, src.field
     # =destroy(blob)
-    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
+    var dummy = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
+    dummy.typ = y.typ
+    if ccgIntroducedPtr(c.g.config, dummy, y.typ):
+      # Because of potential aliasing when the src param is passed by ref, we need to check for equality here,
+      # because the wasMoved(dest) call would zero out src, if dest aliases src.
+      var cond = newTree(nkCall, newSymNode(c.g.getSysMagic(c.info, "==", mEqRef)),
+        newTreeIT(nkAddr, c.info, makePtrType(c.fn, x.typ, c.idgen), x), newTreeIT(nkAddr, c.info, makePtrType(c.fn, y.typ, c.idgen), y))
+      cond.typ = getSysType(c.g, x.info, tyBool)
+      body.add genIf(c, cond, newTreeI(nkReturnStmt, c.info, newNodeI(nkEmpty, c.info)))
+    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
     temp.typ = x.typ
     incl(temp.flags, sfFromGeneric)
     var v = newNodeI(nkVarSection, c.info)
@@ -240,7 +269,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     #body.add newAsgnStmt(blob, x)
 
     var wasMovedCall = newNodeI(nkCall, c.info)
-    wasMovedCall.add(newSymNode(createMagic(c.g, c.idgen, "wasMoved", mWasMoved)))
+    wasMovedCall.add(newSymNode(createMagic(c.g, c.idgen, "`=wasMoved`", mWasMoved)))
     wasMovedCall.add x # mWasMoved does not take the address
     body.add wasMovedCall
 
@@ -261,8 +290,8 @@ proc boolLit*(g: ModuleGraph; info: TLineInfo; value: bool): PNode =
   result.typ = getSysType(g, info, tyBool)
 
 proc getCycleParam(c: TLiftCtx): PNode =
-  assert c.kind == attachedAsgn
-  if c.fn.typ.len == 4:
+  assert c.kind in {attachedAsgn, attachedDup}
+  if c.fn.typ.len == 3 + ord(c.kind == attachedAsgn):
     result = c.fn.typ.n.lastSon
     assert result.kind == nkSym
     assert result.sym.name.s == "cyclic"
@@ -276,30 +305,41 @@ proc newHookCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
   result.add newSymNode(op)
   if sfNeverRaises notin op.flags:
     c.canRaise = true
-  if op.typ.sons[1].kind == tyVar:
+  if op.typ.firstParamType.kind == tyVar:
     result.add genAddr(c, x)
   else:
     result.add x
   if y != nil:
     result.add y
-  if op.typ.len == 4:
+  if op.typ.signatureLen == 4:
     assert y != nil
-    if c.fn.typ.len == 4:
+    if c.fn.typ.signatureLen == 4:
       result.add getCycleParam(c)
     else:
       # assume the worst: A cycle is created:
       result.add boolLit(c.g, y.info, true)
 
 proc newOpCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
-  result = newNodeIT(nkCall, x.info, op.typ[0])
+  result = newNodeIT(nkCall, x.info, op.typ.returnType)
   result.add(newSymNode(op))
   result.add x
   if sfNeverRaises notin op.flags:
     c.canRaise = true
 
+  if c.kind == attachedDup and op.typ.len == 3:
+    assert x != nil
+    if c.fn.typ.len == 3:
+      result.add getCycleParam(c)
+    else:
+      # assume the worst: A cycle is created:
+      result.add boolLit(c.g, x.info, true)
+
 proc newDeepCopyCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
   result = newAsgnStmt(x, newOpCall(c, op, y))
 
+proc newDupCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
+  result = newAsgnStmt(x, newOpCall(c, op, y))
+
 proc usesBuiltinArc(t: PType): bool =
   proc wrap(t: PType): bool {.nimcall.} = ast.isGCedMem(t)
   result = types.searchTypeFor(t, wrap)
@@ -324,9 +364,9 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
                         field: var PSym): bool =
   if optSeqDestructors in c.g.config.globalOptions:
     var op = field
-    let destructorOverriden = destructorOverriden(c.g, t)
+    let destructorOverridden = destructorOverridden(c.g, t)
     if op != nil and op != c.fn and
-        (sfOverriden in op.flags or destructorOverriden):
+        (sfOverridden in op.flags or destructorOverridden):
       if sfError in op.flags:
         incl c.fn.flags, sfError
       #else:
@@ -334,10 +374,12 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
       onUse(c.info, op)
       body.add newHookCall(c, op, x, y)
       result = true
-    elif op == nil and destructorOverriden:
+    elif op == nil and destructorOverridden:
       op = produceSym(c.g, c.c, t, c.kind, c.info, c.idgen)
       body.add newHookCall(c, op, x, y)
       result = true
+    else:
+      result = false
   elif tfHasAsgn in t.flags:
     var op: PSym
     if sameType(t, c.asgnForType):
@@ -367,12 +409,14 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
     assert op.ast[genericParamsPos].kind == nkEmpty
     body.add newHookCall(c, op, x, y)
     result = true
+  else:
+    result = false
 
 proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
   let t = orig.skipTypes(abstractInst - {tyDistinct})
   var op = t.destructor
 
-  if op != nil and sfOverriden in op.flags:
+  if op != nil and sfOverridden in op.flags:
     if op.ast.isGenericRoutine:
       # patch generic destructor:
       op = instantiateGeneric(c, op, t, t.typeInst)
@@ -395,7 +439,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
   case c.kind
   of attachedDestructor:
     var op = t.destructor
-    if op != nil and sfOverriden in op.flags:
+    if op != nil and sfOverridden in op.flags:
 
       if op.ast.isGenericRoutine:
         # patch generic destructor:
@@ -406,17 +450,21 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       onUse(c.info, op)
       body.add destructorCall(c, op, x)
       result = true
+    else:
+      result = false
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
     var op = getAttachedOp(c.g, t, c.kind)
+    if op != nil and sfOverridden in op.flags:
+      if op.ast.isGenericRoutine:
+        # patch generic =trace:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
     result = considerAsgnOrSink(c, t, body, x, y, op)
     if op != nil:
       setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
-  of attachedDispose:
-    var op = getAttachedOp(c.g, t, c.kind)
-    result = considerAsgnOrSink(c, t, body, x, nil, op)
-    if op != nil:
-      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
   of attachedDeepCopy:
     let op = getAttachedOp(c.g, t, attachedDeepCopy)
     if op != nil:
@@ -424,9 +472,43 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       onUse(c.info, op)
       body.add newDeepCopyCall(c, op, x, y)
       result = true
+    else:
+      result = false
+
+  of attachedWasMoved:
+    var op = getAttachedOp(c.g, t, attachedWasMoved)
+    if op != nil and sfOverridden in op.flags:
+
+      if op.ast.isGenericRoutine:
+        # patch generic destructor:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, attachedWasMoved, op)
+
+      #markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add genWasMovedCall(c, op, x)
+      result = true
+    else:
+      result = false
+
+  of attachedDup:
+    var op = getAttachedOp(c.g, t, attachedDup)
+    if op != nil and sfOverridden in op.flags:
+
+      if op.ast.isGenericRoutine:
+        # patch generic destructor:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, attachedDup, op)
+
+      #markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add newDupCall(c, op, x, y)
+      result = true
+    else:
+      result = false
 
 proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
   temp.typ = getSysType(c.g, body.info, tyInt)
   incl(temp.flags, sfFromGeneric)
 
@@ -436,7 +518,7 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   body.add v
 
 proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
   temp.typ = value.typ
   incl(temp.flags, sfFromGeneric)
 
@@ -471,22 +553,40 @@ proc setLenSeqCall(c: var TLiftCtx; t: PType; x, y: PNode): PNode =
   result = newTree(nkCall, newSymNode(op, x.info), x, lenCall)
 
 proc forallElements(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  let counterIdx = body.len
   let i = declareCounter(c, body, toInt64(firstOrd(c.g.config, t)))
   let whileLoop = genWhileLoop(c, i, x)
-  let elemType = t.lastSon
+  let elemType = t.elementType
   let b = if c.kind == attachedTrace: y else: y.at(i, elemType)
   fillBody(c, elemType, whileLoop[1], x.at(i, elemType), b)
-  addIncStmt(c, whileLoop[1], i)
-  body.add whileLoop
+  if whileLoop[1].len > 0:
+    addIncStmt(c, whileLoop[1], i)
+    body.add whileLoop
+  else:
+    body.sons.setLen counterIdx
+
+proc checkSelfAssignment(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  var cond = callCodegenProc(c.g, "sameSeqPayload", c.info,
+      newTreeIT(nkAddr, c.info, makePtrType(c.fn, x.typ, c.idgen), x),
+      newTreeIT(nkAddr, c.info, makePtrType(c.fn, y.typ, c.idgen), y)
+      )
+  cond.typ = getSysType(c.g, c.info, tyBool)
+  body.add genIf(c, cond, newTreeI(nkReturnStmt, c.info, newNodeI(nkEmpty, c.info)))
 
 proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
+  of attachedDup:
+    body.add setLenSeqCall(c, t, x, y)
+    forallElements(c, t, body, x, y)
   of attachedAsgn, attachedDeepCopy:
     # we generate:
+    # if x.p == y.p:
+    #   return
     # setLen(dest, y.len)
     # var i = 0
     # while i < y.len: dest[i] = y[i]; inc(i)
     # This is usually more efficient than a destroy/create pair.
+    checkSelfAssignment(c, t, body, x, y)
     body.add setLenSeqCall(c, t, x, y)
     forallElements(c, t, body, x, y)
   of attachedSink:
@@ -500,19 +600,18 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     forallElements(c, t, body, x, y)
     body.add genBuiltin(c, mDestroy, "destroy", x)
   of attachedTrace:
-    # follow all elements:
-    forallElements(c, t, body, x, y)
-  of attachedDispose:
-    forallElements(c, t, body, x, y)
-    body.add genBuiltin(c, mDestroy, "destroy", x)
+    if canFormAcycle(c.g, t.elemType):
+      # follow all elements:
+      forallElements(c, t, body, x, y)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   createTypeBoundOps(c.g, c.c, t, body.info, c.idgen)
   # recursions are tricky, so we might need to forward the generated
   # operation here:
   var t = t
-  if t.assignment == nil or t.destructor == nil:
-    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
+  if t.assignment == nil or t.destructor == nil or t.dup == nil:
+    let h = sighashes.hashType(t,c.g.config, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.g.canonTypes.getOrDefault(h)
     if canon != nil: t = canon
 
@@ -537,19 +636,22 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     doAssert t.destructor != nil
     body.add destructorCall(c, t.destructor, x)
   of attachedTrace:
+    if t.kind != tyString and canFormAcycle(c.g, t.elemType):
+      let op = getAttachedOp(c.g, t, c.kind)
+      if op == nil:
+        return # protect from recursion
+      body.add newHookCall(c, op, x, y)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
+  of attachedDup:
+    # XXX: replace these with assertions.
     let op = getAttachedOp(c.g, t, c.kind)
     if op == nil:
       return # protect from recursion
-    body.add newHookCall(c, op, x, y)
-  of attachedDispose:
-    let op = getAttachedOp(c.g, t, c.kind)
-    if op == nil:
-      return # protect from recursion
-    body.add newHookCall(c, op, x, nil)
+    body.add newDupCall(c, op, x, y)
 
 proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
-  of attachedAsgn, attachedDeepCopy:
+  of attachedAsgn, attachedDeepCopy, attachedDup:
     body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c, x), y)
   of attachedSink:
     let moveCall = genBuiltin(c, mMove, "move", x)
@@ -557,14 +659,15 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     doAssert t.destructor != nil
     moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
-  of attachedDestructor, attachedDispose:
+  of attachedDestructor:
     body.add genBuiltin(c, mDestroy, "destroy", x)
   of attachedTrace:
     discard "strings are atomic and have no inner elements that are to trace"
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
-proc cyclicType*(t: PType): bool =
+proc cyclicType*(g: ModuleGraph, t: PType): bool =
   case t.kind
-  of tyRef: result = types.canFormAcycle(t.lastSon)
+  of tyRef: result = types.canFormAcycle(g, t.elementType)
   of tyProc: result = t.callConv == ccClosure
   else: result = false
 
@@ -589,13 +692,18 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
   ]#
   var actions = newNodeI(nkStmtList, c.info)
-  let elemType = t.lastSon
+  let elemType = t.elementType
 
   createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen)
-  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(elemType)
+  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(c.g, elemType)
+
+  let isInheritableAcyclicRef = c.g.config.selectedGC == gcOrc and
+                      (not isPureObject(elemType)) and
+                      tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags
+  # dynamic Acyclic refs need to use dyn decRef
 
   let tmp =
-    if isCyclic and c.kind in {attachedAsgn, attachedSink}:
+    if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
       declareTempOf(c, body, x)
     else:
       x
@@ -617,6 +725,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
     else:
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, tmp)
+  elif isInheritableAcyclicRef:
+    cond = callCodegenProc(c.g, "nimDecRefIsLastDyn", c.info, x)
   else:
     cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x)
   cond.typ = getSysType(c.g, x.info, tyBool)
@@ -643,21 +753,25 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
-    if isFinal(elemType):
-      let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
-      typInfo.typ = getSysType(c.g, c.info, tyPointer)
-      body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y)
+    if isCyclic:
+      if isFinal(elemType):
+        let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
+        typInfo.typ = getSysType(c.g, c.info, tyPointer)
+        body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y)
+      else:
+        # If the ref is polymorphic we have to account for this
+        body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y)
+      #echo "can follow ", elemType, " static ", isFinal(elemType)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
+  of attachedDup:
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, y, callCodegenProc(c.g,
+          "nimIncRefCyclic", c.info, y, getCycleParam(c)))
     else:
-      # If the ref is polymorphic we have to account for this
-      body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y)
-  of attachedDispose:
-    # this is crucial! dispose is like =destroy but we don't follow refs
-    # as that is dealt within the cycle collector.
-    when false:
-      let cond = copyTree(x)
-      cond.typ = getSysType(c.g, x.info, tyBool)
-      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
-      body.add genIf(c, cond, actions)
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, y, callCodegenProc(c.g,
+          "nimIncRef", c.info, y))
 
 proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   ## Closures are really like refs except they always use a virtual destructor
@@ -667,7 +781,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
   let isCyclic = c.g.config.selectedGC == gcOrc
   let tmp =
-    if isCyclic and c.kind in {attachedAsgn, attachedSink}:
+    if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
       declareTempOf(c, body, xenv)
     else:
       xenv
@@ -701,19 +815,21 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
       body.add genIf(c, cond, actions)
       body.add newAsgnStmt(x, y)
+  of attachedDup:
+    let yenv = genBuiltin(c, mAccessEnv, "accessEnv", y)
+    yenv.typ = getSysType(c.g, c.info, tyPointer)
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRefCyclic", c.info, yenv, getCycleParam(c)))
+    else:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRef", c.info, yenv))
   of attachedDestructor:
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y)
-  of attachedDispose:
-    # this is crucial! dispose is like =destroy but we don't follow refs
-    # as that is dealt within the cycle collector.
-    when false:
-      let cond = copyTree(xenv)
-      cond.typ = getSysType(c.g, xenv.info, tyBool)
-      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, xenv)
-      body.add genIf(c, cond, actions)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
@@ -726,6 +842,9 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, y, callCodegenProc(c.g, "nimIncRef", c.info, y))
     body.add genIf(c, x, callCodegenProc(c.g, "nimDecWeakRef", c.info, x))
     body.add newAsgnStmt(x, y)
+  of attachedDup:
+    body.add newAsgnStmt(x, y)
+    body.add genIf(c, y, callCodegenProc(c.g, "nimIncRef", c.info, y))
   of attachedDestructor:
     # it's better to prepend the destruction of weak refs in order to
     # prevent wrong "dangling refs exist" problems:
@@ -737,12 +856,13 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     else:
       body.sons.insert(des, 0)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace, attachedDispose: discard
+  of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   var actions = newNodeI(nkStmtList, c.info)
 
-  let elemType = t.lastSon
+  let elemType = t.skipModifier
   #fillBody(c, elemType, actions, genDeref(x), genDeref(y))
   #var disposeCall = genBuiltin(c, mDispose, "dispose", x)
 
@@ -759,10 +879,13 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedSink, attachedAsgn:
     body.add genIf(c, x, actions)
     body.add newAsgnStmt(x, y)
+  of attachedDup:
+    body.add newAsgnStmt(x, y)
   of attachedDestructor:
     body.add genIf(c, x, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace, attachedDispose: discard
+  of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if c.kind == attachedDeepCopy:
@@ -774,7 +897,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     call[1] = y
     body.add newAsgnStmt(x, call)
   elif (optOwnedRefs in c.g.config.globalOptions and
-      optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcOrc}:
+      optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}:
     let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
     xx.typ = getSysType(c.g, c.info, tyPointer)
     case c.kind
@@ -789,6 +912,11 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
       body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       body.add newAsgnStmt(x, y)
+    of attachedDup:
+      let yy = genBuiltin(c, mAccessEnv, "accessEnv", y)
+      yy.typ = getSysType(c.g, c.info, tyPointer)
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
     of attachedDestructor:
       let des = genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       if body.len == 0:
@@ -796,7 +924,8 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       else:
         body.sons.insert(des, 0)
     of attachedDeepCopy: assert(false, "cannot happen")
-    of attachedTrace, attachedDispose: discard
+    of attachedTrace: discard
+    of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
@@ -808,19 +937,22 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedSink, attachedAsgn:
     body.add genIf(c, xx, actions)
     body.add newAsgnStmt(x, y)
+  of attachedDup:
+    body.add newAsgnStmt(x, y)
   of attachedDestructor:
     body.add genIf(c, xx, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace, attachedDispose: discard
+  of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case t.kind
   of tyNone, tyEmpty, tyVoid: discard
-  of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString,
+  of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring,
       tyPtr, tyUncheckedArray, tyVar, tyLent:
     defaultOp(c, t, body, x, y)
   of tyRef:
-    if c.g.config.selectedGC in {gcArc, gcOrc}:
+    if c.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
       atomicRefOp(c, t, body, x, y)
     elif (optOwnedRefs in c.g.config.globalOptions and
         optRefCheck in c.g.config.options):
@@ -829,7 +961,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       defaultOp(c, t, body, x, y)
   of tyProc:
     if t.callConv == ccClosure:
-      if c.g.config.selectedGC in {gcArc, gcOrc}:
+      if c.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
         atomicClosureOp(c, t, body, x, y)
       else:
         closureOp(c, t, body, x, y)
@@ -875,13 +1007,28 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       defaultOp(c, t, body, x, y)
   of tyObject:
     if not considerUserDefinedOp(c, t, body, x, y):
-      if c.kind in {attachedAsgn, attachedSink} and t.sym != nil and sfImportc in t.sym.flags:
-        body.add newAsgnStmt(x, y)
+      if t.sym != nil and sfImportc in t.sym.flags:
+        case c.kind
+        of {attachedAsgn, attachedSink, attachedDup}:
+          body.add newAsgnStmt(x, y)
+        of attachedWasMoved:
+          body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
+        else:
+          fillBodyObjT(c, t, body, x, y)
       else:
-        fillBodyObjT(c, t, body, x, y)
+        if c.kind == attachedDup:
+          var op2 = getAttachedOp(c.g, t, attachedAsgn)
+          if op2 != nil and sfOverridden in op2.flags:
+            #markUsed(c.g.config, c.info, op, c.g.usageSym)
+            onUse(c.info, op2)
+            body.add newHookCall(c, t.assignment, x, y)
+          else:
+            fillBodyObjT(c, t, body, x, y)
+        else:
+          fillBodyObjT(c, t, body, x, y)
   of tyDistinct:
     if not considerUserDefinedOp(c, t, body, x, y):
-      fillBody(c, t[0], body, x, y)
+      fillBody(c, t.elementType, body, x, y)
   of tyTuple:
     fillBodyTup(c, t, body, x, y)
   of tyVarargs, tyOpenArray:
@@ -898,42 +1045,86 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     discard
   of tyOrdinal, tyRange, tyInferred,
      tyGenericInst, tyAlias, tySink:
-    fillBody(c, lastSon(t), body, x, y)
-  of tyConcept: doAssert false
+    fillBody(c, skipModifier(t), body, x, y)
+  of tyConcept, tyIterable: raiseAssert "unreachable"
 
 proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
                             kind: TTypeAttachedOp; info: TLineInfo;
                             idgen: IdGenerator): PSym =
   assert typ.kind == tyDistinct
-  let baseType = typ[0]
+  let baseType = typ.elementType
   if getAttachedOp(g, baseType, kind) == nil:
-    discard produceSym(g, c, baseType, kind, info, idgen)
+    # TODO: fixme `isDistinct` is a fix for #23552; remove it after
+    # `-d:nimPreviewNonVarDestructor` becomes the default
+    discard produceSym(g, c, baseType, kind, info, idgen, isDistinct = true)
   result = getAttachedOp(g, baseType, kind)
   setAttachedOp(g, idgen.module, typ, kind, result)
 
-proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
+proc symDupPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
               info: TLineInfo; idgen: IdGenerator): PSym =
+  let procname = getIdent(g.cache, AttachedOpToStr[kind])
+  result = newSym(skProc, procname, idgen, owner, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), idgen, result, info)
+  let src = newSym(skParam, getIdent(g.cache, "src"),
+                   idgen, result, info)
+  res.typ = typ
+  src.typ = typ
+
+  result.typ = newType(tyProc, idgen, owner)
+  result.typ.n = newNodeI(nkFormalParams, info)
+  rawAddSon(result.typ, res.typ)
+  result.typ.n.add newNodeI(nkEffectList, info)
+
+  result.typ.addParam src
+
+  if g.config.selectedGC == gcOrc and
+    cyclicType(g, typ.skipTypes(abstractInst)):
+    let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"),
+                            idgen, result, info)
+    cycleParam.typ = getSysType(g, info, tyBool)
+    result.typ.addParam cycleParam
+
+  var n = newNodeI(nkProcDef, info, bodyPos+2)
+  for i in 0..
"
 
 # * $analytics: Google analytics location, includes 
-
-
+
 
+
+
 
 
-

-  

-    
$title
$subtitle
-    $content
-    

+  

+    

+      
$title
$subtitle
+      $content
       

         
-        

+        

         Made with Nim. Generated: $date $time UTC
       

     

   

-

-$analytics
+  $analytics
 
 
 """
diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg
index 307b280cc6..4fb1aec908 100644
--- a/config/nimdoc.tex.cfg
+++ b/config/nimdoc.tex.cfg
@@ -3,27 +3,42 @@
 # (c) 2012 Andreas Rumpf
 # Feel free to edit the templates as you need.
 
-split.item.toc = "20"  
+split.item.toc = "20"
 # too long entries in the table of contents wrap around
 # after this number of characters
 
 doc.section = """
-\chapter{$sectionTitle}\label{$sectionID}
-\begin{description}
+\rsthA[$sectionTitle]{$sectionTitle}\label{$sectionID}
+
 $content
-\end{description}
 """
 
 doc.section.toc = ""
 # $sectionID $sectionTitleID $sectionTitle $content
 
 doc.item = """
-\item[\texttt{$header}\label{$itemID}]\mbox{~}\\*
+
+\vspace{1em}
+\phantomsection\addcontentsline{toc}{subsubsection}{$uniqueName}
+\label{$itemSymOrID}\hypertarget{$itemSymOrID}{}
+
+\begin{rstdocitem}
+$header
+\end{rstdocitem}
+
+\begin{addmargin}[0.05\linewidth]{0pt}
 $desc
+\end{addmargin}
+"""
+
+doc.item2 = """
+\phantomsection\addcontentsline{toc}{subsection}{$header_plain}
+\label{$overloadGroupName}\hypertarget{$overloadGroupName}{}
+
+$content
 """
 
 doc.item.toc = ""
-#  \item $name\ref{$itemID}
 
 doc.toc = r"\tableofcontents \newpage"
 
@@ -38,97 +53,33 @@ $moduledesc
 $content
 """
 
+# $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor
+doc.listing_start = "\\begin{rstpre}\n"
+doc.listing_end = "\n\\end{rstpre}\n\n"
+
 doc.file = """
 % This file was generated by Nim.
 % Generated: $date $time UTC
-\documentclass[a4paper]{article}
-\usepackage[left=2cm,right=3cm,top=3cm,bottom=3cm]{geometry}
-\usepackage[utf8]{inputenc}
-\usepackage[T1]{fontenc}
-\usepackage{graphicx}
-\usepackage{lmodern}
-\usepackage{fancyvrb, courier}
-\usepackage{tabularx}
-\usepackage{hyperref}
-\usepackage{enumitem}  % for enumList and rstfootnote
-
-\usepackage{xcolor}
-\usepackage[tikz]{mdframed}
-\usetikzlibrary{shadows}
-\mdfsetup{%
-linewidth=3,
-topline=false,
-rightline=false,
-bottomline=false}
+%
+% Compile it by:   xelatex    (up to 3 times to get labels generated)
+%                  -------
+% For example:
+%   xelatex file.tex
+%   xelatex file.tex
+%   makeindex file
+%   xelatex file.tex
+%
+\documentclass{nimdoc}
 
 \begin{document}
 \title{$title $version $subtitle}
 \author{$author}
 
-\tolerance 1414 
-\hbadness 1414 
-\emergencystretch 1.5em 
-\hfuzz 0.3pt 
-\widowpenalty=10000 
-\vfuzz \hfuzz 
-\raggedbottom 
-
 \maketitle
 
-\newenvironment{rstpre}{\VerbatimEnvironment\begingroup\begin{Verbatim}[fontsize=\footnotesize , commandchars=\\\{\}]}{\end{Verbatim}\endgroup}
-\newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}}
-
-% to pack tabularx into a new environment, special syntax is needed :-(
-\newenvironment{rsttab}[1]{\tabularx{\linewidth}{#1}}{\endtabularx}
-
-\newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}}
-\newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}}
-
-\newcommand{\rsthA}[1]{\section{#1}}
-\newcommand{\rsthB}[1]{\subsection{#1}}
-\newcommand{\rsthC}[1]{\subsubsection{#1}}
-\newcommand{\rsthD}[1]{\paragraph{#1}}
-\newcommand{\rsthE}[1]{\paragraph{#1}}
-
-\newcommand{\rstovA}[1]{\section*{#1}}
-\newcommand{\rstovB}[1]{\subsection*{#1}}
-\newcommand{\rstovC}[1]{\subsubsection*{#1}}
-\newcommand{\rstovD}[1]{\paragraph*{#1}}
-\newcommand{\rstovE}[1]{\paragraph*{#1}}
-
-% Syntax highlighting:
-\newcommand{\spanDecNumber}[1]{#1}
-\newcommand{\spanBinNumber}[1]{#1}
-\newcommand{\spanHexNumber}[1]{#1}
-\newcommand{\spanOctNumber}[1]{#1}
-\newcommand{\spanFloatNumber}[1]{#1}
-\newcommand{\spanIdentifier}[1]{#1}
-\newcommand{\spanKeyword}[1]{\textbf{#1}}
-\newcommand{\spanStringLit}[1]{#1}
-\newcommand{\spanLongStringLit}[1]{#1}
-\newcommand{\spanCharLit}[1]{#1}
-\newcommand{\spanEscapeSequence}[1]{#1}
-\newcommand{\spanOperator}[1]{#1}
-\newcommand{\spanPunctuation}[1]{#1}
-\newcommand{\spanComment}[1]{\emph{#1}}
-\newcommand{\spanLongComment}[1]{\emph{#1}}
-\newcommand{\spanRegularExpression}[1]{#1}
-\newcommand{\spanTagStart}[1]{#1}
-\newcommand{\spanTagEnd}[1]{#1}
-\newcommand{\spanKey}[1]{#1}
-\newcommand{\spanValue}[1]{#1}
-\newcommand{\spanRawData}[1]{#1}
-\newcommand{\spanAssembler}[1]{#1}
-\newcommand{\spanPreprocessor}[1]{#1}
-\newcommand{\spanDirective}[1]{#1}
-\newcommand{\spanCommand}[1]{#1}
-\newcommand{\spanRule}[1]{#1}
-\newcommand{\spanHyperlink}[1]{#1}
-\newcommand{\spanLabel}[1]{#1}
-\newcommand{\spanReference}[1]{#1}
-\newcommand{\spanOther}[1]{#1}
-\newcommand{\spantok}[1]{\frame{#1}}
-
 $content
+
+\printindex
+
 \end{document}
 """
diff --git a/copying.txt b/copying.txt
index a498a95253..819330be30 100644
--- a/copying.txt
+++ b/copying.txt
@@ -1,7 +1,7 @@
 =====================================================
 Nim -- a Compiler for Nim. https://nim-lang.org/
 
-Copyright (C) 2006-2021 Andreas Rumpf. All rights reserved.
+Copyright (C) 2006-2024 Andreas Rumpf. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/doc/advopt.txt b/doc/advopt.txt
index 108c07222f..e4d11081ae 100644
--- a/doc/advopt.txt
+++ b/doc/advopt.txt
@@ -4,9 +4,13 @@ Advanced commands:
   //compileToOC, objc       compile project to Objective C code
   //js                      compile project to Javascript
   //e                       run a Nimscript file
+  //md2html                 convert a Markdown file to HTML
+                            use `--docCmd:skip` to skip compiling snippets
   //rst2html                convert a reStructuredText file to HTML
                             use `--docCmd:skip` to skip compiling snippets
-  //rst2tex                 convert a reStructuredText file to TeX
+  //md2tex                  convert a Markdown file to LaTeX
+  //rst2tex                 convert a reStructuredText file to LaTeX
+  //doc2tex                 extract the documentation to a LaTeX file
   //jsondoc                 extract the documentation to a json file
   //ctags                   create a tags file
   //buildIndex              build an index for the whole documentation
@@ -14,7 +18,8 @@ Advanced commands:
                             module dependency graph
   //dump                    dump all defined conditionals and search paths
                             see also: --dump.format:json (useful with: `| jq`)
-  //check                   checks the project for syntax and semantic
+  //check                   checks the project for syntax and semantics
+                            (can be combined with --defusages)
 
 Runtime checks (see -x):
   --objChecks:on|off        turn obj conversion checks on|off
@@ -27,28 +32,45 @@ Runtime checks (see -x):
   --infChecks:on|off        turn Inf checks on|off
 
 Advanced options:
+  --defusages:FILE,LINE,COL
+                            find the definition and all usages of a symbol
   -o:FILE, --out:FILE       set the output filename
   --outdir:DIR              set the path where the output file will be written
   --usenimcache             will use `outdir=$$nimcache`, whichever it resolves
                             to after all options have been processed
   --stdout:on|off           output to stdout
   --colors:on|off           turn compiler messages coloring on|off
-  --listFullPaths:on|off    list full paths in messages
+  --filenames:abs|canonical|legacyRelProj
+                            customize how filenames are rendered in compiler messages,
+                            defaults to `abs` (absolute)
+  --processing:dots|filenames|off
+                            show files as they're being processed by nim compiler
+  --unitsep:on|off          use the ASCII unit separator (31) between error
+                            messages, useful for IDE-like tooling
   --declaredLocs:on|off     show declaration locations in messages
-  --spellSuggest|:num       show at most `num >= 0` spelling suggestions on typos.
+  --spellSuggest:num        show at most `num >= 0` spelling suggestions on typos.
                             if `num` is not specified (or `auto`), return
                             an implementation defined set of suggestions.
+  --hints:on|off|list.      `on|off` enables or disables hints.
+                            `list` reports which hints are selected.
+  --hint:X:on|off           turn specific hint X on|off. `hint:X` means `hint:X:on`,
+                            as with similar flags. `all` is the set of all hints
+                            (only `all:off` is supported).
+  --hintAsError:X:on|off    turn specific hint X into an error on|off
   -w:on|off|list, --warnings:on|off|list
-                            turn all warnings on|off or list all available
-  --warning[X]:on|off       turn specific warning X on|off
-  --hints:on|off|list       turn all hints on|off or list all available
-  --hint[X]:on|off          turn specific hint X on|off
-  --warningAsError[X]:on|off turn specific warning X into an error on|off
-  --hintAsError[X]:on|off    turn specific hint X into an error on|off
+                            `on|off` enables or disables warnings.
+                            `list` reports which warnings are selected.
+  --warning:X:on|off        turn specific warning X on|off. `warning:X` means `warning:X:on`,
+                            as with similar flags. `all` is the set of all warning
+                            (only `all:off` is supported).
+  --warningAsError:X:on|off
+                            turn specific warning X into an error on|off
   --styleCheck:off|hint|error
                             produce hints or errors for Nim identifiers that
                             do not adhere to Nim's official style guide
                             https://nim-lang.org/docs/nep1.html
+  --styleCheck:usages       only enforce consistent spellings of identifiers,
+                            do not enforce the style on declarations
   --showAllMismatches:on|off
                             show all mismatching candidates in overloading
                             resolution
@@ -83,7 +105,8 @@ Advanced options:
                             if path == @default (the default and most useful), will use
                             best match among @pkg,@path.
                             if these are nonexistent, will use project path
-  -b, --backend:c|cpp|js|objc sets backend to use with commands like `nim doc` or `nim r`
+  -b, --backend:c|cpp|js|objc
+                            sets backend to use with commands like `nim doc` or `nim r`
   --docCmd:cmd              if `cmd == skip`, skips runnableExamples
                             else, runs runnableExamples with given options, e.g.:
                             `--docCmd:"-d:foo --threads:on"`
@@ -93,28 +116,26 @@ Advanced options:
   --lineDir:on|off          generation of #line directive on|off
   --embedsrc:on|off         embeds the original source code as comments
                             in the generated output
-  --threadanalysis:on|off   turn thread analysis on|off
   --tlsEmulation:on|off     turn thread local storage emulation on|off
   --implicitStatic:on|off   turn implicit compile time evaluation on|off
   --trmacros:on|off         turn term rewriting macros on|off
   --multimethods:on|off     turn multi-methods on|off
-  --memTracker:on|off       turn memory tracker on|off
   --hotCodeReloading:on|off
                             turn support for hot code reloading on|off
   --excessiveStackTrace:on|off
                             stack traces use full file paths
   --stackTraceMsgs:on|off   enable user defined stack frame msgs via `setFrameMsg`
-  --seqsv2:on|off           use the new string/seq implementation based on
-                            destructors
   --skipCfg:on|off          do not read the nim installation's configuration file
   --skipUserCfg:on|off      do not read the user's configuration file
   --skipParentCfg:on|off    do not read the parent dirs' configuration files
   --skipProjCfg:on|off      do not read the project's configuration file
-  --gc:refc|arc|orc|markAndSweep|boehm|go|none|regions
-                            select the GC to use; default is 'refc'
-  --exceptions:setjmp|cpp|goto
+  --mm:orc|arc|refc|markAndSweep|boehm|go|none|regions
+                            select which memory management to use; default is 'orc'
+  --exceptions:setjmp|cpp|goto|quirky
                             select the exception handling implementation
-  --index:on|off            turn index file generation on|off
+  --index:on|off|only       docgen: turn index file generation on|off (`only` means
+                            not generate output files like HTML)
+  --noImportdoc:on|off      turn loading documentation ``.idx`` files on|off
   --putenv:key=value        set an environment variable
   --NimblePath:PATH         add a path for Nimble support
   --noNimblePath            deactivate the Nimble path
@@ -122,6 +143,8 @@ Advanced options:
   --cppCompileToNamespace:namespace
                             use the provided namespace for the generated C++ code,
                             if no namespace is provided "Nim" will be used
+  --nimMainPrefix:prefix    use `{prefix}NimMain` instead of `NimMain` in the produced
+                            C/C++ code
   --expandMacro:MACRO       dump every generated AST from MACRO
   --expandArc:PROCNAME      show how PROCNAME looks like after diverse optimizations
                             before the final backend phase (mostly ARC/ORC specific)
@@ -146,12 +169,11 @@ Advanced options:
                             enable experimental language feature
   --legacy:$2
                             enable obsolete/legacy language feature
-  --useVersion:1.0|1.2      emulate Nim version X of the Nim compiler, for testing
-  --profiler:on|off         enable profiling; requires `import nimprof`, and
-                            works better with `--stackTrace:on`
-                            see also https://nim-lang.github.io/Nim/estp.html
-  --benchmarkVM:on|off      enable benchmarking of VM code with cpuTime()
-  --profileVM:on|off        enable compile time VM profiler
-  --sinkInference:on|off    en-/disable sink parameter inference (default: on)
+  --benchmarkVM:on|off      turn benchmarking of VM code with cpuTime() on|off
+  --profileVM:on|off        turn compile time VM profiler on|off
   --panics:on|off           turn panics into process terminations (default: off)
-  --deepcopy:on|off         enable 'system.deepCopy' for ``--gc:arc|orc``
+  --deepcopy:on|off         enable 'system.deepCopy' for ``--mm:arc|orc``
+  --jsbigint64:on|off       toggle the use of BigInt for 64-bit integers for
+                            the JavaScript backend (default: on)
+  --nimBasePattern:nimbase.h
+                            allows to specify a custom pattern for `nimbase.h`
diff --git a/doc/apis.rst b/doc/apis.md
similarity index 93%
rename from doc/apis.rst
rename to doc/apis.md
index d01e75d782..f0b8c93e56 100644
--- a/doc/apis.rst
+++ b/doc/apis.md
@@ -1,9 +1,10 @@
-.. default-role:: code
-
 =================
 API naming design
 =================
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 The API is designed to be **easy to use** and consistent. Ease of use is
 measured by the number of calls to achieve a concrete high-level action.
 
@@ -17,9 +18,9 @@ been renamed to fit this scheme. The ultimate goal is that the programmer can
 *guess* a name.
 
 
--------------------     ------------   --------------------------------------
+===================     ============   ======================================
 English word            To use         Notes
--------------------     ------------   --------------------------------------
+===================     ============   ======================================
 initialize              initT          `init` is used to create a
                                        value type `T`
 new                     newP           `new` is used to create a
@@ -81,4 +82,4 @@ literal                 lit
 string                  str
 identifier              ident
 indentation             indent
--------------------     ------------   --------------------------------------
+===================     ============   ======================================
diff --git a/doc/astspec.txt b/doc/astspec.txt
index c49f7bcc26..7a7053a2d0 100644
--- a/doc/astspec.txt
+++ b/doc/astspec.txt
@@ -6,8 +6,7 @@ The AST consists of nodes (``NimNode``) with a variable number of
 children. Each node has a field named ``kind`` which describes what the node
 contains:
 
-.. code-block:: nim
-
+  ```nim
   type
     NimNodeKind = enum     ## kind of a node; only explanatory
       nnkNone,             ## invalid node kind
@@ -32,6 +31,7 @@ contains:
         strVal: string                 ## the string literal
       else:
         sons: seq[NimNode]             ## the node's sons (or children)
+  ```
 
 For the ``NimNode`` type, the ``[]`` operator has been overloaded:
 ``n[i]`` is ``n``'s ``i``-th child.
@@ -50,9 +50,9 @@ A leaf of the AST often corresponds to a terminal symbol in the concrete
 syntax. Note that the default ``float`` in Nim maps to ``float64`` such that
 the default AST for a float is ``nnkFloat64Lit`` as below.
 
------------------                ---------------------------------------------
+=================                =============================================
 Nim expression                   Corresponding AST
------------------                ---------------------------------------------
+=================                =============================================
 ``42``                           ``nnkIntLit(intVal = 42)``
 ``42'i8``                        ``nnkInt8Lit(intVal = 42)``
 ``42'i16``                       ``nnkInt16Lit(intVal = 42)``
@@ -72,7 +72,7 @@ Nim expression                   Corresponding AST
 ``nil``                          ``nnkNilLit()``
 ``myIdentifier``                 ``nnkIdent(strVal = "myIdentifier")``
 ``myIdentifier``                 after lookup pass: ``nnkSym(strVal = "myIdentifier", ...)``
------------------                ---------------------------------------------
+=================                =============================================
 
 Identifiers are ``nnkIdent`` nodes. After the name lookup pass these nodes
 get transferred into ``nnkSym`` nodes.
@@ -86,17 +86,19 @@ Command call
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   echo "abc", "xyz"
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCommand(
     nnkIdent("echo"),
     nnkStrLit("abc"),
     nnkStrLit("xyz")
   )
+  ```
 
 
 Call with ``()``
@@ -104,17 +106,19 @@ Call with ``()``
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   echo("abc", "xyz")
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCall(
     nnkIdent("echo"),
     nnkStrLit("abc"),
     nnkStrLit("xyz")
   )
+  ```
 
 
 Infix operator call
@@ -122,29 +126,32 @@ Infix operator call
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   "abc" & "xyz"
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkInfix(
     nnkIdent("&"),
     nnkStrLit("abc"),
     nnkStrLit("xyz")
   )
+  ```
 
 Note that with multiple infix operators, the command is parsed by operator
 precedence.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   5 + 3 * 4
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkInfix(
     nnkIdent("+"),
     nnkIntLit(5),
@@ -154,6 +161,7 @@ AST:
       nnkIntLit(4)
     )
   )
+  ```
 
 As a side note, if you choose to use infix operators in a prefix form, the AST
 behaves as a
@@ -162,12 +170,13 @@ behaves as a
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   `+`(3, 4)
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCall(
     nnkAccQuoted(
       nnkIdent("+")
@@ -175,22 +184,25 @@ AST:
     nnkIntLit(3),
     nnkIntLit(4)
   )
+  ```
 
 Prefix operator call
 --------------------
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   ? "xyz"
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkPrefix(
     nnkIdent("?"),
     nnkStrLit("abc")
   )
+  ```
 
 
 Postfix operator call
@@ -201,16 +213,18 @@ Postfix operator call
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   identifier*
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkPostfix(
     nnkIdent("*"),
     nnkIdent("identifier")
   )
+  ```
 
 
 Call with named arguments
@@ -218,12 +232,13 @@ Call with named arguments
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   writeLine(file=stdout, "hallo")
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCall(
     nnkIdent("writeLine"),
     nnkExprEqExpr(
@@ -232,6 +247,7 @@ AST:
     ),
     nnkStrLit("hallo")
   )
+  ```
 
 Call with raw string literal
 ----------------------------
@@ -242,29 +258,33 @@ This is used, for example, in the ``bindSym`` examples
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   echo"abc"
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCallStrLit(
     nnkIdent("echo"),
     nnkRStrLit("hello")
   )
+  ```
 
 Dereference operator ``[]``
 ---------------------------
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   x[]
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkDerefExpr(nnkIdent("x"))
+  ```
 
 
 Addr operator
@@ -272,13 +292,15 @@ Addr operator
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   addr(x)
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkAddr(nnkIdent("x"))
+  ```
 
 
 Cast operator
@@ -286,13 +308,15 @@ Cast operator
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   cast[T](x)
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCast(nnkIdent("T"), nnkIdent("x"))
+  ```
 
 
 Object access operator ``.``
@@ -300,13 +324,15 @@ Object access operator ``.``
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   x.y
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkDotExpr(nnkIdent("x"), nnkIdent("y"))
+  ```
 
 If you use Nim's flexible calling syntax (as in ``x.len()``), the result is the
 same as above but wrapped in an ``nnkCall``.
@@ -317,31 +343,77 @@ Array access operator ``[]``
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   x[y]
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkBracketExpr(nnkIdent("x"), nnkIdent("y"))
+  ```
 
 
 Parentheses
 -----------
 
-Parentheses for affecting operator precedence or tuple construction
-are built with the ``nnkPar`` node.
+Parentheses for affecting operator precedence use the ``nnkPar`` node.
 
 Concrete syntax:
 
-.. code-block:: nim
-  (1, 2, (3))
+  ```nim
+  (a + b) * c
+  ```
 
 AST:
 
-.. code-block:: nim
-  nnkPar(nnkIntLit(1), nnkIntLit(2), nnkPar(nnkIntLit(3)))
+  ```nim
+  nnkInfix(nnkIdent("*"),
+    nnkPar(
+      nnkInfix(nnkIdent("+"), nnkIdent("a"), nnkIdent("b"))),
+    nnkIdent("c"))
+  ```
 
+Tuple Constructors
+------------------
+
+Nodes for tuple construction are built with the ``nnkTupleConstr`` node.
+
+Concrete syntax:
+
+  ```nim
+  (1, 2, 3)
+  (a: 1, b: 2, c: 3)
+  ()
+  ```
+
+AST:
+
+  ```nim
+  nnkTupleConstr(nnkIntLit(1), nnkIntLit(2), nnkIntLit(3))
+  nnkTupleConstr(
+    nnkExprColonExpr(nnkIdent("a"), nnkIntLit(1)),
+    nnkExprColonExpr(nnkIdent("b"), nnkIntLit(2)),
+    nnkExprColonExpr(nnkIdent("c"), nnkIntLit(3)))
+  nnkTupleConstr()
+  ```
+
+Since the one tuple would be syntactically identical to parentheses
+with an expression in them, the parser expects a trailing comma for
+them. For tuple constructors with field names, this is not necessary.
+
+  ```nim
+  (1,)
+  (a: 1)
+  ```
+
+AST:
+
+  ```nim
+  nnkTupleConstr(nnkIntLit(1))
+  nnkTupleConstr(
+    nnkExprColonExpr(nnkIdent("a"), nnkIntLit(1)))
+  ```
 
 Curly braces
 ------------
@@ -350,28 +422,32 @@ Curly braces are used as the set constructor.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   {1, 2, 3}
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCurly(nnkIntLit(1), nnkIntLit(2), nnkIntLit(3))
+  ```
 
 When used as a table constructor, the syntax is different.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   {a: 3, b: 5}
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkTableConstr(
     nnkExprColonExpr(nnkIdent("a"), nnkIntLit(3)),
     nnkExprColonExpr(nnkIdent("b"), nnkIntLit(5))
   )
+  ```
 
 
 Brackets
@@ -381,13 +457,15 @@ Brackets are used as the array constructor.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   [1, 2, 3]
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkBracket(nnkIntLit(1), nnkIntLit(2), nnkIntLit(3))
+  ```
 
 
 Ranges
@@ -399,21 +477,23 @@ AST, construction with ``..`` as an infix operator should be used instead.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   1..3
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkInfix(
     nnkIdent(".."),
     nnkIntLit(1),
     nnkIntLit(3)
   )
+  ```
 
 Example code:
 
-.. code-block:: nim
+  ```nim
   macro genRepeatEcho() =
     result = newNimNode(nnkStmtList)
 
@@ -432,6 +512,7 @@ Example code:
                   # 3
                   # 3
                   # 3
+  ```
 
 
 If expression
@@ -441,17 +522,19 @@ The representation of the ``if`` expression is subtle, but easy to traverse.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   if cond1: expr1 elif cond2: expr2 else: expr3
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkIfExpr(
     nnkElifExpr(cond1, expr1),
     nnkElifExpr(cond2, expr2),
     nnkElseExpr(expr3)
   )
+  ```
 
 Documentation Comments
 ----------------------
@@ -462,19 +545,21 @@ comments are ignored.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   ## This is a comment
   ## This is part of the first comment
   stmt1
   ## Yet another
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCommentStmt() # only appears once for the first two lines!
   stmt1
   nnkCommentStmt() # another nnkCommentStmt because there is another comment
                    # (separate from the first)
+  ```
 
 Pragmas
 -------
@@ -485,30 +570,33 @@ objects, but the standalone ``emit`` pragma shows the basics with the AST.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   {.emit: "#include ".}
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkPragma(
     nnkExprColonExpr(
       nnkIdent("emit"),
       nnkStrLit("#include ") # the "argument"
     )
   )
+  ```
 
 As many ``nnkIdent`` appear as there are pragmas between ``{..}``. Note that
 the declaration of new pragmas is essentially the same:
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   {.pragma: cdeclRename, cdecl.}
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkPragma(
     nnkExprColonExpr(
       nnkIdent("pragma"), # this is always first when declaring a new pragma
@@ -516,6 +604,7 @@ AST:
     ),
     nnkIdent("cdecl")
   )
+  ```
 
 Statements
 ==========
@@ -528,7 +617,7 @@ there is no ``else`` branch, no ``nnkElse`` child exists.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   if cond1:
     stmt1
   elif cond2:
@@ -537,16 +626,18 @@ Concrete syntax:
     stmt3
   else:
     stmt4
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkIfStmt(
     nnkElifBranch(cond1, stmt1),
     nnkElifBranch(cond2, stmt2),
     nnkElifBranch(cond3, stmt3),
     nnkElse(stmt4)
   )
+  ```
 
 
 When statement
@@ -560,13 +651,15 @@ Assignment
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   x = 42
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkAsgn(nnkIdent("x"), nnkIntLit(42))
+  ```
 
 This is not the syntax for assignment when combined with ``var``, ``let``,
 or ``const``.
@@ -576,15 +669,17 @@ Statement list
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   stmt1
   stmt2
   stmt3
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkStmtList(stmt1, stmt2, stmt3)
+  ```
 
 
 Case statement
@@ -592,7 +687,7 @@ Case statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   case expr1
   of expr2, expr3..expr4:
     stmt1
@@ -602,10 +697,11 @@ Concrete syntax:
     stmt3
   else:
     stmt4
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkCaseStmt(
     expr1,
     nnkOfBranch(expr2, nnkRange(expr3, expr4), stmt1),
@@ -613,6 +709,7 @@ AST:
     nnkElifBranch(cond1, stmt3),
     nnkElse(stmt4)
   )
+  ```
 
 The ``nnkElifBranch`` and ``nnkElse`` parts may be missing.
 
@@ -622,14 +719,16 @@ While statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   while expr1:
     stmt1
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkWhileStmt(expr1, stmt1)
+  ```
 
 
 For statement
@@ -637,14 +736,16 @@ For statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   for ident1, ident2 in expr1:
     stmt1
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkForStmt(ident1, ident2, expr1, stmt1)
+  ```
 
 
 Try statement
@@ -652,7 +753,7 @@ Try statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   try:
     stmt1
   except e1, e2:
@@ -663,10 +764,11 @@ Concrete syntax:
     stmt4
   finally:
     stmt5
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkTryStmt(
     stmt1,
     nnkExceptBranch(e1, e2, stmt2),
@@ -674,6 +776,7 @@ AST:
     nnkExceptBranch(stmt4),
     nnkFinally(stmt5)
   )
+  ```
 
 
 Return statement
@@ -681,13 +784,15 @@ Return statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   return expr1
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkReturnStmt(expr1)
+  ```
 
 
 Yield statement
@@ -695,8 +800,9 @@ Yield statement
 
 Like ``return``, but with ``nnkYieldStmt`` kind.
 
-.. code-block:: nim
+  ```nim
   nnkYieldStmt(expr1)
+  ```
 
 
 Discard statement
@@ -704,8 +810,9 @@ Discard statement
 
 Like ``return``, but with ``nnkDiscardStmt`` kind.
 
-.. code-block:: nim
+  ```nim
   nnkDiscardStmt(expr1)
+  ```
 
 
 Continue statement
@@ -713,26 +820,30 @@ Continue statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   continue
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkContinueStmt()
+  ```
 
 Break statement
 ---------------
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   break otherLocation
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkBreakStmt(nnkIdent("otherLocation"))
+  ```
 
 If ``break`` is used without a jump-to location, ``nnkEmpty`` replaces ``nnkIdent``.
 
@@ -741,13 +852,15 @@ Block statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   block name:
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkBlockStmt(nnkIdent("name"), nnkStmtList(...))
+  ```
 
 A ``block`` doesn't need an name, in which case ``nnkEmpty`` is used.
 
@@ -756,18 +869,20 @@ Asm statement
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   asm """
     some asm
   """
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkAsmStmt(
     nnkEmpty(), # for pragmas
     nnkTripleStrLit("some asm"),
   )
+  ```
 
 Import section
 --------------
@@ -777,37 +892,42 @@ on what keywords are present. Let's start with the simplest form.
 
 Concrete syntax:
 
-.. code-block:: nim
-  import math
+  ```nim
+  import std/math
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkImportStmt(nnkIdent("math"))
+  ```
 
 With ``except``, we get ``nnkImportExceptStmt``.
 
 Concrete syntax:
 
-.. code-block:: nim
-  import math except pow
+  ```nim
+  import std/math except pow
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkImportExceptStmt(nnkIdent("math"),nnkIdent("pow"))
+  ```
 
-Note that ``import math as m`` does not use a different node; rather,
+Note that ``import std/math as m`` does not use a different node; rather,
 we use ``nnkImportStmt`` with ``as`` as an infix operator.
 
 Concrete syntax:
 
-.. code-block:: nim
-  import strutils as su
+  ```nim
+  import std/strutils as su
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkImportStmt(
     nnkInfix(
       nnkIdent("as"),
@@ -815,6 +935,7 @@ AST:
       nnkIdent("su")
     )
   )
+  ```
 
 From statement
 --------------
@@ -823,15 +944,17 @@ If we use ``from ... import``, the result is different, too.
 
 Concrete syntax:
 
-.. code-block:: nim
-  from math import pow
+  ```nim
+  from std/math import pow
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkFromStmt(nnkIdent("math"), nnkIdent("pow"))
+  ```
 
-Using ``from math as m import pow`` works identically to the ``as`` modifier
+Using ``from std/math as m import pow`` works identically to the ``as`` modifier
 with the ``import`` statement, but wrapped in ``nnkFromStmt``.
 
 Export statement
@@ -842,26 +965,30 @@ the ``export`` syntax is pretty straightforward.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   export unsigned
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkExportStmt(nnkIdent("unsigned"))
+  ```
 
 Similar to the ``import`` statement, the AST is different for
 ``export ... except``.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   export math except pow # we're going to implement our own exponentiation
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkExportExceptStmt(nnkIdent("math"),nnkIdent("pow"))
+  ```
 
 Include statement
 -----------------
@@ -870,25 +997,28 @@ Like a plain ``import`` statement but with ``nnkIncludeStmt``.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   include blocks
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkIncludeStmt(nnkIdent("blocks"))
+  ```
 
 Var section
 -----------
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   var a = 3
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkVarSection(
     nnkIdentDefs(
       nnkIdent("a"),
@@ -896,6 +1026,7 @@ AST:
       nnkIntLit(3),
     )
   )
+  ```
 
 Note that either the second or third (or both) parameters above must exist,
 as the compiler needs to know the type somehow (which it can infer from
@@ -913,12 +1044,13 @@ This is equivalent to ``var``, but with ``nnkLetSection`` rather than
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   let a = 3
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkLetSection(
     nnkIdentDefs(
       nnkIdent("a"),
@@ -926,18 +1058,20 @@ AST:
       nnkIntLit(3),
     )
   )
+  ```
 
 Const section
 -------------
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   const a = 3
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkConstSection(
     nnkConstDef( # not nnkConstDefs!
       nnkIdent("a"),
@@ -945,6 +1079,7 @@ AST:
       nnkIntLit(3), # required in a const declaration!
     )
   )
+  ```
 
 Type section
 ------------
@@ -954,12 +1089,13 @@ and ``const``.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   type A = int
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkTypeSection(
     nnkTypeDef(
       nnkIdent("A"),
@@ -967,18 +1103,20 @@ AST:
       nnkIdent("int")
     )
   )
+  ```
 
 Declaring ``distinct`` types is similar, with the last ``nnkIdent`` wrapped
 in ``nnkDistinctTy``.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   type MyInt = distinct int
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...
   nnkTypeDef(
     nnkIdent("MyInt"),
@@ -987,17 +1125,19 @@ AST:
       nnkIdent("int")
     )
   )
+  ```
 
 If a type section uses generic parameters, they are treated here:
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   type A[T] = expr1
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkTypeSection(
     nnkTypeDef(
       nnkIdent("A"),
@@ -1012,6 +1152,7 @@ AST:
       expr1,
     )
   )
+  ```
 
 Note that not all ``nnkTypeDef`` utilize ``nnkIdent`` as their
 parameter. One of the most common uses of type declarations
@@ -1019,12 +1160,13 @@ is to work with objects.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   type IO = object of RootObj
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...
   nnkTypeDef(
     nnkIdent("IO"),
@@ -1037,29 +1179,39 @@ AST:
       nnkEmpty()
     )
   )
+  ```
 
 Nim's object syntax is rich. Let's take a look at an involved example in
 its entirety to see some of the complexities.
 
 Concrete syntax:
 
-.. code-block:: nim
-  type Obj[T] = object {.inheritable.}
+  ```nim
+  type Obj[T] {.inheritable.} = object
     name: string
     case isFat: bool
     of true:
       m: array[100_000, T]
     of false:
       m: array[10, T]
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...
+  nnkPragmaExpr(
+    nnkIdent("Obj"),
+    nnkPragma(nnkIdent("inheritable"))
+  ),
+  nnkGenericParams(
+  nnkIdentDefs(
+    nnkIdent("T"),
+    nnkEmpty(),
+    nnkEmpty())
+  ),
   nnkObjectTy(
-    nnkPragma(
-      nnkIdent("inheritable")
-    ),
+    nnkEmpty(),
     nnkEmpty(),
     nnkRecList( # list of object parameters
       nnkIdentDefs(
@@ -1103,55 +1255,61 @@ AST:
       )
     )
   )
+  ```
 
 
 Using an ``enum`` is similar to using an ``object``.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   type X = enum
     First
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...
   nnkEnumTy(
     nnkEmpty(),
     nnkIdent("First") # you need at least one nnkIdent or the compiler complains
   )
+  ```
 
 The usage of ``concept`` (experimental) is similar to objects.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   type Con = concept x,y,z
     (x & y & z) is string
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...
   nnkTypeClassTy( # note this isn't nnkConceptTy!
-    nnkArglist(
+    nnkArgList(
       # ... idents for x, y, z
     )
     # ...
   )
+  ```
 
 Static types, like ``static[int]``, use ``nnkIdent`` wrapped in
 ``nnkStaticTy``.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   type A[T: static[int]] = object
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ... within nnkGenericParams
   nnkIdentDefs(
     nnkIdent("T"),
@@ -1161,13 +1319,14 @@ AST:
     nnkEmpty()
   )
   # ...
+  ```
 
 In general, declaring types mirrors this syntax (i.e., ``nnkStaticTy`` for
 ``static``, etc.). Examples follow (exceptions marked by ``*``):
 
--------------                ---------------------------------------------
+=============                =============================================
 Nim type                     Corresponding AST
--------------                ---------------------------------------------
+=============                =============================================
 ``static``                   ``nnkStaticTy``
 ``tuple``                    ``nnkTupleTy``
 ``var``                      ``nnkVarTy``
@@ -1180,7 +1339,7 @@ Nim type                     Corresponding AST
 ``proc``                     ``nnkProcTy``
 ``iterator``                 ``nnkIteratorTy``
 ``object``                   ``nnkObjectTy``
--------------                ---------------------------------------------
+=============                =============================================
 
 Take special care when declaring types as ``proc``. The behavior is similar
 to ``Procedure declaration``, below, but does not treat ``nnkGenericParams``.
@@ -1188,12 +1347,13 @@ Generic parameters are treated in the type, not the ``proc`` itself.
 
 Concrete syntax:
 
-.. code-block:: nim
-  type MyProc[T] = proc(x: T)
+  ```nim
+  type MyProc[T] = proc(x: T) {.nimcall.}
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...
   nnkTypeDef(
     nnkIdent("MyProc"),
@@ -1203,38 +1363,75 @@ AST:
     nnkProcTy( # behaves like a procedure declaration from here on
       nnkFormalParams(
         # ...
-      )
+      ),
+      nnkPragma(nnkIdent("nimcall"))
     )
   )
+  ```
 
 The same syntax applies to ``iterator`` (with ``nnkIteratorTy``), but
 *does not* apply to ``converter`` or ``template``.
 
+Type class versions of these nodes generally share the same node kind but
+without any child nodes. The ``tuple`` type class is represented by
+``nnkTupleClassTy``, while a ``proc`` or ``iterator`` type class with pragmas
+has an ``nnkEmpty`` node in place of the ``nnkFormalParams`` node of a
+concrete ``proc`` or ``iterator`` type node.
+
+  ```nim
+  type TypeClass = proc {.nimcall.} | ref | tuple
+  ```
+
+AST:
+
+  ```nim
+  nnkTypeDef(
+    nnkIdent("TypeClass"),
+    nnkEmpty(),
+    nnkInfix(
+      nnkIdent("|"),
+      nnkProcTy(
+        nnkEmpty(),
+        nnkPragma(nnkIdent("nimcall"))
+      ),
+      nnkInfix(
+        nnkIdent("|"),
+        nnkRefTy(),
+        nnkTupleClassTy()
+      )
+    )
+  )
+  ```
+
 Mixin statement
 ---------------
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   mixin x
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkMixinStmt(nnkIdent("x"))
+  ```
 
 Bind statement
 --------------
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   bind x
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkBindStmt(nnkIdent("x"))
+  ```
 
 Procedure declaration
 ---------------------
@@ -1244,12 +1441,13 @@ a feel for how procedure calls are broken down.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   proc hello*[T: SomeInteger](x: int = 3, y: float32): int {.inline.} = discard
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkProcDef(
     nnkPostfix(nnkIdent("*"), nnkIdent("hello")), # the exported proc name
     nnkEmpty(), # patterns for term rewriting in templates and macros (not procs)
@@ -1277,6 +1475,7 @@ AST:
     nnkEmpty(), # reserved slot for future use
     nnkStmtList(nnkDiscardStmt(nnkEmpty())) # the meat of the proc
   )
+  ```
 
 There is another consideration. Nim has flexible type identification for
 its procs. Even though ``proc(a: int, b: int)`` and ``proc(a, b: int)``
@@ -1284,12 +1483,13 @@ are equivalent in the code, the AST is a little different for the latter.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   proc(a, b: int)
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...AST as above...
   nnkFormalParams(
     nnkEmpty(), # no return here
@@ -1301,24 +1501,27 @@ AST:
     )
   ),
   # ...
+  ```
 
 When a procedure uses the special ``var`` type return variable, the result
 is different from that of a var section.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   proc hello(): var int
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   # ...
   nnkFormalParams(
     nnkVarTy(
       nnkIdent("int")
     )
   )
+  ```
 
 Iterator declaration
 --------------------
@@ -1328,17 +1531,19 @@ replacing ``nnkProcDef``.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   iterator nonsense[T](x: seq[T]): float {.closure.} = ...
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkIteratorDef(
     nnkIdent("nonsense"),
     nnkEmpty(),
     ...
   )
+  ```
 
 Converter declaration
 ---------------------
@@ -1347,16 +1552,18 @@ A converter is similar to a proc.
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   converter toBool(x: float): bool
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkConverterDef(
     nnkIdent("toBool"),
     # ...
   )
+  ```
 
 Template declaration
 --------------------
@@ -1369,12 +1576,13 @@ the ``nnkEmpty()`` as the second argument to ``nnkProcDef`` and
 
 Concrete syntax:
 
-.. code-block:: nim
+  ```nim
   template optOpt{expr1}(a: int): int
+  ```
 
 AST:
 
-.. code-block:: nim
+  ```nim
   nnkTemplateDef(
     nnkIdent("optOpt"),
     nnkStmtList( # instead of nnkEmpty()
@@ -1382,6 +1590,7 @@ AST:
     ),
     # follows like a proc or iterator
   )
+  ```
 
 If the template does not have types for its parameters, the type identifiers
 inside ``nnkFormalParams`` just becomes ``nnkEmpty``.
@@ -1395,8 +1604,9 @@ Macros behave like templates, but ``nnkTemplateDef`` is replaced with
 Hidden Standard Conversion
 --------------------------
 
-.. code-block:: nim
+  ```nim
   var f: float = 1
+  ```
 
 The type of "f" is ``float`` but the type of "1" is actually ``int``. Inserting 
 ``int`` into a ``float`` is a type error. Nim inserts the ``nnkHiddenStdConv``  
diff --git a/doc/backends.rst b/doc/backends.md
similarity index 51%
rename from doc/backends.rst
rename to doc/backends.md
index 896b0f834d..9f0c548354 100644
--- a/doc/backends.rst
+++ b/doc/backends.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 ================================
    Nim Backend Integration
 ================================
@@ -7,27 +5,33 @@
 :Author: Puppet Master
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
+.. no syntax highlighting here by default:
+
 .. contents::
-  "Heresy grows from idleness." -- Unknown.
+
+> "Heresy grows from idleness." -- Unknown.
 
 
 Introduction
 ============
 
-The `Nim Compiler User Guide `_ documents the typical
-compiler invocation, using the `compile` or `c` command to transform a
-`.nim` file into one or more `.c` files which are then compiled with the
+The [Nim Compiler User Guide](nimc.html) documents the typical
+compiler invocation, using the `compile`:option:
+or `c`:option: command to transform a
+``.nim`` file into one or more ``.c`` files which are then compiled with the
 platform's C compiler into a static binary. However, there are other commands
 to compile to C++, Objective-C, or JavaScript. This document tries to
 concentrate in a single place all the backend and interfacing options.
 
 The Nim compiler supports mainly two backend families: the C, C++ and
-Objective-C targets and the JavaScript target. `The C like targets
-<#backends-the-c-like-targets>`_ creates source files that can be compiled
-into a library or a final executable. `The JavaScript target
-<#backends-the-javascript-target>`_ can generate a `.js` file which you
-reference from an HTML file or create a `standalone Node.js program
-`_.
+Objective-C targets and the JavaScript target. [The C like targets](
+#backends-the-c-like-targets) creates source files that can be compiled
+into a library or a final executable. [The JavaScript target](
+#backends-the-javascript-target) can generate a ``.js`` file which you
+reference from an HTML file or create a [standalone Node.js program](
+http://nodejs.org).
 
 On top of generating libraries or standalone applications, Nim offers
 bidirectional interfacing with the backend targets through generic and
@@ -42,35 +46,37 @@ The C like targets
 
 The commands to compile to either C, C++ or Objective-C are:
 
-  //compileToC, cc          compile project with C code generator
-  //compileToCpp, cpp       compile project to C++ code
-  //compileToOC, objc       compile project to Objective C code
+//compileToC, cc          compile project with C code generator
+//compileToCpp, cpp       compile project to C++ code
+//compileToOC, objc       compile project to Objective C code
 
 The most significant difference between these commands is that if you look
-into the `nimcache` directory you will find `.c`, `.cpp` or `.m`
+into the ``nimcache`` directory you will find ``.c``, ``.cpp`` or ``.m``
 files, other than that all of them will produce a native binary for your
-project.  This allows you to take the generated code and place it directly
+project. This allows you to take the generated code and place it directly
 into a project using any of these languages. Here are some typical command-
-line invocations::
+line invocations:
 
-    $ nim c hallo.nim
-    $ nim cpp hallo.nim
-    $ nim objc hallo.nim
+  ```cmd
+  nim c hallo.nim
+  nim cpp hallo.nim
+  nim objc hallo.nim
+  ```
 
 The compiler commands select the target backend, but if needed you can
-`specify additional switches for cross-compilation
-`_ to select the target CPU, operative system
+[specify additional switches for cross-compilation](
+nimc.html#crossminuscompilation) to select the target CPU, operative system
 or compiler/linker commands.
 
 
 The JavaScript target
 ---------------------
 
-Nim can also generate `JavaScript`:idx: code through the `js` command.
+Nim can also generate `JavaScript`:idx: code through the `js`:option: command.
 
 Nim targets JavaScript 1.5 which is supported by any widely used browser.
 Since JavaScript does not have a portable means to include another module,
-Nim just generates a long `.js` file.
+Nim just generates a long ``.js`` file.
 
 Features or modules that the JavaScript platform does not support are not
 available. This includes:
@@ -83,22 +89,24 @@ available. This includes:
 * some modules of the standard library
 * proper 64-bit integer arithmetic
 
-To compensate, the standard library has modules `catered to the JS backend
-`_
+To compensate, the standard library has modules [catered to the JS backend](
+lib.html#pure-libraries-modules-for-js-backend)
 and more support will come in the future (for instance, Node.js bindings
 to get OS info).
 
-To compile a Nim module into a `.js` file use the `js` command; the
-default is a `.js` file that is supposed to be referenced in an `.html`
+To compile a Nim module into a ``.js`` file use the `js`:option: command; the
+default is a ``.js`` file that is supposed to be referenced in an ``.html``
 file. However, you can also run the code with `nodejs`:idx:
-(``_)::
+(http://nodejs.org):
 
+  ```cmd
   nim js -d:nodejs -r examples/hallo.nim
+  ```
 
 If you experience errors saying that `globalThis` is not defined, be
 sure to run a recent version of Node.js (at least 12.0).
 
-  
+
 Interfacing
 ===========
 
@@ -112,14 +120,14 @@ component?).
 Nim code calling the backend
 ----------------------------
 
-Nim code can interface with the backend through the `Foreign function
-interface `_ mainly through the
-`importc pragma `_.
+Nim code can interface with the backend through the [Foreign function
+interface](manual.html#foreign-function-interface) mainly through the
+[importc pragma](manual.html#foreign-function-interface-importc-pragma).
 The `importc` pragma is the *generic* way of making backend symbols available
-in Nim and is available in all the target backends (JavaScript too).  The C++
-or Objective-C backends have their respective `ImportCpp
-`_ and
-`ImportObjC `_
+in Nim and is available in all the target backends (JavaScript too). The C++
+or Objective-C backends have their respective [ImportCpp](
+manual.html#implementation-specific-pragmas-importcpp-pragma) and
+[ImportObjC](manual.html#implementation-specific-pragmas-importobjc-pragma)
 pragmas to call methods from classes.
 
 Whenever you use any of these pragmas you need to integrate native code into
@@ -131,66 +139,67 @@ However, for the C like targets you need to link external code either
 statically or dynamically. The preferred way of integrating native code is to
 use dynamic linking because it allows you to compile Nim programs without
 the need for having the related development libraries installed. This is done
-through the `dynlib pragma for import
-`_, though
-more specific control can be gained using the `dynlib module `_.
+through the [dynlib pragma for import](
+manual.html#foreign-function-interface-dynlib-pragma-for-import), though
+more specific control can be gained using the [dynlib module](dynlib.html).
 
-The `dynlibOverride `_ command line switch allows
+The [dynlibOverride](nimc.html#dynliboverride) command line switch allows
 to avoid dynamic linking if you need to statically link something instead.
-Nim wrappers designed to statically link source files can use the `compile
-pragma `_ if
+Nim wrappers designed to statically link source files can use the [compile
+pragma](manual.html#implementation-specific-pragmas-compile-pragma) if
 there are few sources or providing them along the Nim code is easier than using
 a system library. Libraries installed on the host system can be linked in with
-the `PassL pragma `_.
+the [PassL pragma](manual.html#implementation-specific-pragmas-passl-pragma).
 
-To wrap native code, take a look at the `c2nim tool `_ which helps
+To wrap native code, take a look at the [c2nim tool](
+https://github.com/nim-lang/c2nim/blob/master/doc/c2nim.rst) which helps
 with the process of scanning and transforming header files into a Nim
 interface.
 
-C invocation example
-~~~~~~~~~~~~~~~~~~~~
+### C invocation example
 
-Create a `logic.c` file with the following content:
+Create a ``logic.c`` file with the following content:
 
-.. code-block:: c
+  ```c
   int addTwoIntegers(int a, int b)
   {
     return a + b;
   }
+  ```
 
-Create a `calculator.nim` file with the following content:
-
-.. code-block:: nim
+Create a ``calculator.nim`` file with the following content:
 
+  ```nim
   {.compile: "logic.c".}
   proc addTwoIntegers(a, b: cint): cint {.importc.}
 
   when isMainModule:
     echo addTwoIntegers(3, 7)
+  ```
 
-With these two files in place, you can run `nim c -r calculator.nim` and
-the Nim compiler will compile the `logic.c` file in addition to
-`calculator.nim` and link both into an executable, which outputs `10` when
+With these two files in place, you can run `nim c -r calculator.nim`:cmd: and
+the Nim compiler will compile the ``logic.c`` file in addition to
+``calculator.nim`` and link both into an executable, which outputs `10` when
 run. Another way to link the C file statically and get the same effect would
-be to remove the line with the `compile` pragma and run the following typical
-Unix commands::
+be to remove the line with the `compile` pragma and run the following
+typical Unix commands:
 
-    $ gcc -c logic.c
-    $ ar rvs mylib.a logic.o
-    $ nim c --passL:mylib.a -r calculator.nim
+  ```cmd
+  gcc -c logic.c
+  ar rvs mylib.a logic.o
+  nim c --passL:mylib.a -r calculator.nim
+  ```
 
-Just like in this example we pass the path to the `mylib.a` library (and we
-could as well pass `logic.o`) we could be passing switches to link any other
+Just like in this example we pass the path to the ``mylib.a`` library (and we
+could as well pass ``logic.o``) we could be passing switches to link any other
 static C library.
 
 
-JavaScript invocation example
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+### JavaScript invocation example
 
-Create a `host.html` file with the following content:
-
-.. code-block::
+Create a ``host.html`` file with the following content:
 
+  ```
   
   
   
   
+  ```
 
-Create a `calculator.nim` file with the following content (or reuse the one
+Create a ``calculator.nim`` file with the following content (or reuse the one
 from the previous section):
 
-.. code-block:: nim
-
+  ```nim
   proc addTwoIntegers(a, b: int): int {.importc.}
 
   when isMainModule:
     echo addTwoIntegers(3, 7)
+  ```
 
 Compile the Nim code to JavaScript with `nim js -o:calculator.js
-calculator.nim` and open `host.html` in a browser. If the browser supports
+calculator.nim`:cmd: and open ``host.html`` in a browser. If the browser supports
 javascript, you should see the value `10` in the browser's console. Use the
-`dom module `_ for specific DOM querying and modification procs
-or take a look at `karax `_ for how to
+[dom module](dom.html) for specific DOM querying and modification procs
+or take a look at [karax](https://github.com/pragmagic/karax) for how to
 develop browser-based applications.
 
 
 Backend code calling Nim
 ------------------------
 
-Backend code can interface with Nim code exposed through the `exportc
-pragma `_. The
+Backend code can interface with Nim code exposed through the [exportc
+pragma](manual.html#foreign-function-interface-exportc-pragma). The
 `exportc` pragma is the *generic* way of making Nim symbols available to
 the backends. By default, the Nim compiler will mangle all the Nim symbols to
 avoid any name collision, so the most significant thing the `exportc` pragma
@@ -237,32 +247,32 @@ Also, C code requires you to specify a forward declaration for functions or
 the compiler will assume certain types for the return value and parameters
 which will likely make your program crash at runtime.
 
-The Nim compiler can generate a C interface header through the `--header`
-command-line switch. The generated header will contain all the exported
-symbols and the `NimMain` proc which you need to call before any other
-Nim code.
+The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch.
+Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`.
+
+When compiling to static or dynamic libraries, they don't call destructors of global variables as normal Nim programs would do. A C API `NimDestroyGlobals` is provided to call these global destructors.
 
 
-Nim invocation example from C
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+### Nim invocation example from C
 
-Create a `fib.nim` file with the following content:
-
-.. code-block:: nim
+Create a ``fib.nim`` file with the following content:
 
+  ```nim
   proc fib(a: cint): cint {.exportc.} =
     if a <= 2:
       result = 1
     else:
       result = fib(a - 1) + fib(a - 2)
+  ```
 
-Create a `maths.c` file with the following content:
+Create a ``maths.c`` file with the following content:
 
-.. code-block:: c
-
-  #include "fib.h"
+  ```c
   #include 
 
+  int fib(int a);
+  void NimMain();
+
   int main(void)
   {
     NimMain();
@@ -270,63 +280,66 @@ Create a `maths.c` file with the following content:
       printf("Fib of %d is %d\n", f, fib(f));
     return 0;
   }
+  ```
 
 Now you can run the following Unix like commands to first generate C sources
 from the Nim code, then link them into a static binary along your main C
-program::
+program:
 
-  $ nim c --noMain --noLinking --header:fib.h fib.nim
-  $ gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c
+  ```cmd
+  nim c --noMain --noLinking fib.nim
+  gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c
+  ```
 
 The first command runs the Nim compiler with three special options to avoid
-generating a `main()` function in the generated files, avoid linking the
-object files into a final binary, and explicitly generate a header file for C
-integration. All the generated files are placed into the `nimcache`
-directory. That's why the next command compiles the `maths.c` source plus
-all the `.c` files from `nimcache`. In addition to this path, you also
-have to tell the C compiler where to find Nim's `nimbase.h` header file.
+generating a `main()`:c: function in the generated files and to avoid linking the
+object files into a final binary. All the generated files are placed into the ``nimcache``
+directory. That's why the next command compiles the ``maths.c`` source plus
+all the ``.c`` files from ``nimcache``. In addition to this path, you also
+have to tell the C compiler where to find Nim's ``nimbase.h`` header file.
 
-Instead of depending on the generation of the individual `.c` files you can
-also ask the Nim compiler to generate a statically linked library::
+Instead of depending on the generation of the individual ``.c`` files you can
+also ask the Nim compiler to generate a statically linked library:
 
-  $ nim c --app:staticLib --noMain --header fib.nim
-  $ gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c
+  ```cmd
+  nim c --app:staticLib fib.nim
+  gcc -o m -Inimcache -Ipath/to/nim/lib maths.c libfib.nim.a
+  ```
 
 The Nim compiler will handle linking the source files generated in the
-`nimcache` directory into the `libfib.nim.a` static library, which you can
-then link into your C program.  Note that these commands are generic and will
+``nimcache`` directory into the ``libfib.nim.a`` static library, which you can
+then link into your C program. Note that these commands are generic and will
 vary for each system. For instance, on Linux systems you will likely need to
-use `-ldl` too to link in required dlopen functionality.
+use `-ldl`:option: too to link in required dlopen functionality.
 
 
-Nim invocation example from JavaScript
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+### Nim invocation example from JavaScript
 
-Create a `mhost.html` file with the following content:
-
-.. code-block::
+Create a ``mhost.html`` file with the following content:
 
+  ```
   
   
   
   
+  ```
 
-Create a `fib.nim` file with the following content (or reuse the one
+Create a ``fib.nim`` file with the following content (or reuse the one
 from the previous section):
 
-.. code-block:: nim
-
+  ```nim
   proc fib(a: cint): cint {.exportc.} =
     if a <= 2:
       result = 1
     else:
       result = fib(a - 1) + fib(a - 2)
+  ```
 
-Compile the Nim code to JavaScript with `nim js -o:fib.js fib.nim` and
-open `mhost.html` in a browser. If the browser supports javascript, you
-should see an alert box displaying the text `Fib for 9 is 34`. As mentioned
+Compile the Nim code to JavaScript with `nim js -o:fib.js fib.nim`:cmd: and
+open ``mhost.html`` in a browser. If the browser supports javascript, you
+should see an alert box displaying the text ``Fib for 9 is 34``. As mentioned
 earlier, JavaScript doesn't require an initialization call to `NimMain` or
 a similar function and you can call the exported Nim proc directly.
 
@@ -337,8 +350,8 @@ Nimcache naming logic
 The `nimcache`:idx: directory is generated during compilation and will hold
 either temporary or final files depending on your backend target. The default
 name for the directory depends on the used backend and on your OS but you can
-use the `--nimcache` `compiler switch
-`_ to change it.
+use the `--nimcache`:option: [compiler switch](
+nimc.html#compiler-usage-commandminusline-switches) to change it.
 
 
 Memory management
@@ -356,32 +369,22 @@ aware of who controls what to avoid crashing.
 Strings and C strings
 ---------------------
 
-The manual mentions that `Nim strings are implicitly convertible to
-cstrings `_ which makes interaction usually
+The manual mentions that [Nim strings are implicitly convertible to
+cstrings](manual.html#types-cstring-type) which makes interaction usually
 painless. Most C functions accepting a Nim string converted to a
 `cstring` will likely not need to keep this string around and by the time
-they return the string won't be needed anymore. However, for the rare cases
-where a Nim string has to be preserved and made available to the C backend
-as a `cstring`, you will need to manually prevent the string data from being
-freed with `GC_ref `_ and `GC_unref
-`_.
+they return the string won't be needed anymore.
 
 A similar thing happens with C code invoking Nim code which returns a
 `cstring`. Consider the following proc:
 
-.. code-block:: nim
-
+  ```nim
   proc gimme(): cstring {.exportc.} =
     result = "Hey there C code! " & $rand(100)
+  ```
 
-Since Nim's garbage collector is not aware of the C code, once the
+Since Nim's reference counting mechanism is not aware of the C code, once the
 `gimme` proc has finished it can reclaim the memory of the `cstring`.
-However, from a practical standpoint, the C code invoking the `gimme`
-function directly will be able to use it since Nim's garbage collector has
-not had a chance to run *yet*. This gives you enough time to make a copy for
-the C side of the program, as calling any further Nim procs *might* trigger
-garbage collection making the previously returned string garbage. Or maybe you
-are `yourself triggering the collection `_.
 
 
 Custom data types
@@ -389,43 +392,15 @@ Custom data types
 
 Just like strings, custom data types that are to be shared between Nim and
 the backend will need careful consideration of who controls who. If you want
-to hand a Nim reference to C code, you will need to use `GC_ref
-`_ to mark the reference as used, so it does not get
-freed. And for the C backend you will need to expose the `GC_unref
-`_ proc to clean up this memory when it is not
+to hand a Nim reference to C code, you will need to use [GC_ref](
+system.html#GC_ref,ref.T) to mark the reference as used, so it does not get
+freed. And for the C backend you will need to expose the [GC_unref](
+system.html#GC_unref,ref.T) proc to clean up this memory when it is not
 required anymore.
 
 Again, if you are wrapping a library which *mallocs* and *frees* data
 structures, you need to expose the appropriate *free* function to Nim so
 you can clean it up. And of course, once cleaned you should avoid accessing it
 from Nim (or C for that matter). Typically C data structures have their own
-`malloc_structure` and `free_structure` specific functions, so wrapping
+`malloc_structure`:c: and `free_structure`:c: specific functions, so wrapping
 these for the Nim side should be enough.
-
-
-Thread coordination
--------------------
-
-When the `NimMain()` function is called Nim initializes the garbage
-collector to the current thread, which is usually the main thread of your
-application. If your C code later spawns a different thread and calls Nim
-code, the garbage collector will fail to work properly and you will crash.
-
-As long as you don't use the threadvar emulation Nim uses native thread
-variables, of which you get a fresh version whenever you create a thread. You
-can then attach a GC to this thread via
-
-.. code-block:: nim
-
-  system.setupForeignThreadGc()
-
-It is **not** safe to disable the garbage collector and enable it after the
-call from your background thread even if the code you are calling is short
-lived.
-
-Before the thread exits, you should tear down the thread's GC to prevent memory
-leaks by calling
-
-.. code-block:: nim
-
-  system.tearDownForeignThreadGc()
diff --git a/doc/basicopt.txt b/doc/basicopt.txt
index a2db1c92ec..e8133d2271 100644
--- a/doc/basicopt.txt
+++ b/doc/basicopt.txt
@@ -1,10 +1,9 @@
-::
 
     nim command [options] [projectfile] [arguments]
 
 Command:
   //compile, c              compile project with default code generator (C)
-  //r                       compile to $nimcache/projname, run with [arguments]
+  //r                       compile to $nimcache/projname, run with `arguments`
                             using backend specified by `--backend` (default: c)
   //doc                     generate the documentation for inputfile for
                             backend specified by `--backend` (default: c)
@@ -27,11 +26,11 @@ Options:
   -a, --assertions:on|off   turn assertions on|off
   --opt:none|speed|size     optimize not at all or for speed|size
                             Note: use -d:release for a release build!
-  --debugger:native         Use native debugger (gdb)
+  --debugger:native         use native debugger (gdb)
   --app:console|gui|lib|staticlib
                             generate a console app|GUI app|DLL|static library
   -r, --run                 run the compiled program with given arguments
-  --eval:cmd                evaluates nim code directly; e.g.: `nim --eval:"echo 1"`
+  --eval:cmd                evaluate nim code directly; e.g.: `nim --eval:"echo 1"`
                             defaults to `e` (nimscript) but customizable:
                             `nim r --eval:'for a in stdin.lines: echo a'`
   --fullhelp                show all command line switches
diff --git a/doc/contributing.rst b/doc/contributing.md
similarity index 50%
rename from doc/contributing.rst
rename to doc/contributing.md
index b7188a436a..420c1438e9 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.md
@@ -1,14 +1,18 @@
-.. default-role:: code
-
 ============
 Contributing
 ============
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 .. contents::
 
 
-Contributing happens via "Pull requests" (PR) on github. Every PR needs to be
+Contributing happens via "Pull requests" (PR) on GitHub. Every PR needs to be
 reviewed before it can be merged and the Continuous Integration should be green.
+The title of a PR should contain a brief description. If it fixes an issue,
+in addition to the number of the issue, the title should also contain a description
+of the issue.
 
 The PR has to be approved by two core developers or by Araq.
 
@@ -19,21 +23,23 @@ Writing tests
 
 There are 4 types of tests:
 
-1. `runnableExamples` documentation comment tests, ran by `nim doc mymod.nim`
+1. `runnableExamples` documentation comment tests, ran by `nim doc mymod.nim`:cmd:
    These end up in documentation and ensure documentation stays in sync with code.
 
-2. separate test files, e.g.: `tests/stdlib/tos.nim`.
-   In nim repo, `testament` (see below) runs all `$nim/tests/*/t*.nim` test files;
+2. separate test files, e.g.: ``tests/stdlib/tos.nim``.
+   In nim repo, `testament`:cmd: (see below) runs all
+   ``$nim/tests/*/t*.nim`` test files;
    for nimble packages, see https://github.com/nim-lang/nimble#tests.
 
-3. (deprecated) tests in `when isMainModule:` block, ran by `nim r mymod.nim`.
-   `nimble test` can run those in nimble packages when specified in a
+3. (deprecated) tests in `when isMainModule:` block, ran by `nim r mymod.nim`:cmd:.
+   `nimble test`:cmd: can run those in nimble packages when specified in a
    `task "test"`.
 
-4. (not preferred) `.. code-block:: nim` RST snippets; these should only be used in rst sources,
+4. (not preferred) ``.. code-block:: nim`` RST snippets;
+   these should only be used in rst sources,
    in nim sources `runnableExamples` should now always be preferred to those for
    several reasons (cleaner syntax, syntax highlights, batched testing, and
-   `rdoccmd` allows customization).
+   parameter `rdoccmd` allows customization).
 
 Not all the tests follow the convention here, feel free to change the ones
 that don't. Always leave the code cleaner than you found it.
@@ -41,8 +47,8 @@ that don't. Always leave the code cleaner than you found it.
 Stdlib
 ------
 
-Each stdlib module (anything under `lib/`, e.g. `lib/pure/os.nim`) should
-preferably have a corresponding separate test file, e.g. `tests/stdlib/tos.nim`.
+Each stdlib module (anything under ``lib/``, e.g. ``lib/pure/os.nim``) should
+preferably have a corresponding separate test file, e.g. ``tests/stdlib/tos.nim``.
 The old convention was to add a `when isMainModule:` block in the source file,
 which only gets executed when the tester is building the file.
 
@@ -53,8 +59,7 @@ things like `echo "done"`. Don't use `unittest.suite` and `unittest.test`.
 
 Sample test:
 
-.. code-block:: nim
-
+  ```nim
   block: # foo
     doAssert foo(1) == 10
 
@@ -70,42 +75,42 @@ Sample test:
                         @[false, false], @[false, false]]
     # doAssert with `not` can now be done as follows:
     doAssert not (1 == 2)
+  ```
 
-Always refer to a GitHub issue using the following exact syntax: `bug #1234` as shown
+Always refer to a GitHub issue using the following exact syntax: ``bug #1234`` as shown
 above, so that it's consistent and easier to search or for tooling. Some browser
 extensions (e.g. https://github.com/sindresorhus/refined-github) will even turn those
 in clickable links when it works.
 
 Rationale for using a separate test file instead of `when isMainModule:` block:
 * allows custom compiler flags or testing options (see details below)
-* faster CI since they can be joined in `megatest` (combined into a single test)
+* faster CI since they can be joined in ``megatest`` (combined into a single test)
 * avoids making the parser do un-necessary work when a source file is merely imported
 * avoids mixing source and test code when reporting line of code statistics or code coverage
 
 Compiler
 --------
 
-The tests for the compiler use a testing tool called `testament`. They are all
-located in `tests/` (e.g.: `tests/destructor/tdestructor3.nim`).
+The tests for the compiler use a testing tool called `testament`:cmd:. They are all
+located in ``tests/`` (e.g.: ``tests/destructor/tdestructor3.nim``).
 Each test has its own file. All test files are prefixed with `t`. If you want
 to create a file for import into another test only, use the prefix `m`.
 
 At the beginning of every test is the expected behavior of the test.
 Possible keys are:
 
-- `cmd`: A compilation command template e.g. `nim $target --threads:on $options $file`
+- `cmd`: A compilation command template e.g. `nim $target --threads:on $options $file`:cmd:
 - `output`: The expected output (stdout + stderr), most likely via `echo`
 - `exitcode`: Exit code of the test (via `exit(number)`)
 - `errormsg`: The expected compiler error message
 - `file`: The file the errormsg was produced at
 - `line`: The line the errormsg was produced at
 
-For a full spec, see here: `testament/specs.nim`
+For a full spec, see here: ``testament/specs.nim``
 
 An example of a test:
 
-.. code-block:: nim
-
+  ```nim
   discard """
     errormsg: "type mismatch: got (PTest)"
   """
@@ -117,6 +122,7 @@ An example of a test:
 
   var buf: PTest
   buf.test()
+  ```
 
 
 Running tests
@@ -124,79 +130,78 @@ Running tests
 
 You can run the tests with
 
-::
-
+  ```cmd
   ./koch tests
+  ```
 
 which will run a good subset of tests. Some tests may fail. If you
 only want to see the output of failing tests, go for
 
-::
-
+  ```cmd
   ./koch tests --failing all
+  ```
 
 You can also run only a single category of tests. A category is a subdirectory
-in the `tests` directory. There are a couple of special categories; for a
-list of these, see `testament/categories.nim`, at the bottom.
+in the ``tests/`` directory. There are a couple of special categories; for a
+list of these, see ``testament/categories.nim``, at the bottom.
 
-::
-
-  ./koch tests c lib # compiles/runs stdlib modules, including `isMainModule` tests
+  ```cmd
+  ./koch tests c lib # compiles / runs stdlib modules, including `isMainModule` tests
   ./koch tests c megatest # runs a set of tests that can be combined into 1
+  ```
 
 To run a single test:
 
-::
-
+  ```cmd
   ./koch test run /    # e.g.: tuples/ttuples_issues
   ./koch test run tests/stdlib/tos.nim # can also provide relative path
+  ```
 
 For reproducible tests (to reproduce an environment more similar to the one
-run by Continuous Integration on travis/appveyor), you may want to disable your
-local configuration (e.g. in `~/.config/nim/nim.cfg`) which may affect some
+run by Continuous Integration on GitHub actions/azure pipelines), you may want to disable your
+local configuration (e.g. in ``~/.config/nim/nim.cfg``) which may affect some
 tests; this can also be achieved by using
-`export XDG_CONFIG_HOME=pathtoAlternateConfig` before running `./koch`
+`export XDG_CONFIG_HOME=pathtoAlternateConfig`:cmd: before running `./koch`:cmd:
 commands.
 
 
 Comparing tests
 ===============
 
-Test failures can be grepped using `Failure:`.
+Test failures can be grepped using ``Failure:``.
 
 The tester can compare two test runs. First, you need to create a
 reference test. You'll also need to the commit id, because that's what
 the tester needs to know in order to compare the two.
 
-::
-
+  ```cmd
   git checkout devel
   DEVEL_COMMIT=$(git rev-parse HEAD)
   ./koch tests
+  ```
 
 Then switch over to your changes and run the tester again.
 
-::
-
+  ```cmd
   git checkout your-changes
   ./koch tests
+  ```
 
-Then you can ask the tester to create a `testresults.html` which will
+Then you can ask the tester to create a ``testresults.html`` which will
 tell you if any new tests passed/failed.
 
-::
-
+  ```cmd
   ./koch tests --print html $DEVEL_COMMIT
+  ```
 
 
 Deprecation
 ===========
 
-Backward compatibility is important, so instead of a rename you need to deprecate
-the old name and introduce a new name:
-
-.. code-block:: nim
+Backwards compatibility is important. When renaming types, procedures, etc. the old name
+must be marked as deprecated using the `deprecated` pragma:
 
+  ```nim
   # for routines (proc/template/macro/iterator) and types:
   proc oldProc(a: int, b: float): bool {.deprecated:
       "deprecated since v1.2.3; use `newImpl: string -> int` instead".} = discard
@@ -208,9 +213,10 @@ the old name and introduce a new name:
   # (likewise with object types and their fields):
   type Bar {.deprecated.} = enum bar0, bar1
   type Barz  = enum baz0, baz1 {.deprecated.}, baz2
+  ```
 
 
-See also `Deprecated `_
+See also [Deprecated](manual.html#pragmas-deprecated-pragma)
 pragma in the manual.
 
 
@@ -219,72 +225,127 @@ Documentation
 
 When contributing new procs, be sure to add documentation, especially if
 the proc is public. Even private procs benefit from documentation and can be
-viewed using `nim doc --docInternal foo.nim`.
+viewed using `nim doc --docInternal foo.nim`:cmd:.
 Documentation begins on the line
 following the `proc` definition, and is prefixed by `##` on each line.
 
 Runnable code examples are also encouraged, to show typical behavior with a few
 test cases (typically 1 to 3 `assert` statements, depending on complexity).
-These `runnableExamples` are automatically run by `nim doc mymodule.nim`
-as well as `testament` and guarantee they stay in sync.
+These `runnableExamples` are automatically run by `nim doc mymodule.nim`:cmd:
+as well as `testament`:cmd: and guarantee they stay in sync.
 
-.. code-block:: nim
+  ```nim
   proc addBar*(a: string): string =
     ## Adds "Bar" to `a`.
     runnableExamples:
       assert "baz".addBar == "bazBar"
     result = a & "Bar"
+  ```
 
-See `parentDir `_ example.
+See [parentDir](os.html#parentDir,string) example.
 
 The RestructuredText Nim uses has a special syntax for including code snippets
-embedded in documentation; these are not run by `nim doc` and therefore are
+embedded in documentation; these are not run by `nim doc`:cmd: and therefore are
 not guaranteed to stay in sync, so `runnableExamples` is almost always preferred:
 
-.. code-block:: nim
-
+  ````nim
   proc someProc*(): string =
     ## Returns "something"
     ##
-    ## .. code-block::
-    ##  echo someProc() # "something"
+    ##   ```
+    ##   echo someProc() # "something"
+    ##   ```
     result = "something" # single-hash comments do not produce documentation
+  ````
 
-The `.. code-block:: nim` followed by a newline and an indentation instructs the
-`nim doc` command to produce syntax-highlighted example code with the
-documentation (`.. code-block::` is sufficient from inside a nim module).
+The \`\`\` followed by a newline and an indentation instructs the
+`nim doc`:cmd: command to produce syntax-highlighted example code with the
+documentation (\`\`\` is sufficient inside a ``.nim`` module, while from
+a ``.md`` one needs to set the language explicitly as \`\`\`nim).
 
 When forward declaration is used, the documentation should be included with the
 first appearance of the proc.
 
-.. code-block:: nim
-
+  ```nim
   proc hello*(): string
     ## Put documentation here
   proc nothing() = discard
   proc hello*(): string =
     ## ignore this
     echo "hello"
+  ```
 
 The preferred documentation style is to begin with a capital letter and use
 the third-person singular. That is, between:
 
-.. code-block:: nim
-
+  ```nim
   proc hello*(): string =
     ## Returns "hello"
     result = "hello"
+  ```
 
 or
 
-.. code-block:: nim
-
+  ```nim
   proc hello*(): string =
     ## say hello
     result = "hello"
+  ```
 
 the first is preferred.
 
+When you specify an *RST role* (highlighting/interpretation marker) do it
+in the postfix form for uniformity, that is after \`text in backticks\`.
+For example an ``:idx:`` role for referencing a topic ("SQLite" in the
+example below) from [Nim Index] can be used in doc comment this way:
+
+  ```nim
+  ## A higher level `SQLite`:idx: database wrapper.
+  ```
+
+.. _`Nim Index`: https://nim-lang.org/docs/theindex.html
+
+Inline monospaced text can be input using \`single backticks\` or
+\`\`double backticks\`\`. The former are syntactically highlighted,
+the latter are not.
+To avoid accidental highlighting follow this rule in ``*.nim`` files:
+
+* Use single backticks for fragments of code in Nim and other
+  programming languages, including identifiers, in ``*.nim`` files.
+
+  For languages other than Nim add a role after final backtick,
+  e.g. for C++ inline highlighting:
+
+      `#include `:cpp:
+
+  For a currently unsupported language add the `:code:` role,
+  like for SQL in this example:
+
+      `SELECT * FROM ;`:code:
+
+  Highlight shell commands by ``:cmd:`` role; for command line options use
+  ``:option:`` role, e.g.: \`--docInternal\`:option:.
+
+* Use double backticks:
+
+  * For file names: \`\`os.nim\`\`
+  * For fragments of strings **not** enclosed by `"` and `"` and not
+    related to code, e.g. text of compiler messages
+  * When code ends with a standalone ``\`` (otherwise a combination of
+    ``\`` and a final \` would get escaped)
+
+.. Note:: ``*.rst`` files have ``:literal:`` as their default role.
+  So for them the rule above is only applicable if the ``:nim:`` role
+  is set up manually as the default [^1]:
+
+      .. role:: nim(code)
+         :language: nim
+      .. default-role:: nim
+
+  The first 2 lines are for other RST implementations,
+  including Github one.
+
+  [^1]: this is fulfilled when ``doc/rstcommon.rst`` is included.
 
 Best practices
 ==============
@@ -298,85 +359,85 @@ New `defined(foo)` symbols need to be prefixed by the nimble package name, or
 by `nim` for symbols in nim sources (e.g. compiler, standard library). This is
 to avoid name conflicts across packages.
 
-.. code-block:: nim
-
+  ```nim
   # if in nim sources
   when defined(allocStats): discard # bad, can cause conflicts
   when defined(nimAllocStats): discard # preferred
-  # if in a pacakge `cligen`:
+  # if in a package `cligen`:
   when defined(debug): discard # bad, can cause conflicts
   when defined(cligenDebug): discard # preferred
+  ```
 
 .. _noimplicitbool:
 Take advantage of no implicit bool conversion
 
-.. code-block:: nim
-
+  ```nim
   doAssert isValid() == true
   doAssert isValid() # preferred
+  ```
 
 .. _design_for_mcs:
 Design with method call syntax chaining in mind
 
-.. code-block:: nim
-
+  ```nim
   proc foo(cond: bool, lines: seq[string]) # bad
   proc foo(lines: seq[string], cond: bool) # preferred
   # can be called as: `getLines().foo(false)`
+  ```
 
 .. _avoid_quit:
-Use exceptions (including assert / doAssert) instead of `quit`
+Use exceptions (including `assert` / `doAssert`) instead of `quit`
 rationale: https://forum.nim-lang.org/t/4089
 
-.. code-block:: nim
-
+  ```nim
   quit() # bad in almost all cases
   doAssert() # preferred
+  ```
 
 .. _tests_use_doAssert:
 Use `doAssert` (or `unittest.check`, `unittest.require`), not `assert` in all
-tests so they'll be enabled even with `--assertions:off`.
-
-.. code-block:: nim
+tests, so they'll be enabled even with `--assertions:off`:option:.
 
+  ```nim
   block: # foo
     assert foo() # bad
     doAssert foo() # preferred
+  ```
 
 .. _runnableExamples_use_assert:
-An exception to the above rule is `runnableExamples` and `code-block` rst blocks
+An exception to the above rule is `runnableExamples` and ``code-block`` rst blocks
 intended to be used as `runnableExamples`, which for brevity use `assert`
-instead of `doAssert`. Note that `nim doc -d:danger main` won't pass `-d:danger` to the
-`runnableExamples`, but `nim doc --doccmd:-d:danger main` would, and so would the
+instead of `doAssert`. Note that `nim doc -d:danger main`:cmd: won't pass `-d:danger`:option: to the
+`runnableExamples`, but `nim doc --doccmd:-d:danger main`:cmd: would, and so would the
 second example below:
 
-.. code-block:: nim
-
+  ```nim
   runnableExamples:
     doAssert foo() # bad
     assert foo() # preferred
 
   runnableExamples("-d:danger"):
     doAssert foo() # `assert` would be disabled here, so `doAssert` makes more sense
+  ```
 
 .. _delegate_printing:
 Delegate printing to caller: return `string` instead of calling `echo`
 rationale: it's more flexible (e.g. allows the caller to call custom printing,
-including prepending location info, writing to log files, etc).
-
-.. code-block:: nim
+including prepending location info, writing to log files, etc.).
 
+  ```nim
   proc foo() = echo "bar" # bad
   proc foo(): string = "bar" # preferred (usually)
+  ```
 
 .. _use_Option:
-[Ongoing debate] Consider using Option instead of return bool + var argument,
+(Ongoing debate) Consider using Option instead of return bool + var argument,
 unless stack allocation is needed (e.g. for efficiency).
 
-.. code-block:: nim
-
+  ```nim
   proc foo(a: var Bar): bool
   proc foo(): Option[Bar]
+  ```
 
 .. _use_doAssert_not_echo:
 Tests (including in testament) should always prefer assertions over `echo`,
@@ -384,14 +445,14 @@ except when that's not possible. It's more precise, easier for readers and
 maintainers to where expected values refer to. See for example
 https://github.com/nim-lang/Nim/pull/9335 and https://forum.nim-lang.org/t/4089
 
-.. code-block:: nim
-
+  ```nim
   echo foo() # adds a line for testament in `output:` block inside `discard`.
   doAssert foo() == [1, 2] # preferred, except when not possible to do so.
+  ```
 
 
-The Git stuff
-=============
+The `git`:cmd: stuff
+====================
 
 General commit rules
 --------------------
@@ -399,14 +460,14 @@ General commit rules
 1. Important, critical bugfixes that have a tiny chance of breaking
    somebody's code should be backported to the latest stable release
    branch (currently 1.4.x) and maybe also all the way back to the 1.0.x branch.
-   The commit message should contain the tag `[backport]` for "backport to all
-   stable releases" and the tag `[backport:$VERSION]` for backporting to the
-   given $VERSION.
+   The commit message should contain the tag ``[backport]`` for "backport to the latest
+   stable release" and the tag ``[backport:$VERSION]`` for backporting back to the
+   given $VERSION (and all newer releases).
 
 2. If you introduce changes which affect backward compatibility,
-   make breaking changes, or have PR which is tagged as `[feature]`,
-   the changes should be mentioned in `the changelog
-   `_.
+   make breaking changes, or have PR which is tagged as ``[feature]``,
+   the changes should be mentioned in [the changelog](
+   https://github.com/nim-lang/Nim/blob/devel/changelog.md).
 
 3. All changes introduced by the commit (diff lines) must be related to the
    subject of the commit.
@@ -415,44 +476,44 @@ General commit rules
    your editor reformatted automatically the code or whatever different reason,
    this should be excluded from the commit.
 
-   *Tip:* Never commit everything as is using `git commit -a`, but review
-   carefully your changes with `git add -p`.
+   *Tip:* Never commit everything as-is using `git commit -a`:cmd:, but review
+   carefully your changes with `git add -p`:cmd:.
 
 4. Changes should not introduce any trailing whitespace.
 
-   Always check your changes for whitespace errors using `git diff --check`
-   or add the following `pre-commit` hook:
+   Always check your changes for whitespace errors using `git diff --check`:cmd:
+   or add the following ``pre-commit`` hook:
 
-   .. code-block:: sh
-
-      #!/bin/sh
-      git diff --check --cached || exit $?
+     ```cmd
+     #!/bin/sh
+     git diff --check --cached || exit $?
+     ```
 5. Describe your commit and use your common sense.
    Example commit message:
 
-   `Fixes #123; refs #124`
+       Fixes #123; refs #124
 
-   indicates that issue `#123` is completely fixed (GitHub may automatically
-   close it when the PR is committed), wheres issue `#124` is referenced
+   indicates that issue ``#123`` is completely fixed (GitHub may automatically
+   close it when the PR is committed), whereas issue ``#124`` is referenced
    (e.g.: partially fixed) and won't close the issue when committed.
 
-6. PR body (not just PR title) should contain references to fixed/referenced github
-   issues, e.g.: `fix #123` or `refs #123`. This is so that you get proper cross
-   referencing from linked issue to the PR (github won't make those links with just
-   PR title, and commit messages aren't always sufficient to ensure that, e.g.
-   can't be changed after a PR is merged).
+6. PR body (not just PR title) should contain references to fixed/referenced GitHub
+   issues, e.g.: ``fix #123`` or ``refs #123``. This is so that you get proper
+   cross-referencing from linked issue to the PR (GitHub won't make those links
+   with just a PR title, and commit messages aren't always sufficient to ensure
+   that, e.g. can't be changed after a PR is merged).
 
-7. Commits should be always be rebased against devel (so a fast forward
+7. Commits should be always be rebased against devel (so a fast-forward
    merge can happen)
 
-   e.g.: use `git pull --rebase origin devel`. This is to avoid messing up
+   e.g.: use `git pull --rebase origin devel`:cmd:. This is to avoid messing up
    git history.
    Exceptions should be very rare: when rebase gives too many conflicts, simply
    squash all commits using the script shown in
    https://github.com/nim-lang/Nim/pull/9356
 
 8. Do not mix pure formatting changes (e.g. whitespace changes, nimpretty) or
-   automated changes (e.g. nimfix) with other code changes: these should be in
+   automated changes with other code changes: these should be in
    separate commits (and the merge on GitHub should not squash these into 1).
 
 
@@ -461,14 +522,17 @@ Continuous Integration (CI)
 
 1. Continuous Integration is by default run on every push in a PR; this clogs
    the CI pipeline and affects other PR's; if you don't need it (e.g. for WIP or
-   documentation only changes), add `[ci skip]` to your commit message title.
-   This convention is supported by `Appveyor
-   `_
-   and `Travis `_.
+   documentation only changes), add ``[skip ci]`` to your commit message title.
+   This convention is supported by our GitHub actions pipelines and our azure pipeline
+   (using custom logic, which should complete in < 1mn) as well as our former other pipelines:
+   [Appveyor](
+   https://www.appveyor.com/docs/how-to/filtering-commits/#skip-directive-in-commit-message)
+   and [Travis](
+   https://docs.travis-ci.com/user/customizing-the-build/#skipping-a-build).
 
 2. Consider enabling CI (azure, GitHub actions and builds.sr.ht) in your own Nim fork, and
    waiting for CI to be green in that fork (fixing bugs as needed) before
-   opening your PR in the original Nim repo, so as to reduce CI congestion. Same
+   opening your PR in the original Nim repo, to reduce CI congestion. Same
    applies for updates on a PR: you can test commits on a separate private
    branch before updating the main PR.
 
@@ -481,18 +545,20 @@ Debugging CI failures, flaky tests, etc
 2. If CI failure seems unrelated to your PR, it could be caused by a flaky test.
    File a bug for it if it isn't already reported. A PR push (or opening/closing PR)
    will re-trigger all CI jobs (even successful ones, which can be wasteful). Instead,
+   request collaboration from the Nim team. The Nim team should
    follow these instructions to only restart the jobs that failed:
 
-  * Azure: if on your own fork, it's possible from inside azure console
-    (e.g. `dev.azure.com/username/username/_build/results?buildId=1430&view=results`) via `rerun failed jobs` on top.
-    If either on you own fork or in Nim repo, it's possible from inside GitHub UI
-    under checks tab, see https://github.com/timotheecour/Nim/issues/211#issuecomment-629751569
-  * GitHub actions: under "Checks" tab, click "Re-run jobs" in the right.
-  * builds.sr.ht: create a sourcehut account so you can restart a PR job as illustrated.
-    builds.sr.ht also allows you to ssh to a CI machine which can help a lot for debugging
-    issues, see docs in https://man.sr.ht/builds.sr.ht/build-ssh.md and
-    https://drewdevault.com/2019/08/19/Introducing-shell-access-for-builds.html; see
-    https://man.sr.ht/tutorials/set-up-account-and-git.md to generate and upload ssh keys.
+   * Azure: if on your own fork, it's possible from inside azure console
+     (e.g. ``dev.azure.com/username/username/_build/results?buildId=1430&view=results``) via
+     ``rerun failed jobs`` on top.
+     If either on you own fork or in Nim repo, it's possible from inside GitHub UI
+     under checks tab, see https://github.com/timotheecour/Nim/issues/211#issuecomment-629751569
+   * GitHub actions: under "Checks" tab, click "Re-run jobs" in the right.
+   * builds.sr.ht: create a SourceHut account so that you can restart a PR job as illustrated.
+     builds.sr.ht also allows you to ssh to a CI machine which can help a lot for debugging
+     issues, see docs in https://man.sr.ht/builds.sr.ht/build-ssh.md and
+     https://drewdevault.com/2019/08/19/Introducing-shell-access-for-builds.html; see
+     https://man.sr.ht/tutorials/set-up-account-and-git.md to generate and upload ssh keys.
 
 
 Code reviews
@@ -503,26 +569,25 @@ Code reviews
    https://forum.nim-lang.org/t/4317
 
 2. When reviewing large diffs that may involve code moving around, GitHub's interface
-   doesn't help much as it doesn't highlight moves. Instead, you can use something
-   like this, see visual results `here `_:
+   doesn't help much, as it doesn't highlight moves. Instead, you can use something
+   like this, see visual results [here](
+   https://github.com/nim-lang/Nim/pull/10431#issuecomment-456968196):
 
-   .. code-block:: sh
-
-      git fetch origin pull/10431/head && git checkout FETCH_HEAD
-      git diff --color-moved-ws=allow-indentation-change --color-moved=blocks HEAD^
+     ```cmd
+     git fetch origin pull/10431/head && git checkout FETCH_HEAD
+     git diff --color-moved-ws=allow-indentation-change --color-moved=blocks HEAD^
+     ```
 
 3. In addition, you can view GitHub-like diffs locally to identify what was changed
-   within a code block using `diff-highlight` or `diff-so-fancy`, e.g.:
+   within a code block using `diff-highlight`:cmd: or `diff-so-fancy`:cmd:, e.g.:
 
-   .. code-block:: sh
-
-      # put this in ~/.gitconfig:
-      [core]
-        pager = "diff-so-fancy | less -R" # or: use: `diff-highlight`
+        # put this in ~/.gitconfig:
+        [core]
+          pager = "diff-so-fancy | less -R" # or: use: `diff-highlight`
 
 
 
-.. include:: docstyle.rst
+.. include:: docstyle.md
 
 
 Evolving the stdlib
@@ -556,7 +621,7 @@ Time handling, especially the `Time` type are also covered by this rule.
 Existing, battle-tested modules stay
 ------------------------------------
 
-Reason: There is no benefit in moving them around just to fullfill some design
+Reason: There is no benefit in moving them around just to fulfill some design
 fashion as in "Nim's core MUST BE SMALL". If you don't like an existing module,
 don't import it. If a compilation target (e.g. JS) cannot support a module,
 document this limitation.
@@ -570,7 +635,7 @@ Syntactic helpers can start as experimental stdlib modules
 
 Reason: Generally speaking as external dependencies they are not exposed
 to enough users so that we can see if the shortcuts provide enough benefit
-or not. Many programmers avoid external dependencies, even moreso for
+or not. Many programmers avoid external dependencies, especially for
 "tiny syntactic improvements". However, this is only true for really good
 syntactic improvements that have the potential to clean up other parts of
 the Nim library substantially. If in doubt, new stdlib modules should start
@@ -595,7 +660,7 @@ to existing modules is acceptable. For two reasons:
    ("Why does sequtils lack a `countIt`?
    Because version 1.0 happens to have lacked it? Silly...")
 2. To encourage contributions. Contributors often start with PRs that
-   add simple things and then they stay and also fix bugs. Nim is an
+   add simple things, then they stay and also fix bugs. Nim is an
    open source project and lives from people's contributions and involvement.
    Newly introduced issues have to be balanced against motivating new people. We know where
    to find perfectly designed pieces of software that have no bugs -- these are the systems
@@ -603,16 +668,131 @@ to existing modules is acceptable. For two reasons:
 
 Conventions
 -----------
-1. New stdlib modules should go under `Nim/lib/std/`. The rationale is to
+1. New stdlib modules should go under ``Nim/lib/std/``. The rationale is to
    require users to import via `import std/foo` instead of `import foo`,
    which would cause potential conflicts with nimble packages.
    Note that this still applies for new modules in existing logical
-   directories, e.g.: use `lib/std/collections/foo.nim`,
-   not `lib/pure/collections/foo.nim`.
+   directories, e.g.: use ``lib/std/collections/foo.nim``,
+   not ``lib/pure/collections/foo.nim``.
 
 2. New module names should prefer plural form whenever possible, e.g.:
-   `std/sums.nim` instead of `std/sum.nim`. In particular, this reduces
+   ``std/sums.nim`` instead of ``std/sum.nim``. In particular, this reduces
    chances of conflicts between module name and the symbols it defines.
    Furthermore, module names should use `snake_case` and not use capital
    letters, which cause issues when going from an OS without case
    sensitivity to an OS with it.
+
+
+Breaking Changes
+================
+
+Introducing breaking changes, no matter how well-intentioned,
+creates long-term problems for the community, in particular those looking to promote
+reusable Nim code in libraries: In the Nim distribution, critical security and bugfixes,
+language changes and community improvements are bundled in a single distribution - it is
+difficult to make partial upgrades with only benign changes. When one library depends on
+a legacy behavior, it can no longer be used together with another library that does not,
+breaking all downstream applications - the standard library is unique in that it sits at
+the root of **all** dependency trees.
+
+There is a big difference between compile-time breaking changes and run-time breaking
+changes.
+
+
+Run-time breaking changes
+-------------------------
+
+Run-time breaking changes are to be avoided at almost all costs: Nim is used for
+mission critical applications which depend on behaviours that
+are not covered by the test suite. As such, it's important that changes to the
+*stable* parts of the standard library are made avoiding changing the existing
+behaviors, even when the test suite continues to pass.
+
+Examples of run-time breaking changes:
+
+- Raising exceptions of a new type, compared to what's currently being raised.
+
+- Adding unconstrained or poorly constrained generic procs or macros
+  ("hash now works for all `ref T`"): This may cause code to behave differently
+  depending only on which modules are imported - common examples include `==` and `hash`.
+
+- Changing behavior of existing functions like:
+
+  * "Nim's path handling procs like `getXDir` now consistently lack the trailing slash"
+  * "Nim's strformat implementation is now more consistent with Python"
+
+Instead, write new code that explicitly announces the feature you think we announced but
+didn't. For example, `strformat` does not say "it's compatible with Python", it
+says "inspired by Python's f-strings". This new code can be submitted to the stdlib
+and the old code can be deprecated or published as a Nimble package.
+
+Sometimes, a run-time breaking change is most desirable: For example, a string
+representation of a floating point number that "roundtrips" is much better than
+a string representation that doesn't. These run-time breaking changes must start in the
+state "opt-in" via a new `-d:nimPreviewX` or command line flag and then should become
+the new default later, in follow-up versions. This way users can track
+regressions more easily. ("git bisect" is not an acceptable alternative, that's for
+Nim compiler developers, not for Nim users.)
+
+Above all else, additive approaches that don't change existing behaviors
+should be preferred.
+
+
+Compile-time breaking changes
+-----------------------------
+
+Compile-time breaking changes are usually easier to handle, but for large code bases
+they can also involve a large amount of work and can hinder the adoption of a new
+Nim release.
+Additive approaches are to be preferred here as well.
+
+Examples of compile-time breaking changes include (but are not limited to):
+
+* Renaming functions and modules, or moving things. Instead of a direct rename,
+  deprecate the old name and introduce a new one.
+* Renaming the parameter names: Thanks to Nim's "named parameter" calling syntax
+  like `f(x = 0, y = 1)` this is a breaking change. Instead, live with the existing
+  parameter names.
+* Adding an enum value to an existing enum. Nim's exhaustive case statements stop
+  compiling after such a change. Instead, consider to introduce new `bool`
+  fields/parameters. This can be impractical though, so we use good judgement
+  and our list of "important packages" to see if it doesn't break too much code
+  out there in practice.
+* Adding a new proc to an existing stdlib module. However, for aesthetic reasons
+  this is often preferred over introducing a new module with just a single proc
+  inside. We use good judgement and our list of "important packages" to see if
+  it doesn't break too much code out there in practice. The new procs need to
+  be annotated with a `.since` annotation.
+
+
+Compiler/language spec bugfixes
+-------------------------------
+
+This can even be applied to compiler "bugfixes": If the compiler should have been
+"pickier" in its handling of `typedesc`, instead of "fixing typedesc handling bugs",
+consider the following solution:
+
+- Spec out how `typedesc` should really work and also spec out the cases where it
+  should not be allowed!
+- Deprecate `typedesc` and name the new metatype something new like `typeArg`.
+- Implement the spec.
+
+
+Non-breaking changes
+--------------------
+
+Examples of changes that are considered non-breaking (or acceptable breaking changes) include:
+
+* Creating a new module.
+* Adding an overload to an already overloaded proc.
+* Adding new default parameters to an existing proc. It is assumed that you do not
+  use Nim's stdlib procs's addresses (that you don't use them as first class entities).
+* Changing the calling convention from `nimcall` to `inline`
+  (but first RFC https://github.com/nim-lang/RFCs/issues/396 needs to be implemented).
+* Changing the behavior from "crashing" into some other, well documented result (including
+  raising a Defect, but not raising an exception that does not inherit from Defect).
+* Adding new fields to an existing object.
+
+Nim's introspection facilities imply that strictly speaking almost every addition can
+break somebody's code. It is impractical to care about these cases, a change that only
+affects introspection is not considered to be a breaking change.
diff --git a/doc/destructors.rst b/doc/destructors.md
similarity index 63%
rename from doc/destructors.rst
rename to doc/destructors.md
index 01e2d2ee9b..e192fd362c 100644
--- a/doc/destructors.rst
+++ b/doc/destructors.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 ==================================
 Nim Destructors and Move Semantics
 ==================================
@@ -7,18 +5,20 @@ Nim Destructors and Move Semantics
 :Authors: Andreas Rumpf
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
 About this document
 ===================
 
-This document describes the upcoming Nim runtime which does
+This document describes the ARC/ORC Nim runtime which does
 not use classical GC algorithms anymore but is based on destructors and
-move semantics. The new runtime's advantages are that Nim programs become
+move semantics. The advantages are that Nim programs become
 oblivious to the involved heap sizes and programs are easier to write to make
 effective use of multi-core machines. As a nice bonus, files and sockets and
-the like will not require manual `close` calls anymore.
+the like can be written not to require manual `close` calls anymore.
 
 This document aims to be a precise specification about how
 move semantics and destructors work in Nim.
@@ -30,23 +30,31 @@ Motivating example
 With the language mechanisms described here, a custom seq could be
 written as:
 
-.. code-block:: nim
-
+  ```nim test
   type
     myseq*[T] = object
       len, cap: int
       data: ptr UncheckedArray[T]
 
-  proc `=destroy`*[T](x: var myseq[T]) =
+  proc `=destroy`*[T](x: myseq[T]) =
     if x.data != nil:
       for i in 0..`_.
+"Lifetime-tracking hooks", which are particular [type bound operators](
+manual.html#procedures-type-bound-operators).
 
-There are 3 different hooks for each (generic or concrete) object type `T` (`T` can also be a
+There are 6 different hooks for each (generic or concrete) object type `T` (`T` can also be a
 `distinct` type) that are called implicitly by the compiler.
 
 (Note: The word "hook" here does not imply any kind of dynamic binding
@@ -111,22 +129,47 @@ other associated resources. Variables are destroyed via this hook when
 they go out of scope or when the routine they were declared in is about
 to return.
 
-The prototype of this hook for a type `T` needs to be:
-
-.. code-block:: nim
-
-  proc `=destroy`(x: var T)
+A `=destroy` hook is allowed to have a parameter of a `var T` or `T` type. Taking a `var T` type is deprecated. The prototype of this hook for a type `T` needs to be:
 
+  ```nim
+  proc `=destroy`(x: T)
+  ```
 
 The general pattern in `=destroy` looks like:
 
-.. code-block:: nim
-
-  proc `=destroy`(x: var T) =
+  ```nim
+  proc `=destroy`(x: T) =
     # first check if 'x' was moved to somewhere else:
     if x.field != nil:
       freeResource(x.field)
+  ```
 
+A `=destroy` is implicitly annotated with `.raises: []`; a destructor
+should not raise exceptions. For backwards compatibility the compiler
+produces a warning for a `=destroy` that does raise.
+
+A `=destroy` can explicitly list the exceptions it can raise, if any,
+but this of little utility as a raising destructor is implementation defined
+behavior. Later versions of the language specification might cover this case precisely.
+
+
+`=wasMoved` hook
+----------------
+
+A `=wasMoved` hook sets the object to a state that signifies to the destructor there is nothing to destroy.
+
+The prototype of this hook for a type `T` needs to be:
+
+  ```nim
+  proc `=wasMoved`(x: var T)
+  ```
+
+Usually some pointer field inside the object is set to `nil`:
+
+  ```nim
+  proc `=wasMoved`(x: var T) =
+    x.field = nil
+  ```
 
 
 `=sink` hook
@@ -139,31 +182,30 @@ not free the resources afterward by setting the object to its default value
 default value is written as `wasMoved(x)`. When not provided the compiler
 is using a combination of `=destroy` and `copyMem` instead. This is efficient
 hence users rarely need to implement their own `=sink` operator, it is enough to
-provide `=destroy` and `=copy`, compiler will take care of the rest.
+provide `=destroy` and `=copy`, the compiler will take care of the rest.
 
 The prototype of this hook for a type `T` needs to be:
 
-.. code-block:: nim
-
+  ```nim
   proc `=sink`(dest: var T; source: T)
-
+  ```
 
 The general pattern in `=sink` looks like:
 
-.. code-block:: nim
+  ```nim
 
   proc `=sink`(dest: var T; source: T) =
     `=destroy`(dest)
     wasMoved(dest)
     dest.field = source.field
-
+  ```
 
 **Note**: `=sink` does not need to check for self-assignments.
 How self-assignments are handled is explained later in this document.
 
 
 `=copy` hook
----------------
+------------
 
 The ordinary assignment in Nim conceptually copies the values. The `=copy` hook
 is called for assignments that couldn't be transformed into `=sink`
@@ -171,42 +213,139 @@ operations.
 
 The prototype of this hook for a type `T` needs to be:
 
-.. code-block:: nim
-
+  ```nim
   proc `=copy`(dest: var T; source: T)
-
+  ```
 
 The general pattern in `=copy` looks like:
 
-.. code-block:: nim
-
+  ```nim
   proc `=copy`(dest: var T; source: T) =
     # protect against self-assignments:
     if dest.field != source.field:
       `=destroy`(dest)
       wasMoved(dest)
       dest.field = duplicateResource(source.field)
-
+  ```
 
 The `=copy` proc can be marked with the `{.error.}` pragma. Then any assignment
 that otherwise would lead to a copy is prevented at compile-time. This looks like:
 
-.. code-block:: nim
-
+  ```nim
   proc `=copy`(dest: var T; source: T) {.error.}
+  ```
 
 but a custom error message (e.g., `{.error: "custom error".}`) will not be emitted
 by the compiler. Notice that there is no `=` before the `{.error.}` pragma.
 
+
+`=trace` hook
+-------------
+
+A custom **container** type can support Nim's cycle collector `--mm:orc` via
+the `=trace` hook. If the container does not implement `=trace`, cyclic data
+structures which are constructed with the help of the container might leak
+memory or resources, but memory safety is not compromised.
+
+The prototype of this hook for a type `T` needs to be:
+
+  ```nim
+  proc `=trace`(dest: var T; env: pointer)
+  ```
+
+`env` is used by ORC to keep track of its internal state, it should be passed around
+to calls of the built-in `=trace` operation.
+
+Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates
+manually allocated resources is also used, and then only when there is a chance of cyclic
+references from items within the manually allocated resources when it is desired that `--mm:orc`
+is able to break and collect these cyclic referenced resources. Currently however, there is a
+mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically
+create a version of the other which will then conflict with the creation of the second of the
+pair. The workaround for this problem is to forward declare the second of the "hooks" to
+prevent the automatic creation.
+
+The general pattern in using `=destroy` with `=trace` looks like:
+
+  ```nim
+  type
+    Test[T] = object
+      size: Natural
+      arr: ptr UncheckedArray[T] # raw pointer field
+
+  proc makeTest[T](size: Natural): Test[T] = # custom allocation...
+    Test[T](size: size, arr: cast[ptr UncheckedArray[T]](alloc0(sizeof(T) * size)))
+
+
+  proc `=destroy`[T](dest: Test[T]) =
+    if dest.arr != nil:
+      for i in 0 ..< dest.size: dest.arr[i].`=destroy`
+      dealloc dest.arr
+
+  proc `=trace`[T](dest: var Test[T]; env: pointer) =
+    if dest.arr != nil:
+      # trace the `T`'s which may be cyclic
+      for i in 0 ..< dest.size: `=trace`(dest.arr[i], env)
+
+  # following may be other custom "hooks" as required...
+  ```
+
+**Note**: The `=trace` hooks (which are only used by `--mm:orc`) are currently more experimental and less refined
+than the other hooks.
+
+
+`=dup` hook
+-----------
+
+A `=dup` hook duplicates an object. `=dup(x)` can be regarded as an optimization replacing a `wasMoved(dest); =copy(dest, x)` operation.
+
+The prototype of this hook for a type `T` needs to be:
+
+  ```nim
+  proc `=dup`(x: T): T
+  ```
+
+The general pattern in implementing `=dup` looks like:
+
+  ```nim
+  type
+    Ref[T] = object
+      data: ptr T
+      rc: ptr int
+
+  proc `=dup`[T](x: Ref[T]): Ref[T] =
+    result = x
+    if x.rc != nil:
+      inc x.rc[]
+  ```
+
 Move semantics
 ==============
 
 A "move" can be regarded as an optimized copy operation. If the source of the
 copy operation is not used afterward, the copy can be replaced by a move. This
 document uses the notation `lastReadOf(x)` to describe that `x` is not
-used afterwards. This property is computed by a static control flow analysis
+used afterward. This property is computed by a static control flow analysis
 but can also be enforced by using `system.move` explicitly.
 
+One can query if the analysis is able to perform a move with `system.ensureMove`.
+`move` enforces a move operation and calls `=wasMoved` whereas `ensureMove` is
+an annotation that implies no runtime operation. An `ensureMove` annotation leads to a static error
+if the compiler cannot prove that a move would be safe.
+
+For example:
+
+  ```nim
+  proc main(normalParam: string; sinkParam: sink string) =
+    var x = "abc"
+    # valid:
+    let valid = ensureMove x
+    # invalid:
+    let invalid = ensureMove normalParam
+    # valid:
+    let alsoValid = ensureMove sinkParam
+  ```
+
 
 Swap
 ====
@@ -245,8 +384,7 @@ not a linear type system.
 The employed static analysis is limited and only concerned with local variables;
 however, object and tuple fields are treated as separate entities:
 
-.. code-block:: nim
-
+  ```nim
   proc consume(x: sink Obj) = discard "no implementation"
 
   proc main =
@@ -254,16 +392,16 @@ however, object and tuple fields are treated as separate entities:
     consume tup[0]
     # ok, only tup[0] was consumed, tup[1] is still alive:
     echo tup[1]
-
+  ```
 
 Sometimes it is required to explicitly `move` a value into its final position:
 
-.. code-block:: nim
-
+  ```nim
   proc main =
     var dest, src: array[10, string]
     # ...
     for i in 0..high(dest): dest[i] = move(src[i])
+  ```
 
 An implementation is allowed, but not required to implement even more move
 optimizations (and the current implementation does not).
@@ -273,20 +411,19 @@ Sink parameter inference
 ========================
 
 The current implementation can do a limited form of sink parameter
-inference. But it has to be enabled via `--sinkInference:on`, either
+inference. But it has to be enabled via `--sinkInference:on`:option:, either
 on the command line or via a `push` pragma.
 
 To enable it for a section of code, one can
-use `{.push sinkInference: on.}`...`{.pop.}`.
+use `{.push sinkInference: on.}` ... `{.pop.}`.
 
 The `.nosinks`:idx: pragma can be used to disable this inference
 for a single routine:
 
-.. code-block:: nim
-
+  ```nim
   proc addX(x: T; child: T) {.nosinks.} =
     x.s.add child
-
+  ```
 
 The details of the inference algorithm are currently undocumented.
 
@@ -303,58 +440,57 @@ Rewrite rules
 The current implementation follows strategy (2). This means that resources are
 destroyed at the scope exit.
 
-::
 
-  var x: T; stmts
-  ---------------             (destroy-var)
-  var x: T; try stmts
-  finally: `=destroy`(x)
+    var x: T; stmts
+    ---------------             (destroy-var)
+    var x: T; try stmts
+    finally: `=destroy`(x)
 
 
-  g(f(...))
-  ------------------------    (nested-function-call)
-  g(let tmp;
-  bitwiseCopy tmp, f(...);
-  tmp)
-  finally: `=destroy`(tmp)
+    g(f(...))
+    ------------------------    (nested-function-call)
+    g(let tmp;
+    bitwiseCopy tmp, f(...);
+    tmp)
+    finally: `=destroy`(tmp)
 
 
-  x = f(...)
-  ------------------------    (function-sink)
-  `=sink`(x, f(...))
+    x = f(...)
+    ------------------------    (function-sink)
+    `=sink`(x, f(...))
 
 
-  x = lastReadOf z
-  ------------------          (move-optimization)
-  `=sink`(x, z)
-  wasMoved(z)
+    x = lastReadOf z
+    ------------------          (move-optimization)
+    `=sink`(x, z)
+    `=wasMoved`(z)
 
 
-  v = v
-  ------------------   (self-assignment-removal)
-  discard "nop"
+    v = v
+    ------------------   (self-assignment-removal)
+    discard "nop"
 
 
-  x = y
-  ------------------          (copy)
-  `=copy`(x, y)
+    x = y
+    ------------------          (copy)
+    `=copy`(x, y)
 
 
-  f_sink(g())
-  -----------------------     (call-to-sink)
-  f_sink(g())
+    f_sink(g())
+    -----------------------     (call-to-sink)
+    f_sink(g())
 
 
-  f_sink(notLastReadOf y)
-  --------------------------     (copy-to-sink)
-  (let tmp; `=copy`(tmp, y);
-  f_sink(tmp))
+    f_sink(notLastReadOf y)
+    --------------------------     (copy-to-sink)
+    (let tmp = `=dup`(y);
+    f_sink(tmp))
 
 
-  f_sink(lastReadOf y)
-  -----------------------     (move-to-sink)
-  f_sink(y)
-  wasMoved(y)
+    f_sink(lastReadOf y)
+    -----------------------     (move-to-sink)
+    f_sink(y)
+    `=wasMoved`(y)
 
 
 Object and array construction
@@ -367,7 +503,7 @@ function has `sink` parameters.
 Destructor removal
 ==================
 
-`wasMoved(x);` followed by a `=destroy(x)` operation cancel each other
+`=wasMoved(x)` followed by a `=destroy(x)` operation cancel each other
 out. An implementation is encouraged to exploit this in order to improve
 efficiency and code sizes. The current implementation does perform this
 optimization.
@@ -376,11 +512,11 @@ optimization.
 Self assignments
 ================
 
-`=sink` in combination with `wasMoved` can handle self-assignments but
+`=sink` in combination with `=wasMoved` can handle self-assignments but
 it's subtle.
 
 The simple case of `x = x` cannot be turned
-into `=sink(x, x); wasMoved(x)` because that would lose `x`'s value.
+into `=sink(x, x); =wasMoved(x)` because that would lose `x`'s value.
 The solution is that simple self-assignments that consist of
 
 - Symbols: `x = x`
@@ -394,8 +530,7 @@ The complex case looks like a variant of `x = f(x)`, we consider
 `x = select(rand() < 0.5, x, y)` here:
 
 
-.. code-block:: nim
-
+  ```nim
   proc select(cond: bool; a, b: sink string): string =
     if cond:
       result = a # moves a into result
@@ -407,21 +542,19 @@ The complex case looks like a variant of `x = f(x)`, we consider
     var y = "xyz"
     # possible self-assignment:
     x = select(true, x, y)
-
+  ```
 
 Is transformed into:
 
-
-.. code-block:: nim
-
+  ```nim
   proc select(cond: bool; a, b: sink string): string =
     try:
       if cond:
         `=sink`(result, a)
-        wasMoved(a)
+        `=wasMoved`(a)
       else:
         `=sink`(result, b)
-        wasMoved(b)
+        `=wasMoved`(b)
     finally:
       `=destroy`(b)
       `=destroy`(a)
@@ -435,15 +568,16 @@ Is transformed into:
       `=sink`(y, "xyz")
       `=sink`(x, select(true,
         let blitTmp = x
-        wasMoved(x)
+        `=wasMoved`(x)
         blitTmp,
         let blitTmp = y
-        wasMoved(y)
+        `=wasMoved`(y)
         blitTmp))
       echo [x]
     finally:
       `=destroy`(y)
       `=destroy`(x)
+  ```
 
 As can be manually verified, this transformation is correct for
 self-assignments.
@@ -465,8 +599,7 @@ that the pointer does not outlive its origin. No destructor call is injected
 for expressions of type `lent T` or of type `var T`.
 
 
-.. code-block:: nim
-
+  ```nim test
   type
     Tree = object
       kids: seq[Tree]
@@ -474,13 +607,13 @@ for expressions of type `lent T` or of type `var T`.
   proc construct(kids: sink seq[Tree]): Tree =
     result = Tree(kids: kids)
     # converted into:
-    `=sink`(result.kids, kids); wasMoved(kids)
+    `=sink`(result.kids, kids); `=wasMoved`(kids)
     `=destroy`(kids)
 
   proc `[]`*(x: Tree; i: int): lent Tree =
     result = x.kids[i]
     # borrows from 'x', this is transformed into:
-    result = addr x.kids[i]
+    # result = addr x.kids[i]
     # This means 'lent' is like 'var T' a hidden pointer.
     # Unlike 'var' this hidden pointer cannot be used to mutate the object.
 
@@ -491,42 +624,43 @@ for expressions of type `lent T` or of type `var T`.
     # everything turned into moves:
     let t = construct(@[construct(@[]), construct(@[])])
     echo t[0] # accessor does not copy the element!
+  ```
 
 
-The .cursor annotation
-======================
+The cursor pragma
+=================
 
-Under the `--gc:arc|orc` modes Nim's `ref` type is implemented via the same runtime
-"hooks" and thus via reference counting. This means that cyclic structures cannot be freed
-immediately (`--gc:orc` ships with a cycle collector). With the `.cursor` annotation
-one can break up cycles declaratively:
-
-.. code-block:: nim
+Under the `--mm:arc|orc`:option: modes Nim's `ref` type is implemented
+via the same runtime "hooks" and thus via reference counting.
+This means that cyclic structures cannot be freed
+immediately (`--mm:orc`:option: ships with a cycle collector).
+With the `cursor` pragma one can break up cycles declaratively:
 
+  ```nim
   type
     Node = ref object
       left: Node # owning ref
       right {.cursor.}: Node # non-owning ref
+  ```
 
 But please notice that this is not C++'s weak_ptr, it means the right field is not
 involved in the reference counting, it is a raw pointer without runtime checks.
 
 Automatic reference counting also has the disadvantage that it introduces overhead
-when iterating over linked structures. The `.cursor` annotation can also be used
+when iterating over linked structures. The `cursor` pragma can also be used
 to avoid this overhead:
 
-.. code-block:: nim
-
+  ```nim
   var it {.cursor.} = listRoot
   while it != nil:
     use(it)
     it = it.next
+  ```
 
-
-In fact, `.cursor` more generally prevents object construction/destruction pairs
+In fact, `cursor` more generally prevents object construction/destruction pairs
 and so can also be useful in other contexts. The alternative solution would be to
 use raw pointers (`ptr`) instead which is more cumbersome and also more dangerous
-for Nim's evolution: Later on, the compiler can try to prove `.cursor` annotations
+for Nim's evolution: Later on, the compiler can try to prove `cursor` pragmas
 to be safe, but for `ptr` the compiler has to remain silent about possible
 problems.
 
@@ -534,25 +668,25 @@ problems.
 Cursor inference / copy elision
 ===============================
 
-The current implementation also performs `.cursor` inference. Cursor inference is
+The current implementation also performs `cursor` inference. Cursor inference is
 a form of copy elision.
 
 To see how and when we can do that, think about this question: In `dest = src` when
 do we really have to *materialize* the full copy? - Only if `dest` or `src` are mutated
-afterwards. If `dest` is a local variable that is simple to analyze. And if `src` is a
+afterward. If `dest` is a local variable that is simple to analyze. And if `src` is a
 location derived from a formal parameter, we also know it is not mutated! In other
 words, we do a compile-time copy-on-write analysis.
 
 This means that "borrowed" views can be written naturally and without explicit pointer
 indirections:
 
-.. code-block:: nim
-
+  ```nim
   proc main(tab: Table[string, string]) =
-    let v = tab["key"] # inferred as .cursor because 'tab' is not mutated.
+    let v = tab["key"] # inferred as cursor because 'tab' is not mutated.
     # no copy into 'v', no destruction of 'v'.
     use(v)
     useItAgain(v)
+  ```
 
 
 Hook lifting
@@ -576,8 +710,7 @@ Hook generation
 
 The ability to override a hook leads to a phase ordering problem:
 
-.. code-block:: nim
-
+  ```nim
   type
     Foo[T] = object
 
@@ -586,11 +719,11 @@ The ability to override a hook leads to a phase ordering problem:
     # error: destructor for 'f' called here before
     # it was seen in this module.
 
-  proc `=destroy`[T](f: var Foo[T]) =
+  proc `=destroy`[T](f: Foo[T]) =
     discard
+  ```
 
-
-The solution is to define `proc `=destroy`[T](f: var Foo[T])` before
+The solution is to define ``proc `=destroy`[T](f: Foo[T])`` before
 it is used. The compiler generates implicit
 hooks for all types in *strategic places* so that an explicitly provided
 hook that comes too "late" can be detected reliably. These *strategic places*
@@ -611,8 +744,7 @@ The experimental `nodestroy`:idx: pragma inhibits hook injections. This can be
 used to specialize the object traversal in order to avoid deep recursions:
 
 
-.. code-block:: nim
-
+  ```nim test
   type Node = ref object
     x, y: int32
     left, right: Node
@@ -620,18 +752,19 @@ used to specialize the object traversal in order to avoid deep recursions:
   type Tree = object
     root: Node
 
-  proc `=destroy`(t: var Tree) {.nodestroy.} =
+  proc `=destroy`(t: Tree) {.nodestroy.} =
     # use an explicit stack so that we do not get stack overflows:
     var s: seq[Node] = @[t.root]
     while s.len > 0:
       let x = s.pop
       if x.left != nil: s.add(x.left)
       if x.right != nil: s.add(x.right)
-      # free the memory explicit:
-      dispose(x)
+      # free the memory explicitly:
+      `=dispose`(x)
     # notice how even the destructor for 's' is not called implicitly
     # anymore thanks to .nodestroy, so we have to call it on our own:
     `=destroy`(s)
+  ```
 
 
 As can be seen from the example, this solution is hardly sufficient and
@@ -641,63 +774,28 @@ should eventually be replaced by a better solution.
 Copy on write
 =============
 
-String literals are implemented as [copy-on-write](https://en.wikipedia.org/wiki/Copy-on-write).
+String literals are implemented as "copy on write".
 When assigning a string literal to a variable, a copy of the literal won't be created.
 Instead the variable simply points to the literal.
 The literal is shared between different variables which are pointing to it.
 The copy operation is deferred until the first write.
 
-.. code-block:: nim
-  var x = "abc"  # no copy
-  var y = x      # no copy
+For example:
 
-The string literal "abc" is stored in static memory and not allocated on the heap.
-The variable `x` points to the literal and the variable `y` points to the literal too.
-There is no copy during assigning operations.
-
-.. code-block:: nim
+  ```nim
   var x = "abc"  # no copy
   var y = x      # no copy
   y[0] = 'h'     # copy
+  ```
 
-The program above shows when the copy operations happen.
-When mutating the variable `y`, the Nim compiler creates a fresh copy of `x`, 
-the variable `y` won't point to the string literal anymore. 
-Instead it points to the copy of `x` of which the memory can be mutated 
-and the variable `y` becomes a mutable string.
+The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown.
+`prepareMutation` needs to be called before the "address of" operation. For example:
 
-.. Note:: The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown. 
-
-Let's look at a silly example demonstrating this behaviour:
-
-.. code-block:: nim
-  var x = "abc"
-  var y = x
-
-  moveMem(addr y[0], addr x[0], 3)
-
-The program fails because we need to prepare a fresh copy for the variable `y`.
-`prepareMutation` should be called before the address operation.
-
-.. code-block:: nim
+  ```nim
   var x = "abc"
   var y = x
 
   prepareMutation(y)
   moveMem(addr y[0], addr x[0], 3)
   assert y == "abc"
-
-Now `prepareMutation` solves the problem.
-It manually creates a fresh copy and makes the variable `y` mutable.
-
-.. code-block:: nim
-  var x = "abc"
-  var y = x
-
-  prepareMutation(y)
-  moveMem(addr y[0], addr x[0], 3)
-  moveMem(addr y[0], addr x[0], 3)
-  moveMem(addr y[0], addr x[0], 3)
-  assert y == "abc"
-
-No matter how many times `moveMem` is called, the program compiles and runs.
+  ```
diff --git a/doc/docgen.md b/doc/docgen.md
new file mode 100644
index 0000000000..3cc75fc180
--- /dev/null
+++ b/doc/docgen.md
@@ -0,0 +1,891 @@
+===================================
+   Nim DocGen Tools Guide
+===================================
+
+:Author: Erik O'Leary
+:Version: |nimversion|
+
+.. default-role:: code
+.. include:: rstcommon.rst
+.. contents::
+
+.. importdoc:: markdown_rst.md, compiler/docgen.nim
+
+Introduction
+============
+
+This document describes the `documentation generation tools`:idx: built into
+the [Nim compiler](nimc.html), which can generate HTML, Latex and JSON output
+from input ``.nim`` files and projects.
+The output documentation will include the module
+dependencies (`import`), any top-level documentation comments (`##`), and
+exported symbols (`*`), including procedures, types, and variables.
+
+===================   ==============
+command               output format
+===================   ==============
+`nim doc`:cmd:        ``.html`` HTML
+`nim doc2tex`:cmd:    ``.tex`` LaTeX
+`nim jsondoc`:cmd:    ``.json`` JSON
+===================   ==============
+
+Nim can generate HTML and LaTeX from input Markdown and
+RST (reStructuredText) files as well, which is intended for writing
+standalone documents like user's guides and technical specifications.
+See [Nim-flavored Markdown and reStructuredText] document for the description
+of this feature and particularly section [Command line usage] for the full
+list of supported commands.
+
+Quick start
+-----------
+
+Generate HTML documentation for a file:
+
+  ```cmd
+  nim doc .nim
+  ```
+
+Generate HTML documentation for a whole project:
+
+  ```cmd
+  # delete any htmldocs/*.idx file before starting
+  nim doc --project --index:on --git.url: --git.commit: --outdir:htmldocs .nim
+  # this will generate html files, a theindex.html index, css and js under `htmldocs`
+  # See also `--docroot` to specify a relative root.
+  # to get search (dochacks.js) to work locally, you need a server otherwise
+  # CORS will prevent opening file:// urls; this works:
+  python3 -m http.server 7029 --directory htmldocs
+  # When --outdir is omitted it defaults to $projectPath/htmldocs,
+  # or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources;
+  # and likewise without `--project`.
+  # Adding `-r` will open in a browser directly.
+  # Use `--showNonExports` to show non-exported fields of an exported type.
+  ```
+
+Documentation Comments
+----------------------
+
+Any comments which are preceded by a double-hash (`##`), are interpreted as
+documentation.  Comments are parsed as RST (see [reference](
+http://docutils.sourceforge.net/docs/user/rst/quickref.html)), providing
+Nim module authors the ability to easily generate richly formatted
+documentation with only their well-documented code!
+Basic Markdown syntax is also supported inside the doc comments.
+
+Example:
+
+  ```nim
+  type Person* = object
+    ## This type contains a description of a person
+    name: string
+    age: int
+  ```
+
+Outputs:
+
+    Person* = object
+      name: string
+      age: int
+
+  This type contains a description of a person
+
+Field documentation comments can be added to fields like so:
+
+  ```nim
+  var numValues: int ## \
+    ## `numValues` stores the number of values
+  ```
+
+Note that without the `*` following the name of the type, the documentation for
+this type would not be generated. Documentation will only be generated for
+*exported* types/procedures/etc.
+
+It's recommended to always add exactly **one** space after `##` for readability
+of comments — this extra space will be cropped from the parsed comments and
+won't influence RST formatting.
+
+.. note:: Generally, this baseline indentation level inside a documentation
+   comment may not be 1: it can be any since it is determined by the offset
+   of the first non-whitespace character in the comment.
+   After that indentation **must** be consistent on the following lines of
+   the same comment.
+   If you still need to add an additional indentation at the very beginning
+   (for RST block quote syntax) use backslash \\ before it:
+
+     ```nim
+     ## \
+     ##
+     ##    Block quote at the first line.
+     ##
+     ## Paragraph.
+     ```
+
+Structuring output directories
+------------------------------
+
+Basic directory for output is set by `--outdir:OUTDIR`:option: switch,
+by default `OUTDIR` is ``htmldocs`` sub-directory in the directory of
+the processed file.
+
+There are 2 basic options as to how generated HTML output files are stored:
+
+1) complex hierarchy when docgen-compiling with `--project`:option:,
+   which follows directory structure of the project itself.
+   So `nim doc`:cmd: replicates project's directory structure
+   inside `--outdir:OUTDIR`:option: directory.
+   `--project`:option: is well suited for projects that have 1 main module.
+   File name clashes are impossible in this case.
+
+2) flattened structure, where user-provided script goes through all
+   needed input files and calls commands like `nim doc`:cmd:
+   with `--outdir:OUTDIR`:option: switch, thus putting all HTML (and
+   ``.idx``) files into 1 directory.
+
+   .. Important:: Make sure that you don't have files with same base name
+     like ``x.nim`` and ``x.md`` in the same package, otherwise you'll
+     have name conflict for ``x.html``.
+
+   .. Tip:: To structure your output directories and avoid file name
+     clashes you can split your project into
+     different *packages* -- parts of your repository that are
+     docgen-compiled with different `--outdir:OUTDIR`:option: options.
+
+     An example of such strategy is Nim repository itself which has:
+
+     * its stdlib ``.nim`` files from different directories and ``.md``
+       documentation from ``doc/`` directory are all docgen-compiled
+       into `--outdir:web/upload//`:option: directory
+     * its ``.nim`` files from ``compiler/`` directory are docgen-compiled
+       into `--outdir:web/upload//compiler/`:option: directory.
+       Interestingly, it's compiled with complex hierarchy using
+       `--project`:option: switch.
+
+     Contents of ``web/upload/`` are then deployed into Nim's
+     Web server.
+
+     This output directory structure allows to work correctly with files like
+     ``compiler/docgen.nim`` (implementation) and ``doc/docgen.md`` (user
+     documentation) in 1 repository.
+
+
+Index files
+-----------
+
+Index (``.idx``) files are used for 2 different purposes:
+
+1. easy cross-referencing between different ``.nim`` and/or ``.md`` / ``.rst``
+   files described in [Nim external referencing]
+2. creating a whole-project index for searching of symbols and keywords,
+   see [Buildindex command].
+
+
+Document Types
+==============
+
+Example of Nim file input
+-------------------------
+
+The following examples will generate documentation for this sample
+*Nim* module, aptly named ``doc/docgen_sample.nim``:
+
+   ```nim file=docgen_sample.nim
+   ```
+
+All the below commands save their output to ``htmldocs`` directory relative to
+the directory of file;
+hence the output for this sample will be in ``doc/htmldocs``.
+
+HTML
+----
+
+The generation of HTML documents is done via the `doc`:option: command. This command
+takes either a single ``.nim`` file, outputting a single ``.html`` file with the same
+base filename, or multiple ``.nim`` files, outputting multiple ``.html`` files and,
+optionally, an index file.
+
+The `doc`:option: command:
+
+  ```cmd
+  nim doc docgen_sample.nim
+  ```
+
+Partial Output:
+
+    ...
+    proc helloWorld(times: int) {.raises: [], tags: [].}
+    ...
+
+The full output can be seen here: [docgen_sample.html](docgen_sample.html).
+It runs after semantic checking and includes pragmas attached implicitly by the
+compiler.
+
+LaTeX
+-----
+
+LaTeX files are intended to be converted to PDF, especially for offline
+reading or making hard copies. (LaTeX output is oftentimes better than
+HTML -> PDF conversion).
+
+The `doc2tex`:option: command:
+
+  ```cmd
+  nim doc2tex docgen_sample.nim
+  cd htmldocs
+  xelatex docgen_sample.tex
+  xelatex docgen_sample.tex
+  # It is usually necessary to run `xelatex` 2 times (or even 3 times for
+  # large documents) to get all labels generated.
+  # That depends on this warning in the end of `xelatex` output:
+  #   LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.
+  ```
+
+The output is ``docgen_sample.pdf``.
+
+JSON
+----
+
+The generation of JSON documents is done via the `jsondoc`:option: command.
+This command takes in a ``.nim`` file and outputs a ``.json`` file with
+the same base filename.
+Note that this tool is built off of the `doc`:option: command
+(previously `doc2`:option:), and contains the same information.
+
+The `jsondoc`:option: command:
+
+  ```cmd
+  nim jsondoc docgen_sample.nim
+  ```
+
+Output:
+
+    {
+      "orig": "docgen_sample.nim",
+      "nimble": "",
+      "moduleDescription": "This module is a sample",
+      "entries": [
+        {
+          "name": "helloWorld",
+          "type": "skProc",
+          "line": 5,
+          "col": 0,
+          "description": "Takes an integer and outputs as many "hello world!"s",
+          "code": "proc helloWorld(times: int) {.raises: [], tags: [].}"
+        }
+      ]
+    }
+
+Similarly to the old `doc`:option: command, the old `jsondoc`:option: command has been
+renamed to `jsondoc0`:option:.
+
+The `jsondoc0`:option: command:
+
+  ```cmd
+  nim jsondoc0 docgen_sample.nim
+  ```
+
+Output:
+
+    [
+      {
+        "comment": "This module is a sample."
+      },
+      {
+        "name": "helloWorld",
+        "type": "skProc",
+        "description": "Takes an integer and outputs as many "hello world!"s",
+        "code": "proc helloWorld*(times: int)"
+      }
+    ]
+
+Note that the `jsondoc`:option: command outputs its JSON without pretty-printing it,
+while `jsondoc0`:option: outputs pretty-printed JSON.
+
+
+Simple documentation links
+==========================
+
+It's possible to use normal Markdown/RST syntax to *manually*
+reference Nim symbols using HTML anchors, however Nim has an *automatic*
+facility that makes referencing inside ``.nim`` and ``.md/.rst`` files and
+between them easy and seamless.
+The point is that such links will be resolved automatically
+by `nim doc`:cmd: (or `md2html`:option:, or `jsondoc`:option:,
+or `doc2tex`:option:, ...). And, unlike manual links, such automatic
+links **check** that their target exists -- a warning is emitted for
+any broken link, so you avoid broken links in your project.
+
+Nim treats both ``.md/.rst`` files and ``.nim`` modules (their doc comment
+part) as *documents* uniformly.
+Hence all directions of referencing are equally possible having the same syntax:
+
+1. ``.md/rst`` -> itself (internal). See [Markup local referencing].
+2. ``.md/rst`` -> external ``.md/rst``. See [Markup external referencing].
+   To summarize, referencing in `.md`/`.rst` files was already described in
+   [Nim-flavored Markdown and reStructuredText]
+   (particularly it described usage of index files for referencing),
+   while in this document we focus on Nim-specific details.
+3. ``.md/rst`` -> external ``.nim``. See [Nim external referencing].
+4. ``.nim`` -> itself (internal). See [Nim local referencing].
+5. ``.nim`` -> external ``.md/rst``. See [Markup external referencing].
+6. ``.nim`` -> external ``.nim``. See [Nim external referencing].
+
+To put it shortly, local referencing always works out of the box,
+external referencing requires to use ``.. importdoc:: ``
+directive to import `file` and to ensure that the corresponding
+``.idx`` file was generated.
+
+Syntax for referencing is basically the same as for normal markup.
+Recall from [Referencing] that our parser supports two equivalent syntaxes
+for referencing, Markdown and RST one.
+So to reference ``proc f`` one should use something like that,
+depending on markup type:
+
+    Markdown                    RST
+
+    Ref. [proc f] or [f]        Ref. `proc f`_ or just f_ for a one-word case
+
+Nim local referencing
+---------------------
+
+You can reference Nim identifiers from Nim documentation comments
+inside their ``.nim`` file (or inside a ``.rst`` file included from
+a ``.nim``).
+This pertains to any exported symbol like `proc`, `const`, `iterator`, etc.
+Link text is either one word or a group of words enclosed by delimiters
+(brackets ``[...]`` for Markdown or backticks `\`...\`_` for RST).
+Link text will be displayed *as is* while *link target* will be set to
+the anchor [^1] of Nim symbol that corresponds to link text.
+
+[^1] anchors' format is described in [HTML anchor generation] section below.
+
+If you have a constant:
+
+  ```Nim
+  const pi* = 3.14
+  ```
+
+then it should be referenced in one of the 2 forms:
+
+A. non-qualified (no symbol kind specification):
+
+       pi_
+
+B. qualified (with symbol kind specification):
+
+       `const pi`_
+
+For routine kinds there are more options. Consider this definition:
+
+  ```Nim
+  proc foo*(a: int, b: float): string
+  ```
+
+Generally following syntax is allowed for referencing `foo`:
+
+*  short (without parameters):
+
+   A. non-qualified:
+
+          foo_
+
+   B. qualified:
+
+          `proc foo`_
+
+*  longer variants (with parameters):
+
+   A. non-qualified:
+
+      1) specifying parameters names:
+
+             `foo(a, b)`_
+
+      2) specifying parameters types:
+
+             `foo(int, float)`_
+
+      3) specifying both names and types:
+
+             `foo(a: int, b: float)`_
+
+      4) output parameter can also be specified if you wish:
+
+             `foo(a: int, b: float): string`_
+
+   B. qualified: all 4 options above are valid.
+      Particularly you can use the full format:
+
+          `proc foo(a: int, b: float): string`_
+
+.. Tip:: Avoid cluttering your text with extraneous information by using
+   one of shorter forms:
+
+       binarySearch_
+       `binarySearch(a, key, cmp)`_
+
+   Brevity is better for reading! If you use a short form and have an
+   ambiguity problem (see below) then just add some additional info.
+
+Symbol kind like `proc` can also be specified in the postfix form:
+
+    `foo proc`_
+    `walkDir(d: string) iterator`_
+
+.. Warning:: An ambiguity in resolving documentation links may arise because of:
+
+   1. clash with other RST anchors
+      * manually setup anchors
+      * automatically set up, e.g. section names
+   2. collision with other Nim symbols:
+
+      * routines with different parameters can exist e.g. for
+        `proc` and `template`. In this case they are split between their
+        corresponding sections in output file. Qualified references are
+        useful in this case -- just disambiguate by referring to these
+        sections explicitly:
+
+            See `foo proc`_ and `foo template`_.
+
+      * because in Nim `proc` and `iterator` belong to different namespaces,
+        so there can be a collision even if parameters are the same.
+        Use `\`proc foo\`_`:literal: or `\`iterator foo\`_`:literal: then.
+
+   Any ambiguity is always reported with Nim compiler warnings and an anchor
+   with higher priority is selected. Manual anchors have highest
+   priority, then go automatic RST anchors; then Nim-generated anchors
+   (while procs have higher priority than other Nim symbol kinds).
+
+Generic parameters can also be used. All in all, this long form will be
+recognized fine:
+
+    `proc binarySearch*[T; K](a: openArray[T], key: K, cmp: proc(T, K)): int`_
+
+**Limitations**:
+
+1. The parameters of a nested routine type can be specified only with types
+   (without parameter names, see form A.2 above).
+   E.g. for this signature:
+
+      ```Nim
+      proc binarySearch*[T, K](a: openArray[T]; key: K;
+                               cmp: proc (x: T; y: K): int {.closure.}): int
+                                          ~~    ~~   ~~~~~
+      ```
+
+   you cannot use names underlined by `~~` so it must be referenced with
+   ``cmp: proc(T, K)``. Hence these forms are valid:
+
+       `binarySearch(a: openArray[T], key: K, cmp: proc(T, K))`_
+       `binarySearch(openArray[T], K, proc(T, K))`_
+       `binarySearch(a, key, cmp)`_
+2. Default values in routine parameters are not recognized, one needs to
+   specify the type and/or name instead. E.g. for referencing `proc f(x = 7)`
+   use one of the mentioned forms:
+
+       `f(int)`_ or `f(x)`_ or `f(x: int)`_.
+3. Generic parameters must be given the same way as in the
+   definition of referenced symbol.
+
+   * their names should be the same
+   * parameters list should be given the same way, e.g. without substitutions
+     between commas (,) and semicolons (;).
+
+.. Note:: A bit special case is operators
+   (as their signature is also defined with `\``):
+
+      ```Nim
+      func `$`(x: MyType): string
+      func `[]`*[T](x: openArray[T]): T
+      ```
+
+   A short form works without additional backticks:
+
+       `$`_
+       `[]`_
+
+   However for fully-qualified reference copy-pasting backticks (`) into other
+   backticks will not work in our RST parser (because we use Markdown-like
+   inline markup rules). You need either to delete backticks or keep
+   them and escape with backslash \\:
+
+       no backticks: `func $`_
+       escaped:  `func \`$\``_
+       no backticks: `func [][T](x: openArray[T]): T`_
+       escaped:  `func \`[]\`[T](x: openArray[T]): T`_
+
+.. Note:: Types that defined as `enum`, or `object`, or `tuple` can also be
+   referenced with those names directly (instead of `type`):
+
+       type CopyFlag = enum
+         ...
+       ## Ref. `CopyFlag enum`_
+
+Nim external referencing
+------------------------
+
+Just like for [Markup external referencing], which saves markup anchors,
+the Nim symbols are also saved in ``.idx`` files, so one needs
+to generate them beforehand, and they should be loaded by
+an ``.. importdoc::`` directive. Arguments to ``.. importdoc::`` is a
+comma-separated list of Nim modules or Markdown/RST documents.
+
+`--index:only`:option: tells Nim to only generate ``.idx`` file and
+do **not** attempt to generate HTML/LaTeX output.
+For ``.nim`` modules there are 2 alternatives to work with ``.idx`` files:
+
+1. using [Project switch] implies generation of ``.idx`` files,
+   however, if ``importdoc`` is called on upper modules as its arguments,
+   their ``.idx`` are not yet created. Thus one should generate **all**
+   required ``.idx`` first:
+     ```cmd
+     nim doc --project --index:only 
.nim
+     nim doc --project 
.nim
+     ```
+2. or run `nim doc --index:only `:cmd: command for **all** (used)
+   Nim modules in your project. Then run `nim doc ` on them for
+   output HTML generation.
+
+   .. Warning:: A mere `nim doc --index:on`:cmd: may fail on an attempt to do
+      ``importdoc`` from another module (for which ``.idx`` was not yet
+      generated), that's why `--index:only`:option: shall be used instead.
+
+   For ``.md``/``.rst`` markup documents point 2 is the only option.
+
+Then, you can freely use something like this in ``your_module.nim``:
+
+  ```nim
+  ## .. importdoc::  user_manual.md, another_module.nim
+
+  ...
+  ## Ref. [some section from User Manual].
+
+  ...
+  ## Ref. [proc f]
+  ## (assuming you have a proc `f` in ``another_module``).
+  ```
+
+and compile it by `nim doc`:cmd:. Note that link text will
+be automatically prefixed by the module name of symbol,
+so you will see something like "Ref. [another_module: proc f](#)"
+in the generated output.
+
+It's also possible to reference a whole module by prefixing or
+suffixing full canonical module name with "module":
+
+    Ref. [module subdir/name] or [subdir/name module].
+
+Markup documents as a whole can be referenced just by their title
+(or by their file name if the title was not set) without any prefix.
+
+.. Tip:: During development process the stage of ``.idx`` files generation
+  can be done only *once*, after that you use already generated ``.idx``
+  files while working with a document *being developed* (unless you do
+  incompatible changes to *referenced* documents).
+
+.. Hint:: After changing a *referenced* document file one may need
+  to regenerate its corresponding ``.idx`` file to get correct results.
+  Of course, when referencing *internally* inside any given ``.nim`` file,
+  it's not needed, one can even immediately use any freshly added anchor
+  (a document's own ``.idx`` file is not used for resolving its internal links).
+
+If an ``importdoc`` directive fails to find a ``.idx``, then an error
+is emitted.
+
+In case of such compilation failures please note that:
+
+* **all** relative paths, given to ``importdoc``, relate to insides of
+  ``OUTDIR``, and **not** project's directory structure.
+
+* ``importdoc`` searches for ``.idx`` in `--outdir:OUTDIR`:option: directory
+  (``htmldocs`` by default) and **not** around original modules, so:
+
+  .. Tip:: look into ``OUTDIR`` to understand what's going on.
+
+* also keep in mind that ``.html`` and ``.idx`` files should always be
+  output to the same directory, so check this and, if it's not true, check
+  that both runs *with* and *without* `--index:only`:option: have all
+  other options the same.
+
+To summarize, for 2 basic options of [Structuring output directories]
+compilation options are different:
+
+1) complex hierarchy with `--project`:option: switch.
+
+   As the **original** project's directory structure is replicated in
+   `OUTDIR`, all passed paths are related to this structure also.
+
+   E.g. if a module ``path1/module.nim`` does
+   ``.. importdoc:: path2/another.nim`` then docgen tries to load file
+   ``OUTDIR/path1/path2/another.idx``.
+
+   .. Note:: markup documents are just placed into the specified directory
+     `OUTDIR`:option: by default (i.e. they are **not** affected by
+     `--project`:option:), so if you have ``PROJECT/doc/manual.md``
+     document and want to use complex hierarchy (with ``doc/``),
+     compile it with `--docroot`:option:\:
+       ```cmd
+       # 1st stage
+       nim md2html --outdir:OUTDIR --docroot:/absolute/path/to/PROJECT \
+            --index:only PROJECT/doc/manual.md
+       ...
+       # 2nd stage
+       nim md2html --outdir:OUTDIR --docroot:/absolute/path/to/PROJECT \
+                         PROJECT/doc/manual.md
+       ```
+
+     Then the output file will be placed as ``OUTDIR/doc/manual.idx``.
+     So if you have ``PROJECT/path1/module.nim``, then ``manual.md`` can
+     be referenced as ``../doc/manual.md``.
+
+2) flattened structure.
+
+   E.g. if a module ``path1/module.nim`` does
+   ``.. importdoc:: path2/another.nim`` then docgen tries to load
+   ``OUTDIR/path2/another.idx``, so the path ``path1``
+   does not matter and providing ``path2`` can be useful only
+   in the case it contains another package that was placed there
+   using `--outdir:OUTDIR/path2`:option:.
+
+   The links' text will be prefixed as ``another: ...`` in both cases.
+
+   .. Warning:: Again, the same `--outdir:OUTDIR`:option: option should
+     be provided to both `doc --index:only`:option: /
+     `md2html --index:only`:option: and final generation by
+     `doc`:option:/`md2html`:option: inside 1 package.
+
+To temporarily disable ``importdoc``, e.g. if you don't need
+correct link resolution at the moment, use a `--noImportdoc`:option: switch
+(only warnings about unresolved links will be generated for external references).
+
+Related Options
+===============
+
+Project switch
+--------------
+
+  ```cmd
+  nim doc --project filename.nim
+  ```
+
+This will recursively generate documentation of all Nim modules imported
+into the input module that belong to the Nimble package that ``filename.nim``
+belongs to. The index files and the corresponding ``theindex.html`` will
+also be generated.
+
+
+Index switch
+------------
+
+  ```cmd
+  nim doc --index:on filename.nim
+  ```
+
+This will generate an index of all the exported symbols in the input Nim
+module, and put it into a neighboring file with the extension of ``.idx``. The
+index file is line-oriented (newlines have to be escaped). Each line
+represents a tab-separated record of several columns, the first two mandatory,
+the rest optional. See the [Index (idx) file format] section for details.
+
+.. Note:: `--index`:option: switch only affects creation of ``.idx``
+  index files, while user-searchable Index HTML file is created by
+  `buildIndex`:option: command.
+
+Buildindex command
+------------------
+
+Once index files have been generated for one or more modules, the Nim
+compiler command `nim buildIndex directory`:cmd: can be run to go over all the index
+files in the specified directory to generate a [theindex.html](theindex.html)
+file:
+
+  ```cmd
+  nim buildIndex -o:path/to/htmldocs/theindex.html path/to/htmldocs
+  ```
+
+See source switch
+-----------------
+
+  ```cmd
+  nim doc --git.url: filename.nim
+  ```
+
+With the `git.url`:option: switch the *See source* hyperlink will appear below each
+documented item in your source code pointing to the implementation of that
+item on a GitHub repository.
+You can click the link to see the implementation of the item.
+
+The `git.commit`:option: switch overrides the hardcoded `devel` branch in
+``config/nimdoc.cfg``.
+This is useful to link to a different branch e.g. `--git.commit:master`:option:,
+or to a tag e.g. `--git.commit:1.2.3`:option: or a commit.
+
+Source URLs are generated as ``href="${url}/tree/${commit}/${path}#L${line}"``
+by default and thus compatible with GitHub but not with GitLab.
+
+Similarly, `git.devel`:option: switch overrides the hardcoded `devel` branch
+for the `Edit` link which is also useful if you have a different working
+branch than `devel` e.g. `--git.devel:master`:option:.
+
+Edit URLs are generated as ``href="${url}/tree/${devel}/${path}#L${line}"``
+by default.
+
+You can edit ``config/nimdoc.cfg`` and modify the ``doc.item.seesrc`` value
+with a hyperlink to your own code repository.
+
+In the case of Nim's own documentation, the `commit` value is just a commit
+hash to append to a formatted URL to https://github.com/nim-lang/Nim.
+
+
+Other Input Formats
+===================
+
+The *Nim compiler* also has support for RST (reStructuredText) files with
+the `rst2html`:option: and `rst2tex`:option: commands. Documents like this one are
+initially written in a dialect of RST which adds support for Nim source code
+highlighting with the ``.. code-block:: nim`` prefix. ``code-block`` also
+supports highlighting of a few other languages supported by the
+[packages/docutils/highlite module](highlite.html).
+
+See [Markdown and RST markup languages](markdown_rst.html) for
+usage of those commands.
+
+HTML anchor generation
+======================
+
+When you run the `rst2html`:option: command, all sections in the RST document will
+get an anchor you can hyperlink to. Usually, you can guess the anchor lower
+casing the section title and replacing spaces with dashes, and in any case, you
+can get it from the table of contents. But when you run the `doc`:option:
+command to generate API documentation, some symbol get one or two anchors at
+the same time: a numerical identifier, or a plain name plus a complex name.
+
+The numerical identifier is just a random number. The number gets assigned
+according to the section and position of the symbol in the file being processed
+and you should not rely on it being constant: if you add or remove a symbol the
+numbers may shuffle around.
+
+The plain name of a symbol is a simplified version of its fully exported
+signature. Variables or constants have the same plain name symbol as their
+complex name. The plain name for procs, templates, and other callable types
+will be their unquoted value after removing parameters, return types, and
+pragmas. The plain name allows short and nice linking of symbols that works
+unless you have a module with collisions due to overloading.
+
+If you hyperlink a plain name symbol and there are other matches on the same
+HTML file, most browsers will go to the first one. To differentiate the rest,
+you will need to use the complex name. A complex name for a callable type is
+made up of several parts:
+
+  (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\*
+
+The first thing to note is that all callable types have at least a comma, even
+if they don't have any parameters. If there are parameters, they are
+represented by their types and will be comma-separated. To the plain symbol a
+suffix may be added depending on the type of the callable:
+
+==============   ==============
+Callable type    Suffix
+==============   ==============
+`proc`, `func`   *empty string*
+`macro`          ``.m``
+`method`         ``.e``
+`iterator`       ``.i``
+`template`       ``.t``
+`converter`      ``.c``
+==============   ==============
+
+The relationship of type to suffix is made by the proc `complexName` in the
+``compiler/docgen.nim`` file. Here are some examples of complex names for
+symbols in the [system module](system.html).
+
+* `type SomeSignedInt = int | int8 | int16 | int32 | int64` **=>**
+  [#SomeSignedInt](system.html#SomeSignedInt)
+* `var globalRaiseHook: proc (e: ref E_Base): bool {.nimcall.}` **=>**
+  [#globalRaiseHook](system.html#globalRaiseHook)
+* `const NimVersion = "0.0.0"` **=>**
+  [#NimVersion](system.html#NimVersion)
+* `proc getTotalMem(): int {.rtl, raises: [], tags: [].}` **=>**
+  [#getTotalMem](system.html#getTotalMem)
+* `proc len[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.}` **=>**
+  [#len,seq[T]](system.html#len,seq[T])
+* `iterator pairs[T](a: seq[T]): tuple[key: int, val: T] {.inline.}` **=>**
+  [#pairs.i,seq[T]](iterators.html#pairs.i,seq[T])
+* `template newException[](exceptn: typedesc; message: string;
+    parentException: ref Exception = nil): untyped` **=>**
+  [#newException.t,typedesc,string,ref.Exception](
+  system.html#newException.t,typedesc,string,ref.Exception)
+
+
+Index (idx) file format
+=======================
+
+Files with the ``.idx`` extension are generated when you use the [Index
+switch] along with commands to generate
+documentation from source or text files. You can programmatically generate
+indices with the [setIndexTerm()](
+rstgen.html#setIndexTerm,RstGenerator,string,string,string,string,string)
+and `writeIndexFile() `_ procs.
+The purpose of `idx` files is to hold the interesting symbols and their HTML
+references so they can be later concatenated into a big index file with
+[mergeIndexes()](rstgen.html#mergeIndexes,string).  This section documents
+the file format in detail.
+
+Index files are line-oriented and tab-separated (newline and tab characters
+have to be escaped). Each line represents a record with 6 fields.
+The content of these columns is:
+
+0. Discriminator tag denoting type of the index entry, allowed values are:
+   `markupTitle`
+   : a title for ``.md``/``.rst`` document
+   `nimTitle`
+   : a title of ``.nim`` module
+   `heading`
+   : heading of sections, can be both in Nim and markup files
+   `idx`
+   : terms marked with :idx: role
+   `nim`
+   : a Nim symbol
+   `nimgrp`
+   : a Nim group for overloadable symbols like `proc`s
+1. Mandatory term being indexed. Terms can include quoting according to
+   Nim's rules (e.g. \`^\`).
+2. Base filename plus anchor hyperlink (e.g. ``algorithm.html#*,int,SortOrder``).
+3. Optional human-readable string to display as a hyperlink. If the value is not
+   present or is the empty string, the hyperlink will be rendered
+   using the term. Prefix whitespace indicates that this entry is
+   not for an API symbol but for a TOC entry.
+4. Optional title or description of the hyperlink. Browsers usually display
+   this as a tooltip after hovering a moment over the hyperlink.
+5. A line number of file where the entry was defined.
+
+The index generation tools differentiate between documentation
+generated from ``.nim`` files and documentation generated from ``.md`` or
+``.rst`` files by tag `nimTitle` or `markupTitle` in the 1st line of
+the ``.idx`` file.
+
+.. TODO Normal symbols are added to the index with surrounding whitespaces removed. An
+  exception to this are the table of content (TOC) entries. TOC entries are added to
+  the index file with their third column having as much prefix spaces as their
+  level is in the TOC (at least 1 character). The prefix whitespace helps to
+  filter TOC entries from API or text symbols. This is important because the
+  amount of spaces is used to replicate the hierarchy for document TOCs in the
+  final index, and TOC entries found in ``.nim`` files are discarded.
+
+
+Additional resources
+====================
+
+* [Nim Compiler User Guide](nimc.html#compiler-usage-commandminusline-switches)
+
+* already mentioned documentation for
+  [Markdown and RST markup languages](markdown_rst.html), which also
+  contains the list of implemented features of these markup languages.
+
+* the implementation is in [module compiler/docgen].
+
+The output for HTML and LaTeX comes from the ``config/nimdoc.cfg`` and
+``config/nimdoc.tex.cfg`` configuration files. You can add and modify these
+files to your project to change the look of the docgen output.
+
+You can import the [packages/docutils/rstgen module](rstgen.html) in your
+programs if you want to reuse the compiler's documentation generation procs.
diff --git a/doc/docgen.rst b/doc/docgen.rst
deleted file mode 100644
index c002ddb836..0000000000
--- a/doc/docgen.rst
+++ /dev/null
@@ -1,412 +0,0 @@
-.. default-role:: code
-
-===================================
-   Nim DocGen Tools Guide
-===================================
-
-:Author: Erik O'Leary
-:Version: |nimversion|
-
-.. contents::
-
-
-Introduction
-============
-
-This document describes the `documentation generation tools`:idx: built into
-the `Nim compiler `_, which can generate HTML and JSON output
-from input .nim files and projects, as well as HTML and LaTeX from input RST
-(reStructuredText) files. The output documentation will include the module
-dependencies (`import`), any top-level documentation comments (##), and
-exported symbols (*), including procedures, types, and variables.
-
-Quick start
------------
-
-Generate HTML documentation for a file:
-
-::
-  nim doc .nim
-
-Generate HTML documentation for a whole project:
-
-::
-  # delete any htmldocs/*.idx file before starting
-  nim doc --project --index:on --git.url: --git.commit: --outdir:htmldocs .nim
-  # this will generate html files, a theindex.html index, css and js under `htmldocs`
-  # See also `--docroot` to specify a relative root.
-  # to get search (dochacks.js) to work locally, you need a server otherwise
-  # CORS will prevent opening file:// urls; this works:
-  python3 -m http.server 7029 --directory htmldocs
-  # When --outdir is omitted it defaults to $projectPath/htmldocs,
-  or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources;
-  and likewise without `--project`.
-  Adding `-r` will open in a browser directly.
-
-
-Documentation Comments
-----------------------
-
-Any comments which are preceded by a double-hash (##), are interpreted as
-documentation.  Comments are parsed as RST (see `reference
-`_), providing
-Nim module authors the ability to easily generate richly formatted
-documentation with only their well-documented code.
-
-Example:
-
-.. code-block:: nim
-  type Person* = object
-    ## This type contains a description of a person
-    name: string
-    age: int
-
-Outputs::
-  Person* = object
-    name: string
-    age: int
-
-This type contains a description of a person
-
-Field documentation comments can be added to fields like so:
-
-.. code-block:: nim
-  var numValues: int ## \
-    ## `numValues` stores the number of values
-
-Note that without the `*` following the name of the type, the documentation for
-this type would not be generated. Documentation will only be generated for
-*exported* types/procedures/etc.
-
-It's recommended to always add exactly **one** space after `##` for readability
-of comments — this extra space will be cropped from the parsed comments and
-won't influence RST formatting.
-
-.. note:: Generally, this baseline indentation level inside a documentation
-   comment may not be 1: it can be any since it is determined by the offset
-   of the first non-whitespace character in the comment.
-   After that indentation **must** be consistent on the following lines of
-   the same comment.
-   If you still need to add an additional indentation at the very beginning
-   (for RST block quote syntax) use backslash \\ before it:
-
-   .. code-block:: nim
-      ## \
-      ##
-      ##    Block quote at the first line.
-      ##
-      ## Paragraph.
-
-Nim file input
------------------
-
-The following examples will generate documentation for the below contrived
-*Nim* module, aptly named 'sample.nim'
-
-sample.nim:
-
-.. code-block:: nim
-  ## This module is a sample.
-
-  import std/strutils
-
-  proc helloWorld*(times: int) =
-    ## Takes an integer and outputs
-    ## as many "hello world!"s
-
-    for i in 0 .. times-1:
-      echo "hello world!"
-
-  helloWorld(5)
-
-
-Document Types
-==============
-
-
-HTML
-----
-
-The generation of HTML documents is done via the `doc` command. This command
-takes either a single .nim file, outputting a single .html file with the same
-base filename, or multiple .nim files, outputting multiple .html files and,
-optionally, an index file.
-
-The `doc` command::
-  nim doc sample
-
-Partial Output::
-  ...
-  proc helloWorld(times: int) {.raises: [], tags: [].}
-  ...
-
-The full output can be seen here: `docgen_sample.html `_.
-It runs after semantic checking and includes pragmas attached implicitly by the
-compiler.
-
-
-JSON
-----
-
-The generation of JSON documents is done via the `jsondoc` command. This command
-takes in a .nim file and outputs a .json file with the same base filename. Note
-that this tool is built off of the `doc` command (previously `doc2`), and
-contains the same information.
-
-The `jsondoc` command::
-  nim jsondoc sample
-
-Output::
-  {
-    "orig": "docgen_sample.nim",
-    "nimble": "",
-    "moduleDescription": "This module is a sample",
-    "entries": [
-      {
-        "name": "helloWorld",
-        "type": "skProc",
-        "line": 5,
-        "col": 0,
-        "description": "Takes an integer and outputs as many "hello world!"s",
-        "code": "proc helloWorld(times: int) {.raises: [], tags: [].}"
-      }
-    ]
-  }
-
-Similarly to the old `doc` command, the old `jsondoc` command has been
-renamed to `jsondoc0`.
-
-The `jsondoc0` command::
-  nim jsondoc0 sample
-
-Output::
-  [
-    {
-      "comment": "This module is a sample."
-    },
-    {
-      "name": "helloWorld",
-      "type": "skProc",
-      "description": "Takes an integer and outputs as many "hello world!"s",
-      "code": "proc helloWorld*(times: int)"
-    }
-  ]
-
-Note that the `jsondoc` command outputs it's JSON without pretty-printing it,
-while `jsondoc0` outputs pretty-printed JSON.
-
-Related Options
-===============
-
-Project switch
---------------
-
-::
-  nim doc --project filename.nim
-
-This will recursively generate documentation of all nim modules imported
-into the input module that belong to the Nimble package that `filename.nim`
-belongs to.
-
-
-Index switch
-------------
-
-::
-  nim doc --index:on filename.nim
-
-This will generate an index of all the exported symbols in the input Nim
-module, and put it into a neighboring file with the extension of `.idx`. The
-index file is line-oriented (newlines have to be escaped). Each line
-represents a tab-separated record of several columns, the first two mandatory,
-the rest optional. See the `Index (idx) file format`_ section for details.
-
-Once index files have been generated for one or more modules, the Nim
-compiler command `buildIndex directory` can be run to go over all the index
-files in the specified directory to generate a `theindex.html `_
-file.
-
-See source switch
------------------
-
-::
-  nim doc --git.url: filename.nim
-
-With the `git.url` switch the *See source* hyperlink will appear below each
-documented item in your source code pointing to the implementation of that
-item on a GitHub repository.
-You can click the link to see the implementation of the item.
-
-The `git.commit` switch overrides the hardcoded `devel` branch in config/nimdoc.cfg.
-This is useful to link to a different branch e.g. `--git.commit:master`,
-or to a tag e.g. `--git.commit:1.2.3` or a commit.
-
-Source URLs are generated as `href="${url}/tree/${commit}/${path}#L${line}"` by default and this compatible with GitHub but not with GitLab.
-
-Similarly, `git.devel` switch overrides the hardcoded `devel` branch for the `Edit` link which is also useful if you have a different working branch than `devel` e.g. `--git.devel:master`.
-
-Edit URLs are generated as `href="${url}/tree/${devel}/${path}#L${line}"` by default.
-
-You can edit `config/nimdoc.cfg` and modify the `doc.item.seesrc` value with a hyperlink to your own code repository.
-
-In the case of Nim's own documentation, the `commit` value is just a commit
-hash to append to a formatted URL to https://github.com/nim-lang/Nim. The
-`tools/nimweb.nim` helper queries the current git commit hash during the doc
-generation, but since you might be working on an unpublished repository, it
-also allows specifying a `githash` value in `web/website.ini` to force a
-specific commit in the output.
-
-
-Other Input Formats
-===================
-
-The *Nim compiler* also has support for RST (reStructuredText) files with
-the `rst2html` and `rst2tex` commands. Documents like this one are
-initially written in a dialect of RST which adds support for nim source code
-highlighting with the `.. code-block:: nim` prefix. `code-block` also
-supports highlighting of C++ and some other c-like languages.
-
-Usage::
-  nim rst2html docgen.txt
-
-Output::
-  You're reading it!
-
-The `rst2tex` command is invoked identically to `rst2html`, but outputs
-a .tex file instead of .html.
-
-
-HTML anchor generation
-======================
-
-When you run the `rst2html` command, all sections in the RST document will
-get an anchor you can hyperlink to. Usually, you can guess the anchor lower
-casing the section title and replacing spaces with dashes, and in any case, you
-can get it from the table of contents. But when you run the `doc`
-command to generate API documentation, some symbol get one or two anchors at
-the same time: a numerical identifier, or a plain name plus a complex name.
-
-The numerical identifier is just a random number. The number gets assigned
-according to the section and position of the symbol in the file being processed
-and you should not rely on it being constant: if you add or remove a symbol the
-numbers may shuffle around.
-
-The plain name of a symbol is a simplified version of its fully exported
-signature. Variables or constants have the same plain name symbol as their
-complex name. The plain name for procs, templates, and other callable types
-will be their unquoted value after removing parameters, return types, and
-pragmas. The plain name allows short and nice linking of symbols that works
-unless you have a module with collisions due to overloading.
-
-If you hyperlink a plain name symbol and there are other matches on the same
-HTML file, most browsers will go to the first one. To differentiate the rest,
-you will need to use the complex name. A complex name for a callable type is
-made up of several parts:
-
-    (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\*
-
-The first thing to note is that all callable types have at least a comma, even
-if they don't have any parameters. If there are parameters, they are
-represented by their types and will be comma-separated. To the plain symbol a
-suffix may be added depending on the type of the callable:
-
--------------   --------------
-Callable type   Suffix
--------------   --------------
-proc            *empty string*
-macro           `.m`
-method          `.e`
-iterator        `.i`
-template        `.t`
-converter       `.c`
--------------   --------------
-
-The relationship of type to suffix is made by the proc `complexName` in the
-`compiler/docgen.nim` file. Here are some examples of complex names for
-symbols in the `system module `_.
-
-* `type SomeSignedInt = int | int8 | int16 | int32 | int64` **=>**
-  `#SomeSignedInt `_
-* `var globalRaiseHook: proc (e: ref E_Base): bool {.nimcall.}` **=>**
-  `#globalRaiseHook `_
-* `const NimVersion = "0.0.0"` **=>**
-  `#NimVersion `_
-* `proc getTotalMem(): int {.rtl, raises: [], tags: [].}` **=>**
-  `#getTotalMem, `_
-* `proc len[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.}` **=>**
-  `#len,seq[T] `_
-* `iterator pairs[T](a: seq[T]): tuple[key: int, val: T] {.inline.}` **=>**
-  `#pairs.i,seq[T] `_
-* `template newException[](exceptn: typedesc; message: string;
-    parentException: ref Exception = nil): untyped` **=>**
-  `#newException.t,typedesc,string,ref.Exception
-  `_
-
-
-Index (idx) file format
-=======================
-
-Files with the `.idx` extension are generated when you use the `Index
-switch <#related-options-index-switch>`_ along with commands to generate
-documentation from source or text files. You can programmatically generate
-indices with the `setIndexTerm()
-`_
-and `writeIndexFile() `_ procs.
-The purpose of `idx` files is to hold the interesting symbols and their HTML
-references so they can be later concatenated into a big index file with
-`mergeIndexes() `_.  This section documents
-the file format in detail.
-
-Index files are line-oriented and tab-separated (newline and tab characters
-have to be escaped). Each line represents a record with at least two fields
-but can have up to four (additional columns are ignored). The content of these
-columns is:
-
-1. Mandatory term being indexed. Terms can include quoting according to
-   Nim's rules (e.g. \`^\`).
-2. Base filename plus anchor hyperlink (e.g. `algorithm.html#*,int,SortOrder`).
-3. Optional human-readable string to display as a hyperlink. If the value is not
-   present or is the empty string, the hyperlink will be rendered
-   using the term. Prefix whitespace indicates that this entry is
-   not for an API symbol but for a TOC entry.
-4. Optional title or description of the hyperlink. Browsers usually display
-   this as a tooltip after hovering a moment over the hyperlink.
-
-The index generation tools try to differentiate between documentation
-generated from `.nim` files and documentation generated from `.txt` or
-`.rst` files. The former are always closely related to source code and
-consist mainly of API entries. The latter are generic documents meant for
-human reading.
-
-To differentiate both types (documents and APIs), the index generator will add
-to the index of documents an entry with the title of the document. Since the
-title is the topmost element, it will be added with a second field containing
-just the filename without any HTML anchor.  By convention, this entry without
-anchor is the *title entry*, and since entries in the index file are added as
-they are scanned, the title entry will be the first line. The title for APIs
-is not present because it can be generated concatenating the name of the file
-to the word **Module**.
-
-Normal symbols are added to the index with surrounding whitespaces removed. An
-exception to this are the table of content (TOC) entries. TOC entries are added to
-the index file with their third column having as much prefix spaces as their
-level is in the TOC (at least 1 character). The prefix whitespace helps to
-filter TOC entries from API or text symbols. This is important because the
-amount of spaces is used to replicate the hierarchy for document TOCs in the
-final index, and TOC entries found in `.nim` files are discarded.
-
-
-Additional resources
-====================
-
-`Nim Compiler User Guide `_
-
-`RST Quick Reference
-`_
-
-The output for HTML and LaTeX comes from the `config/nimdoc.cfg` and
-`config/nimdoc.tex.cfg` configuration files. You can add and modify these
-files to your project to change the look of the docgen output.
-
-You can import the `packages/docutils/rstgen module `_ in your
-programs if you want to reuse the compiler's documentation generation procs.
diff --git a/doc/docgen_sample.nim b/doc/docgen_sample.nim
index 7a167cb457..06b8d7f8e2 100644
--- a/doc/docgen_sample.nim
+++ b/doc/docgen_sample.nim
@@ -1,6 +1,6 @@
 ## This module is a sample.
 
-import strutils
+import std/strutils
 
 proc helloWorld*(times: int) =
   ## Takes an integer and outputs
diff --git a/doc/docs.rst b/doc/docs.md
similarity index 63%
rename from doc/docs.rst
rename to doc/docs.md
index b34383253c..b6ff6d2c70 100644
--- a/doc/docs.rst
+++ b/doc/docs.md
@@ -1,38 +1,38 @@
 The documentation consists of several documents:
 
-- | `Tutorial (part I) `_
+- | [Tutorial (part I)](tut1.html)
   | The Nim tutorial part one deals with the basics.
 
-- | `Tutorial (part II) `_
+- | [Tutorial (part II)](tut2.html)
   | The Nim tutorial part two deals with the advanced language constructs.
 
-- | `Tutorial (part III) `_
+- | [Tutorial (part III)](tut3.html)
   | The Nim tutorial part three about Nim's macro system.
 
-- | `Language Manual `_
+- | [Language Manual](manual.html)
   | The Nim manual is a draft that will evolve into a proper specification.
 
-- | `Library documentation `_
+- | [Library documentation](lib.html)
   | This document describes Nim's standard library.
 
-- | `Compiler user guide `_
+- | [Compiler user guide](nimc.html)
   | The user guide lists command line arguments, special features of the
     compiler, etc.
 
-- | `Tools documentation `_
+- | [Tools documentation](tools.html)
   | Description of some tools that come with the standard distribution.
 
-- | `GC `_
-  | Additional documentation about Nim's multi-paradigm memory management strategies
+- | [Memory management](mm.html)
+  | Additional documentation about Nim's memory management strategies
   | and how to operate them in a realtime setting.
 
-- | `Source code filters `_
+- | [Source code filters](filters.html)
   | The Nim compiler supports source code filters as a simple yet powerful
     builtin templating system.
 
-- | `Internal documentation `_
+- | [Internal documentation](intern.html)
   | The internal documentation describes how the compiler is implemented. Read
     this if you want to hack the compiler.
 
-- | `Index `_
+- | [Index](theindex.html)
   | The generated index. **Index + (Ctrl+F) == Joy**
diff --git a/doc/docstyle.rst b/doc/docstyle.md
similarity index 80%
rename from doc/docstyle.rst
rename to doc/docstyle.md
index 7f3fa8cf20..291a34cf68 100644
--- a/doc/docstyle.rst
+++ b/doc/docstyle.md
@@ -4,51 +4,52 @@ Documentation Style
 General Guidelines
 ------------------
 
-* See also `nep1`_ which should probably be merged here.
+* See also [nep1](nep1.html) which should probably be merged here.
 * Authors should document anything that is exported; documentation for private
-  procs can be useful too (visible via `nim doc --docInternal foo.nim`).
+  procs can be useful too (visible via `nim doc --docInternal foo.nim`:cmd:).
 * Within documentation, a period (`.`) should follow each sentence (or sentence fragment) in a comment block.
   The documentation may be limited to one sentence fragment, but if multiple sentences are within the documentation,
   each sentence after the first should be complete and in present tense.
 * Documentation is parsed as a custom ReStructuredText (RST) with partial markdown support.
 * In nim sources, prefer single backticks to double backticks since it's simpler
-  and `nim doc` supports it. Likewise with rst files: `nim rst2html` will render those as monospace, and
-  adding `.. default-role:: code` to an rst file will also make those render as monospace when rendered directly
+  and `nim doc`:cmd: supports it. Likewise with ``rst`` files: `nim rst2html`:cmd: will render those as monospace, and
+  adding ``.. default-role:: code`` to an ``rst`` file will also make those render as monospace when rendered directly
   in tools such as github.
-* In nim sources, for links, prefer `[link text](link.html)` to `` `link text`_ ``
-  since the syntax is simpler and markdown is more common (likewise, `nim rst2html` also supports it in rst files).
-
-.. code-block:: nim
+* (debatable) In nim sources, for links, prefer ``[link text](link.html)`` to `\`link text\`_`:code:
+  since the syntax is simpler and markdown is more common (likewise, `nim rst2html`:cmd: also supports it in ``rst`` files).
 
+  ```nim
   proc someproc*(s: string, foo: int) =
     ## Use single backticks for inline code, e.g.: `s` or `someExpr(true)`.
     ## Use a backlash to follow with alphanumeric char: `int8`\s are great.
+  ```
 
 
 Module-level documentation
 --------------------------
 
 Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (`##`).
-Sometimes `##[ multiline docs containing code ]##` is preferable, see `lib/pure/times.nim`.
+Sometimes `##[ multiline docs containing code ]##` is preferable, see ``lib/pure/times.nim``.
 Code samples are encouraged, and should follow the general RST syntax:
 
-.. code-block:: Nim
-
+  ````Nim
   ## The `universe` module computes the answer to life, the universe, and everything.
   ##
-  ## .. code-block::
-  ##  doAssert computeAnswerString() == 42
+  ##   ```
+  ##   doAssert computeAnswerString() == 42
+  ##   ```
+  ````
 
 
 Within this top-level comment, you can indicate the authorship and copyright of the code, which will be featured in the produced documentation.
 
-.. code-block:: Nim
-
+  ```Nim
   ## This is the best module ever. It provides answers to everything!
   ##
   ## :Author: Steve McQueen
   ## :Copyright: 1965
   ##
+  ```
 
 Leave a space between the last line of top-level documentation and the beginning of Nim code (the imports, etc.).
 
@@ -57,38 +58,38 @@ Procs, Templates, Macros, Converters, and Iterators
 
 The documentation of a procedure should begin with a capital letter and should be in present tense. Variables referenced in the documentation should be surrounded by single tick marks:
 
-.. code-block:: Nim
-
+  ```Nim
   proc example1*(x: int) =
     ## Prints the value of `x`.
     echo x
+  ```
 
 Whenever an example of usage would be helpful to the user, you should include one within the documentation in RST format as below.
 
-.. code-block:: Nim
-
+  ````Nim
   proc addThree*(x, y, z: int8): int =
     ## Adds three `int8` values, treating them as unsigned and
     ## truncating the result.
     ##
-    ## .. code-block::
-    ##  # things that aren't suitable for a `runnableExamples` go in code-block:
-    ##  echo execCmdEx("git pull")
-    ##  drawOnScreen()
+    ##   ```
+    ##   # things that aren't suitable for a `runnableExamples` go in code block:
+    ##   echo execCmdEx("git pull")
+    ##   drawOnScreen()
+    ##   ```
     runnableExamples:
-      # `runnableExamples` is usually preferred to `code-block`, when possible.
+      # `runnableExamples` is usually preferred to code blocks, when possible.
       doAssert addThree(3, 125, 6) == -122
     result = x +% y +% z
+  ````
 
-The command `nim doc` will then correctly syntax highlight the Nim code within the documentation.
+The command `nim doc`:cmd: will then correctly syntax highlight the Nim code within the documentation.
 
 Types
 -----
 
 Exported types should also be documented. This documentation can also contain code samples, but those are better placed with the functions to which they refer.
 
-.. code-block:: Nim
-
+  ```Nim
   type
     NamedQueue*[T] = object ## Provides a linked data structure with names
                             ## throughout. It is named for convenience. I'm making
@@ -96,12 +97,12 @@ Exported types should also be documented. This documentation can also contain co
       name*: string ## The name of the item
       val*: T ## Its value
       next*: ref NamedQueue[T] ## The next item in the queue
+  ```
 
 
 You have some flexibility when placing the documentation:
 
-.. code-block:: Nim
-
+  ```Nim
   type
     NamedQueue*[T] = object
       ## Provides a linked data structure with names
@@ -110,11 +111,11 @@ You have some flexibility when placing the documentation:
       name*: string ## The name of the item
       val*: T ## Its value
       next*: ref NamedQueue[T] ## The next item in the queue
+  ```
 
 Make sure to place the documentation beside or within the object.
 
-.. code-block:: Nim
-
+  ```Nim
   type
     ## Bad: this documentation disappears because it annotates the `type` keyword
     ## above, not `NamedQueue`.
@@ -123,14 +124,14 @@ Make sure to place the documentation beside or within the object.
                     ## is not what we want.
       val*: T ## Its value
       next*: ref NamedQueue[T] ## The next item in the queue
+  ```
 
 Var, Let, and Const
 -------------------
 
 When declaring module-wide constants and values, documentation is encouraged. The placement of doc comments is similar to the `type` sections.
 
-.. code-block:: Nim
-
+  ```Nim
   const
     X* = 42 ## An awesome number.
     SpreadArray* = [
@@ -138,25 +139,26 @@ When declaring module-wide constants and values, documentation is encouraged. Th
       [2,3,1],
       [3,1,2],
     ] ## Doc comment for `SpreadArray`.
+  ```
 
 Placement of comments in other areas is usually allowed, but will not become part of the documentation output and should therefore be prefaced by a single hash (`#`).
 
-.. code-block:: Nim
-
+  ```Nim
   const
     BadMathVals* = [
       3.14, # pi
       2.72, # e
       0.58, # gamma
     ] ## A bunch of badly rounded values.
+  ```
 
 Nim supports Unicode in comments, so the above can be replaced with the following:
 
-.. code-block:: Nim
-
+  ```Nim
   const
     BadMathVals* = [
       3.14, # π
       2.72, # e
       0.58, # γ
     ] ## A bunch of badly rounded values (including π!).
+  ```
diff --git a/doc/drnim.rst b/doc/drnim.md
similarity index 83%
rename from doc/drnim.rst
rename to doc/drnim.md
index d33a6066e0..1dc2b550f1 100644
--- a/doc/drnim.rst
+++ b/doc/drnim.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 ===================================
    DrNim User Guide
 ===================================
@@ -7,6 +5,8 @@
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
@@ -14,7 +14,7 @@ Introduction
 ============
 
 This document describes the usage of the *DrNim* tool. DrNim combines
-the Nim frontend with the `Z3 `_ proof
+the Nim frontend with the [Z3](https://github.com/Z3Prover/z3) proof
 engine, in order to allow verify/validate software written in Nim.
 DrNim's command-line options are the same as the Nim compiler's.
 
@@ -22,11 +22,11 @@ DrNim's command-line options are the same as the Nim compiler's.
 DrNim currently only checks the sections of your code that are marked
 via `staticBoundChecks: on`:
 
-.. code-block:: nim
-
+  ```nim
   {.push staticBoundChecks: on.}
   # <--- code section here ---->
   {.pop.}
+  ```
 
 DrNim currently only tries to prove array indexing or subrange checks,
 overflow errors are *not* prevented. Overflows will be checked for in
@@ -43,7 +43,8 @@ your code.
 Installation
 ============
 
-Run `koch drnim`, the executable will afterwards be in `$nim/bin/drnim`.
+Run `koch drnim`:cmd:, the executable will afterwards be
+in ``$nim/bin/drnim``.
 
 
 Motivating Example
@@ -52,8 +53,7 @@ Motivating Example
 The follow example highlights what DrNim can easily do, even
 without additional annotations:
 
-.. code-block:: nim
-
+  ```nim
   {.push staticBoundChecks: on.}
 
   proc sum(a: openArray[int]): int =
@@ -63,11 +63,12 @@ without additional annotations:
   {.pop.}
 
   echo sum([1, 2, 3])
+  ```
 
 This program contains a famous "index out of bounds" bug. DrNim
-detects it and produces the following error message::
+detects it and produces the following error message:
 
-  cannot prove: i <= len(a) + -1; counter example: i -> 0 a.len -> 0 [IndexCheck]
+    cannot prove: i <= len(a) + -1; counter example: i -> 0 a.len -> 0 [IndexCheck]
 
 In other words for `i == 0` and `a.len == 0` (for example!) there would be
 an index out of bounds error.
@@ -124,8 +125,7 @@ Example: insertionSort
 
 **Note**: This example does not yet work with DrNim.
 
-.. code-block:: nim
-
+  ```nim
   import std / logic
 
   proc insertionSort(a: var openArray[int]) {.
@@ -141,27 +141,28 @@ Example: insertionSort
         {.invariant: forall(j in 1..k, i in 0..`.
-A `prop` is either a comparison or a compound::
+A `prop` is either a comparison or a compound:
 
-  prop = nim_bool_expression
-       | prop 'and' prop
-       | prop 'or' prop
-       | prop '->' prop # implication
-       | prop '<->' prop
-       | 'not' prop
-       | '(' prop ')' # you can group props via ()
-       | forallProp
-       | existsProp
+    prop = nim_bool_expression
+         | prop 'and' prop
+         | prop 'or' prop
+         | prop '->' prop # implication
+         | prop '<->' prop
+         | 'not' prop
+         | '(' prop ')' # you can group props via ()
+         | forallProp
+         | existsProp
 
-  forallProp = 'forall' '(' quantifierList ',' prop ')'
-  existsProp = 'exists' '(' quantifierList ',' prop ')'
+    forallProp = 'forall' '(' quantifierList ',' prop ')'
+    existsProp = 'exists' '(' quantifierList ',' prop ')'
 
-  quantifierList = quantifier (',' quantifier)*
-  quantifier =  'in' nim_iteration_expression
+    quantifierList = quantifier (',' quantifier)*
+    quantifier =  'in' nim_iteration_expression
 
 
 `nim_iteration_expression` here is an ordinary expression of Nim code
diff --git a/doc/effects.txt b/doc/effects.txt
index 4ed1d09f1e..df624e8b02 100644
--- a/doc/effects.txt
+++ b/doc/effects.txt
@@ -11,13 +11,14 @@ Iff a proc is side effect free and all its argument are evaluable at
 compile time, it can be evaluated by the compiler. However, really
 difficult is the ``newString`` proc: If it is simply wrapped, it
 should not be evaluated at compile time! On other occasions it can
-and should be evaluted:
+and should be evaluated:
 
-.. code-block:: nim
+  ```nim
   proc toUpper(s: string): string =
     result = newString(len(s))
     for i in 0..len(s) - 1:
       result[i] = toUpper(s[i])
+  ```
 
 No, it really can always be evaluated. The code generator should transform
 ``s = "\0\0\0..."`` back into ``s = newString(...)``.
diff --git a/doc/estp.md b/doc/estp.md
new file mode 100644
index 0000000000..8a986bdf30
--- /dev/null
+++ b/doc/estp.md
@@ -0,0 +1,206 @@
+===================================================
+  Embedded Stack Trace Profiler (ESTP) User Guide
+===================================================
+
+.. default-role:: code
+.. include:: rstcommon.rst
+
+:Author: Andreas Rumpf
+:Version: |nimversion|
+
+
+Nim comes with a platform independent profiler -
+the Embedded Stack Trace Profiler (ESTP). The profiler
+is *embedded* into your executable. To activate the profiler you need to do:
+
+* compile your program with the `--profiler:on --stackTrace:on`:option: command
+  line options
+* import the `nimprof` module
+* run your program as usual.
+
+You can in fact look at `nimprof`'s source code to see how to implement
+your own profiler.
+
+The setting `--profiler:on`:option: defines the conditional symbol `profiler`.
+You can use `when compileOption("profiler")` to make the switch seamless.
+If `profiler`:option: is `off`:option:, your program runs normally.
+Otherwise your program is profiled.
+
+```nim
+when compileOption("profiler"):
+  import std/nimprof
+```
+
+After your program has finished the profiler will create a
+file ``profile_results.txt`` containing the profiling results.
+
+Since the profiler works by examining stack traces, it's essential that
+the option `--stackTrace:on`:option: is active! Unfortunately this means that a
+profiling build is much slower than a release build.
+
+
+Memory profiler
+===============
+
+You can also use ESTP as a memory profiler to see which stack traces allocate
+the most memory and thus create the most GC pressure. It may also help to
+find memory leaks. To activate the memory profiler you need to do:
+
+* compile your program with the
+  `--profiler:off --stackTrace:on -d:memProfiler`:option:
+  command line options. Yes it's `--profiler:off`:option:.
+* import the `nimprof` module
+* run your program as usual.
+
+Define the symbol `ignoreAllocationSize` so that only the number of
+allocations is counted and the sizes of the memory allocations do not matter.
+
+
+Example results file
+====================
+
+The results file lists stack traces ordered by significance.
+
+The following example file has been generated by profiling the Nim compiler
+itself: It shows that in total 5.4% of the runtime has been spent
+in `crcFromRope` or its children.
+
+In general the stack traces show you immediately where the problem is because
+the trace acts like an explanation; in traditional profilers you can only find
+expensive leaf functions easily but the *reason* why they are invoked
+often remains mysterious.
+
+    total executions of each stack trace:
+    Entry: 0/3391 Calls: 84/4160 = 2.0% [sum: 84; 84/4160 = 2.0%]
+      newCrcFromRopeAux
+      crcFromRope
+      writeRopeIfNotEqual
+      shouldRecompile
+      writeModule
+      myClose
+      closePasses
+      processModule
+      CompileModule
+      CompileProject
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
+      nim
+    Entry: 1/3391 Calls: 46/4160 = 1.1% [sum: 130; 130/4160 = 3.1%]
+      updateCrc32
+      newCrcFromRopeAux
+      crcFromRope
+      writeRopeIfNotEqual
+      shouldRecompile
+      writeModule
+      myClose
+      closePasses
+      processModule
+      CompileModule
+      CompileProject
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
+      nim
+    Entry: 2/3391 Calls: 41/4160 = 0.99% [sum: 171; 171/4160 = 4.1%]
+      updateCrc32
+      updateCrc32
+      newCrcFromRopeAux
+      crcFromRope
+      writeRopeIfNotEqual
+      shouldRecompile
+      writeModule
+      myClose
+      closePasses
+      processModule
+      CompileModule
+      CompileProject
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
+      nim
+    Entry: 3/3391 Calls: 41/4160 = 0.99% [sum: 212; 212/4160 = 5.1%]
+      crcFromFile
+      writeRopeIfNotEqual
+      shouldRecompile
+      writeModule
+      myClose
+      closePasses
+      processModule
+      CompileModule
+      CompileProject
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
+      nim
+    Entry: 4/3391 Calls: 41/4160 = 0.99% [sum: 253; 253/4160 = 6.1%]
+      updateCrc32
+      crcFromFile
+      writeRopeIfNotEqual
+      shouldRecompile
+      writeModule
+      myClose
+      closePasses
+      processModule
+      CompileModule
+      CompileProject
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
+      nim
+    Entry: 5/3391 Calls: 32/4160 = 0.77% [sum: 285; 285/4160 = 6.9%]
+      pop
+      newCrcFromRopeAux
+      crcFromRope
+      writeRopeIfNotEqual
+      shouldRecompile
+      writeModule
+      myClose
+      closePasses
+      processModule
+      CompileModule
+      CompileProject
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
+      nim
+    Entry: 6/3391 Calls: 17/4160 = 0.41% [sum: 302; 302/4160 = 7.3%]
+      doOperation
+      forAllChildrenAux
+      pop
+      newCrcFromRopeAux
+      crcFromRope
+      writeRopeIfNotEqual
+      shouldRecompile
+      writeModule
+      myClose
+      closePasses
+      processModule
+      CompileModule
+      CompileProject
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
+      ...
+      nim
+    Entry: 7/3391 Calls: 14/4160 = 0.34% [sum: 316; 316/4160 = 7.6%]
+      Contains
+      isAccessible
+      interiorAllocatedPtr
+      gcMark
+      markStackAndRegisters
+      collectCTBody
+      collectCT
+      rawNewObj
+      newObj
+      newNode
+      copyTree
+      matchesAux
+      matches
+      resolveOverloads
+      semOverloadedCall
+      semOverloadedCallAnalyseEffects
+      ...
+      CommandCompileToC
+      MainCommand
+      HandleCmdLine
diff --git a/doc/estp.rst b/doc/estp.rst
deleted file mode 100644
index 8146562b6a..0000000000
--- a/doc/estp.rst
+++ /dev/null
@@ -1,197 +0,0 @@
-.. default-role:: code
-
-===================================================
-  Embedded Stack Trace Profiler (ESTP) User Guide
-===================================================
-
-:Author: Andreas Rumpf
-:Version: |nimversion|
-
-
-Nim comes with a platform independent profiler -
-the Embedded Stack Trace Profiler (ESTP). The profiler
-is *embedded* into your executable. To activate the profiler you need to do:
-
-* compile your program with the `--profiler:on --stackTrace:on` command
-  line options
-* import the `nimprof` module
-* run your program as usual.
-
-You can in fact look at `nimprof`'s source code to see how to implement
-your own profiler.
-
-The setting `--profiler:on` defines the conditional symbol `profiler`.
-
-After your program has finished the profiler will create a
-file `profile_results.txt` containing the profiling results.
-
-Since the profiler works by examining stack traces, it's essential that
-the option `--stackTrace:on` is active! Unfortunately this means that a
-profiling build is much slower than a release build.
-
-
-Memory profiler
-===============
-
-You can also use ESTP as a memory profiler to see which stack traces allocate
-the most memory and thus create the most GC pressure. It may also help to
-find memory leaks. To activate the memory profiler you need to do:
-
-* compile your program with the `--profiler:off --stackTrace:on -d:memProfiler`
-  command line options. Yes it's `--profiler:off`.
-* import the `nimprof` module
-* run your program as usual.
-
-Define the symbol `ignoreAllocationSize` so that only the number of
-allocations is counted and the sizes of the memory allocations do not matter.
-
-
-Example results file
-====================
-
-The results file lists stack traces ordered by significance.
-
-The following example file has been generated by profiling the Nim compiler
-itself: It shows that in total 5.4% of the runtime has been spent
-in `crcFromRope` or its children.
-
-In general the stack traces show you immediately where the problem is because
-the trace acts like an explanation; in traditional profilers you can only find
-expensive leaf functions easily but the *reason* why they are invoked
-often remains mysterious.
-
-::
-  total executions of each stack trace:
-  Entry: 0/3391 Calls: 84/4160 = 2.0% [sum: 84; 84/4160 = 2.0%]
-    newCrcFromRopeAux
-    crcFromRope
-    writeRopeIfNotEqual
-    shouldRecompile
-    writeModule
-    myClose
-    closePasses
-    processModule
-    CompileModule
-    CompileProject
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
-    nim
-  Entry: 1/3391 Calls: 46/4160 = 1.1% [sum: 130; 130/4160 = 3.1%]
-    updateCrc32
-    newCrcFromRopeAux
-    crcFromRope
-    writeRopeIfNotEqual
-    shouldRecompile
-    writeModule
-    myClose
-    closePasses
-    processModule
-    CompileModule
-    CompileProject
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
-    nim
-  Entry: 2/3391 Calls: 41/4160 = 0.99% [sum: 171; 171/4160 = 4.1%]
-    updateCrc32
-    updateCrc32
-    newCrcFromRopeAux
-    crcFromRope
-    writeRopeIfNotEqual
-    shouldRecompile
-    writeModule
-    myClose
-    closePasses
-    processModule
-    CompileModule
-    CompileProject
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
-    nim
-  Entry: 3/3391 Calls: 41/4160 = 0.99% [sum: 212; 212/4160 = 5.1%]
-    crcFromFile
-    writeRopeIfNotEqual
-    shouldRecompile
-    writeModule
-    myClose
-    closePasses
-    processModule
-    CompileModule
-    CompileProject
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
-    nim
-  Entry: 4/3391 Calls: 41/4160 = 0.99% [sum: 253; 253/4160 = 6.1%]
-    updateCrc32
-    crcFromFile
-    writeRopeIfNotEqual
-    shouldRecompile
-    writeModule
-    myClose
-    closePasses
-    processModule
-    CompileModule
-    CompileProject
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
-    nim
-  Entry: 5/3391 Calls: 32/4160 = 0.77% [sum: 285; 285/4160 = 6.9%]
-    pop
-    newCrcFromRopeAux
-    crcFromRope
-    writeRopeIfNotEqual
-    shouldRecompile
-    writeModule
-    myClose
-    closePasses
-    processModule
-    CompileModule
-    CompileProject
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
-    nim
-  Entry: 6/3391 Calls: 17/4160 = 0.41% [sum: 302; 302/4160 = 7.3%]
-    doOperation
-    forAllChildrenAux
-    pop
-    newCrcFromRopeAux
-    crcFromRope
-    writeRopeIfNotEqual
-    shouldRecompile
-    writeModule
-    myClose
-    closePasses
-    processModule
-    CompileModule
-    CompileProject
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
-    ...
-    nim
-  Entry: 7/3391 Calls: 14/4160 = 0.34% [sum: 316; 316/4160 = 7.6%]
-    Contains
-    isAccessible
-    interiorAllocatedPtr
-    gcMark
-    markStackAndRegisters
-    collectCTBody
-    collectCT
-    rawNewObj
-    newObj
-    newNode
-    copyTree
-    matchesAux
-    matches
-    resolveOverloads
-    semOverloadedCall
-    semOverloadedCallAnalyseEffects
-    ...
-    CommandCompileToC
-    MainCommand
-    HandleCmdLine
diff --git a/doc/filelist.txt b/doc/filelist.txt
index c863ba0cf1..5522414fe6 100644
--- a/doc/filelist.txt
+++ b/doc/filelist.txt
@@ -5,7 +5,7 @@ Short description of Nim's modules
 Module          Description
 ==============  ==========================================================
 nim             main module: parses the command line and calls
-                ``main.MainCommand``
+                `main.MainCommand`
 main            implements the top-level command dispatching
 nimconf         implements the config file reader
 syntaxes        dispatcher for the different parsers and filters
@@ -31,16 +31,14 @@ semstmts        contains the semantic checking phase for statements
 semtypes        contains the semantic checking phase for types
 seminst         instantiation of generic procs and types
 semfold         contains code to deal with constant folding
-semthreads      deep program analysis for threads
-evals           contains an AST interpreter for compile time evaluation
+sempass2        Second semantic checking pass over the AST
+vm              contains an AST interpreter for compile time evaluation
 pragmas         semantic checking of pragmas
 
 idents          implements a general mapping from identifiers to an internal
-                representation (``PIdent``) that is used so that a simple
+                representation (`PIdent`) that is used so that a simple
                 id-comparison suffices to establish whether two Nim
                 identifiers are equivalent
-ropes           implements long strings represented as trees for
-                lazy evaluation; used mainly by the code generators
 
 transf          transformations on the AST that need to be done before
                 code generation
diff --git a/doc/filters.rst b/doc/filters.md
similarity index 53%
rename from doc/filters.rst
rename to doc/filters.md
index 52704411fc..9482b0b47f 100644
--- a/doc/filters.rst
+++ b/doc/filters.md
@@ -1,32 +1,32 @@
-.. default-role:: code
-
 ===================
 Source Code Filters
 ===================
 
+.. include:: rstcommon.rst
+.. default-role:: code
 .. contents::
 
 A `Source Code Filter (SCF)`  transforms the input character stream to an in-memory
 output stream before parsing. A filter can be used to provide templating
 systems or preprocessors.
 
-To use a filter for a source file the `#?` notation is used::
+To use a filter for a source file the `#?` notation is used:
 
-  #? stdtmpl(subsChar = '$', metaChar = '#')
-  #proc generateXML(name, age: string): string =
-  #  result = ""
-  
-    $name
-    $age
-  
+    #? stdtmpl(subsChar = '$', metaChar = '#')
+    #proc generateXML(name, age: string): string =
+    #  result = ""
+    
+      $name
+      $age
+    
 
 As the example shows, passing arguments to a filter can be done
 just like an ordinary procedure call with named or positional arguments. The
 available parameters depend on the invoked filter. Before version 0.12.0 of
 the language `#!` was used instead of `#?`.
 
-**Hint:** With `--hint[codeBegin]:on` or `--verbosity:2`
-(or higher) while compiling or `nim check`, Nim lists the processed code after
+**Hint:** With `--hint:codeBegin:on`:option: or `--verbosity:2`:option:
+(or higher) while compiling or `nim check`:cmd:, Nim lists the processed code after
 each filter application.
 
 Usage
@@ -41,23 +41,24 @@ recognize it as Nim source file).
 If we use `generateXML` code shown above and call the SCF file `xmlGen.nimf`
 In your `main.nim`:
 
-.. code-block:: nim
+  ```nim
   include "xmlGen.nimf"
   
   echo generateXML("John Smith","42")
+  ```
 
 Pipe operator
 =============
 
-Filters can be combined with the `|` pipe operator::
+Filters can be combined with the `|` pipe operator:
 
-  #? strip(startswith="<") | stdtmpl
-  #proc generateXML(name, age: string): string =
-  #  result = ""
-  
-    $name
-    $age
-  
+    #? strip(startswith="<") | stdtmpl
+    #proc generateXML(name, age: string): string =
+    #  result = ""
+    
+      $name
+      $age
+    
 
 
 Available filters
@@ -70,11 +71,11 @@ The replace filter replaces substrings in each line.
 
 Parameters and their defaults:
 
-  `sub: string = ""`
-    the substring that is searched for
+* `sub: string = ""`
+  : the substring that is searched for
 
-  `by: string = ""`
-    the string the substring is replaced with
+* `by: string = ""`
+  : the string the substring is replaced with
 
 
 Strip filter
@@ -85,15 +86,15 @@ each line.
 
 Parameters and their defaults:
 
-  `startswith: string = ""`
-    strip only the lines that start with *startswith* (ignoring leading
+* `startswith: string = ""`
+  : strip only the lines that start with *startswith* (ignoring leading
     whitespace). If empty every line is stripped.
 
-  `leading: bool = true`
-    strip leading whitespace
+* `leading: bool = true`
+  : strip leading whitespace
 
-  `trailing: bool = true`
-    strip trailing whitespace
+* `trailing: bool = true`
+  : strip trailing whitespace
 
 
 StdTmpl filter
@@ -107,50 +108,50 @@ statements need `end X` delimiters.
 
 Parameters and their defaults:
 
-  `metaChar: char = '#'`
-    prefix for a line that contains Nim code
+* `metaChar: char = '#'`
+  : prefix for a line that contains Nim code
 
-  `subsChar: char = '$'`
-    prefix for a Nim expression within a template line
+* `subsChar: char = '$'`
+  : prefix for a Nim expression within a template line
 
-  `conc: string = " & "`
-    the operation for concatenation
+* `conc: string = " & "`
+  : the operation for concatenation
 
-  `emit: string = "result.add"`
-    the operation to emit a string literal
+* `emit: string = "result.add"`
+  : the operation to emit a string literal
 
-  `toString: string = "$"`
-    the operation that is applied to each expression
+* `toString: string = "$"`
+  : the operation that is applied to each expression
 
-Example::
+Example:
 
-  #? stdtmpl | standard
-  #proc generateHTMLPage(title, currentTab, content: string,
-  #                      tabs: openArray[string]): string =
-  #  result = ""
-  $title
-  
-    
-    

-      $content
-      A dollar: $$.
-    

-  
+    #? stdtmpl | standard
+    #proc generateHTMLPage(title, currentTab, content: string,
+    #                      tabs: openArray[string]): string =
+    #  result = ""
+    $title
+    
+      
+      

+        $content
+        A dollar: $$.
+      

+    
 
 The filter transforms this into:
 
-.. code-block:: nim
+  ```nim
   proc generateHTMLPage(title, currentTab, content: string,
                         tabs: openArray[string]): string =
     result = ""
@@ -173,6 +174,7 @@ The filter transforms this into:
       "    A dollar: $.\n" &
       "  
\n" &
       "\n")
+  ```
 
 
 Each line that does not start with the meta character (ignoring leading
@@ -181,36 +183,36 @@ whitespace) is converted to a string literal that is added to `result`.
 The substitution character introduces a Nim expression *e* within the
 string literal. *e* is converted to a string with the *toString* operation
 which defaults to `$`. For strong type checking, set `toString` to the
-empty string. *e* must match this PEG pattern::
+empty string. *e* must match this PEG pattern:
 
-  e <- [a-zA-Z\128-\255][a-zA-Z0-9\128-\255_.]* / '{' x '}'
-  x <- '{' x+ '}' / [^}]*
+    e <- [a-zA-Z\128-\255][a-zA-Z0-9\128-\255_.]* / '{' x '}'
+    x <- '{' x+ '}' / [^}]*
 
 To produce a single substitution character it has to be doubled: `$$`
 produces `$`.
 
 The template engine is quite flexible. It is easy to produce a procedure that
-writes the template code directly to a file::
+writes the template code directly to a file:
 
-  #? stdtmpl(emit="f.write") | standard
-  #proc writeHTMLPage(f: File, title, currentTab, content: string,
-  #                   tabs: openArray[string]) =
-  $title
-  
-    
-    

-      $content
-      A dollar: $$.
-    

-  
+    #? stdtmpl(emit="f.write") | standard
+    #proc writeHTMLPage(f: File, title, currentTab, content: string,
+    #                   tabs: openArray[string]) =
+    $title
+    
+      
+      

+        $content
+        A dollar: $$.
+      

+    
diff --git a/doc/grammar.txt b/doc/grammar.txt
index 0d5eef179e..51b3e0053c 100644
--- a/doc/grammar.txt
+++ b/doc/grammar.txt
@@ -1,5 +1,5 @@
 # This file is generated by compiler/parser.nim.
-module = stmt ^* (';' / IND{=})
+module = complexOrSimpleStmt ^* (';' / IND{=})
 comma = ',' COMMENT?
 semicolon = ';' COMMENT?
 colon = ':' COMMENT?
@@ -7,7 +7,7 @@ colcom = ':' COMMENT?
 operator =  OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9
          | 'or' | 'xor' | 'and'
          | 'is' | 'isnot' | 'in' | 'notin' | 'of' | 'as' | 'from'
-         | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'static' | '..'
+         | 'div' | 'mod' | 'shl' | 'shr' | 'not' | '..'
 prefixOperator = operator
 optInd = COMMENT? IND?
 optPar = (IND{>} | IND{=})?
@@ -22,64 +22,70 @@ ampExpr = plusExpr (OP7 optInd plusExpr)*
 plusExpr = mulExpr (OP8 optInd mulExpr)*
 mulExpr = dollarExpr (OP9 optInd dollarExpr)*
 dollarExpr = primary (OP10 optInd primary)*
+operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 |
+            'div' | 'mod' | 'shl' | 'shr' | 'in' | 'notin' |
+            'is' | 'isnot' | 'not' | 'of' | 'as' | 'from' | '..' | 'and' | 'or' | 'xor'
 symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`'
-       | IDENT | KEYW
-exprColonEqExpr = expr (':'|'=' expr)?
+       | IDENT | 'addr' | 'type' | 'static'
+symbolOrKeyword = symbol | KEYW
+exprColonEqExpr = expr ((':'|'=') expr
+                       / doBlock extraPostExprBlock*)?
+exprEqExpr = expr ('=' expr
+                  / doBlock extraPostExprBlock*)?
 exprList = expr ^+ comma
+optionalExprList = expr ^* comma
 exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)?
-dotExpr = expr '.' optInd (symbol | '[:' exprList ']')
-explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )?
-qualifiedIdent = symbol ('.' optInd symbol)?
+qualifiedIdent = symbol ('.' optInd symbolOrKeyword)?
 setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}'
 castExpr = 'cast' ('[' optInd typeDesc optPar ']' '(' optInd expr optPar ')') /
 parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try'
         | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let'
         | 'when' | 'var' | 'mixin'
 par = '(' optInd
-          ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';'
-          | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';'
+          ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';'
+          | ';' (ifExpr / complexOrSimpleStmt) ^+ ';'
           | pragmaStmt
-          | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? )
+          | simpleExpr ( (doBlock extraPostExprBlock*)
+                       | ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? )
                        | (':' expr (',' exprColonEqExpr     ^+ ',' )? ) ) )
           optPar ')'
 literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT
           | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT
           | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT
           | STR_LIT | RSTR_LIT | TRIPLESTR_LIT
-          | CHAR_LIT
+          | CHAR_LIT | CUSTOM_NUMERIC_LIT
           | NIL
 generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT
 identOrLiteral = generalizedLit | symbol | literal
-               | par | arrayConstr | setOrTableConstr
+               | par | arrayConstr | setOrTableConstr | tupleConstr
                | castExpr
 tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')'
 arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']'
 primarySuffix = '(' (exprColonEqExpr comma?)* ')'
-      | '.' optInd symbol generalizedLit?
+      | '.' optInd symbolOrKeyword ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit?
+      | DOTLIKEOP optInd symbolOrKeyword generalizedLit?
       | '[' optInd exprColonEqExprList optPar ']'
       | '{' optInd exprColonEqExprList optPar '}'
-      | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax
 pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}')
 identVis = symbol OPR?  # postfix position
-identVisDot = symbol '.' optInd symbol OPR?
+identVisDot = symbol '.' optInd symbolOrKeyword OPR?
 identWithPragma = identVis pragma?
 identWithPragmaDot = identVisDot pragma?
 declColonEquals = identWithPragma (comma identWithPragma)* comma?
-                  (':' optInd typeDesc)? ('=' optInd expr)?
+                  (':' optInd typeDescExpr)? ('=' optInd expr)?
 identColonEquals = IDENT (comma IDENT)* comma?
-     (':' optInd typeDesc)? ('=' optInd expr)?)
-inlTupleDecl = 'tuple'
-    '[' optInd  (identColonEquals (comma/semicolon)?)*  optPar ']'
-extTupleDecl = 'tuple'
-    COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?
-tupleClass = 'tuple'
+     (':' optInd typeDescExpr)? ('=' optInd expr)?)
+tupleTypeBracket = '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']'
+tupleType = 'tuple' tupleTypeBracket
+tupleDecl = 'tuple' (tupleTypeBracket /
+    COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?)
 paramList = '(' declColonEquals ^* (comma/semicolon) ')'
 paramListArrow = paramList? ('->' optInd typeDesc)?
 paramListColon = paramList? (':' optInd typeDesc)?
 doBlock = 'do' paramListArrow pragma? colcom stmt
-procExpr = 'proc' paramListColon pragma? ('=' COMMENT? stmt)?
-distinct = 'distinct' optInd typeDesc
-forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt
+routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)?
+routineType = ('proc' | 'iterator') paramListColon pragma?
+forStmt = 'for' ((varTuple / identWithPragma) ^+ comma) 'in' expr colcom stmt
 forExpr = forStmt
 expr = (blockExpr
       | ifExpr
@@ -88,24 +94,33 @@ expr = (blockExpr
       | forExpr
       | tryExpr)
       / simpleExpr
-typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple'
-         | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
-primary = typeKeyw optInd typeDesc
-        /  prefixOperator* identOrLiteral primarySuffix*
-        / 'bind' primary
-typeDesc = simpleExpr ('not' expr)?
-typeDefAux = simpleExpr ('not' expr)?
-           | 'concept' typeClass
-postExprBlocks = ':' stmt? ( IND{=} doBlock
-                           | IND{=} 'of' exprList ':' stmt
-                           | IND{=} 'elif' expr ':' stmt
-                           | IND{=} 'except' exprList ':' stmt
-                           | IND{=} 'else' ':' stmt )*
-exprStmt = simpleExpr
-         (( '=' optInd expr colonBody? )
-         / ( expr ^+ comma
-             postExprBlocks
-           ))?
+simplePrimary = SIGILLIKEOP? identOrLiteral primarySuffix*
+commandStart = &('`'|IDENT|literal|'cast'|'addr'|'type'|'var'|'out'|
+                 'static'|'enum'|'tuple'|'object'|'proc')
+primary = simplePrimary (commandStart expr (doBlock extraPostExprBlock*)?)?
+        / operatorB primary
+        / routineExpr
+        / rawTypeDesc
+        / prefixOperator primary
+rawTypeDesc = (tupleType | routineType | 'enum' | 'object' |
+                ('var' | 'out' | 'ref' | 'ptr' | 'distinct') typeDesc?)
+                ('not' primary)?
+typeDescExpr = (routineType / simpleExpr) ('not' primary)?
+typeDesc = rawTypeDesc / typeDescExpr
+typeDefValue = ((tupleDecl | enumDecl | objectDecl | conceptDecl |
+                 ('ref' | 'ptr' | 'distinct') (tupleDecl | objectDecl))
+               / (simpleExpr (exprEqExpr ^+ comma postExprBlocks?)?))
+               ('not' primary)?
+extraPostExprBlock = ( IND{=} doBlock
+                     | IND{=} 'of' exprList ':' stmt
+                     | IND{=} 'elif' expr ':' stmt
+                     | IND{=} 'except' optionalExprList ':' stmt
+                     | IND{=} 'finally' ':' stmt
+                     | IND{=} 'else' ':' stmt )
+postExprBlocks = (doBlock / ':' (extraPostExprBlock / stmt)) extraPostExprBlock*
+exprStmt = simpleExpr postExprBlocks?
+         / simplePrimary (exprEqExpr ^+ comma) postExprBlocks?
+         / simpleExpr '=' optInd (expr postExprBlocks?)
 importStmt = 'import' optInd expr
               ((comma expr)*
               / 'except' optInd (expr ^+ comma))
@@ -125,9 +140,9 @@ condStmt = expr colcom stmt COMMENT?
            (IND{=} 'else' colcom stmt)?
 ifStmt = 'if' condStmt
 whenStmt = 'when' condStmt
-condExpr = expr colcom expr optInd
-        ('elif' expr colcom expr optInd)*
-         'else' colcom expr
+condExpr = expr colcom stmt optInd
+        ('elif' expr colcom stmt optInd)*
+         'else' colcom stmt
 ifExpr = 'if' condExpr
 whenExpr = 'when' condExpr
 whileStmt = 'while' expr colcom stmt
@@ -139,12 +154,11 @@ caseStmt = 'case' expr ':'? COMMENT?
             (IND{>} ofBranches DED
             | IND{=} ofBranches)
 tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally')
-           (IND{=}? 'except' exprList colcom stmt)*
+           (IND{=}? 'except' optionalExprList colcom stmt)*
            (IND{=}? 'finally' colcom stmt)?
 tryExpr = 'try' colcom stmt &(optInd 'except'|'finally')
-           (optInd 'except' exprList colcom stmt)*
+           (optInd 'except' optionalExprList colcom stmt)*
            (optInd 'finally' colcom stmt)?
-exceptBlock = 'except' colcom stmt
 blockStmt = 'block' symbol? colcom stmt
 blockExpr = 'block' symbol? colcom stmt
 staticStmt = 'static' colcom stmt
@@ -159,7 +173,7 @@ routine = optInd identVis pattern? genericParamList?
   paramListColon pragma? ('=' COMMENT? stmt)? indAndComment
 commentStmt = COMMENT
 section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED)
-enum = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+
+enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+
 objectWhen = 'when' expr colcom objectPart COMMENT?
             ('elif' expr colcom objectPart COMMENT?)*
             ('else' colcom objectPart COMMENT?)?
@@ -167,19 +181,19 @@ objectBranch = 'of' exprList colcom objectPart
 objectBranches = objectBranch (IND{=} objectBranch)*
                       (IND{=} 'elif' expr colcom objectPart)*
                       (IND{=} 'else' colcom objectPart)?
-objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT?
+objectCase = 'case' declColonEquals ':'? COMMENT?
             (IND{>} objectBranches DED
             | IND{=} objectBranches)
 objectPart = IND{>} objectPart^+IND{=} DED
            / objectWhen / objectCase / 'nil' / 'discard' / declColonEquals
-object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart
-typeClassParam = ('var' | 'out')? symbol
-typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
+objectDecl = 'object' ('of' typeDesc)? COMMENT? objectPart
+conceptParam = ('var' | 'out' | 'ptr' | 'ref' | 'static' | 'type')? symbol
+conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
               &IND{>} stmt
-typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux
-            indAndComment? / identVisDot genericParamList? pragma '=' optInd typeDefAux
+typeDef = identVisDot genericParamList? pragma '=' optInd typeDefValue
             indAndComment?
-varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr
+varTupleLhs = '(' optInd (identWithPragma / varTupleLhs) ^+ comma optPar ')' (':' optInd typeDescExpr)?
+varTuple = varTupleLhs '=' optInd expr
 colonBody = colcom stmt postExprBlocks?
 variable = (varTuple / identColonEquals) colonBody? indAndComment
 constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment
diff --git a/doc/hcr.rst b/doc/hcr.md
similarity index 92%
rename from doc/hcr.rst
rename to doc/hcr.md
index 7e9d71da39..285a862821 100644
--- a/doc/hcr.rst
+++ b/doc/hcr.md
@@ -1,9 +1,10 @@
-.. default-role:: code
-
 ===================================
       Hot code reloading
 ===================================
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 The `hotCodeReloading`:idx: option enables special compilation mode where
 changes in the code can be applied automatically to a running program.
 The code reloading happens at the granularity of an individual module.
@@ -22,11 +23,10 @@ during development resides.
 
 In this example, we use SDL2 to create a window and we reload the logic
 code when `F9` is pressed. The important lines are marked with `#***`.
-To install SDL2 you can use `nimble install sdl2`.
+To install SDL2 you can use `nimble install sdl2`:cmd:.
 
 
-.. code-block:: nim
-
+  ```nim
   # logic.nim
   import sdl2
 
@@ -78,14 +78,14 @@ To install SDL2 you can use `nimble install sdl2`.
     discard renderer.clear()
     discard renderer.setDrawColor(255, 128, 128, 0)
 
-    var rect = Rect(x: posX - 25, y: posY - 25, w: 50.cint, h: 50.cint)
+    var rect: Rect = (x: posX - 25, y: posY - 25, w: 50.cint, h: 50.cint)
     discard renderer.fillRect(rect)
     delay(16)
     renderer.present()
+  ```
 
 
-.. code-block:: nim
-
+  ```nim
   # mymain.nim
   import logic
 
@@ -96,36 +96,44 @@ To install SDL2 you can use `nimble install sdl2`.
     destroy()
 
   main()
+  ```
 
 
-Compile this example via::
+Compile this example via:
 
+  ```cmd
   nim c --hotcodereloading:on mymain.nim
+  ```
 
 Now start the program and KEEP it running!
 
-::
-
+  ```cmd
   # Unix:
   mymain &
   # or Windows (click on the .exe)
   mymain.exe
   # edit
+  ```
 
-For example, change the line::
+For example, change the line:
 
+  ```nim
   discard renderer.setDrawColor(255, 128, 128, 0)
+  ```
 
-into::
+into:
 
+  ```nim
   discard renderer.setDrawColor(255, 255, 128, 0)
+  ```
 
 (This will change the color of the rectangle.)
 
 Then recompile the project, but do not restart or quit the mymain.exe program!
-::
 
+  ```cmd
   nim c --hotcodereloading:on mymain.nim
+  ```
 
 Now give the `mymain` SDL window the focus, press F9, and watch the
 updated version of the program.
@@ -139,7 +147,7 @@ One can use the special event handlers `beforeCodeReload` and
 `afterCodeReload` to reset the state of a particular variable or to force
 the execution of certain statements:
 
-.. code-block:: Nim
+  ```Nim
   var
    settings = initTable[string, string]()
    lastReload: Time
@@ -152,6 +160,7 @@ the execution of certain statements:
   afterCodeReload:
     lastReload = now()
     resetProgramState()
+  ```
 
 On each code reload, Nim will first execute all `beforeCodeReload`:idx:
 handlers registered in the previous version of the program and then all
@@ -160,7 +169,7 @@ that any handlers appearing in modules that weren't reloaded will also be
 executed. To prevent this behavior, one can guard the code with the
 `hasModuleChanged()`:idx: API:
 
-.. code-block:: Nim
+  ```Nim
   import mydb
 
   var myCache = initTable[Key, Value]()
@@ -168,6 +177,7 @@ executed. To prevent this behavior, one can guard the code with the
   afterCodeReload:
     if hasModuleChanged(mydb):
       resetCache(myCache)
+  ```
 
 The hot code reloading is based on dynamic library hot swapping in the native
 targets and direct manipulation of the global namespace in the JavaScript
@@ -188,16 +198,15 @@ Native code targets
 ===================
 
 Native projects using the hot code reloading option will be implicitly
-compiled with the `-d:useNimRtl` option and they will depend on both
+compiled with the `-d:useNimRtl`:option: option and they will depend on both
 the `nimrtl` library and the `nimhcr` library which implements the
 hot code reloading run-time. Both libraries can be found in the `lib`
 folder of Nim and can be compiled into dynamic libraries to satisfy
-runtime demands of the example code above. An example of compiling 
-`nimhcr.nim` and `nimrtl.nim` when the source dir of Nim is installed
+runtime demands of the example code above. An example of compiling
+``nimhcr.nim`` and ``nimrtl.nim`` when the source dir of Nim is installed
 with choosenim follows.
 
-::
-
+  ```console
   # Unix/MacOS
   # Make sure you are in the directory containing your .nim files
   $ cd your-source-directory
@@ -208,6 +217,7 @@ with choosenim follows.
 
   # verify that you have two files named libnimhcr and libnimrtl in your
   # source directory (.dll for Windows, .so for Unix, .dylib for MacOS)
+  ```
 
 All modules of the project will be compiled to separate dynamic link
 libraries placed in the `nimcache` directory. Please note that during
diff --git a/doc/idetools.rst b/doc/idetools.md
similarity index 75%
rename from doc/idetools.rst
rename to doc/idetools.md
index dcafaf45f4..0388a76c01 100644
--- a/doc/idetools.rst
+++ b/doc/idetools.md
@@ -10,24 +10,21 @@
 .. contents::
 
 
-.. raw:: html
-  

-  "yes, I'm the creator" -- Araq, 2013-07-26 19:28:32.
-  

+> "yes, I'm the creator" -- Araq, 2013-07-26 19:28:32.
 
-Note: this is mostly outdated, see instead `nimsuggest `_
+Note: this is mostly outdated, see instead [nimsuggest](nimsuggest.html)
 
 Nim differs from many other compilers in that it is really fast,
 and being so fast makes it suited to provide external queries for
 text editors about the source code being written. Through the
-`idetools` command of `the compiler `_, any IDE
+`idetools` command of [the compiler](nimc.html), any IDE
 can query a `.nim` source file and obtain useful information like
 definition of symbols or suggestions for completion.
 
 This document will guide you through the available options. If you
 want to look at practical examples of idetools support you can look
-at the test files found in the `Test suite`_ or `various editor
-integrations `_
+at the test files found in the [Test suite] or [various editor
+integrations](https://github.com/Araq/Nim/wiki/Editor-Support)
 already available.
 
 
@@ -39,39 +36,43 @@ Specifying the location of the query
 
 All of the available idetools commands require you to specify a
 query location through the `--track` or `--trackDirty` switches.
-The general idetools invocations are::
+The general idetools invocations are:
 
-    nim idetools --track:FILE,LINE,COL  proj.nim
+  ```cmd
+  nim idetools --track:FILE,LINE,COL  proj.nim
+  ```
 
-Or::
+Or:
 
-    nim idetools --trackDirty:DIRTY_FILE,FILE,LINE,COL  proj.nim
+  ```cmd
+  nim idetools --trackDirty:DIRTY_FILE,FILE,LINE,COL  proj.nim
+  ```
 
 `proj.nim`
-    This is the main *project* filename. Most of the time you will
+:   This is the main *project* filename. Most of the time you will
     pass in the same as **FILE**, but for bigger projects this is
     the file which is used as main entry point for the program, the
     one which users compile to generate a final binary.
 
 ``
-    This would be any of the other idetools available options, like
+:   This would be any of the other idetools available options, like
     `--def` or `--suggest` explained in the following sections.
 
 `COL`
-    An integer with the column you are going to query. For the
+:   An integer with the column you are going to query. For the
     compiler columns start at zero, so the first column will be
     **0** and the last in an 80 column terminal will be **79**.
 
 `LINE`
-    An integer with the line you are going to query. For the compiler
+:   An integer with the line you are going to query. For the compiler
     lines start at **1**.
 
 `FILE`
-    The file you want to perform the query on. Usually you will
+:   The file you want to perform the query on. Usually you will
     pass in the same value as **proj.nim**.
 
 `DIRTY_FILE`
-    The **FILE** parameter is enough for static analysis, but IDEs
+:   The **FILE** parameter is enough for static analysis, but IDEs
     tend to have *unsaved buffers* where the user may still be in
     the middle of typing a line. In such situations the IDE can
     save the current contents to a temporary file and then use the
@@ -127,8 +128,8 @@ to the case insensitiveness of the language (plus underscores as
 separators!).
 
 The typical usage scenario for this option is to call it after the
-user has typed the dot character for `the object oriented call
-syntax `_.
+user has typed the dot character for [the object oriented call
+syntax](tut2.html#object-oriented-programming-method-call-syntax).
 Idetools will try to return the suggestions sorted first by scope
 (from innermost to outermost) and then by item name.
 
@@ -181,14 +182,18 @@ results of the compilation, and subsequent queries should be fast
 in the millisecond range, thus being responsive enough for IDEs.
 
 If you want to start the server using stdin/stdout as communication
-you need to type::
+you need to type:
 
-    nim serve --server.type:stdin proj.nim
+  ```cmd
+  nim serve --server.type:stdin proj.nim
+  ```
 
-If you want to start the server using tcp and a port, you need to type::
+If you want to start the server using tcp and a port, you need to type:
 
-    nim serve --server.type:tcp --server.port:6000 \
+  ```cmd
+  nim serve --server.type:tcp --server.port:6000 \
       --server.address:hostname proj.nim
+  ```
 
 In both cases the server will start up and await further commands.
 The syntax of the commands you can now send to the server is
@@ -199,7 +204,7 @@ of text it thinks necessary plus an empty line to indicate the end
 of the answer.
 
 You can find examples of client/server communication in the idetools
-tests found in the `Test suite`_.
+tests found in the [Test suite].
 
 
 Parsing idetools output
@@ -242,45 +247,48 @@ symbol for which idetools returns valid output.
 skConst
 -------
 
-| **Third column**: module + [n scope nesting] + const name.
+| **Third column**: module + \[n scope nesting] + const name.
 | **Fourth column**: the type of the const value.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    const SOME_SEQUENCE = @[1, 2]
-    --> col 2: $MODULE.SOME_SEQUENCE
-        col 3: seq[int]
-        col 7: ""
+  ```nim
+  const SOME_SEQUENCE = @[1, 2]
+  --> col 2: $MODULE.SOME_SEQUENCE
+      col 3: seq[int]
+      col 7: ""
+  ```
 
 
 skEnumField
 -----------
 
-| **Third column**: module + [n scope nesting] + enum type + enum field name.
+| **Third column**: module + \[n scope nesting] + enum type + enum field name.
 | **Fourth column**: enum type grouping other enum fields.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    Open(filename, fmWrite)
-    --> col 2: system.FileMode.fmWrite
-        col 3: FileMode
-        col 7: ""
+  ```nim
+  Open(filename, fmWrite)
+  --> col 2: system.FileMode.fmWrite
+      col 3: FileMode
+      col 7: ""
+  ```
 
 
 skForVar
 --------
 
-| **Third column**: module + [n scope nesting] + var name.
+| **Third column**: module + \[n scope nesting] + var name.
 | **Fourth column**: type of the var.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    proc looper(filename = "tests.nim") =
-      for letter in filename:
-        echo letter
-    --> col 2: $MODULE.looper.letter
-        col 3: char
-        col 7: ""
+  ```nim
+  proc looper(filename = "tests.nim") =
+    for letter in filename:
+      echo letter
+  --> col 2: $MODULE.looper.letter
+      col 3: char
+      col 7: ""
+  ```
 
 
 skIterator, skClosureIterator
@@ -291,48 +299,51 @@ defined, since at that point in the file the parser hasn't processed
 the full line yet. The signature will be returned complete in
 posterior instances of the iterator.
 
-| **Third column**: module + [n scope nesting] + iterator name.
+| **Third column**: module + \[n scope nesting] + iterator name.
 | **Fourth column**: signature of the iterator including return type.
 | **Docstring**: docstring if available.
 
-.. code-block:: nim
-    let
-      text = "some text"
-      letters = toSeq(runes(text))
-    --> col 2: unicode.runes
-        col 3: iterator (string): Rune
-        col 7: "iterates over any unicode character of the string `s`."
+  ```nim
+  let
+    text = "some text"
+    letters = toSeq(runes(text))
+  --> col 2: unicode.runes
+      col 3: iterator (string): Rune
+      col 7: "iterates over any unicode character of the string `s`."
+  ```
 
 
 skLabel
 -------
 
-| **Third column**: module + [n scope nesting] + name.
+| **Third column**: module + \[n scope nesting] + name.
 | **Fourth column**: always the empty string.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    proc test(text: string) =
-      var found = -1
-      block loops:
-    --> col 2: $MODULE.test.loops
-        col 3: ""
-        col 7: ""
+  ```nim
+  proc test(text: string) =
+    var found = -1
+    block loops:
+  --> col 2: $MODULE.test.loops
+      col 3: ""
+      col 7: ""
+  ```
 
 
 skLet
 -----
 
-| **Third column**: module + [n scope nesting] + let name.
+| **Third column**: module + \[n scope nesting] + let name.
 | **Fourth column**: the type of the let variable.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    let
-      text = "some text"
-    --> col 2: $MODULE.text
-        col 3: string
-        col 7: ""
+  ```nim
+  let
+    text = "some text"
+  --> col 2: $MODULE.text
+      col 3: string
+      col 7: ""
+  ```
 
 
 skMacro
@@ -343,16 +354,17 @@ defined, since at that point in the file the parser hasn't processed
 the full line yet. The signature will be returned complete in
 posterior instances of the macro.
 
-| **Third column**: module + [n scope nesting] + macro name.
+| **Third column**: module + \[n scope nesting] + macro name.
 | **Fourth column**: signature of the macro including return type.
 | **Docstring**: docstring if available.
 
-.. code-block:: nim
-    proc testMacro() =
-      expect(EArithmetic):
-    --> col 2: idetools_api.expect
-        col 3: proc (varargs[expr], stmt): stmt
-        col 7: ""
+  ```nim
+  proc testMacro() =
+    expect(EArithmetic):
+  --> col 2: idetools_api.expect
+      col 3: proc (varargs[expr], stmt): stmt
+      col 7: ""
+  ```
 
 
 skMethod
@@ -363,8 +375,8 @@ defined, since at that point in the file the parser hasn't processed
 the full line yet. The signature will be returned complete in
 posterior instances of the method.
 
-Methods imply `dynamic dispatch
-`_ and
+Methods imply [dynamic dispatch](
+tut2.html#object-oriented-programming-dynamic-dispatch) and
 idetools performs a static analysis on the code. For this reason
 idetools may not return the definition of the correct method you
 are querying because it may be impossible to know until the code
@@ -380,33 +392,35 @@ Note that at the moment the word `proc` is returned for the
 signature of the found method instead of the expected `method`.
 This may change in the future.
 
-| **Third column**: module + [n scope nesting] + method name.
+| **Third column**: module + \[n scope nesting] + method name.
 | **Fourth column**: signature of the method including return type.
 | **Docstring**: docstring if available.
 
-.. code-block:: nim
-    method eval(e: PExpr): int = quit "to override!"
-    method eval(e: PLiteral): int = e.x
-    method eval(e: PPlusExpr): int = eval(e.a) + eval(e.b)
-    echo eval(newPlus(newPlus(newLit(1), newLit(2)), newLit(4)))
-    --> col 2: $MODULE.eval
-        col 3: proc (PPlusExpr): int
-        col 7: ""
+  ```nim
+  method eval(e: PExpr): int = quit "to override!"
+  method eval(e: PLiteral): int = e.x
+  method eval(e: PPlusExpr): int = eval(e.a) + eval(e.b)
+  echo eval(newPlus(newPlus(newLit(1), newLit(2)), newLit(4)))
+  --> col 2: $MODULE.eval
+      col 3: proc (PPlusExpr): int
+      col 7: ""
+  ```
 
 
 skParam
 -------
 
-| **Third column**: module + [n scope nesting] + param name.
+| **Third column**: module + \[n scope nesting] + param name.
 | **Fourth column**: the type of the parameter.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    proc reader(filename = "tests.nim") =
-      let text = readFile(filename)
-    --> col 2: $MODULE.reader.filename
-        col 3: string
-        col 7: ""
+  ```nim
+  proc reader(filename = "tests.nim") =
+    let text = readFile(filename)
+  --> col 2: $MODULE.reader.filename
+      col 3: string
+      col 7: ""
+  ```
 
 
 skProc
@@ -421,34 +435,36 @@ While at the language level a proc is differentiated from others
 by the parameters and return value, the signature of the proc
 returned by idetools returns also the pragmas for the proc.
 
-| **Third column**: module + [n scope nesting] + proc name.
+| **Third column**: module + \[n scope nesting] + proc name.
 | **Fourth column**: signature of the proc including return type.
 | **Docstring**: docstring if available.
 
-.. code-block:: nim
-    open(filename, fmWrite)
-    --> col 2: system.Open
-        col 3: proc (var File, string, FileMode, int): bool
-        col 7:
-    "Opens a file named `filename` with given `mode`.
+  ```nim
+  open(filename, fmWrite)
+  --> col 2: system.Open
+      col 3: proc (var File, string, FileMode, int): bool
+      col 7:
+  "Opens a file named `filename` with given `mode`.
 
-     Default mode is readonly. Returns true iff the file could be opened.
-     This throws no exception if the file could not be opened."
+   Default mode is readonly. Returns true iff the file could be opened.
+   This throws no exception if the file could not be opened."
+  ```
 
 
 skResult
 --------
 
-| **Third column**: module + [n scope nesting] + result.
+| **Third column**: module + \[n scope nesting] + result.
 | **Fourth column**: the type of the result.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    proc getRandomValue() : int =
-      return 4
-    --> col 2: $MODULE.getRandomValue.result
-        col 3: int
-        col 7: ""
+  ```nim
+  proc getRandomValue() : int =
+    return 4
+  --> col 2: $MODULE.getRandomValue.result
+      col 3: int
+      col 7: ""
+  ```
 
 
 skTemplate
@@ -459,11 +475,11 @@ defined, since at that point in the file the parser hasn't processed
 the full line yet. The signature will be returned complete in
 posterior instances of the template.
 
-| **Third column**: module + [n scope nesting] + template name.
+| **Third column**: module + \[n scope nesting] + template name.
 | **Fourth column**: signature of the template including return type.
 | **Docstring**: docstring if available.
 
-.. code-block:: nim
+  `````nim
     let
       text = "some text"
       letters = toSeq(runes(text))
@@ -474,45 +490,49 @@ posterior instances of the template.
 
      Example:
 
-     .. code-block:: nim
+       ```nim
        let
          numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9]
          odd_numbers = toSeq(filter(numeric) do (x: int) -> bool:
            if x mod 2 == 1:
              result = true)
        assert odd_numbers == @[1, 3, 5, 7, 9]"
+       ```
+  `````
 
 
 skType
 ------
 
-| **Third column**: module + [n scope nesting] + type name.
+| **Third column**: module + \[n scope nesting] + type name.
 | **Fourth column**: the type.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    proc writeTempFile() =
-      var output: File
-    --> col 2: system.File
-        col 3: File
-        col 7: ""
+  ```nim
+  proc writeTempFile() =
+    var output: File
+  --> col 2: system.File
+      col 3: File
+      col 7: ""
+  ```
 
 
 skVar
 -----
 
-| **Third column**: module + [n scope nesting] + var name.
+| **Third column**: module + \[n scope nesting] + var name.
 | **Fourth column**: the type of the var.
 | **Docstring**: always the empty string.
 
-.. code-block:: nim
-    proc writeTempFile() =
-      var output: File
-      output.open("/tmp/somefile", fmWrite)
-      output.write("test")
-    --> col 2: $MODULE.writeTempFile.output
-        col 3: File
-        col 7: ""
+  ```nim
+  proc writeTempFile() =
+    var output: File
+    output.open("/tmp/somefile", fmWrite)
+    output.write("test")
+  --> col 2: $MODULE.writeTempFile.output
+      col 3: File
+      col 7: ""
+  ```
 
 
 Test suite
@@ -530,10 +550,12 @@ Running the test suite
 
 At the moment idetools support is still in development so the test
 suite is not integrated with the main test suite and you have to
-run it manually. First you have to compile the tester::
+run it manually. First you have to compile the tester:
 
-	$ cd my/nim/checkout/tests
-	$ nim c testament/caasdriver.nim
+  ```cmd
+  $ cd my/nim/checkout/tests
+  $ nim c testament/caasdriver.nim
+  ```
 
 Running the `caasdriver` without parameters will attempt to process
 all the test cases in all three operation modes. If a test succeeds
@@ -555,9 +577,11 @@ If you don't want to run all the test case files you can pass any
 substring as a parameter to `caasdriver`. Only files matching the
 passed substring will be run. The filtering doesn't use any globbing
 metacharacters, it's a plain match. For example, to run only
-`*-compile*.txt` tests in verbose mode::
+`*-compile*.txt` tests in verbose mode:
 
-	./caasdriver verbose -compile
+  ```cmd
+  ./caasdriver verbose -compile
+  ```
 
 
 Test case file format
@@ -578,7 +602,7 @@ All the `tests/caas/*.txt` files encode a session with the compiler:
   modes, you can prefix a line with the mode and the line will be
   processed only in that mode.
 
-* The rest of the line is treated as a `regular expression `_,
+* The rest of the line is treated as a [regular expression](re.html),
   so be careful escaping metacharacters like parenthesis.
 
 Before the line is processed as a regular expression, some basic
diff --git a/doc/intern.md b/doc/intern.md
new file mode 100644
index 0000000000..6b16bc71fa
--- /dev/null
+++ b/doc/intern.md
@@ -0,0 +1,679 @@
+=========================================
+    Internals of the Nim Compiler
+=========================================
+
+
+:Author: Andreas Rumpf
+:Version: |nimversion|
+
+.. default-role:: code
+.. include:: rstcommon.rst
+.. contents::
+
+> "Abstraction is layering ignorance on top of reality." -- Richard Gabriel
+
+
+Directory structure
+===================
+
+The Nim project's directory structure is:
+
+============   ===================================================
+Path           Purpose
+============   ===================================================
+`bin`          generated binary files
+`build`        generated C code for the installation
+`compiler`     the Nim compiler itself; note that this
+               code has been translated from a bootstrapping
+               version written in Pascal, so the code is **not**
+               a poster child of good Nim code
+`config`       configuration files for Nim
+`dist`         additional packages for the distribution
+`doc`          the documentation; it is a bunch of
+               reStructuredText files
+`lib`          the Nim library
+============   ===================================================
+
+
+Bootstrapping the compiler
+==========================
+
+**Note**: Add ``.`` to your PATH so that `koch`:cmd: can be used without the ``./``.
+
+Compiling the compiler is a simple matter of running:
+
+  ```cmd
+  nim c koch.nim
+  koch boot -d:release
+  ```
+
+For a debug version use:
+
+  ```cmd
+  nim c koch.nim
+  koch boot
+  ```
+
+
+And for a debug version compatible with GDB:
+
+  ```cmd
+  nim c koch.nim
+  koch boot --debuginfo --linedir:on
+  ```
+
+The `koch`:cmd: program is Nim's maintenance script. It is a replacement for
+make and shell scripting with the advantage that it is much more portable.
+More information about its options can be found in the [koch](koch.html)
+documentation.
+
+
+Reproducible builds
+-------------------
+
+Set the compilation timestamp with the `SOURCE_DATE_EPOCH` environment variable.
+
+  ```cmd
+  export SOURCE_DATE_EPOCH=$(git log -n 1 --format=%at)
+  koch boot # or `./build_all.sh`
+  ```
+
+
+Debugging the compiler
+======================
+
+
+Bisecting for regressions
+-------------------------
+
+There are often times when there is a bug that is caused by a regression in the
+compiler or stdlib. Bisecting the Nim repo commits is a useful tool to identify
+what commit introduced the regression.
+
+Even if it's not known whether a bug is caused by a regression, bisection can reduce
+debugging time by ruling it out. If the bug is found to be a regression, then you
+focus on the changes introduced by that one specific commit.
+
+`koch temp`:cmd: returns 125 as the exit code in case the compiler
+compilation fails. This exit code tells `git bisect`:cmd: to skip the
+current commit:
+
+  ```cmd
+  git bisect start bad-commit good-commit
+  git bisect run ./koch temp -r c test-source.nim
+  ```
+
+You can also bisect using custom options to build the compiler, for example if
+you don't need a debug version of the compiler (which runs slower), you can replace
+`./koch temp`:cmd: by explicit compilation command, see [Bootstrapping the compiler].
+
+See also:
+
+- Crossplatform C/Cpp/Valgrind/JS Bisect in GitHub: https://github.com/juancarlospaco/nimrun-action#examples
+
+
+Building an instrumented compiler
+---------------------------------
+
+Considering that a useful method of debugging the compiler is inserting debug
+logging, or changing code and then observing the outcome of a testcase, it is
+fastest to build a compiler that is instrumented for debugging from an
+existing release build. `koch temp`:cmd: provides a convenient method of doing
+just that.
+
+By default, running `koch temp`:cmd: will build a lean version of the compiler
+with `-d:debug`:option: enabled. The compiler is written to `bin/nim_temp` by
+default. A lean version of the compiler lacks JS and documentation generation.
+
+`bin/nim_temp` can be directly used to run testcases, or used with testament
+with `testament --nim:bin/nim_temp r tests/category/tsometest`:cmd:.
+
+`koch temp`:cmd: will build the temporary compiler with the `-d:debug`:option:
+enabled. Here are compiler options that are of interest when debugging:
+
+* `-d:debug`:option:\: enables `assert` statements and stacktraces and all
+  runtime checks
+* `--opt:speed`:option:\: build with optimizations enabled
+* `--debugger:native`:option:\: enables `--debuginfo --lineDir:on`:option: for using
+  a native debugger like GDB, LLDB or CDB
+* `-d:nimDebug`:option: cause calls to `quit` to raise an assertion exception
+* `-d:nimDebugUtils`:option:\: enables various debugging utilities;
+  see `compiler/debugutils`
+* `-d:stacktraceMsgs -d:nimCompilerStacktraceHints`:option:\: adds some additional
+  stacktrace hints; see https://github.com/nim-lang/Nim/pull/13351
+* `-u:leanCompiler`:option:\: enable JS and doc generation
+
+Another method to build and run the compiler is directly through `koch`:cmd:\:
+
+  ```cmd
+  koch temp [options] c test.nim
+
+  # (will build with js support)
+  koch temp [options] js test.nim
+
+  # (will build with doc support)
+  koch temp [options] doc test.nim
+  ```
+
+Debug logging
+-------------
+
+"Printf debugging" is still the most appropriate way to debug many problems
+arising in compiler development. The typical usage of breakpoints to debug
+the code is often less practical, because almost all code paths in the
+compiler will be executed hundreds of times before a particular section of the
+tested program is reached where the newly developed code must be activated.
+
+To work around this problem, you'll typically introduce an if statement in the
+compiler code detecting more precisely the conditions where the tested feature
+is being used. One very common way to achieve this is to use the `mdbg` condition,
+which will be true only in contexts, processing expressions and statements from
+the currently compiled main module:
+
+  ```nim
+  # inside some compiler module
+  if mdbg:
+    debug someAstNode
+  ```
+
+Using the `isCompilerDebug`:nim: condition along with inserting some statements
+into the testcase provides more granular logging:
+
+  ```nim
+  # compilermodule.nim
+  if isCompilerDebug():
+    debug someAstNode
+
+  # testcase.nim
+  proc main =
+    {.define(nimCompilerDebug).}
+    let a = 2.5 * 3
+    {.undef(nimCompilerDebug).}
+  ```
+
+Logging can also be scoped to a specific filename as well. This will of course
+match against every module with that name.
+
+  ```nim
+  if `??`(conf, n.info, "module.nim"):
+    debug(n)
+  ```
+
+The above examples also makes use of the `debug`:nim: proc, which is able to
+print a human-readable form of an arbitrary AST tree. Other common ways to print
+information about the internal compiler types include:
+
+  ```nim
+  # pretty print PNode
+
+  # pretty prints the Nim ast
+  echo renderTree(someNode)
+
+  # pretty prints the Nim ast, but annotates symbol IDs
+  echo renderTree(someNode, {renderIds})
+
+  # pretty print ast as JSON
+  debug(someNode)
+
+  # print as YAML
+  echo treeToYaml(config, someNode)
+
+
+  # pretty print PType
+
+  # print type name
+  echo typeToString(someType)
+
+  # pretty print as JSON
+  debug(someType)
+
+  # print as YAML
+  echo typeToYaml(config, someType)
+
+
+  # pretty print PSym
+
+  # print the symbol's name
+  echo symbol.name.s
+
+  # pretty print as JSON
+  debug(symbol)
+
+  # print as YAML
+  echo symToYaml(config, symbol)
+
+
+  # pretty print TLineInfo
+  lineInfoToStr(lineInfo)
+
+
+  # print the structure of any type
+  repr(someVar)
+  ```
+
+Here are some other helpful utilities:
+
+  ```nim
+  # how did execution reach this location?
+  writeStackTrace()
+  ```
+
+These procs may not already be imported by the module you're editing.
+You can import them directly for debugging:
+
+  ```nim
+  from astalgo import debug
+  from types import typeToString
+  from renderer import renderTree
+  from msgs import `??`
+  ```
+
+Native debugging
+----------------
+
+Stepping through the compiler with a native debugger is a very powerful tool to
+both learn and debug it. However, there is still the need to constrain when
+breakpoints are triggered. The same methods as in [Debug logging] can be applied
+here when combined with calls to the debug helpers `enteringDebugSection()`:nim:
+and `exitingDebugSection()`:nim:.
+
+#. Compile the temp compiler with `--debugger:native -d:nimDebugUtils`:option:
+#. Set your desired breakpoints or watchpoints.
+#. Configure your debugger:
+   * GDB: execute `source tools/compiler.gdb` at startup
+   * LLDB execute `command source tools/compiler.lldb` at startup
+#. Use one of the scoping helpers like so:
+
+  ```nim
+  if isCompilerDebug():
+    enteringDebugSection()
+  else:
+    exitingDebugSection()
+  ```
+
+A caveat of this method is that all breakpoints and watchpoints are enabled or
+disabled. Also, due to a bug, only breakpoints can be constrained for LLDB.
+
+The compiler's architecture
+===========================
+
+Nim uses the classic compiler architecture: A lexer/scanner feeds tokens to a
+parser. The parser builds a syntax tree that is used by the code generators.
+This syntax tree is the interface between the parser and the code generator.
+It is essential to understand most of the compiler's code.
+
+Semantic analysis is separated from parsing.
+
+.. include:: filelist.txt
+
+
+The syntax tree
+---------------
+The syntax tree consists of nodes which may have an arbitrary number of
+children. Types and symbols are represented by other nodes, because they
+may contain cycles. The AST changes its shape after semantic checking. This
+is needed to make life easier for the code generators. See the "ast" module
+for the type definitions. The [macros](macros.html) module contains many
+examples how the AST represents each syntactic structure.
+
+
+Runtimes
+========
+
+Nim has two different runtimes, the "old runtime" and the "new runtime". The old
+runtime supports the old GCs (markAndSweep, refc, Boehm), the new runtime supports
+ARC/ORC. The new runtime is active `when defined(nimV2)`.
+
+
+Coding Guidelines
+=================
+
+* We follow Nim's official style guide, see [NEP1](nep1.html).
+* Max line length is 100 characters.
+* Provide spaces around binary operators if that enhances readability.
+* Use a space after a colon, but not before it.
+* (deprecated) Start types with a capital `T`, unless they are
+  pointers/references which start with `P`.
+* Prefer `import package`:nim: over `from package import symbol`:nim:.
+
+See also the [API naming design](apis.html) document.
+
+
+Porting to new platforms
+========================
+
+Porting Nim to a new architecture is pretty easy, since C is the most
+portable programming language (within certain limits) and Nim generates
+C code, porting the code generator is not necessary.
+
+POSIX-compliant systems on conventional hardware are usually pretty easy to
+port: Add the platform to `platform` (if it is not already listed there),
+check that the OS, System modules work and recompile Nim.
+
+The only case where things aren't as easy is when old runtime's garbage
+collectors need some assembler tweaking to work. The default
+implementation uses C's `setjmp`:c: function to store all registers
+on the hardware stack. It may be necessary that the new platform needs to
+replace this generic code by some assembler code.
+
+Files that may need changed for your platform include:
+
+* `compiler/platform.nim`
+  Add os/cpu properties.
+* `lib/system.nim`
+  Add os/cpu to the documentation for `system.hostOS` and `system.hostCPU`.
+* `compiler/options.nim`
+  Add special os/cpu property checks in `isDefined`.
+* `compiler/installer.ini`
+  Add os/cpu to `Project.Platforms` field.
+* `lib/system/platforms.nim`
+  Add os/cpu.
+* `std/private/osseps.nim`
+  Add os specializations.
+* `lib/pure/distros.nim`
+  Add os, package handler.
+* `tools/niminst/makefile.nimf`
+  Add os/cpu compiler/linker flags.
+* `tools/niminst/buildsh.nimf`
+  Add os/cpu compiler/linker flags.
+
+If the `--os` or `--cpu` options aren't passed to the compiler, then Nim will
+determine the current host os, cpu and endianness from `system.cpuEndian`,
+`system.hostOS` and `system.hostCPU`. Those values are derived from
+`compiler/platform.nim`.
+
+In order for the new platform to be bootstrapped from the `csources`, it must:
+
+* have `compiler/platform.nim` updated
+* have `compiler/installer.ini` updated
+* have `tools/niminst/buildsh.nimf` updated
+* have `tools/niminst/makefile.nimf` updated
+* be backported to the Nim version used by the `csources`
+* the new `csources` must be pushed
+* the new `csources` revision must be updated in `config/build_config.txt`
+
+
+Runtime type information
+========================
+
+**Note**: This section describes the "old runtime".
+
+*Runtime type information* (RTTI) is needed for several aspects of the Nim
+programming language:
+
+Garbage collection
+: The old GCs use the RTTI for traversing arbitrary Nim types, but usually
+  only the `marker` field which contains a proc that does the traversal.
+
+Complex assignments
+: Sequences and strings are implemented as
+  pointers to resizable buffers, but Nim requires copying for
+  assignments. Apart from RTTI the compiler also generates copy procedures
+  as a specialization.
+
+We already know the type information as a graph in the compiler.
+Thus, we need to serialize this graph as RTTI for C code generation.
+Look at the file ``lib/system/hti.nim`` for more information.
+
+
+Magics and compilerProcs
+========================
+
+The `system` module contains the part of the RTL which needs support by
+compiler magic. The C code generator generates the C code for it, just like any other
+module. However, calls to some procedures like `addInt` are inserted by
+the generator. Therefore, there is a table (`compilerprocs`)
+with all symbols that are marked as `compilerproc`. `compilerprocs` are
+needed by the code generator. A `magic` proc is not the same as a
+`compilerproc`: A `magic` is a proc that needs compiler magic for its
+semantic checking, a `compilerproc` is a proc that is used by the code
+generator.
+
+
+Code generation for closures
+============================
+
+Code generation for closures is implemented by `lambda lifting`:idx:.
+
+
+Design
+------
+
+A `closure` proc var can call ordinary procs of the default Nim calling
+convention. But not the other way round! A closure is implemented as a
+`tuple[prc, env]`. `env` can be nil implying a call without a closure.
+This means that a call through a closure generates an `if` but the
+interoperability is worth the cost of the `if`. Thunk generation would be
+possible too, but it's slightly more effort to implement.
+
+Tests with GCC on Amd64 showed that it's really beneficial if the
+'environment' pointer is passed as the last argument, not as the first argument.
+
+Proper thunk generation is harder because the proc that is to wrap
+could stem from a complex expression:
+
+  ```nim
+  receivesClosure(returnsDefaultCC[i])
+  ```
+
+A thunk would need to call `returnsDefaultCC[i]` somehow and that would require
+an *additional* closure generation... Ok, not really, but it requires to pass
+the function to call. So we'd end up with 2 indirect calls instead of one.
+Another much more severe problem with this solution is that it's not GC-safe
+to pass a proc pointer around via a generic `ref` type.
+
+
+Example code:
+
+  ```nim
+  proc add(x: int): proc (y: int): int {.closure.} =
+    return proc (y: int): int =
+      return x + y
+
+  var add2 = add(2)
+  echo add2(5) #OUT 7
+  ```
+
+This should produce roughly this code:
+
+  ```nim
+  type
+    Env = ref object
+      x: int # data
+
+  proc anon(y: int, c: Env): int =
+    return y + c.x
+
+  proc add(x: int): tuple[prc, data] =
+    var env: Env
+    new env
+    env.x = x
+    result = (anon, env)
+
+  var add2 = add(2)
+  let tmp = if add2.data == nil: add2.prc(5) else: add2.prc(5, add2.data)
+  echo tmp
+  ```
+
+
+Beware of nesting:
+
+  ```nim
+  proc add(x: int): proc (y: int): proc (z: int): int {.closure.} {.closure.} =
+    return lambda (y: int): proc (z: int): int {.closure.} =
+      return lambda (z: int): int =
+        return x + y + z
+
+  var add24 = add(2)(4)
+  echo add24(5) #OUT 11
+  ```
+
+This should produce roughly this code:
+
+  ```nim
+  type
+    EnvX = ref object
+      x: int # data
+
+    EnvY = ref object
+      y: int
+      ex: EnvX
+
+  proc lambdaZ(z: int, ey: EnvY): int =
+    return ey.ex.x + ey.y + z
+
+  proc lambdaY(y: int, ex: EnvX): tuple[prc, data: EnvY] =
+    var ey: EnvY
+    new ey
+    ey.y = y
+    ey.ex = ex
+    result = (lambdaZ, ey)
+
+  proc add(x: int): tuple[prc, data: EnvX] =
+    var ex: EnvX
+    ex.x = x
+    result = (lambdaY, ex)
+
+  var tmp = add(2)
+  var tmp2 = tmp.fn(4, tmp.data)
+  var add24 = tmp2.fn(4, tmp2.data)
+  echo add24(5)
+  ```
+
+
+We could get rid of nesting environments by always inlining inner anon procs.
+More useful is escape analysis and stack allocation of the environment,
+however.
+
+
+Accumulator
+-----------
+
+  ```nim
+  proc getAccumulator(start: int): proc (): int {.closure} =
+    var i = start
+    return lambda: int =
+      inc i
+      return i
+
+  proc p =
+    var delta = 7
+    proc accumulator(start: int): proc(): int =
+      var x = start-1
+      result = proc (): int =
+        x = x + delta
+        inc delta
+        return x
+
+    var a = accumulator(3)
+    var b = accumulator(4)
+    echo a() + b()
+  ```
+
+
+Internals
+---------
+
+Lambda lifting is implemented as part of the `transf` pass. The `transf`
+pass generates code to set up the environment and to pass it around. However,
+this pass does not change the types! So we have some kind of mismatch here; on
+the one hand the proc expression becomes an explicit tuple, on the other hand
+the tyProc(ccClosure) type is not changed. For C code generation it's also
+important the hidden formal param is `void*`:c: and not something more
+specialized. However, the more specialized env type needs to passed to the
+backend somehow. We deal with this by modifying `s.ast[paramPos]` to contain
+the formal hidden parameter, but not `s.typ`!
+
+
+Notes on type and AST representation
+====================================
+
+To be expanded.
+
+
+Integer literals
+----------------
+
+In Nim, there is a redundant way to specify the type of an
+integer literal. First, it should be unsurprising that every
+node has a node kind. The node of an integer literal can be any of the
+following values:
+
+    nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit,
+    nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit
+
+On top of that, there is also the `typ` field for the type. The
+kind of the `typ` field can be one of the following ones, and it
+should be matching the literal kind:
+
+    tyInt, tyInt8, tyInt16, tyInt32, tyInt64, tyUInt, tyUInt8,
+    tyUInt16, tyUInt32, tyUInt64
+
+Then there is also the integer literal type. This is a specific type
+that is implicitly convertible into the requested type if the
+requested type can hold the value. For this to work, the type needs to
+know the concrete value of the literal. For example an expression
+`321` will be of type `int literal(321)`. This type is implicitly
+convertible to all integer types and ranges that contain the value
+`321`. That would be all builtin integer types except `uint8` and
+`int8` where `321` would be out of range. When this literal type is
+assigned to a new `var` or `let` variable, it's type will be resolved
+to just `int`, not `int literal(321)` unlike constants. A constant
+keeps the full `int literal(321)` type. Here is an example where that
+difference matters.
+
+
+  ```nim
+  proc foo(arg: int8) =
+    echo "def"
+
+  const tmp1 = 123
+  foo(tmp1)  # OK
+
+  let tmp2 = 123
+  foo(tmp2) # Error
+  ```
+
+In a context with multiple overloads, the integer literal kind will
+always prefer the `int` type over all other types. If none of the
+overloads is of type `int`, then there will be an error because of
+ambiguity.
+
+  ```nim
+  proc foo(arg: int) =
+    echo "abc"
+  proc foo(arg: int8) =
+    echo "def"
+  foo(123) # output: abc
+
+  proc bar(arg: int16) =
+    echo "abc"
+  proc bar(arg: int8) =
+    echo "def"
+
+  bar(123) # Error ambiguous call
+  ```
+
+In the compiler these integer literal types are represented with the
+node kind `nkIntLit`, type kind `tyInt` and the member `n` of the type
+pointing back to the integer literal node in the ast containing the
+integer value. These are the properties that hold true for integer
+literal types.
+
+    n.kind == nkIntLit
+    n.typ.kind == tyInt
+    n.typ.n == n
+
+Other literal types, such as `uint literal(123)` that would
+automatically convert to other integer types, but prefers to
+become a `uint` are not part of the Nim language.
+
+In an unchecked AST, the `typ` field is nil. The type checker will set
+the `typ` field accordingly to the node kind. Nodes of kind `nkIntLit`
+will get the integer literal type (e.g. `int literal(123)`). Nodes of
+kind `nkUIntLit` will get type `uint` (kind `tyUint`), etc.
+
+This also means that it is not possible to write a literal in an
+unchecked AST that will after sem checking just be of type `int` and
+not implicitly convertible to other integer types. This only works for
+all integer types that are not `int`.
diff --git a/doc/intern.rst b/doc/intern.rst
deleted file mode 100644
index 2456b25fde..0000000000
--- a/doc/intern.rst
+++ /dev/null
@@ -1,796 +0,0 @@
-.. default-role:: code
-
-=========================================
-    Internals of the Nim Compiler
-=========================================
-
-
-:Author: Andreas Rumpf
-:Version: |nimversion|
-
-.. contents::
-
-  "Abstraction is layering ignorance on top of reality." -- Richard Gabriel
-
-
-Directory structure
-===================
-
-The Nim project's directory structure is:
-
-============   ===================================================
-Path           Purpose
-============   ===================================================
-`bin`          generated binary files
-`build`        generated C code for the installation
-`compiler`     the Nim compiler itself; note that this
-               code has been translated from a bootstrapping
-               version written in Pascal, so the code is **not**
-               a poster child of good Nim code
-`config`       configuration files for Nim
-`dist`         additional packages for the distribution
-`doc`          the documentation; it is a bunch of
-               reStructuredText files
-`lib`          the Nim library
-`web`          website of Nim; generated by `nimweb`
-               from the `*.txt` and `*.nimf` files
-============   ===================================================
-
-
-Bootstrapping the compiler
-==========================
-
-Compiling the compiler is a simple matter of running::
-
-  nim c koch.nim
-  ./koch boot
-
-For a release version use::
-
-  nim c koch.nim
-  ./koch boot -d:release
-
-And for a debug version compatible with GDB::
-
-  nim c koch.nim
-  ./koch boot --debuginfo --linedir:on
-
-The `koch` program is Nim's maintenance script. It is a replacement for
-make and shell scripting with the advantage that it is much more portable.
-More information about its options can be found in the `koch `_
-documentation.
-
-
-Coding Guidelines
-=================
-
-* Use CamelCase, not underscored_identifiers.
-* Indent with two spaces.
-* Max line length is 80 characters.
-* Provide spaces around binary operators if that enhances readability.
-* Use a space after a colon, but not before it.
-* [deprecated] Start types with a capital `T`, unless they are
-  pointers/references which start with `P`.
-
-See also the `API naming design `_ document.
-
-
-Porting to new platforms
-========================
-
-Porting Nim to a new architecture is pretty easy, since C is the most
-portable programming language (within certain limits) and Nim generates
-C code, porting the code generator is not necessary.
-
-POSIX-compliant systems on conventional hardware are usually pretty easy to
-port: Add the platform to `platform` (if it is not already listed there),
-check that the OS, System modules work and recompile Nim.
-
-The only case where things aren't as easy is when the garbage
-collector needs some assembler tweaking to work. The standard
-version of the GC uses C's `setjmp` function to store all registers
-on the hardware stack. It may be necessary that the new platform needs to
-replace this generic code by some assembler code.
-
-
-Runtime type information
-========================
-
-*Runtime type information* (RTTI) is needed for several aspects of the Nim
-programming language:
-
-Garbage collection
-  The most important reason for RTTI. Generating
-  traversal procedures produces bigger code and is likely to be slower on
-  modern hardware as dynamic procedure binding is hard to predict.
-
-Complex assignments
-  Sequences and strings are implemented as
-  pointers to resizeable buffers, but Nim requires copying for
-  assignments. Apart from RTTI the compiler could generate copy procedures
-  for any type that needs one. However, this would make the code bigger and
-  the RTTI is likely already there for the GC.
-
-We already know the type information as a graph in the compiler.
-Thus we need to serialize this graph as RTTI for C code generation.
-Look at the file `lib/system/hti.nim` for more information.
-
-Rebuilding the compiler
-========================
-
-After an initial build via `sh build_all.sh` on posix or `build_all.bat` on windows,
-you can rebuild the compiler as follows:
-
-* `nim c koch` if you need to rebuild koch
-* `./koch boot -d:release` this ensures the compiler can rebuild itself
-  (use `koch` instead of `./koch` on windows), which builds the compiler 3 times.
-
-A faster approach if you don't need to run the full bootstrapping implied by `koch boot`,
-is the following:
-
-* `pathto/nim c --lib:lib -d:release -o:bin/nim_temp compiler/nim.nim`
-
-Where `pathto/nim` is any nim binary sufficiently recent (e.g. `bin/nim_cources`
-built during bootstrap or `$HOME/.nimble/bin/nim` installed by `choosenim 1.2.0`)
-
-You can pass any additional options such as `-d:leanCompiler` if you don't need
-certain features or `-d:debug --stacktrace:on --excessiveStackTrace --stackTraceMsgs`
-for debugging the compiler. See also `Debugging the compiler`_.
-
-Debugging the compiler
-======================
-
-You can of course use GDB or Visual Studio to debug the
-compiler (via `--debuginfo --lineDir:on`). However, there
-are also lots of procs that aid in debugging:
-
-
-.. code-block:: nim
-  # pretty prints the Nim AST
-  echo renderTree(someNode)
-  # outputs some JSON representation
-  debug(someNode)
-  # pretty prints some type
-  echo typeToString(someType)
-  debug(someType)
-  echo symbol.name.s
-  debug(symbol)
-  # pretty prints the Nim ast, but annotates symbol IDs:
-  echo renderTree(someNode, {renderIds})
-  if `??`(conf, n.info, "temp.nim"):
-    # only output when it comes from "temp.nim"
-    echo renderTree(n)
-  if `??`(conf, n.info, "temp.nim"):
-    # why does it process temp.nim here?
-    writeStackTrace()
-
-These procs may not be imported by a module. You can import them directly for debugging:
-
-.. code-block:: nim
-  from astalgo import debug
-  from types import typeToString
-  from renderer import renderTree
-  from msgs import `??`
-
-To create a new compiler for each run, use `koch temp`::
-
-  ./koch temp c /tmp/test.nim
-
-`koch temp` creates a debug build of the compiler, which is useful
-to create stacktraces for compiler debugging. See also
-`Rebuilding the compiler`_ if you need more control.
-
-Bisecting for regressions
-=========================
-
-`koch temp` returns 125 as the exit code in case the compiler
-compilation fails. This exit code tells `git bisect` to skip the
-current commit.::
-
-  git bisect start bad-commit good-commit
-  git bisect run ./koch temp -r c test-source.nim
-
-You can also bisect using custom options to build the compiler, for example if
-you don't need a debug version of the compiler (which runs slower), you can replace
-`./koch temp` by explicit compilation command, see `Rebuilding the compiler`_.
-
-The compiler's architecture
-===========================
-
-Nim uses the classic compiler architecture: A lexer/scanner feds tokens to a
-parser. The parser builds a syntax tree that is used by the code generator.
-This syntax tree is the interface between the parser and the code generator.
-It is essential to understand most of the compiler's code.
-
-In order to compile Nim correctly, type-checking has to be separated from
-parsing. Otherwise generics cannot work.
-
-.. include:: filelist.txt
-
-
-The syntax tree
----------------
-The syntax tree consists of nodes which may have an arbitrary number of
-children. Types and symbols are represented by other nodes, because they
-may contain cycles. The AST changes its shape after semantic checking. This
-is needed to make life easier for the code generators. See the "ast" module
-for the type definitions. The `macros `_ module contains many
-examples how the AST represents each syntactic structure.
-
-
-How the RTL is compiled
-=======================
-
-The `system` module contains the part of the RTL which needs support by
-compiler magic (and the stuff that needs to be in it because the spec
-says so). The C code generator generates the C code for it, just like any other
-module. However, calls to some procedures like `addInt` are inserted by
-the CCG. Therefore the module `magicsys` contains a table (`compilerprocs`)
-with all symbols that are marked as `compilerproc`. `compilerprocs` are
-needed by the code generator. A `magic` proc is not the same as a
-`compilerproc`: A `magic` is a proc that needs compiler magic for its
-semantic checking, a `compilerproc` is a proc that is used by the code
-generator.
-
-
-Compilation cache
-=================
-
-The implementation of the compilation cache is tricky: There are lots
-of issues to be solved for the front- and backend.
-
-
-General approach: AST replay
-----------------------------
-
-We store a module's AST of a successful semantic check in a SQLite
-database. There are plenty of features that require a sub sequence
-to be re-applied, for example:
-
-.. code-block:: nim
-  {.compile: "foo.c".} # even if the module is loaded from the DB,
-                       # "foo.c" needs to be compiled/linked.
-
-The solution is to **re-play** the module's top level statements.
-This solves the problem without having to special case the logic
-that fills the internal seqs which are affected by the pragmas.
-
-In fact, this describes how the AST should be stored in the database,
-as a "shallow" tree. Let's assume we compile module `m` with the
-following contents:
-
-.. code-block:: nim
-  import std/strutils
-
-  var x*: int = 90
-  {.compile: "foo.c".}
-  proc p = echo "p"
-  proc q = echo "q"
-  static:
-    echo "static"
-
-Conceptually this is the AST we store for the module:
-
-.. code-block:: nim
-  import std/strutils
-
-  var x*
-  {.compile: "foo.c".}
-  proc p
-  proc q
-  static:
-    echo "static"
-
-The symbol's `ast` field is loaded lazily, on demand. This is where most
-savings come from, only the shallow outer AST is reconstructed immediately.
-
-It is also important that the replay involves the `import` statement so
-that dependencies are resolved properly.
-
-
-Shared global compiletime state
--------------------------------
-
-Nim allows `.global, compiletime` variables that can be filled by macro
-invocations across different modules. This feature breaks modularity in a
-severe way. Plenty of different solutions have been proposed:
-
-- Restrict the types of global compiletime variables to `Set[T]` or
-  similar unordered, only-growable collections so that we can track
-  the module's write effects to these variables and reapply the changes
-  in a different order.
-- In every module compilation, reset the variable to its default value.
-- Provide a restrictive API that can load/save the compiletime state to
-  a file.
-
-(These solutions are not mutually exclusive.)
-
-Since we adopt the "replay the top level statements" idea, the natural
-solution to this problem is to emit pseudo top level statements that
-reflect the mutations done to the global variable. However, this is
-MUCH harder than it sounds, for example `squeaknim` uses this
-snippet:
-
-.. code-block:: nim
-  apicall.add(") module: '" & dllName & "'>\C" &
-              "\t^self externalCallFailed\C!\C\C")
-  stCode.add(st & "\C\t\"Generated by NimSqueak\"\C\t" & apicall)
-
-We can "replay" `stCode.add` only if the values of `st`
-and `apicall` are known. And even then a hash table's `add` with its
-hashing mechanism is too hard to replay.
-
-In practice, things are worse still, consider `someGlobal[i][j].add arg`.
-We only know the root is `someGlobal` but the concrete path to the data
-is unknown as is the value that is added. We could compute a "diff" between
-the global states and use that to compute a symbol patchset, but this is
-quite some work, expensive to do at runtime (it would need to run after
-every module has been compiled) and would also break for hash tables.
-
-We need an API that hides the complex aliasing problems by not relying
-on Nim's global variables. The obvious solution is to use string keys
-instead of global variables:
-
-.. code-block:: nim
-
-  proc cachePut*(key: string; value: string)
-  proc cacheGet*(key: string): string
-
-However, the values being strings/json is quite problematic: Many
-lookup tables that are built at compiletime embed *proc vars* and
-types which have no obvious string representation... Seems like
-AST diffing is still the best idea as it will not require to use
-an alien API and works with some existing Nimble packages, at least.
-
-On the other hand, in Nim's future I would like to replace the VM
-by native code. A diff algorithm wouldn't work for that.
-Instead the native code would work with an API like `put`, `get`:
-
-.. code-block:: nim
-
-  proc cachePut*(key: string; value: NimNode)
-  proc cacheGet*(key: string): NimNode
-
-The API should embrace the AST diffing notion: See the
-module `macrocache` for the final details.
-
-
-
-Methods and type converters
----------------------------
-
-In the following
-sections *global* means *shared between modules* or *property of the whole
-program*.
-
-Nim contains language features that are *global*. The best example for that
-are multi methods: Introducing a new method with the same name and some
-compatible object parameter means that the method's dispatcher needs to take
-the new method into account. So the dispatching logic is only completely known
-after the whole program has been translated!
-
-Other features that are *implicitly* triggered cause problems for modularity
-too. Type converters fall into this category:
-
-.. code-block:: nim
-  # module A
-  converter toBool(x: int): bool =
-    result = x != 0
-
-.. code-block:: nim
-  # module B
-  import A
-
-  if 1:
-    echo "ugly, but should work"
-
-If in the above example module `B` is re-compiled, but `A` is not then
-`B` needs to be aware of `toBool` even though  `toBool` is not referenced
-in `B` *explicitly*.
-
-Both the multi method and the type converter problems are solved by the
-AST replay implementation.
-
-
-Generics
-~~~~~~~~
-
-We cache generic instantiations and need to ensure this caching works
-well with the incremental compilation feature. Since the cache is
-attached to the `PSym` datastructure, it should work without any
-special logic.
-
-
-Backend issues
---------------
-
-- Init procs must not be "forgotten" to be called.
-- Files must not be "forgotten" to be linked.
-- Method dispatchers are global.
-- DLL loading via `dlsym` is global.
-- Emulated thread vars are global.
-
-However the biggest problem is that dead code elimination breaks modularity!
-To see why, consider this scenario: The module `G` (for example the huge
-Gtk2 module...) is compiled with dead code elimination turned on. So none
-of `G`'s procs is generated at all.
-
-Then module `B` is compiled that requires `G.P1`. Ok, no problem,
-`G.P1` is loaded from the symbol file and `G.c` now contains `G.P1`.
-
-Then module `A` (that depends on `B` and `G`) is compiled and `B`
-and `G` are left unchanged. `A` requires `G.P2`.
-
-So now `G.c` MUST contain both `P1` and `P2`, but we haven't even
-loaded `P1` from the symbol file, nor do we want to because we then quickly
-would restore large parts of the whole program.
-
-
-Solution
-~~~~~~~~
-
-The backend must have some logic so that if the currently processed module
-is from the compilation cache, the `ast` field is not accessed. Instead
-the generated C(++) for the symbol's body needs to be cached too and
-inserted back into the produced C file. This approach seems to deal with
-all the outlined problems above.
-
-
-Debugging Nim's memory management
-=================================
-
-The following paragraphs are mostly a reminder for myself. Things to keep
-in mind:
-
-* If an assertion in Nim's memory manager or GC fails, the stack trace
-  keeps allocating memory! Thus a stack overflow may happen, hiding the
-  real issue.
-* What seem to be C code generation problems is often a bug resulting from
-  not producing prototypes, so that some types default to `cint`. Testing
-  without the `-w` option helps!
-
-
-The Garbage Collector
-=====================
-
-Introduction
-------------
-
-I use the term *cell* here to refer to everything that is traced
-(sequences, refs, strings).
-This section describes how the GC works.
-
-The basic algorithm is *Deferrent Reference Counting* with cycle detection.
-References on the stack are not counted for better performance and easier C
-code generation.
-
-Each cell has a header consisting of a RC and a pointer to its type
-descriptor. However the program does not know about these, so they are placed at
-negative offsets. In the GC code the type `PCell` denotes a pointer
-decremented by the right offset, so that the header can be accessed easily. It
-is extremely important that `pointer` is not confused with a `PCell`
-as this would lead to a memory corruption.
-
-
-The CellSet data structure
---------------------------
-
-The GC depends on an extremely efficient datastructure for storing a
-set of pointers - this is called a `TCellSet` in the source code.
-Inserting, deleting and searching are done in constant time. However,
-modifying a `TCellSet` during traversal leads to undefined behaviour.
-
-.. code-block:: Nim
-  type
-    TCellSet # hidden
-
-  proc cellSetInit(s: var TCellSet) # initialize a new set
-  proc cellSetDeinit(s: var TCellSet) # empty the set and free its memory
-  proc incl(s: var TCellSet, elem: PCell) # include an element
-  proc excl(s: var TCellSet, elem: PCell) # exclude an element
-
-  proc `in`(elem: PCell, s: TCellSet): bool # tests membership
-
-  iterator elements(s: TCellSet): (elem: PCell)
-
-
-All the operations have to perform efficiently. Because a Cellset can
-become huge a hash table alone is not suitable for this.
-
-We use a mixture of bitset and hash table for this. The hash table maps *pages*
-to a page descriptor. The page descriptor contains a bit for any possible cell
-address within this page. So including a cell is done as follows:
-
-- Find the page descriptor for the page the cell belongs to.
-- Set the appropriate bit in the page descriptor indicating that the
-  cell points to the start of a memory block.
-
-Removing a cell is analogous - the bit has to be set to zero.
-Single page descriptors are never deleted from the hash table. This is not
-needed as the data structures needs to be rebuilt periodically anyway.
-
-Complete traversal is done in this way::
-
-  for each page descriptor d:
-    for each bit in d:
-      if bit == 1:
-        traverse the pointer belonging to this bit
-
-
-Further complications
----------------------
-
-In Nim the compiler cannot always know if a reference
-is stored on the stack or not. This is caused by var parameters.
-Consider this example:
-
-.. code-block:: Nim
-  proc setRef(r: var ref TNode) =
-    new(r)
-
-  proc usage =
-    var
-      r: ref TNode
-    setRef(r) # here we should not update the reference counts, because
-              # r is on the stack
-    setRef(r.left) # here we should update the refcounts!
-
-We have to decide at runtime whether the reference is on the stack or not.
-The generated code looks roughly like this:
-
-.. code-block:: C
-  void setref(TNode** ref) {
-    unsureAsgnRef(ref, newObj(TNode_TI, sizeof(TNode)))
-  }
-  void usage(void) {
-    setRef(&r)
-    setRef(&r->left)
-  }
-
-Note that for systems with a continuous stack (which most systems have)
-the check whether the ref is on the stack is very cheap (only two
-comparisons).
-
-
-Code generation for closures
-============================
-
-Code generation for closures is implemented by `lambda lifting`:idx:.
-
-
-Design
-------
-
-A `closure` proc var can call ordinary procs of the default Nim calling
-convention. But not the other way round! A closure is implemented as a
-`tuple[prc, env]`. `env` can be nil implying a call without a closure.
-This means that a call through a closure generates an `if` but the
-interoperability is worth the cost of the `if`. Thunk generation would be
-possible too, but it's slightly more effort to implement.
-
-Tests with GCC on Amd64 showed that it's really beneficial if the
-'environment' pointer is passed as the last argument, not as the first argument.
-
-Proper thunk generation is harder because the proc that is to wrap
-could stem from a complex expression:
-
-.. code-block:: nim
-  receivesClosure(returnsDefaultCC[i])
-
-A thunk would need to call 'returnsDefaultCC[i]' somehow and that would require
-an *additional* closure generation... Ok, not really, but it requires to pass
-the function to call. So we'd end up with 2 indirect calls instead of one.
-Another much more severe problem which this solution is that it's not GC-safe
-to pass a proc pointer around via a generic `ref` type.
-
-
-Example code:
-
-.. code-block:: nim
-  proc add(x: int): proc (y: int): int {.closure.} =
-    return proc (y: int): int =
-      return x + y
-
-  var add2 = add(2)
-  echo add2(5) #OUT 7
-
-This should produce roughly this code:
-
-.. code-block:: nim
-  type
-    PEnv = ref object
-      x: int # data
-
-  proc anon(y: int, c: PEnv): int =
-    return y + c.x
-
-  proc add(x: int): tuple[prc, data] =
-    var env: PEnv
-    new env
-    env.x = x
-    result = (anon, env)
-
-  var add2 = add(2)
-  let tmp = if add2.data == nil: add2.prc(5) else: add2.prc(5, add2.data)
-  echo tmp
-
-
-Beware of nesting:
-
-.. code-block:: nim
-  proc add(x: int): proc (y: int): proc (z: int): int {.closure.} {.closure.} =
-    return lambda (y: int): proc (z: int): int {.closure.} =
-      return lambda (z: int): int =
-        return x + y + z
-
-  var add24 = add(2)(4)
-  echo add24(5) #OUT 11
-
-This should produce roughly this code:
-
-.. code-block:: nim
-  type
-    PEnvX = ref object
-      x: int # data
-
-    PEnvY = ref object
-      y: int
-      ex: PEnvX
-
-  proc lambdaZ(z: int, ey: PEnvY): int =
-    return ey.ex.x + ey.y + z
-
-  proc lambdaY(y: int, ex: PEnvX): tuple[prc, data: PEnvY] =
-    var ey: PEnvY
-    new ey
-    ey.y = y
-    ey.ex = ex
-    result = (lambdaZ, ey)
-
-  proc add(x: int): tuple[prc, data: PEnvX] =
-    var ex: PEnvX
-    ex.x = x
-    result = (labmdaY, ex)
-
-  var tmp = add(2)
-  var tmp2 = tmp.fn(4, tmp.data)
-  var add24 = tmp2.fn(4, tmp2.data)
-  echo add24(5)
-
-
-We could get rid of nesting environments by always inlining inner anon procs.
-More useful is escape analysis and stack allocation of the environment,
-however.
-
-
-Alternative
------------
-
-Process the closure of all inner procs in one pass and accumulate the
-environments. This is however not always possible.
-
-
-Accumulator
------------
-
-.. code-block:: nim
-  proc getAccumulator(start: int): proc (): int {.closure} =
-    var i = start
-    return lambda: int =
-      inc i
-      return i
-
-  proc p =
-    var delta = 7
-    proc accumulator(start: int): proc(): int =
-      var x = start-1
-      result = proc (): int =
-        x = x + delta
-        inc delta
-        return x
-
-    var a = accumulator(3)
-    var b = accumulator(4)
-    echo a() + b()
-
-
-Internals
----------
-
-Lambda lifting is implemented as part of the `transf` pass. The `transf`
-pass generates code to setup the environment and to pass it around. However,
-this pass does not change the types! So we have some kind of mismatch here; on
-the one hand the proc expression becomes an explicit tuple, on the other hand
-the tyProc(ccClosure) type is not changed. For C code generation it's also
-important the hidden formal param is `void*` and not something more
-specialized. However the more specialized env type needs to passed to the
-backend somehow. We deal with this by modifying `s.ast[paramPos]` to contain
-the formal hidden parameter, but not `s.typ`!
-
-
-Integer literals:
------------------
-
-In Nim, there is a redundant way to specify the type of an
-integer literal. First of all, it should be unsurprising that every
-node has a node kind. The node of an integer literal can be any of the
-following values:
-
-    nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit,
-    nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit
-
-On top of that, there is also the `typ` field for the type. It the
-kind of the `typ` field can be one of the following ones, and it
-should be matching the literal kind:
-
-    tyInt, tyInt8, tyInt16, tyInt32, tyInt64, tyUInt, tyUInt8,
-    tyUInt16, tyUInt32, tyUInt64
-
-Then there is also the integer literal type. This is a specific type
-that is implicitly convertible into the requested type if the
-requested type can hold the value. For this to work, the type needs to
-know the concrete value of the literal. For example an expression
-`321` will be of type `int literal(321)`. This type is implicitly
-convertible to all integer types and ranges that contain the value
-`321`. That would be all builtin integer types except `uint8` and
-`int8` where `321` would be out of range. When this literal type is
-assigned to a new `var` or `let` variable, it's type will be resolved
-to just `int`, not `int literal(321)` unlike constants. A constant
-keeps the full `int literal(321)` type. Here is an example where that
-difference matters.
-
-
-.. code-block:: nim
-
-   proc foo(arg: int8) =
-     echo "def"
-
-   const tmp1 = 123
-   foo(tmp1)  # OK
-
-   let tmp2 = 123
-   foo(tmp2) # Error
-
-In a context with multiple overloads, the integer literal kind will
-always prefer the `int` type over all other types. If none of the
-overloads is of type `int`, then there will be an error because of
-ambiguity.
-
-.. code-block:: nim
-
-   proc foo(arg: int) =
-     echo "abc"
-   proc foo(arg: int8) =
-     echo "def"
-   foo(123) # output: abc
-
-   proc bar(arg: int16) =
-     echo "abc"
-   proc bar(arg: int8) =
-     echo "def"
-
-   bar(123) # Error ambiguous call
-
-In the compiler these integer literal types are represented with the
-node kind `nkIntLit`, type kind `tyInt` and the member `n` of the type
-pointing back to the integer literal node in the ast containing the
-integer value. These are the properties that hold true for integer
-literal types.
-
-    n.kind == nkIntLit
-    n.typ.kind == tyInt
-    n.typ.n == n
-
-Other literal types, such as `uint literal(123)` that would
-automatically convert to other integer types, but prefers to
-become a `uint` are not part of the Nim language.
-
-In an unchecked AST, the `typ` field is nil. The type checker will set
-the `typ` field accordingly to the node kind. Nodes of kind `nkIntLit`
-will get the integer literal type (e.g. `int literal(123)`). Nodes of
-kind `nkUIntLit` will get type `uint` (kind `tyUint`), etc.
-
-This also means that it is not possible to write a literal in an
-unchecked AST that will after sem checking just be of type `int` and
-not implicitly convertible to other integer types. This only works for
-all integer types that are not `int`.
diff --git a/doc/koch.rst b/doc/koch.md
similarity index 64%
rename from doc/koch.rst
rename to doc/koch.md
index 1eb02d7852..8fa19ce448 100644
--- a/doc/koch.rst
+++ b/doc/koch.md
@@ -1,17 +1,14 @@
-.. default-role:: code
-
 ===============================
    Nim maintenance script
 ===============================
 
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
-.. raw:: html
-  

-  "A great chef is an artist that I truly respect" -- Robert Stack.
-  

+> "A great chef is an artist that I truly respect" -- Robert Stack.
 
 
 Introduction
@@ -19,7 +16,7 @@ Introduction
 
 The `koch`:idx: program is Nim's maintenance script. It is a replacement
 for make and shell scripting with the advantage that it is much more portable.
-The word *koch* means *cook* in German. `koch` is used mainly to build the
+The word *koch* means *cook* in German. `koch`:cmd: is used mainly to build the
 Nim compiler, but it can also be used for other tasks. This document
 describes the supported commands and their options.
 
@@ -33,12 +30,14 @@ boot command
 The `boot`:idx: command bootstraps the compiler, and it accepts different
 options:
 
--d:release
-  By default a debug version is created, passing this option will
+-d:release    By default a debug version is created, passing this option will
   force a release build, which is much faster and should be preferred
   unless you are debugging the compiler.
--d:nimUseLinenoise
-  Use the linenoise library for interactive mode (not needed on Windows).
+-d:nimUseLinenoise     Use the linenoise library for interactive mode
+                       (not needed on Windows).
+-d:leanCompiler        Produce a compiler without JS codegen or
+                       documentation generator in order to use less RAM
+                       for bootstrapping.
 
 After compilation is finished you will hopefully end up with the nim
 compiler in the `bin` directory. You can add Nim's `bin` directory to
@@ -49,23 +48,23 @@ csource command
 ---------------
 
 The `csource`:idx: command builds the C sources for installation. It accepts
-the same options as you would pass to the `boot command
-<#commands-boot-command>`_.
+the same options as you would pass to the [boot command](
+#commands-boot-command).
 
 temp command
 ------------
 
 The temp command builds the Nim compiler but with a different final name
-(`nim_temp`), so it doesn't overwrite your normal compiler. You can use
-this command to test different options, the same you would issue for the `boot
-command <#commands-boot-command>`_.
+(`nim_temp`:cmd:), so it doesn't overwrite your normal compiler. You can use
+this command to test different options, the same you would issue for the [boot
+command](#commands-boot-command).
 
 test command
 ------------
 
-The `test`:idx: command can also be invoked with the alias `tests`. This
-command will compile and run `testament/tester.nim`, which is the main
-driver of Nim's test suite. You can pass options to the `test` command,
+The `test`:idx: command can also be invoked with the alias `tests`:option:. This
+command will compile and run ``testament/tester.nim``, which is the main
+driver of Nim's test suite. You can pass options to the `test`:option: command,
 they will be forwarded to the tester. See its source code for available
 options.
 
@@ -74,13 +73,13 @@ web command
 
 The `web`:idx: command converts the documentation in the `doc` directory
 from rst to HTML. It also repeats the same operation but places the result in
-the `web/upload` which can be used to update the website at
+the ``web/upload`` which can be used to update the website at
 https://nim-lang.org.
 
 By default, the documentation will be built in parallel using the number of
 available CPU cores. If any documentation build sub-commands fail, they will
 be rerun in serial fashion so that meaningful error output can be gathered for
-inspection. The `--parallelBuild:n` switch or configuration option can be
+inspection. The `--parallelBuild:n`:option: switch or configuration option can be
 used to force a specific number of parallel jobs or run everything serially
 from the start (`n == 1`).
 
@@ -89,4 +88,4 @@ pdf command
 
 The `pdf`:idx: command builds PDF versions of Nim documentation: Manual,
 Tutorial and a few other documents. To run it one needs to
-`install Latex/pdflatex `_ first.
+[install Latex/xelatex](https://www.latex-project.org/get) first.
diff --git a/doc/lib.md b/doc/lib.md
new file mode 100644
index 0000000000..1507bbaac7
--- /dev/null
+++ b/doc/lib.md
@@ -0,0 +1,682 @@
+====================
+Nim Standard Library
+====================
+
+:Author: Andreas Rumpf
+:Version: |nimversion|
+
+.. default-role:: code
+.. include:: rstcommon.rst
+.. contents::
+
+Nim's library is divided into *pure libraries*, *impure libraries*, and *wrappers*.
+
+Pure libraries do not depend on any external ``*.dll`` or ``lib*.so`` binary
+while impure libraries do. A wrapper is an impure library that is a very
+low-level interface to a C library.
+
+Read [this document](apis.html) for a quick overview of the API design.
+
+
+Nimble
+======
+
+Nim's standard library only covers the basics, check
+out https://nimble.directory/ for a list of 3rd party packages.
+
+
+Pure libraries
+==============
+
+Automatic imports
+-----------------
+
+* [system](system.html)
+  Basic procs and operators that every program needs. It also provides IO
+  facilities for reading and writing text and binary files. It is imported
+  implicitly by the compiler. Do not import it directly. It relies on compiler
+  magic to work.
+
+Core
+----
+
+* [atomics](atomics.html)
+  Types and operations for atomic operations and lockless algorithms.
+
+* [bitops](bitops.html)
+  Provides a series of low-level methods for bit manipulation.
+
+* [compilesettings](compilesettings.html)
+  Querying the compiler about diverse configuration settings from code.
+
+* [cpuinfo](cpuinfo.html)
+  Procs to determine the number of CPUs / cores.
+
+* [effecttraits](effecttraits.html)
+  Access to the inferred .raises effects
+  for Nim's macro system.
+
+* [endians](endians.html)
+  Helpers that deal with different byte orders.
+
+* [locks](locks.html)
+  Locks and condition variables for Nim.
+
+* [macrocache](macrocache.html)
+  Provides an API for macros to collect compile-time information across modules.
+
+* [macros](macros.html)
+  Contains the AST API and documentation of Nim for writing macros.
+
+* [rlocks](rlocks.html)
+  Reentrant locks for Nim.
+
+* [typeinfo](typeinfo.html)
+  Provides (unsafe) access to Nim's run-time type information.
+
+* [typetraits](typetraits.html)
+  Compile-time reflection procs for working with types.
+
+* [volatile](volatile.html)
+  Code for generating volatile loads and stores,
+  which are useful in embedded and systems programming.
+
+
+Algorithms
+----------
+
+* [algorithm](algorithm.html)
+  Some common generic algorithms like sort or binary search.
+
+* [enumutils](enumutils.html)
+  Additional functionality for the built-in `enum` type.
+
+* [sequtils](sequtils.html)
+  Operations for the built-in `seq` type
+  which were inspired by functional programming languages.
+
+* [setutils](setutils.html)
+  Additional functionality for the built-in `set` type.
+
+
+Collections
+-----------
+
+* [critbits](critbits.html)
+  A *crit bit tree* which is an efficient
+  container for a sorted set of strings, or a sorted mapping of strings.
+
+* [deques](deques.html)
+  Implementation of a double-ended queue.
+  The underlying implementation uses a `seq`.
+
+* [heapqueue](heapqueue.html)
+  Implementation of a binary heap data structure that can be used as a priority queue.
+
+* [intsets](intsets.html)
+  Efficient implementation of a set of ints as a sparse bit set.
+
+* [lists](lists.html)
+  Nim linked list support. Contains singly and doubly linked lists and
+  circular lists ("rings").
+
+* [options](options.html)
+  The option type encapsulates an optional value.
+
+* [packedsets](packedsets.html)
+  Efficient implementation of a set of ordinals as a sparse bit set.
+
+* [ropes](ropes.html)
+  A *rope* data type.
+  Ropes can represent very long strings efficiently;
+  in particular, concatenation is done in O(1) instead of O(n).
+
+* [sets](sets.html)
+  Nim hash set support.
+
+* [strtabs](strtabs.html)
+  The `strtabs` module implements an efficient hash table that is a mapping
+  from strings to strings. Supports a case-sensitive, case-insensitive and
+  style-insensitive modes.
+
+* [tables](tables.html)
+  Nim hash table support. Contains tables, ordered tables, and count tables.
+
+
+String handling
+---------------
+
+* [cstrutils](cstrutils.html)
+  Utilities for `cstring` handling.
+
+* [editdistance](editdistance.html)
+  An algorithm to compute the edit distance between two
+  Unicode strings.
+
+* [encodings](encodings.html)
+  Converts between different character encodings. On UNIX, this uses
+  the `iconv` library, on Windows the Windows API.
+
+* [formatfloat](formatfloat.html)
+  Formatting floats as strings.
+
+* [objectdollar](objectdollar.html)
+  A generic `$` operator to convert objects to strings.
+
+* [punycode](punycode.html)
+  Implements a representation of Unicode with the limited ASCII character subset.
+
+* [strbasics](strbasics.html)
+  Some high performance string operations.
+
+* [strformat](strformat.html)
+  Macro based standard string interpolation/formatting. Inspired by
+  Python's f-strings.\
+  **Note:** if you need templating, consider using Nim
+  [Source Code Filters (SCF)](filters.html).
+
+* [strmisc](strmisc.html)
+  Uncommon string handling operations that do not
+  fit with the commonly used operations in [strutils](strutils.html).
+
+* [strscans](strscans.html)
+  A `scanf` macro for convenient parsing of mini languages.
+
+* [strutils](strutils.html)
+  Common string handling operations like changing
+  case of a string, splitting a string into substrings, searching for
+  substrings, replacing substrings.
+
+* [unicode](unicode.html)
+  Support for handling the Unicode UTF-8 encoding.
+
+* [unidecode](unidecode.html)
+  It provides a single proc that does Unicode to ASCII transliterations.
+  Based on Python's Unidecode module.
+
+* [widestrs](widestrs.html)
+  Nim support for C/C++'s wide strings.
+
+* [wordwrap](wordwrap.html)
+  An algorithm for word-wrapping Unicode strings.
+
+
+Time handling
+-------------
+
+* [monotimes](monotimes.html)
+  The `monotimes` module implements monotonic timestamps.
+
+* [times](times.html)
+  The `times` module contains support for working with time.
+
+
+Generic Operating System Services
+---------------------------------
+
+* [appdirs](appdirs.html)
+  Helpers for determining special directories used by apps.
+
+* [cmdline](cmdline.html)
+  System facilities for reading command line parameters.
+
+* [dirs](dirs.html)
+  Directory handling.
+
+* [distros](distros.html)
+  Basics for OS distribution ("distro") detection
+  and the OS's native package manager.
+  Its primary purpose is to produce output for Nimble packages,
+  but it also contains the widely used **Distribution** enum
+  that is useful for writing platform-specific code.
+  See [packaging](packaging.html) for hints on distributing Nim using OS packages.
+
+* [dynlib](dynlib.html)
+  Accessing symbols from shared libraries.
+
+* [envvars](envvars.html)
+  Environment variable handling.
+
+* [exitprocs](exitprocs.html)
+  Adding hooks to program exit.
+
+* [files](files.html)
+  File handling.
+
+* [memfiles](memfiles.html)
+  Support for memory-mapped files (Posix's `mmap`)
+  on the different operating systems.
+
+* [os](os.html)
+  Basic operating system facilities like retrieving environment variables,
+  reading command line arguments, working with directories, running shell
+  commands, etc.
+
+* [oserrors](oserrors.html)
+  OS error reporting.
+
+* [osproc](osproc.html)
+  Module for process communication beyond `os.execShellCmd`.
+
+* [paths](paths.html)
+  Path handling.
+
+* [reservedmem](reservedmem.html)
+  Utilities for reserving portions of the
+  address space of a program without consuming physical memory.
+
+* [streams](streams.html)
+  A stream interface and two implementations thereof:
+  the `FileStream` and the `StringStream` which implement the stream
+  interface for Nim file objects (`File`) and strings. Other modules
+  may provide other implementations for this standard stream interface.
+
+* [symlinks](symlinks.html)
+  Symlink handling.
+
+* [syncio](syncio.html)
+  Various synchronized I/O operations.
+
+* [terminal](terminal.html)
+  A module to control the terminal output (also called *console*).
+  
+* [tempfiles](tempfiles.html)
+  Some utilities for generating temporary path names and
+  creating temporary files and directories.
+
+
+Math libraries
+--------------
+
+* [complex](complex.html)
+  Complex numbers and relevant mathematical operations.
+
+* [fenv](fenv.html)
+  Floating-point environment. Handling of floating-point rounding and
+  exceptions (overflow, zero-divide, etc.).
+
+* [lenientops](lenientops.html)
+  Binary operators for mixed integer/float expressions for convenience.
+
+* [math](math.html)
+  Mathematical operations like cosine, square root.
+
+* [random](random.html)
+  Fast and tiny random number generator.
+
+* [rationals](rationals.html)
+  Rational numbers and relevant mathematical operations.
+
+* [stats](stats.html)
+  Statistical analysis.
+
+* [sysrand](sysrand.html)
+  Cryptographically secure pseudorandom number generator.
+
+
+Internet Protocols and Support
+------------------------------
+
+* [async](async.html)
+  Exports `asyncmacro` and `asyncfutures` for native backends, and `asyncjs` on the JS backend. 
+
+* [asyncdispatch](asyncdispatch.html)
+  An asynchronous dispatcher for IO operations.
+
+* [asyncfile](asyncfile.html)
+  An asynchronous file reading and writing using `asyncdispatch`.
+
+* [asyncftpclient](asyncftpclient.html)
+  An asynchronous FTP client using the `asyncnet` module.
+
+* [asynchttpserver](asynchttpserver.html)
+  An asynchronous HTTP server using the `asyncnet` module.
+
+* [asyncmacro](asyncmacro.html)
+  `async` and `multisync` macros for `asyncdispatch`.
+
+* [asyncnet](asyncnet.html)
+  Asynchronous sockets based on the `asyncdispatch` module.
+
+* [asyncstreams](asyncstreams.html)
+  `FutureStream` - a future that acts as a queue.
+
+* [cgi](cgi.html)
+  Helpers for CGI applications.
+
+* [cookies](cookies.html)
+  Helper procs for parsing and generating cookies.
+
+* [httpclient](httpclient.html)
+  A simple HTTP client with support for both synchronous
+  and asynchronous retrieval of web pages.
+
+* [mimetypes](mimetypes.html)
+  A mimetypes database.
+
+* [nativesockets](nativesockets.html)
+  A low-level sockets API.
+
+* [net](net.html)
+  A high-level sockets API.
+
+* [selectors](selectors.html)
+  A selector API with backends specific to each OS.
+  Supported OS primitives: `epoll`, `kqueue`, `poll`, and `select` on Windows.
+
+* [smtp](smtp.html)
+  A simple SMTP client with support for both synchronous and asynchronous operation.
+
+* [socketstreams](socketstreams.html)
+  An implementation of the streams interface for sockets.
+
+* [uri](uri.html)
+  Functions for working with URIs and URLs.
+
+
+Threading
+---------
+
+* [isolation](isolation.html)
+  The `Isolated[T]` type for
+  safe construction of isolated subgraphs that can be
+  passed efficiently to different channels and threads.
+
+* [tasks](tasks.html)
+  Basic primitives for creating parallel programs.
+
+* [threadpool](threadpool.html)
+  Implements Nim's [spawn](manual_experimental.html#parallel-amp-spawn).
+
+* [typedthreads](typedthreads.html)
+  Basic Nim thread support.
+
+
+Parsers
+-------
+
+* [htmlparser](htmlparser.html)
+  HTML document parser that creates a XML tree representation.
+
+* [json](json.html)
+  High-performance JSON parser.
+
+* [lexbase](lexbase.html)
+  A low-level module that implements an extremely efficient buffering
+  scheme for lexers and parsers. This is used by the diverse parsing modules.
+
+* [parsecfg](parsecfg.html)
+  The `parsecfg` module implements a high-performance configuration file
+  parser. The configuration file's syntax is similar to the Windows ``.ini``
+  format, but much more powerful, as it is not a line based parser. String
+  literals, raw string literals, and triple quote string literals are supported
+  as in the Nim programming language.
+
+* [parsecsv](parsecsv.html)
+  The `parsecsv` module implements a simple high-performance CSV parser.
+
+* [parsejson](parsejson.html)
+  A JSON parser. It is used and exported by the [json](json.html) module, but can also be used in its own right.
+
+* [parseopt](parseopt.html)
+  The `parseopt` module implements a command line option parser.
+
+* [parsesql](parsesql.html)
+  The `parsesql` module implements a simple high-performance SQL parser.
+
+* [parseutils](parseutils.html)
+  Helpers for parsing tokens, numbers, identifiers, etc.
+
+* [parsexml](parsexml.html)
+  The `parsexml` module implements a simple high performance XML/HTML parser.
+  The only encoding that is supported is UTF-8. The parser has been designed
+  to be somewhat error-correcting, so that even some "wild HTML" found on the
+  web can be parsed with it.
+
+* [pegs](pegs.html)
+  Procedures and operators for handling PEGs.
+
+
+Docutils
+--------
+
+* [packages/docutils/highlite](highlite.html)
+  Source highlighter for programming or markup languages. Currently,
+  only a few languages are supported, other languages may be added.
+  The interface supports one language nested in another.
+
+* [packages/docutils/rst](rst.html)
+  A reStructuredText parser. A large subset
+  is implemented. Some features of the markdown wiki syntax are also supported.
+
+* [packages/docutils/rstast](rstast.html)
+  An AST for the reStructuredText parser.
+
+* [packages/docutils/rstgen](rstgen.html)
+  A generator of HTML/Latex from reStructuredText.
+
+
+XML Processing
+--------------
+
+* [xmltree](xmltree.html)
+  A simple XML tree. More efficient and simpler than the DOM. It also
+  contains a macro for XML/HTML code generation.
+
+* [xmlparser](xmlparser.html)
+  XML document parser that creates a XML tree representation.
+
+
+Generators
+----------
+
+* [genasts](genasts.html)
+  AST generation using captured variables for macros.
+
+* [htmlgen](htmlgen.html)
+  A simple XML and HTML code
+  generator. Each commonly used HTML tag has a corresponding macro
+  that generates a string with its HTML representation.
+
+
+Hashing
+-------
+
+* [base64](base64.html)
+  A Base64 encoder and decoder.
+
+* [hashes](hashes.html)
+  Efficient computations of hash values for diverse Nim types.
+
+* [md5](md5.html)
+  The MD5 checksum algorithm.
+
+* [oids](oids.html)
+  An OID is a global ID that consists of a timestamp,
+  a unique counter, and a random value. This combination should suffice to
+  produce a globally distributed unique ID.
+
+* [sha1](sha1.html)
+  The SHA-1 checksum algorithm.
+
+
+Serialization
+-------------
+
+* [jsonutils](jsonutils.html)
+  Hookable (de)serialization for arbitrary types
+  using JSON.
+
+* [marshal](marshal.html)
+  Contains procs for serialization and deserialization of arbitrary Nim
+  data structures.
+
+
+Miscellaneous
+-------------
+
+* [assertions](assertions.html)
+  Assertion handling.
+
+* [browsers](browsers.html)
+  Procs for opening URLs with the user's default
+  browser.
+
+* [colors](colors.html)
+  Color handling.
+
+* [coro](coro.html)
+  Experimental coroutines in Nim.
+
+* [decls](decls.html)
+  Syntax sugar for some declarations.
+
+* [enumerate](enumerate.html)
+  `enumerate` syntactic sugar based on Nim's macro system.
+
+* [importutils](importutils.html)
+  Utilities related to import and symbol resolution.
+
+* [logging](logging.html)
+  A simple logger.
+
+* [segfaults](segfaults.html)
+  Turns access violations or segfaults into a `NilAccessDefect` exception.
+
+* [sugar](sugar.html)
+  Nice syntactic sugar based on Nim's macro system.
+
+* [unittest](unittest.html)
+  Implements a Unit testing DSL.
+
+* [varints](varints.html)
+  Decode variable-length integers that are compatible with SQLite.
+
+* [with](with.html)
+  The `with` macro for easy function chaining.
+
+* [wrapnils](wrapnils.html)
+  Allows evaluating expressions safely against nil dereferences.
+
+
+Modules for the JavaScript backend
+----------------------------------
+
+* [asyncjs](asyncjs.html)
+  Types and macros for writing asynchronous procedures in JavaScript.
+
+* [dom](dom.html)
+  Declaration of the Document Object Model for the JS backend.
+
+* [jsbigints](jsbigints.html)
+  Arbitrary precision integers.
+
+* [jsconsole](jsconsole.html)
+  Wrapper for the `console` object.
+
+* [jscore](jscore.html)
+  The wrapper of core JavaScript functions. For most purposes, you should be using
+  the `math`, `json`, and `times` stdlib modules instead of this module.
+
+* [jsfetch](jsfetch.html)
+  Wrapper for `fetch`.
+
+* [jsffi](jsffi.html)
+  Types and macros for easier interaction with JavaScript.
+
+* [jsre](jsre.html)
+  Regular Expressions for the JavaScript target.
+
+
+Impure libraries
+================
+
+Regular expressions
+-------------------
+
+* [re](re.html)
+  Procedures and operators for handling regular
+  expressions. The current implementation uses PCRE.
+
+* [nre](nre.html)
+
+  Many help functions for handling regular expressions.
+  The current implementation uses PCRE.
+
+Database support
+----------------
+
+* [db_mysql](db_mysql.html)
+  A higher level MySQL database wrapper. The same interface is implemented
+  for other databases too.
+
+* [db_odbc](db_odbc.html)
+  A higher level ODBC database wrapper. The same interface is implemented
+  for other databases too.
+
+* [db_postgres](db_postgres.html)
+  A higher level PostgreSQL database wrapper. The same interface is implemented
+  for other databases too.
+
+* [db_sqlite](db_sqlite.html)
+  A higher level SQLite database wrapper. The same interface is implemented
+  for other databases too.
+
+
+Generic Operating System Services
+---------------------------------
+
+* [rdstdin](rdstdin.html)
+  Code for reading user input from stdin.
+
+
+Wrappers
+========
+
+The generated HTML for some of these wrappers is so huge that it is
+not contained in the distribution. You can then find them on the website.
+
+
+Windows-specific
+----------------
+
+* [winlean](winlean.html)
+  Wrapper for a small subset of the Win32 API.
+* [registry](registry.html)
+  Windows registry support.
+
+
+UNIX specific
+-------------
+
+* [posix](posix.html)
+  Wrapper for the POSIX standard.
+* [posix_utils](posix_utils.html)
+  Contains helpers for the POSIX standard or specialized for Linux and BSDs.
+
+
+Regular expressions
+-------------------
+
+* [pcre](pcre.html)
+  Wrapper for the PCRE library.
+
+
+Database support
+----------------
+
+* [mysql](mysql.html)
+  Wrapper for the mySQL API.
+* [odbcsql](odbcsql.html)
+  interface to the ODBC driver.
+* [postgres](postgres.html)
+  Wrapper for the PostgreSQL API.
+* [sqlite3](sqlite3.html)
+  Wrapper for the SQLite 3 API.
+
+
+Network Programming and Internet Protocols
+------------------------------------------
+
+* [openssl](openssl.html)
+  Wrapper for OpenSSL.
diff --git a/doc/lib.rst b/doc/lib.rst
deleted file mode 100644
index 3202c5a53a..0000000000
--- a/doc/lib.rst
+++ /dev/null
@@ -1,571 +0,0 @@
-.. default-role:: code
-
-====================
-Nim Standard Library
-====================
-
-:Author: Andreas Rumpf
-:Version: |nimversion|
-
-.. contents::
-
-Nim's library is divided into *pure libraries*, *impure libraries*, and *wrappers*.
-
-Pure libraries do not depend on any external `*.dll` or `lib*.so` binary
-while impure libraries do. A wrapper is an impure library that is a very
-low-level interface to a C library.
-
-Read this `document `_ for a quick overview of the API design.
-
-
-Nimble
-======
-
-Nim's standard library only covers the basics, check
-out ``_ for a list of 3rd party packages.
-
-
-Pure libraries
-==============
-
-Automatic imports
------------------
-
-* `system `_
-  Basic procs and operators that every program needs. It also provides IO
-  facilities for reading and writing text and binary files. It is imported
-  implicitly by the compiler. Do not import it directly. It relies on compiler
-  magic to work.
-
-* `threads `_
-  Basic Nim thread support. **Note:** This is part of the system module. Do not
-  import it explicitly. Enabled with `--threads:on`.
-
-* `channels `_
-  Nim message passing support for threads. **Note:** This is part of the
-  system module. Do not import it explicitly. Enabled with `--threads:on`.
-
-
-Core
-----
-
-* `bitops `_
-  Provides a series of low-level methods for bit manipulation.
-
-* `cpuinfo `_
-  This module implements procs to determine the number of CPUs / cores.
-
-* `endians `_
-  This module contains helpers that deal with different byte orders.
-
-* `lenientops `_
-  Provides binary operators for mixed integer/float expressions for convenience.
-
-* `locks `_
-  Locks and condition variables for Nim.
-
-* `macros `_
-  Contains the AST API and documentation of Nim for writing macros.
-
-* `rlocks `_
-  Reentrant locks for Nim.
-
-* `typeinfo `_
-  Provides (unsafe) access to Nim's run-time type information.
-
-* `typetraits `_
-  This module defines compile-time reflection procs for working with types.
-
-* `volatile `_
-  This module contains code for generating volatile loads and stores,
-  which are useful in embedded and systems programming.
-
-
-Algorithms
-----------
-
-* `algorithm `_
-  This module implements some common generic algorithms like sort or binary search.
-
-* `std/enumutils `_
-  This module adds functionality for the built-in `enum` type.
-
-* `sequtils `_
-  This module implements operations for the built-in `seq` type
-  which were inspired by functional programming languages.
-
-* `std/setutils `_
-  This module adds functionality for the built-in `set` type.
-
-
-Collections
------------
-
-* `critbits `_
-  This module implements a *crit bit tree* which is an efficient
-  container for a sorted set of strings, or a sorted mapping of strings.
-
-* `deques `_
-  Implementation of a double-ended queue.
-  The underlying implementation uses a `seq`.
-
-* `heapqueue `_
-  Implementation of a heap data structure that can be used as a priority queue.
-
-* `intsets `_
-  Efficient implementation of a set of ints as a sparse bit set.
-
-* `lists `_
-  Nim linked list support. Contains singly and doubly linked lists and
-  circular lists ("rings").
-
-* `options `_
-  The option type encapsulates an optional value.
-
-* `std/packedsets `_
-  Efficient implementation of a set of ordinals as a sparse bit set.
-
-* `sets `_
-  Nim hash and bit set support.
-
-* `sharedlist `_
-  Nim shared linked list support. Contains a shared singly-linked list.
-
-* `sharedtables `_
-  Nim shared hash table support. Contains shared tables.
-
-* `tables `_
-  Nim hash table support. Contains tables, ordered tables, and count tables.
-
-
-String handling
----------------
-
-* `cstrutils `_
-  Utilities for `cstring` handling.
-
-* `std/editdistance `_
-  This module contains an algorithm to compute the edit distance between two
-  Unicode strings.
-
-* `encodings `_
-  Converts between different character encodings. On UNIX, this uses
-  the `iconv` library, on Windows the Windows API.
-
-* `parseutils `_
-  This module contains helpers for parsing tokens, numbers, identifiers, etc.
-
-* `pegs `_
-  This module contains procedures and operators for handling PEGs.
-
-* `punycode `_
-  Implements a representation of Unicode with the limited ASCII character subset.
-
-* `ropes `_
-  This module contains support for a *rope* data type.
-  Ropes can represent very long strings efficiently;
-  especially concatenation is done in O(1) instead of O(n).
-
-* `strformat `_
-  Macro based standard string interpolation/formatting. Inspired by
-  Python's `f`-strings.
-
-* `strmisc `_
-  This module contains uncommon string handling operations that do not
-  fit with the commonly used operations in strutils.
-
-* `strscans `_
-  This module contains a `scanf` macro for convenient parsing of mini languages.
-
-* `strtabs `_
-  The `strtabs` module implements an efficient hash table that is a mapping
-  from strings to strings. Supports a case-sensitive, case-insensitive and
-  style-insensitive modes.
-
-* `strutils `_
-  This module contains common string handling operations like changing
-  case of a string, splitting a string into substrings, searching for
-  substrings, replacing substrings.
-
-* `unicode `_
-  This module provides support to handle the Unicode UTF-8 encoding.
-
-* `unidecode `_
-  It provides a single proc that does Unicode to ASCII transliterations.
-  Based on Python's Unidecode module.
-
-* `std/wordwrap `_
-  This module contains an algorithm to wordwrap a Unicode string.
-
-
-Time handling
--------------
-
-* `std/monotimes `_
-  The `monotimes` module implements monotonic timestamps.
-
-* `times `_
-  The `times` module contains support for working with time.
-
-
-Generic Operating System Services
----------------------------------
-
-* `distros `_
-  This module implements the basics for OS distribution ("distro") detection
-  and the OS's native package manager.
-  Its primary purpose is to produce output for Nimble packages,
-  but it also contains the widely used **Distribution** enum
-  that is useful for writing platform-specific code.
-  See `packaging `_ for hints on distributing Nim using OS packages.
-
-* `dynlib `_
-  This module implements the ability to access symbols from shared libraries.
-
-* `marshal `_
-  Contains procs for serialization and deserialization of arbitrary Nim
-  data structures.
-
-* `memfiles `_
-  This module provides support for memory-mapped files (Posix's `mmap`)
-  on the different operating systems.
-
-* `os `_
-  Basic operating system facilities like retrieving environment variables,
-  reading command line arguments, working with directories, running shell
-  commands, etc.
-
-* `osproc `_
-  Module for process communication beyond `os.execShellCmd`.
-
-* `streams `_
-  This module provides a stream interface and two implementations thereof:
-  the `FileStream` and the `StringStream` which implement the stream
-  interface for Nim file objects (`File`) and strings. Other modules
-  may provide other implementations for this standard stream interface.
-
-* `terminal `_
-  This module contains a few procedures to control the *terminal*
-  (also called *console*). The implementation simply uses ANSI escape
-  sequences and does not depend on any other module.
-
-
-Math libraries
---------------
-
-* `complex `_
-  This module implements complex numbers and relevant mathematical operations.
-
-* `fenv `_
-  Floating-point environment. Handling of floating-point rounding and
-  exceptions (overflow, zero-divide, etc.).
-
-* `math `_
-  Mathematical operations like cosine, square root.
-
-* `mersenne `_
-  Mersenne twister random number generator.
-
-* `random `_
-  Fast and tiny random number generator.
-
-* `std/sysrand `_
-  Cryptographically secure pseudorandom number generator.
-
-* `rationals `_
-  This module implements rational numbers and relevant mathematical operations.
-
-* `stats `_
-  Statistical analysis
-
-* `std/sums `_
-  Accurate summation functions.
-
-
-Internet Protocols and Support
-------------------------------
-
-* `asyncdispatch `_
-  This module implements an asynchronous dispatcher for IO operations.
-
-* `asyncfile `_
-  This module implements asynchronous file reading and writing using
-  `asyncdispatch`.
-
-* `asyncftpclient `_
-  This module implements an asynchronous FTP client using the `asyncnet`
-  module.
-
-* `asynchttpserver `_
-  This module implements an asynchronous HTTP server using the `asyncnet`
-  module.
-
-* `asyncnet `_
-  This module implements asynchronous sockets based on the `asyncdispatch`
-  module.
-
-* `asyncstreams `_
-  This module provides `FutureStream` - a future that acts as a queue.
-
-* `cgi `_
-  This module implements helpers for CGI applications.
-
-* `cookies `_
-  This module contains helper procs for parsing and generating cookies.
-
-* `httpclient `_
-  This module implements a simple HTTP client which supports both synchronous
-  and asynchronous retrieval of web pages.
-
-* `mimetypes `_
-  This module implements a mimetypes database.
-
-* `nativesockets `_
-  This module implements a low-level sockets API.
-
-* `net `_
-  This module implements a high-level sockets API. It replaces the
-  `sockets` module.
-
-* `selectors `_
-  This module implements a selector API with backends specific to each OS.
-  Currently, epoll on Linux and select on other operating systems.
-
-* `smtp `_
-  This module implements a simple SMTP client.
-
-* `uri `_
-  This module provides functions for working with URIs.
-
-
-Threading
----------
-
-* `threadpool `_
-  Implements Nim's `spawn `_.
-
-
-Parsers
--------
-
-* `htmlparser `_
-  This module parses an HTML document and creates its XML tree representation.
-
-* `json `_
-  High-performance JSON parser.
-
-* `lexbase `_
-  This is a low-level module that implements an extremely efficient buffering
-  scheme for lexers and parsers. This is used by the diverse parsing modules.
-
-* `parsecfg `_
-  The `parsecfg` module implements a high-performance configuration file
-  parser. The configuration file's syntax is similar to the Windows `.ini`
-  format, but much more powerful, as it is not a line based parser. String
-  literals, raw string literals, and triple quote string literals are supported
-  as in the Nim programming language.
-
-* `parsecsv `_
-  The `parsecsv` module implements a simple high-performance CSV parser.
-
-* `parseopt `_
-  The `parseopt` module implements a command line option parser.
-
-* `parsesql `_
-  The `parsesql` module implements a simple high-performance SQL parser.
-
-* `parsexml `_
-  The `parsexml` module implements a simple high performance XML/HTML parser.
-  The only encoding that is supported is UTF-8. The parser has been designed
-  to be somewhat error-correcting, so that even some "wild HTML" found on the
-  web can be parsed with it.
-
-
-Docutils
---------
-
-* `packages/docutils/highlite `_
-  Source highlighter for programming or markup languages. Currently,
-  only a few languages are supported, other languages may be added.
-  The interface supports one language nested in another.
-
-* `packages/docutils/rst `_
-  This module implements a reStructuredText parser. A large subset
-  is implemented. Some features of the markdown wiki syntax are also supported.
-
-* `packages/docutils/rstast `_
-  This module implements an AST for the reStructuredText parser.
-
-* `packages/docutils/rstgen `_
-  This module implements a generator of HTML/Latex from reStructuredText.
-
-
-XML Processing
---------------
-
-* `xmltree `_
-  A simple XML tree. More efficient and simpler than the DOM. It also
-  contains a macro for XML/HTML code generation.
-
-* `xmlparser `_
-  This module parses an XML document and creates its XML tree representation.
-
-
-Generators
-----------
-
-* `htmlgen `_
-  This module implements a simple XML and HTML code
-  generator. Each commonly used HTML tag has a corresponding macro
-  that generates a string with its HTML representation.
-
-
-Hashing
--------
-
-* `base64 `_
-  This module implements a base64 encoder and decoder.
-
-* `hashes `_
-  This module implements efficient computations of hash values for diverse
-  Nim types.
-
-* `md5 `_
-  This module implements the MD5 checksum algorithm.
-
-* `oids `_
-  An OID is a global ID that consists of a timestamp,
-  a unique counter, and a random value. This combination should suffice to
-  produce a globally distributed unique ID. This implementation was extracted
-  from the Mongodb interface and it thus binary compatible with a Mongo OID.
-
-* `std/sha1 `_
-  This module implements a sha1 encoder and decoder.
-
-
-Miscellaneous
--------------
-
-* `browsers `_
-  This module implements procs for opening URLs with the user's default
-  browser.
-
-* `colors `_
-  This module implements color handling for Nim.
-
-* `coro `_
-  This module implements experimental coroutines in Nim.
-
-* `logging `_
-  This module implements a simple logger.
-
-* `segfaults `_
-  Turns access violations or segfaults into a `NilAccessDefect` exception.
-
-* `sugar `_
-  This module implements nice syntactic sugar based on Nim's macro system.
-
-* `unittest `_
-  Implements a Unit testing DSL.
-
-* `std/varints `_
-  Decode variable-length integers that are compatible with SQLite.
-
-
-Modules for JS backend
-----------------------
-
-* `asyncjs `_
-  Types and macros for writing asynchronous procedures in JavaScript.
-
-* `dom `_
-  Declaration of the Document Object Model for the JS backend.
-
-* `jsconsole `_
-  Wrapper for the `console` object.
-
-* `jscore `_
-  The wrapper of core JavaScript functions. For most purposes, you should be using
-  the `math`, `json`, and `times` stdlib modules instead of this module.
-
-* `jsffi `_
-  Types and macros for easier interaction with JavaScript.
-
-
-Impure libraries
-================
-
-Regular expressions
--------------------
-
-* `re `_
-  This module contains procedures and operators for handling regular
-  expressions. The current implementation uses PCRE.
-
-
-Database support
-----------------
-
-* `db_postgres `_
-  A higher level PostgreSQL database wrapper. The same interface is implemented
-  for other databases too.
-
-* `db_mysql `_
-  A higher level MySQL database wrapper. The same interface is implemented
-  for other databases too.
-
-* `db_sqlite `_
-  A higher level SQLite database wrapper. The same interface is implemented
-  for other databases too.
-
-
-Wrappers
-========
-
-The generated HTML for some of these wrappers is so huge that it is
-not contained in the distribution. You can then find them on the website.
-
-
-Windows-specific
-----------------
-
-* `winlean `_
-  Contains a wrapper for a small subset of the Win32 API.
-* `registry `_
-  Windows registry support.
-
-
-UNIX specific
--------------
-
-* `posix `_
-  Contains a wrapper for the POSIX standard.
-* `posix_utils `_
-  Contains helpers for the POSIX standard or specialized for Linux and BSDs.
-
-
-Regular expressions
--------------------
-
-* `pcre `_
-  Wrapper for the PCRE library.
-
-
-Database support
-----------------
-
-* `postgres `_
-  Contains a wrapper for the PostgreSQL API.
-* `mysql `_
-  Contains a wrapper for the mySQL API.
-* `sqlite3 `_
-  Contains a wrapper for SQLite 3 API.
-* `odbcsql `_
-  interface to the ODBC driver.
-
-
-Network Programming and Internet Protocols
-------------------------------------------
-
-* `openssl `_
-  Wrapper for OpenSSL.
diff --git a/doc/manual.rst b/doc/manual.md
similarity index 73%
rename from doc/manual.rst
rename to doc/manual.md
index 1bb47f28b7..bb7b603a85 100644
--- a/doc/manual.rst
+++ b/doc/manual.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 ==========
 Nim Manual
 ==========
@@ -7,12 +5,14 @@ Nim Manual
 :Authors: Andreas Rumpf, Zahary Karadjov
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
-  "Complexity" seems to be a lot like "energy": you can transfer it from the
-  end-user to one/some of the other players, but the total amount seems to remain
-  pretty much constant for a given task. -- Ran
+> "Complexity" seems to be a lot like "energy": you can transfer it from the
+> end-user to one/some of the other players, but the total amount seems to remain
+> pretty much constant for a given task. -- Ran
 
 
 About this document
@@ -22,16 +22,16 @@ About this document
 precise wording. This manual is constantly evolving into a proper specification.
 
 **Note**: The experimental features of Nim are
-covered `here `_.
+covered [here](manual_experimental.html).
 
 **Note**: Assignments, moves, and destruction are specified in
-the `destructors `_ document.
+the [destructors](destructors.html) document.
 
 
 This document describes the lexis, the syntax, and the semantics of the Nim language.
 
 To learn how to compile Nim programs and generate documentation see
-`Compiler User Guide `_ and `DocGen Tools Guide `_.
+the [Compiler User Guide](nimc.html) and the [DocGen Tools Guide](docgen.html).
 
 The language constructs are explained using an extended BNF, in which `(a)*`
 means 0 or more `a`'s, `a+` means 1 or more `a`'s, and `(a)?` means an
@@ -47,16 +47,16 @@ is not ambiguous.
 
 Non-terminals start with a lowercase letter, abstract terminal symbols are in
 UPPERCASE. Verbatim terminal symbols (including keywords) are quoted
-with `'`. An example::
+with `'`. An example:
 
-  ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?
+    ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?
 
 The binary `^*` operator is used as a shorthand for 0 or more occurrences
 separated by its second argument; likewise `^+` means 1 or more
 occurrences: `a ^+ b` is short for `a (b a)*`
-and `a ^* b` is short for `(a (b a)*)?`. Example::
+and `a ^* b` is short for `(a (b a)*)?`. Example:
 
-  arrayConstructor = '[' expr ^* ',' ']'
+    arrayConstructor = '[' expr ^* ',' ']'
 
 Other parts of Nim, like scoping rules or runtime semantics, are
 described informally.
@@ -91,11 +91,11 @@ The nature of this executable depends on the compiler implementation; it may,
 for example, be a native binary or JavaScript source code.
 
 In a typical Nim program, most of the code is compiled into the executable.
-However, some of the code may be executed at
+However, some code may be executed at
 `compile-time`:idx:. This can include constant expressions, macro definitions,
 and Nim procedures used by macro definitions. Most of the Nim language is
-supported at compile-time, but there are some restrictions -- see `Restrictions
-on Compile-Time Execution <#restrictions-on-compileminustime-execution>`_ for
+supported at compile-time, but there are some restrictions -- see [Restrictions
+on Compile-Time Execution] for
 details. We use the term `runtime`:idx: to cover both compile-time execution
 and code execution in the executable.
 
@@ -111,25 +111,26 @@ A `panic`:idx: is an error that the implementation detects
 and reports at runtime. The method for reporting such errors is via
 *raising exceptions* or *dying with a fatal error*. However, the implementation
 provides a means to disable these `runtime checks`:idx:. See the section
-pragmas_ for details.
+[Pragmas] for details.
 
 Whether a panic results in an exception or in a fatal error is
-implementation specific. Thus the following program is invalid; even though the
+implementation specific. Thus, the following program is invalid; even though the
 code purports to catch the `IndexDefect` from an out-of-bounds array access, the
 compiler may instead choose to allow the program to die with a fatal error.
 
-.. code-block:: nim
+  ```nim
   var a: array[0..1, char]
   let i = 5
   try:
     a[i] = 'N'
   except IndexDefect:
     echo "invalid index"
+  ```
 
-The current implementation allows to switch between these different behaviors
-via `--panics:on|off`. When panics are turned on, the program dies with a
+The current implementation allows switching between these different behaviors
+via `--panics:on|off`:option:. When panics are turned on, the program dies with a
 panic, if they are turned off the runtime errors are turned into
-exceptions. The benefit of `--panics:on` is that it produces smaller binary
+exceptions. The benefit of `--panics:on`:option: is that it produces smaller binary
 code and the compiler has more freedom to optimize the code.
 
 An `unchecked runtime error`:idx: is an error that is not guaranteed to be
@@ -189,16 +190,16 @@ is another pseudo terminal that describes the *action* of popping a value
 from the stack, `IND{>}` then implies to push onto the stack.
 
 With this notation we can now easily define the core of the grammar: A block of
-statements (simplified example)::
+statements (simplified example):
 
-  ifStmt = 'if' expr ':' stmt
-           (IND{=} 'elif' expr ':' stmt)*
-           (IND{=} 'else' ':' stmt)?
+    ifStmt = 'if' expr ':' stmt
+             (IND{=} 'elif' expr ':' stmt)*
+             (IND{=} 'else' ':' stmt)?
 
-  simpleStmt = ifStmt / ...
+    simpleStmt = ifStmt / ...
 
-  stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
-       / simpleStmt                 # or a simple statement
+    stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
+         / simpleStmt                 # or a simple statement
 
 
 
@@ -214,15 +215,16 @@ no other tokens between it and the preceding one, it does not start a new
 comment:
 
 
-.. code-block:: nim
+  ```nim
   i = 0     # This is a single comment over multiple lines.
-    # The scanner merges these two pieces.
+    # The lexer merges these two pieces.
     # The comment continues here.
+  ```
 
 
 `Documentation comments`:idx: are comments that start with two `##`.
 Documentation comments are tokens; they are only allowed at certain places in
-the input file as they belong to the syntax tree!
+the input file as they belong to the syntax tree.
 
 
 Multiline comments
@@ -231,26 +233,41 @@ Multiline comments
 Starting with version 0.13.0 of the language Nim supports multiline comments.
 They look like:
 
-.. code-block:: nim
+  ```nim
   #[Comment here.
   Multiple lines
   are not a problem.]#
+  ```
 
 Multiline comments support nesting:
 
-.. code-block:: nim
+  ```nim
   #[  #[ Multiline comment in already
      commented out code. ]#
   proc p[T](x: T) = discard
   ]#
+  ```
 
 Multiline documentation comments also exist and support nesting too:
 
-.. code-block:: nim
+  ```nim
   proc foo =
     ##[Long documentation comment
        here.
     ]##
+  ```
+
+You can also use the [discard statement](#statements-and-expressions-discard-statement) together with
+[triple quoted string literals](#lexical-analysis-triple-quoted-string-literals) to create multiline comments:
+
+  ```nim
+  discard """ You can have any Nim code text commented
+  out inside this with no indentation restrictions.
+        yes("May I ask a pointless question?") """
+  ```
+
+This was how multiline comments were done before version 0.13.0,
+and it is used to provide specifications to [testament](testament.html#writing-unit-tests) test framework.
 
 
 Identifiers & Keywords
@@ -261,11 +278,13 @@ and underscores, with the following restrictions:
 
 * begins with a letter
 * does not end with an underscore `_`
-* two immediate following underscores `__` are not allowed::
+* two immediate following underscores `__` are not allowed:
 
+  ```
   letter ::= 'A'..'Z' | 'a'..'z' | '\x80'..'\xff'
   digit ::= '0'..'9'
   IDENTIFIER ::= letter ( ['_'] (letter | digit) )*
+  ```
 
 Currently, any Unicode character with an ordinal value > 127 (non-ASCII) is
 classified as a `letter` and may thus be part of an identifier but later
@@ -274,8 +293,8 @@ operator characters instead.
 
 The following keywords are reserved and cannot be used as identifiers:
 
-.. code-block:: nim
-   :file: keywords.txt
+  ```nim file="keywords.txt"
+  ```
 
 Some keywords are unused; they are reserved for future developments of the
 language.
@@ -286,10 +305,11 @@ Identifier equality
 
 Two identifiers are considered equal if the following algorithm returns true:
 
-.. code-block:: nim
+  ```nim
   proc sameIdentifier(a, b: string): bool =
     a[0] == b[0] and
       a.replace("_", "").toLowerAscii == b.replace("_", "").toLowerAscii
+  ```
 
 That means only the first letters are compared in a case-sensitive manner. Other
 letters are compared case-insensitively within the ASCII range and underscores are ignored.
@@ -315,23 +335,24 @@ it was not case-sensitive and underscores were ignored and there was not even a
 distinction between `foo` and `Foo`.
 
 
-Stropping
----------
+Keywords as identifiers
+-----------------------
 
 If a keyword is enclosed in backticks it loses its keyword property and becomes an ordinary identifier.
 
 Examples
 
-.. code-block:: nim
+  ```nim
   var `var` = "Hello Stropping"
+  ```
 
-.. code-block:: nim
-  type Type = object
-    `int`: int
+  ```nim
+  type Obj = object
+    `type`: int
 
-  let `object` = Type(`int`: 9)
-  assert `object` is Type
-  assert `object`.`int` == 9
+  let `object` = Obj(`type`: 9)
+  assert `object` is Obj
+  assert `object`.`type` == 9
 
   var `var` = 42
   let `let` = 8
@@ -339,6 +360,7 @@ Examples
 
   const `assert` = true
   assert `assert`
+  ```
 
 
 String literals
@@ -362,7 +384,7 @@ contain the following `escape sequences`:idx:\ :
   ``\\``                   `backslash`:idx:
   ``\"``                   `quotation mark`:idx:
   ``\'``                   `apostrophe`:idx:
-  ``\\`` '0'..'9'+         `character with decimal value d`:idx:;
+  ``\`` '0'..'9'+          `character with decimal value d`:idx:;
                            all decimal digits directly
                            following are used for the character
   ``\a``                   `alert`:idx:
@@ -378,7 +400,7 @@ contain the following `escape sequences`:idx:\ :
 ==================         ===================================================
 
 
-Strings in Nim may contain any 8-bit value, even embedded zeros. However
+Strings in Nim may contain any 8-bit value, even embedded zeros. However,
 some operations may interpret the first binary zero as a terminator.
 
 
@@ -395,12 +417,13 @@ be whitespace between the opening `"""` and the newline),
 the newline (and the preceding whitespace) is not included in the string. The
 ending of the string literal is defined by the pattern `"""[^"]`, so this:
 
-.. code-block:: nim
+  ```nim
   """"long string within quotes""""
+  ```
 
-Produces::
+Produces:
 
-  "long string within quotes"
+    "long string within quotes"
 
 
 Raw string literals
@@ -413,19 +436,19 @@ letter `r` (or `R`) and are delimited by matching double quotes (just
 like ordinary string literals) and do not interpret the escape sequences.
 This is especially convenient for regular expressions or Windows paths:
 
-.. code-block:: nim
-
+  ```nim
   var f = openFile(r"C:\texts\text.txt") # a raw string, so ``\t`` is no tab
+  ```
 
 To produce a single `"` within a raw string literal, it has to be doubled:
 
-.. code-block:: nim
-
+  ```nim
   r"a""b"
+  ```
 
-Produces::
+Produces:
 
-  a"b
+    a"b
 
 `r""""` is not possible with this notation, because the three leading
 quotes introduce a triple quoted string literal. `r"""` is the same
@@ -442,7 +465,7 @@ Terminal symbols in the grammar: `GENERALIZED_STR_LIT`,
 The construct `identifier"string literal"` (without whitespace between the
 identifier and the opening quotation mark) is a
 generalized raw string literal. It is a shortcut for the construct
-`identifier(r"string literal")`, so it denotes a procedure call with a
+`identifier(r"string literal")`, so it denotes a routine call with a
 raw string literal as its only argument. Generalized raw string literals
 are especially convenient for embedding mini languages directly into Nim
 (for example regular expressions).
@@ -457,8 +480,8 @@ Character literals
 Character literals are enclosed in single quotes `''` and can contain the
 same escape sequences as strings - with one exception: the platform
 dependent `newline`:idx: (``\p``)
-is not allowed as it may be wider than one character (often it is the pair
-CR/LF for example).  Here are the valid `escape sequences`:idx: for character
+is not allowed as it may be wider than one character (it can be the pair
+CR/LF). Here are the valid `escape sequences`:idx: for character
 literals:
 
 ==================         ===================================================
@@ -472,7 +495,7 @@ literals:
   ``\\``                   `backslash`:idx:
   ``\"``                   `quotation mark`:idx:
   ``\'``                   `apostrophe`:idx:
-  ``\\`` '0'..'9'+         `character with decimal value d`:idx:;
+  ``\`` '0'..'9'+          `character with decimal value d`:idx:;
                            all decimal digits directly
                            following are used for the character
   ``\a``                   `alert`:idx:
@@ -482,71 +505,117 @@ literals:
                            exactly two hex digits are allowed
 ==================         ===================================================
 
-A character is not a Unicode character but a single byte. The reason for this
-is efficiency: for the overwhelming majority of use-cases, the resulting
-programs will still handle UTF-8 properly as UTF-8 was specially designed for
-this. Another reason is that Nim can thus support `array[char, int]` or
-`set[char]` efficiently as many algorithms rely on this feature.  The `Rune`
-type is used for Unicode characters, it can represent any Unicode character.
-`Rune` is declared in the `unicode module `_.
+A character is not a Unicode character but a single byte.
+
+Rationale: It enables the efficient support of `array[char, int]` or
+`set[char]`.
+
+The `Rune` type can represent any Unicode character.
+`Rune` is declared in the [unicode module](unicode.html).
+
+A character literal that does not end in `'` is interpreted as `'` if there
+is a preceding backtick token. There must be no whitespace between the preceding
+backtick token and the character literal. This special case ensures that a declaration
+like ``proc `'customLiteral`(s: string)`` is valid. ``proc `'customLiteral`(s: string)``
+is the same as ``proc `'\''customLiteral`(s: string)``.
+
+See also [custom numeric literals].
 
 
-Numerical constants
--------------------
+Numeric literals
+----------------
 
-Numerical constants are of a single type and have the form::
+Numeric literals have the form:
 
-  hexdigit = digit | 'A'..'F' | 'a'..'f'
-  octdigit = '0'..'7'
-  bindigit = '0'..'1'
-  HEX_LIT = '0' ('x' | 'X' ) hexdigit ( ['_'] hexdigit )*
-  DEC_LIT = digit ( ['_'] digit )*
-  OCT_LIT = '0' 'o' octdigit ( ['_'] octdigit )*
-  BIN_LIT = '0' ('b' | 'B' ) bindigit ( ['_'] bindigit )*
+    hexdigit = digit | 'A'..'F' | 'a'..'f'
+    octdigit = '0'..'7'
+    bindigit = '0'..'1'
+    unary_minus = '-' # See the section about unary minus
+    HEX_LIT = unary_minus? '0' ('x' | 'X' ) hexdigit ( ['_'] hexdigit )*
+    DEC_LIT = unary_minus? digit ( ['_'] digit )*
+    OCT_LIT = unary_minus? '0' 'o' octdigit ( ['_'] octdigit )*
+    BIN_LIT = unary_minus? '0' ('b' | 'B' ) bindigit ( ['_'] bindigit )*
 
-  INT_LIT = HEX_LIT
-          | DEC_LIT
-          | OCT_LIT
-          | BIN_LIT
+    INT_LIT = HEX_LIT
+            | DEC_LIT
+            | OCT_LIT
+            | BIN_LIT
 
-  INT8_LIT = INT_LIT ['\''] ('i' | 'I') '8'
-  INT16_LIT = INT_LIT ['\''] ('i' | 'I') '16'
-  INT32_LIT = INT_LIT ['\''] ('i' | 'I') '32'
-  INT64_LIT = INT_LIT ['\''] ('i' | 'I') '64'
+    INT8_LIT = INT_LIT ['\''] ('i' | 'I') '8'
+    INT16_LIT = INT_LIT ['\''] ('i' | 'I') '16'
+    INT32_LIT = INT_LIT ['\''] ('i' | 'I') '32'
+    INT64_LIT = INT_LIT ['\''] ('i' | 'I') '64'
 
-  UINT_LIT = INT_LIT ['\''] ('u' | 'U')
-  UINT8_LIT = INT_LIT ['\''] ('u' | 'U') '8'
-  UINT16_LIT = INT_LIT ['\''] ('u' | 'U') '16'
-  UINT32_LIT = INT_LIT ['\''] ('u' | 'U') '32'
-  UINT64_LIT = INT_LIT ['\''] ('u' | 'U') '64'
+    UINT_LIT = INT_LIT ['\''] ('u' | 'U')
+    UINT8_LIT = INT_LIT ['\''] ('u' | 'U') '8'
+    UINT16_LIT = INT_LIT ['\''] ('u' | 'U') '16'
+    UINT32_LIT = INT_LIT ['\''] ('u' | 'U') '32'
+    UINT64_LIT = INT_LIT ['\''] ('u' | 'U') '64'
 
-  exponent = ('e' | 'E' ) ['+' | '-'] digit ( ['_'] digit )*
-  FLOAT_LIT = digit (['_'] digit)* (('.' digit (['_'] digit)* [exponent]) |exponent)
-  FLOAT32_SUFFIX = ('f' | 'F') ['32']
-  FLOAT32_LIT = HEX_LIT '\'' FLOAT32_SUFFIX
-              | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT32_SUFFIX
-  FLOAT64_SUFFIX = ( ('f' | 'F') '64' ) | 'd' | 'D'
-  FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX
-              | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX
+    exponent = ('e' | 'E' ) ['+' | '-'] digit ( ['_'] digit )*
+    FLOAT_LIT = unary_minus? digit (['_'] digit)* (('.' digit (['_'] digit)* [exponent]) |exponent)
+    FLOAT32_SUFFIX = ('f' | 'F') ['32']
+    FLOAT32_LIT = HEX_LIT '\'' FLOAT32_SUFFIX
+                | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT32_SUFFIX
+    FLOAT64_SUFFIX = ( ('f' | 'F') '64' ) | 'd' | 'D'
+    FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX
+                | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX
+
+    CUSTOM_NUMERIC_LIT = (FLOAT_LIT | INT_LIT) '\'' CUSTOM_NUMERIC_SUFFIX
+
+    # CUSTOM_NUMERIC_SUFFIX is any Nim identifier that is not
+    # a pre-defined type suffix.
 
 
-As can be seen in the productions, numerical constants can contain underscores
+As can be seen in the productions, numeric literals can contain underscores
 for readability. Integer and floating-point literals may be given in decimal (no
 prefix), binary (prefix `0b`), octal (prefix `0o`), and hexadecimal
 (prefix `0x`) notation.
 
-There exists a literal for each numerical type that is
-defined. The suffix starting with an apostrophe ('\'') is called a
+The fact that the unary minus `-` in a number literal like `-1` is considered
+to be part of the literal is a late addition to the language. The rationale is that
+an expression `-128'i8` should be valid and without this special case, this would
+be impossible -- `128` is not a valid `int8` value, only `-128` is.
+
+For the `unary_minus` rule there are further restrictions that are not covered
+in the formal grammar. For `-` to be part of the number literal the immediately
+preceding character has to be in the
+set `{' ', '\t', '\n', '\r', ',', ';', '(', '[', '{'}`. This set was designed to
+cover most cases in a natural manner.
+
+In the following examples, `-1` is a single token:
+
+  ```nim
+  echo -1
+  echo(-1)
+  echo [-1]
+  echo 3,-1
+
+  "abc";-1
+  ```
+
+In the following examples, `-1` is parsed as two separate tokens
+(as `-`:tok: `1`:tok:):
+
+  ```nim
+  echo x-1
+  echo (int)-1
+  echo [a]-1
+  "abc"-1
+  ```
+
+
+The suffix starting with an apostrophe ('\'') is called a
 `type suffix`:idx:. Literals without a type suffix are of an integer type
 unless the literal contains a dot or `E|e` in which case it is of
 type `float`. This integer type is `int` if the literal is in the range
-`low(i32)..high(i32)`, otherwise it is `int64`.
+`low(int32)..high(int32)`, otherwise it is `int64`.
 For notational convenience, the apostrophe of a type suffix
 is optional if it is not ambiguous (only hexadecimal floating-point literals
 with a type suffix can be ambiguous).
 
 
-The type suffixes are:
+The pre-defined type suffixes are:
 
 =================    =========================
   Type Suffix        Resulting type of literal
@@ -571,18 +640,53 @@ notation:
 `0B0_10001110100_0000101001000111101011101111111011000101001101001001'f64`
 is approximately 1.72826e35 according to the IEEE floating-point standard.
 
-Literals are bounds checked so that they fit the datatype. Non-base-10
-literals are used mainly for flags and bit pattern representations, therefore
-bounds checking is done on bit width, not value range. If the literal fits in
-the bit width of the datatype, it is accepted.
+Literals must match the datatype, for example, `333'i8` is an invalid literal.
+Non-base-10 literals are used mainly for flags and bit pattern representations,
+therefore the checking is done on bit width and not on value range.
 Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1
 instead of causing an overflow error.
 
+
+### Custom numeric literals
+
+If the suffix is not predefined, then the suffix is assumed to be a call
+to a proc, template, macro or other callable identifier that is passed the
+string containing the literal. The callable identifier needs to be declared
+with a special ``'`` prefix:
+
+  ```nim
+  import std/strutils
+  type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble"
+  proc `'u4`(n: string): u4 =
+    # The leading ' is required.
+    result = (parseInt(n) and 0x0F).u4
+
+  var x = 5'u4
+  ```
+
+More formally, a custom numeric literal `123'custom` is transformed
+to r"123".`'custom` in the parsing step. There is no AST node kind that
+corresponds to this transformation. The transformation naturally handles
+the case that additional parameters are passed to the callee:
+
+  ```nim
+  import std/strutils
+  type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble"
+  proc `'u4`(n: string; moreData: int): u4 =
+    result = (parseInt(n) and 0x0F).u4
+
+  var x = 5'u4(123)
+  ```
+
+Custom numeric literals are covered by the grammar rule named `CUSTOM_NUMERIC_LIT`.
+A custom numeric literal is a single token.
+
+
 Operators
 ---------
 
 Nim allows user defined operators. An operator is any combination of the
-following characters::
+following characters:
 
        =     +     -     *     /     <     >
        @     $     ~     &     %     |
@@ -594,7 +698,7 @@ defined here.)
 These keywords are also operators:
 `and or not xor shl shr div mod in notin is isnot of as from`.
 
-`.`:tok: `=`:tok:, `:`:tok:, `::`:tok: are not available as general operators; they
+`.`:tok:, `=`:tok:, `:`:tok:, `::`:tok: are not available as general operators; they
 are used for other notational purposes.
 
 `*:` is as a special case treated as the two tokens `*`:tok: and `:`:tok:
@@ -603,26 +707,40 @@ are used for other notational purposes.
 The `not` keyword is always a unary operator, `a not b` is parsed
 as `a(not b)`, not as `(a) not (b)`.
 
+Unicode Operators
+-----------------
+
+These Unicode operators are also parsed as operators:
+
+    ∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓   # same priority as * (multiplication)
+    ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔             # same priority as + (addition)
+
+
+Unicode operators can be combined with non-Unicode operator
+symbols. The usual precedence extensions then apply, for example, `⊠=` is an
+assignment like operator just like `*=` is.
+
+No Unicode normalization step is performed.
+
 
 Other tokens
 ------------
 
-The following strings denote other tokens::
+The following strings denote other tokens:
 
     `   (    )     {    }     [    ]    ,  ;   [.    .]  {.   .}  (.  .)  [:
 
 
 The `slice`:idx: operator `..`:tok: takes precedence over other tokens that
-contain a dot: `{..}`:tok: are the three tokens `{`:tok:, `..`:tok:, `}`:tok:
+contain a dot: `{..}` are the three tokens `{`:tok:, `..`:tok:, `}`:tok:
 and not the two tokens `{.`:tok:, `.}`:tok:.
 
 
-
 Syntax
 ======
 
 This section lists Nim's standard syntax. How the parser handles
-the indentation is already described in the `Lexical Analysis`_ section.
+the indentation is already described in the [Lexical Analysis] section.
 
 Nim allows user-definable operators.
 Binary operators have 11 different levels of precedence.
@@ -635,12 +753,13 @@ Associativity
 Binary operators whose first character is `^` are right-associative, all
 other binary operators are left-associative.
 
-.. code-block:: nim
+  ```nim
   proc `^/`(x, y: float): float =
     # a right-associative division operator
     result = x / y
   echo 12 ^/ 4 ^/ 8 # 24.0 (4 / 8 = 0.5, then 12 / 0.5 = 24.0)
   echo 12  / 4  / 8 # 0.375 (12 / 4 = 3.0, then 3 / 8 = 0.375)
+  ```
 
 Precedence
 ----------
@@ -665,11 +784,12 @@ has the second-lowest precedence.
 
 Otherwise, precedence is determined by the first character.
 
+
 ================  =======================================================  ==================  ===============
 Precedence level    Operators                                              First character     Terminal symbol
 ================  =======================================================  ==================  ===============
  10 (highest)                                                              `$  ^`              OP10
-  9               `*    /    div   mod   shl  shr  %`                      ``*  %  \  /``        OP9
+  9               `*    /    div   mod   shl  shr  %`                      `*  %  \  /`        OP9
   8               `+    -`                                                 `+  -  ~  |`        OP8
   7               `&`                                                      `&`                 OP7
   6               `..`                                                     `.`                 OP6
@@ -685,20 +805,31 @@ Precedence level    Operators                                              First
 Whether an operator is used as a prefix operator is also affected by preceding
 whitespace (this parsing change was introduced with version 0.13.0):
 
-.. code-block:: nim
+  ```nim
   echo $foo
   # is parsed as
   echo($foo)
+  ```
 
 
 Spacing also determines whether `(a, b)` is parsed as an argument list
 of a call or whether it is parsed as a tuple constructor:
 
-.. code-block:: nim
+  ```nim
   echo(1, 2) # pass 1 and 2 to echo
+  ```
 
-.. code-block:: nim
+  ```nim
   echo (1, 2) # pass the tuple (1, 2) to echo
+  ```
+
+Dot-like operators
+------------------
+
+Terminal symbol in the grammar: `DOTLIKEOP`.
+
+Dot-like operators are operators starting with `.`, but not with `..`, for e.g. `.?`;
+they have the same precedence as `.`, so that `a.?b.c` is parsed as `(a.?b).c` instead of `a.?(b.c)`.
 
 
 Grammar
@@ -717,9 +848,7 @@ Order of evaluation
 Order of evaluation is strictly left-to-right, inside-out as it is typical for most others
 imperative programming languages:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   var s = ""
 
   proc p(arg: int): int =
@@ -729,14 +858,13 @@ imperative programming languages:
   discard p(p(1) + p(2))
 
   doAssert s == "123"
+  ```
 
 
 Assignments are not special, the left-hand-side expression is evaluated before the
 right-hand side:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   var v = 0
   proc getI(): int =
     result = v
@@ -754,6 +882,7 @@ right-hand side:
   someCopy(b[getI()], getI())
 
   doAssert b == [1, 0, 0]
+  ```
 
 
 Rationale: Consistency with overloaded assignment or assignment-like operations,
@@ -764,9 +893,7 @@ However, the concept of "order of evaluation" is only applicable after the code
 was normalized: The normalization involves template expansions and argument
 reorderings that have been passed to named parameters:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   var s = ""
 
   proc p(): int =
@@ -793,6 +920,7 @@ reorderings that have been passed to named parameters:
   construct(second = q(), first = p())
 
   doAssert s == "qppqpq"
+  ```
 
 
 Rationale: This is far easier to implement than hypothetical alternatives.
@@ -827,37 +955,37 @@ constant expressions that may be surprising to those coming from other
 statically typed languages. For example, the following code echoes the beginning
 of the Fibonacci series **at compile-time**. (This is a demonstration of
 flexibility in defining constants, not a recommended style for solving this
-problem!)
+problem.)
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   import std/strformat
 
-  var fib_n {.compileTime.}: int
-  var fib_prev {.compileTime.}: int
-  var fib_prev_prev {.compileTime.}: int
+  var fibN {.compileTime.}: int
+  var fibPrev {.compileTime.}: int
+  var fibPrevPrev {.compileTime.}: int
 
-  proc next_fib(): int =
-    result = if fib_n < 2:
-      fib_n
+  proc nextFib(): int =
+    result = if fibN < 2:
+      fibN
     else:
-      fib_prev_prev + fib_prev
-    inc(fib_n)
-    fib_prev_prev = fib_prev
-    fib_prev = result
+      fibPrevPrev + fibPrev
+    inc(fibN)
+    fibPrevPrev = fibPrev
+    fibPrev = result
 
-  const f0 = next_fib()
-  const f1 = next_fib()
+  const f0 = nextFib()
+  const f1 = nextFib()
 
-  const display_fib = block:
-    const f2 = next_fib()
+  const displayFib = block:
+    const f2 = nextFib()
     var result = fmt"Fibonacci sequence: {f0}, {f1}, {f2}"
     for i in 3..12:
-      add(result, fmt", {next_fib()}")
+      add(result, fmt", {nextFib()}")
     result
 
   static:
-    echo display_fib
+    echo displayFib
+  ```
 
 
 Restrictions on Compile-Time Execution
@@ -904,10 +1032,12 @@ Ordinal types have the following characteristics:
 - Ordinal types are countable and ordered. This property allows the operation
   of functions such as `inc`, `ord`, and `dec` on ordinal types to
   be defined.
-- Ordinal values have the smallest possible value. Trying to count further
-  down than the smallest value produces a panic or a static error.
-- Ordinal values have the largest possible value. Trying to count further
-  than the largest value produces a panic or a static error.
+- Ordinal types have a smallest possible value, accessible with `low(type)`.
+  Trying to count further down than the smallest value produces a panic or
+  a static error.
+- Ordinal types have a largest possible value, accessible with `high(type)`.
+  Trying to count further up than the largest value produces a panic or
+  a static error.
 
 Integers, bool, characters, and enumeration types (and subranges of these
 types) belong to ordinal types.
@@ -920,22 +1050,24 @@ Pre-defined integer types
 These integer types are pre-defined:
 
 `int`
-  the generic signed integer type; its size is platform-dependent and has the
+: the generic signed integer type; its size is platform-dependent and has the
   same size as a pointer. This type should be used in general. An integer
   literal that has no type suffix is of this type if it is in the range
   `low(int32)..high(int32)` otherwise the literal's type is `int64`.
 
-intXX
-  additional signed integer types of XX bits use this naming scheme
+`int`\ XX
+: additional signed integer types of XX bits use this naming scheme
   (example: int16 is a 16-bit wide integer).
   The current implementation supports `int8`, `int16`, `int32`, `int64`.
   Literals of these types have the suffix 'iXX.
 
 `uint`
-  the generic `unsigned integer`:idx: type; its size is platform-dependent and has the same size as a pointer. An integer literal with the type suffix `'u` is of this type.
+: the generic `unsigned integer`:idx: type; its size is platform-dependent and
+  has the same size as a pointer. An integer literal with the type
+  suffix `'u` is of this type.
 
-uintXX
-  additional unsigned integer types of XX bits use this naming scheme
+`uint`\ XX
+: additional unsigned integer types of XX bits use this naming scheme
   (example: uint16 is a 16-bit wide unsigned integer).
   The current implementation supports `uint8`, `uint16`, `uint32`,
   `uint64`. Literals of these types have the suffix 'uXX.
@@ -954,24 +1086,13 @@ the `%` suffix as convention:
 ======================   ======================================================
 operation                meaning
 ======================   ======================================================
-`a +% b`               unsigned integer addition
-`a -% b`               unsigned integer subtraction
-`a *% b`               unsigned integer multiplication
-`a /% b`               unsigned integer division
-`a %% b`               unsigned integer modulo operation
-`a <% b`               treat `a` and `b` as unsigned and compare
-`a <=% b`              treat `a` and `b` as unsigned and compare
-`ze(a)`                extends the bits of `a` with zeros until it has the
-                         width of the `int` type
-`toU8(a)`              treats `a` as unsigned and converts it to an
-                         unsigned integer of 8 bits (but still the
-                         `int8` type)
-`toU16(a)`             treats `a` as unsigned and converts it to an
-                         unsigned integer of 16 bits (but still the
-                         `int16` type)
-`toU32(a)`             treats `a` as unsigned and converts it to an
-                         unsigned integer of 32 bits (but still the
-                         `int32` type)
+`a +% b`                 unsigned integer addition
+`a -% b`                 unsigned integer subtraction
+`a *% b`                 unsigned integer multiplication
+`a /% b`                 unsigned integer division
+`a %% b`                 unsigned integer modulo operation
+`a <% b`                 treat `a` and `b` as unsigned and compare
+`a <=% b`                treat `a` and `b` as unsigned and compare
 ======================   ======================================================
 
 `Automatic type conversion`:idx: is performed in expressions where different
@@ -982,20 +1103,20 @@ example `int32 -> int16`). A `widening type conversion`:idx: converts a
 smaller type to a larger type (for example `int16 -> int32`). In Nim only
 widening type conversions are *implicit*:
 
-.. code-block:: nim
+  ```nim
   var myInt16 = 5i16
   var myInt: int
   myInt16 + 34     # of type `int16`
   myInt16 + myInt  # of type `int`
   myInt16 + 2i32   # of type `int32`
+  ```
 
 However, `int` literals are implicitly convertible to a smaller integer type
 if the literal's value fits this smaller type and such a conversion is less
 expensive than other implicit conversions, so `myInt16 + 34` produces
 an `int16` result.
 
-For further details, see `Convertible relation
-<#type-relations-convertible-relation>`_.
+For further details, see [Convertible relation].
 
 
 Subrange types
@@ -1004,10 +1125,12 @@ A subrange type is a range of values from an ordinal or floating-point type (the
 type). To define a subrange type, one must specify its limiting values -- the
 lowest and highest value of the type. For example:
 
-.. code-block:: nim
+  ```nim
   type
     Subrange = range[0..5]
     PositiveFloat = range[0.0..Inf]
+    Positive* = range[1..high(int)] # as defined in `system`
+  ```
 
 
 `Subrange` is a subrange of an integer which can only hold the values 0
@@ -1028,22 +1151,21 @@ Pre-defined floating-point types
 The following floating-point types are pre-defined:
 
 `float`
-  the generic floating-point type; its size used to be platform-dependent,
+: the generic floating-point type; its size used to be platform-dependent,
   but now it is always mapped to `float64`.
   This type should be used in general.
 
-floatXX
-  an implementation may define additional floating-point types of XX bits using
-  this naming scheme (example: float64 is a 64-bit wide float). The current
+`float`\ XX
+: an implementation may define additional floating-point types of XX bits using
+  this naming scheme (example: `float64` is a 64-bit wide float). The current
   implementation supports `float32` and `float64`. Literals of these types
   have the suffix 'fXX.
 
 
-Automatic type conversion in expressions with different kinds
-of floating-point types is performed: See `Convertible relation`_ for further
-details. Arithmetic performed on floating-point types follows the IEEE
-standard. Integer types are not converted to floating-point types automatically
-and vice versa.
+Automatic type conversion in expressions with different kinds of floating-point
+types is performed: See [Convertible relation] for further details. Arithmetic
+performed on floating-point types follows the IEEE standard. Integer types are
+not converted to floating-point types automatically and vice versa.
 
 The IEEE standard defines five types of floating-point exceptions:
 
@@ -1067,12 +1189,13 @@ These exceptions inherit from the `FloatingPointDefect`:idx: base class.
 Nim provides the pragmas `nanChecks`:idx: and `infChecks`:idx: to control
 whether the IEEE exceptions are ignored or trap a Nim exception:
 
-.. code-block:: nim
+  ```nim
   {.nanChecks: on, infChecks: on.}
   var a = 1.0
   var b = 0.0
   echo b / b # raises FloatInvalidOpDefect
   echo a / b # raises FloatOverflowDefect
+  ```
 
 In the current implementation `FloatDivByZeroDefect` and `FloatInexactDefect`
 are never raised. `FloatOverflowDefect` is raised instead of
@@ -1085,7 +1208,7 @@ The only operations that are affected by the `floatChecks` pragma are
 the `+`, `-`, `*`, `/` operators for floating-point types.
 
 An implementation should always use the maximum precision available to evaluate
-floating pointer values during semantic analysis; this means expressions like
+floating-point values during semantic analysis; this means expressions like
 `0.09'f32 + 0.01'f32 == 0.09'f64 + 0.01'f64` that are evaluating during
 constant folding are true.
 
@@ -1096,19 +1219,21 @@ The boolean type is named `bool`:idx: in Nim and can be one of the two
 pre-defined values `true` and `false`. Conditions in `while`,
 `if`, `elif`, `when`-statements need to be of type `bool`.
 
-This condition holds::
+This condition holds:
 
+  ```nim
   ord(false) == 0 and ord(true) == 1
+  ```
 
 The operators `not, and, or, xor, <, <=, >, >=, !=, ==` are defined
 for the bool type. The `and` and `or` operators perform short-cut
 evaluation. Example:
 
-.. code-block:: nim
-
+  ```nim
   while p != nil and p.name != "xyz":
     # p.name is not evaluated if p == nil
     p = p.next
+  ```
 
 
 The size of the bool type is one byte.
@@ -1117,14 +1242,10 @@ The size of the bool type is one byte.
 Character type
 --------------
 The character type is named `char` in Nim. Its size is one byte.
-Thus it cannot represent a UTF-8 character, but a part of it.
-The reason for this is efficiency: for the overwhelming majority of use-cases,
-the resulting programs will still handle UTF-8 properly as UTF-8 was especially
-designed for this.
-Another reason is that Nim can support `array[char, int]` or
-`set[char]` efficiently as many algorithms rely on this feature. The
-`Rune` type is used for Unicode characters, it can represent any Unicode
-character. `Rune` is declared in the `unicode module `_.
+Thus, it cannot represent a UTF-8 character, but a part of it.
+
+The `Rune` type is used for Unicode characters, it can represent any Unicode
+character. `Rune` is declared in the [unicode module](unicode.html).
 
 
 
@@ -1134,15 +1255,16 @@ Enumeration types
 Enumeration types define a new type whose values consist of the ones
 specified. The values are ordered. Example:
 
-.. code-block:: nim
-
+  ```nim
   type
     Direction = enum
       north, east, south, west
+  ```
 
 
-Now the following holds::
+Now the following holds:
 
+  ```nim
   ord(north) == 0
   ord(east) == 1
   ord(south) == 2
@@ -1150,9 +1272,10 @@ Now the following holds::
 
   # Also allowed:
   ord(Direction.west) == 3
+  ```
 
-Thus, north < east < south < west. The comparison operators can be used
-with enumeration types. Instead of `north` etc, the enum value can also
+The implied order is: north < east < south < west. The comparison operators can be used
+with enumeration types. Instead of `north` etc., the enum value can also
 be qualified with the enum type that it resides in, `Direction.north`.
 
 For better interfacing to other programming languages, the fields of enum
@@ -1162,12 +1285,13 @@ explicitly given is assigned the value of the previous field + 1.
 
 An explicit ordered enum can have *holes*:
 
-.. code-block:: nim
+  ```nim
   type
     TokenType = enum
       a = 2, b = 4, c = 89 # holes are valid
+  ```
 
-However, it is then not ordinal anymore, so it is not possible to use these
+However, it is then not ordinal anymore, so it is impossible to use these
 enums as an index type for arrays. The procedures `inc`, `dec`, `succ`
 and `pred` are not available for them either.
 
@@ -1176,14 +1300,14 @@ The compiler supports the built-in stringify operator `$` for enumerations.
 The stringify's result can be controlled by explicitly giving the string
 values to use:
 
-.. code-block:: nim
-
+  ```nim
   type
     MyEnum = enum
       valueA = (0, "my value A"),
       valueB = "value B",
       valueC = 2,
       valueD = (3, "abc")
+  ```
 
 As can be seen from the example, it is possible to both specify a field's
 ordinal value and its string value by using a tuple. It is also
@@ -1195,8 +1319,7 @@ as the last attempt. Only non-ambiguous symbols are added to this scope.
 But one can always access these via type qualification written
 as `MyEnum.value`:
 
-.. code-block:: nim
-
+  ```nim
   type
     MyEnum {.pure.} = enum
       valueA, valueB, valueC, valueD, amb
@@ -1208,8 +1331,59 @@ as `MyEnum.value`:
   echo valueA # MyEnum.valueA
   echo amb    # Error: Unclear whether it's MyEnum.amb or OtherEnum.amb
   echo MyEnum.amb # OK.
+  ```
+
+Enum value names are overloadable, much like routines. If both of the enums
+`T` and `U` have a member named `foo`, then the identifier `foo` corresponds
+to a choice between `T.foo` and `U.foo`. During overload resolution,
+the correct type of `foo` is decided from the context. If the type of `foo` is
+ambiguous, a static error will be produced.
+
+  ```nim  test = "nim c $1"
+
+  type
+    E1 = enum
+      value1,
+      value2
+    E2 = enum
+      value1,
+      value2 = 4
+
+  const
+    Lookuptable = [
+      E1.value1: "1",
+      # no need to qualify value2, known to be E1.value2
+      value2: "2"
+    ]
+
+  proc p(e: E1) =
+    # disambiguation in 'case' statements:
+    case e
+    of value1: echo "A"
+    of value2: echo "B"
+
+  p value2
+  ```
+
+In some cases, ambiguity of enums is resolved depending on the relation
+between the current scope and the scope the enums were defined in.
+
+  ```nim
+  # a.nim
+  type Foo* = enum abc
+
+  # b.nim
+  import a
+  type Bar = enum abc
+  echo abc is Bar # true
+
+  block:
+    type Baz = enum abc
+    echo abc is Baz # true
+  ```
+
+To implement bit fields with enums see [Bit fields].
 
-To implement bit fields with enums see `Bit fields <#set-type-bit-fields>`_
 
 String type
 -----------
@@ -1229,14 +1403,14 @@ Most native Nim types support conversion to strings with the special `$` proc.
 When calling the `echo` proc, for example, the built-in stringify operation
 for the parameter is called:
 
-.. code-block:: nim
-
+  ```nim
   echo 3 # calls `$` for `int`
+  ```
 
 Whenever a user creates a specialized object, implementation of this procedure
 provides for `string` representation.
 
-.. code-block:: nim
+  ```nim
   type
     Person = object
       name: string
@@ -1248,6 +1422,7 @@ provides for `string` representation.
                      # is natively an integer to convert it to
                      # a string
             " years old."
+  ```
 
 While `$p.name` can also be used, the `$` operation on a string does
 nothing. Note that we cannot rely on automatic conversion from an `int` to
@@ -1257,18 +1432,18 @@ Strings are compared by their lexicographical order. All comparison operators
 are available. Strings can be indexed like arrays (lower bound is 0). Unlike
 arrays, they can be used in case statements:
 
-.. code-block:: nim
-
+  ```nim
   case paramStr(i)
   of "-v": incl(options, optVerbose)
   of "-h", "-?": incl(options, optHelp)
   else: write(stdout, "invalid command line option!\n")
+  ```
 
 Per convention, all strings are UTF-8 strings, but this is not enforced. For
 example, when reading strings from binary files, they are merely a sequence of
 bytes. The index operation `s[i]` means the i-th *char* of `s`, not the
-i-th *unichar*. The iterator `runes` from the `unicode module
-`_ can be used for iteration over all Unicode characters.
+i-th *unichar*. The iterator `runes` from the [unicode module](unicode.html)
+can be used for iteration over all Unicode characters.
 
 
 cstring type
@@ -1277,7 +1452,7 @@ cstring type
 The `cstring` type meaning `compatible string` is the native representation
 of a string for the compilation backend. For the C backend the `cstring` type
 represents a pointer to a zero-terminated char array
-compatible with the type `char*` in Ansi C. Its primary purpose lies in easy
+compatible with the type `char*` in ANSI C. Its primary purpose lies in easy
 interfacing with C. The index operation `s[i]` means the i-th *char* of
 `s`; however no bounds checking for `cstring` is performed making the
 index operation unsafe.
@@ -1286,40 +1461,46 @@ A Nim `string` is implicitly convertible
 to `cstring` for convenience. If a Nim string is passed to a C-style
 variadic proc, it is implicitly converted to `cstring` too:
 
-.. code-block:: nim
+  ```nim
   proc printf(formatstr: cstring) {.importc: "printf", varargs,
                                     header: "".}
 
   printf("This works %s", "as expected")
+  ```
 
 Even though the conversion is implicit, it is not *safe*: The garbage collector
 does not consider a `cstring` to be a root and may collect the underlying
-memory. However, in practice, this almost never happens as the GC considers
-stack roots conservatively. One can use the builtin procs `GC_ref` and
-`GC_unref` to keep the string data alive for the rare cases where it does
-not work.
+memory. For this reason, the implicit conversion will be removed in future
+releases of the Nim compiler. Certain idioms like conversion of a `const` string
+to `cstring` are safe and will remain to be allowed.
 
-A `$` proc is defined for cstrings that returns a string. Thus to get a nim
+A `$` proc is defined for cstrings that returns a string. Thus, to get a nim
 string from a cstring:
 
-.. code-block:: nim
+  ```nim
   var str: string = "Hello!"
   var cstr: cstring = str
   var newstr: string = $cstr
+  ```
 
 `cstring` literals shouldn't be modified.
 
-.. code-block:: nim
+  ```nim
   var x = cstring"literals"
   x[1] = 'A' # This is wrong!!!
+  ```
 
 If the `cstring` originates from a regular memory (not read-only memory),
 it can be modified:
 
-.. code-block:: nim
+  ```nim
   var x = "123456"
-  var s: cstring = x
+  prepareMutation(x) # call `prepareMutation` before modifying the strings
+  var s: cstring = cstring(x)
   s[0] = 'u' # This is ok
+  ```
+
+`cstring` values may also be used in case statements like strings.
 
 Structured types
 ----------------
@@ -1353,8 +1534,7 @@ A sequence may be passed to a parameter that is of type *open array*.
 
 Example:
 
-.. code-block:: nim
-
+  ```nim
   type
     IntArray = array[0..5, int] # an array that is indexed with 0..5
     IntSeq = seq[int] # a sequence of integers
@@ -1365,6 +1545,7 @@ Example:
   y = @[1, 2, 3, 4, 5, 6] # the @ turns the array into a sequence
 
   let z = [1.0, 2, 3, 4] # the type of z is array[0..3, float]
+  ```
 
 The lower bound of an array or sequence may be received by the built-in proc
 `low()`, the higher bound by `high()`. The length may be
@@ -1378,12 +1559,11 @@ The notation `x[i]` can be used to access the i-th element of `x`.
 
 Arrays are always bounds checked (statically or at runtime). These
 checks can be disabled via pragmas or invoking the compiler with the
-`--boundChecks:off` command-line switch.
+`--boundChecks:off`:option: command-line switch.
 
 An array constructor can have explicit indexes for readability:
 
-.. code-block:: nim
-
+  ```nim
   type
     Values = enum
       valA, valB, valC
@@ -1394,12 +1574,12 @@ An array constructor can have explicit indexes for readability:
       valB: "B",
       valC: "C"
     ]
+  ```
 
 If an index is left out, `succ(lastIndex)` is used as the index
 value:
 
-.. code-block:: nim
-
+  ```nim
   type
     Values = enum
       valA, valB, valC, valD, valE
@@ -1411,6 +1591,7 @@ value:
       valC: "C",
       "D", "e"
     ]
+  ```
 
 
 
@@ -1419,30 +1600,31 @@ Open arrays
 
 Often fixed size arrays turn out to be too inflexible; procedures should
 be able to deal with arrays of different sizes. The `openarray`:idx: type
-allows this; it can only be used for parameters. Openarrays are always
+allows this; it can only be used for parameters. Open arrays are always
 indexed with an `int` starting at position 0. The `len`, `low`
 and `high` operations are available for open arrays too. Any array with
-a compatible base type can be passed to an openarray parameter, the index
+a compatible base type can be passed to an open array parameter, the index
 type does not matter. In addition to arrays, sequences can also be passed
 to an open array parameter.
 
-The openarray type cannot be nested: multidimensional openarrays are not
+The `openarray` type cannot be nested: multidimensional open arrays are not
 supported because this is seldom needed and cannot be done efficiently.
 
-.. code-block:: nim
+  ```nim
   proc testOpenArray(x: openArray[int]) = echo repr(x)
 
   testOpenArray([1,2,3])  # array[]
   testOpenArray(@[1,2,3]) # seq[]
+  ```
 
 Varargs
 -------
 
-A `varargs` parameter is an openarray parameter that additionally
-allows to pass a variable number of arguments to a procedure. The compiler
+A `varargs` parameter is an open array parameter that additionally
+allows a variable number of arguments to be passed to a procedure. The compiler
 converts the list of arguments to an array implicitly:
 
-.. code-block:: nim
+  ```nim
   proc myWriteln(f: File, a: varargs[string]) =
     for s in items(a):
       write(f, s)
@@ -1451,12 +1633,13 @@ converts the list of arguments to an array implicitly:
   myWriteln(stdout, "abc", "def", "xyz")
   # is transformed to:
   myWriteln(stdout, ["abc", "def", "xyz"])
+  ```
 
-This transformation is only done if the varargs parameter is the
+This transformation is only done if the `varargs` parameter is the
 last parameter in the procedure header. It is also possible to perform
 type conversions in this context:
 
-.. code-block:: nim
+  ```nim
   proc myWriteln(f: File, a: varargs[string, `$`]) =
     for s in items(a):
       write(f, s)
@@ -1464,7 +1647,8 @@ type conversions in this context:
 
   myWriteln(stdout, 123, "abc", 4.0)
   # is transformed to:
-  myWriteln(stdout, [$123, $"def", $4.0])
+  myWriteln(stdout, [$123, $"abc", $4.0])
+  ```
 
 In this example `$` is applied to any argument that is passed to the
 parameter `a`. (Note that `$` applied to strings is a nop.)
@@ -1472,21 +1656,23 @@ parameter `a`. (Note that `$` applied to strings is a nop.)
 Note that an explicit array constructor passed to a `varargs` parameter is
 not wrapped in another implicit array construction:
 
-.. code-block:: nim
+  ```nim
   proc takeV[T](a: varargs[T]) = discard
 
   takeV([123, 2, 1]) # takeV's T is "int", not "array of int"
+  ```
 
 
 `varargs[typed]` is treated specially: It matches a variable list of arguments
 of arbitrary type but *always* constructs an implicit array. This is required
 so that the builtin `echo` proc does what is expected:
 
-.. code-block:: nim
+  ```nim
   proc echo*(x: varargs[typed, `$`]) {...}
 
   echo @[1, 2, 3]
   # prints "@[1, 2, 3]" and not "123"
+  ```
 
 
 Unchecked arrays
@@ -1496,20 +1682,22 @@ are not checked. This is often useful to implement customized flexibly sized
 arrays. Additionally, an unchecked array is translated into a C array of
 undetermined size:
 
-.. code-block:: nim
+  ```nim
   type
     MySeq = object
       len, cap: int
       data: UncheckedArray[int]
+  ```
 
 Produces roughly this C code:
 
-.. code-block:: C
+  ```C
   typedef struct {
     NI len;
     NI cap;
     NI data[];
   } MySeq;
+  ```
 
 The base type of the unchecked array may not contain any GC'ed memory but this
 is currently not checked.
@@ -1530,35 +1718,32 @@ heterogeneous storage types with few abstractions. The `()` syntax
 can be used to construct tuples. The order of the fields in the constructor
 must match the order of the tuple's definition. Different tuple-types are
 *equivalent* if they specify the same fields of the same type in the same
-order. The *names* of the fields also have to be identical.
-
-The assignment operator for tuples copies each component.
-The default assignment operator for objects copies each component. Overloading
-of the assignment operator is described `here
-`_.
-
-.. code-block:: nim
+order. The *names* of the fields also have to be the same.
 
+  ```nim
   type
     Person = tuple[name: string, age: int] # type representing a person:
-                                           # a person consists of a name
-                                           # and an age
-  var
-    person: Person
+                                           # it consists of a name and an age.
+  var person: Person
   person = (name: "Peter", age: 30)
-  echo person.name
+  assert person.name == "Peter"
   # the same, but less readable:
   person = ("Peter", 30)
-  echo person[0]
+  assert person[0] == "Peter"
+  assert Person is (string, int)
+  assert (string, int) is Person
+  assert Person isnot tuple[other: string, age: int] # `other` is a different identifier
+  ```
 
 A tuple with one unnamed field can be constructed with the parentheses and a
 trailing comma:
 
-.. code-block:: nim
+  ```nim
   proc echoUnaryTuple(a: (int,)) =
     echo a[0]
 
   echoUnaryTuple (1,)
+  ```
 
 
 In fact, a trailing comma is allowed for every tuple construction.
@@ -1569,11 +1754,12 @@ is compatible with the way the C compiler does it.
 For consistency  with `object` declarations, tuples in a `type` section
 can also be defined with indentation instead of `[]`:
 
-.. code-block:: nim
+  ```nim
   type
     Person = tuple   # type representing a person
       name: string   # a person consists of a name
       age: Natural   # and an age
+  ```
 
 Objects provide many features that tuples do not. Objects provide inheritance
 and the ability to hide fields from other modules. Objects with inheritance
@@ -1581,7 +1767,7 @@ enabled have information about their type at runtime so that the `of` operator
 can be used to determine the object's type. The `of` operator is similar to
 the `instanceof` operator in Java.
 
-.. code-block:: nim
+  ```nim
   type
     Person = object of RootObj
       name*: string   # the * means that `name` is accessible from other modules
@@ -1595,6 +1781,7 @@ the `instanceof` operator in Java.
     person: Person
   assert(student of Student) # is true
   assert(student of Person) # also true
+  ```
 
 Object fields that should be visible from outside the defining module have to
 be marked by `*`. In contrast to tuples, different object types are
@@ -1603,7 +1790,7 @@ Objects that have no ancestor are implicitly `final` and thus have no hidden
 type information. One can use the `inheritable` pragma to
 introduce new object roots apart from `system.RootObj`.
 
-.. code-block:: nim
+  ```nim
   type
     Person = object # example of a final object
       name*: string
@@ -1611,6 +1798,11 @@ introduce new object roots apart from `system.RootObj`.
 
     Student = ref object of Person # Error: inheritance only works with non-final objects
       id: int
+  ```
+
+The assignment operator for tuples and objects copies each component.
+The methods to override this copying behavior are described [here][type
+bound operators].
 
 
 Object construction
@@ -1620,8 +1812,19 @@ Objects can also be created with an `object construction expression`:idx: that
 has the syntax `T(fieldA: valueA, fieldB: valueB, ...)` where `T` is
 an `object` type or a `ref object` type:
 
-.. code-block:: nim
-  var student = Student(name: "Anton", age: 5, id: 3)
+  ```nim
+  type
+    Student = object
+      name: string
+      age: int
+    PStudent = ref Student
+  var a1 = Student(name: "Anton", age: 5)
+  var a2 = PStudent(name: "Anton", age: 5)
+  # this also works directly:
+  var a3 = (ref Student)(name: "Anton", age: 5)
+  # not all fields need to be mentioned, and they can be mentioned out of order:
+  var a4 = Student(age: 5)
+  ```
 
 Note that, unlike tuples, objects require the field names along with their values.
 For a `ref object` type `system.new` is invoked implicitly.
@@ -1636,8 +1839,7 @@ enumerated type used for runtime type flexibility, mirroring the concepts of
 
 An example:
 
-.. code-block:: nim
-
+  ```nim
   # This is an example of how an abstract syntax tree could be modelled in Nim
   type
     NodeKind = enum  # the different node types
@@ -1674,6 +1876,7 @@ An example:
                             rightOp: Node(kind: nkInt, intVal: 2))
   # valid: does not change the active object branch:
   x.kind = nkSub
+  ```
 
 As can be seen from the example, an advantage to an object hierarchy is that
 no casting between different object types is needed. Yet, access to invalid
@@ -1691,12 +1894,12 @@ corresponding discriminator value must be specified as a constant expression.
 Instead of changing the active object branch, replace the old object in memory
 with a new one completely:
 
-.. code-block:: nim
-
+  ```nim
   var x = Node(kind: nkAdd, leftOp: Node(kind: nkInt, intVal: 4),
                             rightOp: Node(kind: nkInt, intVal: 2))
   # change the node's contents:
   x[] = NodeObj(kind: nkString, strVal: "abc")
+  ```
 
 
 Starting with version 0.20 `system.reset` cannot be used anymore to support
@@ -1713,8 +1916,7 @@ valid for the chosen object branch.
 
 A small example:
 
-.. code-block:: nim
-
+  ```nim
   let unknownKind = nkSub
 
   # invalid: unsafe initialization because the kind field is not statically known:
@@ -1731,6 +1933,92 @@ A small example:
   # also valid, since unknownKindBounded can only contain the values nkAdd or nkSub
   let unknownKindBounded = range[nkAdd..nkSub](unknownKind)
   z = Node(kind: unknownKindBounded, leftOp: Node(), rightOp: Node())
+  ```
+
+
+cast uncheckedAssign
+--------------------
+
+Some restrictions for case objects can be disabled via a `{.cast(uncheckedAssign).}` section:
+
+  ```nim  test="nim c $1"
+  type
+    TokenKind* = enum
+      strLit, intLit
+    Token = object
+      case kind*: TokenKind
+      of strLit:
+        s*: string
+      of intLit:
+        i*: int64
+
+  proc passToVar(x: var TokenKind) = discard
+
+  var t = Token(kind: strLit, s: "abc")
+
+  {.cast(uncheckedAssign).}:
+    # inside the 'cast' section it is allowed to pass 't.kind' to a 'var T' parameter:
+    passToVar(t.kind)
+
+    # inside the 'cast' section it is allowed to set field 's' even though the
+    # constructed 'kind' field has an unknown value:
+    t = Token(kind: t.kind, s: "abc")
+
+    # inside the 'cast' section it is allowed to assign to the 't.kind' field directly:
+    t.kind = intLit
+  ```
+
+Default values for object fields
+--------------------------------
+
+Object fields are allowed to have a constant default value. The type of field can be omitted if a default value is given.
+
+```nim test
+type
+  Foo = object
+    a: int = 2
+    b: float = 3.14
+    c = "I can have a default value"
+
+  Bar = ref object
+    a: int = 2
+    b: float = 3.14
+    c = "I can have a default value"
+```
+
+The explicit initialization uses these defaults which includes an `object` created with an object construction expression or the procedure `default`; a `ref object` created with an object construction expression or the procedure `new`; an array or a tuple with a subtype which has a default created with the procedure `default`.
+
+
+```nim test
+type
+  Foo = object
+    a: int = 2
+    b = 3.0
+  Bar = ref object
+    a: int = 2
+    b = 3.0
+
+block: # created with an object construction expression
+  let x = Foo()
+  assert x.a == 2 and x.b == 3.0
+
+  let y = Bar()
+  assert y.a == 2 and y.b == 3.0
+
+block: # created with an object construction expression
+  let x = default(Foo)
+  assert x.a == 2 and x.b == 3.0
+
+  let y = default(array[1, Foo])
+  assert y[0].a == 2 and y[0].b == 3.0
+
+  let z = default(tuple[x: Foo])
+  assert z.x.a == 2 and z.x.b == 3.0
+
+block: # created with the procedure `new`
+  let y = new Bar
+  assert y.a == 2 and y.b == 3.0
+```
 
 Set type
 --------
@@ -1746,25 +2034,24 @@ point to and modify the same location in memory (also called `aliasing`:idx:).
 Nim distinguishes between `traced`:idx: and `untraced`:idx: references.
 Untraced references are also called *pointers*. Traced references point to
 objects of a garbage-collected heap, untraced references point to
-manually allocated objects or objects somewhere else in memory. Thus
+manually allocated objects or objects somewhere else in memory. Thus,
 untraced references are *unsafe*. However, for certain low-level operations
 (accessing the hardware) untraced references are unavoidable.
 
 Traced references are declared with the **ref** keyword, untraced references
-are declared with the **ptr** keyword.  In general, a `ptr T` is implicitly
+are declared with the **ptr** keyword. In general, a `ptr T` is implicitly
 convertible to the `pointer` type.
 
 An empty subscript `[]` notation can be used to de-refer a reference,
 the `addr` procedure returns the address of an item. An address is always
 an untraced reference.
-Thus the usage of `addr` is an *unsafe* feature.
+Thus, the usage of `addr` is an *unsafe* feature.
 
 The `.` (access a tuple/object field operator)
 and `[]` (array/string/sequence index operator) operators perform implicit
 dereferencing operations for reference types:
 
-.. code-block:: nim
-
+  ```nim
   type
     Node = ref NodeObj
     NodeObj = object
@@ -1776,16 +2063,14 @@ dereferencing operations for reference types:
   new(n)
   n.data = 9
   # no need to write n[].data; in fact n[].data is highly discouraged!
-
-Automatic dereferencing can be performed for the first argument of a routine
-call, but this is an experimental feature and is described `here
-`_.
+  ```
 
 In order to simplify structural type checking, recursive tuples are not valid:
 
-.. code-block:: nim
+  ```nim
   # invalid recursion
   type MyTuple = tuple[a: ref MyTuple]
+  ```
 
 Likewise `T = ref T` is an invalid type.
 
@@ -1793,17 +2078,17 @@ As a syntactical extension, `object` types can be anonymous if
 declared in a type section via the `ref object` or `ptr object` notations.
 This feature is useful if an object should only gain reference semantics:
 
-.. code-block:: nim
-
+  ```nim
   type
     Node = ref object
       le, ri: Node
       data: int
+  ```
 
 
 To allocate a new traced object, the built-in procedure `new` has to be used.
 To deal with untraced memory, the procedures `alloc`, `dealloc` and
-`realloc` can be used. The documentation of the `system `_ module
+`realloc` can be used. The documentation of the [system](system.html) module
 contains further information.
 
 
@@ -1820,20 +2105,20 @@ Dereferencing `nil` is an unrecoverable fatal runtime error (and not a panic).
 A successful dereferencing operation `p[]` implies that `p` is not nil. This
 can be exploited by the implementation to optimize code like:
 
-.. code-block:: nim
-
+  ```nim
   p[].field = 3
   if p != nil:
     # if p were nil, `p[]` would have caused a crash already,
     # so we know `p` is always not nil here.
     action()
+  ```
 
 Into:
 
-.. code-block:: nim
-
+  ```nim
   p[].field = 3
   action()
+  ```
 
 
 *Note*: This is not comparable to C's "undefined behavior" for
@@ -1848,7 +2133,7 @@ traced references, strings, or sequences: in order to free everything properly,
 the built-in procedure `reset` has to be called before freeing the untraced
 memory manually:
 
-.. code-block:: nim
+  ```nim
   type
     Data = tuple[x, y: int, s: string]
 
@@ -1863,9 +2148,11 @@ memory manually:
 
   # free the memory:
   dealloc(d)
+  ```
 
 Without the `reset` call the memory allocated for the `d.s` string would
-never be freed. The example also demonstrates two important features for low-level programming: the `sizeof` proc returns the size of a type or value
+never be freed. The example also demonstrates two important features for
+low-level programming: the `sizeof` proc returns the size of a type or value
 in bytes. The `cast` operator can circumvent the type system: the compiler
 is forced to treat the result of the `alloc0` call (which returns an untyped
 pointer) as if it would have the type `ptr Data`. Casting should only be
@@ -1883,23 +2170,21 @@ details like this when mixing garbage-collected data with unmanaged memory.
 Procedural type
 ---------------
 A procedural type is internally a pointer to a procedure. `nil` is
-an allowed value for variables of a procedural type. Nim uses procedural
-types to achieve `functional`:idx: programming techniques.
+an allowed value for a variable of a procedural type.
 
 Examples:
 
-.. code-block:: nim
-
+  ```nim
   proc printItem(x: int) = ...
 
   proc forEach(c: proc (x: int) {.cdecl.}) =
     ...
 
   forEach(printItem)  # this will NOT compile because calling conventions differ
+  ```
 
 
-.. code-block:: nim
-
+  ```nim
   type
     OnMouseMove = proc (x, y: int) {.closure.}
 
@@ -1912,6 +2197,7 @@ Examples:
   # ok, 'onMouseMove' has the default calling convention, which is compatible
   # to 'closure':
   setOnMouseMove(onMouseMove)
+  ```
 
 
 A subtle issue with procedural types is that the calling convention of the
@@ -1923,52 +2209,52 @@ that expects a proc of the calling convention `closure`.
 Nim supports these `calling conventions`:idx:\:
 
 `nimcall`:idx:
-    is the default convention used for a Nim **proc**. It is the
+:   is the default convention used for a Nim **proc**. It is the
     same as `fastcall`, but only for C compilers that support `fastcall`.
 
 `closure`:idx:
-    is the default calling convention for a **procedural type** that lacks
+:   is the default calling convention for a **procedural type** that lacks
     any pragma annotations. It indicates that the procedure has a hidden
     implicit parameter (an *environment*). Proc vars that have the calling
     convention `closure` take up two machine words: One for the proc pointer
     and another one for the pointer to implicitly passed environment.
 
 `stdcall`:idx:
-    This is the stdcall convention as specified by Microsoft. The generated C
+:   This is the stdcall convention as specified by Microsoft. The generated C
     procedure is declared with the `__stdcall` keyword.
 
 `cdecl`:idx:
-    The cdecl convention means that a procedure shall use the same convention
+:   The cdecl convention means that a procedure shall use the same convention
     as the C compiler. Under Windows the generated C procedure is declared with
     the `__cdecl` keyword.
 
 `safecall`:idx:
-    This is the safecall convention as specified by Microsoft. The generated C
+:   This is the safecall convention as specified by Microsoft. The generated C
     procedure is declared with the `__safecall` keyword. The word *safe*
     refers to the fact that all hardware registers shall be pushed to the
     hardware stack.
 
 `inline`:idx:
-    The inline convention means the caller should not call the procedure,
+:   The inline convention means the caller should not call the procedure,
     but inline its code directly. Note that Nim does not inline, but leaves
     this to the C compiler; it generates `__inline` procedures. This is
-    only a hint for the compiler: it may completely ignore it and
+    only a hint for the compiler: it may completely ignore it, and
     it may inline procedures that are not marked as `inline`.
 
 `fastcall`:idx:
-    Fastcall means different things to different C compilers. One gets whatever
+:   Fastcall means different things to different C compilers. One gets whatever
     the C `__fastcall` means.
 
 `thiscall`:idx:
-    This is the thiscall calling convention as specified by Microsoft, used on
+:   This is the thiscall calling convention as specified by Microsoft, used on
     C++ class member functions on the x86 architecture.
 
 `syscall`:idx:
-    The syscall convention is the same as `__syscall` in C. It is used for
+:   The syscall convention is the same as `__syscall`:c: in C. It is used for
     interrupts.
 
 `noconv`:idx:
-    The generated C code will not have any explicit calling convention and thus
+:   The generated C code will not have any explicit calling convention and thus
     use the C compiler's default calling convention. This is needed because
     Nim's default calling convention for procedures is `fastcall` to
     improve speed.
@@ -1994,8 +2280,7 @@ reverse operation.
 A distinct type is an ordinal type if its base type is an ordinal type.
 
 
-Modeling currencies
-~~~~~~~~~~~~~~~~~~~~
+### Modeling currencies
 
 A distinct type can be used to model different physical `units`:idx: with a
 numerical base type, for example. The following example models currencies.
@@ -2003,7 +2288,7 @@ numerical base type, for example. The following example models currencies.
 Different currencies should not be mixed in monetary calculations. Distinct
 types are a perfect tool to model different currencies:
 
-.. code-block:: nim
+  ```nim
   type
     Dollar = distinct int
     Euro = distinct int
@@ -2014,19 +2299,21 @@ types are a perfect tool to model different currencies:
 
   echo d + 12
   # Error: cannot add a number with no unit and a `Dollar`
+  ```
 
 Unfortunately, `d + 12.Dollar` is not allowed either,
 because `+` is defined for `int` (among others), not for `Dollar`. So
 a `+` for dollars needs to be defined:
 
-.. code-block::
+  ```nim
   proc `+` (x, y: Dollar): Dollar =
     result = Dollar(int(x) + int(y))
+  ```
 
 It does not make sense to multiply a dollar with a dollar, but with a
 number without unit; and the same holds for division:
 
-.. code-block::
+  ```nim
   proc `*` (x: Dollar, y: int): Dollar =
     result = Dollar(int(x) * y)
 
@@ -2034,6 +2321,7 @@ number without unit; and the same holds for division:
     result = Dollar(x * int(y))
 
   proc `div` ...
+  ```
 
 This quickly gets tedious. The implementations are trivial and the compiler
 should not generate all this code only to optimize it away later - after all
@@ -2041,21 +2329,20 @@ should not generate all this code only to optimize it away later - after all
 The pragma `borrow`:idx: has been designed to solve this problem; in principle,
 it generates the above trivial implementations:
 
-.. code-block:: nim
+  ```nim
   proc `*` (x: Dollar, y: int): Dollar {.borrow.}
   proc `*` (x: int, y: Dollar): Dollar {.borrow.}
   proc `div` (x: Dollar, y: int): Dollar {.borrow.}
+  ```
 
 The `borrow` pragma makes the compiler use the same implementation as
 the proc that deals with the distinct type's base type, so no code is
 generated.
 
 But it seems all this boilerplate code needs to be repeated for the `Euro`
-currency. This can be solved with templates_.
-
-.. code-block:: nim
-    :test: "nim c $1"
+currency. This can be solved with [templates].
 
+  ```nim  test = "nim c $1"
   template additive(typ: typedesc) =
     proc `+` *(x, y: typ): typ {.borrow.}
     proc `-` *(x, y: typ): typ {.borrow.}
@@ -2084,12 +2371,13 @@ currency. This can be solved with templates_.
 
   defineCurrency(Dollar, int)
   defineCurrency(Euro, int)
+  ```
 
 
 The borrow pragma can also be used to annotate the distinct type to allow
 certain builtin operations to be lifted:
 
-.. code-block:: nim
+  ```nim
   type
     Foo = object
       a, b: int
@@ -2102,18 +2390,18 @@ certain builtin operations to be lifted:
   # field access now valid
   bb.a = 90
   bb.s = "abc"
+  ```
 
 Currently, only the dot accessor can be borrowed in this way.
 
 
-Avoiding SQL injection attacks
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+### Avoiding SQL injection attacks
 
 An SQL statement that is passed from Nim to an SQL database might be
 modeled as a string. However, using string templates and filling in the
 values is vulnerable to the famous `SQL injection attack`:idx:\:
 
-.. code-block:: nim
+  ```nim
   import std/strutils
 
   proc query(db: DbHandle, statement: string) = ...
@@ -2123,12 +2411,13 @@ values is vulnerable to the famous `SQL injection attack`:idx:\:
 
   db.query("SELECT FROM users WHERE name = '$1'" % username)
   # Horrible security hole, but the compiler does not mind!
+  ```
 
 This can be avoided by distinguishing strings that contain SQL from strings
 that don't. Distinct types provide a means to introduce a new string type
 `SQL` that is incompatible with `string`:
 
-.. code-block:: nim
+  ```nim
   type
     SQL = distinct string
 
@@ -2139,13 +2428,14 @@ that don't. Distinct types provide a means to introduce a new string type
 
   db.query("SELECT FROM users WHERE name = '$1'" % username)
   # Static error: `query` expects an SQL string!
+  ```
 
 
 It is an essential property of abstract types that they **do not** imply a
 subtype relation between the abstract type and its base type. Explicit type
 conversions from `string` to `SQL` are allowed:
 
-.. code-block:: nim
+  ```nim
   import std/[strutils, sequtils]
 
   proc properQuote(s: string): SQL =
@@ -2161,12 +2451,13 @@ conversions from `string` to `SQL` are allowed:
     result = SQL(string(frmt) % StrSeq(v))
 
   db.query("SELECT FROM users WHERE name = '$1'".SQL % [username])
+  ```
 
-Now we have compile-time checking against SQL injection attacks.  Since
+Now we have compile-time checking against SQL injection attacks. Since
 `"".SQL` is transformed to `SQL("")` no new syntax is needed for nice
 looking `SQL` string literals. The hypothetical `SQL` type actually
-exists in the library as the `SqlQuery type `_ of
-modules like `db_sqlite `_.
+exists in the library as the [SqlQuery type](db_common.html#SqlQuery) of
+modules like [db_sqlite](db_sqlite.html).
 
 
 Auto type
@@ -2175,18 +2466,21 @@ Auto type
 The `auto` type can only be used for return types and parameters. For return
 types it causes the compiler to infer the type from the routine body:
 
-.. code-block:: nim
+  ```nim
   proc returnsInt(): auto = 1984
+  ```
 
 For parameters it currently creates implicitly generic routines:
 
-.. code-block:: nim
+  ```nim
   proc foo(a, b: auto) = discard
+  ```
 
 Is the same as:
 
-.. code-block:: nim
+  ```nim
   proc foo[T1, T2](a: T1, b: T2) = discard
+  ```
 
 However, later versions of the language might change this to mean "infer the
 parameters' types from the body". Then the above `foo` would be rejected as
@@ -2202,121 +2496,36 @@ describe the type checking done by the compiler.
 
 Type equality
 -------------
+
 Nim uses structural type equivalence for most types. Only for objects,
-enumerations and distinct types name equivalence is used. The following
-algorithm, *in pseudo-code*, determines type equality:
-
-.. code-block:: nim
-  proc typeEqualsAux(a, b: PType,
-                     s: var HashSet[(PType, PType)]): bool =
-    if (a,b) in s: return true
-    incl(s, (a,b))
-    if a.kind == b.kind:
-      case a.kind
-      of int, intXX, float, floatXX, char, string, cstring, pointer,
-          bool, nil, void:
-        # leaf type: kinds identical; nothing more to check
-        result = true
-      of ref, ptr, var, set, seq, openarray:
-        result = typeEqualsAux(a.baseType, b.baseType, s)
-      of range:
-        result = typeEqualsAux(a.baseType, b.baseType, s) and
-          (a.rangeA == b.rangeA) and (a.rangeB == b.rangeB)
-      of array:
-        result = typeEqualsAux(a.baseType, b.baseType, s) and
-                 typeEqualsAux(a.indexType, b.indexType, s)
-      of tuple:
-        if a.tupleLen == b.tupleLen:
-          for i in 0..a.tupleLen-1:
-            if not typeEqualsAux(a[i], b[i], s): return false
-          result = true
-      of object, enum, distinct:
-        result = a == b
-      of proc:
-        result = typeEqualsAux(a.parameterTuple, b.parameterTuple, s) and
-                 typeEqualsAux(a.resultType, b.resultType, s) and
-                 a.callingConvention == b.callingConvention
-
-  proc typeEquals(a, b: PType): bool =
-    var s: HashSet[(PType, PType)] = {}
-    result = typeEqualsAux(a, b, s)
-
-Since types are graphs which can have cycles, the above algorithm needs an
-auxiliary set `s` to detect this case.
-
-
-Type equality modulo type distinction
--------------------------------------
-
-The following algorithm (in pseudo-code) determines whether two types
-are equal with no respect to `distinct` types. For brevity the cycle check
-with an auxiliary set `s` is omitted:
-
-.. code-block:: nim
-  proc typeEqualsOrDistinct(a, b: PType): bool =
-    if a.kind == b.kind:
-      case a.kind
-      of int, intXX, float, floatXX, char, string, cstring, pointer,
-          bool, nil, void:
-        # leaf type: kinds identical; nothing more to check
-        result = true
-      of ref, ptr, var, set, seq, openarray:
-        result = typeEqualsOrDistinct(a.baseType, b.baseType)
-      of range:
-        result = typeEqualsOrDistinct(a.baseType, b.baseType) and
-          (a.rangeA == b.rangeA) and (a.rangeB == b.rangeB)
-      of array:
-        result = typeEqualsOrDistinct(a.baseType, b.baseType) and
-                 typeEqualsOrDistinct(a.indexType, b.indexType)
-      of tuple:
-        if a.tupleLen == b.tupleLen:
-          for i in 0..a.tupleLen-1:
-            if not typeEqualsOrDistinct(a[i], b[i]): return false
-          result = true
-      of distinct:
-        result = typeEqualsOrDistinct(a.baseType, b.baseType)
-      of object, enum:
-        result = a == b
-      of proc:
-        result = typeEqualsOrDistinct(a.parameterTuple, b.parameterTuple) and
-                 typeEqualsOrDistinct(a.resultType, b.resultType) and
-                 a.callingConvention == b.callingConvention
-    elif a.kind == distinct:
-      result = typeEqualsOrDistinct(a.baseType, b)
-    elif b.kind == distinct:
-      result = typeEqualsOrDistinct(a, b.baseType)
+enumerations and distinct types and for generic types name equivalence is used.
 
 
 Subtype relation
 ----------------
-If object `a` inherits from `b`, `a` is a subtype of `b`. This subtype
-relation is extended to the types `var`, `ref`, `ptr`:
 
-.. code-block:: nim
-  proc isSubtype(a, b: PType): bool =
-    if a.kind == b.kind:
-      case a.kind
-      of object:
-        var aa = a.baseType
-        while aa != nil and aa != b: aa = aa.baseType
-        result = aa == b
-      of var, ref, ptr:
-        result = isSubtype(a.baseType, b.baseType)
+If object `a` inherits from `b`, `a` is a subtype of `b`.
 
-.. XXX nil is a special value!
+This subtype relation is extended to the types `var`, `ref`, `ptr`.
+If `A` is a subtype of `B` and `A` and `B` are `object` types then:
 
+- `var A` is a subtype of `var B`
+- `ref A` is a subtype of `ref B`
+- `ptr A` is a subtype of `ptr B`.
 
+**Note**: One of the above pointer-indirections is required for assignment from
+a subtype to its parent type to prevent "object slicing".
 
 
 Convertible relation
 --------------------
+
 A type `a` is **implicitly** convertible to type `b` iff the following
 algorithm returns true:
 
-.. code-block:: nim
-
+  ```nim
   proc isImplicitlyConvertible(a, b: PType): bool =
-    if isSubtype(a, b) or isCovariant(a, b):
+    if isSubtype(a, b):
       return true
     if isIntLiteral(a):
       return b in {int8, int16, int32, int64, int, uint, uint8, uint16,
@@ -2342,7 +2551,13 @@ algorithm returns true:
       result = b == pointer
     of string:
       result = b == cstring
+    of proc:
+      result = typeEquals(a, b) or compatibleParametersAndEffects(a, b)
+  ```
 
+We used the predicate `typeEquals(a, b)` for the "type equality" property
+and the predicate `isSubtype(a, b)` for the "subtype relation".
+`compatibleParametersAndEffects(a, b)` is currently not specified.
 
 Implicit conversions are also performed for Nim's `range` type
 constructor.
@@ -2358,21 +2573,24 @@ are signed integers or if both are unsigned integers.
 A type `a` is **explicitly** convertible to type `b` iff the following
 algorithm returns true:
 
-.. code-block:: nim
+  ```nim
   proc isIntegralType(t: PType): bool =
     result = isOrdinal(t) or t.kind in {float, float32, float64}
 
   proc isExplicitlyConvertible(a, b: PType): bool =
     result = false
     if isImplicitlyConvertible(a, b): return true
-    if typeEqualsOrDistinct(a, b): return true
+    if typeEquals(a, b): return true
+    if a == distinct and typeEquals(a.baseType, b): return true
+    if b == distinct and typeEquals(b.baseType, a): return true
     if isIntegralType(a) and isIntegralType(b): return true
     if isSubtype(a, b) or isSubtype(b, a): return true
+  ```
 
 The convertible relation can be relaxed by a user-defined type
 `converter`:idx:.
 
-.. code-block:: nim
+  ```nim
   converter toInt(x: char): int = result = ord(x)
 
   var
@@ -2385,6 +2603,7 @@ The convertible relation can be relaxed by a user-defined type
   # one can use the explicit form too
   x = chr.toInt
   echo x # => 97
+  ```
 
 The type conversion `T(a)` is an L-value if `a` is an L-value and
 `typeEqualsOrDistinct(T, typeof(a))` holds.
@@ -2397,16 +2616,23 @@ An expression `b` can be assigned to an expression `a` iff `a` is an
 `l-value` and `isImplicitlyConvertible(b.typ, a.typ)` holds.
 
 
-Overloading resolution
-======================
+Overload resolution
+===================
 
-In a call `p(args)` the routine `p` that matches best is selected. If
-multiple routines match equally well, the ambiguity is reported during
-semantic analysis.
+In a call `p(args)` where `p` may refer to more than one
+candidate, it is said to be a symbol choice. Overload resolution will attempt to
+find the best candidate, thus transforming the symbol choice into a resolved symbol.
+The routine `p` that matches best is selected following a series of trials explained below. 
+In order: Catagory matching, Hierarchical Order Comparison, and finally, Complexity Analysis.
 
-Every arg in args needs to match. There are multiple different categories how an
-argument can match. Let `f` be the formal parameter's type and `a` the type
-of the argument.
+If multiple candidates match equally well after all trials have been tested, the ambiguity 
+is reported during semantic analysis.
+
+First Trial: Catagory matching
+--------------------------------
+
+Every arg in `args` needs to match and there are multiple different categories of matches.
+Let `f` be the formal parameter's type and `a` the type of the argument.
 
 1. Exact match: `a` and `f` are of the same type.
 2. Literal match: `a` is an integer literal of value `v`
@@ -2416,7 +2642,7 @@ of the argument.
    range.
 3. Generic match: `f` is a generic type and `a` matches, for
    instance `a` is `int` and `f` is a generic (constrained) parameter
-   type (like in `[T]` or `[T: int|char]`.
+   type (like in `[T]` or `[T: int|char]`).
 4. Subrange or subtype match: `a` is a `range[T]` and `T`
    matches `f` exactly. Or: `a` is a subtype of `f`.
 5. Integral conversion match: `a` is convertible to `f` and `f` and `a`
@@ -2424,14 +2650,21 @@ of the argument.
 6. Conversion match: `a` is convertible to `f`, possibly via a user
    defined `converter`.
 
-These matching categories have a priority: An exact match is better than a
-literal match and that is better than a generic match etc. In the following,
-`count(p, m)` counts the number of matches of the matching category `m`
-for the routine `p`.
+Each operand may fall into one of the categories above; the operand's
+highest priority category. The list above is in order or priority.
+If a candidate has more priority matches than all other candidates, it is selected as the
+resolved symbol.
+
+For example, if a candidate with one exact match is compared to a candidate with multiple
+generic matches and zero exact matches, the candidate with an exact match will win.
+
+Below is a pseudocode interpretation of category matching, `count(p, m)` counts the number 
+of matches of the matching category `m` for the routine `p`.
 
 A routine `p` matches better than a routine `q` if the following
-algorithm returns true::
+algorithm returns true:
 
+  ```nim
   for each matching category m in ["exact match", "literal match",
                                   "generic match", "subtype match",
                                   "integral match", "conversion match"]:
@@ -2441,11 +2674,55 @@ algorithm returns true::
     else:
       return false
   return "ambiguous"
+  ```
 
+Second Trial: Hierarchical Order Comparison
+----------------------------------------------
 
-Some examples:
+The hierarchical order of a type is analogous to its relative specificity. Consider the type defined:
 
-.. code-block:: nim
+```nim
+type A[T] = object
+```
+
+Matching formals for this type include `T`, `object`, `A`, `A[...]` and `A[C]` where `C` is a concrete type, `A[...]`
+is a generic typeclass composition and `T` is an unconstrained generic type variable. This list is in order of 
+specificity with respect to `A` as each subsequent category narrows the set of types that are members of their match set.
+
+In this trail, the formal parameters of candidates are compared in order (1st parameter, 2nd parameter, etc.) to search for
+a candidate that has an unrivaled specificity. If such a formal parameter is found, the candidate it belongs to is chosen 
+as the resolved symbol.
+
+Third Trial: Complexity Analysis
+----------------------------------
+
+A slight clarification: While category matching digests all the formal parameters of a candidate at once (order doesn't matter),
+specificity comparison and complexity analysis operate on each formal parameter at a time. The following
+is the final trial to disambiguate a symbol choice when a pair of formal parameters have the same hierarchical order.
+
+The complexity of a type is essentially its number of modifiers and depth of shape. The definition with the *highest*
+complexity wins. Consider the following types:
+
+```nim
+type
+  A[T] = object
+  B[T, H] = object
+```
+
+Note: The below examples are not exhaustive.
+
+We shall say that:
+
+1. `A[T]` has a higher complexity than `A`
+2. `var A[T]` has a higher complexity than `A[T]`
+3. `A[A[T]]` has a higher complexity than `A[T]`
+4. `B[T, H]` has a higher complexity than `A[T]` (`A` and `B` are not compatible here, but convoluted versions of this exist)
+5. `B[ptr T, H]` has a higher complexity than `B[T, H]`
+
+Some Examples
+---------------
+
+  ```nim
   proc takesInt(x: int) = echo "int"
   proc takesInt[T](x: T) = echo "T"
   proc takesInt(x: int16) = echo "int16"
@@ -2457,14 +2734,14 @@ Some examples:
   takesInt(y) # "int16"
   var z: range[0..4] = 0
   takesInt(z) # "T"
+  ```
 
 
-If this algorithm returns "ambiguous" further disambiguation is performed:
 If the argument `a` matches both the parameter type `f` of `p`
 and `g` of `q` via a subtyping relation, the inheritance depth is taken
 into account:
 
-.. code-block:: nim
+  ```nim
   type
     A = object of RootObj
     B = object of A
@@ -2486,18 +2763,37 @@ into account:
 
   # but this is ambiguous:
   pp(c, c)
+  ```
 
 
 Likewise, for generic matches, the most specialized generic type (that still
 matches) is preferred:
 
-.. code-block:: nim
+  ```nim
   proc gen[T](x: ref ref T) = echo "ref ref T"
   proc gen[T](x: ref T) = echo "ref T"
   proc gen[T](x: T) = echo "T"
 
   var ri: ref int
   gen(ri) # "ref T"
+  ```
+
+Type variables match
+----------------------
+
+When overload resolution is considering candidates, the type variable's definition
+is not overlooked as it is used to define the formal parameter's type via variable substitution.
+
+For example:
+```nim
+type A
+proc p[T: A](param: T)
+proc p[T: object](param: T)
+```
+
+These signatures are not ambiguous for a concrete type of `A` even though the formal parameters match ("T" == "T").
+Instead `T` is treated as a variable in that (`T` ?= `T`) depending on the bound type of `T` at the time of
+overload resolution.
 
 
 Overloading based on 'var T'
@@ -2508,7 +2804,7 @@ in addition to the ordinary type checking,
 the argument is checked to be an `l-value`:idx:.
 `var T` matches better than just `T` then.
 
-.. code-block:: nim
+  ```nim
   proc sayHi(x: int): string =
     # matches a non-var int
     result = $x
@@ -2523,6 +2819,7 @@ the argument is checked to be an `l-value`:idx:.
 
   sayHello(3) # 3
               # 13
+  ```
 
 
 Lazy type resolution for untyped
@@ -2536,10 +2833,11 @@ in overloading resolution, it's essential to have a way to pass unresolved
 expressions to a template or macro. This is what the meta-type `untyped`
 accomplishes:
 
-.. code-block:: nim
+  ```nim
   template rem(x: untyped) = discard
 
   rem unresolvedExpression(undeclaredIdentifier)
+  ```
 
 A parameter of type `untyped` always matches any argument (as long as there is
 any argument passed to it).
@@ -2547,12 +2845,13 @@ any argument passed to it).
 But one has to watch out because other overloads might trigger the
 argument's resolution:
 
-.. code-block:: nim
+  ```nim
   template rem(x: untyped) = discard
   proc rem[T](x: T) = discard
 
   # undeclared identifier: 'unresolvedExpression'
   rem unresolvedExpression(undeclaredIdentifier)
+  ```
 
 `untyped` and `varargs[untyped]` are the only metatype that are lazy in this sense, the other
 metatypes `typed` and `typedesc` are not lazy.
@@ -2561,7 +2860,68 @@ metatypes `typed` and `typedesc` are not lazy.
 Varargs matching
 ----------------
 
-See `Varargs <#types-varargs>`_.
+See [Varargs].
+
+
+iterable
+--------
+
+A called `iterator` yielding type `T` can be passed to a template or macro via
+a parameter typed as `untyped` (for unresolved expressions) or the type class
+`iterable` or `iterable[T]` (after type checking and overload resolution).
+
+  ```nim
+  iterator iota(n: int): int =
+    for i in 0..`_
-for details.
+See [Constants and Constant Expressions] for details.
 
 Static statement/expression
 ---------------------------
@@ -2808,14 +3228,27 @@ Static statement/expression
 A static statement/expression explicitly requires compile-time execution.
 Even some code that has side effects is permitted in a static block:
 
-.. code-block::
-
+  ```nim
   static:
     echo "echo at compile time"
+  ```
+
+`static` can also be used like a routine.
+
+  ```nim
+  proc getNum(a: int): int = a
+
+  # Below calls "echo getNum(123)" at compile time.
+  static:
+    echo getNum(123)
+
+  # Below call evaluates the "getNum(123)" at compile time, but its
+  # result gets used at run time.
+  echo static(getNum(123))
+  ```
 
 There are limitations on what Nim code can be executed at compile time;
-see `Restrictions on Compile-Time Execution
-<#restrictions-on-compileminustime-execution>`_ for details.
+see [Restrictions on Compile-Time Execution] for details.
 It's a static error if the compiler cannot execute the block at compile
 time.
 
@@ -2825,8 +3258,7 @@ If statement
 
 Example:
 
-.. code-block:: nim
-
+  ```nim
   var name = readLine(stdin)
 
   if name == "Andreas":
@@ -2835,10 +3267,11 @@ Example:
     echo "Don't you have a name?"
   else:
     echo "Boring name..."
+  ```
 
 The `if` statement is a simple way to make a branch in the control flow:
 The expression after the keyword `if` is evaluated, if it is true
-the corresponding statements after the `:` are executed. Otherwise
+the corresponding statements after the `:` are executed. Otherwise,
 the expression after the `elif` is evaluated (if there is an
 `elif` branch), if it is true the corresponding statements after
 the `:` are executed. This goes on until the last `elif`. If all
@@ -2851,26 +3284,28 @@ corresponding *then* block.
 For visualization purposes the scopes have been enclosed
 in `{|  |}` in the following example:
 
-.. code-block:: nim
+  ```nim
   if {| (let m = input =~ re"(\w+)=\w+"; m.isMatch):
     echo "key ", m[0], " value ", m[1]  |}
   elif {| (let m = input =~ re""; m.isMatch):
     echo "new m in this scope"  |}
   else: {|
     echo "m not declared here"  |}
+  ```
 
 Case statement
 --------------
 
 Example:
 
-.. code-block:: nim
-
-  case readline(stdin)
+  ```nim
+  let line = readline(stdin)
+  case line
   of "delete-everything", "restart-computer":
     echo "permission denied"
   of "go-for-a-walk":     echo "please yourself"
-  else:                   echo "unknown command"
+  elif line.len == 0:     echo "empty" # optional, must come after `of` branches
+  else:                   echo "unknown command" # ditto
 
   # indentation of the branches is also allowed; and so is an optional colon
   # after the selecting expression:
@@ -2879,21 +3314,29 @@ Example:
       echo "permission denied"
     of "go-for-a-walk":     echo "please yourself"
     else:                   echo "unknown command"
+  ```
 
 
-The `case` statement is similar to the if statement, but it represents
+The `case` statement is similar to the `if` statement, but it represents
 a multi-branch selection. The expression after the keyword `case` is
 evaluated and if its value is in a *slicelist* the corresponding statements
 (after the `of` keyword) are executed. If the value is not in any
-given *slicelist* the `else` part is executed. If there is no `else`
-part and not all possible values that `expr` can hold occur in a
-*slicelist*, a static error occurs. This holds only for expressions of
-ordinal types. "All possible values" of `expr` are determined by `expr`'s
-type. To suppress the static error an `else` part with an
-empty `discard` statement should be used.
+given *slicelist*, trailing `elif` and `else` parts are executed using same
+semantics as for `if` statement, and `elif` is handled just like `else: if`.
+If there are no `else` or `elif` parts and not
+all possible values that `expr` can hold occur in a *slicelist*, a static error occurs.
+This holds only for expressions of ordinal types.
+"All possible values" of `expr` are determined by `expr`'s type.
+To suppress the static error an `else: discard` should be used.
+
+Only ordinal types, floats, strings and cstrings are allowed as values
+in case statements.
 
 For non-ordinal types, it is not possible to list every possible value and so
 these always require an `else` part.
+An exception to this rule is for the `string` type, which currently doesn't
+require a trailing `else` or `elif` branch; it's unspecified whether this will
+keep working in future versions.
 
 Because case statements are checked for exhaustiveness during semantic analysis,
 the value in every `of` branch must be a constant expression.
@@ -2903,7 +3346,7 @@ As a special semantic extension, an expression in an `of` branch of a case
 statement may evaluate to a set or array constructor; the set or array is then
 expanded into a list of its elements:
 
-.. code-block:: nim
+  ```nim
   const
     SymChars: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
 
@@ -2919,11 +3362,12 @@ expanded into a list of its elements:
     of 'a'..'z', 'A'..'Z', '\x80'..'\xFF', '_': echo "an identifier"
     of '0'..'9': echo "a number"
     else: echo "other"
+  ```
 
 The `case` statement doesn't produce an l-value, so the following example
 won't work:
 
-.. code-block:: nim
+  ```nim
   type
     Foo = ref object
       x: seq[string]
@@ -2938,16 +3382,18 @@ won't work:
 
   var foo = Foo(x: @[])
   foo.get_x().add("asd")
+  ```
 
-This can be fixed by explicitly using `return`:
+This can be fixed by explicitly using `result` or `return`:
 
-.. code-block:: nim
+  ```nim
   proc get_x(x: Foo): var seq[string] =
     case true
     of true:
-      return x.x
+      result = x.x
     else:
-      return x.x
+      result = x.x
+  ```
 
 
 When statement
@@ -2955,8 +3401,7 @@ When statement
 
 Example:
 
-.. code-block:: nim
-
+  ```nim
   when sizeof(int) == 2:
     echo "running on a 16 bit system!"
   elif sizeof(int) == 4:
@@ -2965,6 +3410,7 @@ Example:
     echo "running on a 64 bit system!"
   else:
     echo "cannot happen!"
+  ```
 
 The `when` statement is almost identical to the `if` statement with some
 exceptions:
@@ -2989,7 +3435,7 @@ compile-time and the executable.
 
 Example:
 
-.. code-block:: nim
+  ```nim
   proc someProcThatMayRunInCompileTime(): bool =
     when nimvm:
       # This branch is taken at compile time.
@@ -3001,6 +3447,7 @@ Example:
   let rtValue = someProcThatMayRunInCompileTime()
   assert(ctValue == true)
   assert(rtValue == false)
+  ```
 
 A `when nimvm` statement must meet the following requirements:
 
@@ -3017,16 +3464,18 @@ Return statement
 
 Example:
 
-.. code-block:: nim
-  return 40+2
+  ```nim
+  return 40 + 2
+  ```
 
 The `return` statement ends the execution of the current procedure.
 It is only allowed in procedures. If there is an `expr`, this is syntactic
 sugar for:
 
-.. code-block:: nim
+  ```nim
   result = expr
   return result
+  ```
 
 
 `return` without an expression is a short notation for `return result` if
@@ -3034,9 +3483,10 @@ the proc has a return type. The `result`:idx: variable is always the return
 value of the procedure. It is automatically declared by the compiler. As all
 variables, `result` is initialized to (binary) zero:
 
-.. code-block:: nim
+  ```nim
   proc returnZero(): int =
     # implicitly returns 0
+  ```
 
 
 Yield statement
@@ -3044,14 +3494,15 @@ Yield statement
 
 Example:
 
-.. code-block:: nim
+  ```nim
   yield (1, 2, 3)
+  ```
 
 The `yield` statement is used instead of the `return` statement in
 iterators. It is only valid in iterators. Execution is returned to the body
 of the for loop that called the iterator. Yield does not end the iteration
 process, but the execution is passed back to the iterator if the next iteration
-starts. See the section about iterators (`Iterators and the for statement`_)
+starts. See the section about iterators ([Iterators and the for statement])
 for further information.
 
 
@@ -3060,7 +3511,7 @@ Block statement
 
 Example:
 
-.. code-block:: nim
+  ```nim
   var found = false
   block myblock:
     for i in 0..3:
@@ -3069,6 +3520,7 @@ Example:
           found = true
           break myblock # leave the block, in this case both for-loops
   echo found
+  ```
 
 The block statement is a means to group statements to a (named) `block`.
 Inside the block, the `break` statement is allowed to leave the block
@@ -3081,8 +3533,9 @@ Break statement
 
 Example:
 
-.. code-block:: nim
+  ```nim
   break
+  ```
 
 The `break` statement is used to leave a block immediately. If `symbol`
 is given, it is the name of the enclosing block that is to be left. If it is
@@ -3094,12 +3547,13 @@ While statement
 
 Example:
 
-.. code-block:: nim
+  ```nim
   echo "Please tell me your password:"
   var pw = readLine(stdin)
   while pw != "12345":
     echo "Wrong password! Next try:"
     pw = readLine(stdin)
+  ```
 
 
 The `while` statement is executed until the `expr` evaluates to false.
@@ -3114,20 +3568,22 @@ A `continue` statement leads to the immediate next iteration of the
 surrounding loop construct. It is only allowed within a loop. A continue
 statement is syntactic sugar for a nested block:
 
-.. code-block:: nim
+  ```nim
   while expr1:
     stmt1
     continue
     stmt2
+  ```
 
 Is equivalent to:
 
-.. code-block:: nim
+  ```nim
   while expr1:
     block myBlockName:
       stmt1
       break myBlockName
       stmt2
+  ```
 
 
 Assembler statement
@@ -3136,9 +3592,9 @@ Assembler statement
 The direct embedding of assembler code into Nim code is supported
 by the unsafe `asm` statement. Identifiers in the assembler code that refer to
 Nim identifiers shall be enclosed in a special character which can be
-specified in the statement's pragmas. The default special character is ``'`'``:
+specified in the statement's pragmas. The default special character is `'\`'`:
 
-.. code-block:: nim
+  ```nim
   {.push stackTrace:off.}
   proc addInt(a, b: int): int =
     # a in eax, and b in edx
@@ -3150,10 +3606,11 @@ specified in the statement's pragmas. The default special character is ``'`'``:
       theEnd:
     """
   {.pop.}
+  ```
 
 If the GNU assembler is used, quotes and newlines are inserted automatically:
 
-.. code-block:: nim
+  ```nim
   proc addInt(a, b: int): int =
     asm """
       addl %%ecx, %%eax
@@ -3163,10 +3620,11 @@ If the GNU assembler is used, quotes and newlines are inserted automatically:
       :"=a"(`result`)
       :"a"(`a`), "c"(`b`)
     """
+  ```
 
 Instead of:
 
-.. code-block:: nim
+  ```nim
   proc addInt(a, b: int): int =
     asm """
       "addl %%ecx, %%eax\n"
@@ -3176,23 +3634,25 @@ Instead of:
       :"=a"(`result`)
       :"a"(`a`), "c"(`b`)
     """
+  ```
 
 Using statement
 ---------------
 
-The using statement provides syntactic convenience in modules where
+The `using` statement provides syntactic convenience in modules where
 the same parameter names and types are used over and over. Instead of:
 
-.. code-block:: nim
+  ```nim
   proc foo(c: Context; n: Node) = ...
   proc bar(c: Context; n: Node, counter: int) = ...
   proc baz(c: Context; n: Node) = ...
+  ```
 
 One can tell the compiler about the convention that a parameter of
 name `c` should default to type `Context`, `n` should default to
 `Node` etc.:
 
-.. code-block:: nim
+  ```nim
   using
     c: Context
     n: Node
@@ -3206,6 +3666,7 @@ name `c` should default to type `Context`, `n` should default to
     # 'c' is inferred to be of the type 'Context'
     # 'n' is inferred to be of the type 'Node'
     # But 'x' and 'y' are of type 'int'.
+  ```
 
 The `using` section uses the same indentation based grouping syntax as
 a `var` or `let` section.
@@ -3220,27 +3681,28 @@ that are explicitly typed is possible and requires a semicolon between them.
 If expression
 -------------
 
-An `if expression` is almost like an if statement, but it is an expression.
-This feature is similar to `ternary operators` in other languages.
+An `if` expression is almost like an if statement, but it is an expression.
+This feature is similar to *ternary operators* in other languages.
 Example:
 
-.. code-block:: nim
+  ```nim
   var y = if x > 8: 9 else: 10
+  ```
 
-An if expression always results in a value, so the `else` part is
-required. `Elif` parts are also allowed.
+An `if` expression always results in a value, so the `else` part is
+required. `elif` parts are also allowed.
 
 When expression
 ---------------
 
-Just like an `if expression`, but corresponding to the when statement.
+Just like an `if` expression, but corresponding to the `when` statement.
 
 Case expression
 ---------------
 
-The `case expression` is again very similar to the case statement:
+The `case` expression is again very similar to the case statement:
 
-.. code-block:: nim
+  ```nim
   var favoriteFood = case animal
     of "dog": "bones"
     of "cat": "mice"
@@ -3248,6 +3710,7 @@ The `case expression` is again very similar to the case statement:
     else:
       echo "I'm not sure what to serve, but everybody loves ice cream"
       "ice cream"
+  ```
 
 As seen in the above example, the case expression can also introduce side
 effects. When multiple statements are given for a branch, Nim will use
@@ -3256,28 +3719,30 @@ the last expression as the result value.
 Block expression
 ----------------
 
-A `block expression` is almost like a block statement, but it is an expression
+A `block` expression is almost like a block statement, but it is an expression
 that uses the last expression under the block as the value.
 It is similar to the statement list expression, but the statement list expression
 does not open a new block scope.
 
-.. code-block:: nim
+  ```nim
   let a = block:
     var fib = @[0, 1]
     for i in 0..10:
       fib.add fib[^1] + fib[^2]
     fib
+  ```
 
 Table constructor
 -----------------
 
 A table constructor is syntactic sugar for an array constructor:
 
-.. code-block:: nim
+  ```nim
   {"key1": "value1", "key2", "key3": "value2"}
 
   # is the same as:
   [("key1", "value1"), ("key2", "value2"), ("key3", "value2")]
+  ```
 
 
 The empty table can be written `{:}` (in contrast to the empty set
@@ -3310,13 +3775,13 @@ can be used to convert from floating-point to integer or vice versa.
 
 Type conversion can also be used to disambiguate overloaded routines:
 
-.. code-block:: nim
-
+  ```nim
   proc p(x: int) = echo "int"
   proc p(x: string) = echo "string"
 
   let procVar = (proc(x: string))(p)
   procVar("a")
+  ```
 
 Since operations on unsigned numbers wrap around and are unchecked so are
 type conversions to unsigned integers and between unsigned integers. The
@@ -3339,21 +3804,26 @@ Type casts
 as if it would be of another type. Type casts are only needed for low-level
 programming and are inherently unsafe.
 
-.. code-block:: nim
+  ```nim
   cast[int](x)
+  ```
 
 The target type of a cast must be a concrete type, for instance, a target type
 that is a type class (which is non-concrete) would be invalid:
 
-.. code-block:: nim
+  ```nim
   type Foo = int or float
   var x = cast[Foo](1) # Error: cannot cast to a non concrete type: 'Foo'
+  ```
 
 Type casts should not be confused with *type conversions,* as mentioned in the
 prior section. Unlike type conversions, a type cast cannot change the underlying
-bit pattern of the data being casted (aside from that the size of the target type
+bit pattern of the data being cast (aside from that the size of the target type
 may differ from the source type). Casting resembles *type punning* in other
-languages or C++'s `reinterpret_cast` and `bit_cast` features.
+languages or C++'s `reinterpret_cast`:cpp: and `bit_cast`:cpp: features.
+
+If the size of the target type is larger than the size of the source type,
+the remaining memory is zeroed.
 
 The addr operator
 -----------------
@@ -3362,11 +3832,11 @@ location is `T`, the `addr` operator result is of the type `ptr T`. An
 address is always an untraced reference. Taking the address of an object that
 resides on the stack is **unsafe**, as the pointer may live longer than the
 object on the stack and can thus reference a non-existing object. One can get
-the address of variables, but one can't use it on variables declared through
-`let` statements:
-
-.. code-block:: nim
+the address of variables. For easier interoperability with other compiled languages
+such as C, retrieving the address of a `let` variable, a parameter,
+or a `for` loop variable can be accomplished too:
 
+  ```nim
   let t1 = "Hello"
   var
     t2 = t1
@@ -3375,23 +3845,19 @@ the address of variables, but one can't use it on variables declared through
   # --> ref 0x7fff6b71b670 --> 0x10bb81050"Hello"
   echo cast[ptr string](t3)[]
   # --> Hello
-  # The following line doesn't compile:
+  # The following line also works
   echo repr(addr(t1))
-  # Error: expression has no address
-
+  ```
 
 The unsafeAddr operator
 -----------------------
 
-For easier interoperability with other compiled languages such as C, retrieving
-the address of a `let` variable, a parameter, or a `for` loop variable can
-be accomplished by using the `unsafeAddr` operation:
-
-.. code-block:: nim
+The `unsafeAddr` operator is a deprecated alias for the `addr` operator:
 
+  ```nim
   let myArray = [1, 2, 3]
   foreignProcThatTakesAnAddr(unsafeAddr myArray)
-
+  ```
 
 Procedures
 ==========
@@ -3406,7 +3872,7 @@ until either the beginning of the parameter list, a semicolon separator, or an
 already typed parameter, is reached. The semicolon can be used to make
 separation of types and subsequent identifiers more distinct.
 
-.. code-block:: nim
+  ```nim
   # Using only commas
   proc foo(a, b: int, c, d: bool): int
 
@@ -3415,31 +3881,34 @@ separation of types and subsequent identifiers more distinct.
 
   # Will fail: a is untyped since ';' stops type propagation.
   proc foo(a; b: int; c, d: bool): int
+  ```
 
 A parameter may be declared with a default value which is used if the caller
-does not provide a value for the argument.
+does not provide a value for the argument. The value will be reevaluated
+every time the function is called.
 
-.. code-block:: nim
-  # b is optional with 47 as its default value
+  ```nim
+  # b is optional with 47 as its default value.
   proc foo(a: int, b: int = 47): int
+  ```
 
 Parameters can be declared mutable and so allow the proc to modify those
 arguments, by using the type modifier `var`.
 
-.. code-block:: nim
+  ```nim
   # "returning" a value to the caller through the 2nd argument
   # Notice that the function uses no actual return value at all (ie void)
   proc foo(inp: int, outp: var int) =
     outp = inp + 47
+  ```
 
-If the proc declaration has no body, it is a `forward`:idx: declaration. If the
-proc returns a value, the procedure body can access an implicitly declared
+If the proc declaration doesn't have a body, it is a `forward`:idx: declaration.
+If the proc returns a value, the procedure body can access an implicitly declared
 variable named `result`:idx: that represents the return value. Procs can be
 overloaded. The overloading resolution algorithm determines which proc is the
 best match for the arguments. Example:
 
-.. code-block:: nim
-
+  ```nim
   proc toLower(c: char): char = # toLower for characters
     if c in {'A'..'Z'}:
       result = chr(ord(c) + (ord('a') - ord('A')))
@@ -3450,10 +3919,11 @@ best match for the arguments. Example:
     result = newString(len(s))
     for i in 0..len(s) - 1:
       result[i] = toLower(s[i]) # calls toLower for characters; no recursion!
+  ```
 
-Calling a procedure can be done in many different ways:
+Calling a procedure can be done in many ways:
 
-.. code-block:: nim
+  ```nim
   proc callme(x, y: int, s: string = "", c: char, b: bool = false) = ...
 
   # call with positional arguments      # parameter bindings:
@@ -3464,16 +3934,18 @@ Calling a procedure can be done in many different ways:
   callme(c='\t', y=1, x=0)              # (x=0, y=1, s="", c='\t', b=false)
   # call as a command statement: no () needed:
   callme 0, 1, "abc", '\t'              # (x=0, y=1, s="abc", c='\t', b=false)
+  ```
 
 A procedure may call itself recursively.
 
 
 `Operators`:idx: are procedures with a special operator symbol as identifier:
 
-.. code-block:: nim
+  ```nim
   proc `$` (x: int): string =
     # converts an integer to a string; this is a prefix operator.
     result = intToStr(x)
+  ```
 
 Operators with one parameter are prefix operators, operators with two
 parameters are infix operators. (However, the parser distinguishes these from
@@ -3481,15 +3953,16 @@ the operator's position within an expression.) There is no way to declare
 postfix operators: all postfix operators are built-in and handled by the
 grammar explicitly.
 
-Any operator can be called like an ordinary proc with the '`opr`'
+Any operator can be called like an ordinary proc with the \`opr\`
 notation. (Thus an operator can have more than two parameters):
 
-.. code-block:: nim
+  ```nim
   proc `*+` (a, b, c: int): int =
     # Multiply and add
     result = a * b + c
 
   assert `*+`(3, 4, 6) == `+`(`*`(a, b), c)
+  ```
 
 
 Export marker
@@ -3498,8 +3971,7 @@ Export marker
 If a declared symbol is marked with an `asterisk`:idx: it is exported from the
 current module:
 
-.. code-block:: nim
-
+  ```nim
   proc exportedEcho*(s: string) = echo s
   proc `*`*(a: string; b: int): string =
     result = newStringOfCap(a.len * b)
@@ -3510,24 +3982,25 @@ current module:
   type
     ExportedType* = object
       exportedField*: int
+  ```
 
 
 Method call syntax
 ------------------
 
-For object-oriented programming, the syntax `obj.method(args)` can be used
-instead of `method(obj, args)`. The parentheses can be omitted if there are no
-remaining arguments: `obj.len` (instead of `len(obj)`).
+For object-oriented programming, the syntax `obj.methodName(args)` can be used
+instead of `methodName(obj, args)`. The parentheses can be omitted if
+there are no remaining arguments: `obj.len` (instead of `len(obj)`).
 
 This method call syntax is not restricted to objects, it can be used
 to supply any type of first argument for procedures:
 
-.. code-block:: nim
-
+  ```nim
   echo "abc".len # is the same as echo len "abc"
   echo "abc".toUpper()
   echo {'a', 'b', 'c'}.card
   stdout.writeLine("Hallo") # the same as writeLine(stdout, "Hallo")
+  ```
 
 Another way to look at the method call syntax is that it provides the missing
 postfix notation.
@@ -3536,8 +4009,7 @@ The method call syntax conflicts with explicit generic instantiations:
 `p[T](x)` cannot be written as `x.p[T]` because `x.p[T]` is always
 parsed as `(x.p)[T]`.
 
-See also: `Limitations of the method call syntax
-<#templates-limitations-of-the-method-call-syntax>`_.
+See also: [Limitations of the method call syntax].
 
 The `[: ]` notation has been designed to mitigate this issue: `x.p[:T]`
 is rewritten by the parser to `p[T](x)`, `x.p[:T](y)` is rewritten to
@@ -3551,7 +4023,7 @@ Nim has no need for *get-properties*: Ordinary get-procedures that are called
 with the *method call syntax* achieve the same. But setting a value is
 different; for this, a special setter syntax is needed:
 
-.. code-block:: nim
+  ```nim
   # Module asocket
   type
     Socket* = ref object of RootObj
@@ -3570,24 +4042,26 @@ different; for this, a special setter syntax is needed:
     ## `host` because the builtin dot access is preferred if it is
     ## available:
     s.host
+  ```
 
-.. code-block:: nim
+  ```nim
   # module B
   import asocket
   var s: Socket
   new s
   s.host = 34  # same as `host=`(s, 34)
+  ```
 
 A proc defined as `f=` (with the trailing `=`) is called
 a `setter`:idx:. A setter can be called explicitly via the common
 backticks notation:
 
-.. code-block:: nim
-
+  ```nim
   proc `f=`(x: MyObject; value: string) =
     discard
 
   `f=`(myObject, "value")
+  ```
 
 
 `f=` can be called implicitly in the pattern
@@ -3608,7 +4082,7 @@ means `echo f 1, f 2` is parsed as `echo(f(1), f(2))` and not as
 `echo(f(1, f(2)))`. The method call syntax may be used to provide one
 more argument in this case:
 
-.. code-block:: nim
+  ```nim
   proc optarg(x: int, y: int = 0): int = x + y
   proc singlearg(x: int): int = 20*x
 
@@ -3618,9 +4092,10 @@ more argument in this case:
   let x = optarg(1, optarg 8)  # traditional procedure call with 2 arguments
   let y = 1.optarg optarg 8    # same thing as above, w/o the parenthesis
   assert x == y
+  ```
 
 The command invocation syntax also can't have complex expressions as arguments.
-For example: (`anonymous procs <#procedures-anonymous-procs>`_), `if`,
+For example: [anonymous procedures], `if`,
 `case` or `try`. Function calls with no arguments still need () to
 distinguish between a call and the function itself as a first-class value.
 
@@ -3637,45 +4112,85 @@ the closure and its enclosing scope (i.e. any modifications made to them are
 visible in both places). The closure environment may be allocated on the heap
 or on the stack if the compiler determines that this would be safe.
 
-Creating closures in loops
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+### Creating closures in loops
 
 Since closures capture local variables by reference it is often not wanted
-behavior inside loop bodies. See `closureScope
-`_ and `capture
-`_ for details on how to change this behavior.
+behavior inside loop bodies. See [closureScope](
+system.html#closureScope.t,untyped) and [capture](
+sugar.html#capture.m,varargs[typed],untyped) for details on how to change this behavior.
 
-Anonymous Procs
----------------
+Anonymous procedures
+--------------------
 
 Unnamed procedures can be used as lambda expressions to pass into other
 procedures:
 
-.. code-block:: nim
+  ```nim
   var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"]
 
-  cities.sort(proc (x,y: string): int =
-      cmp(x.len, y.len))
+  cities.sort(proc (x, y: string): int =
+    cmp(x.len, y.len))
+  ```
 
 
 Procs as expressions can appear both as nested procs and inside top-level
-executable code. The  `sugar `_ module contains the `=>` macro
+executable code. The  [sugar](sugar.html) module contains the `=>` macro
 which enables a more succinct syntax for anonymous procedures resembling
 lambdas as they are in languages like JavaScript, C#, etc.
 
+Do notation
+-----------
+
+As a special convenience notation that keeps most elements of a
+regular proc expression, the `do` keyword can be used to pass
+anonymous procedures to routines:
+
+  ```nim
+  var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"]
+
+  sort(cities) do (x, y: string) -> int:
+    cmp(x.len, y.len)
+
+  # Less parentheses using the method plus command syntax:
+  cities = cities.map do (x: string) -> string:
+    "City of " & x
+  ```
+
+`do` is written after the parentheses enclosing the regular proc parameters.
+The proc expression represented by the `do` block is appended to the routine
+call as the last argument. In calls using the command syntax, the `do` block
+will bind to the immediately preceding expression rather than the command call.
+
+`do` with a parameter list or pragma list corresponds to an anonymous `proc`,
+however `do` without parameters or pragmas is treated as a normal statement
+list. This allows macros to receive both indented statement lists as an
+argument in inline calls, as well as a direct mirror of Nim's routine syntax.
+
+  ```nim
+  # Passing a statement list to an inline macro:
+  macroResults.add quote do:
+    if not `ex`:
+      echo `info`, ": Check failed: ", `expString`
+
+  # Processing a routine definition in a macro:
+  rpc(router, "add") do (a, b: int) -> int:
+    result = a + b
+  ```
 
 Func
 ----
 
 The `func` keyword introduces a shortcut for a `noSideEffect`:idx: proc.
 
-.. code-block:: nim
+  ```nim
   func binarySearch[T](a: openArray[T]; elem: T): int
+  ```
 
 Is short for:
 
-.. code-block:: nim
+  ```nim
   proc binarySearch[T](a: openArray[T]; elem: T): int {.noSideEffect.}
+  ```
 
 
 
@@ -3689,11 +4204,11 @@ Type bound operators
 
 A type bound operator is a `proc` or `func` whose name starts with `=` but isn't an operator
 (i.e. containing only symbols, such as `==`). These are unrelated to setters
-(see `properties `_), which instead end in `=`.
+(see [Properties]), which instead end in `=`.
 A type bound operator declared for a type applies to the type regardless of whether
 the operator is in scope (including if it is private).
 
-.. code-block:: nim
+  ```nim
   # foo.nim:
   var witness* = 0
   type Foo[T] = object
@@ -3713,37 +4228,50 @@ the operator is in scope (including if it is private).
     doAssert witness == 2
   # will still be called upon exiting scope
   doAssert witness == 3
+  ```
 
-Type bound operators currently include:
-`=destroy`, `=copy`, `=sink`, `=trace`, `=dispose`, `=deepcopy`
-(some of which are still implementation defined and not yet documented).
+Type bound operators are:
+`=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy`, `=wasMoved`, `=dup`.
+
+These operations can be *overridden* instead of *overloaded*. This means that
+the implementation is automatically lifted to structured types. For instance,
+if the type `T` has an overridden assignment operator `=`, this operator is
+also used for assignments of the type `seq[T]`.
+
+Since these operations are bound to a type, they have to be bound to a
+nominal type for reasons of simplicity of implementation; this means an
+overridden `deepCopy` for `ref T` is really bound to `T` and not to `ref T`.
+This also means that one cannot override `deepCopy` for both `ptr T` and
+`ref T` at the same time, instead a distinct or object helper type has to be
+used for one pointer type.
 
 For more details on some of those procs, see
-`lifetimeminustracking-hooks `_.
+[Lifetime-tracking hooks](destructors.html#lifetimeminustracking-hooks).
 
 Nonoverloadable builtins
 ------------------------
 
 The following built-in procs cannot be overloaded for reasons of implementation
-simplicity (they require specialized semantic checking)::
+simplicity (they require specialized semantic checking):
 
-  declared, defined, definedInScope, compiles, sizeof,
-  is, shallowCopy, getAst, astToStr, spawn, procCall
+    declared, defined, definedInScope, compiles, sizeof,
+    is, shallowCopy, getAst, astToStr, spawn, procCall
 
-Thus they act more like keywords than like ordinary identifiers; unlike a
+Thus, they act more like keywords than like ordinary identifiers; unlike a
 keyword however, a redefinition may `shadow`:idx: the definition in
-the `system` module. From this list the following should not be written in dot
+the [system](system.html) module.
+From this list the following should not be written in dot
 notation `x.f` since `x` cannot be type-checked before it gets passed
-to `f`::
+to `f`:
 
-  declared, defined, definedInScope, compiles, getAst, astToStr
+    declared, defined, definedInScope, compiles, getAst, astToStr
 
 
 Var parameters
 --------------
 The type of a parameter may be prefixed with the `var` keyword:
 
-.. code-block:: nim
+  ```nim
   proc divmod(a, b: int; res, remainder: var int) =
     res = a div b
     remainder = a mod b
@@ -3754,6 +4282,7 @@ The type of a parameter may be prefixed with the `var` keyword:
   divmod(8, 5, x, y) # modifies x and y
   assert x == 1
   assert y == 3
+  ```
 
 In the example, `res` and `remainder` are `var parameters`.
 Var parameters can be modified by the procedure and the changes are
@@ -3761,7 +4290,7 @@ visible to the caller. The argument passed to a var parameter has to be
 an l-value. Var parameters are implemented as hidden pointers. The
 above example is equivalent to:
 
-.. code-block:: nim
+  ```nim
   proc divmod(a, b: int; res, remainder: ptr int) =
     res[] = a div b
     remainder[] = a mod b
@@ -3771,11 +4300,12 @@ above example is equivalent to:
   divmod(8, 5, addr(x), addr(y))
   assert x == 1
   assert y == 3
+  ```
 
 In the examples, var parameters or pointers are used to provide two
 return values. This can be done in a cleaner way by returning a tuple:
 
-.. code-block:: nim
+  ```nim
   proc divmod(a, b: int): tuple[res, remainder: int] =
     (a div b, a mod b)
 
@@ -3783,13 +4313,15 @@ return values. This can be done in a cleaner way by returning a tuple:
 
   assert t.res == 1
   assert t.remainder == 3
+  ```
 
 One can use `tuple unpacking`:idx: to access the tuple's fields:
 
-.. code-block:: nim
+  ```nim
   var (x, y) = divmod(8, 5) # tuple unpacking
   assert x == 1
   assert y == 3
+  ```
 
 
 **Note**: `var` parameters are never necessary for efficient parameter
@@ -3803,7 +4335,7 @@ Var return type
 A proc, converter, or iterator may return a `var` type which means that the
 returned value is an l-value and can be modified by the caller:
 
-.. code-block:: nim
+  ```nim
   var g = 0
 
   proc writeAccessToG(): var int =
@@ -3811,36 +4343,39 @@ returned value is an l-value and can be modified by the caller:
 
   writeAccessToG() = 6
   assert g == 6
+  ```
 
 It is a static error if the implicitly introduced pointer could be
 used to access a location beyond its lifetime:
 
-.. code-block:: nim
+  ```nim
   proc writeAccessToG(): var int =
     var g = 0
     result = g # Error!
+  ```
 
 For iterators, a component of a tuple return type can have a `var` type too:
 
-.. code-block:: nim
+  ```nim
   iterator mpairs(a: var seq[string]): tuple[key: int, val: var string] =
     for i in 0..a.high:
       yield (i, a[i])
+  ```
 
 In the standard library every name of a routine that returns a `var` type
 starts with the prefix `m` per convention.
 
 
-.. include:: manual/var_t_return.rst
+.. include:: manual/var_t_return.md
 
-Future directions
-~~~~~~~~~~~~~~~~~
+### Future directions
 
 Later versions of Nim can be more precise about the borrowing rule with
 a syntax like:
 
-.. code-block:: nim
+  ```nim
   proc foo(other: Y; container: var X): var T from container
+  ```
 
 Here `var T from container` explicitly exposes that the
 location is derived from the second parameter (called
@@ -3866,7 +4401,7 @@ receives a hidden mutable parameter representing `result`.
 
 Informally:
 
-.. code-block:: nim
+  ```nim
   proc p(): BigT = ...
 
   var x = p()
@@ -3878,6 +4413,7 @@ Informally:
 
   var x; p(x)
   p(x)
+  ```
 
 
 Let `T`'s be `p`'s return type. NRVO applies for `T`
@@ -3887,8 +4423,7 @@ in other words, it applies for "big" structures.
 If `p` can raise an exception, NRVO applies regardless. This can produce
 observable differences in behavior:
 
-.. code-block:: nim
-
+  ```nim
   type
     BigT = array[16, int]
 
@@ -3905,18 +4440,19 @@ observable differences in behavior:
       doAssert x == [0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0]
 
   main()
+  ```
 
 
-However, the current implementation produces a warning in these cases.
-There are different ways to deal with this warning:
-
-1. Disable the warning via `{.push warning[ObservableStores]: off.}` ... `{.pop.}`.
-   Then one may need to ensure that `p` only raises *before* any stores to `result`
-   happen.
-
-2. One can use a temporary helper variable, for example instead of `x = p(8)`
-   use `let tmp = p(8); x = tmp`.
+The compiler can produce a warning in these cases, however this behavior is
+turned off by default. It can be enabled for a section of code via the
+`warning[ObservableStores]` and `push`/`pop` pragmas. Take the above code
+as an example:
 
+  ```nim
+  {.push warning[ObservableStores]: on.}
+  main()
+  {.pop.}
+  ```
 
 Overloading of the subscript operator
 -------------------------------------
@@ -3931,7 +4467,7 @@ Procedures always use static dispatch. Methods use dynamic
 dispatch. For dynamic dispatch to work on an object it should be a reference
 type.
 
-.. code-block:: nim
+  ```nim
   type
     Expression = ref object of RootObj ## abstract base class for an expression
     Literal = ref object of Expression
@@ -3959,6 +4495,7 @@ type.
     result.b = b
 
   echo eval(newPlus(newPlus(newLit(1), newLit(2)), newLit(4)))
+  ```
 
 In the example the constructors `newLit` and `newPlus` are procs
 because they should use static binding, but `eval` is a method because it
@@ -3978,20 +4515,18 @@ Multi-methods
 --------------
 
 **Note:** Starting from Nim 0.20, to use multi-methods one must explicitly pass
-`--multimethods:on` when compiling.
+`--multimethods:on`:option: when compiling.
 
 In a multi-method, all parameters that have an object type are used for the
 dispatching:
 
-.. code-block:: nim
-    :test: "nim c --multiMethods:on $1"
-
+  ```nim  test = "nim c --multiMethods:on $1"
   type
     Thing = ref object of RootObj
     Unit = ref object of Thing
       x: int
 
-  method collide(a, b: Thing) {.inline.} =
+  method collide(a, b: Thing) {.base, inline.} =
     quit "to override!"
 
   method collide(a: Thing, b: Unit) {.inline.} =
@@ -4004,6 +4539,7 @@ dispatching:
   new a
   new b
   collide(a, b) # output: 2
+  ```
 
 Inhibit dynamic method resolution via procCall
 -----------------------------------------------
@@ -4012,9 +4548,7 @@ Dynamic method resolution can be inhibited via the builtin `system.procCall`:idx
 This is somewhat comparable to the `super`:idx: keyword that traditional OOP
 languages offer.
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     Thing = ref object of RootObj
     Unit = ref object of Thing
@@ -4027,6 +4561,7 @@ languages offer.
     # Call the base method:
     procCall m(Thing(a))
     echo "1"
+  ```
 
 
 Iterators and the for statement
@@ -4049,7 +4584,7 @@ reached, the data is bound to the `for` loop variables and control continues
 in the body of the `for` loop. The iterator's local variables and execution
 state are automatically saved between calls. Example:
 
-.. code-block:: nim
+  ```nim
   # this definition exists in the system module
   iterator items*(a: string): char {.inline.} =
     var i = 0
@@ -4059,15 +4594,17 @@ state are automatically saved between calls. Example:
 
   for ch in items("hello world"): # `ch` is an iteration variable
     echo ch
+  ```
 
-The compiler generates code as if the programmer would have written this:
+The compiler generates code as if the programmer had written this:
 
-.. code-block:: nim
+  ```nim
   var i = 0
   while i < len(a):
     var ch = a[i]
     echo ch
     inc(i)
+  ```
 
 If the iterator yields a tuple, there can be as many iteration variables
 as there are components in the tuple. The i'th iteration variable's type is
@@ -4079,10 +4616,11 @@ Implicit items/pairs invocations
 
 If the for loop expression `e` does not denote an iterator and the for loop
 has exactly 1 variable, the for loop expression is rewritten to `items(e)`;
-ie. an `items` iterator is implicitly invoked:
+i.e. an `items` iterator is implicitly invoked:
 
-.. code-block:: nim
+  ```nim
   for x in [1,2,3]: echo x
+  ```
 
 If the for loop has exactly 2 variables, a `pairs` iterator is implicitly
 invoked.
@@ -4111,7 +4649,7 @@ templates, macros, and other inline iterators.
 
 In contrast to that, a `closure iterator`:idx: can be passed around more freely:
 
-.. code-block:: nim
+  ```nim
   iterator count0(): int {.closure.} =
     yield 0
 
@@ -4126,18 +4664,15 @@ In contrast to that, a `closure iterator`:idx: can be passed around more freely:
 
   invoke(count0)
   invoke(count2)
+  ```
 
 Closure iterators and inline iterators have some restrictions:
 
 1. For now, a closure iterator cannot be executed at compile time.
 2. `return` is allowed in a closure iterator but not in an inline iterator
    (but rarely useful) and ends the iteration.
-3. Neither inline nor closure iterators can be (directly)* recursive.
+3. Inline iterators cannot be recursive.
 4. Neither inline nor closure iterators have the special `result` variable.
-5. Closure iterators are not supported by the JS backend.
-
-(*) Closure iterators can be co-recursive with a factory proc which results
-in similar syntax to a recursive iterator.  More details follow.
 
 Iterators that are neither marked `{.closure.}` nor `{.inline.}` explicitly
 default to being inline, but this may change in future versions of the
@@ -4147,7 +4682,7 @@ The `iterator` type is always of the calling convention `closure`
 implicitly; the following example shows how to use iterators to implement
 a `collaborative tasking`:idx: system:
 
-.. code-block:: nim
+  ```nim
   # simple tasking:
   type
     Task = iterator (ticker: int)
@@ -4177,15 +4712,16 @@ a `collaborative tasking`:idx: system:
       inc ticker
 
   runTasks(a1, a2)
+  ```
 
 The builtin `system.finished` can be used to determine if an iterator has
 finished its operation; no exception is raised on an attempt to invoke an
 iterator that has already finished its work.
 
-Note that `system.finished` is error prone to use because it only returns
+Note that `system.finished` is error-prone to use because it only returns
 `true` one iteration after the iterator has finished:
 
-.. code-block:: nim
+  ```nim
   iterator mycount(a, b: int): int {.closure.} =
     var x = a
     while x <= b:
@@ -4201,15 +4737,17 @@ Note that `system.finished` is error prone to use because it only returns
   2
   3
   0
+  ```
 
-Instead this code has to be used:
+Instead, this code has to be used:
 
-.. code-block:: nim
+  ```nim
   var c = mycount # instantiate the iterator
   while true:
     let value = c(1, 3)
     if finished(c): break # and discard 'value'!
     echo value
+  ```
 
 It helps to think that the iterator actually returns a
 pair `(value, done)` and `finished` is used to access the hidden `done`
@@ -4220,7 +4758,7 @@ Closure iterators are *resumable functions* and so one has to provide the
 arguments to every call. To get around this limitation one can capture
 parameters of an outer factory proc:
 
-.. code-block:: nim
+  ```nim
   proc mycount(a, b: int): iterator (): int =
     result = iterator (): int =
       var x = a
@@ -4232,10 +4770,11 @@ parameters of an outer factory proc:
 
   for f in foo():
     echo f
+  ```
 
 The call can be made more like an inline iterator with a for loop macro:
 
-.. code-block:: nim
+  ```nim
   import std/macros
   macro toItr(x: ForLoopStmt): untyped =
     let expr = x[0]
@@ -4249,43 +4788,49 @@ The call can be made more like an inline iterator with a for loop macro:
 
   for f in toItr(mycount(1, 4)): # using early `proc mycount`
     echo f
+  ```
 
-Because of full backend function call aparatus involvment, closure iterator
-invocation is typically higher cost than inline iterators.  Adornment by
+Because of full backend function call apparatus involvement, closure iterator
+invocation is typically higher cost than inline iterators. Adornment by
 a macro wrapper at the call site like this is a possibly useful reminder.
 
-The factory `proc`, as an ordinary procedure, can be recursive.  The
+The factory `proc`, as an ordinary procedure, can be recursive. The
 above macro allows such recursion to look much like a recursive iterator
-would.  For example:
+would. For example:
 
-.. code-block:: nim
+  ```nim
   proc recCountDown(n: int): iterator(): int =
     result = iterator(): int =
       if n > 0:
         yield n
         for e in toItr(recCountDown(n - 1)):
-            yield e
+          yield e
 
   for i in toItr(recCountDown(6)): # Emits: 6 5 4 3 2 1
     echo i
+  ```
+
+
+See also [iterable] for passing iterators to templates and macros.
 
 Converters
 ==========
 
 A converter is like an ordinary proc except that it enhances
-the "implicitly convertible" type relation (see `Convertible relation`_):
+the "implicitly convertible" type relation (see [Convertible relation]):
 
-.. code-block:: nim
+  ```nim
   # bad style ahead: Nim is not C.
   converter toBool(x: int): bool = x != 0
 
   if 4:
     echo "compiles"
+  ```
 
 
 A converter can also be explicitly invoked for improved readability. Note that
 implicit converter chaining is not supported: If there is a converter from
-type A to type B and from type B to type C the implicit conversion from A to C
+type A to type B and from type B to type C, the implicit conversion from A to C
 is not provided.
 
 
@@ -4294,7 +4839,7 @@ Type sections
 
 Example:
 
-.. code-block:: nim
+  ```nim
   type # example demonstrating mutually recursive types
     Node = ref object  # an object managed by the garbage collector (ref)
       le, ri: Node     # left and right subtrees
@@ -4304,6 +4849,7 @@ Example:
       name: string     # the symbol's name
       line: int        # the line the symbol was declared in
       code: Node       # the symbol's abstract syntax tree
+  ```
 
 A type section begins with the `type` keyword. It contains multiple
 type definitions. A type definition binds a type to a name. Type definitions
@@ -4321,7 +4867,7 @@ Try statement
 
 Example:
 
-.. code-block:: nim
+  ```nim
   # read the first two lines of a text file that should contain numbers
   # and tries to add them
   var
@@ -4333,14 +4879,13 @@ Example:
       echo "sum: " & $(parseInt(a) + parseInt(b))
     except OverflowDefect:
       echo "overflow!"
-    except ValueError:
-      echo "could not convert string to integer"
-    except IOError:
-      echo "IO error!"
-    except:
-      echo "Unknown exception!"
+    except ValueError, IOError:
+      echo "catch multiple exceptions!"
+    except CatchableError:
+      echo "Catchable exception!"
     finally:
       close(f)
+  ```
 
 
 The statements after the `try` are executed in sequential order unless
@@ -4349,9 +4894,6 @@ listed in an `except` clause, the corresponding statements are executed.
 The statements following the `except` clauses are called
 `exception handlers`:idx:.
 
-The empty `except`:idx: clause is executed if there is an exception that is
-not listed otherwise. It is similar to an `else` clause in `if` statements.
-
 If there is a `finally`:idx: clause, it is always executed after the
 exception handlers.
 
@@ -4369,19 +4911,22 @@ Try can also be used as an expression; the type of the `try` branch then
 needs to fit the types of `except` branches, but the type of the `finally`
 branch always has to be `void`:
 
-.. code-block:: nim
+  ```nim test
   from std/strutils import parseInt
 
   let x = try: parseInt("133a")
-          except: -1
+          except ValueError: -1
           finally: echo "hi"
+  ```
 
 
 To prevent confusing code there is a parsing limitation; if the `try`
 follows a `(` it has to be written as a one liner:
 
-.. code-block:: nim
-  let x = (try: parseInt("133a") except: -1)
+  ```nim test
+  from std/strutils import parseInt
+  let x = (try: parseInt("133a") except ValueError: -1)
+  ```
 
 
 Except clauses
@@ -4390,59 +4935,65 @@ Except clauses
 Within an `except` clause it is possible to access the current exception
 using the following syntax:
 
-.. code-block:: nim
+  ```nim
   try:
     # ...
   except IOError as e:
     # Now use "e"
     echo "I/O error: " & e.msg
+  ```
 
 Alternatively, it is possible to use `getCurrentException` to retrieve the
 exception that has been raised:
 
-.. code-block:: nim
+  ```nim
   try:
     # ...
   except IOError:
     let e = getCurrentException()
     # Now use "e"
+  ```
 
 Note that `getCurrentException` always returns a `ref Exception`
 type. If a variable of the proper type is needed (in the example
 above, `IOError`), one must convert it explicitly:
 
-.. code-block:: nim
+  ```nim
   try:
     # ...
   except IOError:
     let e = (ref IOError)(getCurrentException())
     # "e" is now of the proper type
+  ```
 
 However, this is seldom needed. The most common case is to extract an
 error message from `e`, and for such situations, it is enough to use
 `getCurrentExceptionMsg`:
 
-.. code-block:: nim
+  ```nim
   try:
     # ...
-  except:
+  except CatchableError:
     echo getCurrentExceptionMsg()
+  ```
 
 Custom exceptions
 -----------------
 
 It is possible to create custom exceptions. A custom exception is a custom type:
 
-.. code-block:: nim
+  ```nim
   type
     LoadError* = object of Exception
+  ```
 
 Ending the custom exception's name with `Error` is recommended.
 
 Custom exceptions can be raised just like any other exception, e.g.:
 
-.. code-block:: nim
+  ```nim
   raise newException(LoadError, "Failed to load data")
+  ```
 
 Defer statement
 ---------------
@@ -4451,23 +5002,20 @@ Instead of a `try finally` statement a `defer` statement can be used, which
 avoids lexical nesting and offers more flexibility in terms of scoping as shown
 below.
 
-Any statements following the `defer` in the current block will be considered
-to be in an implicit try block:
-
-.. code-block:: nim
-    :test: "nim c $1"
+Any statements following the `defer` will be considered
+to be in an implicit try block in the current block:
 
+  ```nim  test = "nim c $1"
   proc main =
     var f = open("numbers.txt", fmWrite)
     defer: close(f)
     f.write "abc"
     f.write "def"
+  ```
 
 Is rewritten to:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   proc main =
     var f = open("numbers.txt")
     try:
@@ -4475,13 +5023,12 @@ Is rewritten to:
       f.write "def"
     finally:
       close(f)
+  ```
 
 When `defer` is at the outermost scope of a template/macro, its scope extends
-to the block where the template is called from:
-
-.. code-block:: nim
-    :test: "nim c $1"
+to the block where the template/macro is called from:
 
+  ```nim  test = "nim c $1"
   template safeOpenDefer(f, path) =
     var f = open(path, fmWrite)
     defer: close(f)
@@ -4502,6 +5049,7 @@ to the block where the template is called from:
     try:
       f.write "abc" # adds a lexical scope
     finally: close(f)
+  ```
 
 Top-level `defer` statements are not supported
 since it's unclear what such a statement should refer to.
@@ -4512,8 +5060,9 @@ Raise statement
 
 Example:
 
-.. code-block:: nim
+  ```nim
   raise newException(IOError, "IO failed")
+  ```
 
 Apart from built-in operations like array indexing, memory allocation, etc.
 the `raise` statement is the only way to raise an exception.
@@ -4529,7 +5078,7 @@ exception.
 Exception hierarchy
 -------------------
 
-The exception tree is defined in the `system `_ module.
+The exception tree is defined in the [system](system.html) module.
 Every exception inherits from `system.Exception`. Exceptions that indicate
 programming bugs inherit from `system.Defect` (which is a subtype of `Exception`)
 and are strictly speaking not catchable as they can also be mapped to an operation
@@ -4539,6 +5088,38 @@ exceptions inherit from `Defect`.
 Exceptions that indicate any other runtime error that can be caught inherit from
 `system.CatchableError` (which is a subtype of `Exception`).
 
+```
+Exception
+|-- CatchableError
+|   |-- IOError
+|   |   `-- EOFError
+|   |-- OSError
+|   |-- ResourceExhaustedError
+|   `-- ValueError
+|       `-- KeyError
+`-- Defect
+    |-- AccessViolationDefect
+    |-- ArithmeticDefect
+    |   |-- DivByZeroDefect
+    |   `-- OverflowDefect
+    |-- AssertionDefect
+    |-- DeadThreadDefect
+    |-- FieldDefect
+    |-- FloatingPointDefect
+    |   |-- FloatDivByZeroDefect
+    |   |-- FloatInvalidOpDefect
+    |   |-- FloatOverflowDefect
+    |   |-- FloatUnderflowDefect
+    |   `-- InexactDefect
+    |-- IndexDefect
+    |-- NilAccessDefect
+    |-- ObjectAssignmentDefect
+    |-- ObjectConversionDefect
+    |-- OutOfMemoryDefect
+    |-- RangeDefect
+    |-- ReraiseDefect
+    `-- StackOverflowDefect
+```
 
 Imported exceptions
 -------------------
@@ -4547,9 +5128,7 @@ It is possible to raise/catch imported C++ exceptions. Types imported using
 `importcpp` can be raised or caught. Exceptions are raised by value and
 caught by reference. Example:
 
-.. code-block:: nim
-    :test: "nim cpp -r $1"
-
+  ```nim  test = "nim cpp -r $1"
   type
     CStdException {.importcpp: "std::exception", header: "", inheritable.} = object
       ## does not inherit from `RootObj`, so we use `inheritable` instead
@@ -4580,6 +5159,7 @@ caught by reference. Example:
     doAssert $b == "foo3"
 
   fn()
+  ```
 
 **Note:** `getCurrentException()` and `getCurrentExceptionMsg()` are not available
 for imported exceptions from C++. One needs to use the `except ImportedException as x:` syntax
@@ -4589,6 +5169,10 @@ and rely on functionality of the `x` object to get exception details.
 Effect system
 =============
 
+**Note**: The rules for effect tracking changed with the release of version
+1.6 of the Nim compiler.
+
+
 Exception tracking
 ------------------
 
@@ -4596,31 +5180,28 @@ Nim supports exception tracking. The `raises`:idx: pragma can be used
 to explicitly define which exceptions a proc/iterator/method/converter is
 allowed to raise. The compiler verifies this:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   proc p(what: bool) {.raises: [IOError, OSError].} =
     if what: raise newException(IOError, "IO")
     else: raise newException(OSError, "OS")
+  ```
 
 An empty `raises` list (`raises: []`) means that no exception may be raised:
 
-.. code-block:: nim
+  ```nim
   proc p(): bool {.raises: [].} =
     try:
       unsafeCall()
       result = true
-    except:
+    except CatchableError:
       result = false
+  ```
 
 
 A `raises` list can also be attached to a proc type. This affects type
 compatibility:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c $1"  status = 1
   type
     Callback = proc (s: string) {.raises: [IOError].}
   var
@@ -4630,6 +5211,7 @@ compatibility:
     raise newException(OSError, "OS")
 
   c = p # type error
+  ```
 
 
 For a routine `p`, the compiler uses inference rules to determine the set of
@@ -4638,57 +5220,98 @@ possibly raised exceptions; the algorithm operates on `p`'s call graph:
 1. Every indirect call via some proc type `T` is assumed to
    raise `system.Exception` (the base type of the exception hierarchy) and
    thus any exception unless `T` has an explicit `raises` list.
-   However, if the call is of the form `f(...)` where `f` is a parameter of the currently analyzed routine it is ignored. The call is optimistically assumed to have no effect. Rule 2 compensates for this case.
-2. Every expression of some proc type within a call that is not a call
-   itself (and not nil) is assumed to be called indirectly somehow and thus
+   However, if the call is of the form `f(...)` where `f` is a parameter of
+   the currently analyzed routine that is marked as `.effectsOf: f`, it is ignored.
+   The call is optimistically assumed to have no effect.
+   Rule 2 compensates for this case.
+2. Every expression `e` of some proc type within a call that is passed to parameter
+   marked as `.effectsOf` of proc `p` is assumed to be called indirectly and thus
    its raises list is added to `p`'s raises list.
 3. Every call to a proc `q` which has an unknown body (due to a forward
-   declaration or an `importc` pragma) is assumed to
+   declaration) is assumed to
    raise `system.Exception` unless `q` has an explicit `raises` list.
+   Procs that are `importc`'ed are assumed to have `.raises: []`, unless explicitly
+   declared otherwise.
 4. Every call to a method `m` is assumed to
    raise `system.Exception` unless `m` has an explicit `raises` list.
 5. For every other call, the analysis can determine an exact `raises` list.
 6. For determining a `raises` list, the `raise` and `try` statements
    of `p` are taken into consideration.
 
-Rules 1-2 ensure the following works:
 
-.. code-block:: nim
-  proc noRaise(x: proc()) {.raises: [].} =
-    # unknown call that might raise anything, but valid:
-    x()
+Exceptions inheriting from `system.Defect` are not tracked with
+the `.raises: []` exception tracking mechanism. This is more consistent with the
+built-in operations. The following code is valid:
+
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    a div b # can raise an DivByZeroDefect
+  ```
+
+And so is:
+
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    if b == 0: raise newException(DivByZeroDefect, "division by zero")
+    else: result = a div b
+  ```
+
+
+The reason for this is that `DivByZeroDefect` inherits from `Defect` and
+with `--panics:on`:option: Defects become unrecoverable errors.
+(Since version 1.4 of the language.)
+
+
+EffectsOf annotation
+--------------------
+
+Rules 1-2 of the exception tracking inference rules (see the previous section)
+ensure the following works:
+
+  ```nim
+  proc weDontRaiseButMaybeTheCallback(callback: proc()) {.raises: [], effectsOf: callback.} =
+    callback()
 
   proc doRaise() {.raises: [IOError].} =
     raise newException(IOError, "IO")
 
   proc use() {.raises: [].} =
     # doesn't compile! Can raise IOError!
-    noRaise(doRaise)
+    weDontRaiseButMaybeTheCallback(doRaise)
+  ```
+
+As can be seen from the example, a parameter of type `proc (...)` can be
+annotated as `.effectsOf`. Such a parameter allows for effect polymorphism:
+The proc `weDontRaiseButMaybeTheCallback` raises the exceptions
+that `callback` raises.
 
 So in many cases a callback does not cause the compiler to be overly
-conservative in its effect analysis.
+conservative in its effect analysis:
 
-Exceptions inheriting from `system.Defect` are not tracked with
-the `.raises: []` exception tracking mechanism. This is more consistent with the
-built-in operations. The following code is valid:
+  ```nim  test = "nim c $1"  status = 1
+  {.push warningAsError[Effect]: on.}
 
-.. code-block:: nim
+  import std/algorithm
 
-  proc mydiv(a, b): int {.raises: [].} =
-    a div b # can raise an DivByZeroDefect
+  type
+    MyInt = distinct int
 
-And so is:
+  var toSort = @[MyInt 1, MyInt 2, MyInt 3]
 
-.. code-block:: nim
+  proc cmpN(a, b: MyInt): int =
+    cmp(a.int, b.int)
 
-  proc mydiv(a, b): int {.raises: [].} =
-    if b == 0: raise newException(DivByZeroDefect, "division by zero")
-    else: result = a div b
+  proc harmless {.raises: [].} =
+    toSort.sort cmpN
 
+  proc cmpE(a, b: MyInt): int {.raises: [Exception].} =
+    cmp(a.int, b.int)
+
+  proc harmful {.raises: [].} =
+    # does not compile, `sort` can now raise Exception
+    toSort.sort cmpE
+  ```
 
-The reason for this is that `DivByZeroDefect` inherits from `Defect` and
-with `--panics:on` Defects become unrecoverable errors.
-(Since version 1.4 of the language.)
 
 
 Tag tracking
@@ -4698,16 +5321,14 @@ Exception tracking is part of Nim's `effect system`:idx:. Raising an exception
 is an *effect*. Other effects can also be defined. A user defined effect is a
 means to *tag* a routine and to perform checks against this tag:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c --warningAsError:Effect:on $1"  status = 1
   type IO = object ## input/output effect
   proc readLine(): string {.tags: [IO].} = discard
 
-  proc no_IO_please() {.tags: [].} =
+  proc no_effects_please() {.tags: [].} =
     # the compiler prevents this:
     let x = readLine()
+  ```
 
 A tag has to be a type name. A `tags` list - like a `raises` list - can
 also be attached to a proc type. This affects type compatibility.
@@ -4715,6 +5336,126 @@ also be attached to a proc type. This affects type compatibility.
 The inference for tag tracking is analogous to the inference for
 exception tracking.
 
+There is also a way which can be used to forbid certain effects:
+
+  ```nim  test = "nim c --warningAsError:Effect:on $1"  status = 1
+  type IO = object ## input/output effect
+  proc readLine(): string {.tags: [IO].} = discard
+  proc echoLine(): void = discard
+
+  proc no_IO_please() {.forbids: [IO].} =
+    # this is OK because it didn't define any tag:
+    echoLine()
+    # the compiler prevents this:
+    let y = readLine()
+  ```
+
+The `forbids` pragma defines a list of illegal effects - if any statement
+invokes any of those effects, the compilation will fail.
+Procedure types with any disallowed effect are the subtypes of equal
+procedure types without such lists:
+
+  ```nim
+  type MyEffect = object
+  type ProcType1 = proc (i: int): void {.forbids: [MyEffect].}
+  type ProcType2 = proc (i: int): void
+
+  proc caller1(p: ProcType1): void = p(1)
+  proc caller2(p: ProcType2): void = p(1)
+
+  proc effectful(i: int): void {.tags: [MyEffect].} = echo $i
+  proc effectless(i: int): void {.forbids: [MyEffect].} = echo $i
+
+  proc toBeCalled1(i: int): void = effectful(i)
+  proc toBeCalled2(i: int): void = effectless(i)
+
+  ## this will fail because toBeCalled1 uses MyEffect which was forbidden by ProcType1:
+  caller1(toBeCalled1)
+  ## this is OK because both toBeCalled2 and ProcType1 have the same requirements:
+  caller1(toBeCalled2)
+  ## these are OK because ProcType2 doesn't have any effect requirement:
+  caller2(toBeCalled1)
+  caller2(toBeCalled2)
+  ```
+
+`ProcType2` is a subtype of `ProcType1`. Unlike with the `tags` pragma, the parent context - the
+function which calls other functions with forbidden effects - doesn't inherit the forbidden list of effects.
+
+
+Side effects
+------------
+
+The `noSideEffect` pragma is used to mark a proc/iterator that can have only
+side effects through parameters. This means that the proc/iterator only changes locations that are
+reachable from its parameters and the return value only depends on the
+parameters. If none of its parameters have the type `var`, `ref`, `ptr`, `cstring`, or `proc`,
+then no locations are modified.
+
+In other words, a routine has no side effects if it does not access a threadlocal
+or global variable and it does not call any routine that has a side effect.
+
+It is a static error to mark a proc/iterator to have no side effect if the compiler cannot verify this.
+
+As a special semantic rule, the built-in [debugEcho](
+system.html#debugEcho,varargs[typed,]) pretends to be free of side effects
+so that it can be used for debugging routines marked as `noSideEffect`.
+
+`func` is syntactic sugar for a proc with no side effects:
+
+  ```nim
+  func `+` (x, y: int): int
+  ```
+
+
+To override the compiler's side effect analysis a `{.noSideEffect.}`
+`cast` pragma block can be used:
+
+  ```nim
+  func f() =
+    {.cast(noSideEffect).}:
+      echo "test"
+  ```
+
+**Side effects are usually inferred. The inference for side effects is
+analogous to the inference for exception tracking.**
+
+When the compiler cannot infer side effects, as is the case for imported
+functions, one can annotate them with the `sideEffect` pragma.
+
+GC safety effect
+----------------
+
+We call a proc `p` `GC safe`:idx: when it doesn't access any global variable
+that contains GC'ed memory (`string`, `seq`, `ref` or a closure) either
+directly or indirectly through a call to a GC unsafe proc.
+
+**The GC safety property is usually inferred. The inference for GC safety is
+analogous to the inference for exception tracking.**
+
+The `gcsafe`:idx: annotation can be used to mark a proc to be gcsafe,
+otherwise this property is inferred by the compiler. Note that `noSideEffect`
+implies `gcsafe`.
+
+Routines that are imported from C are always assumed to be `gcsafe`.
+
+To override the compiler's gcsafety analysis a `{.cast(gcsafe).}` pragma block can
+be used:
+
+  ```nim
+  var
+    someGlobal: string = "some string here"
+    perThread {.threadvar.}: string
+
+  proc setPerThread() =
+    {.cast(gcsafe).}:
+      deepCopy(perThread, someGlobal)
+  ```
+
+
+See also:
+
+- [Shared heap memory management](mm.html).
+
 
 
 Effects pragma
@@ -4724,13 +5465,14 @@ The `effects` pragma has been designed to assist the programmer with the
 effects analysis. It is a statement that makes the compiler output all inferred
 effects up to the `effects`'s position:
 
-.. code-block:: nim
+  ```nim
   proc p(what: bool) =
     if what:
       raise newException(IOError, "IO")
       {.effects.}
     else:
       raise newException(OSError, "OS")
+  ```
 
 The compiler produces a hint message that `IOError` can be raised. `OSError`
 is not listed as it cannot be raised in the branch the `effects` pragma
@@ -4744,14 +5486,13 @@ Generics are Nim's means to parametrize procs, iterators or types with
 `type parameters`:idx:. Depending on the context, the brackets are used either to
 introduce type parameters or to instantiate a generic proc, iterator, or type.
 
+
 The following example shows how a generic binary tree can be modeled:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     BinaryTree*[T] = ref object # BinaryTree is a generic type with
-                                # generic param `T`
+                                # generic parameter `T`
       le, ri: BinaryTree[T]     # left and right subtrees; may be nil
       data: T                   # the data stored in a node
 
@@ -4802,10 +5543,54 @@ The following example shows how a generic binary tree can be modeled:
   add(root, "world")          # instantiates the second `add` proc
   for str in preorder(root):
     stdout.writeLine(str)
+  ```
 
 The `T` is called a `generic type parameter`:idx: or
 a `type variable`:idx:.
 
+
+Generic Procs
+---------------
+
+Let's consider the anatomy of a generic `proc` to agree on defined terminology.
+
+```nim
+p[T: t](arg1: f): y
+```
+
+- `p`: Callee symbol
+- `[...]`: Generic parameters
+- `T: t`: Generic constraint
+- `T`: Type variable
+- `[T: t](arg1: f): y`: Formal signature
+- `arg1: f`: Formal parameter
+- `f`: Formal parameter type
+- `y`: Formal return type
+
+The use of the word "formal" here is to denote the symbols as they are defined by the programmer,
+not as they may be at compile time contextually. Since generics may be instantiated and
+types bound, we have more than one entity to think about when generics are involved.
+
+The usage of a generic will resolve the formally defined expression into an instance of that
+expression bound to only concrete types. This process is called "instantiation".
+
+Brackets at the site of a generic's formal definition specify the "constraints" as in:
+
+```nim
+type Foo[T] = object
+proc p[H;T: Foo[H]](param: T): H
+```
+
+A constraint definition may have more than one symbol defined by separating each definition by
+a `;`. Notice how `T` is composed of `H` and the return  type of `p` is defined as `H`. When this
+generic proc is instantiated `H` will be bound to a concrete type, thus making `T` concrete and
+the return type of `p` will be bound to the same concrete type used to define `H`.
+
+Brackets at the site of usage can be used to supply concrete types to instantiate the generic in the same
+order that the symbols are defined in the constraint. Alternatively, type bindings may be inferred by the compiler
+in some situations, allowing for cleaner code.
+
+
 Is operator
 -----------
 
@@ -4813,16 +5598,17 @@ The `is` operator is evaluated during semantic analysis to check for type
 equivalence. It is therefore very useful for type specialization within generic
 code:
 
-.. code-block:: nim
+  ```nim
   type
     Table[Key, Value] = object
       keys: seq[Key]
       values: seq[Value]
       when not (Key is string): # empty value for strings used for optimization
         deletedKeys: seq[bool]
+  ```
 
 
-Type Classes
+Type classes
 ------------
 
 A type class is a special pseudo-type that can be used to match against
@@ -4834,9 +5620,9 @@ type class           matches
 ==================   ===================================================
 `object`             any object type
 `tuple`              any tuple type
-
 `enum`               any enumeration
 `proc`               any proc type
+`iterator`           any iterator type
 `ref`                any `ref` type
 `ptr`                any `ptr` type
 `var`                any `var` type
@@ -4853,20 +5639,22 @@ name that will match any instantiation of the generic type.
 Type classes can be combined using the standard boolean operators to form
 more complex type classes:
 
-.. code-block:: nim
+  ```nim
   # create a type class that will match all tuple and object types
-  type RecordType = tuple or object
+  type RecordType = (tuple or object)
 
   proc printFields[T: RecordType](rec: T) =
     for key, value in fieldPairs(rec):
       echo key, " = ", value
+  ```
 
 Type constraints on generic parameters can be grouped with `,` and propagation
 stops with `;`, similarly to parameters for macros and templates:
 
-.. code-block:: nim
+  ```nim
   proc fn1[T; U, V: SomeFloat]() = discard # T is unconstrained
   template fn2(t; u, v: SomeFloat) = discard # t is unconstrained
+  ```
 
 Whilst the syntax of type classes appears to resemble that of ADTs/algebraic data
 types in ML-like languages, it should be understood that type classes are static
@@ -4877,20 +5665,33 @@ runtime type dynamism, unlike object variants or methods.
 
 As an example, the following would not compile:
 
-.. code-block:: nim
+  ```nim
   type TypeClass = int | string
   var foo: TypeClass = 2 # foo's type is resolved to an int here
   foo = "this will fail" # error here, because foo is an int
+  ```
 
 Nim allows for type classes and regular types to be specified
 as `type constraints`:idx: of the generic type parameter:
 
-.. code-block:: nim
+  ```nim
   proc onlyIntOrString[T: int|string](x, y: T) = discard
 
   onlyIntOrString(450, 616) # valid
   onlyIntOrString(5.0, 0.0) # type mismatch
   onlyIntOrString("xy", 50) # invalid as 'T' cannot be both at the same time
+  ```
+
+`proc` and `iterator` type classes also accept a calling convention pragma
+to restrict the calling convention of the matching `proc` or `iterator` type.
+
+  ```nim
+  proc onlyClosure[T: proc {.closure.}](x: T) = discard
+
+  onlyClosure(proc() = echo "hello") # valid
+  proc foo() {.nimcall.} = discard
+  onlyClosure(foo) # type mismatch
+  ```
 
 
 Implicit generics
@@ -4898,25 +5699,25 @@ Implicit generics
 
 A type class can be used directly as the parameter's type.
 
-.. code-block:: nim
-
+  ```nim
   # create a type class that will match all tuple and object types
-  type RecordType = tuple or object
+  type RecordType = (tuple or object)
 
   proc printFields(rec: RecordType) =
     for key, value in fieldPairs(rec):
       echo key, " = ", value
+  ```
 
 
 Procedures utilizing type classes in such a manner are considered to be
 `implicitly generic`:idx:. They will be instantiated once for each unique
-combination of param types used within the program.
+combination of parameter types used within the program.
 
 By default, during overload resolution, each named type class will bind to
 exactly one concrete type. We call such type classes `bind once`:idx: types.
 Here is an example taken directly from the system module to illustrate this:
 
-.. code-block:: nim
+  ```nim
   proc `==`*(x, y: tuple): bool =
     ## requires `x` and `y` to be of the same tuple type
     ## generic `==` operator for tuples that is lifted from the components
@@ -4924,81 +5725,82 @@ Here is an example taken directly from the system module to illustrate this:
     result = true
     for a, b in fields(x, y):
       if a != b: result = false
+  ```
 
 Alternatively, the `distinct` type modifier can be applied to the type class
-to allow each param matching the type class to bind to a different type. Such
+to allow each parameter matching the type class to bind to a different type. Such
 type classes are called `bind many`:idx: types.
 
 Procs written with the implicitly generic style will often need to refer to the
 type parameters of the matched generic type. They can be easily accessed using
 the dot syntax:
 
-.. code-block:: nim
+  ```nim
   type Matrix[T, Rows, Columns] = object
     ...
 
   proc `[]`(m: Matrix, row, col: int): Matrix.T =
     m.data[col * high(Matrix.Columns) + row]
+  ```
 
 
 Here are more examples that illustrate implicit generics:
 
-.. code-block:: nim
-
+  ```nim
   proc p(t: Table; k: Table.Key): Table.Value
 
   # is roughly the same as:
 
   proc p[Key, Value](t: Table[Key, Value]; k: Key): Value
+  ```
 
 
-.. code-block:: nim
-
+  ```nim
   proc p(a: Table, b: Table)
 
   # is roughly the same as:
 
   proc p[Key, Value](a, b: Table[Key, Value])
+  ```
 
 
-.. code-block:: nim
-
+  ```nim
   proc p(a: Table, b: distinct Table)
 
   # is roughly the same as:
 
   proc p[Key, Value, KeyB, ValueB](a: Table[Key, Value], b: Table[KeyB, ValueB])
+  ```
 
 
 `typedesc` used as a parameter type also introduces an implicit
 generic. `typedesc` has its own set of rules:
 
-.. code-block:: nim
-
+  ```nim
   proc p(a: typedesc)
 
   # is roughly the same as:
 
   proc p[T](a: typedesc[T])
+  ```
 
 
 `typedesc` is a "bind many" type class:
 
-.. code-block:: nim
-
+  ```nim
   proc p(a, b: typedesc)
 
   # is roughly the same as:
 
   proc p[T, T2](a: typedesc[T], b: typedesc[T2])
+  ```
 
 
 A parameter of type `typedesc` is itself usable as a type. If it is used
-as a type, it's the underlying type. (In other words, one level
+as a type, it's the underlying type. In other words, one level
 of "typedesc"-ness is stripped off:
 
-.. code-block:: nim
-
+  ```nim
   proc p(a: typedesc; b: a) = discard
 
   # is roughly the same as:
@@ -5007,6 +5809,7 @@ of "typedesc"-ness is stripped off:
   # hence this is a valid call:
   p(int, 4)
   # as parameter 'a' requires a type, but 'b' requires a value.
+  ```
 
 
 Generic inference restrictions
@@ -5015,10 +5818,7 @@ Generic inference restrictions
 The types `var T` and `typedesc[T]` cannot be inferred in a generic
 instantiation. The following is not allowed:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c $1"  status = 1
   proc g[T](f: proc(x: T); x: T) =
     f(x)
 
@@ -5035,14 +5835,14 @@ instantiation. The following is not allowed:
 
   # also not allowed: explicit instantiation via 'var int'
   g[var int](v, i)
+  ```
 
 
 
 Symbol lookup in generics
 -------------------------
 
-Open and Closed symbols
-~~~~~~~~~~~~~~~~~~~~~~~
+### Open and Closed symbols
 
 The symbol binding rules in generics are slightly subtle: There are "open" and
 "closed" symbols. A "closed" symbol cannot be re-bound in the instantiation
@@ -5052,9 +5852,7 @@ and every other symbol is closed.
 Open symbols are looked up in two different contexts: Both the context
 at definition and the context at instantiation are considered:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     Index = distinct int
 
@@ -5064,8 +5862,9 @@ at definition and the context at instantiation are considered:
   var b = (0, 0.Index)
 
   echo a == b # works!
+  ```
 
-In the example, the generic `==` for tuples (as defined in the system module)
+In the example, the [generic `==` for tuples](system.html#%3D%3D%2CT%2CT_2) (as defined in the system module)
 uses the `==` operators of the tuple's components. However, the `==` for
 the `Index` type is defined *after* the `==` for tuples; yet the example
 compiles as the instantiation takes the currently defined symbols into account
@@ -5076,15 +5875,14 @@ Mixin statement
 
 A symbol can be forced to be open by a `mixin`:idx: declaration:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   proc create*[T](): ref T =
     # there is no overloaded 'init' here, so we need to state that it's an
     # open symbol explicitly:
     mixin init
     new result
     init result
+  ```
 
 `mixin` statements only make sense in templates and generics.
 
@@ -5097,7 +5895,7 @@ can be used to explicitly declare identifiers that should be bound early (i.e.
 the identifiers should be looked up in the scope of the template/generic
 definition):
 
-.. code-block:: nim
+  ```nim
   # Module A
   var
     lastId = 0
@@ -5106,12 +5904,14 @@ definition):
     bind lastId
     inc(lastId)
     lastId
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module B
   import A
 
   echo genId()
+  ```
 
 But a `bind` is rarely useful because symbol binding from the definition
 scope is the default.
@@ -5125,16 +5925,15 @@ Delegating bind statements
 The following example outlines a problem that can arise when generic
 instantiations cross multiple different modules:
 
-.. code-block:: nim
-
+  ```nim
   # module A
   proc genericA*[T](x: T) =
     mixin init
     init(x)
+  ```
 
 
-.. code-block:: nim
-
+  ```nim
   import C
 
   # module B
@@ -5143,23 +5942,24 @@ instantiations cross multiple different modules:
     # not available when `genericB` is instantiated:
     bind init
     genericA(x)
+  ```
 
-.. code-block:: nim
-
+  ```nim
   # module C
   type O = object
   proc init*(x: var O) = discard
+  ```
 
-.. code-block:: nim
-
+  ```nim
   # module main
   import B, C
 
   genericB O()
+  ```
 
 In module B has an `init` proc from module C in its scope that is not
 taken into account when `genericB` is instantiated which leads to the
-instantiation of `genericA`. The solution is to `forward`:idx these
+instantiation of `genericA`. The solution is to `forward`:idx: these
 symbols by a `bind` statement inside `genericB`.
 
 
@@ -5174,12 +5974,13 @@ The syntax to *invoke* a template is the same as calling a procedure.
 
 Example:
 
-.. code-block:: nim
+  ```nim
   template `!=` (a, b: untyped): untyped =
-    # this definition exists in the System module
+    # this definition exists in the system module
     not (a == b)
 
   assert(5 != 6) # the compiler rewrites that to: assert(not (5 == 6))
+  ```
 
 The `!=`, `>`, `>=`, `in`, `notin`, `isnot` operators are in fact
 templates:
@@ -5201,24 +6002,21 @@ An `untyped` parameter means that symbol lookups and type resolution is not
 performed before the expression is passed to the template. This means that
 *undeclared* identifiers, for example, can be passed to the template:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template declareInt(x: untyped) =
     var x: int
 
   declareInt(x) # valid
   x = 3
+  ```
 
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c $1"  status = 1
   template declareInt(x: typed) =
     var x: int
 
   declareInt(x) # invalid, because x has not been declared and so it has no type
+  ```
 
 A template where every parameter is `untyped` is called an `immediate`:idx:
 template. For historical reasons, templates can be explicitly annotated with
@@ -5236,9 +6034,7 @@ Passing a code block to a template
 One can pass a block of statements as the last argument to a template
 following the special `:` syntax:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template withFile(f, fn, mode, actions: untyped): untyped =
     var f: File
     if open(f, fn, mode):
@@ -5252,6 +6048,7 @@ following the special `:` syntax:
   withFile(txt, "ttempl3.txt", fmWrite):  # special colon
     txt.writeLine("line 1")
     txt.writeLine("line 2")
+  ```
 
 In the example, the two `writeLine` statements are bound to the `actions`
 parameter.
@@ -5261,10 +6058,7 @@ Usually, to pass a block of code to a template, the parameter that accepts
 the block needs to be of type `untyped`. Because symbol lookups are then
 delayed until template instantiation time:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c $1"  status = 1
   template t(body: typed) =
     proc p = echo "hey"
     block:
@@ -5272,6 +6066,7 @@ delayed until template instantiation time:
 
   t:
     p()  # fails with 'undeclared identifier: p'
+  ```
 
 The above code fails with the error message that `p` is not declared.
 The reason for this is that the `p()` body is type-checked before getting
@@ -5279,9 +6074,7 @@ passed to the `body` parameter and type checking in Nim implies symbol lookups.
 The same code works with `untyped` as the passed body is not required to be
 type-checked:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template t(body: untyped) =
     proc p = echo "hey"
     block:
@@ -5289,6 +6082,7 @@ type-checked:
 
   t:
     p()  # compiles
+  ```
 
 
 Varargs of untyped
@@ -5297,12 +6091,11 @@ Varargs of untyped
 In addition to the `untyped` meta-type that prevents type checking, there is
 also `varargs[untyped]` so that not even the number of parameters is fixed:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template hideIdentifiers(x: varargs[untyped]) = discard
 
   hideIdentifiers(undeclared1, undeclared2)
+  ```
 
 However, since a template cannot iterate over varargs, this feature is
 generally much more useful for macros.
@@ -5314,7 +6107,7 @@ Symbol binding in templates
 A template is a `hygienic`:idx: macro and so opens a new scope. Most symbols are
 bound from the definition scope of the template:
 
-.. code-block:: nim
+  ```nim
   # Module A
   var
     lastId = 0
@@ -5322,12 +6115,14 @@ bound from the definition scope of the template:
   template genId*: untyped =
     inc(lastId)
     lastId
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module B
   import A
 
   echo genId() # Works as 'lastId' has been bound in 'genId's defining scope
+  ```
 
 As in generics, symbol binding can be influenced via `mixin` or `bind`
 statements.
@@ -5339,9 +6134,7 @@ Identifier construction
 
 In templates, identifiers can be constructed with the backticks notation:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template typedef(name: untyped, typ: typedesc) =
     type
       `T name`* {.inject.} = typ
@@ -5349,6 +6142,7 @@ In templates, identifiers can be constructed with the backticks notation:
 
   typedef(myint, int)
   var x: PMyInt
+  ```
 
 In the example, `name` is instantiated with `myint`, so \`T name\` becomes
 `Tmyint`.
@@ -5361,7 +6155,7 @@ A parameter `p` in a template is even substituted in the expression `x.p`.
 Thus, template arguments can be used as field names and a global symbol can be
 shadowed by the same argument name even when fully qualified:
 
-.. code-block:: nim
+  ```nim
   # module 'm'
 
   type
@@ -5375,10 +6169,11 @@ shadowed by the same argument name even when fully qualified:
 
   tstLev(levA)
   # produces: 'levA levA'
+  ```
 
 But the global symbol can properly be captured by a `bind` statement:
 
-.. code-block:: nim
+  ```nim
   # module 'm'
 
   type
@@ -5393,6 +6188,7 @@ But the global symbol can properly be captured by a `bind` statement:
 
   tstLev(levA)
   # produces: 'levA levB'
+  ```
 
 
 Hygiene in templates
@@ -5401,9 +6197,7 @@ Hygiene in templates
 Per default, templates are `hygienic`:idx:\: Local identifiers declared in a
 template cannot be accessed in the instantiation context:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template newException*(exceptn: typedesc, message: string): untyped =
     var
       e: ref exceptn  # e is implicitly gensym'ed here
@@ -5414,6 +6208,7 @@ template cannot be accessed in the instantiation context:
   # so this works:
   let e = "message"
   raise newException(IoError, e)
+  ```
 
 
 Whether a symbol that is declared in a template is exposed to the instantiation
@@ -5425,25 +6220,27 @@ is `gensym` and for `proc`, `iterator`, `converter`, `template`,
 `macro` is `inject`. However, if the name of the entity is passed as a
 template parameter, it is an `inject`'ed symbol:
 
-.. code-block:: nim
+  ```nim
   template withFile(f, fn, mode: untyped, actions: untyped): untyped =
     block:
-      var f: File  # since 'f' is a template param, it's injected implicitly
+      var f: File  # since 'f' is a template parameter, it's injected implicitly
       ...
 
   withFile(txt, "ttempl3.txt", fmWrite):
     txt.writeLine("line 1")
     txt.writeLine("line 2")
+  ```
 
 
 The `inject` and `gensym` pragmas are second class annotations; they have
-no semantics outside of a template definition and cannot be abstracted over:
+no semantics outside a template definition and cannot be abstracted over:
 
-.. code-block:: nim
+  ```nim
   {.pragma myInject: inject.}
 
   template t() =
     var x {.myInject.}: int # does NOT work
+  ```
 
 
 To get rid of hygiene in templates, one can use the `dirty`:idx: pragma for
@@ -5455,9 +6252,7 @@ and `namedParameterCall(field = value)` syntactic constructs.
 
 The reason for this is that code like
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     T = object
       f: int
@@ -5465,6 +6260,7 @@ The reason for this is that code like
   template tmp(x: T) =
     let f = 34
     echo x.f, T(f: 4)
+  ```
 
 
 should work as expected.
@@ -5472,10 +6268,7 @@ should work as expected.
 However, this means that the method call syntax is not available for
 `gensym`'ed symbols:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c $1"  status = 1
   template tmp(x) =
     type
       T {.gensym.} = int
@@ -5483,11 +6276,7 @@ However, this means that the method call syntax is not available for
     echo x.T # invalid: instead use:  'echo T(x)'.
 
   tmp(12)
-
-
-**Note**: The Nim compiler prior to version 1 was more lenient about this
-requirement. Use the `--useVersion:0.19` switch for a transition period.
-
+  ```
 
 
 Limitations of the method call syntax
@@ -5495,53 +6284,31 @@ Limitations of the method call syntax
 
 The expression `x` in `x.f` needs to be semantically checked (that means
 symbol lookup and type checking) before it can be decided that it needs to be
-rewritten to `f(x)`. Therefore the dot syntax has some limitations when it
+rewritten to `f(x)`. Therefore, the dot syntax has some limitations when it
 is used to invoke templates/macros:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c $1"  status = 1
   template declareVar(name: untyped) =
     const name {.inject.} = 45
 
   # Doesn't compile:
   unknownIdentifier.declareVar
+  ```
 
 
-Another common example is this:
-
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
-  from std/sequtils import toSeq
-
-  iterator something: string =
-    yield "Hello"
-    yield "World"
-
-  var info = something().toSeq
-
-The problem here is that the compiler already decided that `something()` as
-an iterator is not callable in this context before `toSeq` gets its
-chance to convert it into a sequence.
-
 It is also not possible to use fully qualified identifiers with module
 symbol in method call syntax. The order in which the dot operator
 binds to symbols prohibits this.
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
+  ```nim  test = "nim c $1"  status = 1
+  import std/sequtils
 
-   import std/sequtils
-
-   var myItems = @[1,3,3,7]
-   let N1 = count(myItems, 3) # OK
-   let N2 = sequtils.count(myItems, 3) # fully qualified, OK
-   let N3 = myItems.count(3) # OK
-   let N4 = myItems.sequtils.count(3) # illegal, `myItems.sequtils` can't be resolved
+  var myItems = @[1,3,3,7]
+  let N1 = count(myItems, 3) # OK
+  let N2 = sequtils.count(myItems, 3) # fully qualified, OK
+  let N3 = myItems.count(3) # OK
+  let N4 = myItems.sequtils.count(3) # illegal, `myItems.sequtils` can't be resolved
+  ```
 
 This means that when for some reason a procedure needs a
 disambiguation through the module name, the call needs to be
@@ -5570,15 +6337,21 @@ twice:
 While macros enable advanced compile-time code transformations, they
 cannot change Nim's syntax.
 
-Debug Example
+**Style note:** For code readability, it is best to use the least powerful
+programming construct that remains expressive. So the "check list" is:
+
+(1) Use an ordinary proc/iterator, if possible.
+(2) Else: Use a generic proc/iterator, if possible.
+(3) Else: Use a template, if possible.
+(4) Else: Use a macro.
+
+Debug example
 -------------
 
 The following example implements a powerful `debug` command that accepts a
 variable number of arguments:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   # to work with Nim syntax trees, we need an API that is defined in the
   # `macros` module:
   import std/macros
@@ -5606,10 +6379,11 @@ variable number of arguments:
   a[1] = 45
 
   debug(a[0], a[1], x)
+  ```
 
 The macro call expands to:
 
-.. code-block:: nim
+  ```nim
   write(stdout, "a[0]")
   write(stdout, ": ")
   writeLine(stdout, a[0])
@@ -5621,6 +6395,7 @@ The macro call expands to:
   write(stdout, "x")
   write(stdout, ": ")
   writeLine(stdout, x)
+  ```
 
 
 Arguments that are passed to a `varargs` parameter are wrapped in an array
@@ -5628,7 +6403,7 @@ constructor expression. This is why `debug` iterates over all of `args`'s
 children.
 
 
-BindSym
+bindSym
 -------
 
 The above `debug` macro relies on the fact that `write`, `writeLine` and
@@ -5637,9 +6412,7 @@ instantiating context. There is a way to use bound identifiers
 (aka `symbols`:idx:) instead of using unbound identifiers. The `bindSym`
 builtin can be used for that:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   import std/macros
 
   macro debug(n: varargs[typed]): untyped =
@@ -5657,10 +6430,11 @@ builtin can be used for that:
   a[1] = 45
 
   debug(a[0], a[1], x)
+  ```
 
 The macro call expands to:
 
-.. code-block:: nim
+  ```nim
   write(stdout, "a[0]")
   write(stdout, ": ")
   writeLine(stdout, a[0])
@@ -5672,48 +6446,45 @@ The macro call expands to:
   write(stdout, "x")
   write(stdout, ": ")
   writeLine(stdout, x)
+  ```
 
-However, the symbols `write`, `writeLine` and `stdout` are already bound
-and are not looked up again. As the example shows, `bindSym` does work with
-overloaded symbols implicitly.
+In this version of `debug`, the symbols `write`, `writeLine` and `stdout`
+are already bound and are not looked up again. As the example shows, `bindSym`
+does work with overloaded symbols implicitly.
 
-Case-Of Macro
--------------
+Note that the symbol names passed to `bindSym` have to be constant. The
+experimental feature `dynamicBindSym` ([experimental manual](
+manual_experimental.html#dynamic-arguments-for-bindsym))
+allows this value to be computed dynamically.
 
-In Nim, it is possible to have a macro with the syntax of a *case-of*
-expression just with the difference that all *of-branches* are passed to
-and processed by the macro implementation. It is then up the macro
-implementation to transform the *of-branches* into a valid Nim
-statement. The following example should show how this feature could be
-used for a lexical analyzer.
+Post-statement blocks
+---------------------
 
-.. code-block:: nim
-  import std/macros
+Macros can receive `of`, `elif`, `else`, `except`, `finally` and `do`
+blocks (including their different forms such as `do` with routine parameters)
+as arguments if called in statement form.
 
-  macro case_token(args: varargs[untyped]): untyped =
-    echo args.treeRepr
-    # creates a lexical analyzer from regular expressions
-    # ... (implementation is an exercise for the reader ;-)
-    discard
+  ```nim
+  macro performWithUndo(task, undo: untyped) = ...
 
-  case_token: # this colon tells the parser it is a macro statement
-  of r"[A-Za-z_]+[A-Za-z_0-9]*":
-    return tkIdentifier
-  of r"0-9+":
-    return tkInteger
-  of r"[\+\-\*\?]+":
-    return tkOperator
+  performWithUndo do:
+    # multiple-line block of code
+    # to perform the task
+  do:
+    # code to undo it
+
+  let num = 12
+  # a single colon may be used if there is no initial block
+  match (num mod 3, num mod 5):
+  of (0, 0):
+    echo "FizzBuzz"
+  of (0, _):
+    echo "Fizz"
+  of (_, 0):
+    echo "Buzz"
   else:
-    return tkUnknown
-
-
-**Style note**: For code readability, it is best to use the least powerful
-programming construct that still suffices. So the "check list" is:
-
-(1) Use an ordinary proc/iterator, if possible.
-(2) Else: Use a generic proc/iterator, if possible.
-(3) Else: Use a template, if possible.
-(4) Else: Use a macro.
+    echo num
+  ```
 
 
 For loop macro
@@ -5722,9 +6493,28 @@ For loop macro
 A macro that takes as its only input parameter an expression of the special
 type `system.ForLoopStmt` can rewrite the entirety of a `for` loop:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
+  import std/macros
 
+  macro example(loop: ForLoopStmt) =
+    result = newTree(nnkForStmt)    # Create a new For loop.
+    result.add loop[^3]             # This is "item".
+    result.add loop[^2][^1]         # This is "[1, 2, 3]".
+    result.add newCall(bindSym"echo", loop[0])
+
+  for item in example([1, 2, 3]): discard
+  ```
+
+Expands to:
+
+  ```nim
+  for item in items([1, 2, 3]):
+    echo item
+  ```
+
+Another example:
+
+  ```nim  test = "nim c $1"
   import std/macros
 
   macro enumerate(x: ForLoopStmt): untyped =
@@ -5756,19 +6546,60 @@ type `system.ForLoopStmt` can rewrite the entirety of a `for` loop:
   # names for `a` and `b` here to avoid redefinition errors
   for a, b in enumerate(10, [1, 2, 3, 5]):
     echo a, " ", b
+  ```
 
 
+Case statement macros
+---------------------
+
+Macros named `` `case` `` can provide implementations of `case` statements
+for certain types. The following is an example of such an implementation
+for tuples, leveraging the existing equality operator for tuples
+(as provided in `system.==`):
+
+  ```nim  test = "nim c $1"
+  import std/macros
+
+  macro `case`(n: tuple): untyped =
+    result = newTree(nnkIfStmt)
+    let selector = n[0]
+    for i in 1 ..< n.len:
+      let it = n[i]
+      case it.kind
+      of nnkElse, nnkElifBranch, nnkElifExpr, nnkElseExpr:
+        result.add it
+      of nnkOfBranch:
+        for j in 0..it.len-2:
+          let cond = newCall("==", selector, it[j])
+          result.add newTree(nnkElifBranch, cond, it[^1])
+      else:
+        error "custom 'case' for tuple cannot handle this node", it
+
+  case ("foo", 78)
+  of ("foo", 78): echo "yes"
+  of ("bar", 88): echo "no"
+  else: discard
+  ```
+
+`case` macros are subject to overload resolution. The type of the
+`case` statement's selector expression is matched against the type
+of the first argument of the `case` macro. Then the complete `case`
+statement is passed in place of the argument and the macro is evaluated.
+
+In other words, the macro needs to transform the full `case` statement
+but only the statement's selector expression is used to determine which
+macro to call.
+
 
 Special Types
 =============
 
-static[T]
----------
+static\[T]
+----------
 
 As their name suggests, static parameters must be constant expressions:
 
-.. code-block:: nim
-
+  ```nim
   proc precompiledRegex(pattern: static string): RegEx =
     var res {.global.} = re(pattern)
     return res
@@ -5778,16 +6609,16 @@ As their name suggests, static parameters must be constant expressions:
 
   precompiledRegex(paramStr(1)) # Error, command-line options
                                 # are not constant expressions
+  ```
 
 
-For the purposes of code generation, all static params are treated as
-generic params - the proc will be compiled separately for each unique
+For the purposes of code generation, all static parameters are treated as
+generic parameters - the proc will be compiled separately for each unique
 supplied value (or combination of values).
 
-Static params can also appear in the signatures of generic types:
-
-.. code-block:: nim
+Static parameters can also appear in the signatures of generic types:
 
+  ```nim
   type
     Matrix[M,N: static int; T: Number] = array[0..(M*N - 1), T]
       # Note how `Number` is just a type constraint here, while
@@ -5798,25 +6629,27 @@ Static params can also appear in the signatures of generic types:
 
   var m1: AffineTransform3D[float]  # OK
   var m2: AffineTransform2D[string] # Error, `string` is not a `Number`
+  ```
 
 Please note that `static T` is just a syntactic convenience for the underlying
-generic type `static[T]`. The type param can be omitted to obtain the type
+generic type `static[T]`. The type parameter can be omitted to obtain the type
 class of all constant expressions. A more specific type class can be created by
 instantiating `static` with another type class.
 
 One can force an expression to be evaluated at compile time as a constant
 expression by coercing it to a corresponding `static` type:
 
-.. code-block:: nim
+  ```nim
   import std/math
 
   echo static(fac(5)), " ", static[bool](16.isPowerOfTwo)
+  ```
 
 The compiler will report any failure to evaluate the expression or a
 possible type mismatch error.
 
-typedesc[T]
------------
+typedesc\[T]
+------------
 
 In many contexts, Nim treats the names of types as regular
 values. These values exist only during the compilation phase, but since
@@ -5824,48 +6657,46 @@ all values must have a type, `typedesc` is considered their special type.
 
 `typedesc` acts as a generic type. For instance, the type of the symbol
 `int` is `typedesc[int]`. Just like with regular generic types, when the
-generic param is omitted, `typedesc` denotes the type class of all types.
+generic parameter is omitted, `typedesc` denotes the type class of all types.
 As a syntactic convenience, one can also use `typedesc` as a modifier.
 
-Procs featuring `typedesc` params are considered implicitly generic.
+Procs featuring `typedesc` parameters are considered implicitly generic.
 They will be instantiated for each unique combination of supplied types,
-and within the body of the proc, the name of each param will refer to
+and within the body of the proc, the name of each parameter will refer to
 the bound concrete type:
 
-.. code-block:: nim
-
+  ```nim
   proc new(T: typedesc): ref T =
     echo "allocating ", T.name
     new(result)
 
   var n = Node.new
   var tree = new(BinaryTree[int])
+  ```
 
-When multiple type params are present, they will bind freely to different
-types. To force a bind-once behavior, one can use an explicit generic param:
+When multiple type parameters are present, they will bind freely to different
+types. To force a bind-once behavior, one can use an explicit generic parameter:
 
-.. code-block:: nim
+  ```nim
   proc acceptOnlyTypePairs[T, U](A, B: typedesc[T]; C, D: typedesc[U])
+  ```
 
-Once bound, type params can appear in the rest of the proc signature:
-
-.. code-block:: nim
-    :test: "nim c $1"
+Once bound, type parameters can appear in the rest of the proc signature:
 
+  ```nim  test = "nim c $1"
   template declareVariableWithType(T: typedesc, value: T) =
     var x: T = value
 
   declareVariableWithType int, 42
+  ```
 
 
 Overload resolution can be further influenced by constraining the set
-of types that will match the type param. This works in practice by
+of types that will match the type parameter. This works in practice by
 attaching attributes to types via templates. The constraint can be a
 concrete type or a type class.
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template maxval(T: typedesc[int]): int = high(int)
   template maxval(T: typedesc[float]): float = Inf
 
@@ -5881,13 +6712,13 @@ concrete type or a type class.
   echo "is int a number? ", isNumber(int)
   echo "is float a number? ", isNumber(float)
   echo "is RootObj a number? ", isNumber(RootObj)
+  ```
 
 Passing `typedesc` is almost identical, just with the difference that
 the macro is not instantiated generically. The type expression is
 simply passed as a `NimNode` to the macro, like everything else.
 
-.. code-block:: nim
-
+  ```nim
   import std/macros
 
   macro forwardType(arg: typedesc): typedesc =
@@ -5896,6 +6727,7 @@ simply passed as a `NimNode` to the macro, like everything else.
     result = tmp
 
   var tmp: forwardType(int)
+  ```
 
 typeof operator
 ---------------
@@ -5907,10 +6739,10 @@ One can obtain the type of a given expression by constructing a `typeof`
 value from it (in many other languages this is known as the `typeof`:idx:
 operator):
 
-.. code-block:: nim
-
+  ```nim
   var x = 0
   var y: typeof(x) # y has type int
+  ```
 
 
 If `typeof` is used to determine the result type of a proc/iterator/converter
@@ -5919,16 +6751,15 @@ interpretation, where `c` is an iterator, is preferred over the
 other interpretations, but this behavior can be changed by
 passing `typeOfProc` as the second argument to `typeof`:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   iterator split(s: string): string = discard
   proc split(s: string): seq[string] = discard
 
-  # since an iterator is the preferred interpretation, `y` has the type `string`:
+  # since an iterator is the preferred interpretation, this has the type `string`:
   assert typeof("a b c".split) is string
 
   assert typeof("a b c".split, typeOfProc) is seq[string]
+  ```
 
 
 
@@ -5937,11 +6768,11 @@ Modules
 Nim supports splitting a program into pieces by a module concept.
 Each module needs to be in its own file and has its own `namespace`:idx:.
 Modules enable `information hiding`:idx: and `separate compilation`:idx:.
-A module may gain access to symbols of another module by the `import`:idx:
+A module may gain access to the symbols of another module by the `import`:idx:
 statement. `Recursive module dependencies`:idx: are allowed, but are slightly
 subtle. Only top-level symbols that are marked with an asterisk (`*`) are
 exported. A valid module name can only be a valid Nim identifier (and thus its
-filename is `identifier.nim`).
+filename is ``identifier.nim``).
 
 The algorithm for compiling modules is:
 
@@ -5952,7 +6783,7 @@ The algorithm for compiling modules is:
 
 This is best illustrated by an example:
 
-.. code-block:: nim
+  ```nim
   # Module A
   type
     T1* = int  # Module A exports the type `T1`
@@ -5962,9 +6793,10 @@ This is best illustrated by an example:
     var i = p(3) # works because B has been parsed completely here
 
   main()
+  ```
 
 
-.. code-block:: nim
+  ```nim
   # Module B
   import A  # A is not parsed here! Only the already known symbols
             # of A are imported.
@@ -5973,31 +6805,32 @@ This is best illustrated by an example:
     # this works because the compiler has already
     # added T1 to A's interface symbol table
     result = x + 1
+  ```
 
 
 Import statement
 ----------------
 
-After the `import` statement, a list of module names can follow or a single
+After the `import` keyword, a list of module names can follow or a single
 module name followed by an `except` list to prevent some symbols from being
 imported:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
+  ```nim  test = "nim c $1"  status = 1
   import std/strutils except `%`, toUpperAscii
 
   # doesn't work then:
   echo "$1" % "abc".toUpperAscii
+  ```
 
 
 It is not checked that the `except` list is really exported from the module.
-This feature allows us to compile against an older version of the module that
-does not export these identifiers.
+This feature allows us to compile against different versions of the module,
+even when one version does not export some of these identifiers.
 
 The `import` statement is only allowed at the top level.
 
+String literals can be used for import/include statements.
+The compiler performs [path substitution](nimc.html#compiler-usage-commandminusline-switches) when used.
 
 Include statement
 -----------------
@@ -6006,51 +6839,58 @@ The `include` statement does something fundamentally different than
 importing a module: it merely includes the contents of a file. The `include`
 statement is useful to split up a large module into several files:
 
-.. code-block:: nim
+  ```nim
   include fileA, fileB, fileC
+  ```
 
-The `include` statement can be used outside of the top level, as such:
+The `include` statement can be used outside the top level, as such:
 
-.. code-block:: nim
+  ```nim
   # Module A
   echo "Hello World!"
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module B
   proc main() =
     include A
 
   main() # => Hello World!
+  ```
 
 
 Module names in imports
 -----------------------
 
-A module alias can be introduced via the `as` keyword:
+A module alias can be introduced via the `as` keyword, after which the original module name
+is inaccessible:
 
-.. code-block:: nim
+  ```nim
   import std/strutils as su, std/sequtils as qu
 
   echo su.format("$1", "lalelu")
+  ```
 
-The original module name is then not accessible. The notations
-`path/to/module` or `"path/to/module"` can be used to refer to a module
+The notations `path/to/module` or `"path/to/module"` can be used to refer to a module
 in subdirectories:
 
-.. code-block:: nim
+  ```nim
   import lib/pure/os, "lib/pure/times"
+  ```
 
-Note that the module name is still `strutils` and not `lib/pure/strutils`
-and so one **cannot** do:
+Note that the module name is still `strutils` and not `lib/pure/strutils`,
+thus one **cannot** do:
 
-.. code-block:: nim
+  ```nim
   import lib/pure/strutils
   echo lib/pure/strutils.toUpperAscii("abc")
+  ```
 
 Likewise, the following does not make sense as the name is `strutils` already:
 
-.. code-block:: nim
+  ```nim
   import lib/pure/strutils as strutils
+  ```
 
 
 Collective imports from a directory
@@ -6062,8 +6902,9 @@ from the same directory.
 Path names are syntactically either Nim identifiers or string literals. If the path
 name is not a valid Nim identifier it needs to be a string literal:
 
-.. code-block:: nim
+  ```nim
   import "gfx/3d/somemodule" # in quotes because '3d' is not a valid Nim identifier
+  ```
 
 
 Pseudo import/include paths
@@ -6075,29 +6916,29 @@ avoid ambiguity when there are multiple modules with the same path.
 There are two pseudo directories:
 
 1. `std`: The `std` pseudo directory is the abstract location of Nim's standard
-library. For example, the syntax `import std / strutils` is used to unambiguously
-refer to the standard library's `strutils` module.
+   library. For example, the syntax `import std / strutils` is used to unambiguously
+   refer to the standard library's `strutils` module.
 2. `pkg`: The `pkg` pseudo directory is used to unambiguously refer to a Nimble
-package. However, for technical details that lie outside the scope of this document,
-its semantics are: *Use the search path to look for module name but ignore the standard
-library locations*. In other words, it is the opposite of `std`.
+   package. However, for technical details that lie outside the scope of this document,
+   its semantics are: *Use the search path to look for module name but ignore the standard
+   library locations*. In other words, it is the opposite of `std`.
 
+It is recommended and preferred but not currently enforced that all stdlib module imports include the std/ "pseudo directory" as part of the import name.
 
 From import statement
 ---------------------
 
-After the `from` statement, a module name follows followed by
+After the `from` keyword, a module name followed by
 an `import` to list the symbols one likes to use without explicit
 full qualification:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   from std/strutils import `%`
 
   echo "$1" % "abc"
   # always possible: full qualification:
   echo strutils.replace("abc", "a", "z")
+  ```
 
 It's also possible to use `from module import nil` if one wants to import
 the module but wants to enforce fully qualified access to every symbol
@@ -6110,34 +6951,38 @@ Export statement
 An `export` statement can be used for symbol forwarding so that client
 modules don't need to import a module's dependencies:
 
-.. code-block:: nim
+  ```nim
   # module B
   type MyObject* = object
+  ```
 
-.. code-block:: nim
+  ```nim
   # module A
   import B
   export B.MyObject
 
   proc `$`*(x: MyObject): string = "my object"
+  ```
 
 
-.. code-block:: nim
+  ```nim
   # module C
   import A
 
   # B.MyObject has been imported implicitly here:
   var x: MyObject
   echo $x
+  ```
 
 When the exported symbol is another module, all of its definitions will
 be forwarded. One can use an `except` list to exclude some of the symbols.
 
 Notice that when exporting, one needs to specify only the module name:
 
-.. code-block:: nim
+  ```nim
   import foo/bar/baz
   export baz
+  ```
 
 
 
@@ -6148,8 +6993,8 @@ the block in which the declaration occurred. The range where the identifier
 is known is the scope of the identifier. The exact scope of an
 identifier depends on the way it was declared.
 
-Block scope
-~~~~~~~~~~~
+### Block scope
+
 The *scope* of a variable declared in the declaration part of a block
 is valid from the point of declaration until the end of the block. If a
 block contains a second block, in which the identifier is redeclared,
@@ -6159,8 +7004,8 @@ identifier cannot be redefined in the same block, except if valid for
 procedure or iterator overloading purposes.
 
 
-Tuple or object scope
-~~~~~~~~~~~~~~~~~~~~~
+### Tuple or object scope
+
 The field identifiers inside a tuple or object definition are valid in the
 following places:
 
@@ -6168,32 +7013,99 @@ following places:
 * Field designators of a variable of the given tuple/object type.
 * In all descendant types of the object type.
 
-Module scope
-~~~~~~~~~~~~
+### Module scope
+
 All identifiers of a module are valid from the point of declaration until
 the end of the module. Identifiers from indirectly dependent modules are *not*
 available. The `system`:idx: module is automatically imported in every module.
 
-If a module imports an identifier by two different modules, each occurrence of
-the identifier has to be qualified unless it is an overloaded procedure or
-iterator in which case the overloading resolution takes place:
+If a module imports the same identifier from two different modules, the
+identifier is considered ambiguous, which can be resolved in the following ways:
 
-.. code-block:: nim
+* Qualifying the identifier as `module.identifier` resolves ambiguity
+  between modules. (See below for the case that the module name itself
+  is ambiguous.)
+* Calling the identifier as a routine makes overload resolution take place,
+  which resolves ambiguity in the case that one overload matches stronger
+  than the others.
+* Using the identifier in a context where the compiler can infer the type
+  of the identifier resolves ambiguity in the case that one definition
+  matches the type stronger than the others.
+
+  ```nim
   # Module A
   var x*: string
+  proc foo*(a: string) =
+    echo "A: ", a
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module B
   var x*: int
+  proc foo*(b: int) =
+    echo "B: ", b
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module C
   import A, B
+
+  foo("abc") # A: abc
+  foo(123) # B: 123
+  let inferred: proc (x: string) = foo
+  foo("def") # A: def
+
   write(stdout, x) # error: x is ambiguous
   write(stdout, A.x) # no error: qualifier used
+  
+  proc bar(a: int): int = a + 1
+  assert bar(x) == x + 1 # no error: only A.x of type int matches
 
   var x = 4
   write(stdout, x) # not ambiguous: uses the module C's x
+  ```
+Modules can share their name, however, when trying to qualify an identifier with the module name the compiler will fail with ambiguous identifier error. One can qualify the identifier by aliasing the module.
+
+
+```nim
+# Module A/C
+proc fb* = echo "fizz"
+```
+
+
+```nim
+# Module B/C
+proc fb* = echo "buzz"
+```
+
+
+```nim
+import A/C
+import B/C
+
+C.fb() # Error: ambiguous identifier: 'C'
+```
+
+
+```nim
+import A/C as fizz
+import B/C
+
+fizz.fb() # Works
+```
+
+
+Packages
+--------
+A collection of modules in a file tree with an ``identifier.nimble`` file in the
+root of the tree is called a Nimble package. A valid package name can only be a
+valid Nim identifier and thus its filename is ``identifier.nimble`` where
+``identifier`` is the desired package name. A module without a ``.nimble`` file
+is assigned the package identifier: `unknown`.
+
+The distinction between packages allows diagnostic compiler messages to be
+scoped to the current project's package vs foreign packages.
+
 
 
 Compiler Messages
@@ -6221,65 +7133,18 @@ deprecated pragma
 
 The deprecated pragma is used to mark a symbol as deprecated:
 
-.. code-block:: nim
+  ```nim
   proc p() {.deprecated.}
   var x {.deprecated.}: char
+  ```
 
 This pragma can also take in an optional warning string to relay to developers.
 
-.. code-block:: nim
+  ```nim
   proc thing(x: bool) {.deprecated: "use thong instead".}
+  ```
 
 
-noSideEffect pragma
--------------------
-
-The `noSideEffect` pragma is used to mark a proc/iterator that can have only
-side effects through parameters. This means that the proc/iterator only changes locations that are
-reachable from its parameters and the return value only depends on the
-parameters. If none of its parameters have the type `var`, `ref`, `ptr`, `cstring`, or `proc`,
-then no locations are modified.
-
-It is a static error to mark a proc/iterator to have no side effect if the compiler cannot verify this.
-
-As a special semantic rule, the built-in `debugEcho
-`_ pretends to be free of side effects
-so that it can be used for debugging routines marked as `noSideEffect`.
-
-`func` is syntactic sugar for a proc with no side effects:
-
-.. code-block:: nim
-  func `+` (x, y: int): int
-
-
-To override the compiler's side effect analysis a `{.noSideEffect.}`
-`cast` pragma block can be used:
-
-.. code-block:: nim
-
-  func f() =
-    {.cast(noSideEffect).}:
-      echo "test"
-
-When a `noSideEffect` proc has proc params `bar`, whether it can be used inside a `noSideEffect` context
-depends on what the compiler knows about `bar`:
-
-.. code-block:: nim
-    :test: "nim c $1"
-
-  func foo(bar: proc(): int): int = bar()
-  var count = 0
-  proc fn1(): int = 1
-  proc fn2(): int = (count.inc; count)
-  func fun1() = discard foo(fn1) # ok because fn1 is inferred as `func`
-  # func fun2() = discard foo(fn2) # would give: Error: 'fun2' can have side effects
-
-  # with callbacks, the compiler is conservative, ie that bar will have side effects
-  var foo2: type(foo) = foo
-  func main() =
-    discard foo(fn1) # ok
-    # discard foo2(fn1) # now this errors
-
 
 compileTime pragma
 ------------------
@@ -6289,23 +7154,23 @@ procs are useful as helpers for macros. Since version 0.12.0 of the language, a
 proc that uses `system.NimNode` within its parameter types is implicitly
 declared `compileTime`:
 
-.. code-block:: nim
+  ```nim
   proc astHelper(n: NimNode): NimNode =
     result = n
+  ```
 
 Is the same as:
 
-.. code-block:: nim
+  ```nim
   proc astHelper(n: NimNode): NimNode {.compileTime.} =
     result = n
+  ```
 
 `compileTime` variables are available at runtime too. This simplifies certain
 idioms where variables are filled at compile-time (for example, lookup tables)
 but accessed at runtime:
 
-.. code-block:: nim
-    :test: "nim c -r $1"
-
+  ```nim  test = "nim c -r $1"
   import std/macros
 
   var nameToProc {.compileTime.}: seq[(string, proc (): string {.nimcall.})]
@@ -6323,9 +7188,10 @@ but accessed at runtime:
   proc baz: string {.registerProc.} = "baz"
 
   doAssert nameToProc[2][1]() == "baz"
+  ```
 
 
-noReturn pragma
+noreturn pragma
 ---------------
 The `noreturn` pragma is used to mark a proc that never returns.
 
@@ -6336,20 +7202,22 @@ The `acyclic` pragma can be used for object types to mark them as acyclic
 even though they seem to be cyclic. This is an **optimization** for the garbage
 collector to not consider objects of this type as part of a cycle:
 
-.. code-block:: nim
+  ```nim
   type
     Node = ref NodeObj
     NodeObj {.acyclic.} = object
       left, right: Node
       data: string
+  ```
 
 Or if we directly use a ref object:
 
-.. code-block:: nim
+  ```nim
   type
     Node {.acyclic.} = ref object
       left, right: Node
       data: string
+  ```
 
 In the example, a tree structure is declared with the `Node` type. Note that
 the type definition is recursive and the GC has to assume that objects of
@@ -6377,7 +7245,7 @@ because the semantics of Nim require deep copying of sequences and strings.
 This can be expensive, especially if sequences are used to build a tree
 structure:
 
-.. code-block:: nim
+  ```nim
   type
     NodeKind = enum nkLeaf, nkInner
     Node {.shallow.} = object
@@ -6386,6 +7254,7 @@ structure:
         strVal: string
       of nkInner:
         children: seq[Node]
+  ```
 
 
 pure pragma
@@ -6403,7 +7272,7 @@ asmNoStackFrame pragma
 A proc can be marked with the `asmNoStackFrame` pragma to tell the compiler
 it should not generate a stack frame for the proc. There are also no exit
 statements like `return result;` generated and the generated C function is
-declared as `__declspec(naked)` or `__attribute__((naked))` (depending on
+declared as `__declspec(naked)`:c: or `__attribute__((naked))`:c: (depending on
 the used C compiler).
 
 **Note**: This pragma should only be used by procs which consist solely of
@@ -6420,9 +7289,10 @@ annotate a symbol (like an iterator or proc). The *usage* of the symbol then
 triggers a static error. This is especially useful to rule out that some
 operation is valid due to overloading and type conversions:
 
-.. code-block:: nim
+  ```nim
   ## check that underlying int values are compared and not the pointers:
   proc `==`(x, y: ptr int): bool {.error.}
+  ```
 
 
 fatal pragma
@@ -6431,9 +7301,10 @@ The `fatal` pragma is used to make the compiler output an error message
 with the given content. In contrast to the `error` pragma, the compilation
 is guaranteed to be aborted by this pragma. Example:
 
-.. code-block:: nim
+  ```nim
   when not defined(objc):
     {.fatal: "Compile this program with the objc command!".}
+  ```
 
 warning pragma
 --------------
@@ -6450,17 +7321,17 @@ line pragma
 The `line` pragma can be used to affect line information of the annotated
 statement, as seen in stack backtraces:
 
-.. code-block:: nim
-
+  ```nim
   template myassert*(cond: untyped, msg = "") =
     if not cond:
       # change run-time line information of the 'raise' statement:
       {.line: instantiationInfo().}:
-        raise newException(EAssertionFailed, msg)
+        raise newException(AssertionDefect, msg)
+  ```
 
-If the `line` pragma is used with a parameter, the parameter needs be a
+If the `line` pragma is used with a parameter, the parameter needs to be a
 `tuple[filename: string, line: int]`. If it is used without a parameter,
-`system.InstantiationInfo()` is used.
+`system.instantiationInfo()` is used.
 
 
 linearScanEnd pragma
@@ -6469,7 +7340,7 @@ The `linearScanEnd` pragma can be used to tell the compiler how to
 compile a Nim `case`:idx: statement. Syntactically it has to be used as a
 statement:
 
-.. code-block:: nim
+  ```nim
   case myInt
   of 0:
     echo "most common case"
@@ -6478,9 +7349,10 @@ statement:
     echo "second most common case"
   of 2: echo "unlikely: use branch table"
   else: echo "unlikely too: use branch table for ", myInt
+  ```
 
 In the example, the case branches `0` and `1` are much more common than
-the other cases. Therefore the generated assembler code should test for these
+the other cases. Therefore, the generated assembler code should test for these
 values first so that the CPU's branch predictor has a good chance to succeed
 (avoiding an expensive CPU pipeline stall). The other cases might be put into a
 jump table for O(1) overhead but at the cost of a (very likely) pipeline
@@ -6497,8 +7369,7 @@ The `computedGoto` pragma can be used to tell the compiler how to
 compile a Nim `case`:idx: in a `while true` statement.
 Syntactically it has to be used as a statement inside the loop:
 
-.. code-block:: nim
-
+  ```nim
   type
     MyEnum = enum
       enumA, enumB, enumC, enumD, enumE
@@ -6530,6 +7401,7 @@ Syntactically it has to be used as a statement inside the loop:
       inc(pc)
 
   vm()
+  ```
 
 As the example shows, `computedGoto` is mostly useful for interpreters. If
 the underlying backend (C compiler) does not support the computed goto
@@ -6539,9 +7411,24 @@ extension the pragma is simply ignored.
 immediate pragma
 ----------------
 
-The immediate pragma is obsolete. See `Typed vs untyped parameters
-<#templates-typed-vs-untyped-parameters>`_.
+The immediate pragma is obsolete. See [Typed vs untyped parameters].
 
+redefine pragma
+---------------
+
+Redefinition of template symbols with the same signature is allowed.
+This can be made explicit with the `redefine` pragma:
+
+```nim
+template foo: int = 1
+echo foo() # 1
+template foo: int {.redefine.} = 2
+echo foo() # 2
+# warning: implicit redefinition of template
+template foo: int = 3
+```
+
+This is mostly intended for macro generated code.
 
 compilation option pragmas
 --------------------------
@@ -6579,9 +7466,10 @@ callconv         cdecl|...        Specifies the default calling convention for
 
 Example:
 
-.. code-block:: nim
+  ```nim
   {.checks: off, optimization: speed.}
   # compile without runtime checks and optimize for speed
+  ```
 
 
 push and pop pragmas
@@ -6589,16 +7477,17 @@ push and pop pragmas
 The `push/pop`:idx: pragmas are very similar to the option directive,
 but are used to override the settings temporarily. Example:
 
-.. code-block:: nim
+  ```nim
   {.push checks: off.}
   # compile this section without runtime checks as it is
   # speed critical
   # ... some code ...
   {.pop.} # restore old settings
+  ```
 
 `push/pop`:idx: can switch on/off some standard library pragmas, example:
 
-.. code-block:: nim
+  ```nim
   {.push inline.}
   proc thisIsInlined(): int = 42
   func willBeInlined(): float = 42.0
@@ -6609,9 +7498,10 @@ but are used to override the settings temporarily. Example:
   template example(): string = "https://nim-lang.org"
   {.pop.}
 
-  {.push deprecated, hint[LineTooLong]: off, used, stackTrace: off.}
+  {.push deprecated, used, stackTrace: off.}
   proc sample(): bool = true
   {.pop.}
+  ```
 
 For third party pragmas, it depends on its implementation but uses the same syntax.
 
@@ -6633,10 +7523,11 @@ The `global` pragma can be applied to a variable within a proc to instruct
 the compiler to store it in a global location and initialize it once at program
 startup.
 
-.. code-block:: nim
+  ```nim
   proc isHexNumber(s: string): bool =
     var pattern {.global.} = re"[0-9a-fA-F]+"
     result = s.match(pattern)
+  ```
 
 When used within a generic proc, a separate unique global variable will be
 created for each instantiation of the proc. The order of initialization of
@@ -6646,13 +7537,15 @@ and before any variable in a module that imports it.
 
 Disabling certain messages
 --------------------------
-Nim generates some warnings and hints ("line too long") that may annoy the
+Nim generates some warnings and hints that may annoy the
 user. A mechanism for disabling certain messages is provided: Each hint
-and warning message contains a symbol in brackets. This is the message's
-identifier that can be used to enable or disable it:
+and warning message is associated with a symbol. This is the message's
+identifier, which can be used to enable or disable the message by putting it
+in brackets following the pragma:
 
-.. code-block:: Nim
-  {.hint[LineTooLong]: off.} # turn off the hint about too long lines
+  ```Nim
+  {.hint[XDeclaredButNotUsed]: off.} # Turn off the hint about declared but not used symbols.
+  ```
 
 This is often better than disabling all warnings at once.
 
@@ -6664,7 +7557,7 @@ Nim produces a warning for symbols that are not exported and not used either.
 The `used` pragma can be attached to a symbol to suppress this warning. This
 is particularly useful when the symbol was generated by a macro:
 
-.. code-block:: nim
+  ```nim
   template implementArithOps(T) =
     proc echoAdd(a, b: T) {.used.} =
       echo a + b
@@ -6674,18 +7567,19 @@ is particularly useful when the symbol was generated by a macro:
   # no warning produced for the unused 'echoSub'
   implementArithOps(int)
   echoAdd 3, 5
+  ```
 
 `used` can also be used as a top-level statement to mark a module as "used".
 This prevents the "Unused import" warning:
 
-.. code-block:: nim
-
+  ```nim
   # module: debughelper.nim
   when defined(nimHasUsed):
     # 'import debughelper' is so useful for debugging
     # that Nim shouldn't produce a warning for that import,
     # even if currently unused:
     {.used.}
+  ```
 
 
 experimental pragma
@@ -6694,11 +7588,12 @@ experimental pragma
 The `experimental` pragma enables experimental language features. Depending
 on the concrete feature, this means that the feature is either considered
 too unstable for an otherwise stable release or that the future of the feature
-is uncertain (it may be removed at any time).
+is uncertain (it may be removed at any time). See the
+[experimental manual](manual_experimental.html) for more details.
 
 Example:
 
-.. code-block:: nim
+  ```nim
   import std/threadpool
   {.experimental: "parallel".}
 
@@ -6711,6 +7606,7 @@ Example:
         spawn threadedEcho("echo in parallel", i)
 
   useParallel()
+  ```
 
 
 As a top-level statement, the experimental pragma enables a feature for the
@@ -6718,8 +7614,7 @@ rest of the module it's enabled in. This is problematic for macro and generic
 instantiations that cross a module scope. Currently, these usages have to be
 put into a `.push/pop` environment:
 
-.. code-block:: nim
-
+  ```nim
   # client.nim
   proc useParallel*[T](unused: T) =
     # use a generic T here to show the problem.
@@ -6729,12 +7624,13 @@ put into a `.push/pop` environment:
         echo "echo in parallel"
 
     {.pop.}
+  ```
 
 
-.. code-block:: nim
-
+  ```nim
   import client
   useParallel(1)
+  ```
 
 
 Implementation Specific Pragmas
@@ -6749,17 +7645,50 @@ Bitsize pragma
 The `bitsize` pragma is for object field members. It declares the field as
 a bitfield in C/C++.
 
-.. code-block:: Nim
+  ```Nim
   type
     mybitfield = object
       flag {.bitsize:1.}: cuint
+  ```
 
 generates:
 
-.. code-block:: C
+  ```C
   struct mybitfield {
     unsigned int flag:1;
   };
+  ```
+
+
+size pragma
+-----------
+Nim automatically determines the size of an enum.
+But when wrapping a C enum type, it needs to be of a specific size.
+The `size pragma` allows specifying the size of the enum type.
+
+  ```Nim
+  type
+    EventType* {.size: sizeof(uint32).} = enum
+      QuitEvent,
+      AppTerminating,
+      AppLowMemory
+
+  doAssert sizeof(EventType) == sizeof(uint32)
+  ```
+
+The `size pragma` can also specify the size of an `importc` incomplete object type
+so that one can get the size of it at compile time even if it was declared without fields.
+
+  ```Nim
+    type
+      AtomicFlag* {.importc: "atomic_flag", header: "", size: 1.} = object
+
+    static:
+      # if AtomicFlag didn't have the size pragma, this code would result in a compile time error.
+      echo sizeof(AtomicFlag)
+  ```
+
+The `size pragma` accepts only the values 1, 2, 4 or 8.
 
 
 Align pragma
@@ -6772,33 +7701,48 @@ alignments that are weaker than other align pragmas on the same
 declaration are ignored. Alignments that are weaker than the
 alignment requirement of the type are ignored.
 
-.. code-block:: Nim
+  ```Nim
+  type
+    sseType = object
+      sseData {.align(16).}: array[4, float32]
 
-   type
-     sseType = object
-       sseData {.align(16).}: array[4, float32]
+    # every object will be aligned to 128-byte boundary
+    Data = object
+      x: char
+      cacheline {.align(128).}: array[128, char] # over-aligned array of char,
 
-     # every object will be aligned to 128-byte boundary
-     Data = object
-       x: char
-       cacheline {.align(128).}: array[128, char] # over-aligned array of char,
+  proc main() =
+    echo "sizeof(Data) = ", sizeof(Data), " (1 byte + 127 bytes padding + 128-byte array)"
+    # output: sizeof(Data) = 256 (1 byte + 127 bytes padding + 128-byte array)
+    echo "alignment of sseType is ", alignof(sseType)
+    # output: alignment of sseType is 16
+    var d {.align(2048).}: Data # this instance of data is aligned even stricter
 
-   proc main() =
-     echo "sizeof(Data) = ", sizeof(Data), " (1 byte + 127 bytes padding + 128-byte array)"
-     # output: sizeof(Data) = 256 (1 byte + 127 bytes padding + 128-byte array)
-     echo "alignment of sseType is ", alignof(sseType)
-     # output: alignment of sseType is 16
-     var d {.align(2048).}: Data # this instance of data is aligned even stricter
-
-   main()
+  main()
+  ```
 
 This pragma has no effect on the JS backend.
 
 
+Noalias pragma
+--------------
+
+Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables
+and parameters. It is mapped directly to C/C++'s `restrict`:c: keyword and means that
+the underlying pointer is pointing to a unique location in memory, no other aliases to
+this location exist. It is *unchecked* that this alias restriction is followed. If the
+restriction is violated, the backend optimizer is free to miscompile the code.
+This is an **unsafe** language feature.
+
+Ideally in later versions of the language, the restriction will be enforced at
+compile time. (This is also why the name `noalias` was chosen instead of a more
+verbose name like `unsafeAssumeNoAlias`.)
+
+
 Volatile pragma
 ---------------
 The `volatile` pragma is for variables only. It declares the variable as
-`volatile`, whatever that means in C/C++ (its semantics are not well defined
+`volatile`:c:, whatever that means in C/C++ (its semantics are not well-defined
 in C/C++).
 
 **Note**: This pragma will not exist for the LLVM backend.
@@ -6811,10 +7755,11 @@ type, etc.) and is sometimes useful for interoperability with C:
 It tells Nim that it should not generate a declaration for the symbol in
 the C code. For example:
 
-.. code-block:: Nim
+  ```Nim
   var
     EACCES {.importc, nodecl.}: cint # pretend EACCES was a variable, as
                                      # Nim does not know its value
+  ```
 
 However, the `header` pragma is often the better alternative.
 
@@ -6825,17 +7770,18 @@ Header pragma
 -------------
 The `header` pragma is very similar to the `nodecl` pragma: It can be
 applied to almost any symbol and specifies that it should not be declared
-and instead, the generated code should contain an `#include`:
+and instead, the generated code should contain an `#include`:c:\:
 
-.. code-block:: Nim
+  ```Nim
   type
     PFile {.importc: "FILE*", header: "".} = distinct pointer
       # import C's FILE* type; Nim will treat it as a new pointer type
+  ```
 
 The `header` pragma always expects a string constant. The string constant
 contains the header file: As usual for C, a system header file is enclosed
-in angle brackets: `<>`. If no angle brackets are given, Nim
-encloses the header file in `""` in the generated C code.
+in angle brackets: `<>`:c:. If no angle brackets are given, Nim
+encloses the header file in `""`:c: in the generated C code.
 
 **Note**: This will not work for the LLVM backend.
 
@@ -6843,12 +7789,13 @@ encloses the header file in `""` in the generated C code.
 IncompleteStruct pragma
 -----------------------
 The `incompleteStruct` pragma tells the compiler to not use the
-underlying C `struct` in a `sizeof` expression:
+underlying C `struct`:c: in a `sizeof` expression:
 
-.. code-block:: Nim
+  ```Nim
   type
     DIR* {.importc: "DIR", header: "",
            pure, incompleteStruct.} = object
+  ```
 
 
 Compile pragma
@@ -6856,17 +7803,25 @@ Compile pragma
 The `compile` pragma can be used to compile and link a C/C++ source file
 with the project:
 
-.. code-block:: Nim
+This pragma can take three forms. The first is a simple file input:
+  ```Nim
   {.compile: "myfile.cpp".}
+  ```
+
+The second form is a tuple where the second arg is the output name strutils formatter:
+  ```Nim
+  {.compile: ("file.c", "$1.o").}
+  ```
 
 **Note**: Nim computes a SHA1 checksum and only recompiles the file if it
-has changed. One can use the `-f` command-line option to force the recompilation
-of the file.
+has changed. One can use the `-f`:option: command-line option to force
+the recompilation of the file.
 
 Since 1.4 the `compile` pragma is also available with this syntax:
 
-.. code-block:: Nim
+  ```Nim
   {.compile("myfile.cpp", "--custom flags here").}
+  ```
 
 As can be seen in the example, this new variant allows for custom flags
 that are passed to the C compiler when the file is recompiled.
@@ -6876,52 +7831,58 @@ Link pragma
 -----------
 The `link` pragma can be used to link an additional file with the project:
 
-.. code-block:: Nim
+  ```Nim
   {.link: "myfile.o".}
+  ```
 
 
-PassC pragma
+passc pragma
 ------------
 The `passc` pragma can be used to pass additional parameters to the C
-compiler like one would using the command-line switch `--passc`:
+compiler like one would use the command-line switch `--passc`:option:\:
 
-.. code-block:: Nim
+  ```Nim
   {.passc: "-Wall -Werror".}
+  ```
 
-Note that one can use `gorge` from the `system module `_ to
+Note that one can use `gorge` from the [system module](system.html) to
 embed parameters from an external command that will be executed
 during semantic analysis:
 
-.. code-block:: Nim
+  ```Nim
   {.passc: gorge("pkg-config --cflags sdl").}
+  ```
 
 
-LocalPassc pragma
+localPassC pragma
 -----------------
-The `localPassc` pragma can be used to pass additional parameters to the C
+The `localPassC` pragma can be used to pass additional parameters to the C
 compiler, but only for the C/C++ file that is produced from the Nim module
 the pragma resides in:
 
-.. code-block:: Nim
+  ```Nim
   # Module A.nim
   # Produces: A.nim.cpp
-  {.localPassc: "-Wall -Werror".} # Passed when compiling A.nim.cpp
+  {.localPassC: "-Wall -Werror".} # Passed when compiling A.nim.cpp
+  ```
 
 
-PassL pragma
+passl pragma
 ------------
-The `passL` pragma can be used to pass additional parameters to the linker
-like one would be using the command-line switch `--passL`:
+The `passl` pragma can be used to pass additional parameters to the linker
+like one would be using the command-line switch `--passl`:option:\:
 
-.. code-block:: Nim
-  {.passL: "-lSDLmain -lSDL".}
+  ```Nim
+  {.passl: "-lSDLmain -lSDL".}
+  ```
 
-Note that one can use `gorge` from the `system module `_ to
+Note that one can use `gorge` from the [system module](system.html) to
 embed parameters from an external command that will be executed
 during semantic analysis:
 
-.. code-block:: Nim
-  {.passL: gorge("pkg-config --libs sdl").}
+  ```Nim
+  {.passl: gorge("pkg-config --libs sdl").}
+  ```
 
 
 Emit pragma
@@ -6933,7 +7894,7 @@ extremely useful for interfacing with `C++`:idx: or `Objective C`:idx: code.
 
 Example:
 
-.. code-block:: Nim
+  ```Nim
   {.emit: """
   static int cvariable = 420;
   """.}
@@ -6946,27 +7907,29 @@ Example:
   {.pop.}
 
   embedsC()
+  ```
 
-`nimbase.h` defines `NIM_EXTERNC` C macro that can be used for
-`extern "C"` code to work with both `nim c` and `nim cpp`, e.g.:
+``nimbase.h`` defines `NIM_EXTERNC`:c: C macro that can be used for
+`extern "C"`:cpp: code to work with both `nim c`:cmd: and `nim cpp`:cmd:, e.g.:
 
-.. code-block:: Nim
+  ```Nim
   proc foobar() {.importc:"$1".}
   {.emit: """
   #include 
   NIM_EXTERNC
   void fun(){}
   """.}
+  ```
 
-For backward compatibility, if the argument to the `emit` statement
-is a single string literal, Nim symbols can be referred to via backticks.
-This usage is however deprecated.
+.. note:: For backward compatibility, if the argument to the `emit` statement
+  is a single string literal, Nim symbols can be referred to via backticks.
+  This usage is however deprecated.
 
 For a top-level emit statement, the section where in the generated C/C++ file
-the code should be emitted can be influenced via the
-prefixes `/*TYPESECTION*/` or `/*VARSECTION*/` or `/*INCLUDESECTION*/`:
+the code should be emitted can be influenced via the prefixes
+`/*TYPESECTION*/`:c: or `/*VARSECTION*/`:c: or `/*INCLUDESECTION*/`:c:\:
 
-.. code-block:: Nim
+  ```Nim
   {.emit: """/*TYPESECTION*/
   struct Vector3 {
   public:
@@ -6980,24 +7943,25 @@ prefixes `/*TYPESECTION*/` or `/*VARSECTION*/` or `/*INCLUDESECTION*/`:
     x: cfloat
 
   proc constructVector3(a: cfloat): Vector3 {.importcpp: "Vector3(@)", nodecl}
+  ```
 
 
 ImportCpp pragma
 ----------------
 
-**Note**: `c2nim `_ can parse a large subset of C++ and knows
+**Note**: [c2nim](https://github.com/nim-lang/c2nim/blob/master/doc/c2nim.rst)
+can parse a large subset of C++ and knows
 about the `importcpp` pragma pattern language. It is not necessary
 to know all the details described here.
 
 
-Similar to the `importc pragma for C
-<#foreign-function-interface-importc-pragma>`_, the
+Similar to the [importc pragma] for C, the
 `importcpp` pragma can be used to import `C++`:idx: methods or C++ symbols
 in general. The generated code then uses the C++ method calling
-syntax: `obj->method(arg)`.  In combination with the `header` and `emit`
+syntax: `obj->method(arg)`:cpp:. In combination with the `header` and `emit`
 pragmas this allows *sloppy* interfacing with libraries written in C++:
 
-.. code-block:: Nim
+  ```Nim
   # Horrible example of how to interface with a C++ engine ... ;-)
 
   {.link: "/usr/lib/libIrrlicht.so".}
@@ -7023,138 +7987,147 @@ pragmas this allows *sloppy* interfacing with libraries written in C++:
     header: irr, importcpp: "createDevice(@)".}
   proc run(device: IrrlichtDevice): bool {.
     header: irr, importcpp: "#.run(@)".}
+  ```
 
-The compiler needs to be told to generate C++ (command `cpp`) for
+The compiler needs to be told to generate C++ (command `cpp`:option:) for
 this to work. The conditional symbol `cpp` is defined when the compiler
 emits C++ code.
 
-Namespaces
-~~~~~~~~~~
+### Namespaces
 
-The *sloppy interfacing* example uses `.emit` to produce `using namespace`
+The *sloppy interfacing* example uses `.emit` to produce `using namespace`:cpp:
 declarations. It is usually much better to instead refer to the imported name
-via the `namespace::identifier` notation:
+via the `namespace::identifier`:cpp: notation:
 
-.. code-block:: nim
+  ```nim
   type
     IrrlichtDeviceObj {.header: irr,
                         importcpp: "irr::IrrlichtDevice".} = object
+  ```
 
 
-Importcpp for enums
-~~~~~~~~~~~~~~~~~~~
+### Importcpp for enums
 
 When `importcpp` is applied to an enum type the numerical enum values are
-annotated with the C++ enum type, like in this example: `((TheCppEnum)(3))`.
+annotated with the C++ enum type, like in this example:
+`((TheCppEnum)(3))`:cpp:.
 (This turned out to be the simplest way to implement it.)
 
 
-Importcpp for procs
-~~~~~~~~~~~~~~~~~~~
+### Importcpp for procs
 
 Note that the `importcpp` variant for procs uses a somewhat cryptic pattern
 language for maximum flexibility:
 
-- A hash `#` symbol is replaced by the first or next argument.
-- A dot following the hash `#.` indicates that the call should use C++'s dot
+- A hash ``#`` symbol is replaced by the first or next argument.
+- A dot following the hash ``#.`` indicates that the call should use C++'s dot
   or arrow notation.
-- An at symbol `@` is replaced by the remaining arguments, separated by commas.
+- An at symbol ``@`` is replaced by the remaining arguments,
+  separated by commas.
 
 For example:
 
-.. code-block:: nim
+  ```nim
   proc cppMethod(this: CppObj, a, b, c: cint) {.importcpp: "#.CppMethod(@)".}
   var x: ptr CppObj
   cppMethod(x[], 1, 2, 3)
+  ```
 
 Produces:
 
-.. code-block:: C
+  ```C
   x->CppMethod(1, 2, 3)
+  ```
 
 As a special rule to keep backward compatibility with older versions of the
 `importcpp` pragma, if there is no special pattern
-character (any of `# ' @`) at all, C++'s
+character (any of ``# ' @``) at all, C++'s
 dot or arrow notation is assumed, so the above example can also be written as:
 
-.. code-block:: nim
+  ```nim
   proc cppMethod(this: CppObj, a, b, c: cint) {.importcpp: "CppMethod".}
+  ```
 
 Note that the pattern language naturally also covers C++'s operator overloading
 capabilities:
 
-.. code-block:: nim
+  ```nim
   proc vectorAddition(a, b: Vec3): Vec3 {.importcpp: "# + #".}
   proc dictLookup(a: Dict, k: Key): Value {.importcpp: "#[#]".}
+  ```
 
 
-- An apostrophe `'` followed by an integer `i` in the range 0..9
+- An apostrophe ``'`` followed by an integer ``i`` in the range 0..9
   is replaced by the i'th parameter *type*. The 0th position is the result
   type. This can be used to pass types to C++ function templates. Between
-  the `'` and the digit, an asterisk can be used to get to the base type
-  of the type. (So it "takes away a star" from the type; `T*` becomes `T`.)
+  the ``'`` and the digit, an asterisk can be used to get to the base type
+  of the type. (So it "takes away a star" from the type; `T*`:c: becomes `T`.)
   Two stars can be used to get to the element type of the element type etc.
 
 For example:
 
-.. code-block:: nim
-
+  ```nim
   type Input {.importcpp: "System::Input".} = object
   proc getSubsystem*[T](): ptr T {.importcpp: "SystemManager::getSubsystem<'*0>()", nodecl.}
 
   let x: ptr Input = getSubsystem[Input]()
+  ```
 
 Produces:
 
-.. code-block:: C
+  ```C
   x = SystemManager::getSubsystem()
+  ```
 
 
-- `#@` is a special case to support a `cnew` operation. It is required so
+- ``#@`` is a special case to support a `cnew` operation. It is required so
   that the call expression is inlined directly, without going through a
   temporary location. This is only required to circumvent a limitation of the
   current code generator.
 
-For example C++'s `new` operator can be "imported" like this:
+For example C++'s `new`:cpp: operator can be "imported" like this:
 
-.. code-block:: nim
+  ```nim
   proc cnew*[T](x: T): ptr T {.importcpp: "(new '*0#@)", nodecl.}
 
   # constructor of 'Foo':
   proc constructFoo(a, b: cint): Foo {.importcpp: "Foo(@)".}
 
   let x = cnew constructFoo(3, 4)
+  ```
 
 Produces:
 
-.. code-block:: C
+  ```C
   x = new Foo(3, 4)
+  ```
 
-However, depending on the use case `new Foo` can also be wrapped like this
+However, depending on the use case `new Foo`:cpp: can also be wrapped like this
 instead:
 
-.. code-block:: nim
+  ```nim
   proc newFoo(a, b: cint): ptr Foo {.importcpp: "new Foo(@)".}
 
   let x = newFoo(3, 4)
+  ```
 
 
-Wrapping constructors
-~~~~~~~~~~~~~~~~~~~~~
+### Wrapping constructors
 
 Sometimes a C++ class has a private copy constructor and so code like
-`Class c = Class(1,2);` must not be generated but instead `Class c(1,2);`.
+`Class c = Class(1,2);`:cpp: must not be generated but instead
+`Class c(1,2);`:cpp:.
 For this purpose the Nim proc that wraps a C++ constructor needs to be
 annotated with the `constructor`:idx: pragma. This pragma also helps to generate
 faster C++ code since construction then doesn't invoke the copy constructor:
 
-.. code-block:: nim
+  ```nim
   # a better constructor of 'Foo':
   proc constructFoo(a, b: cint): Foo {.importcpp: "Foo(@)", constructor.}
+  ```
 
 
-Wrapping destructors
-~~~~~~~~~~~~~~~~~~~~
+### Wrapping destructors
 
 Since Nim generates C++ directly, any destructor is called implicitly by the
 C++ compiler at the scope exits. This means that often one can get away with
@@ -7162,20 +8135,18 @@ not wrapping the destructor at all! However, when it needs to be invoked
 explicitly, it needs to be wrapped. The pattern language provides
 everything that is required:
 
-.. code-block:: nim
+  ```nim
   proc destroyFoo(this: var Foo) {.importcpp: "#.~Foo()".}
+  ```
 
 
-Importcpp for objects
-~~~~~~~~~~~~~~~~~~~~~
+### Importcpp for objects
 
 Generic `importcpp`'ed objects are mapped to C++ templates. This means that
 one can import C++'s templates rather easily without the need for a pattern
 language for object types:
 
-.. code-block:: nim
-    :test: "nim cpp $1"
-
+  ```nim  test = "nim cpp $1"
   type
     StdMap[K, V] {.importcpp: "std::map", header: "".} = object
   proc `[]=`[K, V](this: var StdMap[K, V]; key: K; val: V) {.
@@ -7183,56 +8154,57 @@ language for object types:
 
   var x: StdMap[cint, cdouble]
   x[6] = 91.4
+  ```
 
 
 Produces:
 
-.. code-block:: C
+  ```C
   std::map x;
   x[6] = 91.4;
+  ```
 
 
 - If more precise control is needed, the apostrophe `'` can be used in the
   supplied pattern to denote the concrete type parameters of the generic type.
   See the usage of the apostrophe operator in proc patterns for more details.
 
-.. code-block:: nim
+    ```nim
+    type
+      VectorIterator[T] {.importcpp: "std::vector<'0>::iterator".} = object
 
-  type
-    VectorIterator {.importcpp: "std::vector<'0>::iterator".} [T] = object
+    var x: VectorIterator[cint]
+    ```
 
-  var x: VectorIterator[cint]
+  Produces:
 
+    ```C
 
-Produces:
-
-.. code-block:: C
-
-  std::vector::iterator x;
+    std::vector::iterator x;
+    ```
 
 
 ImportJs pragma
 ---------------
 
-Similar to the `importcpp pragma for C++ <#implementation-specific-pragmas-importcpp-pragma>`_,
+Similar to the [importcpp pragma] for C++,
 the `importjs` pragma can be used to import Javascript methods or
 symbols in general. The generated code then uses the Javascript method
-calling syntax: `obj.method(arg)`.
+calling syntax: ``obj.method(arg)``.
 
 
 ImportObjC pragma
 -----------------
-Similar to the `importc pragma for C
-<#foreign-function-interface-importc-pragma>`_, the `importobjc` pragma can
-be used to import `Objective C`:idx: methods.  The generated code then uses the
-Objective C method calling syntax: `[obj method param1: arg]`.
+Similar to the [importc pragma] for C, the `importobjc` pragma can
+be used to import `Objective C`:idx: methods. The generated code then uses the
+Objective C method calling syntax: ``[obj method param1: arg]``.
 In addition with the `header` and `emit` pragmas this
 allows *sloppy* interfacing with libraries written in Objective C:
 
-.. code-block:: Nim
+  ```Nim
   # horrible example of how to interface with GNUStep ...
 
-  {.passL: "-lobjc".}
+  {.passl: "-lobjc".}
   {.emit: """
   #include 
   @interface Greeter:Object
@@ -7264,9 +8236,10 @@ allows *sloppy* interfacing with libraries written in Objective C:
   var g = newGreeter()
   g.greet(12, 34)
   g.free()
+  ```
 
-The compiler needs to be told to generate Objective C (command `objc`) for
-this to work. The conditional symbol `objc` is defined when the compiler
+The compiler needs to be told to generate Objective C (command `objc`:option:) for
+this to work. The conditional symbol ``objc`` is defined when the compiler
 emits Objective C code.
 
 
@@ -7274,68 +8247,87 @@ CodegenDecl pragma
 ------------------
 
 The `codegenDecl` pragma can be used to directly influence Nim's code
-generator. It receives a format string that determines how the variable
-or proc is declared in the generated code.
+generator. It receives a format string that determines how the variable,
+proc or object type is declared in the generated code.
 
-For variables, $1 in the format string represents the type of the variable
-and $2 is the name of the variable.
+For variables, $1 in the format string represents the type of the variable,
+$2 is the name of the variable, and each appearance of $# represents $1/$2
+respectively according to its position.
 
 The following Nim code:
 
-.. code-block:: nim
+  ```nim
   var
     a {.codegenDecl: "$# progmem $#".}: int
+  ```
 
 will generate this C code:
 
-.. code-block:: c
+  ```c
   int progmem a
+  ```
 
 For procedures, $1 is the return type of the procedure, $2 is the name of
-the procedure, and $3 is the parameter list.
+the procedure, $3 is the parameter list, and each appearance of $# represents
+$1/$2/$3 respectively according to its position.
 
 The following nim code:
 
-.. code-block:: nim
+  ```nim
   proc myinterrupt() {.codegenDecl: "__interrupt $# $#$#".} =
     echo "realistic interrupt handler"
+  ```
 
 will generate this code:
 
-.. code-block:: c
+  ```c
   __interrupt void myinterrupt()
+  ```
 
+For object types, the $1 represents the name of the object type, $2 is the list of
+fields and $3 is the base type.
+
+```nim
+
+const strTemplate = """
+  struct $1 {
+    $2
+  };
+"""
+type Foo {.codegenDecl:strTemplate.} = object
+  a, b: int
+```
+
+will generate this code:
+
+
+```c
+struct Foo {
+  NI a;
+  NI b;
+};
+```
 
 `cppNonPod` pragma
 ------------------
 
-The `.cppNonPod` pragma should be used for non-POD `importcpp` types so that they
+The `cppNonPod` pragma should be used for non-POD `importcpp` types so that they
 work properly (in particular regarding constructor and destructor) for
-`.threadvar` variables. This requires `--tlsEmulation:off`.
+`threadvar` variables. This requires `--tlsEmulation:off`:option:.
 
-.. code-block:: nim
+  ```nim
   type Foo {.cppNonPod, importcpp, header: "funs.h".} = object
     x: cint
   proc main()=
     var a {.threadvar.}: Foo
+  ```
 
-InjectStmt pragma
------------------
-
-The `injectStmt` pragma can be used to inject a statement before every
-other statement in the current module. It is only supposed to be used for
-debugging:
-
-.. code-block:: nim
-  {.injectStmt: gcInvariants().}
-
-  # ... complex code here that produces crashes ...
 
 compile-time define pragmas
 ---------------------------
 
 The pragmas listed here can be used to optionally accept values from
-the -d/--define option at compile time.
+the `-d/--define`:option: option at compile time.
 
 The implementation currently provides the following possible options (various
 others may be added later).
@@ -7348,19 +8340,40 @@ pragma             description
 `booldefine`:idx:  Reads in a build-time define as a bool
 =================  ============================================
 
-.. code-block:: nim
-   const FooBar {.intdefine.}: int = 5
-   echo FooBar
+  ```nim
+  const FooBar {.intdefine.}: int = 5
+  echo FooBar
+  ```
 
-::
-   nim c -d:FooBar=42 foobar.nim
+  ```cmd
+  nim c -d:FooBar=42 foobar.nim
+  ```
 
-In the above example, providing the -d flag causes the symbol
+In the above example, providing the `-d`:option: flag causes the symbol
 `FooBar` to be overwritten at compile-time, printing out 42. If the
-`-d:FooBar=42` were to be omitted, the default value of 5 would be
+`-d:FooBar=42`:option: were to be omitted, the default value of 5 would be
 used. To see if a value was provided, `defined(FooBar)` can be used.
 
-The syntax `-d:flag` is actually just a shortcut for `-d:flag=true`.
+The syntax `-d:flag`:option: is actually just a shortcut for
+`-d:flag=true`:option:.
+
+These pragmas also accept an optional string argument for qualified
+define names.
+
+  ```nim
+  const FooBar {.intdefine: "package.FooBar".}: int = 5
+  echo FooBar
+  ```
+
+  ```cmd
+  nim c -d:package.FooBar=42 foobar.nim
+  ```
+
+This helps disambiguate define names in different packages.
+
+See also the [generic `define` pragma](manual_experimental.html#generic-nimdefine-pragma)
+for a version of these pragmas that detects the type of the define based on
+the constant value.
 
 User-defined pragmas
 ====================
@@ -7370,19 +8383,21 @@ pragma pragma
 -------------
 
 The `pragma` pragma can be used to declare user-defined pragmas. This is
-useful because Nim's templates and macros do not affect pragmas. User-defined pragmas are in a different module-wide scope than all other symbols.
+useful because Nim's templates and macros do not affect pragmas.
+User-defined pragmas are in a different module-wide scope than all other symbols.
 They cannot be imported from a module.
 
 Example:
 
-.. code-block:: nim
+  ```nim
   when appType == "lib":
     {.pragma: rtl, exportc, dynlib, cdecl.}
   else:
     {.pragma: rtl, importc, dynlib: "client.dll", cdecl.}
 
   proc p*(a, b: int): int {.rtl.} =
-    result = a+b
+    result = a + b
+  ```
 
 In the example, a new pragma named `rtl` is introduced that either imports
 a symbol from a dynamic library or exports the symbol for dynamic library
@@ -7395,17 +8410,18 @@ It is possible to define custom typed pragmas. Custom pragmas do not affect
 code generation directly, but their presence can be detected by macros.
 Custom pragmas are defined using templates annotated with pragma `pragma`:
 
-.. code-block:: nim
+  ```nim
   template dbTable(name: string, table_space: string = "") {.pragma.}
   template dbKey(name: string = "", primary_key: bool = false) {.pragma.}
   template dbForeignKey(t: typedesc) {.pragma.}
   template dbIgnore {.pragma.}
+  ```
 
 
 Consider this stylized example of a possible Object Relation Mapping (ORM)
 implementation:
 
-.. code-block:: nim
+  ```nim
   const tblspace {.strdefine.} = "dev" # switch for dev, test and prod environments
 
   type
@@ -7420,7 +8436,8 @@ implementation:
       user_id {.dbForeignKey: User.}: int
       read_access: bool
       write_access: bool
-      admin_acess: bool
+      admin_access: bool
+  ```
 
 In this example, custom pragmas are used to describe how Nim objects are
 mapped to the schema of the relational database. Custom pragmas can have
@@ -7435,7 +8452,7 @@ definitions, statements, etc.
 
 The macros module includes helpers which can be used to simplify custom pragma
 access `hasCustomPragma`, `getCustomPragmaVal`. Please consult the
-`macros `_ module documentation for details. These macros are not
+[macros](macros.html) module documentation for details. These macros are not
 magic, everything they do can also be achieved by walking the AST of the object
 representation.
 
@@ -7443,65 +8460,68 @@ More examples with custom pragmas:
 
 - Better serialization/deserialization control:
 
-.. code-block:: nim
-  type MyObj = object
-    a {.dontSerialize.}: int
-    b {.defaultDeserialize: 5.}: int
-    c {.serializationKey: "_c".}: string
+    ```nim
+    type MyObj = object
+      a {.dontSerialize.}: int
+      b {.defaultDeserialize: 5.}: int
+      c {.serializationKey: "_c".}: string
+    ```
 
 - Adopting type for gui inspector in a game engine:
 
-.. code-block:: nim
-  type MyComponent = object
-    position {.editable, animatable.}: Vector3
-    alpha {.editRange: [0.0..1.0], animatable.}: float32
+    ```nim
+    type MyComponent = object
+      position {.editable, animatable.}: Vector3
+      alpha {.editRange: [0.0..1.0], animatable.}: float32
+    ```
 
 
 Macro pragmas
 -------------
 
-All macros and templates can also be used as pragmas. They can be attached
-to routines (procs, iterators, etc), type names, or type expressions. The
-compiler will perform the following simple syntactic transformations:
+Macros and templates can sometimes be called with the pragma syntax. Cases
+where this is possible include when attached to routine (procs, iterators, etc.)
+declarations or routine type expressions. The compiler will perform the
+following simple syntactic transformations:
 
-.. code-block:: nim
+  ```nim
   template command(name: string, def: untyped) = discard
 
   proc p() {.command("print").} = discard
+  ```
 
 This is translated to:
 
-.. code-block:: nim
+  ```nim
   command("print"):
     proc p() = discard
+  ```
 
 ------
 
-.. code-block:: nim
+  ```nim
   type
     AsyncEventHandler = proc (x: Event) {.async.}
+  ```
 
 This is translated to:
 
-.. code-block:: nim
+  ```nim
   type
     AsyncEventHandler = async(proc (x: Event))
+  ```
 
 ------
 
-.. code-block:: nim
-  type
-    MyObject {.schema: "schema.protobuf".} = object
-
-This is translated to a call to the `schema` macro with a `nnkTypeDef`
-AST node capturing both the left-hand side and right-hand side of the
-definition. The macro can return a potentially modified `nnkTypeDef` tree
-which will replace the original row in the type section.
-
-When multiple macro pragmas are applied to the same definition, the
-compiler will apply them consequently from left to right. Each macro
-will receive as input the output of the previous one.
+When multiple macro pragmas are applied to the same definition, the first one
+from left to right will be evaluated. This macro can then choose to keep
+the remaining macro pragmas in its output, and those will be evaluated in
+the same way.
 
+There are a few more applications of macro pragmas, such as in type,
+variable and constant declarations, but this behavior is considered to be
+experimental and is documented in the [experimental manual](
+manual_experimental.html#extended-macro-pragmas) instead.
 
 
 Foreign function interface
@@ -7519,60 +8539,67 @@ from C. The optional argument is a string containing the C identifier. If
 the argument is missing, the C name is the Nim identifier *exactly as
 spelled*:
 
-.. code-block::
+  ```nim
   proc printf(formatstr: cstring) {.header: "", importc: "printf", varargs.}
+  ```
 
 When `importc` is applied to a `let` statement it can omit its value which
-will then be expected to come from C. This can be used to import a C `const`:
+will then be expected to come from C. This can be used to import a C `const`:c:\:
 
-.. code-block::
+  ```nim
   {.emit: "const int cconst = 42;".}
 
   let cconst {.importc, nodecl.}: cint
 
   assert cconst == 42
+  ```
 
 Note that this pragma has been abused in the past to also work in the
 JS backend for JS objects and functions. Other backends do provide
 the same feature under the same name. Also, when the target language
 is not set to C, other pragmas are available:
 
- * `importcpp `_
- * `importobjc `_
- * `importjs `_
+ * [importcpp][importcpp pragma]
+ * [importobjc][importobjc pragma]
+ * [importjs][importjs pragma]
 
-.. code-block:: Nim
+The string literal passed to `importc` can be a format string:
+
+  ```Nim
   proc p(s: cstring) {.importc: "prefix$1".}
+  ```
 
-In the example, the external name of `p` is set to `prefixp`. Only `$1`
-is available and a literal dollar sign must be written as `$$`.
+In the example, the external name of `p` is set to `prefixp`. Only ``$1``
+is available and a literal dollar sign must be written as ``$$``.
 
 
 Exportc pragma
 --------------
 The `exportc` pragma provides a means to export a type, a variable, or a
 procedure to C. Enums and constants can't be exported. The optional argument
-is a string containing the C identifier.  If the argument is missing, the C
+is a string containing the C identifier. If the argument is missing, the C
 name is the Nim identifier *exactly as spelled*:
 
-.. code-block:: Nim
+  ```Nim
   proc callme(formatstr: cstring) {.exportc: "callMe", varargs.}
+  ```
 
 Note that this pragma is somewhat of a misnomer: Other backends do provide
 the same feature under the same name.
 
 The string literal passed to `exportc` can be a format string:
 
-.. code-block:: Nim
+  ```Nim
   proc p(s: string) {.exportc: "prefix$1".} =
     echo s
+  ```
 
-In the example, the external name of `p` is set to `prefixp`. Only `$1`
-is available and a literal dollar sign must be written as `$$`.
+In the example, the external name of `p` is set to `prefixp`. Only ``$1``
+is available and a literal dollar sign must be written as ``$$``.
 
 If the symbol should also be exported to a dynamic library, the `dynlib`
 pragma should be used in addition to the `exportc` pragma. See
-`Dynlib pragma for export <#foreign-function-interface-dynlib-pragma-for-export>`_.
+[Dynlib pragma for export].
 
 
 Extern pragma
@@ -7580,32 +8607,93 @@ Extern pragma
 Like `exportc` or `importc`, the `extern` pragma affects name
 mangling. The string literal passed to `extern` can be a format string:
 
-.. code-block:: Nim
+  ```Nim
   proc p(s: string) {.extern: "prefix$1".} =
     echo s
+  ```
 
-In the example, the external name of `p` is set to `prefixp`. Only `$1`
-is available and a literal dollar sign must be written as `$$`.
+In the example, the external name of `p` is set to `prefixp`. Only ``$1``
+is available and a literal dollar sign must be written as ``$$``.
 
 
 Bycopy pragma
 -------------
 
-The `bycopy` pragma can be applied to an object or tuple type and
-instructs the compiler to pass the type by value to procs:
+The `bycopy` pragma can be applied to an object or tuple type or a proc param. It instructs the compiler to pass the type by value to procs:
 
-.. code-block:: nim
+  ```nim
   type
     Vector {.bycopy.} = object
       x, y, z: float
+  ```
 
+The Nim compiler automatically determines whether a parameter is passed by value or
+by reference based on the parameter type's size. If a parameter must be passed by value
+or by reference, (such as when interfacing with a C library) use the bycopy or byref pragmas.
+Notice params marked as `byref` takes precedence over types marked as `bycopy`.
 
 Byref pragma
 ------------
 
-The `byref` pragma can be applied to an object or tuple type and instructs
-the compiler to pass the type by reference (hidden pointer) to procs.
+The `byref` pragma can be applied to an object or tuple type or a proc param.
+When applied to a type it instructs the compiler to pass the type by reference
+(hidden pointer) to procs. When applied to a param it will take precedence, even
+if the the type was marked as `bycopy`. When an `importc` type has a `byref` pragma or
+parameters are marked as `byref` in an `importc` proc, these params translate to pointers.
+When an `importcpp` type has a `byref` pragma, these params translate to
+C++ references `&`.
 
+  ```Nim
+  {.emit: """/*TYPESECTION*/
+  typedef struct {
+    int x;
+  } CStruct;
+  """.}
+
+  {.emit: """
+  #ifdef __cplusplus
+  extern "C"
+  #endif
+  int takesCStruct(CStruct* x) {
+    return x->x;
+  }
+  """.}
+
+  type
+    CStruct {.importc, byref.} = object
+      x: cint
+
+  proc takesCStruct(x: CStruct): cint {.importc.}
+  ```
+
+  or
+
+
+  ```Nim
+  type
+    CStruct {.importc.} = object
+      x: cint
+
+  proc takesCStruct(x {.byref.}: CStruct): cint {.importc.}
+  ```
+
+  ```Nim
+  {.emit: """/*TYPESECTION*/
+  struct CppStruct {
+    int x;
+
+    int takesCppStruct(CppStruct& y) {
+      return x + y.x;
+    }
+  };
+  """.}
+
+  type
+    CppStruct {.importcpp, byref.} = object
+      x: cint
+
+  proc takesCppStruct(x, y: CppStruct): cint {.importcpp.}
+  ```
 
 Varargs pragma
 --------------
@@ -7614,17 +8702,18 @@ types). It tells Nim that the proc can take a variable number of parameters
 after the last specified parameter. Nim string values will be converted to C
 strings automatically:
 
-.. code-block:: Nim
+  ```Nim
   proc printf(formatstr: cstring) {.nodecl, varargs.}
 
   printf("hallo %s", "world") # "world" will be passed as C string
+  ```
 
 
 Union pragma
 ------------
 The `union` pragma can be applied to any `object` type. It means all
-of the object's fields are overlaid in memory. This produces a `union`
-instead of a `struct` in the generated C/C++ code. The object declaration
+of an object's fields are overlaid in memory. This produces a `union`:c:
+instead of a `struct`:c: in the generated C/C++ code. The object declaration
 then must not use inheritance or any GC'ed memory but this is currently not
 checked.
 
@@ -7647,12 +8736,13 @@ a static error. Usage with inheritance should be defined and documented.
 Dynlib pragma for import
 ------------------------
 With the `dynlib` pragma, a procedure or a variable can be imported from
-a dynamic library (`.dll` files for Windows, `lib*.so` files for UNIX).
+a dynamic library (``.dll`` files for Windows, ``lib*.so`` files for UNIX).
 The non-optional argument has to be the name of the dynamic library:
 
-.. code-block:: Nim
+  ```Nim
   proc gtk_image_new(): PGtkWidget
     {.cdecl, dynlib: "libgtk-x11-2.0.so", importc.}
+  ```
 
 In general, importing a dynamic library does not require any special linker
 options or linking with import libraries. This also implies that no *devel*
@@ -7660,25 +8750,26 @@ packages need to be installed.
 
 The `dynlib` import mechanism supports a versioning scheme:
 
-.. code-block:: nim
+  ```nim
   proc Tcl_Eval(interp: pTcl_Interp, script: cstring): int {.cdecl,
     importc, dynlib: "libtcl(|8.5|8.4|8.3).so.(1|0)".}
+  ```
 
-At runtime, the dynamic library is searched for (in this order)::
+At runtime, the dynamic library is searched for (in this order):
 
-  libtcl.so.1
-  libtcl.so.0
-  libtcl8.5.so.1
-  libtcl8.5.so.0
-  libtcl8.4.so.1
-  libtcl8.4.so.0
-  libtcl8.3.so.1
-  libtcl8.3.so.0
+    libtcl.so.1
+    libtcl.so.0
+    libtcl8.5.so.1
+    libtcl8.5.so.0
+    libtcl8.4.so.1
+    libtcl8.4.so.0
+    libtcl8.3.so.1
+    libtcl8.3.so.0
 
 The `dynlib` pragma supports not only constant strings as an argument but also
 string expressions in general:
 
-.. code-block:: nim
+  ```nim
   import std/os
 
   proc getDllName: string =
@@ -7689,16 +8780,17 @@ string expressions in general:
     quit("could not load dynamic library")
 
   proc myImport(s: cstring) {.cdecl, importc, dynlib: getDllName().}
+  ```
 
-**Note**: Patterns like `libtcl(|8.5|8.4).so` are only supported in constant
+**Note**: Patterns like ``libtcl(|8.5|8.4).so`` are only supported in constant
 strings, because they are precompiled.
 
 **Note**: Passing variables to the `dynlib` pragma will fail at runtime
 because of order of initialization problems.
 
 **Note**: A `dynlib` import can be overridden with
-the `--dynlibOverride:name` command-line option. The
-`Compiler User Guide `_ contains further information.
+the `--dynlibOverride:name`:option: command-line option. The
+[Compiler User Guide](nimc.html) contains further information.
 
 
 Dynlib pragma for export
@@ -7708,29 +8800,21 @@ With the `dynlib` pragma, a procedure can also be exported to
 a dynamic library. The pragma then has no argument and has to be used in
 conjunction with the `exportc` pragma:
 
-.. code-block:: Nim
+  ```Nim
   proc exportme(): int {.cdecl, exportc, dynlib.}
+  ```
 
 This is only useful if the program is compiled as a dynamic library via the
-`--app:lib` command-line option.
-
+`--app:lib`:option: command-line option.
 
 
 Threads
 =======
 
-To enable thread support the `--threads:on` command-line switch needs to
-be used. The `system` module then contains several threading primitives.
-See the `threads `_ and `channels `_ modules
-for the low-level thread API. There are also high-level parallelism constructs
-available. See `spawn `_ for
+The `--threads:on`:option: command-line switch is enabled by default. The [typedthreads module](typedthreads.html) module then contains several threading primitives. See [spawn](manual_experimental.html#parallel-amp-spawn) for
 further details.
 
-Nim's memory model for threads is quite different than that of other common
-programming languages (C, Pascal, Java): Each thread has its own (garbage
-collected) heap, and sharing of memory is restricted to global variables. This
-helps to prevent race conditions. GC efficiency is improved quite a lot,
-because the GC never has to stop other threads and see what they reference.
+The only ways to create a thread is via `spawn` or `createThread`.
 
 
 Thread pragma
@@ -7742,47 +8826,9 @@ violations of the `no heap sharing restriction`:idx:\: This restriction implies
 that it is invalid to construct a data structure that consists of memory
 allocated from different (thread-local) heaps.
 
-A thread proc is passed to `createThread` or `spawn` and invoked
-indirectly; so the `thread` pragma implies `procvar`.
+A thread proc can be passed to `createThread` or `spawn`.
 
 
-GC safety
----------
-
-We call a proc `p` `GC safe`:idx: when it doesn't access any global variable
-that contains GC'ed memory (`string`, `seq`, `ref` or a closure) either
-directly or indirectly through a call to a GC unsafe proc.
-
-The `gcsafe`:idx: annotation can be used to mark a proc to be gcsafe,
-otherwise this property is inferred by the compiler. Note that `noSideEffect`
-implies `gcsafe`. The only way to create a thread is via `spawn` or
-`createThread`. The invoked proc must not use `var` parameters nor must
-any of its parameters contain a `ref` or `closure` type. This enforces
-the *no heap sharing restriction*.
-
-Routines that are imported from C are always assumed to be `gcsafe`.
-To disable the GC-safety checking the `--threadAnalysis:off` command-line
-switch can be used. This is a temporary workaround to ease the porting effort
-from old code to the new threading model.
-
-To override the compiler's gcsafety analysis a `{.cast(gcsafe).}` pragma block can
-be used:
-
-.. code-block:: nim
-
-  var
-    someGlobal: string = "some string here"
-    perThread {.threadvar.}: string
-
-  proc setPerThread() =
-    {.cast(gcsafe).}:
-      deepCopy(perThread, someGlobal)
-
-
-See also:
-
-- `Shared heap memory management `_.
-
 
 Threadvar pragma
 ----------------
@@ -7791,8 +8837,9 @@ A variable can be marked with the `threadvar` pragma, which makes it a
 `thread-local`:idx: variable; Additionally, this implies all the effects
 of the `global` pragma.
 
-.. code-block:: nim
+  ```nim
   var checkpoints* {.threadvar.}: seq[string]
+  ```
 
 Due to implementation restrictions, thread-local variables cannot be
 initialized within the `var` section. (Every thread-local variable needs to
@@ -7804,4 +8851,139 @@ Threads and exceptions
 
 The interaction between threads and exceptions is simple: A *handled* exception
 in one thread cannot affect any other thread. However, an *unhandled* exception
-in one thread terminates the whole *process*!
+in one thread terminates the whole *process*.
+
+
+Guards and locks
+================
+
+Nim provides common low level concurrency mechanisms like locks, atomic
+intrinsics or condition variables.
+
+Nim significantly improves on the safety of these features via additional
+pragmas:
+
+1) A `guard`:idx: annotation is introduced to prevent data races.
+2) Every access of a guarded memory location needs to happen in an
+   appropriate `locks`:idx: statement.
+
+
+Guards and locks sections
+-------------------------
+
+### Protecting global variables
+
+Object fields and global variables can be annotated via a `guard` pragma:
+
+  ```nim
+  import std/locks
+
+  var glock: Lock
+  var gdata {.guard: glock.}: int
+  ```
+
+The compiler then ensures that every access of `gdata` is within a `locks`
+section:
+
+  ```nim
+  proc invalid =
+    # invalid: unguarded access:
+    echo gdata
+
+  proc valid =
+    # valid access:
+    {.locks: [glock].}:
+      echo gdata
+  ```
+
+Top level accesses to `gdata` are always allowed so that it can be initialized
+conveniently. It is *assumed* (but not enforced) that every top level statement
+is executed before any concurrent action happens.
+
+The `locks` section deliberately looks ugly because it has no runtime
+semantics and should not be used directly! It should only be used in templates
+that also implement some form of locking at runtime:
+
+  ```nim
+  template lock(a: Lock; body: untyped) =
+    pthread_mutex_lock(a)
+    {.locks: [a].}:
+      try:
+        body
+      finally:
+        pthread_mutex_unlock(a)
+  ```
+
+
+The guard does not need to be of any particular type. It is flexible enough to
+model low level lockfree mechanisms:
+
+  ```nim
+  var dummyLock {.compileTime.}: int
+  var atomicCounter {.guard: dummyLock.}: int
+
+  template atomicRead(x): untyped =
+    {.locks: [dummyLock].}:
+      memoryReadBarrier()
+      x
+
+  echo atomicRead(atomicCounter)
+  ```
+
+
+The `locks` pragma takes a list of lock expressions `locks: [a, b, ...]`
+in order to support *multi lock* statements.
+
+
+### Protecting general locations
+
+The `guard` annotation can also be used to protect fields within an object.
+The guard then needs to be another field within the same object or a
+global variable.
+
+Since objects can reside on the heap or on the stack, this greatly enhances
+the expressiveness of the language:
+
+  ```nim
+  import std/locks
+
+  type
+    ProtectedCounter = object
+      v {.guard: L.}: int
+      L: Lock
+
+  proc incCounters(counters: var openArray[ProtectedCounter]) =
+    for i in 0..counters.high:
+      lock counters[i].L:
+        inc counters[i].v
+  ```
+
+The access to field `x.v` is allowed since its guard `x.L`  is active.
+After template expansion, this amounts to:
+
+  ```nim
+  proc incCounters(counters: var openArray[ProtectedCounter]) =
+    for i in 0..counters.high:
+      pthread_mutex_lock(counters[i].L)
+      {.locks: [counters[i].L].}:
+        try:
+          inc counters[i].v
+        finally:
+          pthread_mutex_unlock(counters[i].L)
+  ```
+
+There is an analysis that checks that `counters[i].L` is the lock that
+corresponds to the protected location `counters[i].v`. This analysis is called
+`path analysis`:idx: because it deals with paths to locations
+like `obj.field[i].fieldB[j]`.
+
+The path analysis is **currently unsound**, but that doesn't make it useless.
+Two paths are considered equivalent if they are syntactically the same.
+
+This means the following compiles (for now) even though it really should not:
+
+  ```nim
+  {.locks: [a[i].L].}:
+    inc i
+    access a[i].v
+  ```
diff --git a/doc/manual/var_t_return.rst b/doc/manual/var_t_return.md
similarity index 95%
rename from doc/manual/var_t_return.rst
rename to doc/manual/var_t_return.md
index e34993e3ef..15d908c74b 100644
--- a/doc/manual/var_t_return.rst
+++ b/doc/manual/var_t_return.md
@@ -1,11 +1,12 @@
 .. default-role:: code
+.. include:: ../rstcommon.rst
 
 Memory safety for returning by `var T` is ensured by a simple borrowing
 rule: If `result` does not refer to a location pointing to the heap
 (that is in `result = X` the `X` involves a `ptr` or `ref` access)
 then it has to be derived from the routine's first parameter:
 
-.. code-block:: nim
+  ```nim
   proc forward[T](x: var T): var T =
     result = x # ok, derived from the first parameter.
 
@@ -16,6 +17,7 @@ then it has to be derived from the routine's first parameter:
     result = forward(x) # Error: location is derived from `x`
                         # which is not p's first parameter and lives
                         # on the stack.
+  ```
 
 In other words, the lifetime of what `result` points to is attached to the
 lifetime of the first parameter and that is enough knowledge to verify
diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.md
similarity index 55%
rename from doc/manual_experimental.rst
rename to doc/manual_experimental.md
index cf2e0c2470..9b52fbd2a4 100644
--- a/doc/manual_experimental.rst
+++ b/doc/manual_experimental.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 =========================
 Nim Experimental Features
 =========================
@@ -7,6 +5,8 @@ Nim Experimental Features
 :Authors: Andreas Rumpf
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
@@ -15,69 +15,30 @@ About this document
 
 This document describes features of Nim that are to be considered experimental.
 Some of these are not covered by the `.experimental` pragma or
-`--experimental` switch because they are already behind a special syntax and
+`--experimental`:option: switch because they are already behind a special syntax and
 one may want to use Nim libraries using these features without using them
 oneself.
 
-**Note**: Unless otherwise indicated, these features are not to be removed,
-but refined and overhauled.
-
-
-Package level objects
-=====================
-
-Every Nim module resides in a (nimble) package. An object type can be attached
-to the package it resides in. If that is done, the type can be referenced from
-other modules as an `incomplete`:idx: object type. This feature allows to
-break up recursive type dependencies across module boundaries. Incomplete
-object types are always passed `byref` and can only be used in pointer like
-contexts (`var/ref/ptr IncompleteObject`) in general since the compiler does
-not yet know the size of the object. To complete an incomplete object
-the `package` pragma has to be used. `package` implies `byref`.
-
-As long as a type `T` is incomplete, neither `sizeof(T)` nor runtime
-type information for `T` is available.
-
-
-Example:
-
-.. code-block:: nim
-
-  # module A (in an arbitrary package)
-  type
-    Pack.SomeObject = object ## declare as incomplete object of package 'Pack'
-    Triple = object
-      a, b, c: ref SomeObject ## pointers to incomplete objects are allowed
-
-  ## Incomplete objects can be used as parameters:
-  proc myproc(x: SomeObject) = discard
-
-
-.. code-block:: nim
-
-  # module B (in package "Pack")
-  type
-    SomeObject* {.package.} = object ## Use 'package' to complete the object
-      s, t: string
-      x, y: int
+.. note:: Unless otherwise indicated, these features are not to be removed,
+  but refined and overhauled.
 
 
 Void type
 =========
 
-The `void` type denotes the absence of any type. Parameters of
-type `void` are treated as non-existent, `void` as a return type means that
-the procedure does not return a value:
+The `void` type denotes the absence of any value, i.e. it is the type that contains no values. Consequently, no value can be provided for parameters of
+type `void`, and no value can be returned from a function with return type `void`:
 
-.. code-block:: nim
+  ```nim
   proc nothing(x, y: void): void =
     echo "ha"
 
   nothing() # writes "ha" to stdout
+  ```
 
 The `void` type is particularly useful for generic code:
 
-.. code-block:: nim
+  ```nim
   proc callProc[T](p: proc (x: T), x: T) =
     when T is void:
       p()
@@ -89,135 +50,235 @@ The `void` type is particularly useful for generic code:
 
   callProc[int](intProc, 12)
   callProc[void](emptyProc)
+  ```
 
 However, a `void` type cannot be inferred in generic code:
 
-.. code-block:: nim
+  ```nim
   callProc(emptyProc)
   # Error: type mismatch: got (proc ())
   # but expected one of:
   # callProc(p: proc (T), x: T)
+  ```
 
 The `void` type is only valid for parameters and return types; other symbols
 cannot have the type `void`.
 
-
-
-Covariance
-==========
-
-Covariance in Nim can be introduced only through pointer-like types such
-as `ptr` and `ref`. Sequence, Array and OpenArray types, instantiated
-with pointer-like types will be considered covariant if and only if they
-are also immutable. The introduction of a `var` modifier or additional
-`ptr` or `ref` indirections would result in invariant treatment of
-these types.
-
-`proc` types are currently always invariant, but future versions of Nim
-may relax this rule.
-
-User-defined generic types may also be covariant with respect to some of
-their parameters. By default, all generic params are considered invariant,
-but you may choose the apply the prefix modifier `in` to a parameter to
-make it contravariant or `out` to make it covariant:
-
-.. code-block:: nim
-  type
-    AnnotatedPtr[out T] =
-      metadata: MyTypeInfo
-      p: ref T
-
-    RingBuffer[out T] =
-      startPos: int
-      data: seq[T]
-
-    Action {.importcpp: "std::function".} [in T] = object
-
-When the designated generic parameter is used to instantiate a pointer-like
-type as in the case of `AnnotatedPtr` above, the resulting generic type will
-also have pointer-like covariance:
-
-.. code-block:: nim
-  type
-    GuiWidget = object of RootObj
-    Button = object of GuiWidget
-    ComboBox = object of GuiWidget
-
-  var
-    widgetPtr: AnnotatedPtr[GuiWidget]
-    buttonPtr: AnnotatedPtr[Button]
-
-  ...
-
-  proc drawWidget[T](x: AnnotatedPtr[GuiWidget]) = ...
-
-  # you can call procs expecting base types by supplying a derived type
-  drawWidget(buttonPtr)
-
-  # and you can convert more-specific pointer types to more general ones
-  widgetPtr = buttonPtr
-
-Just like with regular pointers, covariance will be enabled only for immutable
-values:
-
-.. code-block:: nim
-  proc makeComboBox[T](x: var AnnotatedPtr[GuiWidget]) =
-    x.p = new(ComboBox)
-
-  makeComboBox(buttonPtr) # Error, AnnotatedPtr[Button] cannot be modified
-                          # to point to a ComboBox
-
-On the other hand, in the `RingBuffer` example above, the designated generic
-param is used to instantiate the non-pointer `seq` type, which means that
-the resulting generic type will have covariance that mimics an array or
-sequence (i.e. it will be covariant only when instantiated with `ptr` and
-`ref` types):
-
-.. code-block:: nim
-
-  type
-    Base = object of RootObj
-    Derived = object of Base
-
-  proc consumeBaseValues(b: RingBuffer[Base]) = ...
-
-  var derivedValues: RingBuffer[Derived]
-
-  consumeBaseValues(derivedValues) # Error, Base and Derived values may differ
-                                   # in size
-
-  proc consumeBasePointers(b: RingBuffer[ptr Base]) = ...
-
-  var derivedPointers: RingBuffer[ptr Derived]
-
-  consumeBaseValues(derivedPointers) # This is legal
-
-Please note that Nim will treat the user-defined pointer-like types as
-proper alternatives to the built-in pointer types. That is, types such
-as `seq[AnnotatedPtr[T]]` or `RingBuffer[AnnotatedPtr[T]]` will also be
-considered covariant and you can create new pointer-like types by instantiating
-other user-defined pointer-like types.
-
-The contravariant parameters introduced with the `in` modifier are currently
-useful only when interfacing with imported types having such semantics.
-
-
-Automatic dereferencing
+Generic `define` pragma
 =======================
 
-Automatic dereferencing is performed for the first argument of a routine call.
-This feature has to be enabled via `{.experimental: "implicitDeref".}`:
+Aside the [typed define pragmas for constants](manual.html#implementation-specific-pragmas-compileminustime-define-pragmas),
+there is a generic `{.define.}` pragma that interprets the value of the define
+based on the type of the constant value.
 
-.. code-block:: nim
-  {.experimental: "implicitDeref".}
+  ```nim
+  const foo {.define: "package.foo".} = 123
+  const bar {.define: "package.bar".} = false
+  ```
 
-  proc depth(x: NodeObj): int = ...
+  ```cmd
+  nim c -d:package.foo=456 -d:package.bar foobar.nim
+  ```
+
+The following types are supported:
+
+* `string` and `cstring`
+* Signed and unsigned integer types
+* `bool`
+* Enums
+
+Top-down type inference
+=======================
+
+In expressions such as:
+
+```nim
+let a: T = ex
+```
+
+Normally, the compiler type checks the expression `ex` by itself, then
+attempts to statically convert the type-checked expression to the given type
+`T` as much as it can, while making sure it matches the type. The extent of
+this process is limited however due to the expression usually having
+an assumed type that might clash with the given type.
+
+With top-down type inference, the expression is type checked with the
+extra knowledge that it is supposed to be of type `T`. For example,
+the following code is does not compile with the former method, but
+compiles with top-down type inference:
+
+```nim
+let foo: (float, uint8, cstring) = (1, 2, "abc")
+```
+
+The tuple expression has an expected type of `(float, uint8, cstring)`.
+Since it is a tuple literal, we can use this information to assume the types
+of its elements. The expected types for the expressions `1`, `2` and `"abc"`
+are respectively `float`, `uint8`, and `cstring`; and these expressions can be
+statically converted to these types.
+
+Without this information, the type of the tuple expression would have been
+assumed to be `(int, int, string)`. Thus the type of the tuple expression
+would not match the type of the variable, and an error would be given.
+
+The extent of this varies, but there are some notable special cases.
+
+
+Inferred generic parameters
+---------------------------
+
+In expressions making use of generic procs or templates, the expected
+(unbound) types are often able to be inferred based on context.
+This feature has to be enabled via `{.experimental: "inferGenericTypes".}`
+
+  ```nim  test = "nim c $1"
+  {.experimental: "inferGenericTypes".}
+
+  import std/options
+
+  var x = newSeq[int](1)
+  # Do some work on 'x'...
+
+  # Works!
+  # 'x' is 'seq[int]' so 'newSeq[int]' is implied
+  x = newSeq(10)
+
+  # Works!
+  # 'T' of 'none' is bound to the 'T' of 'noneProducer', passing it along.
+  # Effectively 'none.T = noneProducer.T'
+  proc noneProducer[T](): Option[T] = none()
+  let myNone = noneProducer[int]()
+
+  # Also works
+  # 'myOtherNone' binds its 'T' to 'float' and 'noneProducer' inherits it
+  # noneProducer.T = myOtherNone.T
+  let myOtherNone: Option[float] = noneProducer()
+
+  # Works as well
+  # none.T = myOtherOtherNone.T
+  let myOtherOtherNone: Option[int] = none()
+  ```
+
+This is achieved by reducing the types on the lhs and rhs until the *lhs* is left with only types such as `T`.
+While lhs and rhs are reduced together, this does *not* mean that the *rhs* will also only be left
+with a flat type `Z`, it may be of the form `MyType[Z]`.
+
+After the types have been reduced, the types `T` are bound to the types that are left on the rhs.
+
+If bindings *cannot be inferred*, compilation will fail and manual specification is required.
+
+An example for *failing inference* can be found when passing a generic expression
+to a function/template call:
+
+  ```nim  test = "nim c $1"  status = 1
+  {.experimental: "inferGenericTypes".}
+
+  proc myProc[T](a, b: T) = discard
+
+  # Fails! Unable to infer that 'T' is supposed to be 'int'
+  myProc(newSeq[int](), newSeq(1))
+
+  # Works! Manual specification of 'T' as 'int' necessary
+  myProc(newSeq[int](), newSeq[int](1))
+  ```
+
+Combination of generic inference with the `auto` type is also unsupported:
+
+  ```nim  test = "nim c $1"  status = 1
+  {.experimental: "inferGenericTypes".}
+
+  proc produceValue[T]: auto = default(T)
+  let a: int = produceValue() # 'auto' cannot be inferred here
+  ```
+
+**Note**: The described inference does not permit the creation of overrides based on
+the return type of a procedure. It is a mapping mechanism that does not attempt to 
+perform deeper inference, nor does it modify what is a valid override.
+
+  ```nim  test = "nim c $1"  status = 1
+  # Doesn't affect the following code, it is invalid either way
+  {.experimental: "inferGenericTypes".}
+
+  proc a: int = 0
+  proc a: float = 1.0 # Fails! Invalid code and not recommended
+  ```
+
+
+Sequence literals
+-----------------
+
+Top-down type inference applies to sequence literals.
+
+```nim
+let x: seq[seq[float]] = @[@[1, 2, 3], @[4, 5, 6]]
+```
+
+This behavior is tied to the `@` overloads in the `system` module,
+so overloading `@` can disable this behavior. This can be circumvented by
+specifying the `` system.`@` `` overload.
+
+```nim
+proc `@`(x: string): string = "@" & x
+
+# does not compile:
+let x: seq[float] = @[1, 2, 3]
+# compiles:
+let x: seq[float] = system.`@`([1, 2, 3])
+```
+
+
+Package level objects
+=====================
+
+Every Nim module resides in a (nimble) package. An object type can be attached
+to the package it resides in. If that is done, the type can be referenced from
+other modules as an `incomplete`:idx: object type. This feature allows to
+break up recursive type dependencies across module boundaries. Incomplete
+object types are always passed `byref` and can only be used in pointer like
+contexts (`var/ref/ptr IncompleteObject`) in general, since the compiler does
+not yet know the size of the object. To complete an incomplete object,
+the `package` pragma has to be used. `package` implies `byref`.
+
+As long as a type `T` is incomplete, no runtime type information for `T` is
+available.
+
+
+Example:
+
+  ```nim
+  # module A (in an arbitrary package)
+  type
+    Pack.SomeObject = object # declare as incomplete object of package 'Pack'
+    Triple = object
+      a, b, c: ref SomeObject # pointers to incomplete objects are allowed
+
+  # Incomplete objects can be used as parameters:
+  proc myproc(x: SomeObject) = discard
+  ```
+
+
+  ```nim
+  # module B (in package "Pack")
+  type
+    SomeObject* {.package.} = object # Use 'package' to complete the object
+      s, t: string
+      x, y: int
+  ```
+
+This feature will likely be superseded in the future by support for
+recursive module dependencies.
+
+
+Importing private symbols
+=========================
+
+In some situations, it may be useful to import all symbols (public or private)
+from a module. The syntax `import foo {.all.}` can be used to import all
+symbols from the module `foo`. Note that importing private symbols is
+generally not recommended.
+
+See also the experimental [importutils](importutils.html) module.
 
-  var
-    n: Node
-  new(n)
-  echo n.depth
-  # no need to write n[].depth either
 
 Code reordering
 ===============
@@ -263,8 +324,7 @@ preface definitions inside a module.
 
 Example:
 
-.. code-block:: nim
-
+  ```nim
   {.experimental: "codeReordering".}
 
   proc foo(x: int) =
@@ -274,13 +334,14 @@ Example:
     echo(x)
 
   foo(10)
+  ```
 
 Variables can also be reordered as well. Variables that are *initialized* (i.e.
 variables that have their declaration and assignment combined in a single
 statement) can have their entire initialization statement reordered. Be wary of
 what code is executed at the top level:
 
-.. code-block:: nim
+  ```nim
   {.experimental: "codeReordering".}
 
   proc a() =
@@ -289,6 +350,7 @@ what code is executed at the top level:
   var foo = 5
 
   a() # outputs: "5"
+  ```
 
 ..
    TODO: Let's table this for now. This is an *experimental feature* and so the
@@ -299,7 +361,7 @@ what code is executed at the top level:
    code reordering process, and not after. As an example, the output of this
    code is the same as it would be with code reordering disabled.
 
-   .. code-block:: nim
+     ```nim
      {.experimental: "codeReordering".}
 
      proc x() =
@@ -308,11 +370,12 @@ what code is executed at the top level:
      var foo = 4
 
      x() # "false"
+     ```
 
 It is important to note that reordering *only* works for symbols at top level
 scope. Therefore, the following will *fail to compile:*
 
-.. code-block:: nim
+  ```nim
   {.experimental: "codeReordering".}
 
   proc a() =
@@ -321,64 +384,10 @@ scope. Therefore, the following will *fail to compile:*
       echo("Hello!")
 
   a()
+  ```
 
-
-Named argument overloading
-==========================
-
-Routines with the same type signature can be called differently if a parameter
-has different names. This does not need an `experimental` switch, but is an
-unstable feature.
-
-.. code-block::nim
-  proc foo(x: int) =
-    echo "Using x: ", x
-  proc foo(y: int) =
-    echo "Using y: ", y
-
-  foo(x = 2)
-  # Using x: 2
-  foo(y = 2)
-  # Using y: 2
-
-
-Do notation
-===========
-
-As a special more convenient notation, proc expressions involved in procedure
-calls can use the `do` keyword:
-
-.. code-block:: nim
-  sort(cities) do (x,y: string) -> int:
-    cmp(x.len, y.len)
-
-  # Less parenthesis using the method plus command syntax:
-  cities = cities.map do (x:string) -> string:
-    "City of " & x
-
-  # In macros, the do notation is often used for quasi-quoting
-  macroResults.add quote do:
-    if not `ex`:
-      echo `info`, ": Check failed: ", `expString`
-
-`do` is written after the parentheses enclosing the regular proc params.
-The proc expression represented by the do block is appended to them.
-In calls using the command syntax, the do block will bind to the immediately
-preceding expression, transforming it in a call.
-
-`do` with parentheses is an anonymous `proc`; however a `do` without
-parentheses is just a block of code. The `do` notation can be used to
-pass multiple blocks to a macro:
-
-.. code-block:: nim
-  macro performWithUndo(task, undo: untyped) = ...
-
-  performWithUndo do:
-    # multiple-line block of code
-    # to perform the task
-  do:
-    # code to undo it
-
+This feature will likely be replaced with a better solution to remove
+the need for forward declarations.
 
 Special Operators
 =================
@@ -386,8 +395,8 @@ Special Operators
 dot operators
 -------------
 
-**Note**: Dot operators are still experimental and so need to be enabled
-via `{.experimental: "dotOperators".}`.
+.. note:: Dot operators are still experimental and so need to be enabled
+  via `{.experimental: "dotOperators".}`.
 
 Nim offers a special family of dot operators that can be used to
 intercept and rewrite proc call and field access attempts, referring
@@ -402,19 +411,21 @@ for a dot operator that can be matched against a re-written form of
 the expression, where the unknown field or proc name is passed to
 an `untyped` parameter:
 
-.. code-block:: nim
+  ```nim
   a.b # becomes `.`(a, b)
   a.b(c, d) # becomes `.`(a, b, c, d)
+  ```
 
 The matched dot operators can be symbols of any callable kind (procs,
 templates and macros), depending on the desired effect:
 
-.. code-block:: nim
+  ```nim
   template `.`(js: PJsonNode, field: untyped): JSON = js[astToStr(field)]
 
   var js = parseJson("{ x: 1, y: 2}")
   echo js.x # outputs 1
   echo js.y # outputs 2
+  ```
 
 The following dot operators are available:
 
@@ -433,8 +444,9 @@ operator `.=`
 -------------
 This operator will be matched against assignments to missing fields.
 
-.. code-block:: nim
+  ```nim
   a.b = c # becomes `.=`(a, b, c)
+  ```
 
 Call operator
 -------------
@@ -443,7 +455,7 @@ precedence over dot operators, however it does not match missing overloads
 for existing routines. The experimental `callOperator` switch must be enabled
 to use this operator.
 
-.. code-block:: nim
+  ```nim
   {.experimental: "callOperator".}
 
   template `()`(a: int, b: float): untyped = $(a, b)
@@ -467,21 +479,95 @@ to use this operator.
 
     doAssert not compiles(a.b(c)) # gives a type mismatch error same as b(a, c)
     doAssert (a.b)(c) == `()`(a.b, c)
+  ```
+
+
+Extended macro pragmas
+======================
+
+Macro pragmas as described in [the manual](manual.html#userminusdefined-pragmas-macro-pragmas)
+can also be applied to type, variable and constant declarations.
+
+For types:
+
+  ```nim
+  type
+    MyObject {.schema: "schema.protobuf".} = object
+  ```
+
+This is translated to a call to the `schema` macro with a `nnkTypeDef`
+AST node capturing the left-hand side, remaining pragmas and the right-hand
+side of the definition. The macro can return either a type section or
+another `nnkTypeDef` node, both of which will replace the original row
+in the type section.
+
+In the future, this `nnkTypeDef` argument may be replaced with a unary
+type section node containing the type definition, or some other node that may
+be more convenient to work with. The ability to return nodes other than type
+definitions may also be supported, however currently this is not convenient
+when dealing with mutual type recursion. For now, macros can return an unused
+type definition where the right-hand node is of kind `nnkStmtListType`.
+Declarations in this node will be attached to the same scope as
+the parent scope of the type section.
+
+------
+
+For variables and constants, it is largely the same, except a unary node with
+the same kind as the section containing a single definition is passed to macros,
+and macros can return any expression.
+
+  ```nim
+  var
+    a = ...
+    b {.importc, foo, nodecl.} = ...
+    c = ...
+  ```
+
+Assuming `foo` is a macro or a template, this is roughly equivalent to:
+
+  ```nim
+  var a = ...
+  foo:
+    var b {.importc, nodecl.} = ...
+  var c = ...
+  ```
+
+
+Symbols as template/macro calls (alias syntax)
+==============================================
+
+Templates and macros that have no generic parameters and no required arguments
+can be called as lone symbols, i.e. without parentheses. This is useful for
+repeated uses of complex expressions that cannot conveniently be represented
+as runtime values.
+
+  ```nim
+  type Foo = object
+    bar: int
+
+  var foo = Foo(bar: 10)
+  template bar: int = foo.bar
+  assert bar == 10
+  bar = 15
+  assert bar == 15
+  ```
 
 
 Not nil annotation
 ==================
 
 **Note:** This is an experimental feature. It can be enabled with
-`{.experimental: "notnil"}`.
+`{.experimental: "notnil".}`.
 
-All types for which `nil` is a valid value can be annotated with the `not
-nil` annotation to exclude `nil` as a valid value:
+All types for which `nil` is a valid value can be annotated with the
+`not nil` annotation to exclude `nil` as a valid value. Note that only local
+symbols are checked.
 
-.. code-block:: nim
-  {.experimental: "notnil"}
+  ```nim
+  {.experimental: "notnil".}
 
   type
+    TObj = object
     PObject = ref TObj not nil
     TProc = (proc (x, y: int)) not nil
 
@@ -492,14 +578,323 @@ nil` annotation to exclude `nil` as a valid value:
   p(nil)
 
   # and also this:
-  var x: PObject
-  p(x)
+  proc foo =
+    var x: PObject
+    p(x)
+
+  foo()
+  ```
 
 The compiler ensures that every code path initializes variables which contain
 non-nilable pointers. The details of this analysis are still to be specified
 here.
 
-.. include:: manual_experimental_strictnotnil.rst
+.. include:: manual_experimental_strictnotnil.md
+
+
+Aliasing restrictions in parameter passing
+==========================================
+
+.. note:: The aliasing restrictions are currently not enforced by the
+  implementation and need to be fleshed out further.
+
+"Aliasing" here means that the underlying storage locations overlap in memory
+at runtime. An "output parameter" is a parameter of type `var T`,
+an input parameter is any parameter that is not of type `var`.
+
+1. Two output parameters should never be aliased.
+2. An input and an output parameter should not be aliased.
+3. An output parameter should never be aliased with a global or thread local
+   variable referenced by the called proc.
+4. An input parameter should not be aliased with a global or thread local
+   variable updated by the called proc.
+
+One problem with rules 3 and 4 is that they affect specific global or thread
+local variables, but Nim's effect tracking only tracks "uses no global variable"
+via `.noSideEffect`. The rules 3 and 4 can also be approximated by a different rule:
+
+5. A global or thread local variable (or a location derived from such a location)
+   can only passed to a parameter of a `.noSideEffect` proc.
+
+
+Strict funcs
+============
+
+Since version 1.4, a stricter definition of "side effect" is available.
+In addition to the existing rule that a side effect is calling a function
+with side effects, the following rule is also enforced:
+
+A store to the heap via a `ref` or `ptr` indirection is not allowed.
+
+For example:
+
+  ```nim
+  {.experimental: "strictFuncs".}
+
+  type
+    Node = ref object
+      le, ri: Node
+      data: string
+
+  func len(n: Node): int =
+    # valid: len does not have side effects
+    var it = n
+    while it != nil:
+      inc result
+      it = it.ri
+
+  func mut(n: Node) =
+    var it = n
+    while it != nil:
+      it.data = "yeah" # forbidden mutation
+      it = it.ri
+
+  ```
+
+
+View types
+==========
+
+.. tip::  `--experimental:views`:option: is more effective
+  with `--experimental:strictFuncs`:option:.
+
+A view type is a type that is or contains one of the following types:
+
+- `lent T` (view into `T`)
+- `openArray[T]` (pair of (pointer to array of `T`, size))
+
+For example:
+
+  ```nim
+  type
+    View1 = openArray[byte]
+    View2 = lent string
+    View3 = Table[openArray[char], int]
+  ```
+
+
+Exceptions to this rule are types constructed via `ptr` or `proc`.
+For example, the following types are **not** view types:
+
+  ```nim
+  type
+    NotView1 = proc (x: openArray[int])
+    NotView2 = ptr openArray[char]
+    NotView3 = ptr array[4, lent int]
+  ```
+
+
+The mutability aspect of a view type is not part of the type but part
+of the locations it's derived from. More on this later.
+
+A *view* is a symbol (a let, var, const, etc.) that has a view type.
+
+Since version 1.4, Nim allows view types to be used as local variables.
+This feature needs to be enabled via `{.experimental: "views".}`.
+
+A local variable of a view type *borrows* from the locations and
+it is statically enforced that the view does not outlive the location
+it was borrowed from.
+
+For example:
+
+  ```nim
+  {.experimental: "views".}
+
+  proc take(a: openArray[int]) =
+    echo a.len
+
+  proc main(s: seq[int]) =
+    var x: openArray[int] = s # 'x' is a view into 's'
+    # it is checked that 'x' does not outlive 's' and
+    # that 's' is not mutated.
+    for i in 0 .. high(x):
+      echo x[i]
+    take(x)
+
+    take(x.toOpenArray(0, 1)) # slicing remains possible
+    let y = x  # create a view from a view
+    take y
+    # it is checked that 'y' does not outlive 'x' and
+    # that 'x' is not mutated as long as 'y' lives.
+
+
+  main(@[11, 22, 33])
+  ```
+
+
+A local variable of a view type can borrow from a location
+derived from a parameter, another local variable, a global `const` or `let`
+symbol or a thread-local `var` or `let`.
+
+Let `p` the proc that is analysed for the correctness of the borrow operation.
+
+Let `source` be one of:
+
+- A formal parameter of `p`. Note that this does not cover parameters of
+  inner procs.
+- The `result` symbol of `p`.
+- A local `var` or `let` or `const` of `p`. Note that this does
+  not cover locals of inner procs.
+- A thread-local `var` or `let`.
+- A global `let` or `const`.
+- A constant array/seq/object/tuple constructor.
+
+
+Path expressions
+----------------
+
+A location derived from `source` is then defined as a path expression that
+has `source` as the owner. A path expression `e` is defined recursively:
+
+- `source` itself is a path expression.
+- Container access like `e[i]` is a path expression.
+- Tuple access `e[0]` is a path expression.
+- Object field access `e.field` is a path expression.
+- `system.toOpenArray(e, ...)` is a path expression.
+- Pointer dereference `e[]` is a path expression.
+- An address `addr e` is a path expression.
+- A type conversion `T(e)` is a path expression.
+- A cast expression `cast[T](e)` is a path expression.
+- `f(e, ...)` is a path expression if `f`'s return type is a view type.
+  Because the view can only have been borrowed from `e`, we then know
+  that the owner of `f(e, ...)` is `e`.
+
+
+If a view type is used as a return type, the location must borrow from a location
+that is derived from the first parameter that is passed to the proc.
+See [the manual](manual.html#procedures-var-return-type)
+for details about how this is done for `var T`.
+
+A mutable view can borrow from a mutable location, an immutable view can borrow
+from both a mutable or an immutable location.
+
+If a view borrows from a mutable location, the view can be used to update the
+location. Otherwise it cannot be used for mutations.
+
+The *duration* of a borrow is the span of commands beginning from the assignment
+to the view and ending with the last usage of the view.
+
+For the duration of the borrow operation, no mutations to the borrowed locations
+may be performed except via the view that borrowed from the
+location. The borrowed location is said to be *sealed* during the borrow.
+
+  ```nim
+  {.experimental: "views".}
+
+  type
+    Obj = object
+      field: string
+
+  proc dangerous(s: var seq[Obj]) =
+    let v: lent Obj = s[0] # seal 's'
+    s.setLen 0  # prevented at compile-time because 's' is sealed.
+    echo v.field
+  ```
+
+
+The scope of the view does not matter:
+
+  ```nim
+  proc valid(s: var seq[Obj]) =
+    let v: lent Obj = s[0]  # begin of borrow
+    echo v.field            # end of borrow
+    s.setLen 0  # valid because 'v' isn't used afterwards
+  ```
+
+
+The analysis requires as much precision about mutations as is reasonably obtainable,
+so it is more effective with the experimental [strict funcs]
+feature. In other words `--experimental:views`:option: works better
+with `--experimental:strictFuncs`:option:.
+
+The analysis is currently control flow insensitive:
+
+  ```nim
+  proc invalid(s: var seq[Obj]) =
+    let v: lent Obj = s[0]
+    if false:
+      s.setLen 0
+    echo v.field
+  ```
+
+In this example, the compiler assumes that `s.setLen 0` invalidates the
+borrow operation of `v` even though a human being can easily see that it
+will never do that at runtime.
+
+
+Start of a borrow
+-----------------
+
+A borrow starts with one of the following:
+
+- The assignment of a non-view-type to a view-type.
+- The assignment of a location that is derived from a local parameter
+  to a view-type.
+
+
+End of a borrow
+---------------
+
+A borrow operation ends with the last usage of the view variable.
+
+
+Reborrows
+---------
+
+A view `v` can borrow from multiple different locations. However, the borrow
+is always the full span of `v`'s lifetime and every location that is borrowed
+from is sealed during `v`'s lifetime.
+
+
+Algorithm
+---------
+
+The following section is an outline of the algorithm that the current implementation
+uses. The algorithm performs two traversals over the AST of the procedure or global
+section of code that uses a view variable. No fixpoint iterations are performed, the
+complexity of the analysis is O(N) where N is the number of nodes of the AST.
+
+The first pass over the AST computes the lifetime of each local variable based on
+a notion of an "abstract time", in the implementation it's a simple integer that is
+incremented for every visited node.
+
+In the second pass, information about the underlying object "graphs" is computed.
+Let `v` be a parameter or a local variable. Let `G(v)` be the graph
+that `v` belongs to. A graph is defined by the set of variables that belong
+to the graph. Initially for all `v`: `G(v) = {v}`. Every variable can only
+be part of a single graph.
+
+Assignments like `a = b` "connect" two variables, both variables end up in the
+same graph `{a, b} = G(a) = G(b)`. Unfortunately, the pattern to look for is
+much more complex than that and can involve multiple assignment targets
+and sources:
+
+    f(x, y) = g(a, b)
+
+connects `x` and `y` to `a` and `b`: `G(x) = G(y) = G(a) = G(b) = {x, y, a, b}`.
+A type based alias analysis rules out some of these combinations, for example
+a `string` value cannot possibly be connected to a `seq[int]`.
+
+A pattern like `v[] = value` or `v.field = value` marks `G(v)` as mutated.
+After the second pass a set of disjoint graphs was computed.
+
+For strict functions it is then enforced that there is no graph that is both mutated
+and has an element that is an immutable parameter (that is a parameter that is not
+of type `var T`).
+
+For borrow checking, a different set of checks is performed. Let `v` be the view
+and `b` the location that is borrowed from.
+
+- The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of
+  a parameter is the complete proc body.
+- If `v` is used for a mutation, `b` must be a mutable location too.
+- During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if
+  `v` is a mutable view).
+- If `v` is `result` then `b` has to be a location derived from the first
+  formal parameter or from a constant location.
+- A view cannot be used for a read or a write access before it was assigned to.
+
 
 Concepts
 ========
@@ -509,7 +904,7 @@ arbitrary set of requirements that the matched type must satisfy.
 
 Concepts are written in the following form:
 
-.. code-block:: nim
+  ```nim
   type
     Comparable = concept x, y
       (x < y) is bool
@@ -522,11 +917,13 @@ Concepts are written in the following form:
 
       for value in s:
         value is T
+  ```
 
-The concept is a match if:
+The concept matches if:
 
-a) all of the expressions within the body can be compiled for the tested type
-b) all statically evaluable boolean expressions in the body must be true
+a) all expressions within the body can be compiled for the tested type
+b) all statically evaluable boolean expressions in the body are true
+c) all type modifiers specified match their respective definitions
 
 The identifiers following the `concept` keyword represent instances of the
 currently matched type. You can apply any of the standard type modifiers such
@@ -534,26 +931,28 @@ as `var`, `ref`, `ptr` and `static` to denote a more specific type of
 instance. You can also apply the `type` modifier to create a named instance of
 the type itself:
 
-.. code-block:: nim
+  ```nim
   type
     MyConcept = concept x, var v, ref r, ptr p, static s, type T
       ...
+  ```
 
 Within the concept body, types can appear in positions where ordinary values
 and parameters are expected. This provides a more convenient way to check for
 the presence of callable symbols with specific signatures:
 
-.. code-block:: nim
+  ```nim
   type
     OutputStream = concept var s
       s.write(string)
+  ```
 
 In order to check for symbols accepting `type` params, you must prefix
 the type with the explicit `type` modifier. The named instance of the
 type, following the `concept` keyword is also considered to have the
 explicit modifier and will be matched only as a type.
 
-.. code-block:: nim
+  ```nim
   type
     # Let's imagine a user-defined casting framework with operators
     # such as `val.to(string)` and `val.to(JSonValue)`. We can test
@@ -571,6 +970,7 @@ explicit modifier and will be matched only as a type.
       x is AdditiveMonoid
       -x is T
       x - y is T
+  ```
 
 Please note that the `is` operator allows one to easily verify the precise
 type signatures of the required operations, but since type inference and
@@ -590,11 +990,12 @@ When you need to understand why the compiler is not matching a particular
 concept and, as a result, a wrong overload is selected, you can apply the
 `explain` pragma to either the concept body or a particular call-site.
 
-.. code-block:: nim
+  ```nim
   type
     MyConcept {.explain.} = concept ...
 
   overloadedProc(x, y, z) {.explain.}
+  ```
 
 This will provide Hints in the compiler output either every time the concept is
 not matched or only on the particular call-site.
@@ -605,7 +1006,7 @@ Generic concepts and type binding rules
 
 The concept types can be parametric just like the regular generic types:
 
-.. code-block:: nim
+  ```nim
   ### matrixalgo.nim
 
   import std/typetraits
@@ -665,6 +1066,7 @@ The concept types can be parametric just like the regular generic types:
 
   echo m.transposed.determinant
   setPerspectiveProjection projectionMatrix
+  ```
 
 When the concept type is matched against a concrete type, the unbound type
 parameters are inferred from the body of the concept in a way that closely
@@ -678,10 +1080,11 @@ and `x.data is seq[T]`.
 Unbound static params will be inferred from expressions involving the `==`
 operator and also when types dependent on them are being matched:
 
-.. code-block:: nim
+  ```nim
   type
     MatrixReducer[M, N: static int; T] = concept x
       x.reduce(SquareMatrix[N, T]) is array[M, int]
+  ```
 
 The Nim compiler includes a simple linear equation solver, allowing it to
 infer static params in some situations where integer arithmetic is involved.
@@ -692,7 +1095,7 @@ modifier to any of the otherwise inferable types to get a type that will be
 matched without permanently inferring it. This may be useful when you need
 to match several procs accepting the same wide class of types:
 
-.. code-block:: nim
+  ```nim
   type
     Enumerable[T] = concept e
       for v in e:
@@ -709,12 +1112,13 @@ to match several procs accepting the same wide class of types:
       # it's also possible to give an alias name to a `bind many` type class
       type Enum = distinct Enumerable
       o.baz is Enum
+  ```
 
 On the other hand, using `bind once` types allows you to test for equivalent
 types used in multiple signatures, without actually requiring any concrete
 types, thus allowing you to encode implementation-defined types:
 
-.. code-block:: nim
+  ```nim
   type
     MyConcept = concept x
       type T1 = auto
@@ -725,6 +1129,7 @@ types, thus allowing you to encode implementation-defined types:
       x.alpha(T2)
       x.omega(T2) # both procs must accept the same type
                   # and it must be a numeric sequence
+  ```
 
 As seen in the previous examples, you can refer to generic concepts such as
 `Enumerable[T]` just by their short name. Much like the regular generic types,
@@ -742,9 +1147,7 @@ in any required way. For example, here is how one might define the classic
 `Functor` concept from Haskell and then demonstrate that Nim's `Option[T]`
 type is an instance of it:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   import std/[sugar, typetraits]
 
   type
@@ -765,6 +1168,7 @@ type is an instance of it:
 
   import std/options
   echo Option[int] is Functor # prints true
+  ```
 
 
 Concept derived values
@@ -774,7 +1178,7 @@ All top level constants or types appearing within the concept body are
 accessible through the dot operator in procs where the concept was successfully
 matched to a concrete type:
 
-.. code-block:: nim
+  ```nim
   type
     DateTime = concept t1, t2, type T
       const Min = T.MinDate
@@ -796,6 +1200,7 @@ matched to a concrete type:
 
       deviation: float
     ...
+  ```
 
 
 Concept refinement
@@ -808,7 +1213,7 @@ overload resolution, Nim will assign a higher precedence to the most specific
 one. As an alternative way of defining concept refinements, you can use the
 object inheritance syntax involving the `of` keyword:
 
-.. code-block:: nim
+  ```nim
   type
     Graph = concept g, type G of EquallyComparable, Copyable
       type
@@ -842,6 +1247,7 @@ object inheritance syntax involving the `of` keyword:
   proc f(g: IncidendeGraph)
   proc f(g: BidirectionalGraph) # this one will be preferred if we pass a type
                                 # matching the BidirectionalGraph concept
+  ```
 
 ..
   Converter type classes
@@ -851,7 +1257,7 @@ object inheritance syntax involving the `of` keyword:
   a small set of simpler types. This is achieved with a `return` statement within
   the concept body:
 
-  .. code-block:: nim
+    ```nim
     type
       Stringable = concept x
         $x is string
@@ -875,6 +1281,7 @@ object inheritance syntax involving the `of` keyword:
     # the same call at the cost of additional instantiations
     # the varargs param will be converted to a tuple
     proc log(format: static string, varargs[distinct StringRef])
+    ```
 
 
 ..
@@ -902,7 +1309,7 @@ object inheritance syntax involving the `of` keyword:
   a converter type class, which converts the regular instances of the matching
   types to the corresponding VTable type.
 
-  .. code-block:: nim
+    ```nim
     type
       IntEnumerable = vtref Enumerable[int]
 
@@ -915,6 +1322,7 @@ object inheritance syntax involving the `of` keyword:
 
     proc addStream(o: var MyObject, e: OutputStream.vtref) =
       o.streams.add e
+    ```
 
   The procs that will be included in the vtable are derived from the concept
   body and include all proc calls for which all param types were specified as
@@ -934,97 +1342,44 @@ object inheritance syntax involving the `of` keyword:
   the `vtptr` magic produced types bound to `ptr` types.
 
 
-Type bound operations
-=====================
+..
+  deepCopy
+  --------
+  `=deepCopy` is a builtin that is invoked whenever data is passed to
+  a `spawn`'ed proc to ensure memory safety. The programmer can override its
+  behaviour for a specific `ref` or `ptr` type `T`. (Later versions of the
+  language may weaken this restriction.)
 
-There are 4 operations that are bound to a type:
+  The signature has to be:
 
-1. Assignment
-2. Moves
-3. Destruction
-4. Deep copying for communication between threads
+    ```nim
+    proc `=deepCopy`(x: T): T
+    ```
 
-These operations can be *overridden* instead of *overloaded*. This means the
-implementation is automatically lifted to structured types. For instance if type
-`T` has an overridden assignment operator `=` this operator is also used
-for assignments of the type `seq[T]`. Since these operations are bound to a
-type they have to be bound to a nominal type for reasons of simplicity of
-implementation: This means an overridden `deepCopy` for `ref T` is really
-bound to `T` and not to `ref T`. This also means that one cannot override
-`deepCopy` for both `ptr T` and `ref T` at the same time; instead a
-helper distinct or object type has to be used for one pointer type.
+  This mechanism will be used by most data structures that support shared memory,
+  like channels, to implement thread safe automatic memory management.
 
-Assignments, moves and destruction are specified in
-the `destructors `_ document.
+  The builtin `deepCopy` can even clone closures and their environments. See
+  the documentation of [spawn][spawn statement] for details.
 
 
-deepCopy
---------
+Dynamic arguments for bindSym
+=============================
 
-`=deepCopy` is a builtin that is invoked whenever data is passed to
-a `spawn`'ed proc to ensure memory safety. The programmer can override its
-behaviour for a specific `ref` or `ptr` type `T`. (Later versions of the
-language may weaken this restriction.)
+This experimental feature allows the symbol name argument of `macros.bindSym`
+to be computed dynamically.
 
-The signature has to be:
-
-.. code-block:: nim
-  proc `=deepCopy`(x: T): T
-
-This mechanism will be used by most data structures that support shared memory
-like channels to implement thread safe automatic memory management.
-
-The builtin `deepCopy` can even clone closures and their environments. See
-the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details.
-
-
-Case statement macros
-=====================
-
-Macros named `case` can rewrite `case` statements for certain types in order to
-implement `pattern matching`:idx:. The following example implements a
-simplistic form of pattern matching for tuples, leveraging the existing
-equality operator for tuples (as provided in `system.==`):
-
-.. code-block:: nim
-    :test: "nim c $1"
-
-  {.experimental: "caseStmtMacros".}
+  ```nim
+  {.experimental: "dynamicBindSym".}
 
   import std/macros
 
-  macro `case`(n: tuple): untyped =
-    result = newTree(nnkIfStmt)
-    let selector = n[0]
-    for i in 1 ..< n.len:
-      let it = n[i]
-      case it.kind
-      of nnkElse, nnkElifBranch, nnkElifExpr, nnkElseExpr:
-        result.add it
-      of nnkOfBranch:
-        for j in 0..it.len-2:
-          let cond = newCall("==", selector, it[j])
-          result.add newTree(nnkElifBranch, cond, it[^1])
-      else:
-        error "custom 'case' for tuple cannot handle this node", it
+  macro callOp(opName, arg1, arg2): untyped =
+    result = newCall(bindSym($opName), arg1, arg2)
 
-  case ("foo", 78)
-  of ("foo", 78): echo "yes"
-  of ("bar", 88): echo "no"
-  else: discard
-
-
-Currently case statement macros must be enabled explicitly
-via `{.experimental: "caseStmtMacros".}`.
-
-`case` macros are subject to overload resolution. The type of the
-`case` statement's selector expression is matched against the type
-of the first argument of the `case` macro. Then the complete `case`
-statement is passed in place of the argument and the macro is evaluated.
-
-In other words, the macro needs to transform the full `case` statement
-but only the statement's selector expression is used to determine which
-macro to call.
+  echo callOp("+", 1, 2)
+  echo callOp("-", 5, 4)
+  ```
 
 
 Term rewriting macros
@@ -1035,48 +1390,51 @@ a *name* but also a *pattern* that is searched for after the semantic checking
 phase of the compiler: This means they provide an easy way to enhance the
 compilation pipeline with user defined optimizations:
 
-.. code-block:: nim
-  template optMul{`*`(a, 2)}(a: int): int = a+a
+  ```nim
+  template optMul{`*`(a, 2)}(a: int): int = a + a
 
   let x = 3
   echo x * 2
+  ```
 
 The compiler now rewrites `x * 2` as `x + x`. The code inside the
-curlies is the pattern to match against. The operators `*`,  `**`,
+curly brackets is the pattern to match against. The operators `*`,  `**`,
 `|`, `~` have a special meaning in patterns if they are written in infix
 notation, so to match verbatim against `*` the ordinary function call syntax
 needs to be used.
 
-Term rewriting macro are applied recursively, up to a limit. This means that
+Term rewriting macros are applied recursively, up to a limit. This means that
 if the result of a term rewriting macro is eligible for another rewriting,
 the compiler will try to perform it, and so on, until no more optimizations
 are applicable. To avoid putting the compiler into an infinite loop, there is
 a hard limit on how many times a single term rewriting macro can be applied.
 Once this limit has been passed, the term rewriting macro will be ignored.
 
-Unfortunately optimizations are hard to get right and even the tiny example
+Unfortunately optimizations are hard to get right and even this tiny example
 is **wrong**:
 
-.. code-block:: nim
-  template optMul{`*`(a, 2)}(a: int): int = a+a
+  ```nim
+  template optMul{`*`(a, 2)}(a: int): int = a + a
 
   proc f(): int =
     echo "side effect!"
     result = 55
 
   echo f() * 2
+  ```
 
 We cannot duplicate 'a' if it denotes an expression that has a side effect!
 Fortunately Nim supports side effect analysis:
 
-.. code-block:: nim
-  template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a
+  ```nim
+  template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a + a
 
   proc f(): int =
     echo "side effect!"
     result = 55
 
   echo f() * 2 # not optimized ;-)
+  ```
 
 You can make one overload matching with a constraint and one without, and the
 one with a constraint will have precedence, and so you can handle both cases
@@ -1086,13 +1444,15 @@ So what about `2 * a`? We should tell the compiler `*` is commutative. We
 cannot really do that however as the following code only swaps arguments
 blindly:
 
-.. code-block:: nim
-  template mulIsCommutative{`*`(a, b)}(a, b: int): int = b*a
+  ```nim
+  template mulIsCommutative{`*`(a, b)}(a, b: int): int = b * a
+  ```
 
 What optimizers really need to do is a *canonicalization*:
 
-.. code-block:: nim
-  template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b*a
+  ```nim
+  template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b * a
+  ```
 
 The `int{lit}` parameter pattern matches against an expression of
 type `int`, but only if it's a literal.
@@ -1109,7 +1469,7 @@ The `parameter constraint`:idx: expression can use the operators `|` (or),
 Predicate                Meaning
 ===================      =====================================================
 `atom`                   The matching node has no children.
-`lit`                    The matching node is a literal like "abc", 12.
+`lit`                    The matching node is a literal like `"abc"`, `12`.
 `sym`                    The matching node must be a symbol (a bound
                          identifier).
 `ident`                  The matching node must be an identifier (an unbound
@@ -1150,11 +1510,31 @@ The `alias` and `noalias` predicates refer not only to the matching AST,
 but also to every other bound parameter; syntactically they need to occur after
 the ordinary AST predicates:
 
-.. code-block:: nim
+  ```nim
   template ex{a = b + c}(a: int{noalias}, b, c: int) =
     # this transformation is only valid if 'b' and 'c' do not alias 'a':
     a = b
     inc a, c
+  ```
+
+Another example:
+
+  ```nim
+  proc somefunc(s: string)                 = assert s == "variable"
+  proc somefunc(s: string{nkStrLit})       = assert s == "literal"
+  proc somefunc(s: string{nkRStrLit})      = assert s == r"raw"
+  proc somefunc(s: string{nkTripleStrLit}) = assert s == """triple"""
+  proc somefunc(s: static[string])         = assert s == "constant"
+
+  # Use parameter constraints to provide overloads based on both the input parameter type and form.
+  var variable = "variable"
+  somefunc(variable)
+  const constant = "constant"
+  somefunc(constant)
+  somefunc("literal")
+  somefunc(r"raw")
+  somefunc("""triple""")
+  ```
 
 
 Pattern operators
@@ -1164,50 +1544,50 @@ The operators `*`,  `**`, `|`, `~` have a special meaning in patterns
 if they are written in infix notation.
 
 
-The `|` operator
-~~~~~~~~~~~~~~~~~~
+### The `|` operator
 
 The `|` operator if used as infix operator creates an ordered choice:
 
-.. code-block:: nim
+  ```nim
   template t{0|1}(): untyped = 3
   let a = 1
   # outputs 3:
   echo a
+  ```
 
 The matching is performed after the compiler performed some optimizations like
 constant folding, so the following does not work:
 
-.. code-block:: nim
+  ```nim
   template t{0|1}(): untyped = 3
   # outputs 1:
   echo 1
+  ```
 
 The reason is that the compiler already transformed the 1 into "1" for
 the `echo` statement. However, a term rewriting macro should not change the
-semantics anyway. In fact they can be deactivated with the `--patterns:off`
+semantics anyway. In fact, they can be deactivated with the `--patterns:off`:option:
 command line option or temporarily with the `patterns` pragma.
 
 
-The `{}` operator
-~~~~~~~~~~~~~~~~~~~
+### The `{}` operator
 
 A pattern expression can be bound to a pattern parameter via the `expr{param}`
 notation:
 
-.. code-block:: nim
-  template t{(0|1|2){x}}(x: untyped): untyped = x+1
+  ```nim
+  template t{(0|1|2){x}}(x: untyped): untyped = x + 1
   let a = 1
   # outputs 2:
   echo a
+  ```
 
 
-The `~` operator
-~~~~~~~~~~~~~~~~~~
+### The `~` operator
 
-The `~` operator is the **not** operator in patterns:
+The `~` operator is the 'not' operator in patterns:
 
-.. code-block:: nim
+  ```nim
   template t{x = (~x){y} and (~x){z}}(x, y, z: bool) =
     x = y
     if x: x = z
@@ -1218,15 +1598,15 @@ The `~` operator is the **not** operator in patterns:
     c = false
   a = b and c
   echo a
+  ```
 
 
-The `*` operator
-~~~~~~~~~~~~~~~~~~
+### The `*` operator
 
 The `*` operator can *flatten* a nested binary expression like `a & b & c`
 to `&(a, b, c)`:
 
-.. code-block:: nim
+  ```nim
   var
     calls = 0
 
@@ -1242,6 +1622,7 @@ to `&(a, b, c)`:
 
   # check that it's been optimized properly:
   doAssert calls == 1
+  ```
 
 
 The second operator of `*` must be a parameter; it is used to gather all the
@@ -1250,17 +1631,17 @@ is passed to `optConc` in `a` as a special list (of kind `nkArgList`)
 which is flattened into a call expression; thus the invocation of `optConc`
 produces:
 
-.. code-block:: nim
-   `&&`("my", space & "awe", "some ", "concat")
+  ```nim
+  `&&`("my", space & "awe", "some ", "concat")
+  ```
 
 
-The `**` operator
-~~~~~~~~~~~~~~~~~~~
+### The `**` operator
 
 The `**` is much like the `*` operator, except that it gathers not only
 all the arguments, but also the matched operators in reverse polish notation:
 
-.. code-block:: nim
+  ```nim
   import std/macros
 
   type
@@ -1281,35 +1662,58 @@ all the arguments, but also the matched operators in reverse polish notation:
   var x, y, z: Matrix
 
   echo x + y * z - x
+  ```
 
 This passes the expression `x + y * z - x` to the `optM` macro as
-an `nnkArglist` node containing::
+an `nnkArgList` node containing:
 
-  Arglist
-    Sym "x"
-    Sym "y"
-    Sym "z"
-    Sym "*"
-    Sym "+"
-    Sym "x"
-    Sym "-"
+    Arglist
+      Sym "x"
+      Sym "y"
+      Sym "z"
+      Sym "*"
+      Sym "+"
+      Sym "x"
+      Sym "-"
 
-(Which is the reverse polish notation of `x + y * z - x`.)
+(This is the reverse polish notation of `x + y * z - x`.)
 
 
 Parameters
 ----------
 
 Parameters in a pattern are type checked in the matching process. If a
-parameter is of the type `varargs` it is treated specially and it can match
+parameter is of the type `varargs`, it is treated specially and can match
 0 or more arguments in the AST to be matched against:
 
-.. code-block:: nim
+  ```nim
   template optWrite{
     write(f, x)
     ((write|writeLine){w})(f, y)
   }(x, y: varargs[untyped], f: File, w: untyped) =
     w(f, x, y)
+  ```
+
+
+noRewrite pragma
+----------------
+
+Term rewriting macros and templates are currently greedy and
+they will rewrite as long as there is a match.
+There was no way to ensure some rewrite happens only once,
+e.g. when rewriting term to same term plus extra content.
+
+`noRewrite` pragma can actually prevent further rewriting on marked code,
+e.g. with given example `echo("ab")` will be rewritten just once:
+
+  ```nim
+  template pwnEcho{echo(x)}(x: untyped) =
+    {.noRewrite.}: echo("pwned!")
+
+  echo "ab"
+  ```
+
+`noRewrite` pragma can be useful to control term-rewriting macros recursion.
 
 
 
@@ -1319,12 +1723,13 @@ Example: Partial evaluation
 The following example shows how some simple partial evaluation can be
 implemented with term rewriting:
 
-.. code-block:: nim
+  ```nim
   proc p(x, y: int; cond: bool): int =
     result = if cond: x + y else: x - y
 
   template optP1{p(x, y, true)}(x, y: untyped): untyped = x + y
   template optP2{p(x, y, false)}(x, y: untyped): untyped = x - y
+  ```
 
 
 Example: Hoisting
@@ -1332,7 +1737,7 @@ Example: Hoisting
 
 The following example shows how some form of hoisting can be implemented:
 
-.. code-block:: nim
+  ```nim
   import std/pegs
 
   template optPeg{peg(pattern)}(pattern: string{lit}): Peg =
@@ -1342,6 +1747,7 @@ The following example shows how some form of hoisting can be implemented:
   for i in 0 .. 3:
     echo match("(a b c)", peg"'(' @ ')'")
     echo match("W_HI_Le", peg"\y 'while'")
+  ```
 
 The `optPeg` template optimizes the case of a peg constructor with a string
 literal, so that the pattern will only be parsed once at program startup and
@@ -1353,9 +1759,9 @@ AST based overloading
 =====================
 
 Parameter constraints can also be used for ordinary routine parameters; these
-constraints affect ordinary overloading resolution then:
+constraints then affect ordinary overloading resolution:
 
-.. code-block:: nim
+  ```nim
   proc optLit(a: string{lit|`const`}) =
     echo "string literal"
   proc optLit(a: string) =
@@ -1370,6 +1776,7 @@ constraints affect ordinary overloading resolution then:
   optLit("literal")
   optLit(constant)
   optLit(variable)
+  ```
 
 However, the constraints `alias` and `noalias` are not available in
 ordinary routines.
@@ -1384,29 +1791,32 @@ Nim has two flavors of parallelism:
 
 Nim has a builtin thread pool that can be used for CPU intensive tasks. For
 IO intensive tasks the `async` and `await` features should be
-used instead. Both parallel and spawn need the `threadpool `_
+used instead. Both parallel and spawn need the [threadpool](threadpool.html)
 module to work.
 
 Somewhat confusingly, `spawn` is also used in the `parallel` statement
 with slightly different semantics. `spawn` always takes a call expression of
-the form `f(a, ...)`. Let `T` be `f`'s return type. If `T` is `void`
-then `spawn`'s return type is also `void` otherwise it is `FlowVar[T]`.
+the form `f(a, ...)`. Let `T` be `f`'s return type. If `T` is `void`,
+then `spawn`'s return type is also `void`, otherwise it is `FlowVar[T]`.
 
-Within a `parallel` section sometimes the `FlowVar[T]` is eliminated
+Within a `parallel` section, the `FlowVar[T]` is sometimes eliminated
 to `T`. This happens when `T` does not contain any GC'ed memory.
 The compiler can ensure the location in `location = spawn f(...)` is not
 read prematurely within a `parallel` section and so there is no need for
 the overhead of an indirection via `FlowVar[T]` to ensure correctness.
 
-**Note**: Currently exceptions are not propagated between `spawn`'ed tasks!
+.. note:: Currently exceptions are not propagated between `spawn`'ed tasks!
+
+This feature is likely to be removed in the future as external packages
+can have better solutions.
 
 
 Spawn statement
 ---------------
 
-`spawn`:idx: can be used to pass a task to the thread pool:
+The `spawn`:idx: statement can be used to pass a task to the thread pool:
 
-.. code-block:: nim
+  ```nim
   import std/threadpool
 
   proc processLine(line: string) =
@@ -1415,6 +1825,7 @@ Spawn statement
   for x in lines("myinput.txt"):
     spawn processLine(x)
   sync()
+  ```
 
 For reasons of type safety and implementation simplicity the expression
 that `spawn` takes is restricted:
@@ -1425,10 +1836,10 @@ that `spawn` takes is restricted:
 * `f`'s parameters may not be of type `var`.
   This means one has to use raw `ptr`'s for data passing reminding the
   programmer to be careful.
-* `ref` parameters are deeply copied which is a subtle semantic change and
-  can cause performance problems but ensures memory safety. This deep copy
-  is performed via `system.deepCopy` and so can be overridden.
-* For *safe* data exchange between `f` and the caller a global `TChannel`
+* `ref` parameters are deeply copied, which is a subtle semantic change and
+  can cause performance problems, but ensures memory safety. This deep copy
+  is performed via `system.deepCopy`, so it can be overridden.
+* For *safe* data exchange between `f` and the caller, a global `Channel`
   needs to be used. However, since spawn can return a result, often no further
   communication is required.
 
@@ -1438,7 +1849,7 @@ a `data flow variable`:idx: `FlowVar[T]` that can be read from. The reading
 with the `^` operator is **blocking**. However, one can use `blockUntilAny` to
 wait on multiple flow variables at the same time:
 
-.. code-block:: nim
+  ```nim
   import std/threadpool, ...
 
   # wait until 2 out of 3 servers received the update:
@@ -1450,11 +1861,12 @@ wait on multiple flow variables at the same time:
     assert index >= 0
     responses.del(index)
     discard blockUntilAny(responses)
+  ```
 
-Data flow variables ensure that no data races
-are possible. Due to technical limitations not every type `T` is possible in
-a data flow variable: `T` has to be of the type `ref`, `string`, `seq`
-or of a type that doesn't contain a type that is garbage collected. This
+Data flow variables ensure that no data races are possible. Due to
+technical limitations, not every type `T` can be used in
+a data flow variable: `T` has to be a `ref`, `string`, `seq`
+or of a type that doesn't contain any GC'd type. This
 restriction is not hard to work-around in practice.
 
 
@@ -1464,17 +1876,15 @@ Parallel statement
 
 Example:
 
-.. code-block:: nim
-    :test: "nim c --threads:on $1"
-
-  # Compute PI in an inefficient way
+  ```nim  test = "nim c --threads:on $1"
+  # Compute pi in an inefficient way
   import std/[strutils, math, threadpool]
   {.experimental: "parallel".}
 
   proc term(k: float): float = 4 * math.pow(-1, k) / (2*k + 1)
 
   proc pi(n: int): float =
-    var ch = newSeq[float](n+1)
+    var ch = newSeq[float](n + 1)
     parallel:
       for k in 0..ch.high:
         ch[k] = spawn term(float(k))
@@ -1482,19 +1892,20 @@ Example:
       result += ch[k]
 
   echo formatFloat(pi(5000))
+  ```
 
 
 The parallel statement is the preferred mechanism to introduce parallelism in a
-Nim program. A subset of the Nim language is valid within a `parallel`
+Nim program. Only a subset of the Nim language is valid within a `parallel`
 section. This subset is checked during semantic analysis to be free of data
 races. A sophisticated `disjoint checker`:idx: ensures that no data races are
-possible even though shared memory is extensively supported!
+possible, even though shared memory is extensively supported!
 
 The subset is in fact the full language with the following
 restrictions / changes:
 
 * `spawn` within a `parallel` section has special semantics.
-* Every location of the form `a[i]` and `a[i..j]` and `dest` where
+* Every location of the form `a[i]`, `a[i..j]` and `dest` where
   `dest` is part of the pattern `dest = spawn f(...)` has to be
   provably disjoint. This is called the *disjoint check*.
 * Every other complex location `loc` that is used in a spawned
@@ -1508,569 +1919,696 @@ restrictions / changes:
   yet performed for ordinary slices outside of a `parallel` section.
 
 
-Guards and locks
-================
+Strict definitions and `out` parameters
+=======================================
 
-Apart from `spawn` and `parallel` Nim also provides all the common low level
-concurrency mechanisms like locks, atomic intrinsics or condition variables.
+With `experimental: "strictDefs"` *every* local variable must be initialized explicitly before it can be used:
 
-Nim significantly improves on the safety of these features via additional
-pragmas:
+  ```nim
+  {.experimental: "strictDefs".}
 
-1) A `guard`:idx: annotation is introduced to prevent data races.
-2) Every access of a guarded memory location needs to happen in an
-   appropriate `locks`:idx: statement.
-3) Locks and routines can be annotated with `lock levels`:idx: to allow
-   potential deadlocks to be detected during semantic analysis.
+  proc test =
+    var s: seq[string]
+    s.add "abc" # invalid!
 
+  ```
 
-Guards and the locks section
-----------------------------
+Needs to be written as:
 
-Protecting global variables
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  ```nim
+  {.experimental: "strictDefs".}
 
-Object fields and global variables can be annotated via a `guard` pragma:
+  proc test =
+    var s: seq[string] = @[]
+    s.add "abc" # valid!
 
-.. code-block:: nim
-  var glock: TLock
-  var gdata {.guard: glock.}: int
+  ```
 
-The compiler then ensures that every access of `gdata` is within a `locks`
-section:
+A control flow analysis is performed in order to prove that a variable has been written to
+before it is used. Thus the following is valid:
 
-.. code-block:: nim
-  proc invalid =
-    # invalid: unguarded access:
-    echo gdata
+  ```nim
+  {.experimental: "strictDefs".}
 
-  proc valid =
-    # valid access:
-    {.locks: [glock].}:
-      echo gdata
-
-Top level accesses to `gdata` are always allowed so that it can be initialized
-conveniently. It is *assumed* (but not enforced) that every top level statement
-is executed before any concurrent action happens.
-
-The `locks` section deliberately looks ugly because it has no runtime
-semantics and should not be used directly! It should only be used in templates
-that also implement some form of locking at runtime:
-
-.. code-block:: nim
-  template lock(a: TLock; body: untyped) =
-    pthread_mutex_lock(a)
-    {.locks: [a].}:
-      try:
-        body
-      finally:
-        pthread_mutex_unlock(a)
-
-
-The guard does not need to be of any particular type. It is flexible enough to
-model low level lockfree mechanisms:
-
-.. code-block:: nim
-  var dummyLock {.compileTime.}: int
-  var atomicCounter {.guard: dummyLock.}: int
-
-  template atomicRead(x): untyped =
-    {.locks: [dummyLock].}:
-      memoryReadBarrier()
-      x
-
-  echo atomicRead(atomicCounter)
-
-
-The `locks` pragma takes a list of lock expressions `locks: [a, b, ...]`
-in order to support *multi lock* statements. Why these are essential is
-explained in the `lock levels <#guards-and-locks-lock-levels>`_ section.
-
-
-Protecting general locations
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The `guard` annotation can also be used to protect fields within an object.
-The guard then needs to be another field within the same object or a
-global variable.
-
-Since objects can reside on the heap or on the stack this greatly enhances the
-expressivity of the language:
-
-.. code-block:: nim
-  type
-    ProtectedCounter = object
-      v {.guard: L.}: int
-      L: TLock
-
-  proc incCounters(counters: var openArray[ProtectedCounter]) =
-    for i in 0..counters.high:
-      lock counters[i].L:
-        inc counters[i].v
-
-The access to field `x.v` is allowed since its guard `x.L`  is active.
-After template expansion, this amounts to:
-
-.. code-block:: nim
-  proc incCounters(counters: var openArray[ProtectedCounter]) =
-    for i in 0..counters.high:
-      pthread_mutex_lock(counters[i].L)
-      {.locks: [counters[i].L].}:
-        try:
-          inc counters[i].v
-        finally:
-          pthread_mutex_unlock(counters[i].L)
-
-There is an analysis that checks that `counters[i].L` is the lock that
-corresponds to the protected location `counters[i].v`. This analysis is called
-`path analysis`:idx: because it deals with paths to locations
-like `obj.field[i].fieldB[j]`.
-
-The path analysis is **currently unsound**, but that doesn't make it useless.
-Two paths are considered equivalent if they are syntactically the same.
-
-This means the following compiles (for now) even though it really should not:
-
-.. code-block:: nim
-  {.locks: [a[i].L].}:
-    inc i
-    access a[i].v
-
-
-
-Lock levels
------------
-
-Lock levels are used to enforce a global locking order in order to detect
-potential deadlocks during semantic analysis. A lock level is an constant
-integer in the range 0..1_000. Lock level 0 means that no lock is acquired at
-all.
-
-If a section of code holds a lock of level `M` than it can also acquire any
-lock of level `N < M`. Another lock of level `M` cannot be acquired. Locks
-of the same level can only be acquired *at the same time* within a
-single `locks` section:
-
-.. code-block:: nim
-  var a, b: TLock[2]
-  var x: TLock[1]
-  # invalid locking order: TLock[1] cannot be acquired before TLock[2]:
-  {.locks: [x].}:
-    {.locks: [a].}:
-      ...
-  # valid locking order: TLock[2] acquired before TLock[1]:
-  {.locks: [a].}:
-    {.locks: [x].}:
-      ...
-
-  # invalid locking order: TLock[2] acquired before TLock[2]:
-  {.locks: [a].}:
-    {.locks: [b].}:
-      ...
-
-  # valid locking order, locks of the same level acquired at the same time:
-  {.locks: [a, b].}:
-    ...
-
-
-Here is how a typical multilock statement can be implemented in Nim. Note how
-the runtime check is required to ensure a global ordering for two locks `a`
-and `b` of the same lock level:
-
-.. code-block:: nim
-  template multilock(a, b: ptr TLock; body: untyped) =
-    if cast[ByteAddress](a) < cast[ByteAddress](b):
-      pthread_mutex_lock(a)
-      pthread_mutex_lock(b)
+  proc test(cond: bool) =
+    var s: seq[string]
+    if cond:
+      s = @["y"]
     else:
-      pthread_mutex_lock(b)
-      pthread_mutex_lock(a)
-    {.locks: [a, b].}:
-      try:
-        body
-      finally:
-        pthread_mutex_unlock(a)
-        pthread_mutex_unlock(b)
+      s = @[]
+    s.add "abc" # valid!
+  ```
+
+In this example every path does set `s` to a value before it is used.
+
+  ```nim
+  {.experimental: "strictDefs".}
+
+  proc test(cond: bool) =
+    let s: seq[string]
+    if cond:
+      s = @["y"]
+    else:
+      s = @[]
+  ```
+
+With `experimental: "strictDefs"`, `let` statements are allowed to not have an initial value, but every path should set `s` to a value before it is used.
 
 
-Whole routines can also be annotated with a `locks` pragma that takes a lock
-level. This then means that the routine may acquire locks of up to this level.
-This is essential so that procs can be called within a `locks` section:
-
-.. code-block:: nim
-  proc p() {.locks: 3.} = discard
-
-  var a: TLock[4]
-  {.locks: [a].}:
-    # p's locklevel (3) is strictly less than a's (4) so the call is allowed:
-    p()
-
-
-As usual `locks` is an inferred effect and there is a subtype
-relation: `proc () {.locks: N.}` is a subtype of `proc () {.locks: M.}`
-iff (M <= N).
-
-The `locks` pragma can also take the special value `"unknown"`. This
-is useful in the context of dynamic method dispatching. In the following
-example, the compiler can infer a lock level of 0 for the `base` case.
-However, one of the overloaded methods calls a procvar which is
-potentially locking. Thus, the lock level of calling `g.testMethod`
-cannot be inferred statically, leading to compiler warnings. By using
-`{.locks: "unknown".}`, the base method can be marked explicitly as
-having unknown lock level as well:
-
-.. code-block:: nim
-  type SomeBase* = ref object of RootObj
-  type SomeDerived* = ref object of SomeBase
-    memberProc*: proc ()
-
-  method testMethod(g: SomeBase) {.base, locks: "unknown".} = discard
-  method testMethod(g: SomeDerived) =
-    if g.memberProc != nil:
-      g.memberProc()
-
-
-noRewrite pragma
+`out` parameters
 ----------------
 
-Term rewriting macros and templates are currently greedy and
-they will rewrite as long as there is a match.
-There was no way to ensure some rewrite happens only once,
-e.g. when rewriting term to same term plus extra content.
+An `out` parameter is like a `var` parameter but it must be written to before it can be used:
 
-`noRewrite` pragma can actually prevent further rewriting on marked code,
-e.g. with given example `echo("ab")` will be rewritten just once:
+  ```nim
+  proc myopen(f: out File; name: string): bool =
+    f = default(File)
+    result = open(f, name)
+  ```
 
-.. code-block:: nim
-  template pwnEcho{echo(x)}(x: untyped) =
-    {.noRewrite.}: echo("pwned!")
+While it is usually the better style to use the return type in order to return results API and ABI
+considerations might make this infeasible. Like for `var T` Nim maps `out T` to a hidden pointer.
+For example POSIX's `stat` routine can be wrapped as:
 
-  echo "ab"
+  ```nim
+  proc stat*(a1: cstring, a2: out Stat): cint {.importc, header: "".}
+  ```
 
-`noRewrite` pragma can be useful to control term-rewriting macros recursion.
+When the implementation of a routine with output parameters is analysed, the compiler
+checks that every path before the (implicit or explicit) return does set every output
+parameter:
+
+  ```nim
+  proc p(x: out int; y: out string; cond: bool) =
+    x = 4
+    if cond:
+      y = "abc"
+    # error: not every path initializes 'y'
+  ```
 
 
-Aliasing restrictions in parameter passing
-==========================================
+Out parameters and exception handling
+-------------------------------------
 
-**Note**: The aliasing restrictions are currently not enforced by the
-implementation and need to be fleshed out further.
+The analysis should take exceptions into account (but currently does not):
 
-"Aliasing" here means that the underlying storage locations overlap in memory
-at runtime. An "output parameter" is a parameter of type `var T`,
-an input parameter is any parameter that is not of type `var`.
+  ```nim
+  proc p(x: out int; y: out string; cond: bool) =
+    x = canRaise(45)
+    y = "abc" # <-- error: not every path initializes 'y'
+  ```
 
-1. Two output parameters should never be aliased.
-2. An input and an output parameter should not be aliased.
-3. An output parameter should never be aliased with a global or thread local
-   variable referenced by the called proc.
-4. An input parameter should not be aliased with a global or thread local
-   variable updated by the called proc.
+Once the implementation takes exceptions into account it is easy enough to
+use `outParam = default(typeof(outParam))` in the beginning of the proc body.
 
-One problem with rules 3 and 4 is that they affect specific global or thread
-local variables, but Nim's effect tracking only tracks "uses no global variable"
-via `.noSideEffect`. The rules 3 and 4 can also be approximated by a different rule:
+Out parameters and inheritance
+------------------------------
 
-5. A global or thread local variable (or a location derived from such a location)
-   can only passed to a parameter of a `.noSideEffect` proc.
+It is not valid to pass an lvalue of a supertype to an `out T` parameter:
+
+  ```nim
+  type
+    Superclass = object of RootObj
+      a: int
+    Subclass = object of Superclass
+      s: string
+
+  proc init(x: out Superclass) =
+    x = Superclass(a: 8)
+
+  var v: Subclass
+  init v
+  use v.s # the 's' field was never initialized!
+  ```
+
+However, in the future this could be allowed and provide a better way to write object
+constructors that take inheritance into account.
 
 
-Noalias annotation
-==================
-
-Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables
-and parameters. It is mapped directly to C/C++'s `restrict` keyword and means that
-the underlying pointer is pointing to a unique location in memory, no other aliases to
-this location exist. It is *unchecked* that this alias restriction is followed, if the
-restriction is violated, the backend optimizer is free to miscompile the code.
-This is an **unsafe** language feature.
-
-Ideally in later versions of the language, the restriction will be enforced at
-compile time. (Which is also why the name `noalias` was choosen instead of a more
-verbose name like `unsafeAssumeNoAlias`.)
+**Note**: The implementation of "strict definitions" and "out parameters" is experimental but the concept
+is solid and it is expected that eventually this mode becomes the default in later versions.
 
 
-Strict funcs
-============
+Strict case objects
+===================
 
-Since version 1.4 a stricter definition of "side effect" is available. In addition
-to the existing rule that a side effect is calling a function with side effects
-the following rule is also enforced:
+With `experimental: "strictCaseObjects"` *every* field access is checked to be valid at compile-time.
+The field is within a `case` section of an `object`.
 
-Any mutation to an object does count as a side effect if that object is reachable
-via a parameter that is not declared as a `var` parameter.
-
-For example:
-
-.. code-block:: nim
-
-  {.experimental: "strictFuncs".}
+  ```nim
+  {.experimental: "strictCaseObjects".}
 
   type
-    Node = ref object
-      le, ri: Node
-      data: string
+    Foo = object
+      case b: bool
+      of false:
+        s: string
+      of true:
+        x: int
 
-  func len(n: Node): int =
-    # valid: len does not have side effects
-    var it = n
-    while it != nil:
-      inc result
-      it = it.ri
+  var x = Foo(b: true, x: 4)
+  case x.b
+  of true:
+    echo x.x # valid
+  of false:
+    echo "no"
 
-  func mut(n: Node) =
-    let m = n # is the statement that connected the mutation to the parameter
-    m.data = "yeah" # the mutation is here
-    # Error: 'mut' can have side effects
-    # an object reachable from 'n' is potentially mutated
+  case x.b
+  of false:
+    echo x.x # error: field access outside of valid case branch: x.x
+  of true:
+    echo "no"
+
+  ```
+
+**Note**: The implementation of "strict case objects" is experimental but the concept
+is solid and it is expected that eventually this mode becomes the default in later versions.
 
 
-The algorithm behind this analysis is described in
-the `view types section <#view-types-algorithm>`_.
+Quirky routines
+===============
+
+The default code generation strategy of exceptions under the ARC/ORC model is the so called
+`--exceptions:goto` implementation. This implementation inserts a check after every call that
+can potentially raise an exception. A typical instruction sequence for this on
+for a x86 64 bit machine looks like:
+
+  ```
+  cmp DWORD PTR [rbx], 0
+  je  .L1
+  ```
+
+This is a memory fetch followed by jump. (An ideal implementation would
+use the carry flag and a single instruction like ``jc .L1``.)
+
+This overhead might not be desired and depending on the semantics of the routine may not be required
+either.
+So it can be disabled via a `.quirky` annotation:
+
+  ```nim
+  proc wontRaise(x: int) {.quirky.} =
+    if x != 0:
+      # because of `quirky` this will continue even if `write` raised an IO exception:
+      write x
+      wontRaise(x-1)
+
+  wontRaise 10
+
+  ```
+
+If the used exception model is not `--exceptions:goto` then the `quirky` pragma has no effect and is
+ignored.
+
+The `quirky` pragma can also be be pushed in order to affect a group of routines and whether
+the compiler supports the pragma can be checked with `defined(nimHasQuirky)`:
+
+  ```nim
+  when defined(nimHasQuirky):
+    {.push quirky: on.}
+
+  proc doRaise() = raise newException(ValueError, "")
+
+  proc f(): string = "abc"
+
+  proc q(cond: bool) =
+    if cond:
+      doRaise()
+    echo f()
+
+  q(true)
+
+  when defined(nimHasQuirky):
+    {.pop.}
+  ```
+
+**Warning**: The `quirky` pragma only affects code generation, no check for validity is performed!
 
 
-View types
-==========
+Threading under ARC/ORC
+=======================
 
-**Note**:  `--experimental:views` is more effective
-with `--experimental:strictFuncs`.
-
-A view type is a type that is or contains one of the following types:
-
-- `var T` (mutable view into `T`)
-- `lent T` (immutable view into `T`)
-- `openArray[T]` (pair of (pointer to array of `T`, size))
-
-For example:
-
-.. code-block:: nim
-
-  type
-    View1 = var int
-    View2 = openArray[byte]
-    View3 = lent string
-    View4 = Table[openArray[char], int]
+ARC/ORC supports a shared heap out of the box. This means that messages can be sent between
+threads without copies. However, without copying the data there is an inherent danger of
+data races. Data races are prevented at compile-time if it is enforced that
+only **isolated** subgraphs can be sent around.
 
 
-Exceptions to this rule are types constructed via `ptr` or `proc`.
-For example, the following types are **not** view types:
+Isolation
+---------
 
-.. code-block:: nim
+The standard library module `isolation.nim` provides a generic type `Isolated[T]` that
+captures the important notion that nothing else can reference the graph that is wrapped
+inside `Isolated[T]`. It is what a channel implementation should use in order to enforce
+the freedom of data races:
 
-  type
-    NotView1 = proc (x: openArray[int])
-    NotView2 = ptr openArray[char]
-    NotView3 = ptr array[4, var int]
+  ```nim
+  proc send*[T](c: var Channel[T]; msg: sink Isolated[T])
+  proc recv*[T](c: var Channel[T]): T
+    ## Note: Returns T, not Isolated[T] for convenience.
+
+  proc recvIso*[T](c: var Channel[T]): Isolated[T]
+    ## remembers the data is Isolated[T].
+  ```
+
+In order to create an `Isolated` graph one has to use either `isolate` or `unsafeIsolate`.
+`unsafeIsolate` is as its name says unsafe and no checking is performed. It should be considered
+to be as dangerous as a `cast` operation.
 
 
-A *mutable* view type is a type that is or contains a `var T` type.
-An *immutable* view type is a view type that is not a mutable view type.
+Construction must ensure that the invariant holds, namely that the wrapped `T`
+is free of external aliases into it. `isolate` ensures this invariant. It is
+inspired by Pony's `recover` construct:
 
-A *view* is a symbol (a let, var, const, etc.) that has a view type.
-
-Since version 1.4 Nim allows view types to be used as local variables.
-This feature needs to be enabled via `{.experimental: "views".}`.
-
-A local variable of a view type *borrows* from the locations and
-it is statically enforced that the view does not outlive the location
-it was borrowed from.
-
-For example:
-
-.. code-block:: nim
-
-  {.experimental: "views".}
-
-  proc take(a: openArray[int]) =
-    echo a.len
-
-  proc main(s: seq[int]) =
-    var x: openArray[int] = s # 'x' is a view into 's'
-    # it is checked that 'x' does not outlive 's' and
-    # that 's' is not mutated.
-    for i in 0 .. high(x):
-      echo x[i]
-    take(x)
-
-    take(x.toOpenArray(0, 1)) # slicing remains possible
-    let y = x  # create a view from a view
-    take y
-    # it is checked that 'y' does not outlive 'x' and
-    # that 'x' is not mutated as long as 'y' lives.
+  ```nim
+  func isolate(x: sink T): Isolated[T] {.magic: "Isolate".}
+  ```
 
 
-  main(@[11, 22, 33])
+As you can see, this is a new builtin because the check it performs on `x` is non-trivial:
+
+If `T` does not contain a `ref` or `closure` type, it is isolated. Else the syntactic
+structure of `x` is analyzed:
+
+- Literals like `nil`, `4`, `"abc"` are isolated.
+- A local variable or a routine parameter is isolated if either of these conditions is true:
+  1. Its type is annotated with the `.sendable` pragma. Note `Isolated[T]` is annotated as
+     `.sendable`.
+  2. Its type contains the potentially dangerous `ref` and `proc {.closure}` types
+     only in places that are protected via a `.sendable` container.
+
+- An array constructor `[x...]` is isolated if every element `x` is isolated.
+- An object constructor `Obj(x...)` is isolated if every element `x` is isolated.
+- An `if` or `case` expression is isolated if all possible values the expression
+  may return are isolated.
+- A type conversion `C(x)` is isolated if `x` is isolated. Analogous for `cast`
+  expressions.
+- A function call `f(x...)` is isolated if `f` is `.noSideEffect` and for every argument `x`:
+  - `x` is isolated **or**
+  - `f`'s return type cannot *alias* `x`'s type. This is checked via a form of alias analysis as explained in the next paragraph.
 
 
-A local variable of a view type can borrow from a location
-derived from a parameter, another local variable, a global `const` or `let`
-symbol or a thread-local `var` or `let`.
 
-Let `p` the proc that is analysed for the correctness of the borrow operation.
+Alias analysis
+--------------
 
-Let `source` be one of:
+We start with an important, simple case that must be valid: Sending the result
+of `parseJson` to a channel. Since the signature
+is `func parseJson(input: string): JsonNode` it is easy to see that JsonNode
+can never simply be a view into `input` which is a `string`.
 
-- A formal parameter of `p`. Note that this does not cover parameters of
-  inner procs.
-- The `result` symbol of `p`.
-- A local `var` or `let` or `const` of `p`. Note that this does
-  not cover locals of inner procs.
-- A thread-local `var` or `let`.
-- A global `let` or `const`.
-- A constant array/seq/object/tuple constructor.
+A different case is the identity function `id`, `send id(myJsonGraph)` must be
+invalid because we do not know how many aliases into `myJsonGraph` exist
+elsewhere.
+
+In general type `A` can alias type `T` if:
+
+- `A` and `T` are the same types.
+- `A` is a distinct type derived from `T`.
+- `A` is a field inside `T` if `T` is a final object type.
+- `T` is an inheritable object type. (An inherited type could always contain
+  a `field: A`).
+- `T` is a closure type. Reason: `T`'s environment can contain a field of
+  type `A`.
+- `A` is the element type of `T` if `T` is an array, sequence or pointer type.
 
 
-Path expressions
-----------------
-
-A location derived from `source` is then defined as a path expression that
-has `source` as the owner. A path expression `e` is defined recursively:
-
-- `source` itself is a path expression.
-- Container access like `e[i]` is a path expression.
-- Tuple access `e[0]` is a path expression.
-- Object field access `e.field` is a path expression.
-- `system.toOpenArray(e, ...)` is a path expression.
-- Pointer dereference `e[]` is a path expression.
-- An address `addr e`, `unsafeAddr e` is a path expression.
-- A type conversion `T(e)` is a path expression.
-- A cast expression `cast[T](e)` is a path expression.
-- `f(e, ...)` is a path expression if `f`'s return type is a view type.
-  Because the view can only have been borrowed from `e`, we then know
-  that owner of `f(e, ...)` is `e`.
 
 
-If a view type is used as a return type, the location must borrow from a location
-that is derived from the first parameter that is passed to the proc.
-See https://nim-lang.org/docs/manual.html#procedures-var-return-type for
-details about how this is done for `var T`.
-
-A mutable view can borrow from a mutable location, an immutable view can borrow
-from both a mutable or an immutable location.
-
-The *duration* of a borrow is the span of commands beginning from the assignment
-to the view and ending with the last usage of the view.
-
-For the duration of the borrow operation, no mutations to the borrowed locations
-may be performed except via the potentially mutable view that borrowed from the
-location. The borrowed location is said to be *sealed* during the borrow.
-
-.. code-block:: nim
-
-  {.experimental: "views".}
-
-  type
-    Obj = object
-      field: string
-
-  proc dangerous(s: var seq[Obj]) =
-    let v: lent Obj = s[0] # seal 's'
-    s.setLen 0  # prevented at compile-time because 's' is sealed.
-    echo v.field
-
-
-The scope of the view does not matter:
-
-.. code-block:: nim
-
-  proc valid(s: var seq[Obj]) =
-    let v: lent Obj = s[0]  # begin of borrow
-    echo v.field            # end of borrow
-    s.setLen 0  # valid because 'v' isn't used afterwards
-
-
-The analysis requires as much precision about mutations as is reasonably obtainable,
-so it is more effective with the experimental `strict funcs <#strict-funcs>`_
-feature. In other words `--experimental:views` works better
-with `--experimental:strictFuncs`.
-
-The analysis is currently control flow insensitive:
-
-.. code-block:: nim
-
-  proc invalid(s: var seq[Obj]) =
-    let v: lent Obj = s[0]
-    if false:
-      s.setLen 0
-    echo v.field
-
-In this example, the compiler assumes that `s.setLen 0` invalidates the
-borrow operation of `v` even though a human being can easily see that it
-will never do that at runtime.
-
-
-Start of a borrow
------------------
-
-A borrow starts with one of the following:
-
-- The assignment of a non-view-type to a view-type.
-- The assignment of a location that is derived from a local parameter
-  to a view-type.
-
-
-End of a borrow
+Sendable pragma
 ---------------
 
-A borrow operation ends with the last usage of the view variable.
+A container type can be marked as `.sendable`. `.sendable` declares that the type
+encapsulates a `ref` type effectively so that a variable of this container type
+can be used in an `isolate` context:
+
+  ```nim
+  type
+    Isolated*[T] {.sendable.} = object ## Isolated data can only be moved, not copied.
+      value: T
+
+  proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.}
+
+  proc `=sink`*[T](dest: var Isolated[T]; src: Isolated[T]) {.inline.} =
+    # delegate to value's sink operation
+    `=sink`(dest.value, src.value)
+
+  proc `=destroy`*[T](dest: var Isolated[T]) {.inline.} =
+    # delegate to value's destroy operation
+    `=destroy`(dest.value)
+  ```
+
+The `.sendable` pragma itself is an experimenal, unchecked, unsafe annotation. It is
+currently only used by `Isolated[T]`.
+
+Virtual pragma
+==============
+
+`virtual` is designed to extend or create virtual functions when targeting the cpp backend. When a proc is marked with virtual, it forward declares the proc header within the type's body.
+
+Here's an example of how to use the virtual pragma:
+
+```nim
+proc newCpp*[T](): ptr T {.importcpp: "new '*0()".}
+type
+  Foo = object of RootObj
+  FooPtr = ptr Foo
+  Boo = object of Foo
+  BooPtr = ptr Boo
+
+proc salute(self: FooPtr) {.virtual.} =
+  echo "hello foo"
+
+proc salute(self: BooPtr) {.virtual.} =
+  echo "hello boo"
+
+let foo = newCpp[Foo]()
+let boo = newCpp[Boo]()
+let booAsFoo = cast[FooPtr](newCpp[Boo]())
+
+foo.salute() # prints hello foo
+boo.salute() # prints hello boo
+booAsFoo.salute() # prints hello boo
+```
+In this example, the `salute` function is virtual in both Foo and Boo types. This allows for polymorphism.
+
+The virtual pragma also supports a special syntax to express Cpp constraints. Here's how it works:
+
+`$1` refers to the function name
+`'idx` refers to the type of the argument at the position idx. Where idx = 1 is the `this` argument.
+`#idx` refers to the argument name.
+
+The return type can be referred to as `-> '0`, but this is optional and often not needed.
+
+ ```nim
+ {.emit:"""/*TYPESECTION*/
+#include 
+  class CppPrinter {
+  public:
+
+    virtual void printConst(char* message) const {
+        std::cout << "Const Message: " << message << std::endl;
+    }
+    virtual void printConstRef(char* message, const int& flag) const {
+        std::cout << "Const Ref Message: " << message << std::endl;
+    }
+};
+""".}
+
+type
+  CppPrinter {.importcpp, inheritable.} = object
+  NimPrinter {.exportc.} = object of CppPrinter
+
+proc printConst(self: CppPrinter; message:cstring) {.importcpp.}
+CppPrinter().printConst(message)
+
+# override is optional.
+proc printConst(self: NimPrinter; message: cstring) {.virtual: "$1('2 #2) const override".} =
+  echo "NimPrinter: " & $message
+
+proc printConstRef(self: NimPrinter; message: cstring; flag:int32) {.virtual: "$1('2 #2, const '3& #3 ) const override".} =
+  echo "NimPrinterConstRef: " & $message
+
+NimPrinter().printConst(message)
+var val: int32 = 10
+NimPrinter().printConstRef(message, val)
+
+```
+
+Constructor pragma
+==================
+
+The `constructor` pragma can be used in two ways: in conjunction with `importcpp` to import a C++ constructor, and to declare constructors that operate similarly to `virtual`.
+
+Consider:
+
+```nim
+type Foo* = object
+  x: int32
+
+proc makeFoo(x: int32): Foo {.constructor.} =
+  result.x = x
+```
+
+It forward declares the constructor in the type definition. When the constructor has parameters, it also generates a default constructor. One can avoid this behaviour by using `noDecl` in a default constructor.
+
+Like `virtual`, `constructor` also supports a syntax that allows to express C++ constraints.
+
+For example:
+
+```nim
+{.emit:"""/*TYPESECTION*/
+struct CppClass {
+  int x;
+  int y;
+  CppClass(int inX, int inY) {
+    this->x = inX;
+    this->y = inY;
+  }
+  //CppClass() = default;
+};
+""".}
+
+type
+  CppClass* {.importcpp, inheritable.} = object
+    x: int32
+    y: int32
+  NimClass* = object of CppClass
+
+proc makeNimClass(x: int32): NimClass {.constructor:"NimClass('1 #1) : CppClass(0, #1)".} =
+  result.x = x
+
+# Optional: define the default constructor explicitly
+proc makeCppClass(): NimClass {.constructor: "NimClass() : CppClass(0, 0)".} =
+  result.x = 1
+```
+
+In the example above `CppClass` has a deleted default constructor. Notice how by using the constructor syntax, one can call the appropriate constructor.
+
+Notice when calling a constructor in the section of a global variable initialization, it will be called before `NimMain` meaning Nim is not fully initialized.
+
+Constructor Initializer
+=======================
+
+By default Nim initializes `importcpp` types with `{}`. This can be problematic when importing
+types with a deleted default constructor. In order to avoid this, one can specify default values for a constructor by specifying default values for the proc params in the `constructor` proc.
+
+For example:
+
+```nim
+
+{.emit: """/*TYPESECTION*/
+struct CppStruct {
+  CppStruct(int x, char* y): x(x), y(y){}
+  int x;
+  char* y;
+};
+""".}
+type
+  CppStruct {.importcpp, inheritable.} = object
+
+proc makeCppStruct(a: cint = 5, b:cstring = "hello"): CppStruct {.importcpp: "CppStruct(@)", constructor.}
+
+(proc (s: CppStruct) = echo "hello")(makeCppStruct()) 
+# If one removes a default value from the constructor and passes it to the call explicitly, the C++ compiler will complain.
+
+```
+Skip initializers in fields members
+===================================
+
+By using `noInit` in a type or field declaration, the compiler will skip the initializer. By doing so one can explicitly initialize those values in the constructor of the type owner.
+
+For example:
+
+```nim
+
+{.emit: """/*TYPESECTION*/
+  struct Foo {
+    Foo(int a){};
+  };
+  struct Boo {
+    Boo(int a){};
+  };
+
+  """.}
+
+type 
+  Foo {.importcpp.} = object
+  Boo {.importcpp, noInit.} = object
+  Test {.exportc.} = object
+    foo {.noInit.}: Foo
+    boo: Boo
+
+proc makeTest(): Test {.constructor: "Test() : foo(10), boo(1)".} = 
+  discard
+
+proc main() = 
+  var t = makeTest()
+
+main()
+
+```
+
+Will produce: 
+
+```cpp
+
+struct Test {
+	Foo foo; 
+	Boo boo;
+  N_LIB_PRIVATE N_NOCONV(, Test)(void);
+};
+
+```
+
+Notice that without `noInit` it would produce `Foo foo {}` and `Boo boo {}`
 
 
-Reborrows
----------
+Member pragma
+=============
 
-A view `v` can borrow from multiple different locations. However, the borrow
-is always the full span of `v`'s lifetime and every location that is borrowed
-from is sealed during `v`'s lifetime.
+Like the `constructor` and `virtual` pragmas, the `member` pragma can be used to attach a procedure to a C++ type. It's more flexible than the `virtual` pragma in the sense that it accepts not only names but also operators and destructors.
+
+For example:
+
+```nim
+proc print(s: cstring) {.importcpp: "printf(@)", header: "".}
+
+type
+  Doo {.exportc.} = object
+    test: int
+
+proc memberProc(f: Doo) {.member.} = 
+  echo $f.test
+
+proc destructor(f: Doo) {.member: "~'1()", used.} = 
+  print "destructing\n"
+
+proc `==`(self, other: Doo): bool {.member: "operator==('2 const & #2) const -> '0".} = 
+  self.test == other.test
+
+let doo = Doo(test: 2)
+doo.memberProc()
+echo doo == Doo(test: 1)
+
+```
+
+Will print:
+```
+2
+false
+destructing
+destructing
+```
+
+Notice how the C++ destructor is called automatically. Also notice the double implementation of `==` as an operator in Nim but also in C++. This is useful if you need the type to match some C++ `concept` or `trait` when interoping. 
+
+A side effect of being able to declare C++ operators, is that you can now also create a
+C++ functor to have seamless interop with C++ lambdas (syntactic sugar for functors).
+
+For example:
+
+```nim
+type
+  NimFunctor = object
+    discard
+proc invoke(f: NimFunctor; n: int) {.member: "operator ()('2 #2)".} = 
+  echo "FunctorSupport!"
+
+{.experimental: "callOperator".}
+proc `()`(f: NimFunctor; n:int) {.importcpp: "#(@)" .} 
+NimFunctor()(1)
+```
+Notice we use the overload of `()` to have the same semantics in Nim, but on the `importcpp` we import the functor as a function. 
+This allows to easy interop with functions that accepts for example a `const` operator in its signature. 
 
 
-Algorithm
----------
+Injected symbols in generic procs
+=================================
 
-The following section is an outline of the algorithm that the current implementation
-uses. The algorithm performs two traversals over the AST of the procedure or global
-section of code that uses a view variable. No fixpoint iterations are performed, the
-complexity of the analysis is O(N) where N is the number of nodes of the AST.
+With the experimental option `genericsOpenSym`, captured symbols in generic
+routine bodies may be replaced by symbols injected locally by templates/macros
+at instantiation time. `bind` may be used to keep the captured symbols over
+the injected ones regardless of enabling the option.
+  
+Since this change may affect runtime behavior, the experimental switch
+`genericsOpenSym` needs to be enabled, and a warning is given in the case
+where an injected symbol would replace a captured symbol not bound by `bind`
+and the experimental switch isn't enabled.
 
-The first pass over the AST computes the lifetime of each local variable based on
-a notion of an "abstract time", in the implementation it's a simple integer that is
-incremented for every visited node.
+```nim
+const value = "captured"
+template foo(x: int, body: untyped) =
+  let value {.inject.} = "injected"
+  body
 
-In the second pass information about the underlying object "graphs" is computed.
-Let `v` be a parameter or a local variable. Let `G(v)` be the graph
-that `v` belongs to. A graph is defined by the set of variables that belong
-to the graph. Initially for all `v`: `G(v) = {v}`. Every variable can only
-be part of a single graph.
+proc old[T](): string =
+  foo(123):
+    return value # warning: a new `value` has been injected, use `bind` or turn on `experimental:genericsOpenSym`
+echo old[int]() # "captured"
 
-Assignments like `a = b` "connect" two variables, both variables end up in the
-same graph `{a, b} = G(a) = G(b)`. Unfortunately, the pattern to look for is
-much more complex than that and can involve multiple assignment targets
-and sources::
+{.experimental: "genericsOpenSym".}
 
-  f(x, y) = g(a, b)
+proc bar[T](): string =
+  foo(123):
+    return value
+assert bar[int]() == "injected" # previously it would be "captured"
 
-connects `x` and `y` to `a` and `b`: `G(x) = G(y) = G(a) = G(b) = {x, y, a, b}`.
-A type based alias analysis rules out some of these combinations, for example
-a `string` value cannot possibly be connected to a `seq[int]`.
+proc baz[T](): string =
+  bind value
+  foo(123):
+    return value
+assert baz[int]() == "captured"
+```
 
-A pattern like `v[] = value` or `v.field = value` marks `G(v)` as mutated.
-After the second pass a set of disjoint graphs was computed.
 
-For strict functions it is then enforced that there is no graph that is both mutated
-and has an element that is an immutable parameter (that is a parameter that is not
-of type `var T`).
+VTable for methods
+==================
 
-For borrow checking a different set of checks is performed. Let `v` be the view
-and `b` the location that is borrowed from.
+Methods now support implementations based on a VTable by using `--experimental:vtables`. Note that the option needs to enabled
+globally. The virtual method table is stored in the type info of
+an object, which is an array of function pointers.
 
-- The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of
-  a parameter is the complete proc body.
-- If `v` is a mutable view and `v` is used to actually mutate the
-  borrowed location, then `b` has to be a mutable location.
-  Note: If it is not actually used for mutation, borrowing a mutable view from an
-  immutable location is allowed! This allows for many important idioms and will be
-  justified in an upcoming RFC.
-- During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if
-  `v` is a mutable view).
-- If `v` is `result` then `b` has to be a location derived from the first
-  formal parameter or from a constant location.
-- A view cannot be used for a read or a write access before it was assigned to.
+```nim
+method foo(x: Base, ...) {.base.}
+method foo(x: Derived, ...) {.base.}
+```
+
+It roughly generates a dispatcher like
+
+```nim
+proc foo_dispatch(x: Base, ...) =
+  x.typeinfo.vtable[method_index](x, ...) # method_index is the index of the sorted order of a method
+```
+
+Methods are required to be in the same module where their type has been defined.
+
+```nim
+# types.nim
+type
+  Base* = ref object
+```
+
+```nim
+import types
+
+method foo(x: Base) {.base.} = discard
+```
+
+It gives an error: method `foo` can be defined only in the same module with its type (Base).
+
+
+asmSyntax pragma
+================
+
+The `asmSyntax` pragma is used to specify target inline assembler syntax in an `asm` statement.
+
+It prevents compiling code with different of the target CC inline asm syntax, i.e. it will not allow gcc inline asm code to be compiled with vcc.
+
+```nim
+proc nothing() =
+  asm {.asmSyntax: "gcc".}"""
+    nop
+  """
+```
+
+The current C(C++) backend implementation cannot generate code for gcc and for vcc at the same time. For example, `{.asmSyntax: "vcc".}` with the ICC compiler will not generate code with intel asm syntax, even though ICC can use both gcc-like and vcc-like asm.
diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.md
similarity index 73%
rename from doc/manual_experimental_strictnotnil.rst
rename to doc/manual_experimental_strictnotnil.md
index c7c045683a..a6fa6cda8b 100644
--- a/doc/manual_experimental_strictnotnil.rst
+++ b/doc/manual_experimental_strictnotnil.md
@@ -1,17 +1,20 @@
-.. default-role:: code
-
 Strict not nil checking
 =========================
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 **Note:** This feature is experimental, you need to enable it with
 
-.. code-block:: nim
+  ```nim
   {.experimental: "strictNotNil".}
+  ```
 
 or 
 
-.. code-block:: bash
+  ```cmd
   nim c --experimental:strictNotNil 
+  ```
 
 In the second case it would check builtin and imported modules as well.
 
@@ -38,7 +41,7 @@ not nil
 
 You can annotate a type where nil isn't a valid value with `not nil`.
 
-.. code-block:: nim
+  ```nim
     type
       NilableObject = ref object
         a: int
@@ -56,32 +59,40 @@ You can annotate a type where nil isn't a valid value with `not nil`.
       p(x)
     else:
       p(x) # ok
+  ```
 
 
 
 If a type can include `nil` as a valid value, dereferencing values of the type
-is checked by the compiler: if a value which might be nil is derefenced, this produces a warning by default, you can turn this into an error using the compiler options `--warningAsError:strictNotNil`
+is checked by the compiler: if a value which might be nil is dereferenced, this
+produces a warning by default, you can turn this into an error using
+the compiler options `--warningAsError:strictNotNil`:option:.
 
 If a type is nilable, you should dereference its values only after a `isNil` or equivalent check.
 
 local turn on/off
 ---------------------
 
-You can still turn off nil checking on function/module level by using a `{.strictNotNil: off}.` pragma.
-Note: test that/TODO for code/manual.
+You can still turn off nil checking on function/module level by using a `{.strictNotNil: off.}` pragma.
+
+..
+  Note: test that/TODO for code/manual.
 
 nilability state
 -----------------
 
-Currently a nilable value can be `Safe`, `MaybeNil` or `Nil` : we use internally `Parent` and `Unreachable` but this is an implementation detail(a parent layer has the actual nilability).
+Currently, a nilable value can be `Safe`, `MaybeNil` or `Nil` : we use internally `Parent` and `Unreachable` but this is an implementation detail(a parent layer has the actual nilability).
 
-`Safe` means it shouldn't be nil at that point: e.g. after assignment to a non-nil value or `not a.isNil` check
-`MaybeNil` means it might be nil, but it might not be nil: e.g. an argument, a call argument or a value after an `if` and `else`.
-`Nil` means it should be nil at that point; e.g. after an assignment to `nil` or a `.isNil` check.
+- `Safe` means it shouldn't be nil at that point: e.g. after assignment to
+  a non-nil value or `not a.isNil` check
+- `MaybeNil` means it might be nil, but it might not be nil: e.g. an argument,
+  a call argument or a value after an `if` and `else`.
+- `Nil` means it should be nil at that point; e.g. after an assignment to
+  `nil` or a `.isNil` check.
+- `Unreachable` means it shouldn't be possible to access this in this branch:
+  so we do generate a warning as well.
 
-`Unreachable` means it shouldn't be possible to access this in this branch: so we do generate a warning as well.
-
-We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc) which is of a tracked expression which is
+We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc.) which is of a tracked expression which is
 in `MaybeNil` or `Nil` state.
 
 
@@ -92,7 +103,8 @@ Types are either nilable or non-nilable.
 When you pass a param or a default value, we use the type : for nilable types we return `MaybeNil`
 and for non-nilable `Safe`.
 
-TODO: fix the manual here. (This is not great, as default values for non-nilables and nilables are usually actually `nil` , so we should think a bit more about this section.)
+..
+  TODO: fix the manual here. (This is not great, as default values for non-nilables and nilables are usually actually `nil` , so we should think a bit more about this section.)
 
 params rules
 ------------
@@ -112,38 +124,41 @@ call args rules
 
 When we call with arguments, we have two cases when we might change the nilability.
 
-.. code-block:: nim
+  ```nim
   callByVar(a)
+  ```
 
 Here `callByVar` can re-assign `a`, so this might change `a`'s nilability, so we change it to `MaybeNil`.
 This is also a possible aliasing `move out` (moving out of a current alias set).
 
-.. code-block:: nim
+  ```nim
   call(a)
+  ```
 
-Here `call` can change a field or element of `a`, so if we have a dependant expression of `a` : e.g. `a.field`. Dependats become `MaybeNil`.
+Here `call` can change a field or element of `a`, so if we have a dependant expression of `a` : e.g. `a.field`. Dependants become `MaybeNil`.
 
 
 branches rules
 ---------------
 
 Branches are the reason we do nil checking like this: with flow checking. 
-Sources of brancing are `if`, `while`, `for`, `and`, `or`, `case`, `try` and combinations with `return`, `break`, `continue` and `raise`
+Sources of branching are `if`, `while`, `for`, `and`, `or`, `case`, `try` and combinations with `return`, `break`, `continue` and `raise`
 
 We create a new layer/"scope" for each branch where we map expressions to nilability. This happens when we "fork": usually on the beginning of a construct.
 When branches "join" we usually unify their expression maps or/and nilabilities.
 
 Merging usually merges maps and alias sets: nilabilities are merged like this:
 
-.. code-block:: nim
+  ```nim
   template union(l: Nilability, r: Nilability): Nilability =
     ## unify two states
     if l == r:
       l
     else:
       MaybeNil
+  ```
 
-Special handling is for `.isNil` and ` == nil`, also for `not`, `and` and `or`.
+Special handling is for `.isNil` and `== nil`, also for `not`, `and` and `or`.
 
 `not` reverses the nilability, `and` is similar to "forking" : the right expression is checked in the layer resulting from the left one and `or` is similar to "merging": the right and left expression should be both checked in the original layer.
 
@@ -158,7 +173,7 @@ We want to track also field(dot) and index(bracket) expressions.
 We track some of those compound expressions which might be nilable as dependants of their bases: `a.field` is changed if `a` is moved (re-assigned), 
 similarly `a[index]` is dependent on `a` and `a.field.field` on `a.field`.
 
-When we move the base, we update dependants to `MaybeNil`. Otherwise we usually start with type nilability.
+When we move the base, we update dependants to `MaybeNil`. Otherwise, we usually start with type nilability.
 
 When we call args, we update the nilability of their dependants to `MaybeNil` as the calls usually can change them.
 We might need to check for `strictFuncs` pure funcs and not do that then.
@@ -166,10 +181,10 @@ We might need to check for `strictFuncs` pure funcs and not do that then.
 For field expressions `a.field`, we calculate an integer value based on a hash of the tree and just accept equivalent trees as equivalent expressions.
 
 For item expression `a[index]`, we also calculate an integer value based on a hash of the tree and accept equivalent trees as equivalent expressions: for static values only.
-For now we support only constant indices: we dont track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the `{.warning[StrictCheckNotNil]:off}.`.
+For now, we support only constant indices: we don't track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the `{.warning[StrictNotNil]:off.}`.
 
 For bracket expressions, in the future we might count `a[]` as the same general expression.
-This means we should should the index but otherwise handle it the same for assign (maybe "aliasing" all the non-static elements) and differentiate only for static: e.g. `a[0]` and `a[1]`.
+This means we should the index but otherwise handle it the same for assign (maybe "aliasing" all the non-static elements) and differentiate only for static: e.g. `a[0]` and `a[1]`.
 
 element tracking
 -----------------
@@ -177,8 +192,9 @@ element tracking
 When we assign an object construction, we should track the fields as well: 
 
 
-.. code-block:: nim
+  ```nim
   var a = Nilable(field: Nilable()) # a : Safe, a.field: Safe
+  ```
 
 Usually we just track the result of an expression: probably this should apply for elements in other cases as well.
 Also related to tracking initialization of expressions/fields.
@@ -187,14 +203,15 @@ unstructured control flow rules
 -------------------------------
 
 Unstructured control flow keywords as `return`, `break`, `continue`, `raise` mean that we jump from a branch out.
-This means that if there is code after the finishing of the branch, it would be ran if one hasn't hit the direct parent branch of those: so it is similar to an `else`. In those cases we should use the reverse nilabilities for the local to the condition expressions. E.g.
+This means that if there is code after the finishing of the branch, it would be run if one hasn't hit the direct parent branch of those: so it is similar to an `else`. In those cases we should use the reverse nilabilities for the local to the condition expressions. E.g.
 
-.. code-block:: nim
+  ```nim
   for a in c:
     if not a.isNil:
       b()
       break
     code # here a: Nil , because if not, we would have breaked
+  ```
 
 
 aliasing
@@ -208,28 +225,30 @@ Assignments and other changes to nilability can move / move out expressions of s
 `move`: Moving `left` to `right` means we remove `left` from its current set and unify it with the `right`'s set.
 This means it stops being aliased with its previous aliases.
 
-.. code-block:: nim
+  ```nim
   var left = b
   left = right # moving left to right
+  ```
 
 `move out`: Moving out `left` might remove it from the current set and ensure that it's in its own set as a single element.
 e.g.
 
 
-.. code-block:: nim
+  ```nim
   var left = b
   left = nil # moving out
+  ```
 
+..
+  initialization of non nilable and nilable values
+  -------------------------------------------------
 
-initialization of non nilable and nilable values
--------------------------------------------------
-
-TODO
+  TODO
 
 warnings and errors
 ---------------------
 
-We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc) which is of a tracked expression which is
+We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc.) which is of a tracked expression which is
 in `MaybeNil` or `Nil` state.
 
 We might also show a history of the transitions and the reasons for them that might change the nilability of the expression.
diff --git a/doc/markdown_rst.md b/doc/markdown_rst.md
new file mode 100644
index 0000000000..c7977f75a7
--- /dev/null
+++ b/doc/markdown_rst.md
@@ -0,0 +1,349 @@
+==========================================
+Nim-flavored Markdown and reStructuredText
+==========================================
+
+:Author: Andrey Makarov
+:Version: |nimversion|
+
+.. default-role:: code
+.. include:: rstcommon.rst
+.. contents::
+
+.. importdoc:: docgen.md
+
+Both `Markdown`:idx: (md) and `reStructuredText`:idx: (RST) are markup
+languages whose goal is to typeset texts with complex structure,
+formatting and references using simple plaintext representation.
+
+Command line usage
+==================
+
+Usage (to convert Markdown into HTML):
+
+  ```cmd
+  nim md2html markdown_rst.md
+  ```
+
+Output:
+
+    You're reading it!
+
+The `md2tex`:option: command is invoked identically to `md2html`:option:,
+but outputs a ``.tex`` file instead of ``.html``.
+
+These tools embedded into Nim compiler; the compiler can output
+the result to HTML [^html] or Latex [^latex].
+
+[^html]: commands `nim doc`:cmd: for ``*.nim`` files and
+   `nim rst2html`:cmd: for ``*.rst`` files
+
+[^latex]: commands `nim doc2tex`:cmd: for ``*.nim`` and
+   `nim rst2tex`:cmd: for ``*.rst``.
+
+Full list of supported commands:
+
+===================   ======================   ============   ==============
+command               runs on...               input format   output format
+===================   ======================   ============   ==============
+`nim md2html`:cmd:    standalone md files      ``.md``        ``.html`` HTML
+`nim md2tex`:cmd:     same                     same           ``.tex`` LaTeX
+`nim rst2html`:cmd:   standalone rst files     ``.rst``       ``.html`` HTML
+`nim rst2tex`:cmd:    same                     same           ``.tex`` LaTeX
+`nim doc`:cmd:        documentation comments   ``.nim``       ``.html`` HTML
+`nim doc2tex`:cmd:    same                     same           ``.tex`` LaTeX
+`nim jsondoc`:cmd:    same                     same           ``.json`` JSON
+===================   ======================   ============   ==============
+
+
+Basic markup
+============
+
+If you are new to Markdown/RST please consider reading the following:
+
+1) [Markdown Basic Syntax]
+2) a long specification of Markdown: [CommonMark Spec]
+3) a short [quick introduction] to RST
+4) an [RST reference]: a comprehensive cheatsheet for RST
+5) a more formal 50-page [RST specification].
+
+Features
+--------
+
+A large subset is implemented with some [limitations] and
+[additional Nim-specific features].
+
+Supported common RST/Markdown features:
+
+* body elements
+  + sections
+  + transitions
+  + paragraphs
+  + bullet lists using \+, \*, \-
+  + enumerated lists using arabic numerals or alphabet
+    characters:  1. ... 2. ... *or* a. ... b. ... *or* A. ... B. ...
+  + footnotes (including manually numbered, auto-numbered, auto-numbered
+    with label, and auto-symbol footnotes) and citations
+  + field lists
+  + option lists
+  + quoted literal blocks
+  + line blocks
+  + simple tables
+  + directives (see official documentation in [RST directives list]):
+    - ``image``, ``figure`` for including images and videos
+    - ``code``
+    - ``contents`` (table of contents), ``container``, ``raw``
+    - ``include``
+    - admonitions: "attention", "caution", "danger", "error", "hint",
+      "important", "note", "tip", "warning", "admonition"
+    - substitution definitions: `replace` and `image`
+  + comments
+* inline markup
+  + *emphasis*, **strong emphasis**,
+    ``inline literals``, hyperlink references (including embedded URI),
+    substitution references, standalone hyperlinks,
+    internal links (inline and outline)
+  + \`interpreted text\` with roles ``:literal:``, ``:strong:``,
+    ``emphasis``, ``:sub:``/``:subscript:``, ``:sup:``/``:superscript:``
+    (see [RST roles list] for description).
+  + inline internal targets
+
+RST mode only features
+----------------------
+
++ RST syntax for definition lists (that is additional indentation after
+  a definition line)
++ indented literal blocks starting from ``::``
+
+Markdown-specific features
+--------------------------
+
+* Markdown tables
+* Markdown code blocks. For them the same additional arguments as for RST
+  code blocks can be provided (e.g. `test` or `number-lines`) but with
+  a one-line syntax like this:
+
+      ```nim test number-lines=10
+      echo "ok"
+      ```
+* Markdown links ``[...](...)``
+* Pandoc syntax for automatic links ``[...]``, see [Referencing] for description
+* Pandoc syntax for footnotes, including ``[^10]`` (manually numbered)
+  and ``[^someName]`` (auto-numbered with a label)
+* Markdown literal blocks indented by 4 or more spaces
+* Markdown headlines
+* Markdown block quotes
+* Markdown syntax for definition lists
+* using ``1`` as auto-enumerator in enumerated lists like RST ``#``
+  (auto-enumerator ``1`` can not be used with ``#`` in the same list)
+
+Additional Nim-specific features
+--------------------------------
+
+* referencing to definitions in external files, see
+  [Markup external referencing] section
+* directives: ``code-block`` [^Sphinx], ``title``,
+  ``index`` [^Sphinx]
+* predefined roles
+  - ``:nim:`` (default), ``:c:`` (C programming language),
+    ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#).
+    That is every language that [highlite](highlite.html) supports.
+    They turn on appropriate syntax highlighting in inline code.
+
+    .. Note:: default role for Nim files is ``:nim:``,
+              for ``*.rst`` it's currently ``:literal:``.
+
+  - generic command line highlighting roles:
+    - ``:cmd:`` for commands and common shells syntax
+    - ``:console:`` the same  for interactive sessions
+      (commands should be prepended by ``$``)
+    - ``:program:`` for executable names [^Sphinx]
+      (one can just use ``:cmd:`` on single word)
+    - ``:option:`` for command line options [^Sphinx]
+  - ``:tok:``, a role for highlighting of programming language tokens
+* ***triple emphasis*** (bold and italic) using \*\*\*
+* ``:idx:`` role for \`interpreted text\` to include the link to this
+  text into an index (example: [Nim index]).
+* double slash `//` in option lists serves as a prefix for any option that
+  starts from a word (without any leading symbols like `-`, `--`, `/`):
+
+      //compile   compile the project
+      //doc       generate documentation
+
+  Here the dummy `//` will disappear, while options `compile`:option:
+  and `doc`:option: will be left in the final document.
+* emoji / smiley symbols
+
+[^Sphinx]: similar but different from the directives of
+   Python [Sphinx directives] and [Sphinx roles] extensions
+
+.. Note:: By default Nim has ``roSupportMarkdown`` and
+   ``roSupportRawDirective`` turned **on**.
+
+.. warning:: Using Nim-specific features can cause other Markdown and
+  RST implementations to fail on your document.
+
+Referencing
+===========
+
+To be able to copy and share links Nim generates anchors for all
+main document elements:
+
+* headlines (including document title)
+* footnotes
+* explicitly set anchors: RST internal cross-references and
+  inline internal targets
+* Nim symbols (external referencing), see [Nim DocGen Tools Guide] for details.
+
+But direct use of those anchors have 2 problems:
+
+1. the anchors are usually mangled (e.g. spaces substituted to minus
+   signs, etc).
+2. manual usage of anchors is not checked, so it's easy to get broken
+   links inside your project if e.g. spelling has changed for a heading
+   or you use a wrong relative path to your document.
+
+That's why Nim implementation has syntax for using
+*original* labels for referencing.
+Such referencing can be either local/internal or external:
+
+* Local referencing (inside any given file) is defined by
+  RST standard or Pandoc Markdown User guide.
+* External (cross-document) referencing is a Nim-specific feature,
+  though it's not really different from local referencing by its syntax.
+
+Markup local referencing
+------------------------
+
+There are 2 syntax option available for referencing to objects
+inside any given file, e.g. for headlines:
+
+    Markdown                  RST
+
+    Some headline             Some headline
+    =============             =============
+
+    Ref. [Some headline]      Ref. `Some headline`_
+
+
+Markup external referencing
+---------------------------
+
+The syntax is the same as for local referencing, but the anchors are
+saved in ``.idx`` files, so one needs to generate them beforehand,
+and they should be loaded by an `.. importdoc::` directive.
+E.g. if we want to reference section "Some headline" in ``file1.md``
+from ``file2.md``, then ``file2.md`` may look like:
+
+```
+.. importdoc:: file1.md
+
+Ref. [Some headline]
+```
+
+```cmd
+nim md2html --index:only file1.md  # creates ``htmldocs/file1.idx``
+nim md2html file2.md               # creates ``htmldocs/file2.html``
+```
+
+To allow cross-references between any files in any order (especially, if
+circular references are present), it's strongly recommended
+to make a run for creating all the indexes first:
+
+```cmd
+nim md2html --index:only file1.md  # creates ``htmldocs/file1.idx``
+nim md2html --index:only file2.md  # creates ``htmldocs/file2.idx``
+nim md2html file1.md               # creates ``htmldocs/file1.html``
+nim md2html file2.md               # creates ``htmldocs/file2.html``
+```
+
+and then one can freely reference any objects as if these 2 documents
+are actually 1 file.
+
+Other
+=====
+
+Idiosyncrasies
+--------------
+
+Currently we do **not** aim at 100% Markdown or RST compatibility in inline
+markup recognition rules because that would provide very little user value.
+This parser has 2 modes for inline markup:
+
+1) Markdown-like mode which is enabled by `roPreferMarkdown` option
+   (turned **on** by default).
+
+   .. Note:: RST features like directives are still turned **on**
+
+2) Compatibility mode which is RST rules.
+
+.. Note:: in both modes the parser interpretes text between single
+     backticks (code) identically:
+     backslash does not escape; the only exception: ``\`` folowed by `
+     does escape so that we can always input a single backtick ` in
+     inline code. However that makes impossible to input code with
+     ``\`` at the end in *single* backticks, one must use *double*
+     backticks:
+
+         `\`   -- WRONG
+         ``\`` -- GOOD
+         So single backticks can always be input: `\`` will turn to ` code
+
+.. Attention::
+   We don't support some obviously poor design choices of Markdown (or RST).
+
+   - no support for the rule of 2 spaces causing a line break in Markdown
+     (use RST "line blocks" syntax for making line breaks)
+
+   - interpretation of Markdown block quotes is also slightly different,
+     e.g. case
+
+         >>> foo
+         > bar
+         >>baz
+
+     is a single 3rd-level quote `foo bar baz` in original Markdown, while
+     in Nim we naturally see it as 3rd-level quote `foo` + 1st level `bar` +
+     2nd level `baz`:
+
+     >>> foo
+     > bar
+     >>baz
+
+Limitations
+-----------
+
+* no Unicode support in character width calculations
+* body elements
+  - no roman numerals in enumerated lists
+  - no doctest blocks
+  - no grid tables
+  - some directives are missing (check official [RST directives list]):
+    ``parsed-literal``, ``sidebar``, ``topic``, ``math``, ``rubric``,
+    ``epigraph``, ``highlights``, ``pull-quote``, ``compound``,
+    ``table``, ``csv-table``, ``list-table``, ``section-numbering``,
+    ``header``, ``footer``, ``meta``, ``class``
+    - no ``role`` directives and no custom interpreted text roles
+    - some standard roles are not supported (check [RST roles list])
+    - no generic admonition support
+* inline markup
+  - no simple-inline-markup
+  - no embedded aliases
+
+Additional resources
+--------------------
+
+* See [Nim DocGen Tools Guide](docgen.html) for the details about
+  `nim doc`:cmd: command and idiosyncrasies of documentation markup in
+  ``.nim`` files and Nim programming language projects.
+* See also documentation for [rst module](rst.html) -- Nim RST/Markdown parser.
+
+.. _Markdown Basic Syntax: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax
+.. _CommonMark Spec: https://spec.commonmark.org/0.30
+.. _quick introduction: https://docutils.sourceforge.io/docs/user/rst/quickstart.html
+.. _RST reference: https://docutils.sourceforge.io/docs/user/rst/quickref.html
+.. _RST specification: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html
+.. _RST directives list: https://docutils.sourceforge.io/docs/ref/rst/directives.html
+.. _RST roles list: https://docutils.sourceforge.io/docs/ref/rst/roles.html
+.. _Nim index: https://nim-lang.org/docs/theindex.html
+.. _Sphinx directives: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html
+.. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
diff --git a/doc/mm.md b/doc/mm.md
new file mode 100644
index 0000000000..5e0d2f3b94
--- /dev/null
+++ b/doc/mm.md
@@ -0,0 +1,95 @@
+=======================
+Nim's Memory Management
+=======================
+
+.. default-role:: code
+.. include:: rstcommon.rst
+
+:Author: Andreas Rumpf
+:Version: |nimversion|
+
+..
+
+
+> "The road to hell is paved with good intentions."
+
+
+Multi-paradigm Memory Management Strategies
+===========================================
+
+.. default-role:: option
+
+Nim offers multiple different memory management strategies.
+To choose the memory management strategy use the `--mm:` switch.
+
+ .. hint:: **The recommended switch for newly written Nim code is `--mm:orc`.**
+
+
+ARC/ORC
+-------
+
+ORC is the default memory management strategy. It is a memory
+management mode primarily based on reference counting. Reference cycles are
+handled by a cycle collection mechanism based on "trial deletion".
+Since algorithms based on "tracing" are not used, the runtime behavior is oblivious to the involved heap and stack sizes.
+
+The reference counting operations (= "RC ops") do not use atomic instructions and do not have to --
+instead entire subgraphs are *moved* between threads. The Nim compiler also aggressively
+optimizes away RC ops and exploits [move semantics](destructors.html#move-semantics).
+
+Nim performs a fair share of optimizations for ARC/ORC; you can inspect what it did
+to your time critical function via `--expandArc:functionName`. Likewise, you can inspect the whole module via `--expandArc:fileName`.
+
+`--mm:arc` uses the same mechanism as `--mm:orc`, but it leaves out the cycle collector.
+Both ARC and ORC offer deterministic performance for `hard realtime`:idx: systems, but
+ARC can be easier to reason about for people coming from Ada/C++/C -- roughly speaking
+the memory for a variable is freed when it goes "out of scope".
+
+We generally advise you to use the `acyclic` annotation in order to optimize away the
+cycle collector's overhead
+but `--mm:orc` also produces more machine code than `--mm:arc`, so if you're on a target
+where code size matters and you know that your code does not produce cycles, you can
+use `--mm:arc`. Notice that the default `async`:idx: implementation produces cycles
+and leaks memory with `--mm:arc`, in other words, for `async` you need to use `--mm:orc`.
+
+
+
+Other MM modes
+--------------
+
+.. note:: The `refc` GC is incremental, thread-local and not "stop-the-world".
+
+--mm:refc    It's a deferred reference counting based garbage collector
+  with a simple Mark&Sweep backup GC in order to collect cycles.
+  Heaps are thread-local. [This document](refc.html) contains further information.
+--mm:markAndSweep  Simple Mark-And-Sweep based garbage collector.
+  Heaps are thread-local.
+--mm:boehm    Boehm based garbage collector, it offers a shared heap.
+--mm:go    Go's garbage collector, useful for interoperability with Go.
+  Offers a shared heap.
+
+--mm:none    No memory management strategy nor a garbage collector. Allocated memory is
+  simply never freed. You should use `--mm:arc` instead.
+
+Here is a comparison of the different memory management modes:
+
+================== ======== ================= ============== ====== =================== ===================
+Memory Management  Heap     Reference Cycles  Stop-The-World Atomic Valgrind compatible Command line switch
+================== ======== ================= ============== ====== =================== ===================
+ORC                Shared   Cycle Collector   No             No     Yes                 `--mm:orc`
+ARC                Shared   Leak              No             No     Yes                 `--mm:arc`
+Atomic ARC         Shared   Leak              No             Yes    Yes                 `--mm:atomicArc`
+RefC               Local    Cycle Collector   No             No     No                  `--mm:refc`
+Mark & Sweep       Local    Cycle Collector   No             No     No                  `--mm:markAndSweep`
+Boehm              Shared   Cycle Collector   Yes            No     No                  `--mm:boehm`
+Go                 Shared   Cycle Collector   Yes            No     No                  `--mm:go`
+None               Manual   Manual            Manual         Manual Manual              `--mm:none`
+================== ======== ================= ============== ====== =================== ===================
+
+.. default-role:: code
+.. include:: rstcommon.rst
+
+JavaScript's garbage collector is used for the [JavaScript and NodeJS](
+backends.html#backends-the-javascript-target) compilation targets.
+The [NimScript](nims.html) target uses the memory management strategy built into
+the Nim compiler.
diff --git a/doc/nep1.rst b/doc/nep1.md
similarity index 80%
rename from doc/nep1.rst
rename to doc/nep1.md
index 3bae6a00bb..3d2a0cef3e 100644
--- a/doc/nep1.rst
+++ b/doc/nep1.md
@@ -1,11 +1,12 @@
-.. default-role:: code
+============================
+Standard Library Style Guide
+============================
 
-==========================================================
-Nim Enhancement Proposal #1 - Standard Library Style Guide
-==========================================================
 :Author: Clay Sweetser, Dominik Picheta
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
@@ -20,7 +21,7 @@ library should follow.
 Note that there can be exceptions to these rules. Nim being as flexible as it
 is, there will be parts of this style guide that don't make sense in certain
 contexts. Furthermore, just as
-`Python's style guide`_ changes
+[Python's style guide](http://legacy.python.org/dev/peps/pep-0008/) changes
 over time, this style guide will too.
 
 These rules will only be enforced for contributions to the Nim
@@ -48,7 +49,7 @@ Spacing and Whitespace Conventions
   Not all editors support automatic alignment of code sections, and re-aligning
   long sections of code by hand can quickly become tedious.
 
-  .. code-block:: nim
+    ```nim
     # This is bad, as the next time someone comes
     # to edit this code block, they
     # must re-align all the assignments again:
@@ -59,6 +60,7 @@ Spacing and Whitespace Conventions
       CalId*       = int
       LongLong*    = int64
       LongLongPtr* = ptr LongLong
+    ```
 
 
 Naming Conventions
@@ -68,7 +70,7 @@ Naming Conventions
   camelCase with the exception of constants which **may** use PascalCase but
   are not required to.
 
-  .. code-block:: nim
+    ```nim
     # Constants can start with either a lower case or upper case letter.
     const aConstant = 42
     const FooBar = 4.2
@@ -78,6 +80,7 @@ Naming Conventions
     # Types must start with an uppercase letter.
     type
       FooBar = object
+    ```
 
   For constants coming from a C/C++ wrapper, ALL_UPPERCASE are allowed, but ugly.
   (Why shout CONSTANT? Constants do no harm, variables do!)
@@ -88,41 +91,45 @@ Naming Conventions
   that will be used the most, add the suffixes to the pointer variants only. The
   same applies to C/C++ wrappers.
 
-  .. code-block:: nim
+    ```nim
     type
       Handle = object # Will be used most often
         fd: int64
       HandleRef = ref Handle # Will be used less often
+    ```
 
 - Exception and Error types should have the "Error" or "Defect" suffix.
 
-  .. code-block:: nim
+    ```nim
     type
       ValueError = object of CatchableError
       AssertionDefect = object of Defect
       Foo = object of Exception # bad style, try to inherit CatchableError or Defect
+    ```
 
 - Unless marked with the `{.pure.}` pragma, members of enums should have an
   identifying prefix, such as an abbreviation of the enum's name.
 
-  .. code-block:: nim
+    ```nim
     type
       PathComponent = enum
         pcDir
         pcLinkToDir
         pcFile
         pcLinkToFile
+    ```
 
 - Non-pure enum values should use camelCase whereas pure enum values should use
   PascalCase.
 
-  .. code-block:: nim
+    ```nim
     type
       PathComponent {.pure.} = enum
         Dir
         LinkToDir
         File
         LinkToFile
+    ```
 
 - In the age of HTTP, HTML, FTP, TCP, IP, UTF, WWW it is foolish to pretend
   these are somewhat special words requiring all uppercase. Instead treat them
@@ -139,7 +146,7 @@ Naming Conventions
   - rotate and rotated
 
 - When the 'returns transformed copy' version already exists like `strutils.replace`
-  an in-place version should get an `-In` suffix (`replaceIn` for this example).
+  an in-place version should get an ``-In`` suffix (`replaceIn` for this example).
 
 
 - Use `subjectVerb`, not `verbSubject`, e.g.: `fileExists`, not `existsFile`.
@@ -152,12 +159,13 @@ The library uses a simple naming scheme that makes use of common abbreviations
 to keep the names short but meaningful.
 
 
--------------------     ------------   --------------------------------------
+===================     ============   ======================================
 English word            To use         Notes
--------------------     ------------   --------------------------------------
+===================     ============   ======================================
 initialize              initFoo        initializes a value type `Foo`
 new                     newFoo         initializes a reference type `Foo`
-                                       via `new`
+                                       via `new` or a value type `Foo`
+                                       with reference semantics.
 this or self            self           for method like procs, e.g.:
                                        `proc fun(self: Foo, a: int)`
                                        rationale: `self` is more unique in English
@@ -218,22 +226,23 @@ literal                 lit
 string                  str
 identifier              ident
 indentation             indent
--------------------     ------------   --------------------------------------
+===================     ============   ======================================
 
 
 Coding Conventions
 ------------------
 
-- The 'return' statement should ideally be used when its control-flow properties
-  are required. Use a procedure's implicit 'result' variable whenever possible.
+- The `return` statement should ideally be used when its control-flow properties
+  are required. Use a procedure's implicit `result` variable whenever possible.
   This improves readability.
 
-  .. code-block:: nim
+    ```nim
     proc repeat(text: string, x: int): string =
       result = ""
 
       for i in 0..x:
         result.add($i)
+    ```
 
 - Use a proc when possible, only using the more powerful facilities of macros,
   templates, iterators, and converters when necessary.
@@ -247,32 +256,45 @@ Coding Conventions
 Conventions for multi-line statements and expressions
 -----------------------------------------------------
 
-- Tuples which are longer than one line should indent their parameters to
-  align with the parameters above it.
+- Tuples which are longer than one line should indent their parameters.
 
-  .. code-block:: nim
+    ```nim
     type
-      LongTupleA = tuple[wordyTupleMemberOne: int, wordyTupleMemberTwo: string,
-                         wordyTupleMemberThree: float]
+      LongTupleA = tuple[
+        wordyTupleMemberOne: int, wordyTupleMemberTwo: string,
+        wordyTupleMemberThree: float]
+    ```
 
 - Similarly, any procedure and procedure type declarations that are longer
-  than one line should do the same thing.
+  than one line should do the same thing. Double indent may be used to
+  distinguish them from the body that follows - this applies to all constructs
+  with a body (if, while, etc).
 
-  .. code-block:: nim
+    ```nim
     type
-      EventCallback = proc (timeReceived: Time, errorCode: int, event: Event,
-                            output: var string)
+      EventCallback = proc(
+        timeReceived: Time, errorCode: int, event: Event,
+        output: var string)
 
-    proc lotsOfArguments(argOne: string, argTwo: int, argThree: float,
-                         argFour: proc(), argFive: bool): int
-                        {.heyLookALongPragma.} =
+    proc lotsOfArguments(
+        argOne: string, argTwo: int, argThree: float,
+        argFour: proc(), argFive: bool, argSix: int
+    ): GenericType[int, string] {.heyLookALongPragma.} =
+      discard
+    ```
 
-- Multi-line procedure calls should continue on the same column as the opening
-  parenthesis (like multi-line procedure declarations).
+- Multi-line procedure calls should continue indented (like multi-line procedure
+  declarations).
 
-  .. code-block:: nim
-    startProcess(nimExecutable, currentDirectory, compilerArguments
-                 environment, processOptions)
+    ```nim
+    startProcess(
+      nimExecutable, currentDirectory, compilerArguments
+      environment, processOptions)
+    ```
+
+Previous versions of this guide advocated vertical alignment along the opening
+brace / parenthesis - both styles are permissible with a preference for the
+current style in new code.
 
 Miscellaneous
 -------------
@@ -288,15 +310,26 @@ Miscellaneous
 
   use this:
 
-  .. code-block:: nim
+    ```nim
     let a = """
     foo
     bar
     """
+    ```
 
   instead of:
 
-  .. code-block:: nim
+    ```nim
     let a = """foo
     bar
     """
+    ```
+
+- A getter API for a private field `foo` should preferably be named `foo`, not `getFoo`.
+  A getter-like API should preferably be named `getFoo`, not `foo` if:
+    * the API has side effects
+    * or the cost is not `O(1)`
+  For in between cases, there is no clear guideline.
+
+- Likewise with a setter API, replacing `foo` with `foo=` and `getFoo` with `setFoo`
+  in the above text.
diff --git a/doc/nimc.rst b/doc/nimc.md
similarity index 53%
rename from doc/nimc.rst
rename to doc/nimc.md
index aad889591d..25acf31e83 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 ===================================
    Nim Compiler User Guide
 ===================================
@@ -7,11 +5,15 @@
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
-  "Look at you, hacker. A pathetic creature of meat and bone, panting and
-  sweating as you run through my corridors. How can you challenge a perfect,
-  immortal machine?"
+..
+
+> "Look at you, hacker. A pathetic creature of meat and bone, panting and
+> sweating as you run through my corridors. How can you challenge a perfect,
+> immortal machine?"
 
 
 Introduction
@@ -19,10 +21,10 @@ Introduction
 
 This document describes the usage of the *Nim compiler*
 on the different supported platforms. It is not a definition of the Nim
-programming language (which is covered in the `manual `_).
+programming language (which is covered in the [manual](manual.html)).
 
 Nim is free software; it is licensed under the
-`MIT License `_.
+[MIT License](http://www.opensource.org/licenses/mit-license.php).
 
 
 Compiler Usage
@@ -30,8 +32,22 @@ Compiler Usage
 
 Command-line switches
 ---------------------
+All options that take a `PATH` or `DIR` argument are subject to path substitution:
+
+- `$nim`: The global nim prefix path
+- `$lib`: The stdlib path
+- `$home` and `~`: The user's home path
+- `$config`: The directory of the module currently being compiled
+- `$projectname`: The project file's name without file extension
+- `$projectpath` and `$projectdir`: The project file's path
+- `$nimcache`: The nimcache path
+
+
 Basic command-line switches are:
 
+.. no syntax highlighting in the below included files at the moment
+.. default-role:: code
+
 Usage:
 
 .. include:: basicopt.txt
@@ -43,12 +59,13 @@ Advanced command-line switches are:
 .. include:: advopt.txt
 
 
+.. include:: rstcommon.rst
 
 List of warnings
 ----------------
 
-Each warning can be activated individually with `--warning[NAME]:on|off` or
-in a `push` pragma.
+Each warning can be activated individually with `--warning:NAME:on|off`:option: or
+in a `push` pragma with `{.warning[NAME]:on|off.}`.
 
 ==========================       ============================================
 Name                             Description
@@ -64,6 +81,14 @@ SmallLshouldNotBeUsed            The letter 'l' should not be used as an
                                  identifier.
 EachIdentIsTuple                 The code contains a confusing `var`
                                  declaration.
+CStringConv                      Warn about dangerous implicit conversions
+                                 to `cstring`.
+EnumConv                         Warn about conversions from enum to enum.
+AnyEnumConv                      Warn about any conversions to an enum type.
+HoleEnumConv                     Warn about conversion to an enum with
+                                 holes. These conversions are unsafe.
+ResultUsed                       Warn about the usage of the
+                                 built-in `result` variable.
 User                             Some user-defined warning.
 ==========================       ============================================
 
@@ -71,8 +96,8 @@ User                             Some user-defined warning.
 List of hints
 -------------
 
-Each hint can be activated individually with `--hint[NAME]:on|off` or in a
-`push` pragma.
+Each hint can be activated individually with `--hint:NAME:on|off`:option: or in a
+`push` pragma with `{.hint[NAME]:on|off.}`.
 
 ==========================       ============================================
 Name                             Description
@@ -90,7 +115,6 @@ ExprAlwaysX
 ExtendedContext
 GCStats                          Dumps statistics about the Garbage Collector.
 GlobalVar                        Shows global variables declarations.
-LineTooLong                      Line exceeds the maximum length.
 Link                             Linking phase.
 Name
 Path                             Search paths modifications.
@@ -116,13 +140,13 @@ Level  Description
 =====  ============================================
 0      Minimal output level for the compiler.
 1      Displays compilation of all the compiled files, including those imported
-       by other modules or through the `compile pragma
-       `_.
+       by other modules or through the [compile pragma](
+       manual.html#implementation-specific-pragmas-compile-pragma).
        This is the default level.
 2      Displays compilation statistics, enumerates the dynamic
        libraries that will be loaded by the final binary, and dumps to
-       standard output the result of applying `a filter to the source code
-       `_ if any filter was used during compilation.
+       standard output the result of applying [a filter to the source code](
+       filters.html) if any filter was used during compilation.
 3      In addition to the previous levels dumps a debug stack trace
        for compiler developers.
 =====  ============================================
@@ -131,54 +155,89 @@ Level  Description
 Compile-time symbols
 --------------------
 
-Through the `-d:x` or `--define:x` switch you can define compile-time
+Through the `-d:x`:option: or `--define:x`:option: switch you can define compile-time
 symbols for conditional compilation. The defined switches can be checked in
-source code with the `when statement
-`_ and
-`defined proc `_. The typical use of this switch is
-to enable builds in release mode (`-d:release`) where optimizations are
-enabled for better performance. Another common use is the `-d:ssl` switch to
+source code with the [when statement](
+manual.html#statements-and-expressions-when-statement) and
+[defined proc](system.html#defined,untyped). The typical use of this switch is
+to enable builds in release mode (`-d:release`:option:) where optimizations are
+enabled for better performance. Another common use is the `-d:ssl`:option: switch to
 activate SSL sockets.
 
-Additionally, you may pass a value along with the symbol: `-d:x=y`
-which may be used in conjunction with the `compile-time define
-pragmas`_
+Additionally, you may pass a value along with the symbol: `-d:x=y`:option:
+which may be used in conjunction with the [compile-time define
+pragmas](manual.html#implementation-specific-pragmas-compileminustime-define-pragmas)
 to override symbols during build time.
 
 Compile-time symbols are completely **case insensitive** and underscores are
-ignored too. `--define:FOO` and `--define:foo` are identical.
+ignored too. `--define:FOO`:option: and `--define:foo`:option: are identical.
 
 Compile-time symbols starting with the `nim` prefix are reserved for the
 implementation and should not be used elsewhere.
 
+==========================       ============================================
+Name                             Description
+==========================       ============================================
+nimStdSetjmp                     Use the standard `setjmp()/longjmp()` library
+                                 functions for setjmp-based exceptions. This is
+                                 the default on most platforms.
+nimSigSetjmp                     Use `sigsetjmp()/siglongjmp()` for setjmp-based exceptions.
+nimRawSetjmp                     Use `_setjmp()/_longjmp()` on POSIX and `_setjmp()/longjmp()`
+                                 on Windows, for setjmp-based exceptions. It's the default on
+                                 BSDs and BSD-like platforms, where it's significantly faster
+                                 than the standard functions.
+nimBuiltinSetjmp                 Use `__builtin_setjmp()/__builtin_longjmp()` for setjmp-based
+                                 exceptions. This will not work if an exception is being thrown
+                                 and caught inside the same procedure. Useful for benchmarking.
+==========================       ============================================
+
 
 Configuration files
 -------------------
 
-**Note:** The *project file name* is the name of the `.nim` file that is
+**Note:** The *project file name* is the name of the ``.nim`` file that is
 passed as a command-line argument to the compiler.
 
 
-The `nim` executable processes configuration files in the following
+The `nim`:cmd: executable processes configuration files in the following
 directories (in this order; later files overwrite previous settings):
 
-1) `$nim/config/nim.cfg`, `/etc/nim/nim.cfg` (UNIX) or ``\config\nim.cfg`` (Windows). This file can be skipped with the `--skipCfg` command line option.
-2) If environment variable `XDG_CONFIG_HOME` is defined, `$XDG_CONFIG_HOME/nim/nim.cfg` or `~/.config/nim/nim.cfg` (POSIX) or `%APPDATA%/nim/nim.cfg` (Windows). This file can be skipped with the `--skipUserCfg` command line option.
-3) `$parentDir/nim.cfg` where `$parentDir` stands for any parent  directory of the project file's path. These files can be skipped with the `--skipParentCfg` command-line option.
-4) `$projectDir/nim.cfg` where `$projectDir` stands for the project  file's path. This file can be skipped with the `--skipProjCfg` command-line option.
-5) A project can also have a project-specific configuration file named `$project.nim.cfg` that resides in the same directory as `$project.nim`. This file can be skipped with the `--skipProjCfg` command-line option.
+1) ``$nim/config/nim.cfg``, ``/etc/nim/nim.cfg`` (UNIX) or
+   ``\config\nim.cfg`` (Windows).
+   This file can be skipped with the `--skipCfg`:option: command line option.
+2) If environment variable `XDG_CONFIG_HOME` is defined,
+   ``$XDG_CONFIG_HOME/nim/nim.cfg`` or ``~/.config/nim/nim.cfg`` (POSIX) or
+   ``%APPDATA%/nim/nim.cfg`` (Windows).
+   This file can be skipped with the `--skipUserCfg`:option: command line
+   option.
+3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent
+   directory of the project file's path.
+   These files can be skipped with the `--skipParentCfg`:option:
+   command-line option.
+4) ``$projectDir/nim.cfg`` where ``$projectDir`` stands for the project
+   file's path.
+   This file can be skipped with the `--skipProjCfg`:option:
+   command-line option.
+5) A project can also have a project-specific configuration file named
+   ``$project.nim.cfg`` that resides in the same directory as ``$project.nim``.
+   This file can be skipped with the `--skipProjCfg`:option:
+   command-line option.
 
 
 Command-line settings have priority over configuration file settings.
 
 The default build of a project is a `debug build`:idx:. To compile a
-`release build`:idx: define the `release` symbol::
+`release build`:idx: define the `release` symbol:
 
+  ```cmd
   nim c -d:release myproject.nim
+  ```
 
- To compile a `dangerous release build`:idx: define the `danger` symbol::
+To compile a `dangerous release build`:idx: define the `danger` symbol:
 
+  ```cmd
   nim c -d:danger myproject.nim
+  ```
 
 
 Search path handling
@@ -188,39 +247,39 @@ Nim has the concept of a global search path (PATH) that is queried to
 determine where to find imported modules or include files. If multiple files are
 found an ambiguity error is produced.
 
-`nim dump` shows the contents of the PATH.
+`nim dump`:cmd: shows the contents of the PATH.
 
 However before the PATH is used the current directory is checked for the
-file's existence. So if PATH contains `$lib` and `$lib/bar` and the
-directory structure looks like this::
+file's existence. So if PATH contains ``$lib`` and ``$lib/bar`` and the
+directory structure looks like this:
 
-  $lib/x.nim
-  $lib/bar/x.nim
-  foo/x.nim
-  foo/main.nim
-  other.nim
+    $lib/x.nim
+    $lib/bar/x.nim
+    foo/x.nim
+    foo/main.nim
+    other.nim
 
 And `main` imports `x`, `foo/x` is imported. If `other` imports `x`
-then both `$lib/x.nim` and `$lib/bar/x.nim` match but `$lib/x.nim` is used
+then both ``$lib/x.nim`` and ``$lib/bar/x.nim`` match but ``$lib/x.nim`` is used
 as it is the first match.
 
 
 Generated C code directory
 --------------------------
 The generated files that Nim produces all go into a subdirectory called
-`nimcache`. Its full path is
+``nimcache``. Its full path is
 
-- `$XDG_CACHE_HOME/nim/$projectname(_r|_d)` or `~/.cache/nim/$projectname(_r|_d)`
+- ``$XDG_CACHE_HOME/nim/$projectname(_r|_d)`` or ``~/.cache/nim/$projectname(_r|_d)``
   on Posix
-- `$HOME/nimcache/$projectname(_r|_d)` on Windows.
+- ``$HOME\nimcache\$projectname(_r|_d)`` on Windows.
 
 The `_r` suffix is used for release builds, `_d` is for debug builds.
 
 This makes it easy to delete all generated files.
 
-The `--nimcache`
-`compiler switch <#compiler-usage-commandminusline-switches>`_ can be used to
-to change the `nimcache` directory.
+The `--nimcache`:option:
+[compiler switch][command-line switches] can be used to
+to change the ``nimcache`` directory.
 
 However, the generated C code is not platform-independent. C code generated for
 Linux does not compile on Windows, for instance. The comment on top of the
@@ -230,60 +289,75 @@ C file lists the OS, CPU, and CC the file has been compiled for.
 Compiler Selection
 ==================
 
-To change the compiler from the default compiler (at the command line)::
+To change the compiler from the default compiler (at the command line):
 
+  ```cmd
   nim c --cc:llvm_gcc --compile_only myfile.nim
+  ```
 
-This uses the configuration defined in ``config\nim.cfg`` for `lvm_gcc`.
+This uses the configuration defined in ``config\nim.cfg`` for `llvm_gcc`:cmd:.
 
 If nimcache already contains compiled code from a different compiler for the same project,
-add the `-f` flag to force all files to be recompiled.
+add the `-f`:option: flag to force all files to be recompiled.
 
 The default compiler is defined at the top of ``config\nim.cfg``.
-Changing this setting affects the compiler used by `koch` to (re)build Nim.
+Changing this setting affects the compiler used by `koch`:cmd: to (re)build Nim.
 
-To use the `CC` environment variable, use `nim c --cc:env myfile.nim`. To use the
-`CXX` environment variable, use `nim cpp --cc:env myfile.nim`. `--cc:env` is available
-since Nim version 1.4.
+To use the `CC` environment variable, use `nim c --cc:env myfile.nim`:cmd:.
+To use the `CXX` environment variable, use `nim cpp --cc:env myfile.nim`:cmd:.
+`--cc:env`:option: is available since Nim version 1.4.
 
 
 Cross-compilation
 =================
 
-To cross compile, use for example::
+To cross compile, use for example:
 
+  ```cmd
   nim c --cpu:i386 --os:linux --compileOnly --genScript myproject.nim
+  ```
 
-Then move the C code and the compile script `compile_myproject.sh` to your
+Then move the C code and the compile script `compile_myproject.sh`:cmd: to your
 Linux i386 machine and run the script.
 
-Another way is to make Nim invoke a cross compiler toolchain::
+Another way is to make Nim invoke a cross compiler toolchain:
 
+  ```cmd
   nim c --cpu:arm --os:linux myproject.nim
+  ```
 
-For cross compilation, the compiler invokes a C compiler named
-like `$cpu.$os.$cc` (for example arm.linux.gcc) and the configuration
-system is used to provide meaningful defaults. For example for `ARM` your
-configuration file should contain something like::
+For cross compilation, the compiler invokes a C compiler named like
+`$cpu.$os.$cc` (for example `arm.linux.gcc`) with options defined in
+`$cpu.$os.$cc.options.always`. The configuration system is used to provide
+meaningful defaults. For example, for Linux on a 32-bit ARM CPU, your
+configuration file should contain something like:
 
-  arm.linux.gcc.path = "/usr/bin"
-  arm.linux.gcc.exe = "arm-linux-gcc"
-  arm.linux.gcc.linkerexe = "arm-linux-gcc"
+    arm.linux.gcc.path = "/usr/bin"
+    arm.linux.gcc.exe = "arm-linux-gcc"
+    arm.linux.gcc.linkerexe = "arm-linux-gcc"
+    arm.linux.gcc.options.always = "-w -fmax-errors=3"
 
 Cross-compilation for Windows
 =============================
 
-To cross-compile for Windows from Linux or macOS using the MinGW-w64 toolchain::
+To cross-compile for Windows from Linux or macOS using the MinGW-w64 toolchain:
 
+  ```cmd
   nim c -d:mingw myproject.nim
+  # `nim r` also works, running the binary via `wine` or `wine64`:
+  nim r -d:mingw --eval:'import os; echo "a" / "b"'
+  ```
 
-Use `--cpu:i386` or `--cpu:amd64` to switch the CPU architecture.
+Use `--cpu:i386`:option: or `--cpu:amd64`:option: to switch the CPU architecture.
 
-The MinGW-w64 toolchain can be installed as follows::
+The MinGW-w64 toolchain can be installed as follows:
 
-  Ubuntu: apt install mingw-w64
-  CentOS: yum install mingw32-gcc | mingw64-gcc - requires EPEL
-  OSX: brew install mingw-w64
+  ```cmd
+  apt install mingw-w64   # Ubuntu
+  yum install mingw32-gcc
+  yum install mingw64-gcc # CentOS - requires EPEL
+  brew install mingw-w64  # OSX
+  ```
 
 
 Cross-compilation for Android
@@ -293,36 +367,40 @@ There are two ways to compile for Android: terminal programs (Termux) and with
 the NDK (Android Native Development Kit).
 
 The first one is to treat Android as a simple Linux and use
-`Termux `_ to connect and run the Nim compiler
+[Termux](https://wiki.termux.com) to connect and run the Nim compiler
 directly on android as if it was Linux. These programs are console-only
 programs that can't be distributed in the Play Store.
 
-Use regular `nim c` inside termux to make Android terminal programs.
+Use regular `nim c`:cmd: inside termux to make Android terminal programs.
 
 Normal Android apps are written in Java, to use Nim inside an Android app
 you need a small Java stub that calls out to a native library written in
-Nim using the `NDK `_. You can also use
-`native-activity `_
+Nim using the [NDK](https://developer.android.com/ndk). You can also use
+[native-activity](https://developer.android.com/ndk/samples/sample_na)
 to have the Java stub be auto-generated for you.
 
-Use `nim c -c --cpu:arm --os:android -d:androidNDK --noMain:on` to
+Use `nim c -c --cpu:arm --os:android -d:androidNDK --noMain:on`:cmd: to
 generate the C source files you need to include in your Android Studio
 project. Add the generated C files to CMake build script in your Android
 project. Then do the final compile with Android Studio which uses Gradle
 to call CMake to compile the project.
 
-Because Nim is part of a library it can't have its own c style `main()`
-so you would need to define your own `android_main` and init the Java
+Because Nim is part of a library it can't have its own C-style `main()`:c:
+so you would need to define your own `android_main`:c: and init the Java
 environment, or use a library like SDL2 or GLFM to do it. After the Android
-stuff is done, it's very important to call `NimMain()` in order to
+stuff is done, it's very important to call `NimMain()`:c: in order to
 initialize Nim's garbage collector and to run the top level statements
 of your program.
 
-.. code-block:: Nim
-
+  ```Nim
   proc NimMain() {.importc.}
   proc glfmMain*(display: ptr GLFMDisplay) {.exportc.} =
     NimMain() # initialize garbage collector memory, types and stack
+  ```
+
+
+The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch.
+Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`.
 
 
 Cross-compilation for iOS
@@ -333,85 +411,97 @@ Normal languages for iOS development are Swift and Objective C. Both of these
 use LLVM and can be compiled into object files linked together with C, C++
 or Objective C code produced by Nim.
 
-Use `nim c -c --os:ios --noMain:on` to generate C files and include them in
+Use `nim c -c --os:ios --noMain:on`:cmd: to generate C files and include them in
 your XCode project. Then you can use XCode to compile, link, package and
 sign everything.
 
-Because Nim is part of a library it can't have its own c style `main()` so you
+Because Nim is part of a library it can't have its own C-style `main()`:c: so you
 would need to define `main` that calls `autoreleasepool` and
 `UIApplicationMain` to do it, or use a library like SDL2 or GLFM. After
-the iOS setup is done, it's very important to call `NimMain()` to
+the iOS setup is done, it's very important to call `NimMain()`:c: to
 initialize Nim's garbage collector and to run the top-level statements
 of your program.
 
-.. code-block:: Nim
-
+  ```Nim
   proc NimMain() {.importc.}
   proc glfmMain*(display: ptr GLFMDisplay) {.exportc.} =
     NimMain() # initialize garbage collector memory, types and stack
+  ```
 
 Note: XCode's "make clean" gets confused about the generated nim.c files,
 so you need to clean those files manually to do a clean build.
 
+The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch.
+Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`.
+
 
 Cross-compilation for Nintendo Switch
 =====================================
 
-Simply add --os:nintendoswitch
-to your usual `nim c` or `nim cpp` command and set the `passC`
-and `passL` command line switches to something like:
+Simply add `--os:nintendoswitch`:option:
+to your usual `nim c`:cmd: or `nim cpp`:cmd: command and set the `passC`:option:
+and `passL`:option: command line switches to something like:
 
-.. code-block:: console
-  nim c ... --passC="-I$DEVKITPRO/libnx/include" ...
+  ```cmd
+  nim c ... --d:nimAllocPagesViaMalloc --mm:orc --passC="-I$DEVKITPRO/libnx/include" ...
   --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx"
+  ```
 
-or setup a nim.cfg file like so:
+or setup a ``nim.cfg`` file like so:
 
-.. code-block:: Nim
-  #nim.cfg
-  --passC="-I$DEVKITPRO/libnx/include"
-  --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx"
+    #nim.cfg
+    --mm:orc
+    --d:nimAllocPagesViaMalloc
+    --define:nimInheritHandles
+    --passC="-I$DEVKITPRO/libnx/include"
+    --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx"
 
-The DevkitPro setup must be the same as the default with their new installer
-`here for Mac/Linux `_ or
-`here for Windows `_.
+The devkitPro setup must be the same as the default with their new installer
+[here for Mac/Linux](https://github.com/devkitPro/pacman/releases) or
+[here for Windows](https://github.com/devkitPro/installer/releases).
 
-For example, with the above-mentioned config::
+For example, with the above-mentioned config:
 
+  ```cmd
   nim c --os:nintendoswitch switchhomebrew.nim
+  ```
 
-This will generate a file called `switchhomebrew.elf` which can then be turned into
-an nro file with the `elf2nro` tool in the DevkitPro release. Examples can be found at
-`the nim-libnx github repo `_.
+This will generate a file called ``switchhomebrew.elf`` which can then be turned into
+an nro file with the `elf2nro`:cmd: tool in the devkitPro release. Examples can be found at
+[the nim-libnx github repo](https://github.com/jyapayne/nim-libnx.git).
 
-There are a few things that don't work because the DevkitPro libraries don't support them.
+There are a few things that don't work because the devkitPro libraries don't support them.
 They are:
 
 1. Waiting for a subprocess to finish. A subprocess can be started, but right
    now it can't be waited on, which sort of makes subprocesses a bit hard to use
-2. Dynamic calls. DevkitPro libraries have no dlopen/dlclose functions.
-3. Command line parameters. It doesn't make sense to have these for a console
-   anyways, so no big deal here.
-4. mqueue. Sadly there are no mqueue headers.
-5. ucontext. No headers for these either. No coroutines for now :(
-6. nl_types. No headers for this.
+2. Dynamic calls. Switch OS (Horizon) doesn't support dynamic libraries, so dlopen/dlclose are not available.
+3. mqueue. Sadly there are no mqueue headers.
+4. ucontext. No headers for these either. No coroutines for now :(
+5. nl_types. No headers for this.
+6. As mmap is not supported, the nimAllocPagesViaMalloc option has to be used.
+
 
 DLL generation
 ==============
 
+**Note**: The same rules apply to `lib*.so` shared object files on UNIX. For better
+readability only the DLL version is described here.
+
 Nim supports the generation of DLLs. However, there must be only one
 instance of the GC per process/address space. This instance is contained in
-`nimrtl.dll`. This means that every generated Nim DLL depends
-on `nimrtl.dll`. To generate the "nimrtl.dll" file, use the command::
+``nimrtl.dll``. This means that every generated Nim DLL depends
+on ``nimrtl.dll``. To generate the "nimrtl.dll" file, use the command:
 
+  ```cmd
   nim c -d:release lib/nimrtl.nim
+  ```
 
-To link against `nimrtl.dll` use the command::
+To link against ``nimrtl.dll`` use the command:
 
+  ```cmd
   nim c -d:useNimRtl myprog.nim
-
-**Note**: Currently the creation of `nimrtl.dll` with thread support has
-never been tested and is unlikely to work!
+  ```
 
 
 Additional compilation switches
@@ -426,20 +516,20 @@ Define                   Effect
 ======================   =========================================================
 `release`                Turns on the optimizer.
                          More aggressive optimizations are possible, e.g.:
-                         `--passC:-ffast-math` (but see issue #10305)
+                         `--passC:-ffast-math`:option: (but see issue #10305)
 `danger`                 Turns off all runtime checks and turns on the optimizer.
-`useFork`                Makes `osproc` use `fork` instead of `posix_spawn`.
-`useNimRtl`              Compile and link against `nimrtl.dll`.
+`useFork`                Makes `osproc` use `fork`:c: instead of `posix_spawn`:c:.
+`useNimRtl`              Compile and link against ``nimrtl.dll``.
 `useMalloc`              Makes Nim use C's `malloc`:idx: instead of Nim's
                          own memory manager, albeit prefixing each allocation with
                          its size to support clearing memory on reallocation.
-                         This only works with `gc:none`, `gc:arc` and
-                         `--gc:orc`.
+                         This only works with `--mm:none`:option:,
+                         `--mm:arc`:option: and `--mm:orc`:option:.
 `useRealtimeGC`          Enables support of Nim's GC for *soft* realtime
-                         systems. See the documentation of the `gc `_
+                         systems. See the documentation of the [refc](refc.html)
                          for further information.
 `logGC`                  Enable GC logging to stdout.
-`nodejs`                 The JS target is actually `node.js`.
+`nodejs`                 The JS target is actually ``node.js``.
 `ssl`                    Enables OpenSSL support for the sockets module.
 `memProfiler`            Enables memory profiling for the native GC.
 `uClibc`                 Use uClibc instead of libc. (Relevant for Unix-like OSes)
@@ -447,18 +537,25 @@ Define                   Effect
                          what's in the Nim file with what's in the C header.
                          This may become enabled by default in the future.
 `tempDir`                This symbol takes a string as its value, like
-                         `--define:tempDir:/some/temp/path` to override the
-                         temporary directory returned by `os.getTempDir()`.
+                         `--define:tempDir:/some/temp/path`:option: to override
+                         the temporary directory returned by `os.getTempDir()`.
                          The value **should** end with a directory separator
                          character. (Relevant for the Android platform)
 `useShPath`              This symbol takes a string as its value, like
-                         `--define:useShPath:/opt/sh/bin/sh` to override the
-                         path for the `sh` binary, in cases where it is not
-                         located in the default location `/bin/sh`.
-`noSignalHandler`        Disable the crash handler from `system.nim`.
-`globalSymbols`          Load all `{.dynlib.}` libraries with the `RTLD_GLOBAL`
+                         `--define:useShPath:/opt/sh/bin/sh`:option: to override
+                         the path for the `sh`:cmd: binary, in cases where it is
+                         not located in the default location ``/bin/sh``.
+`noSignalHandler`        Disable the crash handler from ``system.nim``.
+`globalSymbols`          Load all `{.dynlib.}` libraries with the `RTLD_GLOBAL`:c:
                          flag on Posix systems to resolve symbols in subsequently
                          loaded libraries.
+`lto`                    Enable link-time optimization in the backend compiler and
+                         linker.
+`lto_incremental`        Enable link-time optimization and additionally enable
+                         incremental linking for compilers that support it.
+                         Currently only clang and vcc.
+`strip`                  Strip debug symbols added by the backend compiler from
+                         the executable.
 ======================   =========================================================
 
 
@@ -473,20 +570,21 @@ generator and are subject to change.
 
 LineDir option
 --------------
-The `lineDir` option can be turned on or off. If turned on the
-generated C code contains `#line` directives. This may be helpful for
+The `--lineDir`:option: option can be turned on or off. If turned on the
+generated C code contains `#line`:c: directives. This may be helpful for
 debugging with GDB.
 
 
 StackTrace option
 -----------------
-If the `stackTrace` option is turned on, the generated C contains code to
+If the `--stackTrace`:option: option is turned on, the generated C contains code to
 ensure that proper stack traces are given if the program crashes or some uncaught exception is raised.
 
 
 LineTrace option
 ----------------
-The `lineTrace` option implies the `stackTrace` option. If turned on,
+The `--lineTrace`:option: option implies the `stackTrace`:option: option.
+If turned on,
 the generated C contains code to ensure that proper stack traces with line
 number information are given if the program crashes or an uncaught exception
 is raised.
@@ -496,37 +594,39 @@ DynlibOverride
 ==============
 
 By default Nim's `dynlib` pragma causes the compiler to generate
-`GetProcAddress` (or their Unix counterparts)
-calls to bind to a DLL. With the `dynlibOverride` command line switch this
-can be prevented and then via `--passL` the static library can be linked
+`GetProcAddress`:cpp: (or their Unix counterparts)
+calls to bind to a DLL. With the `dynlibOverride`:option: command line switch this
+can be prevented and then via `--passL`:option: the static library can be linked
 against. For instance, to link statically against Lua this command might work
-on Linux::
+on Linux:
 
+  ```cmd
   nim c --dynlibOverride:lua --passL:liblua.lib program.nim
+  ```
 
 
 Backend language options
 ========================
 
-The typical compiler usage involves using the `compile` or `c` command to
-transform a `.nim` file into one or more `.c` files which are then
+The typical compiler usage involves using the `compile`:option: or `c`:option:
+command to transform a ``.nim`` file into one or more ``.c`` files which are then
 compiled with the platform's C compiler into a static binary. However, there
 are other commands to compile to C++, Objective-C, or JavaScript. More details
-can be read in the `Nim Backend Integration document `_.
+can be read in the [Nim Backend Integration document](backends.html).
 
 
 Nim documentation tools
 =======================
 
 Nim provides the `doc`:idx: command to generate HTML
-documentation from `.nim` source files. Only exported symbols will appear in
-the output. For more details `see the docgen documentation `_.
+documentation from ``.nim`` source files. Only exported symbols will appear in
+the output. For more details see [the docgen documentation](docgen.html).
 
 Nim idetools integration
 ========================
 
 Nim provides language integration with external IDEs through the
-idetools command. See the documentation of `idetools `_
+idetools command. See the documentation of [idetools](idetools.html)
 for further information.
 
 ..
@@ -557,19 +657,21 @@ embedded microprocessors with only a few kilobytes of memory.
 A good start is to use the `any` operating target together with the
 `malloc` memory allocator and the `arc` garbage collector. For example:
 
-`nim c --os:any --gc:arc -d:useMalloc [...] x.nim`
+  ```cmd
+  nim c --os:any --mm:arc -d:useMalloc [...] x.nim
+  ```
 
-- `--gc:arc` will enable the reference counting memory management instead
+- `--mm:arc`:option: will enable the reference counting memory management instead
   of the default garbage collector. This enables Nim to use heap memory which
   is required for strings and seqs, for example.
 
-- The `--os:any` target makes sure Nim does not depend on any specific
+- The `--os:any`:option: target makes sure Nim does not depend on any specific
   operating system primitives. Your platform should support only some basic
   ANSI C library `stdlib` and `stdio` functions which should be available
   on almost any platform.
 
-- The `-d:useMalloc` option configures Nim to use only the standard C memory
-  manage primitives `malloc()`, `free()`, `realloc()`.
+- The `-d:useMalloc`:option: option configures Nim to use only the standard C memory
+  manage primitives `malloc()`:c:, `free()`:c:, `realloc()`:c:.
 
 If your platform does not provide these functions it should be trivial to
 provide an implementation for them and link these to your program.
@@ -579,20 +681,65 @@ additional flags to both the Nim compiler and the C compiler and/or linker
 to optimize the build for size. For example, the following flags can be used
 when targeting a gcc compiler:
 
-`--opt:size --passC:-flto --passL:-flto`
+`--opt:size -d:lto -d:strip`:option:
 
-The `--opt:size` flag instructs Nim to optimize code generation for small
-size (with the help of the C compiler), the `flto` flags enable link-time
-optimization in the compiler and linker.
+The `--opt:size`:option: flag instructs Nim to optimize code generation for small
+size (with the help of the C compiler), the `-d:lto`:option: flags enable link-time
+optimization in the compiler and linker, the `-d:strip`:option: strips debug symbols.
 
-Check the `Cross-compilation` section for instructions on how to compile the
+Check the [Cross-compilation] section for instructions on how to compile the
 program for your target.
 
+
+nimAllocPagesViaMalloc
+----------------------
+
+Nim's default allocator is based on TLSF, this algorithm was designed for embedded
+devices. This allocator gets blocks/pages of memory via a currently undocumented
+`osalloc` API which usually uses POSIX's `mmap` call. On many environments `mmap`
+is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc`
+define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently
+only supported with `--mm:arc` or `--mm:orc`. (Since version 1.6)
+
+nimPage256 / nimPage512 / nimPage1k
+===================================
+
+Adjust the page size for Nim's GC allocator. This enables using
+`nimAllocPagesViaMalloc` on devices with less RAM. The default
+page size requires too much RAM to work.
+
+Recommended settings:
+
+- < 32 kB of RAM use `nimPage256`
+
+- < 512 kB of RAM use `nimPage512`
+
+- < 2 MB of RAM use `nimPage1k`
+
+Initial testing hasn't shown much difference between 512B or 1kB page sizes
+in terms of performance or latency. Using `nimPages256` will limit the
+total amount of allocatable RAM.
+
+nimMemAlignTiny
+===============
+
+Sets `MemAlign` to `4` bytes which reduces the memory alignment
+to better match some embedded devices.
+
+Thread stack size
+=================
+
+Nim's thread API provides a simple wrapper around more advanced
+RTOS task features. Customizing the stack size and stack guard size can
+be done by setting `-d:nimThreadStackSize=16384` or `-d:nimThreadStackGuard=32`.
+
+Currently only Zephyr, NuttX and FreeRTOS support these configurations.
+
 Nim for realtime systems
 ========================
 
-See the documentation of Nim's soft realtime `GC `_ for further
-information.
+See the `--mm:arc` or `--mm:orc` memory management settings in
+[MM](mm.html) for further information.
 
 
 Signal handling in Nim
@@ -602,7 +749,7 @@ The Nim programming language has no concept of Posix's signal handling
 mechanisms. However, the standard library offers some rudimentary support
 for signal handling, in particular, segmentation faults are turned into
 fatal errors that produce a stack trace. This can be disabled with the
-`-d:noSignalHandler` switch.
+`-d:noSignalHandler`:option: switch.
 
 
 Optimizing for Nim
@@ -635,38 +782,28 @@ passed to subroutines. The compiler does not copy strings that are a result of
 a procedure call, because the callee returns a new string anyway.
 Thus it is efficient to do:
 
-.. code-block:: Nim
+  ```Nim
   var s = procA() # assignment will not copy the string; procA allocates a new
                   # string already
+  ```
 
 However, it is not efficient to do:
 
-.. code-block:: Nim
+  ```Nim
   var s = varA    # assignment has to copy the whole string into a new buffer!
+  ```
 
 For `let` symbols a copy is not always necessary:
 
-.. code-block:: Nim
+  ```Nim
   let s = varA    # may only copy a pointer if it safe to do so
-
-
-If you know what you're doing, you can also mark single-string (or sequence)
-objects as `shallow`:idx:\:
-
-.. code-block:: Nim
-  var s = "abc"
-  shallow(s) # mark 's' as a shallow string
-  var x = s  # now might not copy the string!
-
-Usage of `shallow` is always safe once you know the string won't be modified
-anymore, similar to Ruby's `freeze`:idx:.
-
+  ```
 
 The compiler optimizes string case statements: A hashing scheme is used for them
 if several different string constants are used. So code like this is reasonably
 efficient:
 
-.. code-block:: Nim
+  ```Nim
   case normalize(k.key)
   of "name": c.name = v
   of "displayname": c.displayName = v
@@ -682,3 +819,4 @@ efficient:
     else: quit(errorStr(p, "expected: console or gui"))
   of "license": c.license = UnixToNativePath(k.value)
   else: quit(errorStr(p, "unknown variable: " & k.key))
+  ```
diff --git a/doc/nimdoc.cls b/doc/nimdoc.cls
new file mode 100644
index 0000000000..37039f1302
--- /dev/null
+++ b/doc/nimdoc.cls
@@ -0,0 +1,196 @@
+\ProvidesClass{nimdoc}[2022/04/17, 2018/01/01 LaTeX2e nonstandard class]
+
+\LoadClass[a4paper,11pt]{article}
+
+\usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry}
+
+%   for 2-sided printing with larger inner "binding" margin
+%\usepackage[a4paper,xetex,twoside,left=4cm,right=2cm,top=1.5cm,bottom=2cm]{geometry}
+%   for e-readers with 1.77:1 aspect ratio (e.g. 1920x1080)
+%\usepackage[xetex,paperheight=27.6cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+%   for e-readers with 1.45:1 aspect ratio (e.g. 1200x825)
+%\usepackage[xetex,paperheight=22.5cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+%   for e-readers with 1.33:1 aspect ratio (e.g. 1872x1404)
+%\usepackage[xetex,paperheight=20.7cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+
+\usepackage{fontspec}
+% logic to select default font with some fall-back fonts.
+\IfFontExistsTF{Times New Roman}{%
+  \setmainfont{Times New Roman}  % the default font
+  \typeout{========================================= nim: using Times New Roman}
+}{
+  \IfFontExistsTF{FreeSerif}{%
+    \setmainfont{FreeSerif}  % fallback #1 - official GNU font, resembles Times
+    \typeout{========================================= nim: using FreeSerif}
+  }{
+    \IfFontExistsTF{DejaVuSerif}{%
+      \setmainfont{DejaVuSerif}  % fallback #2 - very widespread free font
+      \typeout{========================================= nim: using DejaVuSerif}
+    }{
+      \typeout{!!!!!!!!!!!!!!!!!!! Fonts not found !!!!!!!!!!!!!!!!!!!!!!!}
+    }
+  }
+}
+
+% default monospace font for code:
+\usepackage{GoMono}
+\usepackage{relsize}
+% make this monospace font 2 steps smaller to hold 80-character line
+\newcommand{\rstverbblockfont}{\smaller[2]}
+\newcommand{\rstverbinlinefont}{\smaller}
+
+\usepackage{parskip}  % paragraphs delimited by vertical space, no indent
+\usepackage{graphicx}
+
+\usepackage{makeidx}
+\newcommand{\nimindexterm}[2]{#2\index{#2}\label{#1}}
+\makeindex
+
+\usepackage{dingbat} % for \carriagereturn, etc
+\usepackage{fvextra}  % for code blocks (works better than original fancyvrb)
+\fvset{
+  breaklines,
+  breakafter={=}:|\_\{\}[](){,}.;+-*/'",
+  breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}},
+  breaksymbolright=\color{red}{\small\carriagereturn}
+}
+\fvinlineset{%
+   breaklines,
+   breakafter={=}:|\_\{\}[](){,}.;+-*/'",
+     % that does not work at all when we underline inline code by ulem :-(
+   commandchars=\\\{\}
+}
+
+\usepackage{scrextend}  % for the `addmargin` environment
+
+\usepackage[table]{xcolor}
+\usepackage[urlbordercolor=blue,linkbordercolor=cyan,
+            pdfborderstyle={/S/U/W 1}]{hyperref}
+\usepackage{enumitem}  % for option list, enumList, and rstfootnote
+
+\usepackage[most]{tcolorbox}  % boxes around admonitions, code blocks, doc.item
+
+\newtcolorbox{rstadmonition}[1][]{blanker, breakable,
+     left=3mm, right=0mm, top=1mm, bottom=1mm,
+     before upper=\indent, parbox=false, #1}
+
+\newtcolorbox{rstquote}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     parbox=false,
+     borderline west={0.3em}{0pt}{lightgray},
+     borderline north={0.05em}{0pt}{lightgray},
+     borderline east={0.05em}{0pt}{lightgray},
+     borderline south={0.05em}{0pt}{lightgray}}
+
+\definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6}
+
+\usepackage{booktabs}
+\belowrulesep=0ex
+\aboverulesep=0ex
+\renewcommand{\arraystretch}{1.1}
+
+\newtcolorbox{rstprebox}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     borderline ={0.1em}{0pt}{rstframecolor},
+     before upper=\indent, parbox=false, #1}
+
+\newenvironment{rstpre}{%
+\VerbatimEnvironment\begingroup\begin{rstprebox}%
+\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}%
+{\end{Verbatim}\end{rstprebox}\endgroup}
+
+\newtcolorbox{rstdocitembox}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     borderline ={1pt}{0pt}{cyan},
+     before upper=\indent, parbox=false, #1}
+
+% Inline code formatting: grey underline,
+% use \Verb from fvextras e.g. to display -- correctly as double -
+\usepackage[normalem]{ulem}
+\newcommand\rstuline{\bgroup\markoverwith{\textcolor{rstframecolor}{\rule[-0.8ex]{2pt}{1.0pt}}}\ULon}
+
+\newcommand{\rstcode}[1]{%
+{\rstverbinlinefont\Verb{\rstuline{#1}}}%
+}
+
+\newcommand{\rstcodeitem}[1]{\Verb{#1}}
+
+\newenvironment{rstdocitem}{%
+\VerbatimEnvironment\begingroup\begin{rstdocitembox}%
+\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}%
+{\end{Verbatim}\end{rstdocitembox}\endgroup}
+
+
+\newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}}
+\ifdim\linewidth<30em
+  \def\rstoptleftmargin{0.4\linewidth}
+  \def\rstoptlabelwidth{0.35\linewidth}
+\else
+  \def\rstoptleftmargin{12em}
+  \def\rstoptlabelwidth{10.5em}
+\fi
+\newenvironment{rstoptlist}{%
+\begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}}
+
+\usepackage{multirow}
+\usepackage{tabulary}  % tables with adjustable cell width and no overflow
+% make tabulary prevent overflows (https://tex.stackexchange.com/a/195088)
+\tymin=60pt
+\tymax=\maxdimen
+% to pack tabulary into a new environment, special syntax is needed :-(
+\newenvironment{rsttab}[1]{\tabulary{\linewidth}{#1}}{\endtabulary}
+
+\newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}}
+\newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}}
+
+\newcommand{\rsthA}[2][]{\section[#1]{#2}}
+\newcommand{\rsthB}[2][]{\subsection[#1]{#2}}
+\newcommand{\rsthC}[2][]{\subsubsection[#1]{#2}}
+\newcommand{\rsthD}[2][]{\paragraph[#1]{#2}}
+\newcommand{\rsthE}[2][]{\paragraph[#1]{#2}}
+
+\newcommand{\rstovA}[2][]{\section*[#1]{#2}}
+\newcommand{\rstovB}[2][]{\subsection*[#1]{#2}}
+\newcommand{\rstovC}[2][]{\subsubsection*[#1]{#2}}
+\newcommand{\rstovD}[2][]{\paragraph*[#1]{#2}}
+\newcommand{\rstovE}[2][]{\paragraph*[#1]{#2}}
+
+% Syntax highlighting:
+\newcommand{\spanDecNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanBinNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanHexNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanOctNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanFloatNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanIdentifier}[1]{#1}
+\newcommand{\spanKeyword}[1]{\textbf{#1}}
+\newcommand{\spanStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanLongStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanCharLit}[1]{#1}
+\newcommand{\spanEscapeSequence}[1]{#1}
+\newcommand{\spanOperator}[1]{\textbf{#1}}
+\newcommand{\spanPunctuation}[1]{#1}
+\newcommand{\spanComment}[1]{\emph{#1}}
+\newcommand{\spanLongComment}[1]{\emph{#1}}
+\newcommand{\spanRegularExpression}[1]{#1}
+\newcommand{\spanTagStart}[1]{#1}
+\newcommand{\spanTagEnd}[1]{#1}
+\newcommand{\spanKey}[1]{#1}
+\newcommand{\spanValue}[1]{#1}
+\newcommand{\spanRawData}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanAssembler}[1]{#1}
+\newcommand{\spanPreprocessor}[1]{#1}
+\newcommand{\spanDirective}[1]{#1}
+\newcommand{\spanCommand}[1]{#1}
+\newcommand{\spanRule}[1]{#1}
+\newcommand{\spanHyperlink}[1]{#1}
+\newcommand{\spanLabel}[1]{#1}
+\newcommand{\spanReference}[1]{#1}
+\newcommand{\spanOther}[1]{#1}
+\newcommand{\spantok}[1]{\fbox{#1}}
+\newcommand{\spanPrompt}[1]{\textcolor{red}{\textbf{#1}}}
+\newcommand{\spanProgramOutput}[1]{\textcolor{darkgray}{\textbf{#1}}}
+\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}}
+\newcommand{\spanoption}[1]{\textbf{\textcolor{darkgray}{#1}}}
+
+% Never allow text overflow to margin:
+\setlength\emergencystretch{\hsize}\hbadness=10000
diff --git a/doc/nimdoc.css b/doc/nimdoc.css
index db9a7ce979..a9e4ac9c6a 100644
--- a/doc/nimdoc.css
+++ b/doc/nimdoc.css
@@ -35,7 +35,13 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #c4891b;
   --number: #252dbe;
   --literal: #a4255b;
+  --program: #6060c0;
+  --option: #508000;
   --raw-data: #a4255b;
+
+  --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E");
+  --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E");
+  --clipboard-image: var(--clipboard-image-normal)
 }
 
 [data-theme="dark"] {
@@ -63,68 +69,56 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #bd93f9;
   --number: #bd93f9;
   --literal: #f1fa8c;
+  --program: #9090c0;
+  --option: #90b010;
   --raw-data: #8be9fd;
+
+  --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E");
+  --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E");
+  --clipboard-image: var(--clipboard-image-normal);
 }
 
-.theme-switch-wrapper {
+@media (prefers-color-scheme: dark) {
+  [data-theme="auto"] {
+    --primary-background: #171921;
+    --secondary-background: #1e202a;
+    --third-background: #2b2e3b;
+    --info-background: #008000;
+    --warning-background: #807000;
+    --error-background: #c03000;
+    --border: #0e1014;
+    --text: #fff;
+    --anchor: #8be9fd;
+    --anchor-focus: #8be9fd;
+    --input-focus: #8be9fd;
+    --strong: #bd93f9;
+    --hint: #7A7C85;
+    --nim-sprite-base64: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAARMAAABMCAYAAABOBlMuAAAACXBIWXMAAAsTAAALEwEAmpwYAAAFFmlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgNS42LWMxNDggNzkuMTY0MDM2LCAyMDE5LzA4LzEzLTAxOjA2OjU3ICAgICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cGhvdG9zaG9wPSJodHRwOi8vbnMuYWRvYmUuY29tL3Bob3Rvc2hvcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHhtcDpDcmVhdGVEYXRlPSIyMDE5LTEyLTAzVDAxOjE4OjIyKzAxOjAwIiB4bXA6TW9kaWZ5RGF0ZT0iMjAxOS0xMi0wM1QwMToyMDoxMCswMTowMCIgeG1wOk1ldGFkYXRhRGF0ZT0iMjAxOS0xMi0wM1QwMToyMDoxMCswMTowMCIgZGM6Zm9ybWF0PSJpbWFnZS9wbmciIHBob3Rvc2hvcDpDb2xvck1vZGU9IjMiIHBob3Rvc2hvcDpJQ0NQcm9maWxlPSJzUkdCIElFQzYxOTY2LTIuMSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDplZGViMzU3MC1iNmZjLWQyNDQtYTExZi0yMjc5YmY4NDNhYTAiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6ZWRlYjM1NzAtYjZmYy1kMjQ0LWExMWYtMjI3OWJmODQzYWEwIiB4bXBNTTpPcmlnaW5hbERvY3VtZW50SUQ9InhtcC5kaWQ6ZWRlYjM1NzAtYjZmYy1kMjQ0LWExMWYtMjI3OWJmODQzYWEwIj4gPHhtcE1NOkhpc3Rvcnk+IDxyZGY6U2VxPiA8cmRmOmxpIHN0RXZ0OmFjdGlvbj0iY3JlYXRlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDplZGViMzU3MC1iNmZjLWQyNDQtYTExZi0yMjc5YmY4NDNhYTAiIHN0RXZ0OndoZW49IjIwMTktMTItMDNUMDE6MTg6MjIrMDE6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyMS4wIChXaW5kb3dzKSIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz4JZNR8AAAfG0lEQVR4nO2deViTZ7r/7yxkJaxJ2MK+GCBAMCwS1kgUFQSKK4XWWqsz1jpjp3b0tDP1V+eqU391fqfT/mpPPd20drTFDS0KFEVWJSGAEgLIZpAICBJACIRs549Rj1WILAkBfD/XlevySp68z/0S3+/7vPdzLyidTgcLkU2bd+z39/f/q1gshsrKSoJELFCa2iaEuU9K6kb+8uXxv54/fzE8L/eswNT2zCfQpjbAGKS8lPFKSEjIXiaTCSEhIeDj4xNnapsQ5j6rktZGp6UlfxIdzQVzCplmanvmG1hTG2BIAtlc26CgoDfT0tL2e3l5AQCAjY0NkMnk/a9s2k6rrKw8UV8n1JjYTIQ5RlAw14KzmL3xze1vfJyUuMJaq9UCFovFm9qu+YbBxcSPFUYkk8l2Q0NDsvo6ocrQx5+I8Ih4bz6f/0l8fHyKlZXV4/dRKBQwmcwwMpn8A4FAoPgHhH9bV1sxa488wZxoaycnJ/a9e/duCa5fkc3WvAiTI4Ib77p+XdqHG9anbfLy8gAAgLGxMdBpF+bjvzExqJj4scKI0dHRnwQHB++orq7+AgDeMuTxJ2Jl4rqU9PT0EwEBAUQCgTDuGAaDAampqYepVKpHUHDk325Ulw0a266YuFW+Gzdu/MDPz29jfn7+XgA4aOw5ESZP6kvpCXv3vnM8NiaSamVl+fj9BepGNDoGFRN7e/slcXFxO1xcXMDJyWnH7j//H/fi4uJdgutXmgw5z5O8smn7X9euXbvf29sbMBjMhONQKBRYWVlBbGzsbjMzM3JoOG+/sKKwy1h2rd/4elpGRsYuLy+vaDweD2w2Oy1h5ZrCvEunEaeeiVnMiabyl/F2/+X9P+8JDPQHHA5napMWBAYTk6DgSNuEhIS9DAYDAP7tq1i6dOkqOp3OWbNu0wens44emeoxA9lcWwKBYEMkEm2JRKIdHo+3QKFQWJ1Op8ZgMER3d/dVq1evTnFycpr0MSkUCsTExGzH4/Gk1LTME/39/TI0Go1FoVCg1WrVY2NjipGRkcGRkRH5dPwrEZHLXMPCwjJSUlIy3dzcfB+97+rqGhYSEpIOAIiYmBguN3zL77dt3uPh4W5qUxYUBhMTb2/vjeHh4cvR6P/dILK0tITIyEg7BweHr363/Z3Ampqaf1Zcu/zMKiVsyVJvMplsRyKR7IhEor2FhYUbhUJhJCYm2pFIJB6JRAIymQx4PB7QaDRoNBowMzMDJycnwOOn7icjEokQGxu7icFgbLp///7jFY1WqwWlUgkjIyOgUCgO7Ni5Rz48PCwfHh7uGRkZeaBQKOSjo6ODCoVCXlNVKn/6uCsT13FXrVr1emho6BYKhfLMnP7+/omrU9LPX8g+UThloxEMxqJFXjxESAyPQcSEExrLWLNmzW57e/txP/fw8ABHR8cdDAaDt3xF2ru9vb03sVgs0cbGxs/FxWVZUlISj0aj+dna2oKtrS1M5PcwJCgUCry8vODRrs84vPfoH6OjoyCXy6Gvr+/R6+CWrX9s7evrk/b19bWr1Wqli4sLZ8OGDe95eXmxUSjUuAd0cHDwjoqK2sYKXFIhvnldYYTTQpgU4/8+jyASCYDGoCd+ZkYYF8OICYezl8PhuOkbQyAQIDo62s/NzS2np6cHbGxsgEajAYFAAAwGA1gsFia6CE0NgUAABwcHsLe3B61WC2q1eo9WqwWNRgNKpRLUajUQiUSgUCh6zwGHwwGTydzo5+eXBQBnZu8MEJ5keHhYPqyYWMtHR0ZBpVIhYj9FUDONgOUvT12+du3avMDAQJjssdRqNWCxCyrEZdLodDoQi8Ulx44de628NL/V1Pa8iERE8l2dHB2CJvpcq9Nqbt1qKURWj1Njxld0ZGTkAW9v70kLCQC8sEIC8O/HKx8fn2gmk8kHgCk7pRFmzrWyAikASE1tx0Jj2uH0EZHL/N7YtuvT4OBgzmz4OBYSeDweIiMjt2S++vtMP1YYEmmJsCCY8mNOIJtr6+zsHBcZGXmIw+G4mZubG8m0hU9HRwcUFxe/KxQKTyDRsQjznSmJCS9+dVRERMTfQ0NDo2xtbfUGiSFMjtHRUaitrc3Jzc09kHvxVLmp7UFAmC6oZQkvrZLL5RJhReHtiQb5scKIXC7371FRUX90dnYGIpE4JR8Jgn40Gg20t7fXFxYWfnr9+vWjz8sdYi+Osh4vzgUBwZSgtu94V+fs7Hx7YGCgra6u7khLS0u2RCwYeTQgKmYFh8fj/f/g4OAldnZ2prR1wdPd3Q1CofBQSUnJkdLi3N8E93FCY6k+Pj48FxcXjlar1ZSWlh65VvYr4kREmDNg79+/D3FxcW5OTk5uXl5evNbW1tL0jK3ZXV1d1ykUintycvInoaGhdkj+gvGxs7MDPp+/m0AgWMQvS/lyeHhYTqPRPJycnIJSU1NZ3t7eW2g0Gly/fv2oWq1Gij0hzClQ/gHhpLS0tEM8Hm/7I8Ho7++HlpYWsLa2Bg8PDxOb+OKhUCigqakJ7t+/D25ubuDu7g4oFAp0Oh08ePAAvv7666TTWUdzTG0nAsKTYMU3ryuSU18+4+bmFrZo0SIOAICVlRUsXrx4zkakLnRIJBI8CgJ8MtdJp9NBZ2enqL29XWRC8xAQxgUNAHD+3L8KGhoaCp78ABES04JCoX4jJAAAAwMDUFtbe96YpRMQEKbL41DU5ubmko6Ojj2PSgggzD36+/vrb9y4cX425zzw93/8EBjon2is44+NjSkePBjqGRwc7G5v7xBV19w8U5B/3qgrr9+/uWtXUuKKD/TZ9MXh/066/OuFmunO8dGBQ98HBbGSp/t9U6LRaDXK0dHBoeFhuVzeL22/0yFqamopufjLqRJ933ssJi0tLSXV1dWHGAzGbuObOzs8ubqa71vZKpUKOjo6blwpOF8zm/Mu5cVkLlkSaswprAHAaVihgK7O7oSGxltvfXLon3nXK4RHT2cdN4pfKDCAlZyUuMJan02nTmczAaBmunPw4qI3cbnh0/36XICq0+lgcPABp7OrK629vUP5z8++LLh2XXD05L++yxrvC4/F5EZ12WBS8saLS5Ys2U2lUufUY45SqQSlUgkqlQrUavXj19jYGGg0GtBoNKDT6UCn05VotVq1TqfToFAojFar1eh0Og0Wi8XhcDgeGo1+/PhgZmYGOBwOsFgsmJmZ/eY1F+nt7YXa2trs2Z73wdCQBgCMHp1IJpHA09MdPD3dLRIS+OtKisvWvbP7vf2lZdePVFwzbHTwyMiI3hidkZFRUKvUYzOZ48HQkBIA5nWqBAqFAktLC7C0tADmIh88Pz4uMSyUk7hn776DV4tKPn/6d/lNxp1MJqsRCASf8vn8XdMpOjRTVCoVjI2NgUqlAq1WCyMjI9DX1wf379+Hvr6+/Q8ePOgdGRmRKxSKx0WLFAqFXKlUKnQ6nUar1arHq47mxwrD4/F4Eg6HI2GxWDwej7cgkUjWFAqFam5uTjU3N6eRyeQPLSwswNraGqysrIBAIDwWFywW+zja11Qi29LSclIikeSZZPJZBovBAI8XA8HBQR9kZZ3lR8cmvFZSlGe00p8IkwONRkNERBj4+i7a4+XpHv307/IbMakWlciXJbx0nMPh7Jqo0JGh0el0MDo6Cl1dXSCVSkEmk7177969W319fe1DQ0M9KpVKoVarlWq1WjndNhUPG3ApAWDcOxLTLwSDwWAOotFoDBaLxRMIBAsrKysne3t7Xzqd7k2n0/c4OzsDlUoFHA4364IyMDAATU1NxdWikhcq6tXKyhJezljPJZKI2eERS5cZeoWCMD2srCwhPX0tVzk2djiCG//GtfLLUoBxShB0dHTU3Lx580sLC4vtJBLJKMZoNBqQSqUglUqPdnR01PT09DT19/fLHjx40DM0NNQ72933GiSVGgB4JFQK+LfoSAGgnL04yppEIh2xtLS0t7GxcaFSqR7Ozs4fMRgMcHR0nJX8pJs3b54Ui8UXjT7RHIRMIkFK8irfwcEHPwQELUmqvYHUGJkLmJubw8YNa/i9vfffY/px3myQiDTPiEl9nVDDX576jaenZ7SnpyfLUJNrNBqQyWRw+/bt4x0dHTdkMlltV1dXw/XygjkdEv4wB0YOAK0AUM70C8HQ6fSzdDrdm0qlejg6OrLc3Ny2MBiMadWjfR4PHjyAmzdvZs/1v5MxoVAokJK8iicWS95k+nH+s0EiQhqpzQGoVFtYk5a87ba0XQAA34xbpagg/5zoT7s/OGNnZ8eaaYkBuVwOnZ2d5VKpVNTS0lLS2NhYWFVZ3Dujg5qQh6uY+ocvCAiKIPn4+Jz19PSMdnV15VCpVL6Dg4NBViw6nQ5EItHRpqamqzM+2DzHzo4O69amftLQeKsAZrDLgmBY/PyYsCIhfs+SiKUFE5Y8EwqFx11cXDihoaFTjjFAoVAwPDwMHR0dourq6jNCofDHhZqUVnvjmgIAcgAgJyg40mLRokX8kJCQjT4+PussLS1n1JPl7t27UFxcfHguB6mNjY2B7G4naNRTWyygUCjAYDGAx+PB0sICSCSi3vFYLBbCwjjA8vddBQtATKb7d3saBwc7IJPJBpsHjUGDGRYLJBIJLK0sAfucmyIGg4FFi3y8AwNZtycUk5KiS02vvf7WWQaDkejg4DApQwAeh3xDaWnpPoFAcPxFqnP6sEvgGf+A8Bx3d/cvIyIiNi1evHjT8wpNj8fAwACUlZW9P9dD5+/ckcFbf9gd2dcnn9LNAovF4inmZHtXNxdOdBR3+/JlS33pdP29wolEInA4weuiYxOy5vvuTkeHDHb+8c8xvb33Z3R9/N+Df+uIjYk02DwkEsna2trS1d/fNyGeF7uTyw1/7g3R3t4O2OxA/TVghULhcQqFQk1JSfmYSNR/5wD4d6EfgUBwvLS09IhUKhW9qAV5H9YjKQwJi6uvrKw8ERoamhkSEpKp7w7yJEqlEiQSyZmysrJv53qjdaVSCZdyTk+3qFMrAJRHRPLPN95qeifj5fU7mYt8JhyMRqMhMJDFdnF25gDAvBYTpXIMWlpay2fq/8m5mDcIABYGnEcGAGI/VlhBZWX1yZdSkz55OX0dV5+7w9bGGvz8mPrFpK62QskJjf2GTqd7x8bGbpnID4BCoUAmk0lLSkqOiESik2UleS/MakQflYKrXQDQxY1a3tTe3i6KiIjY5OXlxX7e9+rr6wsuXbr0t4ffn9OgMWjghMZQRcLp+8GulRVI/QPC37Wxtnal0ajJtjY2E451ZjiBra31vE9lR2PQQKFQaAAwo98Yi8Xq9fpPd56HO6rlvKWJv/PwcK+JilyCmajWMw6HAzs7+rMFpQOCIn6zHywSFvXm5eUdFAqFZ9Rq9bgHa2trq79w4cK+zz49cAARkmcpL81v/a/Dhz49d+7c3qqqqjyVSjXuOJ1OBxKJpDw3N/fA5V+zax6978cKw/sHhM/raMrnUVdboSy4fPWQSFSjd5yFBQWIRNKEd2IEw1J4JUd88WL+R51d3XrHWVDMnxUTa2tr1zXrNiUGsrmPf7DS4tymCxcu7Kuurs55+kKQSqVN586d23vs+8NHDXUCC5Wzp3/Iy8rKeruysvLM2Nhvo7VVKhXU1tYWnj17du/T7UOdnZ2D7OzsfGGB09raVi4S1RzXl0eFw+EAj8chYjKLVFffyOrq1C8mJBLpWTFRKBRyDofzC4vFWvXk+1ev/CLOzs7eKxAIslQqFeh0Oujp6enKzs7em/XTd7OayTqfKb56sT4rK+sPAoHg5KO/o0KhAKFQmHXy5MkdF3/5+TeZmctXpIXZ29v7zqVcKWNRX1epuXu3U/y8pEw0GmndOZt0dnXVDw0P6/W5oNHoZ30mQ0NDPb29vfvj4+Pf3rR5B/7od188XnEUXr4gDgmL+0NfX5/U19d3d3l5+YGfTnyDtLmcIhXXLsu4UcvfR6PRGGtra9eysrIjYrE45+kt4Fheou/69es/unnz5vm7d+/Wmsre2WRkZGTQ1DYg/JYGiUiTm1ugBAC9IfHPiEmDpFITE7fqJI/H27lmzZpDq5LWtz55t6wUXO3ihMYerK+vz2tpaUFaM0yT8tL81ujYle+TSCTrvEunBU9/voTLd92wYcPHVCqV39XVdXCu7+oYCp1O90Kc50Jk3I5+xVcv1jc3N5d4enpSMzIyvkpK3sh78nORsKg3++yPBS/q1q+hKCm61DSekERGJ3ikp6d/ERsbm1xVVXWwtbX1hRFtFAqFPMLMUyZsDyoQCI7LZDKIiIjwzczM/GpV0vro2TTsRSUqZoX3+vXrP1u9enXi0NAQiESirIdRtggIc5oJ40zq6uryGhoa8ry8vBJCQ0O9USjU94mrN7yWc+EnvaXb5gJMvxCMp6cnl0Kh2Le1tZVXXLs8L1LXefGrWRkZGZ/x+XyeUqkEkUh0vqenZ14HZyG8OEwoJjdrygd37NxTEBkZmWBtbQ3BwcEeKBTq+/UbX3/355Pfzlmn66qk9dGbN29+k8PhbCSRSNDZ2Snb9ae/HCkpKTksEhbN2QTD5NSX+Vu3bj0cHBzsjcFg4O7du1BWVvbNwxB9BIQ5j94I2Fu3bhXW19cDl8sFLBYLHA7Hg0wmf/e77e84ffXlPz6fLSMnQ2paZkJ4eHjmtm3b+B4eHvZkMhlQKBTY29s72dvbfxgUFJT8x7ffP1NRUfHjXErnZ/qFYKKjo7dt3rz5g8DAQPtH/XHa2tpqGhsbC55/BASEuYFeMblz505NTU3NgfDw8PcwGAygUCjw9fW1IJPJn/1130Hv0tLSI4WXL4hny9inYS+Osvbz80tgMpn8jIwMPovFch2vpoiDgwM4ODhwfH19OYsWLeJv3/Hu+cbGxquzXZz5aZYlvMRJT0/fFhkZue3JZmfd3d0gEolOIr4ShPmEXjFpkFRqXlrzSnFnZ+d7Tk5OjzNfXVxcICMjY6ezszNnVdL6vU8HWhmbgKAIkrOzMyc1NTXz0YU4maAuOp0OK1as4EVFRfGEQqHg1dfePHzr1q2rs71S8WOF4f38/BLS09M/iIyM5DxdxLq5uVlcVVU1bgVwBIS5il4xAQCQyWRigUBwJikpKe3JVGQcDgdLly7l2tranti0ecf7IpEoy9hbxX6sMDydTvdevXr1ltjY2F3u7u6AxT73FJ7B3Nwc4uLiwthsdphQKCzZkL7l0/r6+oKbNeVG90+EhMXZL1++fFtycvKHrq6uz4igUqmE5ubmEiTHCWG+8dwrUXD9imz9xtd/jIuLS7N5KpsTjUZDUFCQE4PB+F4oFGYmJW888Mv5k4UTHGpGxC9LYaenp78VEhKyxdHRESgUyoyOh0KhwNraGuLi4qIDAgKi6+rqyjekb/mHMSN6N6RvSdu+ffseNpsdZm09ftuW+vp6EIvFSB9hhHnHpG7rUqm0orW1tdXS0tLj6TIEaDQaaDQaxMfH811dXTl/3Xfw+JUrVz411J01cfWG6IiIiC07d+5McHNzs7ewMGyOFw6HAwcHB6BSqVx3d/fwz7/4rkAgEBwXCoUnHpZonDGrU9J5MTEx27du3Zrm4uKC0beaqq6u/ry+vj7XEPMiIMwmkxKTimuXZe/u+fCkp6fnexPdUfF4PPj7+1szGIydLi4unF1/+kvenTt3RG1tbRXTqfma8lIG39/fP/HVV19NZrFYHpMpzjQTzMzMwNPTE+Pp6Zng6emZ4Ofnl5CesfV8bW1tznQe3/wDwvFeXl7Rvr6+Ca+88kpaUFCQh74GXzqdDrq7u6GpqankRQmdR1hYTNrhUFVVlcXj8d6ysrKy0OfstLS0hPj4eC6Xy+U2NzeDRCI5/sa2XeX37t1rGhwc7BoYGJBN1P+FFbiE5OzszGaxWImvvvrqpoCAAKfp+ERmCpPJBCaTmcnhcDJLS0u/TE59+YxUKhXoi/lg+oVgrKysGJaWlna2trYeaWlpXDabvTMgIGDSfp2KiorzbW1tL0zoPMLCYtJX6uVfs2u++PKowMPDgz+ZIslEIhECAgKAxWJlajSazJ6eHmhra4PW1tZvtmz9o6Czs7O+r6+vfWxsbFir1WosLCzsV6xYkcnj8d7z9vaelmPV0Hh5eYGnp+f2mJiY7UVFRZ/HL0v5tru7+5ZGo1FisVg8Docj4fF4CxsbG1c+nx/m7e39sYeHB7i4uIC5ufmU6r4ODQ1BZWXlifkSrYuA8DRTumIrKytPent78728vCb9HRQKBVgsFhwcHIBOpwObzd4yNja2RaVSwdDQEHR1dcHo6CjQaDRwdXWdsWPV0KBQKPDw8AA7O7udERERO2tra2FgYACoVCo4OTkBjUYDMpkMeDz+8WuqaLVaaGxsbL19+/YzSX8ICPOFqYrJidDQ0AwvLy/e80c/CwaDARKJBI86BdJoNHB3dwe1Wj0nViL6IJPJwGQywdnZGZRKJRAIBDBUx8OBgQEoLS39BtkORpjPTJg1PB61N64pmpqarvb39xvUiLkuJE9CJpPBxsbGYEICANDZ2SlHgtQQ5jtTEhMAgLq6ulyJRFJvDGNeREZGRkAikRSUFuci2cEI85opi0l+7hmBWCzOeV6dToTJcfv27cHr168jxbgR5j1TFhMAgObm5hKZDNl0MAQtLS3Xzpw6hkS8Isx7piUmUqlUIBAIJuyjgzA5Ojs7QSKRINGuCAuCaYmJsKKw68qVK59KJJIu5HFneiiVSigqKjouEolOmtoWBARDMC0xAQC4+MvPJadOnXq3ra1N8yL0dDEkOp0OSktLy/Pz8w8+3d4CAWG+Mm0xAQA4fuy/jl+8ePGju3fvGsqeBY9Wq4XKysrWU6dOvX31yi8mKyyFgGBoZiQmAAD/79D+fadPn96PCMrz0el0UFVV1frtt9+mj9fiAgFhPjNjMQEAyMvLO3Ds2LE/tLS0INmuerh27Vr9999//xoiJAgLEYOEntbVVigB4PNNm3cMpqSkfMRms50McdyFgkqlgqKiovJTp069nZ97BhEShAWJQePYj373xdF1GzbLFQrFx6Ghob766ne8KNy7dw+KiopO5ubmfmTK4tsICMbG4EkxWT99d35l4rre/v7+D0NCQvh0Ot3QU8wL1Go1SKVSTX5+/sH8/PyDSP8bhIWOUTLsLuVklQcFR65pbGzcvnLlyvfc3NwsCASCMaaac+h0OhgaGoLq6uqaCxcu/OV01tGcTw7uM7VZCAhGx2jpug/vxAd58atzoqKitq1cuXKnvb29saabE+h0Oqiurpbm5eUdrK6uPlspuDrvY0hmO4YIhUIBGq1/X2CmNqFQKL3/79HomZ/z82xEowyy9zFr80zGDqPn/hdeviBmL47ad+fOnRsRERGbQkNDo62srIw97azT2dkJxcXFx0tKSo7Mdh8hY4LD4TDPH2U4MFjMc6tLmZmZzaj+Aw6H0/t9PB4PGCxmRudNJBL0ngeZTAI0Gj3jv+1szfM88Hic8cUEAKCmqlQOAN/ELU2qkEgkySwWK3HRokVcBoMxG9MbDZ1OB83NzdDU1FRQW1t7XiAQHJ+ovu18pbr6Rg6L5ZtoM0EhcUPT0tJW8tWRb0vQqIkvgKqqmhnVfrl2TfANXo+gjKlUio4OWc1M5sjOzjnQUH8rbqLPu3t6moaGhmfc+3q25tGHUqmECoEIUKbIrVkcEkONiIh4jcvlvu7s7OxLo9GmVe7QVCgUCujq6oKGhoaCioqKo9XV1WeM3YDMVPDik1gpyas+XrVyeaKXl8czjyANjbcgI/MNmkg49Q4ECPOH3NyC4RUr+M8IcHt7B1y9WlKRl3/5kElKnD1sfXEoJCzueEBAQGJYWFgGk8nk2djYAIFAgLm4pTw6Ogqjo6Mgl8vhxo0b50tLS4/U19fnLvS2FIWXfxEDQNLmLW9ueW1TxtchHDaQyWRTm4VgYkZHR6G+vhF+/NfP+y5e+vVjiVgwZpKVydOwF0dZW1lZOTGZTD6bzU4LCAiIptPp8HTDL1MwOjoKLS0tUFdXd1IsFudIpdKKgYGB7tloJTrX4MUnsVJTEj9etzY10dHRAQAAGm81wcsZW5CVyQInL69gNCGBjwcAGBx8ANnncypOnTr3H9nn/reD55wovvrQpyIHAHFUzIocGo3mQaPRfBwdHVlubm7bXF1dgcFgABqNNvruglwuh7t374JMJoOOjo7P79y5I+ru7m7q7e1tXQi7MzOh8PIv4pCw2DdaWtte37Au7aPIyCWAxWABjUbPif9HCMbjURtKiaQBfvr5zH9evlJ0uLQ4r/nJMXNiZTIRrMAlJAcHB18HBweWo6Mjy8rKajeJRAJLS0uwtLQECwsLoFAogMfjAYvFgpmZ2XNXMyqVCoaHh2FoaAiGh4cfvwYGBqCvrw+6u7vfvnfvXlNvb29rT09Pq0QsUM7S6c4rNqS/lrZ5U+YPRBKR9M7u9xwqBUUvtNAudH766XSLE8PR49ixE78/8tVnX403Zk7fUR46NUUAIPIPCMdTKJTdNjY2QKPRgE6nA51OB1tbWyCRSIDD4YBAIAAejwcCgfDYUajVakGlUoFarQadTvfY79HX1wf9/f0gl8tBLpfDvXv3HvXw+dxQPYYXMj+d+P7Mmzv+5OHr6/OJWq1GBHeB09TcUiKuq/coKS3/eqIx/wPkiIXC3w6YjAAAAABJRU5ErkJggg==");
+
+    --keyword: #ff79c6;
+    --identifier: #f8f8f2;
+    --comment: #6272a4;
+    --operator: #ff79c6;
+    --punctuation: #f8f8f2;
+    --other: #f8f8f2;
+    --escapeSequence: #bd93f9;
+    --number: #bd93f9;
+    --literal: #f1fa8c;
+    --program: #9090c0;
+    --option: #90b010;
+    --raw-data: #8be9fd;
+
+    --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E");
+    --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E");
+    --clipboard-image: var(--clipboard-image-normal);
+  }
+}
+
+.theme-select-wrapper {
   display: flex;
   align-items: center;
 }
 
-.theme-switch-wrapper em {
-  margin-left: 10px;
-  font-size: 1rem;
-}
-
-.theme-switch {
-  display: inline-block;
-  height: 22px;
-  position: relative;
-  width: 50px;
-}
-
-.theme-switch input {
-  display: none;
-}
-
-.slider {
-  background-color: #ccc;
-  bottom: 0;
-  cursor: pointer;
-  left: 0;
-  position: absolute;
-  right: 0;
-  top: 0;
-  transition: .4s;
-}
-
-.slider:before {
-  background-color: #fff;
-  bottom: 4px;
-  content: "";
-  height: 13px;
-  left: 4px;
-  position: absolute;
-  transition: .4s;
-  width: 13px;
-}
-
-input:checked + .slider {
-  background-color: #66bb6a;
-}
-
-input:checked + .slider:before {
-  transform: translateX(26px);
-}
-
-.slider.round {
-  border-radius: 17px;
-}
-
-.slider.round:before {
-  border-radius: 50%;
-}
-
 html {
   font-size: 100%;
   -webkit-text-size-adjust: 100%;
@@ -147,24 +141,39 @@ body {
   padding: 0;
   box-sizing: border-box; }
 
-.column,
-.columns {
+.column, .columns {
   width: 100%;
   float: left;
   box-sizing: border-box;
-  margin-left: 1%;
+  margin-left: 1%; }
+
+@media print {
+  #global-links, .link-seesrc, .theme-switch-wrapper, #searchInputDiv, .search-groupby {
+    display:none;
+  }
+  .columns {
+    width:100% !important;
+  }
 }
 
-.column:first-child,
-.columns:first-child {
+.column:first-child, .columns:first-child {
   margin-left: 0; }
 
+.container .row {
+  display: flex; }
+
 .three.columns {
-  width: 22%;
+  width: 25.0%;
+  height: 100vh;
+  position: sticky;
+  top: 0px;
+  overflow-y: auto;
+  padding: 2px;
 }
 
 .nine.columns {
-  width: 77.0%; }
+  width: 75.0%;
+  padding-left: 1.5em; }
 
 .twelve.columns {
   width: 100%;
@@ -230,6 +239,12 @@ select:focus {
 }
 
 /* Docgen styles */
+
+:target {
+  border: 2px solid #B5651D;
+  border-style: dotted;
+}
+
 /* Links */
 a {
   color: var(--anchor);
@@ -245,27 +260,32 @@ a.reference-toplevel {
   font-weight: bold;
 }
 
+a.nimdoc {
+  word-spacing: 0.3em;
+}
+
 a.toc-backref {
   text-decoration: none;
-  color: var(--text); }
+  color: var(--text);
+}
 
 a.link-seesrc {
   color: #607c9f;
   font-size: 0.9em;
-  font-style: italic; }
+  font-style: italic;
+}
 
-a:hover,
-a:focus {
+a:hover, a:focus {
   color: var(--anchor-focus);
-  text-decoration: underline; }
+  text-decoration: underline;
+}
 
 a:hover span.Identifier {
   color: var(--anchor);
 }
 
 
-sub,
-sup {
+sub, sup {
   position: relative;
   font-size: 75%;
   line-height: 0;
@@ -292,8 +312,7 @@ img {
     background: transparent !important;
     box-shadow: none !important; }
 
-  a,
-  a:visited {
+  a, a:visited {
     text-decoration: underline; }
 
   a[href]:after {
@@ -307,16 +326,14 @@ img {
   a[href^="#"]:after {
     content: ""; }
 
-  pre,
-  blockquote {
+  pre, blockquote {
     border: 1px solid #999;
     page-break-inside: avoid; }
 
   thead {
     display: table-header-group; }
 
-  tr,
-  img {
+  tr, img {
     page-break-inside: avoid; }
 
   img {
@@ -331,22 +348,18 @@ img {
   h1.title {
     page-break-before: avoid; }
 
-  p,
-  h2,
-  h3 {
+  p, h2, h3 {
     orphans: 3;
     widows: 3; }
 
-  h2,
-  h3 {
+  h2, h3 {
     page-break-after: avoid; }
 }
 
 
 p {
   margin-top: 0.5em;
-  margin-bottom: 0.5em;
-}
+  margin-bottom: 0.5em; }
 
 small {
   font-size: 85%; }
@@ -354,8 +367,7 @@ small {
 strong {
   font-weight: 600;
   font-size: 0.95em;
-  color: var(--strong);
-}
+  color: var(--strong); }
 
 em {
   font-style: italic; }
@@ -376,8 +388,7 @@ h1.title {
   text-align: center;
   font-weight: 900;
   margin-top: 0.75em;
-  margin-bottom: 0em;
-}
+  margin-bottom: 0em; }
 
 h2 {
   font-size: 1.3em;
@@ -404,36 +415,29 @@ h6 {
   font-size: 1.1em; }
 
 
-ul,
-ol {
+ul, ol {
   padding: 0;
   margin-top: 0.5em;
   margin-left: 0.75em; }
 
-ul ul,
-ul ol,
-ol ol,
-ol ul {
+ul ul, ul ol, ol ol, ol ul {
   margin-bottom: 0;
   margin-left: 1.25em; }
 
 ul.simple > li {
-    list-style-type: circle;
-}
+  list-style-type: circle; }
 
 ul.simple-boot li {
-    list-style-type: none;
-    margin-left: 0em;
-    margin-bottom: 0.5em;
-}
+  list-style-type: none;
+  margin-left: 0em;
+  margin-bottom: 0.5em; }
 
 ol.simple > li, ul.simple > li {
   margin-bottom: 0.2em;
   margin-left: 0.4em }
 
 ul.simple.simple-toc > li {
-    margin-top: 1em;
-}
+  margin-top: 1em; }
 
 ul.simple-toc {
   list-style: none;
@@ -442,8 +446,7 @@ ul.simple-toc {
   margin-top: 1em; }
 
 ul.simple-toc > li {
-    list-style-type: none;
-}
+  list-style-type: none; }
 
 ul.simple-toc-section {
   list-style-type: circle;
@@ -453,12 +456,10 @@ ul.simple-toc-section {
 ul.nested-toc-section {
   list-style-type: circle;
   margin-left: -0.75em;
-  color: var(--text);
-}
+  color: var(--text); }
 
 ul.nested-toc-section > li {
-  margin-left: 1.25em;
-}
+  margin-left: 1.25em; }
 
 
 ol.arabic {
@@ -505,12 +506,39 @@ hr.footnote {
   margin-top: 0.15em;
 }
 div.footnote-group {
-  margin-left: 1em; }
+  margin-left: 1em;
+}
 div.footnote-label {
   display: inline-block;
   min-width: 1.7em;
 }
 
+div.option-list {
+  border: 0.1em solid var(--border);
+}
+div.option-list-item {
+  padding-left: 12em;
+  padding-right: 0;
+  padding-bottom: 0.3em;
+  padding-top: 0.3em;
+}
+div.odd {
+  background-color: var(--secondary-background);
+}
+div.option-list-label {
+  margin-left: -11.5em;
+  margin-right: 0em;
+  min-width: 11.5em;
+  display: inline-block;
+  vertical-align: top;
+}
+div.option-list-description {
+  width: calc(100% - 1em);
+  padding-left: 1em;
+  padding-right: 0;
+  display: inline-block;
+}
+
 blockquote {
   font-size: 0.9em;
   font-style: italic;
@@ -519,7 +547,12 @@ blockquote {
   border-left: 5px solid #bbc;
 }
 
-.pre {
+blockquote.markdown-quote {
+  font-size: 0.9rem;  /* use rem to avoid recursion */
+  font-style: normal;
+}
+
+.pre, span.tok {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   font-weight: 500;
   font-size: 0.85em;
@@ -530,6 +563,16 @@ blockquote {
   border-radius: 4px;
 }
 
+span.tok {
+  border: 1px solid #808080;
+  padding-bottom: 0.1em;
+  margin-right: 0.2em;
+}
+
+.copyToClipBoard {
+  position: relative;
+}
+
 pre {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   color: var(--text);
@@ -548,7 +591,25 @@ pre {
   border: 1px solid var(--border);
   -webkit-border-radius: 6px;
   -moz-border-radius: 6px;
-  border-radius: 6px; }
+  border-radius: 6px;
+}
+
+.copyToClipBoardBtn {
+  visibility: hidden;
+  position: absolute;
+  width: 24px;
+  border-radius: 4px;
+  background-image: var(--clipboard-image);
+  right: 5px;
+  top: 13px;
+  background-color: var(--secondary-background);
+  padding: 11px;
+  border: 0;
+}
+
+.copyToClipBoard:hover .copyToClipBoardBtn {
+  visibility: visible;
+}
 
 .pre-scrollable {
   max-height: 340px;
@@ -613,8 +674,8 @@ table th {
   font-weight: bold; }
 
 table th.docinfo-name {
-    background-color: transparent;
-    text-align: right;
+  background-color: transparent;
+  text-align: right;
 }
 
 table tr:hover {
@@ -631,31 +692,31 @@ table.borderless td, table.borderless th {
   padding: 0 0.5em 0 0 !important; }
 
 .admonition {
-    padding: 0.3em;
-    background-color: var(--secondary-background);
-    border-left: 0.4em solid #7f7f84;
-    margin-bottom: 0.5em;
-    -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2);
-       -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2);
-            box-shadow: 0 5px 8px -6px rgba(0,0,0,.2);
+  padding: 0.3em;
+  background-color: var(--secondary-background);
+  border-left: 0.4em solid #7f7f84;
+  margin-bottom: 0.5em;
+  -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2);
+      -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2);
+          box-shadow: 0 5px 8px -6px rgba(0,0,0,.2);
 }
 .admonition-info {
-    border-color: var(--info-background);
+  border-color: var(--info-background);
 }
 .admonition-info-text {
-    color: var(--info-background);
+  color: var(--info-background);
 }
 .admonition-warning {
-    border-color: var(--warning-background);
+  border-color: var(--warning-background);
 }
 .admonition-warning-text {
-    color: var(--warning-background);
+  color: var(--warning-background);
 }
 .admonition-error {
-    border-color: var(--error-background);
+  border-color: var(--error-background);
 }
 .admonition-error-text {
-    color: var(--error-background);
+  color: var(--error-background);
 }
 
 .first {
@@ -689,8 +750,7 @@ div.footer, div.header {
   font-size: smaller; }
 
 div.footer {
-    padding-top: 5em;
-}
+  padding-top: 5em; }
 
 div.line-block {
   display: block;
@@ -707,19 +767,23 @@ div.topic {
 
 div.search_results {
   background-color: var(--third-background);
-  margin: 3em;
+  margin: 3vh 5vw;
   padding: 1em;
   border: 1px solid #4d4d4d;
-}
+  position: fixed;
+  top: 10px;
+  isolation: isolate;
+  max-width: calc(100vw - 6em);
+  z-index: 1;
+  max-height: calc(100vh - 6em);
+  overflow-y: scroll;}
 
 div#global-links ul {
   margin-left: 0;
-  list-style-type: none;
-}
+  list-style-type: none; }
 
 div#global-links > simple-boot {
-    margin-left: 3em;
-}
+  margin-left: 3em; }
 
 hr.docutils {
   width: 75%; }
@@ -817,9 +881,6 @@ span.classifier {
 span.classifier-delimiter {
   font-weight: bold; }
 
-span.option {
-  white-space: nowrap; }
-
 span.problematic {
   color: #b30000; }
 
@@ -899,8 +960,29 @@ span.Preprocessor {
 span.Directive {
   color: #252dbe; }
 
-span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
-span.Other {
+span.option {
+  font-weight: bold;
+  font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
+  color: var(--option); }
+
+span.Prompt {
+  font-weight: bold;
+  color: red; }
+
+span.ProgramOutput {
+  font-weight: bold;
+  color: #808080; }
+
+span.program {
+  font-weight: bold;
+  color: var(--program);
+  text-decoration: underline;
+  text-decoration-color: var(--hint);
+  text-decoration-thickness: 0.05em;
+  text-underline-offset: 0.15em; }
+
+span.Command, span.Rule, span.Hyperlink,
+span.Label, span.Reference, span.Other {
   color: var(--other); }
 
 /* Pop type, const, proc, and iterator defs in nim def blocks */
@@ -938,17 +1020,14 @@ span.pragmadots {
   border-radius: 4px;
   margin: 0 2px;
   cursor: pointer;
-  font-size: 0.8em;
-}
+  font-size: 0.8em; }
 
 span.pragmadots:hover {
-  background-color: var(--hint);
-}
+  background-color: var(--hint); }
+
 span.pragmawrap {
-  display: none;
-}
+  display: none; }
 
 span.attachedType {
   display: none;
-  visibility: hidden;
-}
+  visibility: hidden; }
diff --git a/doc/nimfix.rst b/doc/nimfix.rst
deleted file mode 100644
index d105346da2..0000000000
--- a/doc/nimfix.rst
+++ /dev/null
@@ -1,58 +0,0 @@
-.. default-role:: code
-
-=====================
-  Nimfix User Guide
-=====================
-
-:Author: Andreas Rumpf
-:Version: |nimversion|
-
-**WARNING**: Nimfix is currently beta-quality.
-
-Nimfix is a tool to help you upgrade from Nimrod (<= version 0.9.6) to
-Nim (=> version 0.10.0).
-
-It performs 3 different actions:
-
-1. It makes your code case consistent.
-2. It renames every symbol that has a deprecation rule. So if a module has a
-   rule `{.deprecated: [TFoo: Foo].}` then `TFoo` is replaced by `Foo`.
-3. It can also check that your identifiers adhere to the official style guide
-   and optionally modify them to do so (via `--styleCheck:auto`).
-
-Note that `nimfix` defaults to **overwrite** your code unless you
-use `--overwriteFiles:off`! But hey, if you do not use a version control
-system by this day and age, your project is already in big trouble.
-
-
-Installation
-------------
-
-Nimfix is part of the compiler distribution. Compile via::
-
-  nim c compiler/nimfix/nimfix.nim
-  mv compiler/nimfix/nimfix bin
-
-Or on windows::
-
-  nim c compiler\nimfix\nimfix.nim
-  move compiler\nimfix\nimfix.exe bin
-
-Usage
------
-
-Usage:
-  nimfix [options] projectfile.nim
-
-Options:
-
-  --overwriteFiles:on|off       overwrite the original nim files. DEFAULT is ON!
-  --wholeProject                overwrite every processed file.
-  --checkExtern:on|off          style check also extern names
-  --styleCheck:on|off|auto      performs style checking for identifiers
-                                and suggests an alternative spelling;
-                                'auto' corrects the spelling.
-
-In addition, all command line options of Nim are supported.
-
-
diff --git a/doc/nimgrep.md b/doc/nimgrep.md
new file mode 100644
index 0000000000..63f7600513
--- /dev/null
+++ b/doc/nimgrep.md
@@ -0,0 +1,128 @@
+=========================
+  nimgrep User's manual
+=========================
+
+:Author: Andreas Rumpf
+:Version: 1.6.0
+
+.. default-role:: option
+.. contents::
+
+Nimgrep is a command line tool for search and replace tasks. It can search for
+regex or peg patterns and can search whole directories at once. User
+confirmation for every single replace operation can be requested.
+
+Nimgrep has particularly good support for Nim's
+eccentric *style insensitivity* (see option `-y` below).
+Apart from that it is a generic text manipulation tool.
+
+
+Installation
+============
+
+Compile nimgrep with the command:
+
+  ```cmd
+  nim c -d:release tools/nimgrep.nim
+  ```
+
+And copy the executable somewhere in your ``$PATH``.
+
+
+Command line switches
+=====================
+
+.. include:: nimgrep_cmdline.txt
+
+Path filter options
+-------------------
+
+Let us assume we have file `dirA/dirB/dirC/file.nim`.
+Filesystem path options will match for these parts of the path:
+
+| option              | matches for                        |
+| :------------------ | :--------------------------------  |
+| `--[not]extensions` | ``nim``                            |
+| `--[not]filename`   | ``file.nim``                       |
+| `--[not]dirname`    | ``dirA`` and ``dirB`` and ``dirC`` |
+| `--[not]dirpath`    | ``dirA/dirB/dirC``                 |
+
+Combining multiple filter options together and negating them
+------------------------------------------------------------
+
+Options for filtering can be provided multiple times so they form a list,
+which works as:
+* positive filters
+  `--filename`, `--dirname`, `--dirpath`, `--inContext`,
+  `--inFile` accept files/matches if *any* pattern from the list is hit
+* negative filters
+  `--notfilename`, `--notdirname`, `--notdirpath`, `--notinContext`,
+  `--notinFile` accept files/matches if *no* pattern from the list is hit.
+
+In other words the same filtering option repeated many times means logical OR.
+
+.. Important::
+  Different filtering options are related by logical AND: they all must
+  be true for a match to be accepted.
+  E.g. `--filename:F --dirname:D1 --notdirname:D2` means
+  `filename(F) AND dirname(D1) AND (NOT dirname(D2))`.
+
+So negative filtering patterns are effectively related by logical OR also:
+`(NOT PAT1) AND (NOT PAT2) == NOT (PAT1 OR PAT2)`:literal: in pseudo-code.
+
+That means you can always use only 1 such an option with logical OR, e.g.
+`--notdirname:PAT1 --notdirname:PAT2` is fully equivalent to
+`--notdirname:'PAT1|PAT2'`.
+
+.. Note::
+   If you want logical AND on patterns you should compose 1 appropriate pattern,
+   possibly combined with multi-line mode `(?s)`:literal:.
+   E.g. to require that multi-line context of matches has occurrences of
+   **both** PAT1 and PAT2 use positive lookaheads (`(?=PAT)`:literal:):
+     ```cmd
+     nimgrep --inContext:'(?s)(?=.*PAT1)(?=.*PAT2)'
+     ```
+
+Meaning of `^`:literal: and `$`:literal:
+========================================
+
+`nimgrep`:cmd: PCRE engine is run in a single-line mode so
+`^`:literal: matches the beginning of whole input *file* and
+`$`:literal: matches the end of *file* (or whole input *string* for
+options like `--filename`).
+
+Add the `(?m)`:literal: modifier to the beginning of your pattern for
+`^`:literal: and `$`:literal: to match the beginnings and ends of *lines*.
+
+Examples
+========
+
+All examples below use default PCRE Regex patterns:
+
++ To search recursively in Nim files using style-insensitive identifiers:
+
+    ```cmd
+    nimgrep --recursive --ext:'nim|nims' --ignoreStyle
+    # short: -r --ext:'nim|nims' -y
+    ```
+
+  .. Note:: we used `'` quotes to avoid special treatment of `|` symbol
+    for shells like Bash
+
++ To exclude version control directories (Git, Mercurial=hg, Subversion=svn)
+  from the search:
+    ```cmd
+    nimgrep --notdirname:'^\.git$' --notdirname:'^\.hg$' --notdirname:'^\.svn$'
+    # short: --ndi:'^\.git$' --ndi:'^\.hg$' --ndi:'^\.svn$'
+    ```
++ To search only in paths containing the `tests`:literal: sub-directory
+  recursively:
+    ```cmd
+    nimgrep --recursive --dirname:'^tests$'
+    # short: -r --di:'^tests$'
+    # or using --dirpath:
+    nimgrep --recursive --dirpath:'(^|/)tests($|/)'
+    # short: -r --pa:'(^|/)tests($|/)'
+    ```
++ Nimgrep can search multi-line, e.g. to find files containing `import`:literal:
+  and then `strutils`:literal: use pattern `'import(.|\n)*?strutils'`:literal:.
diff --git a/doc/nimgrep.rst b/doc/nimgrep.rst
deleted file mode 100644
index 5b0fe0dbb7..0000000000
--- a/doc/nimgrep.rst
+++ /dev/null
@@ -1,52 +0,0 @@
-.. default-role:: code
-
-=========================
-  nimgrep User's manual
-=========================
-
-:Author: Andreas Rumpf
-:Version: 0.9
-
-
-Nimgrep is a command line tool for search&replace tasks. It can search for
-regex or peg patterns and can search whole directories at once. User
-confirmation for every single replace operation can be requested.
-
-Nimgrep has particularly good support for Nim's
-eccentric *style insensitivity*. Apart from that it is a generic text
-manipulation tool.
-
-
-Installation
-============
-
-Compile nimgrep with the command::
-
-  nim c -d:release tools/nimgrep.nim
-
-And copy the executable somewhere in your `$PATH`.
-
-
-Command line switches
-=====================
-
-Usage:
-  nimgrep [options] [pattern] [replacement] (file/directory)*
-Options:
-  --find, -f          find the pattern (default)
-  --replace, -r       replace the pattern
-  --peg               pattern is a peg
-  --re                pattern is a regular expression (default); extended
-                      syntax for the regular expression is always turned on
-  --recursive         process directories recursively
-  --confirm           confirm each occurrence/replacement; there is a chance
-                      to abort any time without touching the file
-  --stdin             read pattern from stdin (to avoid the shell's confusing
-                      quoting rules)
-  --word, -w          the match should have word boundaries (buggy for pegs!)
-  --ignoreCase, -i    be case insensitive
-  --ignoreStyle, -y   be style insensitive
-  --ext:EX1|EX2|...   only search the files with the given extension(s)
-  --verbose           be verbose: list every processed file
-  --help, -h          shows this help
-  --version, -v       shows the version
diff --git a/doc/nimgrep_cmdline.txt b/doc/nimgrep_cmdline.txt
new file mode 100644
index 0000000000..6f6887bc4e
--- /dev/null
+++ b/doc/nimgrep_cmdline.txt
@@ -0,0 +1,136 @@
+
+Usage:
+
+* To search:
+
+      nimgrep [options] PATTERN [(FILE/DIRECTORY)*/-]
+
+* To replace:
+
+      nimgrep [options] PATTERN --replace REPLACEMENT (FILE/DIRECTORY)*/-
+
+* To list file names:
+
+      nimgrep [options] --filenames [PATTERN] [(FILE/DIRECTORY)*]
+
+Positional arguments, from left to right:
+1) PATTERN is either Regex (default) or Peg if `--peg` is specified.
+   PATTERN and REPLACEMENT should be skipped when `--stdin` is specified.
+2) REPLACEMENT supports `$1`, `$#` notations for captured groups in PATTERN.
+
+   .. DANGER:: `--replace` mode **DOES NOT** ask confirmation
+      unless `--confirm` is specified!
+
+3) Final arguments are a list of paths (FILE/DIRECTORY) or a standalone
+   minus `-` or not specified (empty):
+
+   * empty, current directory `.` is assumed (not with `--replace`)
+
+     .. Note:: so when no FILE/DIRECTORY/`-` is specified nimgrep
+       does **not** read the pipe, but searches files in the current
+       dir instead!
+   * `-`, read buffer once from stdin: pipe or terminal input;
+     in `--replace` mode the result is directed to stdout;
+     it's not compatible with `--stdin`, `--filenames`, or `--confirm`
+
+
+   For any given DIRECTORY nimgrep searches only its immediate files without
+   traversing subdirectories unless `--recursive` is specified.
+
+In replacement mode we require all 3 positional arguments to avoid damaging.
+
+Options:
+* Mode of operation:
+  --find, -f          find the PATTERN (default)
+  --replace, -!       replace the PATTERN to REPLACEMENT, rewriting the files
+  --confirm           confirm each occurrence/replacement; there is a chance
+                      to abort any time without touching the file
+  --filenames         just list filenames. Provide a PATTERN to find it in
+                      the filenames (not in the contents of a file) or run
+                      with empty pattern to just list all files:
+
+                          nimgrep --filenames               # In current dir
+                          nimgrep --filenames "" DIRECTORY
+                           # Note empty pattern "", lists all files in DIRECTORY
+* Interprete patterns:
+  --peg               PATTERN and PAT are Peg
+  --re                PATTERN and PAT are regular expressions (default)
+  --rex, -x           use the "extended" syntax for the regular expression
+                      so that whitespace is not significant
+  --word, -w          matches should have word boundaries (buggy for pegs!)
+  --ignoreCase, -i    be case-insensitive in PATTERN and PAT
+  --ignoreStyle, -y   be style insensitive in PATTERN and PAT
+  .. Note:: PATTERN and patterns PAT (see below in other options) are all either
+     Regex or Peg simultaneously and options `--rex`, `--word`, `--ignoreCase`,
+     and `--ignoreStyle` are applied to all of them.
+
+* File system walk:
+  --recursive, -r      process directories recursively
+  --follow             follow all symlinks when processing recursively
+  --sortTime, -s[:asc|desc]
+                       order files by the last modification time (default: off):
+                       ascending (recent files go last) or descending
+
+* Filter files (based on filesystem paths):
+
+  .. Hint:: Instead of `not` you can type just `n` for negative options below.
+
+  --ex[tensions]:EX1|EX2|...
+                       only search the files with the given extension(s),
+                       empty one (`--ex`) means files with missing extension
+  --notex[tensions]:EX1|EX2|...
+                       exclude files having given extension(s), use empty one to
+                       skip files with no extension (like some binary files are)
+  --fi[lename]:PAT     search only files whose name matches pattern PAT
+  --notfi[lename]:PAT  skip files whose name matches pattern PAT
+  --di[rname]:PAT      select files that in their path have a directory name
+                       that matches pattern PAT
+  --notdi[rname]:PAT   do not descend into directories whose name (not path)
+                       matches pattern PAT
+  --dirp[ath]:PAT      select only files whose whole relative directory path
+                       matches pattern PAT
+  --notdirp[ath]:PAT   skip files whose whole relative directory path
+                       matches pattern PAT
+
+* Filter files (based on file contents):
+  --inF[ile]:PAT      select files containing a (not displayed) match of PAT
+  --notinF[ile]:PAT   skip files containing a match of PAT
+  --bin:on|off|only   process binary files? (detected by \0 in first 1K bytes)
+                      (default: on - binary and text files treated the same way)
+  --text, -t          process only text files, the same as `--bin:off`
+
+* Filter matches:
+  --inC[ontext]:PAT   select only matches containing a match of PAT in their
+                      surrounding context (multiline with `-c`, `-a`, `-b`)
+  --notinC[ontext]:PAT
+                      skip matches not containing a match of PAT
+                      in their surrounding context
+
+* Represent results:
+  --nocolor           output will be given without any colors
+  --color[:on]        force color even if output is redirected (default: auto)
+  --colorTheme:THEME  select color THEME from `simple` (default),
+                      `bnw` (black and white), `ack`, or `gnu` (GNU grep)
+  --count             only print counts of matches for files that matched
+  --context:N, -c:N   print N lines of leading context before every match and
+                      N lines of trailing context after it (default N: 0)
+  --afterContext:N, -a:N
+                      print N lines of trailing context after every match
+  --beforeContext:N, -b:N
+                      print N lines of leading context before every match
+  --group, -g         group matches by file
+  --newLine, -l       display every matching line starting from a new line
+  --cols[:N]          limit max displayed columns/width of output lines from
+                      files by N characters, cropping overflows (default: off)
+  --cols:auto, -%     calculate columns from terminal width for every line
+  --onlyAscii, -@     display only printable ASCII Latin characters 0x20-0x7E
+                      substitutions: 0 -> ^@, 1 -> ^A, ... 0x1F -> ^_,
+                                     0x7F -> '7F, ..., 0xFF -> 'FF
+
+* Miscellaneous:
+  --threads:N, -j:N   speed up search by N additional workers (default: 0, off)
+  --stdin             read PATTERN from stdin (to avoid the shell's confusing
+                      quoting rules) and, if `--replace` given, REPLACEMENT
+  --verbose           be verbose: list every processed file
+  --help, -h          shows this help
+  --version, -v       shows the version
diff --git a/doc/niminst.rst b/doc/niminst.md
similarity index 77%
rename from doc/niminst.rst
rename to doc/niminst.md
index 3ccb47cc85..cc399c57a3 100644
--- a/doc/niminst.rst
+++ b/doc/niminst.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 =========================
   niminst User's manual
 =========================
@@ -7,6 +5,8 @@
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 Introduction
@@ -14,7 +14,7 @@ Introduction
 
 niminst is a tool to generate an installer for a Nim program. Currently
 it can create an installer for Windows
-via `Inno Setup `_ as well as
+via [Inno Setup](http://www.jrsoftware.org/isinfo.php) as well as
 installation/deinstallation scripts for UNIX. Later versions will support
 Linux' package management systems.
 
@@ -26,7 +26,7 @@ systems.
 Configuration file
 ==================
 
-niminst uses the Nim `parsecfg `_ module to parse the
+niminst uses the Nim [parsecfg](parsecfg.html) module to parse the
 configuration file. Here's an example of how the syntax looks like:
 
 .. include:: mytest.cfg
@@ -34,7 +34,7 @@ configuration file. Here's an example of how the syntax looks like:
 
 The value of a key-value pair can reference user-defined variables via
 the `$variable` notation: They can be defined in the command line with the
-`--var:name=value` switch. This is useful to not hard-coding the
+`--var:name=value`:option: switch. This is useful to not hard-coding the
 program's version number into the configuration file, for instance.
 
 It follows a description of each possible section and how it affects the
@@ -49,20 +49,20 @@ contain the following key-value pairs:
 ====================   =======================================================
 Key                    description
 ====================   =======================================================
-`Name`               the project's name; this needs to be a single word
-`DisplayName`        the project's long name; this can contain spaces. If
+`Name`                 the project's name; this needs to be a single word
+`DisplayName`          the project's long name; this can contain spaces. If
                        not specified, this is the same as `Name`.
-`Version`            the project's version
-`OS`                 the OSes to generate C code for; for example:
+`Version`              the project's version
+`OS`                   the OSes to generate C code for; for example:
                        `"windows;linux;macosx"`
-`CPU`                the CPUs to generate C code for; for example:
+`CPU`                  the CPUs to generate C code for; for example:
                        `"i386;amd64;powerpc"`
-`Authors`            the project's authors
-`Description`        the project's description
-`App`                the application's type: "Console" or "GUI". If
+`Authors`              the project's authors
+`Description`          the project's description
+`App`                  the application's type: "Console" or "GUI". If
                        "Console", niminst generates a special batch file
                        for Windows to open up the command-line shell.
-`License`            the filename of the application's license
+`License`              the filename of the application's license
 ====================   =======================================================
 
 
@@ -72,11 +72,11 @@ Key                    description
 Many sections support the `files` key. Listed filenames
 can be separated by semicolon or the `files` key can be repeated. Wildcards
 in filenames are supported. If it is a directory name, all files in the
-directory are used::
+directory are used:
 
-  [Config]
-  Files: "configDir"
-  Files: "otherconfig/*.conf;otherconfig/*.cfg"
+    [Config]
+    Files: "configDir"
+    Files: "otherconfig/*.conf;otherconfig/*.cfg"
 
 
 Config section
@@ -91,10 +91,10 @@ Documentation section
 
 The `documentation` section supports the `files` key.
 Listed files will be installed into the OS's native documentation directory
-(which might be `$appdir/doc`).
+(which might be ``$appdir/doc``).
 
 There is a `start` key which determines whether the Windows installer
-generates a link to e.g. the `index.html` of your documentation.
+generates a link to e.g. the ``index.html`` of your documentation.
 
 
 Other section
@@ -125,9 +125,9 @@ Other possible options are:
 ====================   =======================================================
 Key                    description
 ====================   =======================================================
-`BinPath`            paths to add to the Windows `%PATH%` environment
+`BinPath`              paths to add to the Windows `%PATH%` environment
                        variable. Example: ``BinPath: r"bin;dist\mingw\bin"``
-`InnoSetup`          boolean flag whether an Inno Setup installer should be
+`InnoSetup`            boolean flag whether an Inno Setup installer should be
                        generated for Windows. Example: `InnoSetup: "Yes"`
 ====================   =======================================================
 
@@ -137,7 +137,7 @@ UnixBin section
 
 The `UnixBin` section currently only supports the `files` key.
 Listed files will be installed into the OS's native bin directory
-(e.g. `/usr/local/bin`). The exact location depends on the
+(e.g. ``/usr/local/bin``). The exact location depends on the
 installation path the user specifies when running the `install.sh` script.
 
 
@@ -149,9 +149,9 @@ Possible options are:
 ====================   =======================================================
 Key                    description
 ====================   =======================================================
-`InstallScript`      boolean flag whether an installation shell script
+`InstallScript`        boolean flag whether an installation shell script
                        should be generated. Example: `InstallScript: "Yes"`
-`UninstallScript`    boolean flag whether a de-installation shell script
+`UninstallScript`      boolean flag whether a de-installation shell script
                        should be generated.
                        Example: `UninstallScript: "Yes"`
 ====================   =======================================================
@@ -165,9 +165,9 @@ Possible options are:
 ====================   =======================================================
 Key                    description
 ====================   =======================================================
-`path`               Path to Inno Setup.
+`path`                 Path to Inno Setup.
                        Example: ``path = r"c:\inno setup 5\iscc.exe"``
-`flags`              Flags to pass to Inno Setup.
+`flags`                Flags to pass to Inno Setup.
                        Example: `flags = "/Q"`
 ====================   =======================================================
 
@@ -180,8 +180,8 @@ Possible options are:
 ====================   =======================================================
 Key                    description
 ====================   =======================================================
-`path`               Path to the C compiler.
-`flags`              Flags to pass to the C Compiler.
+`path`                 Path to the C compiler.
+`flags`                Flags to pass to the C Compiler.
                        Example: `flags = "-w"`
 ====================   =======================================================
 
diff --git a/doc/nims.rst b/doc/nims.md
similarity index 65%
rename from doc/nims.rst
rename to doc/nims.md
index f81637d734..42cc6e124d 100644
--- a/doc/nims.rst
+++ b/doc/nims.md
@@ -1,34 +1,35 @@
-.. default-role:: code
-
 ================================
           NimScript
 ================================
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 Strictly speaking, `NimScript` is the subset of Nim that can be evaluated
 by Nim's builtin virtual machine (VM). This VM is used for Nim's compiletime
 function evaluation features.
 
-The `nim` executable processes the `.nims` configuration files in
+The `nim`:cmd: executable processes the ``.nims`` configuration files in
 the following directories (in this order; later files overwrite
 previous settings):
 
 1) If environment variable `XDG_CONFIG_HOME` is defined,
-   `$XDG_CONFIG_HOME/nim/config.nims` or
-   `~/.config/nim/config.nims` (POSIX) or
-   `%APPDATA%/nim/config.nims` (Windows). This file can be skipped
-   with the `--skipUserCfg` command line option.
-2) `$parentDir/config.nims` where `$parentDir` stands for any
+   ``$XDG_CONFIG_HOME/nim/config.nims`` or
+   ``~/.config/nim/config.nims`` (POSIX) or
+   ``%APPDATA%/nim/config.nims`` (Windows). This file can be skipped
+   with the `--skipUserCfg`:option: command line option.
+2) ``$parentDir/config.nims`` where ``$parentDir`` stands for any
    parent directory of the project file's path. These files can be
-   skipped with the `--skipParentCfg` command line option.
-3) `$projectDir/config.nims` where `$projectDir` stands for the
-   project's path. This file can be skipped with the `--skipProjCfg`
+   skipped with the `--skipParentCfg`:option: command line option.
+3) ``$projectDir/config.nims`` where ``$projectDir`` stands for the
+   project's path. This file can be skipped with the `--skipProjCfg`:option:
    command line option.
 4) A project can also have a project specific configuration file named
-   `$project.nims` that resides in the same directory as
-   `$project.nim`. This file can be skipped with the same
-   `--skipProjCfg` command line option.
+   ``$project.nims`` that resides in the same directory as
+   ``$project.nim``. This file can be skipped with the same
+   `--skipProjCfg`:option: command line option.
 
-For available procs and implementation details see `nimscript `_.
+For available procs and implementation details see [nimscript](nimscript.html).
 
 
 Limitations
@@ -60,84 +61,87 @@ Standard library modules
 
 At least the following standard library modules are available:
 
-* `macros `_
-* `os `_
-* `strutils `_
-* `math `_
-* `distros `_
-* `sugar `_
-* `algorithm `_
-* `base64 `_
-* `bitops `_
-* `chains `_
-* `colors `_
-* `complex `_
-* `htmlgen `_
-* `httpcore `_
-* `lenientops `_
-* `mersenne `_
-* `options `_
-* `parseutils `_
-* `punycode `_
-* `random `_
-* `stats `_
-* `strformat `_
-* `strmisc `_
-* `strscans `_
-* `unicode `_
-* `uri `_
-* `std/editdistance `_
-* `std/wordwrap `_
-* `std/sums `_
-* `parsecsv `_
-* `parsecfg `_
-* `parsesql `_
-* `xmlparser `_
-* `htmlparser `_
-* `ropes `_
-* `json `_
-* `parsejson `_
-* `strtabs `_
-* `unidecode `_
+* [macros](macros.html)
+* [os](os.html)
+* [strutils](strutils.html)
+* [math](math.html)
+* [distros](distros.html)
+* [sugar](sugar.html)
+* [algorithm](algorithm.html)
+* [base64](base64.html)
+* [bitops](bitops.html)
+* [chains](chains.html)
+* [colors](colors.html)
+* [complex](complex.html)
+* [htmlgen](htmlgen.html)
+* [httpcore](httpcore.html)
+* [lenientops](lenientops.html)
+* [options](options.html)
+* [parseutils](parseutils.html)
+* [punycode](punycode.html)
+* [random](random.html)
+* [stats](stats.html)
+* [strformat](strformat.html)
+* [strmisc](strmisc.html)
+* [strscans](strscans.html)
+* [unicode](unicode.html)
+* [uri](uri.html)
+* [std/editdistance](editdistance.html)
+* [std/wordwrap](wordwrap.html)
+* [parsecsv](parsecsv.html)
+* [parsecfg](parsecfg.html)
+* [parsesql](parsesql.html)
+* [xmlparser](xmlparser.html)
+* [htmlparser](htmlparser.html)
+* [ropes](ropes.html)
+* [json](json.html)
+* [parsejson](parsejson.html)
+* [strtabs](strtabs.html)
+* [unidecode](unidecode.html)
 
-In addition to the standard Nim syntax (`system `_ module),
+In addition to the standard Nim syntax ([system](system.html) module),
 NimScripts support the procs and templates defined in the
-`nimscript `_ module too.
+[nimscript](nimscript.html) module too.
 
 See also:
-* `Check the tests for more information about modules compatible with NimScript. `_
+* [Check the tests for more information about modules compatible with NimScript](
+  https://github.com/nim-lang/Nim/blob/devel/tests/test_nimscript.nims)
 
 
 NimScript as a configuration file
 =================================
 
-A command-line switch `--FOO` is written as `switch("FOO")` in
-NimScript. Similarly, command-line `--FOO:VAL` translates to
+A command-line switch `--FOO`:option: is written as `switch("FOO")` in
+NimScript. Similarly, command-line `--FOO:VAL`:option: translates to
 `switch("FOO", "VAL")`.
 
 Here are few examples of using the `switch` proc:
 
-.. code-block:: nim
+  ```nim
   # command-line: --opt:size
   switch("opt", "size")
-  # command-line: --define:foo or -d:foo
-  switch("define", "foo")
+  # command-line: --define:release or -d:release
+  switch("define", "release")
   # command-line: --forceBuild
   switch("forceBuild")
+  # command-line: --hint[Conf]:off or --hint:Conf:off
+  switch("hint", "[Conf]:off")
+  ```
 
-NimScripts also support `--` templates for convenience, which look
+NimScripts also support `--`:option: templates for convenience, which look
 like command-line switches written as-is in the NimScript file. So the
 above example can be rewritten as:
 
-.. code-block:: nim
+  ```nim
   --opt:size
-  --define:foo
+  --define:release
   --forceBuild
+  ```
 
 **Note**: In general, the *define* switches can also be set in
-NimScripts using `switch` or `--`, as shown in above
-examples. Only the `release` define (`-d:release`) cannot be set
-in NimScripts.
+NimScripts using `switch` or `--`, as shown in above examples. Few
+`define` switches such as `-d:strip`:option:, `-d:lto`:option: and
+`-d:lto_incremental`:option: cannot be set in NimScripts.
 
 
 NimScript as a build tool
@@ -145,11 +149,12 @@ NimScript as a build tool
 
 The `task` template that the `system` module defines allows a NimScript
 file to be used as a build tool. The following example defines a
-task `build` that is an alias for the `c` command:
+task `build` that is an alias for the `c`:option: command:
 
-.. code-block:: nim
+  ```nim
   task build, "builds an example":
     setCommand "c"
+  ```
 
 
 In fact, as a convention the following tasks should be available:
@@ -159,20 +164,20 @@ Task          Description
 =========     ===================================================
 `help`        List all the available NimScript tasks along with their docstrings.
 `build`       Build the project with the required
-              backend (`c`, `cpp` or `js`).
+              backend (`c`:option:, `cpp`:option: or `js`:option:).
 `tests`       Runs the tests belonging to the project.
 `bench`       Runs benchmarks belonging to the project.
 =========     ===================================================
 
 
-Look at the module `distros `_ for some support of the
+Look at the module [distros](distros.html) for some support of the
 OS's native package managers.
 
 
 Nimble integration
 ==================
 
-See the `Nimble readme `_
+See the [Nimble readme](https://github.com/nim-lang/nimble#readme)
 for more information.
 
 
@@ -180,11 +185,10 @@ Standalone NimScript
 ====================
 
 NimScript can also be used directly as a portable replacement for Bash and
-Batch files. Use `nim myscript.nims` to run `myscript.nims`. For example,
+Batch files. Use `nim myscript.nims`:cmd: to run ``myscript.nims``. For example,
 installation of Nimble could be accomplished with this simple script:
 
-.. code-block:: nim
-
+  ```nim
   mode = ScriptMode.Verbose
 
   var id = 0
@@ -197,18 +201,19 @@ installation of Nimble could be accomplished with this simple script:
     exec "nim c nimble"
 
   mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe
+  ```
 
 On Unix, you can also use the shebang `#!/usr/bin/env nim`, as long as your filename
-ends with `.nims`:
-
-.. code-block:: nim
+ends with ``.nims``:
 
+  ```nim
   #!/usr/bin/env nim
   mode = ScriptMode.Silent
 
   echo "hello world"
+  ```
 
-Use `#!/usr/bin/env -S nim --hints:off` to disable hints.
+Use `#!/usr/bin/env -S nim e --hints:off` to disable hints and relax the file extension constraint.
 
 
 Benefits
@@ -228,8 +233,7 @@ allowing the same script to support a lot of systems.
 
 See the following (incomplete) example:
 
-.. code-block:: nim
-
+  ```nim
   import std/distros
 
   # Architectures.
@@ -255,6 +259,7 @@ See the following (incomplete) example:
     echo "Distro is ArchLinux"
   elif detectOs(Debian):
     echo "Distro is Debian"
+  ```
 
 
 Uniform Syntax
@@ -276,24 +281,24 @@ making it ideal for functional scripting metaprogramming.
 This is an example of a third party module that uses macros and templates to
 translate text strings on unmodified NimScript:
 
-.. code-block:: nim
-
+  ```nim
   import nimterlingua
   nimterlingua("translations.cfg")
   echo "cat"  # Run with -d:RU becomes "kot", -d:ES becomes "gato", ...
+  ```
 
 translations.cfg
 
-.. code-block:: none
-
+  ```none
   [cat]
   ES = gato
   IT = gatto
   RU = kot
   FR = chat
+  ```
 
 
-* `Nimterlingua `_
+* [Nimterlingua](https://nimble.directory/pkg/nimterlingua)
 
 
 Graceful Fallback
@@ -304,8 +309,7 @@ but often a graceful and seamless fallback degradation is used.
 
 See the following NimScript:
 
-.. code-block:: nim
-
+  ```nim
   if likely(true):
     discard
   elif unlikely(false):
@@ -315,6 +319,7 @@ See the following NimScript:
 
   static:
     echo CompileDate
+  ```
 
 
 `likely()`, `unlikely()`, `static:` and `{.compiletime.}`
@@ -325,14 +330,15 @@ Evolving Scripting language
 ---------------------------
 
 NimScript evolves together with Nim,
-`occasionally new features might become available on NimScript `_ ,
+[occasionally new features might become available on NimScript](
+https://github.com/nim-lang/Nim/pulls?q=nimscript+is%3Amerged),
 adapted from compiled Nim or added as new features on both.
 
 Scripting Language with a Package Manager
 -----------------------------------------
 
 You can create your own modules to be compatible with NimScript,
-and check `Nimble `_
+and check [Nimble](https://nimble.directory)
 to search for third party modules that may work on NimScript.
 
 DevOps Scripting
@@ -341,4 +347,5 @@ DevOps Scripting
 You can use NimScript to deploy to production, run tests, build projects, do benchmarks,
 generate documentation, and all kinds of DevOps/SysAdmin specific tasks.
 
-* `An example of a third party NimScript that can be used as a project-agnostic tool. `_
+* [An example of a third party NimScript that can be used as a project-agnostic
+  tool.](https://github.com/kaushalmodi/nim_config#list-available-tasks)
diff --git a/doc/nimsuggest.rst b/doc/nimsuggest.md
similarity index 80%
rename from doc/nimsuggest.rst
rename to doc/nimsuggest.md
index 509f72e8a7..3d076a6f50 100644
--- a/doc/nimsuggest.rst
+++ b/doc/nimsuggest.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 ================================
   Nim IDE Integration Guide
 ================================
@@ -7,49 +5,53 @@
 :Author: Unknown
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
 Nim differs from many other compilers in that it is really fast,
 and being so fast makes it suited to provide external queries for
 text editors about the source code being written. Through the
-`nimsuggest` tool, any IDE
-can query a `.nim` source file and obtain useful information like
+`nimsuggest`:cmd: tool, any IDE
+can query a ``.nim`` source file and obtain useful information like
 definition of symbols or suggestions for completion.
 
 This document will guide you through the available options. If you
 want to look at practical examples of nimsuggest support you can look
 at the
-`various editor integrations `_
+[various editor integrations](https://github.com/Araq/Nim/wiki/Editor-Support)
 already available.
 
 
 Installation
 ============
 
-Nimsuggest is part of Nim's core. Build it via::
+Nimsuggest is part of Nim's core. Build it via:
 
+  ```cmd
   koch nimsuggest
+  ```
 
 
 Nimsuggest invocation
 =====================
 
-Run it via `nimsuggest --stdin --debug myproject.nim`. Nimsuggest is a
+Run it via `nimsuggest --stdin --debug myproject.nim`:cmd:. Nimsuggest is a
 server that takes queries that are related to `myproject`. There is some
-support so that you can throw random `.nim` files which are not part
+support so that you can throw random ``.nim`` files which are not part
 of `myproject` at Nimsuggest too, but usually the query refer to modules/files
 that are part of `myproject`.
 
-`--stdin` means that Nimsuggest reads the query from `stdin`. This is great
+`--stdin`:option: means that Nimsuggest reads the query from `stdin`. This is great
 for testing things out and playing with it but for an editor communication
 via sockets is more reasonable so that is the default. It listens to port 6000
 by default.
 
-Nimsuggest is basically a frontend for the nim compiler so `--path` flags and
-`config files `_
+Nimsuggest is basically a frontend for the nim compiler so `--path`:option: flags and
+[config files](nimc.html#compiler-usage-configuration-files)
 can be used to specify additional dependencies like 
-`nimsuggest --stdin --debug --path:"dependencies" myproject.nim`. 
+`nimsuggest --stdin --debug --path:"dependencies" myproject.nim`:cmd:.
 
 
 Specifying the location of the query
@@ -60,26 +62,26 @@ cryptic 3 letter "command" `def` or `con` or `sug` or `use` followed by
 a location. A query location consists of:
 
 
-`file.nim`
-    This is the name of the module or include file the query refers to.
+``file.nim``
+:   This is the name of the module or include file the query refers to.
 
-`dirtyfile.nim`
-    This is optional.
+``dirtyfile.nim``
+:   This is optional.
 
     The `file` parameter is enough for static analysis, but IDEs
     tend to have *unsaved buffers* where the user may still be in
     the middle of typing a line. In such situations the IDE can
     save the current contents to a temporary file and then use the
-    `dirtyfile.nim` option to tell Nimsuggest that `foobar.nim` should
-    be taken from `temporary/foobar.nim`.
+    ``dirtyfile.nim`` option to tell Nimsuggest that ``foobar.nim`` should
+    be taken from ``temporary/foobar.nim``.
 
 
-`line`
-    An integer with the line you are going to query. For the compiler
+``line``
+:   An integer with the line you are going to query. For the compiler
     lines start at **1**.
 
-`col`
-    An integer with the column you are going to query. For the
+``col``
+:   An integer with the column you are going to query. For the
     compiler columns start at **0**.
 
 
@@ -111,8 +113,8 @@ The `sug` Nimsuggest command performs a query about possible
 completion symbols at some point in the file.
 
 The typical usage scenario for this option is to call it after the
-user has typed the dot character for `the object oriented call
-syntax `_.
+user has typed the dot character for [the object oriented call
+syntax](tut2.html#object-oriented-programming-method-call-syntax).
 Nimsuggest will try to return the suggestions sorted first by scope
 (from innermost to outermost) and then by item name.
 
@@ -149,9 +151,9 @@ tab characters (``\t``). The values of each column are:
 1. Three characters indicating the type of returned answer (e.g.
    `def` for definition, `sug` for suggestion, etc).
 2. Type of the symbol. This can be `skProc`, `skLet`, and just
-   about any of the enums defined in the module `compiler/ast.nim`.
+   about any of the enums defined in the module ``compiler/ast.nim``.
 3. Fully qualified path of the symbol. If you are querying a symbol
-   defined in the `proj.nim` file, this would have the form
+   defined in the ``proj.nim`` file, this would have the form
    `proj.symbolName`.
 4. Type/signature. For variables and enums this will contain the
    type of the symbol, for procs, methods and templates this will
diff --git a/doc/overview.rst b/doc/overview.md
similarity index 86%
rename from doc/overview.rst
rename to doc/overview.md
index e01520d7c8..b21eb1e689 100644
--- a/doc/overview.rst
+++ b/doc/overview.md
@@ -5,5 +5,5 @@ Nim Documentation Overview
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
-.. include:: docs.rst
+.. include:: docs.md
 
diff --git a/doc/packaging.rst b/doc/packaging.md
similarity index 53%
rename from doc/packaging.rst
rename to doc/packaging.md
index ecde73f61a..b742bef282 100644
--- a/doc/packaging.rst
+++ b/doc/packaging.md
@@ -4,7 +4,10 @@ Packaging Nim
 
 This page provide hints on distributing Nim using OS packages.
 
-See `distros `_ for tools to detect Linux distribution at runtime.
+See [distros](distros.html) for tools to detect Linux distribution at runtime.
+
+See [here](intern.html#bootstrapping-the-compiler-reproducible-builds) for how to
+compile reproducible builds.
 
 Supported architectures
 -----------------------
@@ -45,28 +48,33 @@ The Debian package ships bash and ksh completion and manpages that can be reused
 
 Hints on the build process:
 
-::
+  ```cmd
+  # build from C sources and then using koch
+  make -j   # supports parallel build
+  # alternatively: ./build.sh --os $os_type --cpu $cpu_arch
+  ./bin/nim c -d:release koch
+  ./koch boot -d:release
 
-   # build from C sources and then using koch
-   ./build.sh --os $os_type --cpu $cpu_arch
-   ./bin/nim c koch
-   ./koch boot -d:release
+  # optionally generate docs into doc/html
+  ./koch docs
 
-   # optionally generate docs into doc/html
-   ./koch docs
+  ./koch tools
 
-   ./koch tools -d:release
+  # extract files to be really installed
+  ./install.sh 
 
-   # extract files to be really installed
-   ./install.sh 
-
-   # also include the tools
-   for fn in nimble nimsuggest nimgrep; do cp ./bin/$fn /nim/bin/; done
+  # also include the tools
+  for fn in nimble nimsuggest nimgrep; do cp ./bin/$fn /nim/bin/; done
+  ```
 
 What to install:
 
-- The expected stdlib location is /usr/lib/nim
-- Global configuration files under /etc/nim
+- The expected stdlib location is `/usr/lib/nim/lib`, previously it was just `/usr/lib/nim`
+- `nimdoc.css` and `nimdoc.cls` from the `doc` folder should go into `/usr/lib/nim/doc/`
+- `tools/debug/nim-gdb.py` should go into `/usr/lib/nim/tools/`
+- `tools/dochack/dochack.js` should be installed to `/usr/lib/nim/tools/dochack/`
+- Global configuration files under `/etc/nim`
 - Optionally: manpages, documentation, shell completion
 - When installing documentation, .idx files are not required
 - The "compiler" directory contains compiler sources and should not be part of the compiler binary package
+
diff --git a/doc/pegdocs.txt b/doc/pegdocs.txt
index 4c557aed8f..0a8fd81878 100644
--- a/doc/pegdocs.txt
+++ b/doc/pegdocs.txt
@@ -13,12 +13,12 @@ notation           meaning
 ``A / ... / Z``    Ordered choice: Apply expressions `A`, ..., `Z`, in this
                    order, to the text ahead, until one of them succeeds and
                    possibly consumes some text. Indicate success if one of
-                   expressions succeeded. Otherwise do not consume any text
+                   expressions succeeded. Otherwise, do not consume any text
                    and indicate failure.
 ``A ... Z``        Sequence: Apply expressions `A`, ..., `Z`, in this order,
                    to consume consecutive portions of the text ahead, as long
                    as they succeed. Indicate success if all succeeded.
-                   Otherwise do not consume any text and indicate failure.
+                   Otherwise, do not consume any text and indicate failure.
                    The sequence's precedence is higher than that of ordered
                    choice: ``A B / C`` means ``(A B) / Z`` and
                    not ``A (B / Z)``.
@@ -27,7 +27,10 @@ notation           meaning
 ``{E}``            Capture: Apply expression `E` and store the substring
                    that matched `E` into a *capture* that can be accessed
                    after the matching process.
-``$i``             Back reference to the ``i``th capture. ``i`` counts from 1.
+``{}``             Empty capture: Delete the last capture. No character
+                   is consumed.
+``$i``             Back reference to the ``i``th capture. ``i`` counts forwards
+                   from 1 or backwards (last capture to first) from ^1.
 ``$``              Anchor: Matches at the end of the input. No character
                    is consumed. Same as ``!.``.
 ``^``              Anchor: Matches at the start of the input. No character
@@ -41,20 +44,20 @@ notation           meaning
 ``E+``             One or more: Apply expression `E` repeatedly to match
                    the text ahead, as long as it succeeds. Consume the matched
                    text (if any) and indicate success if there was at least
-                   one match. Otherwise indicate failure.
+                   one match. Otherwise, indicate failure.
 ``E*``             Zero or more: Apply expression `E` repeatedly to match
                    the text ahead, as long as it succeeds. Consume the matched
                    text (if any). Always indicate success.
 ``E?``             Zero or one: If expression `E` matches the text ahead,
                    consume it. Always indicate success.
 ``[s]``            Character class: If the character ahead appears in the
-                   string `s`, consume it and indicate success. Otherwise
+                   string `s`, consume it and indicate success. Otherwise,
                    indicate failure.
 ``[a-b]``          Character range: If the character ahead is one from the
                    range `a` through `b`, consume it and indicate success.
-                   Otherwise indicate failure.
+                   Otherwise, indicate failure.
 ``'s'``            String: If the text ahead is the string `s`, consume it
-                   and indicate success. Otherwise indicate failure.
+                   and indicate success. Otherwise, indicate failure.
 ``i's'``           String match ignoring case.
 ``y's'``           String match ignoring style.
 ``v's'``           Verbatim string match: Use this to override a global
@@ -63,15 +66,15 @@ notation           meaning
 ``y$j``            String match ignoring style for back reference.
 ``v$j``            Verbatim string match for back reference.
 ``.``              Any character: If there is a character ahead, consume it
-                   and indicate success. Otherwise (that is, at the end of
+                   and indicate success. Otherwise, (that is, at the end of
                    input) indicate failure.
-``_``              Any Unicode character: If there is an UTF-8 character
-                   ahead, consume it and indicate success. Otherwise indicate
+``_``              Any Unicode character: If there is a UTF-8 character
+                   ahead, consume it and indicate success. Otherwise, indicate
                    failure.
 ``@E``             Search: Shorthand for ``(!E .)* E``. (Search loop for the
                    pattern `E`.)
 ``{@} E``          Captured Search: Shorthand for ``{(!E .)*} E``. (Search
-                   loop for the pattern `E`.) Everything until and exluding
+                   loop for the pattern `E`.) Everything until and excluding
                    `E` is captured.
 ``@@ E``           Same as ``{@} E``.
 ``A <- E``         Rule: Bind the expression `E` to the *nonterminal symbol*
@@ -79,7 +82,7 @@ notation           meaning
                    matching engine.**
 ``\identifier``    Built-in macro for a longer expression.
 ``\ddd``           Character with decimal code *ddd*.
-``\"``, etc        Literal ``"``, etc.
+``\"``, etc.       Literal ``"``, etc.
 ===============    ============================================================
 
 
@@ -128,51 +131,53 @@ notation           meaning
 Supported PEG grammar
 ---------------------
 
-The PEG parser implements this grammar (written in PEG syntax)::
+The PEG parser implements this grammar (written in PEG syntax):
 
-  # Example grammar of PEG in PEG syntax.
-  # Comments start with '#'.
-  # First symbol is the start symbol.
+    # Example grammar of PEG in PEG syntax.
+    # Comments start with '#'.
+    # First symbol is the start symbol.
 
-  grammar <- rule* / expr
+    grammar <- rule* / expr
 
-  identifier <- [A-Za-z][A-Za-z0-9_]*
-  charsetchar <- "\\" . / [^\]]
-  charset <- "[" "^"? (charsetchar ("-" charsetchar)?)+ "]"
-  stringlit <- identifier? ("\"" ("\\" . / [^"])* "\"" /
-                            "'" ("\\" . / [^'])* "'")
-  builtin <- "\\" identifier / [^\13\10]
+    identifier <- [A-Za-z][A-Za-z0-9_]*
+    charsetchar <- "\\" . / [^\]]
+    charset <- "[" "^"? (charsetchar ("-" charsetchar)?)+ "]"
+    stringlit <- identifier? ("\"" ("\\" . / [^"])* "\"" /
+                              "'" ("\\" . / [^'])* "'")
+    builtin <- "\\" identifier / [^\13\10]
 
-  comment <- '#' @ \n
-  ig <- (\s / comment)* # things to ignore
+    comment <- '#' @ \n
+    ig <- (\s / comment)* # things to ignore
 
-  rule <- identifier \s* "<-" expr ig
-  identNoArrow <- identifier !(\s* "<-")
-  prefixOpr <- ig '&' / ig '!' / ig '@' / ig '{@}' / ig '@@'
-  literal <- ig identifier? '$' [0-9]+ / '$' / '^' /
-             ig identNoArrow /
-             ig charset /
-             ig stringlit /
-             ig builtin /
-             ig '.' /
-             ig '_' /
-             (ig "(" expr ig ")")
-  postfixOpr <- ig '?' / ig '*' / ig '+'
-  primary <- prefixOpr* (literal postfixOpr*)
+    rule <- identifier \s* "<-" expr ig
+    identNoArrow <- identifier !(\s* "<-")
+    prefixOpr <- ig '&' / ig '!' / ig '@' / ig '{@}' / ig '@@'
+    literal <- ig identifier? '$' '^'? [0-9]+ / '$' / '^' /
+               ig identNoArrow /
+               ig charset /
+               ig stringlit /
+               ig builtin /
+               ig '.' /
+               ig '_' /
+               (ig "(" expr ig ")") /
+               (ig "{" expr? ig "}")
+    postfixOpr <- ig '?' / ig '*' / ig '+'
+    primary <- prefixOpr* (literal postfixOpr*)
 
-  # Concatenation has higher priority than choice:
-  # ``a b / c`` means ``(a b) / c``
+    # Concatenation has higher priority than choice:
+    # ``a b / c`` means ``(a b) / c``
 
-  seqExpr <- primary+
-  expr <- seqExpr (ig "/" expr)*
+    seqExpr <- primary+
+    expr <- seqExpr (ig "/" expr)*
 
 
 **Note**: As a special syntactic extension if the whole PEG is only a single
 expression, identifiers are not interpreted as non-terminals, but are
 interpreted as verbatim string:
 
-.. code-block:: nim
+  ```nim
   abc =~ peg"abc" # is true
+  ```
 
 So it is not necessary to write ``peg" 'abc' "`` in the above example.
 
@@ -182,22 +187,25 @@ Examples
 
 Check if `s` matches Nim's "while" keyword:
 
-.. code-block:: nim
+  ```nim
   s =~ peg" y'while'"
+  ```
 
 Exchange (key, val)-pairs:
 
-.. code-block:: nim
+  ```nim
   "key: val; key2: val2".replacef(peg"{\ident} \s* ':' \s* {\ident}", "$2: $1")
+  ```
 
 Determine the ``#include``'ed files of a C file:
 
-.. code-block:: nim
+  ```nim
   for line in lines("myfile.c"):
     if line =~ peg"""s <- ws '#include' ws '"' {[^"]+} '"' ws
                      comment <- '/*' @ '*/' / '//' .*
                      ws <- (comment / \s+)* """:
       echo matches[0]
+  ```
 
 PEG vs regular expression
 -------------------------
diff --git a/doc/readme.txt b/doc/readme.txt
index 6f4cece877..1157752b93 100644
--- a/doc/readme.txt
+++ b/doc/readme.txt
@@ -1,7 +1,7 @@
 ============================
-Nim's documenation system
+Nim's documentation system
 ============================
 
 This folder contains Nim's documentation. The documentation
-is written in a format called *reStructuredText*, a markup language that reads
+is written in a format called *Markdown*, a markup language that reads
 like ASCII and can be converted to HTML automatically!
diff --git a/doc/gc.rst b/doc/refc.md
similarity index 59%
rename from doc/gc.rst
rename to doc/refc.md
index 4455afcbe7..4023748e67 100644
--- a/doc/gc.rst
+++ b/doc/refc.md
@@ -1,73 +1,3 @@
-.. default-role:: code
-
-=======================
-Nim's Memory Management
-=======================
-
-:Author: Andreas Rumpf
-:Version: |nimversion|
-
-..
-
-
-  "The road to hell is paved with good intentions."
-
-
-Introduction
-============
-
-A memory-management algorithm optimal for every use-case cannot exist.
-Nim provides multiple paradigms for needs ranging from large multi-threaded
-applications, to games, hard-realtime systems and small microcontrollers.
-
-This document describes how the management strategies work;
-How to tune the garbage collectors for your needs, like (soft) `realtime systems`:idx:,
-and how the memory management strategies other than garbage collectors work.
-
-.. note:: the default GC is incremental, thread-local and not "stop-the-world"
-
-Multi-paradigm Memory Management Strategies
-===========================================
-
-To choose the memory management strategy use the `--gc:` switch.
-
-- `--gc:refc`. This is the default GC. It's a
-  deferred reference counting based garbage collector
-  with a simple Mark&Sweep backup GC in order to collect cycles. Heaps are thread-local.
-- `--gc:markAndSweep`. Simple Mark-And-Sweep based garbage collector. Heaps are thread-local.
-- `--gc:boehm`. Boehm based garbage collector, it offers a shared heap.
-- `--gc:go`. Go's garbage collector, useful for interoperability with Go. Offers a shared heap.
-- `--gc:arc`. Plain reference counting with
-  `move semantic optimizations `_, offers a shared heap.
-  It offers deterministic performance for `hard realtime`:idx: systems. Reference cycles
-  cause memory leaks, beware.
-
-- `--gc:orc`. Same as `--gc:arc` but adds a cycle collector based on "trial deletion".
-  Unfortunately, that makes its performance profile hard to reason about so it is less
-  useful for hard real-time systems.
-
-- `--gc:none`. No memory management strategy nor a garbage collector. Allocated memory is
-  simply never freed. You should use `--gc:arc` instead.
-
-
-================== ======== ================= ============== ===================
-Memory Management  Heap     Reference Cycles  Stop-The-World Command line switch
-================== ======== ================= ============== ===================
-RefC               Local    Cycle Collector   No             `--gc:refc`
-Mark & Sweep       Local    Cycle Collector   No             `--gc:markAndSweep`
-ARC                Shared   Leak              No             `--gc:arc`
-ORC                Shared   Cycle Collector   No             `--gc:orc`
-Boehm              Shared   Cycle Collector   Yes            `--gc:boehm`
-Go                 Shared   Cycle Collector   Yes            `--gc:go`
-None               Manual   Manual            Manual         `--gc:none`
-================== ======== ================= ============== ===================
-
-JavaScript's garbage collector is used for the `JavaScript and NodeJS
-`_ compilation targets.
-The `NimScript `_ target uses the memory management strategy built into
-the Nim compiler.
-
-
 Tweaking the refc GC
 ====================
 
@@ -82,13 +12,14 @@ Soft real-time support
 ----------------------
 
 To enable real-time support, the symbol `useRealtimeGC`:idx: needs to be
-defined via `--define:useRealtimeGC` (you can put this into your config
+defined via `--define:useRealtimeGC`:option: (you can put this into your config
 file as well).
 With this switch the garbage collector supports the following operations:
 
-.. code-block:: nim
+  ```nim
   proc GC_setMaxPause*(maxPauseInUs: int)
   proc GC_step*(us: int, strongAdvice = false, stackSize = -1)
+  ```
 
 The unit of the parameters `maxPauseInUs` and `us` is microseconds.
 
@@ -132,7 +63,7 @@ Time measurement with garbage collectors
 ----------------------------------------
 
 The garbage collectors' way of measuring time uses
-(see `lib/system/timers.nim` for the implementation):
+(see ``lib/system/timers.nim`` for the implementation):
 
 1) `QueryPerformanceCounter` and `QueryPerformanceFrequency` on Windows.
 2) `mach_absolute_time` on Mac OS X.
@@ -156,13 +87,41 @@ that up to 100 objects are traversed and freed before it checks again. Thus
 highly specialized environments or for older hardware.
 
 
+Thread coordination
+-------------------
+
+When the `NimMain()` function is called Nim initializes the garbage
+collector to the current thread, which is usually the main thread of your
+application. If your C code later spawns a different thread and calls Nim
+code, the garbage collector will fail to work properly and you will crash.
+
+As long as you don't use the threadvar emulation Nim uses native thread
+variables, of which you get a fresh version whenever you create a thread. You
+can then attach a GC to this thread via
+
+  ```nim
+  system.setupForeignThreadGc()
+  ```
+
+It is **not** safe to disable the garbage collector and enable it after the
+call from your background thread even if the code you are calling is short
+lived.
+
+Before the thread exits, you should tear down the thread's GC to prevent memory
+leaks by calling
+
+  ```nim
+  system.tearDownForeignThreadGc()
+  ```
+
+
 Keeping track of memory
 =======================
 
 If you need to pass around memory allocated by Nim to C, you can use the
 procs `GC_ref` and `GC_unref` to mark objects as referenced to avoid them
 being freed by the garbage collector.
-Other useful procs from `system `_ you can use to keep track of memory are:
+Other useful procs from [system](system.html) you can use to keep track of memory are:
 
 * `getTotalMem()` Returns the amount of total memory managed by the garbage collector.
 * `getOccupiedMem()` Bytes reserved by the garbage collector and used by objects.
@@ -170,7 +129,7 @@ Other useful procs from `system `_ you can use to keep track of mem
 * `GC_getStatistics()` Garbage collector statistics as a human-readable string.
 
 These numbers are usually only for the running thread, not for the whole heap,
-with the exception of `--gc:boehm` and `--gc:go`.
+with the exception of `--mm:boehm`:option: and `--mm:go`:option:.
 
 In addition to `GC_ref` and `GC_unref` you can avoid the garbage collector by manually
 allocating memory with procs like `alloc`, `alloc0`, `allocShared`, `allocShared0` or `allocCStringArray`.
@@ -179,12 +138,14 @@ The garbage collector won't try to free them, you need to call their respective
 when you are done with them or they will leak.
 
 
+
 Heap dump
 =========
 
 The heap dump feature is still in its infancy, but it already proved
 useful for us, so it might be useful for you. To get a heap dump, compile
-with `-d:nimTypeNames` and call `dumpNumberOfInstances` at a strategic place in your program.
+with `-d:nimTypeNames`:option: and call `dumpNumberOfInstances`
+at a strategic place in your program.
 This produces a list of the used types in your program and for every type
 the total amount of object instances for this type as well as the total
 amount of bytes these instances take up.
diff --git a/doc/regexprs.txt b/doc/regexprs.txt
index b7370d858c..fa7f9d24ab 100644
--- a/doc/regexprs.txt
+++ b/doc/regexprs.txt
@@ -47,16 +47,18 @@ full documentation of Perl's regular expressions.
 Because the backslash ``\`` is a meta character both in the Nim
 programming language and in regular expressions, it is strongly
 recommended that one uses the *raw* strings of Nim, so that
-backslashes are interpreted by the regular expression engine::
+backslashes are interpreted by the regular expression engine:
 
+```nim
   r"\S"  # matches any character that is not whitespace
+```
 
 A regular expression is a pattern that is matched against a subject string
 from left to right. Most characters stand for themselves in a pattern, and
 match the corresponding characters in the subject. As a trivial example,
-the pattern::
+the pattern:
 
-  The quick brown fox
+    The quick brown fox
 
 matches a portion of a subject string that is identical to itself.
 The power of regular expressions comes from the ability to include
@@ -128,7 +130,7 @@ in patterns in a visible manner. There is no restriction on the appearance of
 non-printing characters, apart from the binary zero that terminates a pattern,
 but when a pattern is being prepared by text editing, it is usually easier to
 use one of the following escape sequences than the binary character it
-represents::
+represents:
 
 ==============     ============================================================
 character          meaning
@@ -146,7 +148,7 @@ character          meaning
 After ``\x``, from zero to two hexadecimal digits are read (letters can be in
 upper or lower case). In UTF-8 mode, any number of hexadecimal digits may
 appear between ``\x{`` and ``}``, but the value of the character code must be
-less than 2**31 (that is, the maximum hexadecimal value is 7FFFFFFF). If
+less than 2^31 (that is, the maximum hexadecimal value is 7FFFFFFF). If
 characters other than hexadecimal digits appear between ``\x{`` and ``}``, or
 if there is no terminating ``}``, this form of escape is not recognized.
 Instead, the initial ``\x`` will be interpreted as a basic hexadecimal escape,
@@ -244,7 +246,7 @@ The fourth use of backslash is for certain `simple assertions`:idx:. An
 assertion specifies a condition that has to be met at a particular point in
 a match, without consuming any characters from the subject string. The use of
 subpatterns for more complicated assertions is described below. The
-backslashed assertions are::
+backslashed assertions are:
 
 ==============     ============================================================
 assertion          meaning
diff --git a/doc/rstcommon.rst b/doc/rstcommon.rst
new file mode 100644
index 0000000000..e9cc615db2
--- /dev/null
+++ b/doc/rstcommon.rst
@@ -0,0 +1,51 @@
+..
+  Usage of this file:
+     Add this in the beginning of *.rst file:
+
+         .. default-role:: code
+         .. include:: rstcommon.rst
+
+     It's the current trick for brevity and compatibility with both Github and
+     rst2html.py, considering that Github cannot highlight Nim in
+     RST files anyway and it does not include files.
+     This way interpreted text is displayed with monospaced font in Github
+     and it's displayed an Nim code in both rst2html.py
+     (note ".. default-role:: Nim" above) and `nim rst2html`.
+
+     For files that are user manual and consist of stuff like cmdline
+     option description, use 'code' as a **real** default role:
+
+     .. include:: rstcommon.rst
+     .. default-role:: code
+
+.. define language roles explicitly (for compatibility with rst2html.py):
+
+.. role:: nim(code)
+   :language: nim
+
+.. default-role:: nim
+
+.. role:: c(code)
+   :language: c
+
+.. role:: cpp(code)
+   :language: cpp
+
+.. role:: yaml(code)
+   :language: yaml
+
+.. role:: python(code)
+   :language: python
+
+.. role:: java(code)
+   :language: java
+
+.. role:: csharp(code)
+   :language: csharp
+
+.. role:: cmd(code)
+
+.. role:: program(code)
+
+.. role:: option(code)
+
diff --git a/doc/sets_fragment.txt b/doc/sets_fragment.txt
index 8436190a08..35b1fc0238 100644
--- a/doc/sets_fragment.txt
+++ b/doc/sets_fragment.txt
@@ -1,27 +1,35 @@
 The set type models the mathematical notion of a set. The set's basetype can
 only be an ordinal type of a certain size, namely:
 
-* ``int8``-``int16``
-* ``uint8``/``byte``-``uint16``
-* ``char``
-* ``enum``
+* `int8`-`int16`
+* `uint8`/`byte`-`uint16`
+* `char`
+* `enum`
+* Ordinal subrange types, i.e. `range[-10..10]`
 
-or equivalent. For signed integers the set's base type is defined to be in the
-range ``0 .. MaxSetElements-1`` where ``MaxSetElements`` is currently always
-2^16.
+or equivalent. When constructing a set with signed integer literals, the set's
+base type is defined to be in the range `0 .. DefaultSetElements-1` where
+`DefaultSetElements` is currently always 2^8. The maximum range length for the
+base type of a set is `MaxSetElements` which is currently always 2^16. Types
+with a bigger range length are coerced into the range `0 .. MaxSetElements-1`.
 
 The reason is that sets are implemented as high performance bit vectors.
 Attempting to declare a set with a larger type will result in an error:
 
-.. code-block:: nim
+```nim
+  var s: set[int64] # Error: set is too large; use `std/sets` for ordinal types
+                    # with more than 2^16 elements
+```
 
-  var s: set[int64] # Error: set is too large
 
-Sets can be constructed via the set constructor: ``{}`` is the empty set. The
+**Note:** Nim also offers [hash sets](sets.html) (which you need to import
+with `import std/sets`), which have no such restrictions.
+
+Sets can be constructed via the set constructor: `{}` is the empty set. The
 empty set is type compatible with any concrete set type. The constructor
 can also be used to include elements (and ranges of elements):
 
-.. code-block:: nim
+  ```nim
   type
     CharSet = set[char]
   var
@@ -29,37 +37,44 @@ can also be used to include elements (and ranges of elements):
   x = {'a'..'z', '0'..'9'} # This constructs a set that contains the
                            # letters from 'a' to 'z' and the digits
                            # from '0' to '9'
+  ```
+
+The module [`std/setutils`](setutils.html) provides a way to initialize a set from an iterable:
+
+```nim
+import std/setutils
+
+let uniqueChars = myString.toSet
+```
 
 These operations are supported by sets:
 
 ==================    ========================================================
 operation             meaning
 ==================    ========================================================
-``A + B``             union of two sets
-``A * B``             intersection of two sets
-``A - B``             difference of two sets (A without B's elements)
-``A == B``            set equality
-``A <= B``            subset relation (A is subset of B or equal to B)
-``A < B``             strict subset relation (A is a proper subset of B)
-``e in A``            set membership (A contains element e)
-``e notin A``         A does not contain element e
-``contains(A, e)``    A contains element e
-``card(A)``           the cardinality of A (number of elements in A)
-``incl(A, elem)``     same as ``A = A + {elem}``
-``excl(A, elem)``     same as ``A = A - {elem}``
+`A + B`               union of two sets
+`A * B`               intersection of two sets
+`A - B`               difference of two sets (A without B's elements)
+`A == B`              set equality
+`A <= B`              subset relation (A is subset of B or equal to B)
+`A < B`               strict subset relation (A is a proper subset of B)
+`e in A`              set membership (A contains element e)
+`e notin A`           A does not contain element e
+`contains(A, e)`      A contains element e
+`card(A)`             the cardinality of A (number of elements in A)
+`incl(A, elem)`       same as `A = A + {elem}`
+`excl(A, elem)`       same as `A = A - {elem}`
 ==================    ========================================================
 
-Bit fields
-~~~~~~~~~~
+### Bit fields
 
 Sets are often used to define a type for the *flags* of a procedure.
 This is a cleaner (and type safe) solution than defining integer
-constants that have to be ``or``'ed together.
+constants that have to be `or`'ed together.
 
 Enum, sets and casting can be used together as in:
 
-.. code-block:: nim
-
+  ```nim
   type
     MyFlag* {.size: sizeof(cint).} = enum
       A
@@ -77,10 +92,11 @@ Enum, sets and casting can be used together as in:
   assert toNum({A, C}) == 5
   assert toFlags(0) == {}
   assert toFlags(7) == {A, B, C}
+  ```
 
 Note how the set turns enum values into powers of 2.
 
 If using enums and sets with C, use distinct cint.
 
 For interoperability with C see also the
-`bitsize pragma `_.
+[bitsize pragma](manual.html#implementation-specific-pragmas-bitsize-pragma).
diff --git a/doc/spawn.txt b/doc/spawn.txt
index c437e8aa32..ed25ad5fdd 100644
--- a/doc/spawn.txt
+++ b/doc/spawn.txt
@@ -6,7 +6,7 @@ Nim has two flavors of parallelism:
 1) `Structured`:idx parallelism via the ``parallel`` statement.
 2) `Unstructured`:idx: parallelism via the standalone ``spawn`` statement.
 
-Both need the `threadpool `_ module to work.
+Both need the [threadpool](threadpool.html) module to work.
 
 Somewhat confusingly, ``spawn`` is also used in the ``parallel`` statement
 with slightly different semantics. ``spawn`` always takes a call expression of
@@ -28,7 +28,7 @@ the passed expression on the thread pool and returns a `data flow variable`:idx:
 **blocking**. However, one can use ``blockUntilAny`` to wait on multiple flow
 variables at the same time:
 
-.. code-block:: nim
+  ```nim
   import std/threadpool, ...
 
   # wait until 2 out of 3 servers received the update:
@@ -40,6 +40,7 @@ variables at the same time:
     assert index >= 0
     responses.del(index)
     discard blockUntilAny(responses)
+  ```
 
 Data flow variables ensure that no data races
 are possible. Due to technical limitations not every type ``T`` is possible in
@@ -54,7 +55,7 @@ Parallel statement
 
 Example:
 
-.. code-block:: nim
+  ```nim
   # Compute PI in an inefficient way
   import std/[strutils, math, threadpool]
 
@@ -69,6 +70,7 @@ Example:
       result += ch[k]
 
   echo formatFloat(pi(5000))
+  ```
 
 
 The parallel statement is the preferred mechanism to introduce parallelism
diff --git a/doc/subexes.txt b/doc/subexes.txt
index 62ddd1ec8f..1bfd602131 100644
--- a/doc/subexes.txt
+++ b/doc/subexes.txt
@@ -47,8 +47,7 @@ Notation                meaning
 Examples
 ========
 
-.. code-block:: nim
-
+  ```nim
   subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)"
 
   subex"$1 $[files|file|files]{1} copied" % ["1"] == "1 file copied"
@@ -57,5 +56,5 @@ Examples
 
   subex("type\n  TEnum = enum\n    $', '40c'\n    '{..}") % [
     "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]
-
+  ```
 
diff --git a/doc/testament.md b/doc/testament.md
new file mode 100644
index 0000000000..0ff3591ac9
--- /dev/null
+++ b/doc/testament.md
@@ -0,0 +1,390 @@
+===========
+ Testament
+===========
+
+.. default-role:: code
+.. include:: rstcommon.rst
+.. contents::
+
+Testament is an advanced automatic unittests runner for Nim tests, is used for the development of Nim itself,
+offers process isolation for your tests, it can generate statistics about test cases,
+supports multiple targets (C, C++, ObjectiveC, JavaScript, etc.),
+simulated [Dry-Runs](https://en.wikipedia.org/wiki/Dry_run_(testing)),
+has logging, can generate HTML reports, skip tests from a file, and more,
+so can be useful to run your tests, even the most complex ones.
+
+
+Test files location
+===================
+
+By default, Testament looks for test files on ``"./tests/category/*.nim"``.
+You can overwrite this pattern glob using `pattern `:option:.
+The default working directory path can be changed using
+`--directory:"folder/subfolder/"`:option:.
+
+Testament uses the `nim`:cmd: compiler on `PATH`.
+You can change that using `--nim:"folder/subfolder/nim"`:option:.
+Running JavaScript tests with `--targets:"js"`:option: requires
+a working NodeJS on `PATH`.
+
+Commands
+========
+
+p|pat|pattern         run all the tests matching the given pattern
+all                         run all tests inside of category folders
+c|cat|category    run all the tests of a certain category
+r|run                 run single test file
+html                        generate testresults.html from the database
+
+
+Options
+=======
+
+--print                   print results to the console
+--verbose                 print commands (compiling and running tests)
+--simulate                see what tests would be run but don't run them (for debugging)
+--failing                 only show failing/ignored tests
+--targets:"c cpp js objc" run tests for specified targets (default: c)
+--nim:path                use a particular nim executable (default: $PATH/nim)
+--directory:dir           Change to directory dir before reading the tests or doing anything else.
+--colors:on|off           Turn messages coloring on|off.
+--backendLogging:on|off   Disable or enable backend logging. By default turned on.
+--megatest:on|off         Enable or disable megatest. Default is on.
+--valgrind:on|off         Enable or disable valgrind support. Default is on.
+--skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
+
+
+Running a single test
+=====================
+
+This is a minimal example to understand the basics,
+not very useful for production, but easy to understand:
+
+  ```console
+  $ mkdir -p tests/category
+  $ echo "assert 42 == 42" > tests/category/test0.nim
+  $ testament run tests/category/test0.nim
+  PASS: tests/category/test0.nim c                           ( 0.2 sec)
+  $ testament r tests/category/test0
+  PASS: tests/category/test0.nim C                           ( 0.2 sec)
+  ```
+
+
+Running all tests from a directory
+==================================
+
+ This will run all tests in the top level tests/ directory. NOTE: these
+ tests are skipped by `testament all`.
+ 
+  ```console
+  $ testament pattern "tests/*.nim"
+  ```
+
+To search for tests deeper in a directory, use
+
+  ```console
+  $ testament pattern "tests/**/*.nim"    # one level deeper
+  $ testament pattern "tests/**/**/*.nim" # two levels deeper
+  ```
+
+HTML Reports
+============
+
+Generate HTML Reports ``testresults.html`` from unittests,
+you have to run at least 1 test *before* generating a report:
+
+  ```console
+  $ testament html
+  ```
+
+
+Writing Unit tests
+==================
+
+Example "template" **to edit** and write a Testament unittest:
+
+  ```nim
+  discard """
+
+    # What actions to expect completion on.
+    # Options:
+    #   "compile": expect successful compilation
+    #   "run": expect successful compilation and execution
+    #   "reject": expect failed compilation. The "reject" action can catch
+    #             {.error.} pragmas but not {.fatal.} pragmas because
+    #             {.error.} calls are expected to originate from the test-file, 
+    #             and can explicitly be specified using the "file", "line" and
+    #             "column" options.
+    #             {.fatal.} pragmas guarantee that compilation will be aborted.
+    action: "run"
+    
+    # For testing failed compilations you can specify the expected origin of the 
+    # compilation error.
+    # With the "file", "line" and "column" options you can define the file, 
+    # line and column that a compilation-error should have originated from.
+    # Use only with action: "reject" as it expects a failed compilation.
+    # Requires errormsg or msg to be defined.
+    # file: ""
+    # line: ""
+    # column: ""
+
+    # The exit code that the test is expected to return. Typically, the default
+    # value of 0 is fine. Note that if the test will be run by valgrind, then
+    # the test will exit with either a code of 0 on success or 1 on failure.
+    exitcode: 0
+
+    # Provide an `output` string to assert that the test prints to standard out
+    # exactly the expected string. Provide an `outputsub` string to assert that
+    # the string given here is a substring of the standard out output of the
+    # test (the output includes both the compiler and test execution output).
+    output: ""
+    outputsub: ""
+
+    # Whether to sort the compiler output lines before comparing them to the 
+    # expected output.
+    sortoutput: true
+
+    # Provide a `nimout` string to assert that the compiler during compilation
+    # prints the defined lines to the standard out.
+    # The lines must match in order, but there may be more lines that appear 
+    # before, after, or in between them. 
+    nimout: '''
+  a very long,
+  multi-line
+  string'''
+
+    # This is the Standard Input the test should take, if any.
+    input: ""
+
+    # Error message the test should print, if any.
+    errormsg: ""
+
+    # Can be run in batch mode, or not.
+    batchable: true
+
+    # Can be run Joined with other tests to run all together, or not.
+    joinable: true
+
+    # On Linux 64-bit machines, whether to use Valgrind to check for bad memory
+    # accesses or memory leaks. On other architectures, the test will be run
+    # as-is, without Valgrind.
+    # Options:
+    #   true: run the test with Valgrind
+    #   false: run the without Valgrind
+    #   "leaks": run the test with Valgrind, but do not check for memory leaks
+    valgrind: false   # Can use Valgrind to check for memory leaks, or not (Linux 64Bit only).
+
+    # Checks that the specified piece of C-code is within the generated C-code.
+    ccodecheck: "'Assert error message'"
+
+    # Command the test should use to run. If left out or an empty string is
+    # provided, the command is taken to be:
+    # "nim $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file"
+    # Subject to variable interpolation.
+    cmd: "nim c -r $file"
+
+    # Maximum generated temporary intermediate code file size for the test.
+    maxcodesize: 666
+
+    # Timeout seconds to run the test. Fractional values are supported.
+    timeout: 1.5
+
+    # Targets to run the test into (c, cpp, objc, js). Defaults to c.
+    targets: "c js"
+
+    # flags with which to run the test, delimited by `;`
+    matrix: "; -d:release; -d:caseFoo -d:release"
+
+    # Conditions that will skip this test. Use of multiple "disabled" clauses
+    # is permitted.
+    disabled: "bsd"   # Can disable OSes...
+    disabled: "win"
+    disabled: "32bit" # ...or architectures
+    disabled: "i386"
+    disabled: "azure" # ...or pipeline runners
+    disabled: true    # ...or can disable the test entirely
+
+  """
+  assert true
+  assert 42 == 42, "Assert error message"
+  ```
+
+
+* As you can see the "Spec" is just a `discard """ """`.
+* Spec has sane defaults, so you don't need to provide them all, any simple assert will work just fine.
+* This is not the full spec of Testament, check [the Testament Spec on GitHub,
+  see parseSpec()](https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L317).
+* Nim itself uses Testament, so [there are plenty of test examples](
+  https://github.com/nim-lang/Nim/tree/devel/tests).
+* Has some built-in CI compatibility, like Azure Pipelines, etc.
+
+
+Inline hints, warnings and errors (notes)
+-----------------------------------------
+
+Testing the line, column, kind and message of hints, warnings and errors can
+be written inline like so:
+  ```nim
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+  ```
+
+The opening `#[tt.` marks the message line.
+The `^` marks the message column.
+
+Inline messages can be combined with `nimout` when `nimoutFull` is false (default).
+This allows testing for expected messages from other modules:
+
+  ```nim
+  discard """
+    nimout: "config.nims(1, 1) Hint: some hint message [User]"
+  """
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+  ```
+
+Multiple messages for a line can be checked by delimiting messages with ';':
+
+  ```nim
+  discard """
+    matrix: "--errorMax:0 --styleCheck:error"
+  """
+
+  proc generic_proc*[T](a_a: int) = #[tt.Error
+       ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                        ^ 'a_a' should be: 'aA' ]#
+    discard
+  ```
+
+Use `--errorMax:0` in `matrix`, or `cmd: "nim check $file"` when testing
+for multiple 'Error' messages.
+
+Output message variable interpolation
+-------------------------------------
+
+`errormsg`, `nimout`, and inline messages are subject to these variable interpolations:
+
+* `${/}` - platform's directory separator
+* `$file` - the filename (without directory) of the test
+
+All other `$` characters need escaped as `$$`.
+
+Cmd variable interpolation
+--------------------------
+
+The `cmd` option is subject to these variable interpolations:
+
+* `$target` - the compilation target, e.g. `c`.
+* `$options` - the options for the compiler.
+* `$file` - the file path of the test.
+* `$filedir` - the directory of the test file.
+
+.. code-block:: nim
+
+  discard """
+    cmd: "nim $target --nimblePath:./nimbleDir/simplePkgs $options $file"
+  """
+
+All other `$` characters need escaped as `$$`.
+
+
+Unit test Examples
+==================
+
+Expected to fail:
+
+  ```nim
+  discard """
+    errormsg: "undeclared identifier: 'not_defined'"
+  """
+  assert not_defined == "not_defined", "not_defined is not defined"
+  ```
+
+Expected to fail with error thrown from another file:
+```nim
+# test.nim
+discard """
+  action: "reject"
+  errorMsg: "I break"
+  file: "breakPragma.nim"
+"""
+import ./breakPragma
+
+proc x() {.justDo.} = discard
+
+# breakPragma.nim
+import std/macros
+
+macro justDo*(procDef: typed): untyped =
+  error("I break")
+  return procDef
+```
+
+Expecting generated C to contain a given piece of code:
+
+  ```nim
+  discard """
+    # Checks that the string "Assert error message" is in the generated 
+    # C code.
+    ccodecheck: "'Assert error message'"
+  """
+  assert 42 == 42, "Assert error message"
+  ```
+
+
+Non-Zero exit code:
+
+  ```nim
+  discard """
+    exitcode: 1
+  """
+  quit "Non-Zero exit code", 1
+  ```
+
+Standard output checking:
+
+  ```nim
+  discard """
+
+    output: '''
+  0
+  1
+  2
+  3
+  4
+  5
+  '''
+
+  """
+  for i in 0..5: echo i
+  ```
+
+JavaScript tests:
+
+  ```nim
+  discard """
+    targets: "js"
+  """
+  when defined(js):
+    import std/jsconsole
+    console.log("My Frontend Project")
+  ```
+
+Compile-time tests:
+
+  ```nim
+  discard """
+    action: "compile"
+  """
+  static: assert 9 == 9, "Compile time assert"
+  ```
+
+Tests without Spec:
+
+  ```nim
+  assert 1 == 1
+  ```
+
+
+See also:
+* [Unittest](unittest.html)
diff --git a/doc/testament.rst b/doc/testament.rst
deleted file mode 100644
index 04c966ffe1..0000000000
--- a/doc/testament.rst
+++ /dev/null
@@ -1,252 +0,0 @@
-.. default-role:: code
-
-Testament is an advanced automatic unittests runner for Nim tests, is used for the development of Nim itself,
-offers process isolation for your tests, it can generate statistics about test cases,
-supports multiple targets (C, C++, ObjectiveC, JavaScript, etc),
-simulated `Dry-Runs `_,
-has logging, can generate HTML reports, skip tests from a file, and more,
-so can be useful to run your tests, even the most complex ones.
-
-
-Test files location
-===================
-
-By default Testament looks for test files on `"./tests/*.nim"`.
-You can overwrite this pattern glob using `pattern `.
-The default working directory path can be changed using
-`--directory:"folder/subfolder/"`.
-
-Testament uses the `nim` compiler on `PATH`.
-You can change that using `--nim:"folder/subfolder/nim"`.
-Running JavaScript tests with `--targets:"js"` requires a working NodeJS on
-`PATH`.
-
-
-Options
-=======
-
-* `--print`                   Also print results to the console
-* `--simulate`                See what tests would be run but don't run them (for debugging)
-* `--failing`                 Only show failing/ignored tests
-* `--targets:"c cpp js objc"` Run tests for specified targets (default: all)
-* `--nim:path`                Use a particular nim executable (default: `$PATH/nim`)
-* `--directory:dir`           Change to directory dir before reading the tests or doing anything else.
-* `--colors:on|off`           Turn messages coloring on|off.
-* `--backendLogging:on|off`   Disable or enable backend logging. By default turned on.
-* `--skipFrom:file`           Read tests to skip from `file` - one test per line, # comments ignored
-
-
-Running a single test
-=====================
-
-This is a minimal example to understand the basics,
-not very useful for production, but easy to understand:
-
-.. code::
-
-  $ mkdir tests
-  $ echo "assert 42 == 42" > tests/test0.nim
-  $ testament run test0.nim
-    PASS: tests/test0.nim C                                       ( 0.2 sec)
-
-  $ testament r test0
-    PASS: tests/test0.nim C                                       ( 0.2 sec)
-
-
-Running all tests from a directory
-==================================
-
-.. code::
-
-  $ testament pattern "tests/*.nim"
-
-To search for tests deeper in a directory, use
-
-.. code::
-
-  $ testament pattern "tests/**/*.nim"    # one level deeper
-  $ testament pattern "tests/**/**/*.nim" # two levels deeper
-
-HTML Reports
-============
-
-Generate HTML Reports `testresults.html` from unittests,
-you have to run at least 1 test *before* generating a report:
-
-.. code::
-
-  $ testament html
-
-
-Writing Unitests
-================
-
-Example "template" **to edit** and write a Testament unittest:
-
-.. code-block:: nim
-
-  discard """
-
-    # What actions to expect completion on.
-    # Options:
-    #   "compile": expect successful compilation
-    #   "run": expect successful compilation and execution
-    #   "reject": expect failed compilation. The "reject" action can catch
-    #             {.error.} pragmas but not {.fatal.} pragmas because
-    #             {.fatal.} pragmas guarantee that compilation will be aborted.
-    action: "run"
-
-    # The exit code that the test is expected to return. Typically, the default
-    # value of 0 is fine. Note that if the test will be run by valgrind, then
-    # the test will exit with either a code of 0 on success or 1 on failure.
-    exitcode: 0
-
-    # Provide an `output` string to assert that the test prints to standard out
-    # exatly the expected string. Provide an `outputsub` string to assert that
-    # the string given here is a substring of the standard out output of the
-    # test.
-    output: ""
-    outputsub: ""
-
-    # Whether to sort the output lines before comparing them to the desired
-    # output.
-    sortoutput: true
-
-    # Each line in the string given here appears in the same order in the
-    # compiler output, but there may be more lines that appear before, after, or
-    # in between them.
-    nimout: '''
-  a very long,
-  multi-line
-  string'''
-
-    # This is the Standard Input the test should take, if any.
-    input: ""
-
-    # Error message the test should print, if any.
-    errormsg: ""
-
-    # Can be run in batch mode, or not.
-    batchable: true
-
-    # Can be run Joined with other tests to run all togheter, or not.
-    joinable: true
-
-    # On Linux 64-bit machines, whether to use Valgrind to check for bad memory
-    # accesses or memory leaks. On other architectures, the test will be run
-    # as-is, without Valgrind.
-    # Options:
-    #   true: run the test with Valgrind
-    #   false: run the without Valgrind
-    #   "leaks": run the test with Valgrind, but do not check for memory leaks
-    valgrind: false   # Can use Valgrind to check for memory leaks, or not (Linux 64Bit only).
-
-    # Command the test should use to run. If left out or an empty string is
-    # provided, the command is taken to be:
-    # "nim $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file"
-    # You can use the $target, $options, and $file placeholders in your own
-    # command, too.
-    cmd: "nim c -r $file"
-
-    # Maximum generated temporary intermediate code file size for the test.
-    maxcodesize: 666
-
-    # Timeout seconds to run the test. Fractional values are supported.
-    timeout: 1.5
-
-    # Targets to run the test into (c, cpp, objc, js).
-    targets: "c js"
-
-    # flags with which to run the test, delimited by `;`
-    matrix: "; -d:release; -d:caseFoo -d:release"
-
-    # Conditions that will skip this test. Use of multiple "disabled" clauses
-    # is permitted.
-    disabled: "bsd"   # Can disable OSes...
-    disabled: "win"
-    disabled: "32bit" # ...or architectures
-    disabled: "i386"
-    disabled: "azure" # ...or pipeline runners
-    disabled: true    # ...or can disable the test entirely
-
-  """
-  assert true
-  assert 42 == 42, "Assert error message"
-
-
-* As you can see the "Spec" is just a `discard """ """`.
-* Spec has sane defaults, so you don't need to provide them all, any simple assert will work just fine.
-* `This is not the full spec of Testament, check the Testament Spec on GitHub, see parseSpec(). `_
-* `Nim itself uses Testament, so there are plenty of test examples. `_
-* Has some built-in CI compatibility, like Azure Pipelines, etc.
-* `Testament supports inlined error messages on Unittests, basically comments with the expected error directly on the code. `_
-
-
-Unitests Examples
-=================
-
-Expected to fail:
-
-.. code-block:: nim
-
-  discard """
-    errormsg: "undeclared identifier: 'not_defined'"
-  """
-  assert not_defined == "not_defined", "not_defined is not defined"
-
-Non-Zero exit code:
-
-.. code-block:: nim
-
-  discard """
-    exitcode: 1
-  """
-  quit "Non-Zero exit code", 1
-
-Standard output checking:
-
-.. code-block:: nim
-
-  discard """
-
-    output: '''
-  0
-  1
-  2
-  3
-  4
-  5
-  '''
-
-  """
-  for i in 0..5: echo i
-
-JavaScript tests:
-
-.. code-block:: nim
-
-  discard """
-    targets: "js"
-  """
-  when defined(js):
-    import std/jsconsole
-    console.log("My Frontend Project")
-
-Compile-time tests:
-
-.. code-block:: nim
-
-  discard """
-    action: "compile"
-  """
-  static: assert 9 == 9, "Compile time assert"
-
-Tests without Spec:
-
-.. code-block:: nim
-
-  assert 1 == 1
-
-
-See also:
-* `Unittest `_
diff --git a/doc/tools.rst b/doc/tools.md
similarity index 54%
rename from doc/tools.rst
rename to doc/tools.md
index e0044e1ca9..baf7ce3864 100644
--- a/doc/tools.rst
+++ b/doc/tools.md
@@ -1,41 +1,45 @@
-.. default-role:: code
-
 ========================
 Tools available with Nim
 ========================
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 The standard distribution ships with the following tools:
 
-- | `Hot code reloading `_
+- | [atlas](atlas.html)
+  | `atlas`:cmd: is a simple package cloner tool. It manages an isolated workspace that
+    contains projects and dependencies.
+
+- | [Hot code reloading](hcr.html)
   | The "Hot code reloading" feature is built into the compiler but has its own
     document explaining how it works.
 
-- | `Documentation generator `_
-  | The builtin document generator `nim doc` generates HTML documentation
-    from `.nim` source files.
+- | [Documentation generator](docgen.html)
+  | The builtin document generator `nim doc`:cmd: generates HTML documentation
+    from ``.nim`` source files.
 
-- | `Nimsuggest for IDE support `_
-  | Through the `nimsuggest` tool, any IDE can query a `.nim` source file
+- | [Nimsuggest for IDE support](nimsuggest.html)
+  | Through the `nimsuggest`:cmd: tool, any IDE can query a ``.nim`` source file
     and obtain useful information like the definition of symbols or suggestions for
     completion.
 
-- | `C2nim `_
+- | [C2nim](https://github.com/nim-lang/c2nim/blob/master/doc/c2nim.rst)
   | C to Nim source converter. Translates C header files to Nim.
 
-- | `niminst `_
+- | [niminst](niminst.html)
   | niminst is a tool to generate an installer for a Nim program.
 
-- | `nimgrep `_
+- | [nimgrep](nimgrep.html)
   | Nim search and replace utility.
 
 - | nimpretty
-  | `nimpretty` is a Nim source code beautifier,
+  | `nimpretty`:cmd: is a Nim source code beautifier,
     to format code according to the official style guide.
 
-- | `testament `_
-  | `testament` is an advanced automatic *unittests runner* for Nim tests,
+- | [testament](testament.html)
+  | `testament`:cmd: is an advanced automatic *unittests runner* for Nim tests,
     is used for the development of Nim itself, offers process isolation for your tests,
     it can generate statistics about test cases, supports multiple targets (C, JS, etc),
-    `simulated Dry-Runs `_,
     has logging, can generate HTML reports, skip tests from a file, and more,
     so can be useful to run your tests, even the most complex ones.
diff --git a/doc/tut1.rst b/doc/tut1.md
similarity index 79%
rename from doc/tut1.rst
rename to doc/tut1.md
index d2d6f8fe7d..2e83effa3b 100644
--- a/doc/tut1.rst
+++ b/doc/tut1.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 =====================
 Nim Tutorial (Part I)
 =====================
@@ -7,28 +5,31 @@ Nim Tutorial (Part I)
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 Introduction
 ============
 
-.. raw:: html
-  

-  "Der Mensch ist doch ein Augentier -- schöne Dinge wünsch ich mir."
-  

+>  "Der Mensch ist doch ein Augentier -- Schöne Dinge wünsch' ich mir."
 
 
 This document is a tutorial for the programming language *Nim*.
 
 This tutorial assumes that you are familiar with basic programming concepts
 like variables, types, or statements.
-If you would like to have a gentle introduction of those concepts, we recommend
-`Nim Basics tutorial `_.
-On the other hand, the `manual `_ contains many more examples of
-the advanced language features.
+
+Here are several other resources for learning Nim:
+
+* [Nim Basics tutorial](https://narimiran.github.io/nim-basics/) - a gentle 
+  introduction of the concepts mentioned above
+* [Learn Nim in 5 minutes](https://learnxinyminutes.com/docs/nim/) - quick,
+  five-minute introduction to Nim
+* [The Nim manual](manual.html) - many more examples of the advanced language features
 
 All code examples in this tutorial, as well as the ones found in the rest of
-Nim's documentation, follow the `Nim style guide `_.
+Nim's documentation, follow the [Nim style guide](nep1.html).
 
 
 The first program
@@ -36,36 +37,46 @@ The first program
 
 We start the tour with a modified "hello world" program:
 
-.. code-block:: Nim
-    :test: "nim c $1"
+  ```Nim  test = "nim c $1"
   # This is a comment
   echo "What's your name? "
   var name: string = readLine(stdin)
   echo "Hi, ", name, "!"
+  ```
 
 
-Save this code to the file "greetings.nim". Now compile and run it::
-
+Save this code to the file "greetings.nim". Now compile and run it:
+  ```cmd
   nim compile --run greetings.nim
+  ```
 
-With the `--run` `switch `_ Nim
+With the ``--run`` [switch](nimc.html#compiler-usage-commandminusline-switches) Nim
 executes the file automatically after compilation. You can give your program
-command-line arguments by appending them after the filename::
-
+command-line arguments by appending them after the filename:
+  ```cmd
   nim compile --run greetings.nim arg1 arg2
+  ```
 
-Commonly used commands and switches have abbreviations, so you can also use::
-
+Commonly used commands and switches have abbreviations, so you can also use:
+  ```cmd
   nim c -r greetings.nim
+  ```
 
-To compile a release version use::
-
+This is a **debug version**.
+To compile a release version use:
+  ```cmd
   nim c -d:release greetings.nim
+  ```
 
 By default, the Nim compiler generates a large number of runtime checks
-aiming for your debugging pleasure. With `-d:release` some checks are
-`turned off and optimizations are turned on
-`_.
+aiming for your debugging pleasure. With ``-d:release`` some checks are
+[turned off and optimizations are turned on](
+nimc.html#compiler-usage-compileminustime-symbols).
+
+For benchmarking or production code, use the ``-d:release`` switch.
+For comparing the performance with unsafe languages like C, use the ``-d:danger`` switch
+in order to get meaningful, comparable results. Otherwise, Nim might be handicapped
+by checks that are **not even available** for C.
 
 Though it should be pretty obvious what the program does, I will explain the
 syntax: statements which are not indented are executed when the program
@@ -74,21 +85,21 @@ done with spaces only, tabulators are not allowed.
 
 String literals are enclosed in double-quotes. The `var` statement declares
 a new variable named `name` of type `string` with the value that is
-returned by the `readLine `_ procedure. Since the
-compiler knows that `readLine `_ returns a string,
+returned by the [readLine](syncio.html#readLine,File) procedure. Since the
+compiler knows that [readLine](syncio.html#readLine,File) returns a string,
 you can leave out the type in the declaration (this is called `local type
 inference`:idx:). So this will work too:
 
-.. code-block:: Nim
-    :test: "nim c $1"
+  ```Nim  test = "nim c $1"
   var name = readLine(stdin)
+  ```
 
 Note that this is basically the only form of type inference that exists in
 Nim: it is a good compromise between brevity and readability.
 
 The "hello world" program contains several identifiers that are already known
-to the compiler: `echo`, `readLine `_, etc.
-These built-ins are declared in the system_ module which is implicitly
+to the compiler: `echo`, [readLine](syncio.html#readLine,File), etc.
+These built-ins are declared in the [system](system.html) module which is implicitly
 imported by any other module.
 
 
@@ -104,17 +115,18 @@ String and character literals
 -----------------------------
 
 String literals are enclosed in double-quotes; character literals in single
-quotes. Special characters are escaped with ``\\``: ``\n`` means newline, ``\t``
+quotes. Special characters are escaped with ``\``: ``\n`` means newline, ``\t``
 means tabulator, etc. There are also *raw* string literals:
 
-.. code-block:: Nim
+  ```Nim
   r"C:\program files\nim"
+  ```
 
 In raw literals, the backslash is not an escape character.
 
 The third and last way to write string literals is *long-string literals*.
 They are written with three quotes: `""" ... """`; they can span over
-multiple lines and the ``\\`` is not an escape character either. They are very
+multiple lines and the ``\`` is not an escape character either. They are very
 useful for embedding HTML code templates for example.
 
 
@@ -124,11 +136,11 @@ Comments
 Comments start anywhere outside a string or character literal with the
 hash character `#`. Documentation comments start with `##`:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   # A comment.
 
   var myVariable: int ## a documentation comment
+  ```
 
 
 Documentation comments are tokens; they are only allowed at certain places in
@@ -138,8 +150,7 @@ documentation generators.
 Multiline comments are started with `#[` and terminated with `]#`.  Multiline
 comments can also be nested.
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   #[
   You can have any Nim code text commented
   out inside this with no indentation restrictions.
@@ -148,6 +159,7 @@ comments can also be nested.
        Note: these can be nested!!
     ]#
   ]#
+  ```
 
 
 Numbers
@@ -165,49 +177,19 @@ The var statement
 =================
 The var statement declares a new local or global variable:
 
-.. code-block::
+  ```nim
   var x, y: int # declares x and y to have the type `int`
+  ```
 
 Indentation can be used after the `var` keyword to list a whole section of
 variables:
 
-.. code-block::
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   var
     x, y: int
     # a comment can occur here too
     a, b, c: string
-
-
-The assignment statement
-========================
-
-The assignment statement assigns a new value to a variable or more generally
-to a storage location:
-
-.. code-block::
-  var x = "abc" # introduces a new variable `x` and assigns a value to it
-  x = "xyz"     # assigns a new value to `x`
-
-`=` is the *assignment operator*. The assignment operator can be
-overloaded. You can declare multiple variables with a single assignment
-statement and all the variables will have the same value:
-
-.. code-block::
-    :test: "nim c $1"
-  var x, y = 3  # assigns 3 to the variables `x` and `y`
-  echo "x ", x  # outputs "x 3"
-  echo "y ", y  # outputs "y 3"
-  x = 42        # changes `x` to 42 without changing `y`
-  echo "x ", x  # outputs "x 42"
-  echo "y ", y  # outputs "y 3"
-
-Note that declaring multiple variables with a single assignment that calls a
-procedure can have unexpected results: the compiler will *unroll* the
-assignments and end up calling the procedure several times. If the result of
-the procedure depends on side effects, your variables may end up having
-different values! For safety use side-effect-free procedures if making multiple
-assignments.
+  ```
 
 
 Constants
@@ -217,20 +199,20 @@ Constants are symbols which are bound to a value. The constant's value
 cannot change. The compiler must be able to evaluate the expression in a
 constant declaration at compile time:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   const x = "abc" # the constant x contains the string "abc"
+  ```
 
 Indentation can be used after the `const` keyword to list a whole section of
 constants:
 
-.. code-block::
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   const
     x = 1
     # a comment can occur here too
     y = 2
     z = y + 5 # computations are possible
+  ```
 
 
 The let statement
@@ -239,20 +221,47 @@ The `let` statement works like the `var` statement but the declared
 symbols are *single assignment* variables: After the initialization their
 value cannot change:
 
-.. code-block::
+  ```nim
   let x = "abc" # introduces a new variable `x` and binds a value to it
   x = "xyz"     # Illegal: assignment to `x`
+  ```
 
 The difference between `let` and `const` is: `let` introduces a variable
 that can not be re-assigned, `const` means "enforce compile time evaluation
 and put it into a data section":
 
-.. code-block::
+  ```nim
   const input = readLine(stdin) # Error: constant expression expected
+  ```
 
-.. code-block::
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   let input = readLine(stdin)   # works
+  ```
+
+
+The assignment statement
+========================
+
+The assignment statement assigns a new value to a variable or more generally
+to a storage location:
+
+  ```nim
+  var x = "abc" # introduces a new variable `x` and assigns a value to it
+  x = "xyz"     # assigns a new value to `x`
+  ```
+
+`=` is the *assignment operator*. The assignment operator can be
+overloaded. You can declare multiple variables with a single assignment
+statement and all the variables will have the same value:
+
+  ```nim  test = "nim c $1"
+  var x, y = 3  # assigns 3 to the variables `x` and `y`
+  echo "x ", x  # outputs "x 3"
+  echo "y ", y  # outputs "y 3"
+  x = 42        # changes `x` to 42 without changing `y`
+  echo "x ", x  # outputs "x 42"
+  echo "y ", y  # outputs "y 3"
+  ```
 
 
 Control flow statements
@@ -268,8 +277,7 @@ If statement
 
 The if statement is one way to branch the control flow:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   let name = readLine(stdin)
   if name == "":
     echo "Poor soul, you lost your name?"
@@ -277,6 +285,7 @@ The if statement is one way to branch the control flow:
     echo "Very funny, your name is name."
   else:
     echo "Hi, ", name, "!"
+  ```
 
 There can be zero or more `elif` parts, and the `else` part is optional.
 The keyword `elif` is short for `else if`, and is useful to avoid
@@ -287,11 +296,10 @@ characters.)
 Case statement
 --------------
 
-Another way to branch is provided by the case statement. A case statement is
-a multi-branch:
+Another way to branch is provided by the case statement. A case statement allows
+for multiple branches:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   let name = readLine(stdin)
   case name
   of "":
@@ -302,6 +310,7 @@ a multi-branch:
     echo "Cool name!"
   else:
     echo "Hi, ", name, "!"
+  ```
 
 As it can be seen, for an `of` branch a comma-separated list of values is also
 allowed.
@@ -310,7 +319,7 @@ The case statement can deal with integers, other ordinal types, and strings.
 (What an ordinal type is will be explained soon.)
 For integers or other ordinal types value ranges are also possible:
 
-.. code-block:: nim
+  ```nim
   # this statement will be explained later:
   from std/strutils import parseInt
 
@@ -319,21 +328,23 @@ For integers or other ordinal types value ranges are also possible:
   case n
   of 0..2, 4..7: echo "The number is in the set: {0, 1, 2, 4, 5, 6, 7}"
   of 3, 8: echo "The number is 3 or 8"
+  ```
 
-However, the above code does not compile: the reason is that you have to cover
+However, the above code **does not compile**: the reason is that you have to cover
 every value that `n` may contain, but the code only handles the values
 `0..8`. Since it is not very practical to list every other possible integer
 (though it is possible thanks to the range notation), we fix this by telling
 the compiler that for every other value nothing should be done:
 
-.. code-block:: nim
+  ```nim
   ...
   case n
   of 0..2, 4..7: echo "The number is in the set: {0, 1, 2, 4, 5, 6, 7}"
   of 3, 8: echo "The number is 3 or 8"
   else: discard
+  ```
 
-The empty `discard statement <#procedures-discard-statement>`_ is a *do
+The empty [discard statement] is a *do
 nothing* statement. The compiler knows that a case statement with an else part
 cannot fail and thus the error disappears. Note that it is impossible to cover
 all possible string values: that is why string cases always need an `else`
@@ -349,15 +360,13 @@ While statement
 
 The while statement is a simple looping construct:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   echo "What's your name? "
   var name = readLine(stdin)
   while name == "":
     echo "Please tell me your name: "
-    name = readLine(stdin)
-    # no `var`, because we do not declare a new variable here
+    name = readLine(stdin) # no `var`, because we do not declare a new variable here
+  ```
 
 The example uses a while loop to keep asking the users for their name, as long
 as the user types in nothing (only presses RETURN).
@@ -367,112 +376,117 @@ For statement
 -------------
 
 The `for` statement is a construct to loop over any element an *iterator*
-provides. The example uses the built-in `countup
-`_ iterator:
+provides. The example uses the built-in [countup](
+system.html#countup.i,T,T,Positive) iterator:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   echo "Counting to ten: "
   for i in countup(1, 10):
     echo i
   # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines
+  ```
 
 The variable `i` is implicitly declared by the
-`for` loop and has the type `int`, because that is what `countup
-`_ returns. `i` runs through the values
+`for` loop and has the type `int`, because that is what [countup](
+system.html#countup.i,T,T,Positive) returns. `i` runs through the values
 1, 2, .., 10. Each value is `echo`-ed. This code does the same:
 
-.. code-block:: nim
+  ```nim
   echo "Counting to 10: "
   var i = 1
   while i <= 10:
     echo i
-    inc(i) # increment i by 1
+    inc i # increment i by 1
   # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines
+  ```
+
+Since counting up occurs so often in programs, Nim also has a [..](
+system.html#...i,T,T) iterator that does the same:
+
+  ```nim
+  for i in 1 .. 10:
+    ...
+  ```
 
 Counting down can be achieved as easily (but is less often needed):
 
-.. code-block:: nim
+  ```nim
   echo "Counting down from 10 to 1: "
   for i in countdown(10, 1):
     echo i
   # --> Outputs 10 9 8 7 6 5 4 3 2 1 on different lines
-
-Since counting up occurs so often in programs, Nim also has a `..
-`_ iterator that does the same:
-
-.. code-block:: nim
-  for i in 1 .. 10:
-    ...
+  ```
 
 Zero-indexed counting has two shortcuts `..<` and `.. ^1`
-(`backward index operator `_) to simplify
+([backward index operator](system.html#^.t%2Cint)) to simplify
 counting to one less than the higher index:
 
-.. code-block:: nim
+  ```nim
   for i in 0 ..< 10:
-    ...  # 0 .. 9
+    ...  # the same as 0 .. 9
+  ```
 
 or
 
-.. code-block:: nim
+  ```nim
   var s = "some string"
   for i in 0 ..< s.len:
     ...
+  ```
 
 or
 
-.. code-block:: nim
+  ```nim
   var s = "some string"
   for idx, c in s[0 .. ^1]:
-    ...
+    ... # ^1 is the last element, ^2 would be one before it, and so on
+  ```
 
 Other useful iterators for collections (like arrays and sequences) are
 * `items` and `mitems`, which provides immutable and mutable elements respectively, and
 * `pairs` and `mpairs` which provides the element and an index number (immutable and mutable respectively)
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   for index, item in ["a","b"].pairs:
     echo item, " at index ", index
   # => a at index 0
   # => b at index 1
+  ```
 
 Scopes and the block statement
 ------------------------------
+
 Control flow statements have a feature not covered yet: they open a
 new scope. This means that in the following example, `x` is not accessible
 outside the loop:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
+  ```nim  test = "nim c $1"  status = 1
   while false:
     var x = "hi"
   echo x # does not work
+  ```
 
 A while (for) statement introduces an implicit block. Identifiers
 are only visible within the block they have been declared. The `block`
 statement can be used to open a new block explicitly:
 
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
+  ```nim  test = "nim c $1"  status = 1
   block myblock:
     var x = "hi"
   echo x # does not work either
+  ```
 
 The block's *label* (`myblock` in the example) is optional.
 
 
 Break statement
 ---------------
+
 A block can be left prematurely with a `break` statement. The break statement
 can leave a `while`, `for`, or a `block` statement. It leaves the
 innermost construct, unless a label of a block is given:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   block myblock:
     echo "entering block"
     while true:
@@ -488,19 +502,20 @@ innermost construct, unless a label of a block is given:
       break myblock2 # leaves the block (and the loop)
     echo "still in block" # it won't be printed
   echo "outside the block"
+  ```
 
 
 Continue statement
 ------------------
+
 Like in many other programming languages, a `continue` statement starts
 the next iteration immediately:
 
-.. code-block:: nim
-    :test: "nim c $1"
-  while true:
-    let x = readLine(stdin)
-    if x == "": continue
-    echo x
+  ```nim  test = "nim c $1"
+  for i in 1 .. 5:
+    if i <= 3: continue
+    echo i # will only print 4 and 5
+  ```
 
 
 When statement
@@ -508,9 +523,7 @@ When statement
 
 Example:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   when system.hostOS == "windows":
     echo "running on Windows!"
   elif system.hostOS == "linux":
@@ -519,6 +532,7 @@ Example:
     echo "running on Mac OS X!"
   else:
     echo "unknown operating system"
+  ```
 
 The `when` statement is almost identical to the `if` statement, but with these
 differences:
@@ -530,7 +544,7 @@ differences:
   that belong to the first condition that evaluates to `true`.
 
 The `when` statement is useful for writing platform-specific code, similar to
-the `#ifdef` construct in the C programming language.
+the `#ifdef`:c: construct in the C programming language.
 
 
 Statements and indentation
@@ -546,7 +560,7 @@ statements. *Complex statements* like `if`, `when`, `for`, `while` can
 contain other statements. To avoid ambiguities, complex statements must always
 be indented, but single simple statements do not:
 
-.. code-block:: nim
+  ```nim
   # no indentation needed for single-assignment statement:
   if x: x = false
 
@@ -561,18 +575,19 @@ be indented, but single simple statements do not:
   if x:
     x = false
     y = false
+  ```
 
 
 *Expressions* are parts of a statement that usually result in a value. The
 condition in an if statement is an example of an expression. Expressions can
 contain indentation at certain places for better readability:
 
-.. code-block:: nim
-
+  ```nim
   if thisIsaLongCondition() and
       thisIsAnotherLongCondition(1,
          2, 3, 4):
     x = true
+  ```
 
 As a rule of thumb, indentation within expressions is allowed after operators,
 an open parenthesis and after commas.
@@ -580,22 +595,23 @@ an open parenthesis and after commas.
 With parenthesis and semicolons `(;)` you can use statements where only
 an expression is allowed:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   # computes fac(4) at compile time:
   const fac4 = (var x = 1; for i in 1..4: x *= i; x)
+  ```
 
 
 Procedures
 ==========
 
-To define new commands like `echo `_
-and `readLine `_ in the examples, the concept of a
-`procedure` is needed. (Some languages call them *methods* or *functions*.)
-In Nim new procedures are defined with the `proc` keyword:
+To define new commands like [echo](system.html#echo,varargs[typed,])
+and [readLine](syncio.html#readLine,File) in the examples, the concept of a
+*procedure* is needed. You might be used to them being called *methods* or
+*functions* in other languages, but Nim
+[differentiates these concepts](tut1.html#procedures-funcs-and-methods). In
+Nim, new procedures are defined with the `proc` keyword:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc yes(question: string): bool =
     echo question, " (y/n)"
     while true:
@@ -608,6 +624,7 @@ In Nim new procedures are defined with the `proc` keyword:
     echo "I'm sorry Dave, I'm afraid I can't do that."
   else:
     echo "I think you know what the problem is just as well as I do."
+  ```
 
 This example shows a procedure named `yes` that asks the user a `question`
 and returns true if they answered "yes" (or something similar) and returns
@@ -625,23 +642,24 @@ Some terminology: in the example `question` is called a (formal) *parameter*,
 
 Result variable
 ---------------
+
 A procedure that returns a value has an implicit `result` variable declared
-that represents the return value. A `return` statement with no expression is 
+that represents the return value. A `return` statement with no expression is
 shorthand for `return result`. The `result` value is always returned
 automatically at the end of a procedure if there is no `return` statement at
 the exit.
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc sumTillNegative(x: varargs[int]): int =
     for i in x:
       if i < 0:
         return
       result = result + i
 
-  echo sumTillNegative() # echos 0
-  echo sumTillNegative(3, 4, 5) # echos 12
-  echo sumTillNegative(3, 4 , -1 , 6) # echos 7
+  echo sumTillNegative() # echoes 0
+  echo sumTillNegative(3, 4, 5) # echoes 12
+  echo sumTillNegative(3, 4 , -1 , 6) # echoes 7
+  ```
 
 The `result` variable is already implicitly declared at the start of the
 function, so declaring it again with 'var result', for example, would shadow it
@@ -654,33 +672,33 @@ A procedure that does not have any `return` statement and does not use the
 special `result` variable returns the value of its last expression. For example,
 this procedure
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc helloWorld(): string =
-      "Hello, World!"
+    "Hello, World!"
+  ```
 
 returns the string "Hello, World!".
 
 Parameters
 ----------
+
 Parameters are immutable in the procedure body. By default, their value cannot be
 changed because this allows the compiler to implement parameter passing in the
 most efficient way. If a mutable variable is needed inside the procedure, it has
 to be declared with `var` in the procedure body. Shadowing the parameter name
 is possible, and actually an idiom:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc printSeq(s: seq, nprinted: int = -1) =
     var nprinted = if nprinted == -1: s.len else: min(nprinted, s.len)
     for i in 0 ..< nprinted:
       echo s[i]
+  ```
 
 If the procedure needs to modify the argument for the
 caller, a `var` parameter can be used:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc divmod(a, b: int; res, remainder: var int) =
     res = a div b        # integer division
     remainder = a mod b  # integer modulo operation
@@ -690,6 +708,7 @@ caller, a `var` parameter can be used:
   divmod(8, 5, x, y) # modifies x and y
   echo x
   echo y
+  ```
 
 In the example, `res` and `remainder` are `var parameters`.
 Var parameters can be modified by the procedure and the changes are
@@ -699,23 +718,25 @@ a tuple as a return value instead of using var parameters.
 
 Discard statement
 -----------------
+
 To call a procedure that returns a value just for its side effects and ignoring
 its return value, a `discard` statement **must** be used. Nim does not
 allow silently throwing away a return value:
 
-.. code-block:: nim
+  ```nim
   discard yes("May I ask a pointless question?")
+  ```
 
 
 The return value can be ignored implicitly if the called proc/iterator has
 been declared with the `discardable` pragma:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc p(x, y: int): int {.discardable.} =
     return x + y
 
   p(3, 4) # now valid
+  ```
 
 
 Named arguments
@@ -723,41 +744,45 @@ Named arguments
 
 Often a procedure has many parameters and it is not clear in which order the
 parameters appear. This is especially true for procedures that construct a
-complex data type. Therefore the arguments to a procedure can be named, so
+complex data type. Therefore, the arguments to a procedure can be named, so
 that it is clear which argument belongs to which parameter:
 
-.. code-block:: nim
+  ```nim
   proc createWindow(x, y, width, height: int; title: string;
                     show: bool): Window =
      ...
 
   var w = createWindow(show = true, title = "My Application",
                        x = 0, y = 0, height = 600, width = 800)
+  ```
 
 Now that we use named arguments to call `createWindow` the argument order
 does not matter anymore. Mixing named arguments with ordered arguments is
 also possible, but not very readable:
 
-.. code-block:: nim
+  ```nim
   var w = createWindow(0, 0, title = "My Application",
                        height = 600, width = 800, true)
+  ```
 
 The compiler checks that each parameter receives exactly one argument.
 
 
 Default values
 --------------
+
 To make the `createWindow` proc easier to use it should provide `default
 values`; these are values that are used as arguments if the caller does not
 specify them:
 
-.. code-block:: nim
+  ```nim
   proc createWindow(x = 0, y = 0, width = 500, height = 700,
                     title = "unknown",
                     show = true): Window =
      ...
 
   var w = createWindow(title = "My Application", height = 600, width = 800)
+  ```
 
 Now the call to `createWindow` only needs to set the values that differ
 from the defaults.
@@ -768,9 +793,10 @@ no need to write `title: string = "unknown"`, for example.
 
 Overloaded procedures
 ---------------------
+
 Nim provides the ability to overload procedures similar to C++:
 
-.. code-block:: nim
+  ```nim
   proc toString(x: int): string =
     result =
       if x < 0: "negative"
@@ -784,20 +810,20 @@ Nim provides the ability to overload procedures similar to C++:
 
   assert toString(13) == "positive" # calls the toString(x: int) proc
   assert toString(true) == "yep"    # calls the toString(x: bool) proc
+  ```
 
-(Note that `toString` is usually the `$ `_ operator in
+(Note that `toString` is usually the [$](dollars.html) operator in
 Nim.) The compiler chooses the most appropriate proc for the `toString`
 calls. How this overloading resolution algorithm works exactly is not
-discussed here (it will be specified in the manual soon).  However, it does
-not lead to nasty surprises and is based on a quite simple unification
-algorithm. Ambiguous calls are reported as errors.
+discussed here -- see the manual for details. Ambiguous calls are reported as errors.
 
 
 Operators
 ---------
-The Nim library makes heavy use of overloading - one reason for this is that
+
+The Nim standard library makes heavy use of overloading - one reason for this is that
 each operator like `+` is just an overloaded proc. The parser lets you
-use operators in `infix notation` (`a + b`) or `prefix notation` (`+ a`).
+use operators in *infix notation* (`a + b`) or *prefix notation* (`+ a`).
 An infix operator always receives two arguments, a prefix operator always one.
 (Postfix operators are not possible, because this would be ambiguous: does
 `a @ @ b` mean `(a) @ (@b)` or `(a@) @ (b)`? It always means
@@ -805,27 +831,28 @@ An infix operator always receives two arguments, a prefix operator always one.
 
 Apart from a few built-in keyword operators such as `and`, `or`, `not`,
 operators always consist of these characters:
-``+  -  *  \  /  <  >  =  @  $  ~  &  %  !  ?  ^  .  |``
+`+  -  *  \  /  <  >  =  @  $  ~  &  %  !  ?  ^  .  |`
 
 User-defined operators are allowed. Nothing stops you from defining your own
 `@!?+~` operator, but doing so may reduce readability.
 
 The operator's precedence is determined by its first character. The details
-can be found in the manual.
+can be [found in the manual](manual.html#syntax-precedence).
 
 To define a new operator enclose the operator in backticks "`":
 
-.. code-block:: nim
+  ```nim
   proc `$` (x: myDataType): string = ...
   # now the $ operator also works with myDataType, overloading resolution
   # ensures that $ works for built-in types just like before
+  ```
 
 The "`" notation can also be used to call an operator just like any other
 procedure:
 
-.. code-block:: nim
-    :test: "nim c $1"
-  if `==`( `+`(3, 4), 7): echo "True"
+  ```nim  test = "nim c $1"
+  if `==`( `+`(3, 4), 7): echo "true"
+  ```
 
 
 Forward declarations
@@ -836,11 +863,12 @@ Every variable, procedure, etc. needs to be declared before it can be used.
 language that supports metaprogramming as extensively as Nim does.)
 However, this cannot be done for mutually recursive procedures:
 
-.. code-block:: nim
+  ```nim
   # forward declaration:
   proc even(n: int): bool
+  ```
 
-.. code-block:: nim
+  ```nim
   proc odd(n: int): bool =
     assert(n >= 0) # makes sure we don't run into negative recursion
     if n == 0: false
@@ -852,12 +880,13 @@ However, this cannot be done for mutually recursive procedures:
     if n == 1: false
     else:
       n == 0 or odd(n-1)
+  ```
 
 Here `odd` depends on `even` and vice versa. Thus `even` needs to be
 introduced to the compiler before it is completely defined. The syntax for
 such a forward declaration is simple: just omit the `=` and the
 procedure's body. The `assert` just adds border conditions, and will be
-covered later in `Modules`_ section.
+covered later in [Modules] section.
 
 Later versions of the language will weaken the requirements for forward
 declarations.
@@ -866,40 +895,66 @@ The example also shows that a proc's body can consist of a single expression
 whose value is then returned implicitly.
 
 
+Funcs and methods
+-----------------
+
+As mentioned in the introduction, Nim differentiates between procedures,
+functions, and methods, defined by the `proc`, `func`, and `method` keywords
+respectively. In some ways, Nim is a bit more pedantic in its definitions than
+other languages.
+
+Functions are closer to the concept of a pure mathematical
+function, which might be familiar to you if you've ever done functional
+programming. Essentially they are procedures with additional limitations set on
+them: they can't access global state (except `const`) and can't produce
+side-effects. The `func` keyword is basically an alias for `proc` tagged
+with `{.noSideEffects.}`. Functions can still change their mutable arguments
+however, which are those marked as `var`, along with any `ref` objects.
+
+Unlike procedures, methods are dynamically dispatched. This sounds a bit
+complicated, but it is a concept closely related to inheritance and object-oriented
+programming. If you overload a procedure (two procedures with the same name but
+of different types or with different sets of arguments are said to be overloaded), the procedure to use is determined
+at compile-time. Methods, on the other hand, depend on objects that inherit from
+the `RootObj`. This is something that is covered in much greater depth in
+the [second part of the tutorial](tut2.html#object-oriented-programming-dynamic-dispatch).
+
+
 Iterators
 =========
 
 Let's return to the simple counting example:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   echo "Counting to ten: "
   for i in countup(1, 10):
     echo i
+  ```
 
-Can a `countup `_ proc be written that
-supports this loop? Lets try:
+Can a [countup](system.html#countup.i,T,T,Positive) proc be written that
+supports this loop? Let's try:
 
-.. code-block:: nim
+  ```nim
   proc countup(a, b: int): int =
     var res = a
     while res <= b:
       return res
       inc(res)
+  ```
 
 However, this does not work. The problem is that the procedure should not
 only `return`, but return and **continue** after an iteration has
 finished. This *return and continue* is called a `yield` statement. Now
 the only thing left to do is to replace the `proc` keyword by `iterator`
-and here it is - our first iterator:
+and here it is -- our first iterator:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   iterator countup(a, b: int): int =
     var res = a
     while res <= b:
       yield res
       inc(res)
+  ```
 
 Iterators look very similar to procedures, but there are several
 important differences:
@@ -909,16 +964,17 @@ important differences:
   `yield` statement).
 * Iterators have no implicit `result` variable.
 * Iterators do not support recursion.
-* Iterators cannot be forward declared, because the compiler must be able to inline an iterator. (This restriction will be gone in a
-  future version of the compiler.)
+* Iterators cannot be forward declared, because the compiler must be able to inline an iterator.
+  (This restriction will be gone in a future version of the compiler.)
 
-However, you can also use a `closure` iterator to get a different set of
-restrictions. See `first-class iterators `_
+However, you can also use a closure iterator to get a different set of
+restrictions. See [first-class iterators](
+manual.html#iterators-and-the-for-statement-firstminusclass-iterators)
 for details. Iterators can have the same name and parameters as a proc since
-essentially they have their own namespaces. Therefore it is common practice to
+essentially they have their own namespaces. Therefore, it is common to
 wrap iterators in procs of the same name which accumulate the result of the
-iterator and return it as a sequence, like `split` from the `strutils module
-`_.
+iterator and return it as a sequence, like `split` from the [strutils module](
+strutils.html).
 
 
 Basic types
@@ -931,25 +987,26 @@ Booleans
 --------
 
 Nim's boolean type is called `bool` and consists of the two
-pre-defined values `true` and `false`. Conditions in while,
-if, elif, and when statements must be of type bool.
+pre-defined values `true` and `false`. Conditions in `while`,
+`if`, `elif`, and `when` statements must be of type bool.
 
 The operators `not, and, or, xor, <, <=, >, >=, !=, ==` are defined
 for the bool type. The `and` and `or` operators perform short-circuit
 evaluation. For example:
 
-.. code-block:: nim
-
+  ```nim
   while p != nil and p.name != "xyz":
     # p.name is not evaluated if p == nil
     p = p.next
+  ```
 
 
 Characters
 ----------
-The `character type` is called `char`. Its size is always one byte, so
+
+The *character type* is called `char`. Its size is always one byte, so
 it cannot represent most UTF-8 characters, but it *can* represent one of the bytes
-that makes up a multi-byte UTF-8 character.
+that makes up a multibyte UTF-8 character.
 The reason for this is efficiency: for the overwhelming majority of use-cases,
 the resulting programs will still handle UTF-8 properly as UTF-8 was especially
 designed for this.
@@ -963,6 +1020,7 @@ Converting from an integer to a `char` is done with the `chr` proc.
 
 Strings
 -------
+
 String variables are **mutable**, so appending to a string
 is possible, and quite efficient. Strings in Nim are both zero-terminated and have a
 length field. A string's length can be retrieved with the builtin `len`
@@ -984,6 +1042,7 @@ A string variable is initialized with the empty string `""`.
 
 Integers
 --------
+
 Nim has these integer types built-in:
 `int int8 int16 int32 int64 uint uint8 uint16 uint32 uint64`.
 
@@ -991,13 +1050,13 @@ The default integer type is `int`. Integer literals can have a *type suffix*
 to specify a non-default integer type:
 
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   let
     x = 0     # x is of type `int`
     y = 0'i8  # y is of type `int8`
-    z = 0'i64 # z is of type `int64`
+    z = 0'i32 # z is of type `int32`
     u = 0'u   # u is of type `uint`
+  ```
 
 Most often integers are used for counting objects that reside in memory, so
 `int` has the same size as a pointer.
@@ -1014,12 +1073,13 @@ errors.
 
 Lossless `Automatic type conversion`:idx: is performed in expressions where different
 kinds of integer types are used. However, if the type conversion
-would cause loss of information, the `EOutOfRange`:idx: exception is raised (if the error
+would cause loss of information, the `RangeDefect`:idx: is raised (if the error
 cannot be detected at compile time).
 
 
 Floats
 ------
+
 Nim has these floating-point types built-in: `float float32 float64`.
 
 The default float type is `float`. In the current implementation,
@@ -1028,51 +1088,51 @@ The default float type is `float`. In the current implementation,
 Float literals can have a *type suffix* to specify a non-default float
 type:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   var
     x = 0.0      # x is of type `float`
     y = 0.0'f32  # y is of type `float32`
     z = 0.0'f64  # z is of type `float64`
+  ```
 
 The common operators `+ - * /  <  <=  ==  !=  >  >=` are defined for
 floats and follow the IEEE-754 standard.
 
 Automatic type conversion in expressions with different kinds of floating-point types is performed: the smaller type is converted to the larger. Integer
 types are **not** converted to floating-point types automatically, nor vice
-versa. Use the `toInt `_ and
-`toFloat `_ procs for these conversions.
+versa. Use the [toInt](system.html#toInt,float) and
+[toFloat](system.html#toFloat,int) procs for these conversions.
 
 
 Type Conversion
 ---------------
+
 Conversion between numerical types is performed by using the
 type as a function:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   var
     x: int32 = 1.int32   # same as calling int32(1)
     y: int8  = int8('a') # 'a' == 97'i8
     z: float = 2.5       # int(2.5) rounds down to 2
     sum: int = int(x) + int(y) + int(z) # sum == 100
+  ```
 
 
 Internal type representation
 ============================
 
-As mentioned earlier, the built-in `$ `_ (stringify) operator
+As mentioned earlier, the built-in [$](dollars.html) (stringify) operator
 turns any basic type into a string, which you can then print to the console
 using the `echo` proc. However, advanced types, and your own custom types,
 won't work with the `$` operator until you define it for them.
 Sometimes you just want to debug the current value of a complex type without
-having to write its `$` operator.  You can use then the `repr
-`_ proc which works with any type and even complex data
+having to write its `$` operator.  You can use then the [repr](
+system.html#repr,T) proc which works with any type and even complex data
 graphs with cycles. The following example shows that even for basic types
 there is a difference between the `$` and `repr` outputs:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   var
     myBool = true
     myCharacter = 'n'
@@ -1088,7 +1148,8 @@ there is a difference between the `$` and `repr` outputs:
   echo myInteger, ":", repr(myInteger)
   # --> 42:42
   echo myFloat, ":", repr(myFloat)
-  # --> 3.1400000000000001e+00:3.1400000000000001e+00
+  # --> 3.14:3.14
+  ```
 
 
 Advanced types
@@ -1096,11 +1157,11 @@ Advanced types
 
 In Nim new types can be defined within a `type` statement:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     biggestInt = int64      # biggest integer type that is available
     biggestFloat = float64  # biggest float type that is available
+  ```
 
 Enumeration and object types may only be defined within a
 `type` statement.
@@ -1108,20 +1169,20 @@ Enumeration and object types may only be defined within a
 
 Enumerations
 ------------
+
 A variable of an enumeration type can only be assigned one of the enumeration's specified values.
 These values are a set of ordered symbols. Each symbol is mapped
 to an integer value internally. The first symbol is represented
 at runtime by 0, the second by 1, and so on. For example:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     Direction = enum
       north, east, south, west
 
   var x = south     # `x` is of type `Direction`; its value is `south`
-  echo x            # writes "south" to `stdout`
+  echo x            # prints "south"
+  ```
 
 All the comparison operators can be used with enumeration types.
 
@@ -1138,14 +1199,15 @@ must be in ascending order.
 
 Ordinal types
 -------------
+
 Enumerations, integer types, `char` and `bool` (and
 subranges) are called ordinal types. Ordinal types have quite
 a few special operations:
 
 
------------------     --------------------------------------------------------
+=================     ========================================================
 Operation             Comment
------------------     --------------------------------------------------------
+=================     ========================================================
 `ord(x)`              returns the integer value that is used to
                       represent `x`'s value
 `inc(x)`              increments `x` by one
@@ -1156,24 +1218,25 @@ Operation             Comment
 `succ(x, n)`          returns the `n`'th successor of `x`
 `pred(x)`             returns the predecessor of `x`
 `pred(x, n)`          returns the `n`'th predecessor of `x`
------------------     --------------------------------------------------------
+=================     ========================================================
 
 
-The `inc `_, `dec `_, `succ
-`_ and `pred `_ operations can
-fail by raising an `EOutOfRange` or `EOverflow` exception. (If the code has been
+The [inc](system.html#inc,T,int), [dec](system.html#dec,T,int), [succ](
+system.html#succ,T,int) and [pred](system.html#pred,T,int) operations can
+fail by raising an `RangeDefect` or `OverflowDefect`. (If the code has been
 compiled with the proper runtime checks turned on.)
 
 
 Subranges
 ---------
+
 A subrange type is a range of values from an integer or enumeration type
 (the base type). Example:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     MySubrange = range[0..5]
+  ```
 
 
 `MySubrange` is a subrange of `int` which can only hold the values 0
@@ -1181,8 +1244,8 @@ to 5. Assigning any other value to a variable of type `MySubrange` is a
 compile-time or runtime error. Assignments from the base type to one of its
 subrange types (and vice versa) are allowed.
 
-The `system` module defines the important `Natural `_
-type as `range[0..high(int)]` (`high `_ returns
+The `system` module defines the important [Natural](system.html#Natural)
+type as `range[0..high(int)]` ([high](system.html#high,typedesc[T]) returns
 the maximal value). Other programming languages may suggest the use of unsigned
 integers for natural numbers. This is often **unwise**: you don't want unsigned
 arithmetic (which wraps around) just because the numbers cannot be negative.
@@ -1196,37 +1259,36 @@ Sets
 
 Arrays
 ------
+
 An array is a simple fixed-length container. Each element in
 an array has the same type. The array's index type can be any ordinal type.
 
 Arrays can be constructed using `[]`:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     IntArray = array[0..5, int] # an array that is indexed with 0..5
   var
     x: IntArray
   x = [1, 2, 3, 4, 5, 6]
-  for i in low(x)..high(x):
+  for i in low(x) .. high(x):
     echo x[i]
+  ```
 
 The notation `x[i]` is used to access the i-th element of `x`.
 Array access is always bounds checked (at compile-time or at runtime). These
 checks can be disabled via pragmas or invoking the compiler with the
-`--bound_checks:off` command line switch.
+``--bound_checks:off`` command line switch.
 
 Arrays are value types, like any other Nim type. The assignment operator
 copies the whole array contents.
 
-The built-in `len `_ proc returns the array's
-length. `low(a) `_ returns the lowest valid index
-for the array `a` and `high(a) `_ the highest
+The built-in [len](system.html#len,TOpenArray) proc returns the array's
+length. [low(a)](system.html#low,openArray[T]) returns the lowest valid index
+for the array `a` and [high(a)](system.html#high,openArray[T]) the highest
 valid index.
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     Direction = enum
       north, east, south, west
@@ -1238,10 +1300,11 @@ valid index.
   level[north] = on
   level[south] = slowBlink
   level[east] = fastBlink
-  echo repr(level)  # --> [on, fastBlink, slowBlink, off]
+  echo level        # --> [on, fastBlink, slowBlink, off]
   echo low(level)   # --> north
   echo len(level)   # --> 4
   echo high(level)  # --> west
+  ```
 
 The syntax for nested arrays (multidimensional) in other languages is a matter
 of appending more brackets because usually each dimension is restricted to the
@@ -1251,7 +1314,7 @@ the previous example where a level is defined as an array of enums indexed by
 yet another enum, we can add the following lines to add a light tower type
 subdivided into height levels accessed through their integer index:
 
-.. code-block:: nim
+  ```nim
   type
     LightTower = array[1..10, LevelSetting]
   var
@@ -1260,25 +1323,26 @@ subdivided into height levels accessed through their integer index:
   tower[1][east] = mediumBlink
   echo len(tower)     # --> 10
   echo len(tower[1])  # --> 4
-  echo repr(tower)    # --> [[slowBlink, mediumBlink, ...more output..
+  echo tower          # --> [[slowBlink, mediumBlink, ...more output..
   # The following lines don't compile due to type mismatch errors
   #tower[north][east] = on
   #tower[0][1] = on
+  ```
 
 Note how the built-in `len` proc returns only the array's first dimension
 length.  Another way of defining the `LightTower` to better illustrate its
 nested nature would be to omit the previous definition of the `LevelSetting`
 type and instead write it embedded directly as the type of the first dimension:
 
-.. code-block:: nim
+  ```nim
   type
     LightTower = array[1..10, array[north..west, BlinkLights]]
+  ```
 
 It is quite common to have arrays start at zero, so there's a shortcut syntax
 to specify a range from zero to the specified index minus one:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     IntArray = array[0..5, int] # an array that is indexed with 0..5
     QuickArray = array[6, int]  # an array that is indexed with 0..5
@@ -1287,50 +1351,50 @@ to specify a range from zero to the specified index minus one:
     y: QuickArray
   x = [1, 2, 3, 4, 5, 6]
   y = x
-  for i in low(x)..high(x):
+  for i in low(x) .. high(x):
     echo x[i], y[i]
+  ```
 
 
 Sequences
 ---------
+
 Sequences are similar to arrays but of dynamic length which may change
 during runtime (like strings). Since sequences are resizable they are always
 allocated on the heap and garbage collected.
 
-Sequences are always indexed with an `int` starting at position 0.  The `len
-`_, `low `_ and `high
-`_ operations are available for sequences too.
+Sequences are always indexed with an `int` starting at position 0.  The [len](
+system.html#len,seq[T]), [low](system.html#low,openArray[T]) and [high](
+system.html#high,openArray[T]) operations are available for sequences too.
 The notation `x[i]` can be used to access the i-th element of `x`.
 
 Sequences can be constructed by the array constructor `[]` in conjunction
 with the array to sequence operator `@`. Another way to allocate space for
-a sequence is to call the built-in `newSeq `_ procedure.
+a sequence is to call the built-in [newSeq](system.html#newSeq) procedure.
 
 A sequence may be passed to an openarray parameter.
 
 Example:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   var
     x: seq[int] # a reference to a sequence of integers
   x = @[1, 2, 3, 4, 5, 6] # the @ turns the array into a sequence allocated on the heap
+  ```
 
 Sequence variables are initialized with `@[]`.
 
 The `for` statement can be used with one or two variables when used with a
 sequence. When you use the one variable form, the variable will hold the value
 provided by the sequence. The `for` statement is looping over the results
-from the `items() `_ iterator from the `system
-`_ module.  But if you use the two-variable form, the first
+from the [items()](iterators.html#items.i,seq[T]) iterator from the [system](
+system.html) module.  But if you use the two-variable form, the first
 variable will hold the index position and the second variable will hold the
 value. Here the `for` statement is looping over the results from the
-`pairs() `_ iterator from the `system
-`_ module.  Examples:
+[pairs()](iterators.html#pairs.i,seq[T]) iterator from the [system](
+system.html) module.  Examples:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   for value in @[3, 4, 5]:
     echo value
   # --> 3
@@ -1342,22 +1406,23 @@ value. Here the `for` statement is looping over the results from the
   # --> index: 0, value:3
   # --> index: 1, value:4
   # --> index: 2, value:5
+  ```
 
 
 Open arrays
 -----------
+
 **Note**: Openarrays can only be used for parameters.
 
 Often fixed-size arrays turn out to be too inflexible; procedures should be
 able to deal with arrays of different sizes. The `openarray`:idx: type allows
 this. Openarrays are always indexed with an `int` starting at position 0.
-The `len `_, `low `_
-and `high `_ operations are available for open
+The [len](system.html#len,TOpenArray), [low](system.html#low,openArray[T])
+and [high](system.html#high,openArray[T]) operations are available for open
 arrays too.  Any array with a compatible base type can be passed to an
 openarray parameter, the index type does not matter.
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   var
     fruits:   seq[string]       # reference to a sequence of strings that is initialized with '@[]'
     capitals: array[3, string]  # array of strings with a fixed size
@@ -1371,6 +1436,7 @@ openarray parameter, the index type does not matter.
 
   assert openArraySize(fruits) == 2     # procedure accepts a sequence as parameter
   assert openArraySize(capitals) == 3   # but also an array type
+  ```
 
 The openarray type cannot be nested: multidimensional openarrays are not
 supported because this is seldom needed and cannot be done efficiently.
@@ -1384,8 +1450,7 @@ also a means to implement passing a variable number of
 arguments to a procedure. The compiler converts the list of arguments
 to an array automatically:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc myWriteln(f: File, a: varargs[string]) =
     for s in items(a):
       write(f, s)
@@ -1394,13 +1459,13 @@ to an array automatically:
   myWriteln(stdout, "abc", "def", "xyz")
   # is transformed by the compiler to:
   myWriteln(stdout, ["abc", "def", "xyz"])
+  ```
 
 This transformation is only done if the varargs parameter is the
 last parameter in the procedure header. It is also possible to perform
 type conversions in this context:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc myWriteln(f: File, a: varargs[string, `$`]) =
     for s in items(a):
       write(f, s)
@@ -1409,9 +1474,10 @@ type conversions in this context:
   myWriteln(stdout, 123, "abc", 4.0)
   # is transformed by the compiler to:
   myWriteln(stdout, [$123, $"abc", $4.0])
+  ```
 
-In this example `$ `_ is applied to any argument that is passed
-to the parameter `a`. Note that `$ `_ applied to strings is a
+In this example [$](dollars.html) is applied to any argument that is passed
+to the parameter `a`. Note that [$](dollars.html) applied to strings is a
 nop.
 
 
@@ -1423,9 +1489,7 @@ context. A slice is just an object of type Slice which contains two bounds,
 `a` and `b`. By itself a slice is not very useful, but other collection types
 define operators which accept Slice objects to define ranges.
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   var
     a = "Nim is a programming language"
     b = "Slices are useless."
@@ -1433,29 +1497,30 @@ define operators which accept Slice objects to define ranges.
   echo a[7 .. 12] # --> 'a prog'
   b[11 .. ^2] = "useful"
   echo b # --> 'Slices are useful.'
+  ```
 
 In the previous example slices are used to modify a part of a string. The
 slice's bounds can hold any value supported by
 their type, but it is the proc using the slice object which defines what values
 are accepted.
 
-To understand some of the different ways of specifying the indices of
+To understand the different ways of specifying the indices of
 strings, arrays, sequences, etc., it must be remembered that Nim uses
 zero-based indices.
 
 So the string `b` is of length 19, and two different ways of specifying the
 indices are
 
-.. code-block:: nim
-
+  ```nim
   "Slices are useless."
    |          |     |
    0         11    17   using indices
   ^19        ^8    ^2   using ^ syntax
+  ```
 
 where `b[0 .. ^1]` is equivalent to `b[0 .. b.len-1]` and `b[0 ..< b.len]`, and it
-can be seen that the `^1` provides a short-hand way of specifying the `b.len-1`. See
-the `backwards index operator `_.
+can be seen that the `^1` provides a shorthand way of specifying the `b.len-1`. See
+the [backwards index operator](system.html#^.t%2Cint).
 
 In the above example, because the string ends in a period, to get the portion of the
 string that is "useless" and replace it with "useful".
@@ -1466,7 +1531,7 @@ string that is "useless" and replace it with "useful".
 Note 1: alternate ways of writing this are `b[^8 .. ^2] = "useful"` or
 as `b[11 .. b.len-2] = "useful"` or as `b[11 ..< b.len-1] = "useful"`.
 
-Note 2: As the `^` template returns a `distinct int `_
+Note 2: As the `^` template returns a [distinct int](manual.html#types-distinct-type)
 of type `BackwardsIndex`, we can have a `lastIndex` constant defined as `const lastIndex = ^1`,
 and later used as `b[0 .. lastIndex]`.
 
@@ -1482,7 +1547,7 @@ Each object type `Foo` has a constructor `Foo(field: value, ...)`
 where all of its fields can be initialized. Unspecified fields will
 get their default value.
 
-.. code-block:: nim
+  ```nim
   type
     Person = object
       name: string
@@ -1509,18 +1574,18 @@ get their default value.
   # unspecified members will be initialized with their default
   # values. In this case it is the empty string.
   doAssert person4.name == ""
+  ```
 
 
 Object fields that should be visible from outside the defining module have to
 be marked with `*`.
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     Person* = object # the type is visible from other modules
       name*: string  # the field of this type is visible from other modules
       age*: int
+  ```
 
 Tuples
 ------
@@ -1542,8 +1607,7 @@ tuple's field. Another notation that is not available for objects is
 `t[i]` to access the `i`'th field. Here `i` must be a constant
 integer.
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     # type representing a person:
     # A person consists of a name and an age.
@@ -1592,43 +1656,60 @@ integer.
   #person = building
   # --> Error: type mismatch: got (tuple[street: string, number: int])
   #     but expected 'Person'
+  ```
 
 Even though you don't need to declare a type for a tuple to use it, tuples
 created with different field names will be considered different objects despite
 having the same field types.
 
-Tuples can be *unpacked* during variable assignment (and only then!). This can
+Tuples can be *unpacked* during variable assignment. This can
 be handy to assign directly the fields of the tuples to individually named
-variables. An example of this is the `splitFile `_
-proc from the `os module `_ which returns the directory, name, and
+variables. An example of this is the [splitFile](os.html#splitFile,string)
+proc from the [os module](os.html) which returns the directory, name, and
 extension of a path at the same time. For tuple unpacking to work you must
 use parentheses around the values you want to assign the unpacking to,
 otherwise, you will be assigning the same value to all the individual
 variables! For example:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   import std/os
 
   let
     path = "usr/local/nimc.html"
     (dir, name, ext) = splitFile(path)
     baddir, badname, badext = splitFile(path)
-  echo dir      # outputs `usr/local`
-  echo name     # outputs `nimc`
-  echo ext      # outputs `.html`
+  echo dir      # outputs "usr/local"
+  echo name     # outputs "nimc"
+  echo ext      # outputs ".html"
   # All the following output the same line:
-  # `(dir: usr/local, name: nimc, ext: .html)`
+  # "(dir: usr/local, name: nimc, ext: .html)"
   echo baddir
   echo badname
   echo badext
+  ```
 
-Fields of tuples are always public, they don't need to be explicity
+Tuple unpacking is also supported in for-loops:
+
+  ```nim  test = "nim c $1"
+  let a = [(10, 'a'), (20, 'b'), (30, 'c')]
+
+  for (x, c) in a:
+    echo x
+  # This will output: 10; 20; 30
+
+  # Accessing the index is also possible:
+  for i, (x, c) in a:
+    echo i, c
+  # This will output: 0a; 1b; 2c
+  ```
+
+Fields of tuples are always public, they don't need to be explicitly
 marked to be exported, unlike for example fields in an object type.
 
+
 Reference and pointer types
 ---------------------------
+
 References (similar to pointers in other programming languages) are a
 way to introduce many-to-one relationships. This means different references can
 point to and modify the same location in memory.
@@ -1636,9 +1717,9 @@ point to and modify the same location in memory.
 Nim distinguishes between `traced`:idx: and `untraced`:idx: references.
 Untraced references are also called *pointers*. Traced references point to
 objects in a garbage-collected heap, untraced references point to
-manually allocated objects or objects elsewhere in memory. Thus
+manually allocated objects or objects elsewhere in memory. Thus,
 untraced references are *unsafe*. However, for certain low-level operations
-(e.g., accessing the hardware), untraced references are necessary.
+(e.g. accessing the hardware), untraced references are necessary.
 
 Traced references are declared with the **ref** keyword; untraced references
 are declared with the **ptr** keyword.
@@ -1648,22 +1729,26 @@ meaning to retrieve the item the reference points to. The `.` (access a
 tuple/object field operator) and `[]` (array/string/sequence index operator)
 operators perform implicit dereferencing operations for reference types:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     Node = ref object
       le, ri: Node
       data: int
-  var
-    n: Node
-  new(n)
-  n.data = 9
-  # no need to write n[].data; in fact n[].data is highly discouraged!
 
-To allocate a new traced object, the built-in procedure `new` must be used.
+  var n = Node(data: 9)
+  echo n.data
+  # no need to write n[].data; in fact n[].data is highly discouraged!
+  ```
+
+To allocate a new traced object, the built-in procedure `new` can be used:
+
+  ```nim
+  var n: Node
+  new(n)
+  ```
+
 To deal with untraced memory, the procedures `alloc`, `dealloc` and
-`realloc` can be used. The `system `_
+`realloc` can be used. The [system](system.html)
 module's documentation contains further details.
 
 If a reference points to *nothing*, it has the value `nil`.
@@ -1671,6 +1756,7 @@ If a reference points to *nothing*, it has the value `nil`.
 
 Procedural type
 ---------------
+
 A procedural type is a (somewhat abstract) pointer to a procedure.
 `nil` is an allowed value for a variable of a procedural type.
 Nim uses procedural types to achieve `functional`:idx: programming
@@ -1678,41 +1764,45 @@ techniques.
 
 Example:
 
-.. code-block:: nim
-    :test: "nim c $1"
-  proc echoItem(x: int) = echo x
+  ```nim  test = "nim c $1"
+  proc greet(name: string): string =
+    "Hello, " & name & "!"
 
-  proc forEach(action: proc (x: int)) =
-    const
-      data = [2, 3, 5, 7, 11]
-    for d in items(data):
-      action(d)
+  proc bye(name: string): string =
+    "Goodbye, " & name & "."
 
-  forEach(echoItem)
+  proc communicate(greeting: proc (x: string): string, name: string) =
+    echo greeting(name)
+
+  communicate(greet, "John")
+  communicate(bye, "Mary")
+  ```
 
 A subtle issue with procedural types is that the calling convention of the
 procedure influences the type compatibility: procedural types are only compatible
 if they have the same calling convention. The different calling conventions are
-listed in the `manual `_.
+listed in the [manual](manual.html#types-procedural-type).
 
 Distinct type
 -------------
+
 A Distinct type allows for the creation of a new type that "does not imply a
 subtype relationship between it and its base type".
 You must **explicitly** define all behavior for the distinct type.
 To help with this, both the distinct type and its base type can cast from one
 type to the other.
-Examples are provided in the `manual `_.
+Examples are provided in the [manual](manual.html#types-distinct-type).
 
 Modules
 =======
-Nim supports splitting a program into pieces with a module concept.
+
+Nim supports splitting a program into pieces with a *module* concept.
 Each module is in its own file. Modules enable `information hiding`:idx: and
 `separate compilation`:idx:. A module may gain access to the symbols of another
 module by using the `import`:idx: statement. Only top-level symbols that are marked
 with an asterisk (`*`) are exported:
 
-.. code-block:: nim
+  ```nim
   # Module A
   var
     x*, y: int
@@ -1721,11 +1811,12 @@ with an asterisk (`*`) are exported:
     # allocate a new sequence:
     newSeq(result, len(a))
     # multiply two int sequences:
-    for i in 0..len(a)-1: result[i] = a[i] * b[i]
+    for i in 0 ..< len(a): result[i] = a[i] * b[i]
 
   when isMainModule:
     # test the new `*` operator for sequences:
     assert(@[1, 2, 3] * @[1, 2, 3] == @[1, 4, 9])
+  ```
 
 The above module exports `x` and `*`, but not `y`.
 
@@ -1741,15 +1832,17 @@ a symbol is ambiguous, it *must* be qualified. A symbol is ambiguous
 if it is defined in two (or more) different modules and both modules are
 imported by a third one:
 
-.. code-block:: nim
+  ```nim
   # Module A
   var x*: string
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module B
   var x*: int
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module C
   import A, B
   write(stdout, x) # error: x is ambiguous
@@ -1757,20 +1850,23 @@ imported by a third one:
 
   var x = 4
   write(stdout, x) # not ambiguous: uses the module C's x
+  ```
 
 
 But this rule does not apply to procedures or iterators. Here the overloading
 rules apply:
 
-.. code-block:: nim
+  ```nim
   # Module A
   proc x*(a: int): string = $a
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module B
   proc x*(a: string): string = $a
+  ```
 
-.. code-block:: nim
+  ```nim
   # Module C
   import A, B
   write(stdout, x(3))   # no error: A.x is called
@@ -1778,6 +1874,7 @@ rules apply:
 
   proc x*(a: int): string = discard
   write(stdout, x(3))   # ambiguous: which `x` is to call?
+  ```
 
 
 Excluding symbols
@@ -1787,8 +1884,9 @@ The normal `import` statement will bring in all exported symbols.
 These can be limited by naming symbols that should be excluded using
 the `except` qualifier.
 
-.. code-block:: nim
+  ```nim
   import mymodule except y
+  ```
 
 
 From statement
@@ -1798,42 +1896,48 @@ We have already seen the simple `import` statement that just imports all
 exported symbols. An alternative that only imports listed symbols is the
 `from import` statement:
 
-.. code-block:: nim
+  ```nim
   from mymodule import x, y, z
+  ```
 
 The `from` statement can also force namespace qualification on
 symbols, thereby making symbols available, but needing to be qualified
 in order to be used.
 
-.. code-block:: nim
+  ```nim
   from mymodule import x, y, z
 
   x()           # use x without any qualification
+  ```
 
-.. code-block:: nim
+  ```nim
   from mymodule import nil
 
   mymodule.x()  # must qualify x with the module name as prefix
 
   x()           # using x here without qualification is a compile error
+  ```
 
 Since module names are generally long to be descriptive, you can also
 define a shorter alias to use when qualifying symbols.
 
-.. code-block:: nim
+  ```nim
   from mymodule as m import nil
 
   m.x()         # m is aliasing mymodule
+  ```
 
 
 Include statement
 -----------------
+
 The `include` statement does something fundamentally different than
 importing a module: it merely includes the contents of a file. The `include`
 statement is useful to split up a large module into several files:
 
-.. code-block:: nim
+  ```nim
   include fileA, fileB, fileC
+  ```
 
 
 
@@ -1841,7 +1945,7 @@ Part 2
 ======
 
 So, now that we are done with the basics, let's see what Nim offers apart
-from a nice syntax for procedural programming: `Part II `_
+from a nice syntax for procedural programming: [Part II](tut2.html)
 
 
 .. _strutils: strutils.html
diff --git a/doc/tut2.rst b/doc/tut2.md
similarity index 91%
rename from doc/tut2.rst
rename to doc/tut2.md
index 3aef6bb0f2..1b59288d5b 100644
--- a/doc/tut2.rst
+++ b/doc/tut2.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 ======================
 Nim Tutorial (Part II)
 ======================
@@ -7,17 +5,19 @@ Nim Tutorial (Part II)
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
 Introduction
 ============
 
-  "Repetition renders the ridiculous reasonable." -- Norman Wildberger
+> "Repetition renders the ridiculous reasonable." -- Norman Wildberger
 
 This document is a tutorial for the advanced constructs of the *Nim*
 programming language. **Note that this document is somewhat obsolete as the**
-`manual `_ **contains many more examples of the advanced language
+[manual](manual.html) **contains many more examples of the advanced language
 features.**
 
 
@@ -27,8 +27,8 @@ Pragmas
 Pragmas are Nim's method to give the compiler additional information/
 commands without introducing a massive number of new keywords. Pragmas are
 enclosed in the special `{.` and `.}` curly dot brackets. This tutorial
-does not cover pragmas. See the `manual `_ or `user guide
-`_ for a description of the available
+does not cover pragmas. See the [manual](manual.html#pragmas) or [user guide](
+nimc.html#additional-features) for a description of the available
 pragmas.
 
 
@@ -53,8 +53,7 @@ types with inheritance are also marked as `ref` types even though
 this isn't strictly enforced. To check at runtime if an object is of a certain
 type, the `of` operator can be used.
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     Person = ref object of RootObj
       name*: string  # the * means that `name` is accessible from other modules
@@ -70,6 +69,7 @@ type, the `of` operator can be used.
   # object construction:
   student = Student(name: "Anton", age: 5, id: 2)
   echo student[]
+  ```
 
 Inheritance is done with the `object of` syntax. Multiple inheritance is
 currently not supported. If an object type has no suitable ancestor, `RootObj`
@@ -79,7 +79,7 @@ to introduce new object roots apart from `system.RootObj`. (This is used
 in the GTK wrapper for instance.)
 
 Ref objects should be used whenever inheritance is used. It isn't strictly
-necessary, but with non-ref objects assignments such as `let person: Person =
+necessary, but with non-ref objects, assignments such as `let person: Person =
 Student(id: 123)` will truncate subclass fields.
 
 **Note**: Composition (*has-a* relation) is often preferable to inheritance
@@ -97,8 +97,7 @@ would require arbitrary symbol lookahead which slows down compilation.)
 
 Example:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     Node = ref object  # a reference to an object with the following field:
       le, ri: Node     # left and right subtrees
@@ -108,6 +107,7 @@ Example:
       name: string     # the symbol's name
       line: int        # the line the symbol was declared in
       code: Node       # the symbol's abstract syntax tree
+  ```
 
 
 Type conversions
@@ -124,9 +124,10 @@ raised.
 The syntax for type conversions is `destination_type(expression_to_convert)`
 (like an ordinary call):
 
-.. code-block:: nim
+  ```nim
   proc getID(x: Person): int =
     Student(x).id
+  ```
 
 The `InvalidObjectConversionDefect` exception is raised if `x` is not a
 `Student`.
@@ -139,9 +140,7 @@ variant types are needed.
 
 An example:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   # This is an example how an abstract syntax tree could be modelled in Nim
   type
     NodeKind = enum  # the different node types
@@ -165,8 +164,9 @@ An example:
   # the following statement raises an `FieldDefect` exception, because
   # n.kind's value does not fit:
   n.strVal = ""
+  ```
 
-As can been seen from the example, an advantage to an object hierarchy is that
+As can be seen from the example, an advantage to an object hierarchy is that
 no conversion between different object types is needed. Yet, access to invalid
 object fields raises an exception.
 
@@ -175,34 +175,35 @@ Method call syntax
 ------------------
 
 There is a syntactic sugar for calling routines:
-The syntax `obj.method(args)` can be used instead of `method(obj, args)`.
+The syntax `obj.methodName(args)` can be used
+instead of `methodName(obj, args)`.
 If there are no remaining arguments, the parentheses can be omitted:
 `obj.len` (instead of `len(obj)`).
 
 This method call syntax is not restricted to objects, it can be used
 for any type:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   import std/strutils
 
   echo "abc".len # is the same as echo len("abc")
   echo "abc".toUpperAscii()
   echo({'a', 'b', 'c'}.card)
   stdout.writeLine("Hallo") # the same as writeLine(stdout, "Hallo")
+  ```
 
 (Another way to look at the method call syntax is that it provides the missing
 postfix notation.)
 
 So "pure object oriented" code is easy to write:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   import std/[strutils, sequtils]
 
   stdout.writeLine("Give a list of numbers (separated by spaces): ")
   stdout.write(stdin.readLine.splitWhitespace.map(parseInt).max.`$`)
   stdout.writeLine(" is the maximum!")
+  ```
 
 
 Properties
@@ -212,9 +213,7 @@ Ordinary get-procedures that are called with the *method call syntax* achieve
 the same. But setting a value is different; for this a special setter syntax
 is needed:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   type
     Socket* = ref object of RootObj
       h: int # cannot be accessed from the outside of the module due to missing star
@@ -230,6 +229,7 @@ is needed:
   var s: Socket
   new s
   s.host = 34  # same as `host=`(s, 34)
+  ```
 
 (The example also shows `inline` procedures.)
 
@@ -237,8 +237,7 @@ is needed:
 The `[]` array access operator can be overloaded to provide
 `array properties`:idx:\ :
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     Vector* = object
       x, y, z: float
@@ -258,6 +257,7 @@ The `[]` array access operator can be overloaded to provide
     of 1: result = v.y
     of 2: result = v.z
     else: assert(false)
+  ```
 
 The example is silly, since a vector is better modelled by a tuple which
 already provides `v[]` access.
@@ -269,8 +269,7 @@ Dynamic dispatch
 Procedures always use static dispatch. For dynamic dispatch replace the
 `proc` keyword by `method`:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     Expression = ref object of RootObj ## abstract base class for an expression
     Literal = ref object of Expression
@@ -290,20 +289,19 @@ Procedures always use static dispatch. For dynamic dispatch replace the
   proc newPlus(a, b: Expression): PlusExpr = PlusExpr(a: a, b: b)
 
   echo eval(newPlus(newPlus(newLit(1), newLit(2)), newLit(4)))
+  ```
 
 Note that in the example the constructors `newLit` and `newPlus` are procs
 because it makes more sense for them to use static binding, but `eval` is a
 method because it requires dynamic binding.
 
 **Note:** Starting from Nim 0.20, to use multi-methods one must explicitly pass
-`--multimethods:on` when compiling.
+``--multimethods:on`` when compiling.
 
 In a multi-method all parameters that have an object type are used for the
 dispatching:
 
-.. code-block:: nim
-    :test: "nim c --multiMethods:on $1"
-
+  ```nim  test = "nim c --multiMethods:on $1"
   type
     Thing = ref object of RootObj
     Unit = ref object of Thing
@@ -322,6 +320,7 @@ dispatching:
   new a
   new b
   collide(a, b) # output: 2
+  ```
 
 
 As the example demonstrates, invocation of a multi-method cannot be ambiguous:
@@ -338,7 +337,7 @@ Exceptions
 ==========
 
 In Nim exceptions are objects. By convention, exception types are
-suffixed with 'Error'. The `system `_ module defines an
+suffixed with 'Error'. The [system](system.html) module defines an
 exception hierarchy that you might want to stick to. Exceptions derive from
 `system.Exception`, which provides the common interface.
 
@@ -354,20 +353,21 @@ Raise statement
 ---------------
 Raising an exception is done with the `raise` statement:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   var
     e: ref OSError
   new(e)
   e.msg = "the request to the OS failed"
   raise e
+  ```
 
 If the `raise` keyword is not followed by an expression, the last exception
 is *re-raised*. For the purpose of avoiding repeating this common code pattern,
 the template `newException` in the `system` module can be used:
 
-.. code-block:: nim
+  ```nim
   raise newException(OSError, "the request to the OS failed")
+  ```
 
 
 Try statement
@@ -375,8 +375,7 @@ Try statement
 
 The `try` statement handles exceptions:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   from std/strutils import parseInt
 
   # read the first two lines of a text file that should contain numbers
@@ -394,12 +393,13 @@ The `try` statement handles exceptions:
       echo "could not convert string to integer"
     except IOError:
       echo "IO error!"
-    except:
+    except CatchableError:
       echo "Unknown exception!"
       # reraise the unknown exception:
       raise
     finally:
       close(f)
+  ```
 
 The statements after the `try` are executed unless an exception is
 raised. Then the appropriate `except` part is executed.
@@ -417,19 +417,20 @@ the rest of the procedure - that is not within a `finally` clause -
 is not executed (if an exception occurs).
 
 If you need to *access* the actual exception object or message inside an
-`except` branch you can use the `getCurrentException()
-`_ and `getCurrentExceptionMsg()
-`_ procs from the `system `_
+`except` branch you can use the [getCurrentException()](
+system.html#getCurrentException) and [getCurrentExceptionMsg()](
+system.html#getCurrentExceptionMsg) procs from the [system](system.html)
 module. Example:
 
-.. code-block:: nim
+  ```nim
   try:
     doSomethingHere()
-  except:
+  except CatchableError:
     let
       e = getCurrentException()
       msg = getCurrentExceptionMsg()
     echo "Got exception ", repr(e), " with message ", msg
+  ```
 
 
 Annotating procs with raised exceptions
@@ -442,12 +443,13 @@ instance, if you specify that a proc raises `IOError`, and at some point it
 (or one of the procs it calls) starts raising a new exception the compiler will
 prevent that proc from compiling. Usage example:
 
-.. code-block:: nim
+  ```nim
   proc complexProc() {.raises: [IOError, ArithmeticDefect].} =
     ...
 
   proc simpleProc() {.raises: [].} =
     ...
+  ```
 
 Once you have code like this in place, if the list of raised exception changes
 the compiler will stop with an error specifying the line of the proc which
@@ -459,10 +461,10 @@ If you want to add the `{.raises.}` pragma to existing code, the compiler can
 also help you. You can add the `{.effects.}` pragma statement to your proc and
 the compiler will output all inferred effects up to that point (exception
 tracking is part of Nim's effect system). Another more roundabout way to
-find out the list of exceptions raised by a proc is to use the Nim `doc`
+find out the list of exceptions raised by a proc is to use the Nim ``doc``
 command which generates documentation for a whole module and decorates all
 procs with the list of raised exceptions. You can read more about Nim's
-`effect system and related pragmas in the manual `_.
+[effect system and related pragmas in the manual](manual.html#effect-system).
 
 
 Generics
@@ -473,8 +475,7 @@ with `type parameters`:idx:. Generic parameters are written within square
 brackets, for example `Foo[T]`. They are most useful for efficient type safe
 containers:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   type
     BinaryTree*[T] = ref object # BinaryTree is a generic type with
                                 # generic param `T`
@@ -529,6 +530,7 @@ containers:
   add(root, "world")          # instantiates the second `add` proc
   for str in preorder(root):
     stdout.writeLine(str)
+  ```
 
 The example shows a generic binary tree. Depending on context, the brackets are
 used either to introduce type parameters or to instantiate a generic proc,
@@ -538,8 +540,7 @@ is not hidden and is used in the `preorder` iterator.
 
 There is a special `[:T]` syntax when using generics with the method call syntax:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   proc foo[T](i: T) =
     discard
 
@@ -548,6 +549,7 @@ There is a special `[:T]` syntax when using generics with the method call syntax
   # i.foo[int]() # Error: expression 'foo(i)' has no type (or is ambiguous)
 
   i.foo[:int]() # Success
+  ```
 
 
 Templates
@@ -562,12 +564,13 @@ To *invoke* a template, call it like a procedure.
 
 Example:
 
-.. code-block:: nim
+  ```nim
   template `!=` (a, b: untyped): untyped =
     # this definition exists in the System module
     not (a == b)
 
   assert(5 != 6) # the compiler rewrites that to: assert(not (5 == 6))
+  ```
 
 The `!=`, `>`, `>=`, `in`, `notin`, `isnot` operators are in fact
 templates: this has the benefit that if you overload the `==` operator,
@@ -581,8 +584,7 @@ for IEEE floating point numbers - NaN breaks basic boolean logic.)
 Templates are especially useful for lazy evaluation purposes. Consider a
 simple proc for logging:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   const
     debug = true
 
@@ -592,6 +594,7 @@ simple proc for logging:
   var
     x = 4
   log("x has the value: " & $x)
+  ```
 
 This code has a shortcoming: if `debug` is set to false someday, the quite
 expensive `$` and `&` operations are still performed! (The argument
@@ -599,8 +602,7 @@ evaluation for procedures is *eager*).
 
 Turning the `log` proc into a template solves this problem:
 
-.. code-block:: nim
-    :test: "nim c $1"
+  ```nim  test = "nim c $1"
   const
     debug = true
 
@@ -610,6 +612,7 @@ Turning the `log` proc into a template solves this problem:
   var
     x = 4
   log("x has the value: " & $x)
+  ```
 
 The parameters' types can be ordinary types or the meta types `untyped`,
 `typed`, or `type`. `type` suggests that only a type symbol may be given
@@ -621,9 +624,7 @@ If the template has no explicit return type,
 
 To pass a block of statements to a template, use `untyped` for the last parameter:
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   template withFile(f: untyped, filename: string, mode: FileMode,
                     body: untyped) =
     let fn = filename
@@ -639,6 +640,7 @@ To pass a block of statements to a template, use `untyped` for the last paramete
   withFile(txt, "ttempl3.txt", fmWrite):
     txt.writeLine("line 1")
     txt.writeLine("line 2")
+  ```
 
 In the example the two `writeLine` statements are bound to the `body`
 parameter. The `withFile` template contains boilerplate code and helps to
@@ -649,8 +651,7 @@ once.
 Example: Lifting Procs
 ----------------------
 
-.. code-block:: nim
-    :test: "nim c $1"
+  `````nim  test = "nim c $1"
   import std/math
 
   template liftScalarProc(fname) =
@@ -659,9 +660,10 @@ Example: Lifting Procs
     ## to provide templated procs that can handle a single
     ## parameter of seq[T] or nested seq[seq[]] or the same type
     ##
-    ## .. code-block:: Nim
-    ##  liftScalarProc(abs)
-    ##  # now abs(@[@[1,-2], @[-2,-3]]) == @[@[1,2], @[2,3]]
+    ##   ```Nim
+    ##   liftScalarProc(abs)
+    ##   # now abs(@[@[1,-2], @[-2,-3]]) == @[@[1,2], @[2,3]]
+    ##   ```
     proc fname[T](x: openarray[T]): auto =
       var temp: T
       type outType = typeof(fname(temp))
@@ -671,6 +673,7 @@ Example: Lifting Procs
 
   liftScalarProc(sqrt)   # make sqrt() work for sequences
   echo sqrt(@[4.0, 16.0, 25.0, 36.0])   # => @[2.0, 4.0, 5.0, 6.0]
+  `````
 
 Compilation to JavaScript
 =========================
@@ -691,4 +694,4 @@ JavaScript-compatible code you should remember the following:
 Part 3
 ======
 
-The next part is entirely about metaprogramming via macros: `Part III `_
+The next part is entirely about metaprogramming via macros: [Part III](tut3.html).
diff --git a/doc/tut3.rst b/doc/tut3.md
similarity index 91%
rename from doc/tut3.rst
rename to doc/tut3.md
index 358a9b45e4..67f49c8793 100644
--- a/doc/tut3.rst
+++ b/doc/tut3.md
@@ -1,5 +1,3 @@
-.. default-role:: code
-
 =======================
 Nim Tutorial (Part III)
 =======================
@@ -7,13 +5,15 @@ Nim Tutorial (Part III)
 :Author: Arne Döring
 :Version: |nimversion|
 
+.. default-role:: code
+.. include:: rstcommon.rst
 .. contents::
 
 
 Introduction
 ============
 
-  "With Great Power Comes Great Responsibility." -- Spider Man's Uncle
+>  "With Great Power Comes Great Responsibility." -- Spider Man's Uncle
 
 This document is a tutorial about Nim's macro system.
 A macro is a function that is executed at compile-time and transforms
@@ -91,14 +91,14 @@ in the expression `foo(x)`, `x` needs to be an integer constant,
 but in the macro body `arg` is just like a normal parameter of type
 `int`.
 
-.. code-block:: nim
-
+  ```nim
   import std/macros
 
   macro myMacro(arg: static[int]): untyped =
     echo arg # just an int (7), not `NimNode`
 
   myMacro(1 + 2 * 3)
+  ```
 
 
 Code Blocks as Arguments
@@ -108,12 +108,12 @@ It is possible to pass the last argument of a call expression in a
 separate code block with indentation. For example, the following code
 example is a valid (but not a recommended) way to call `echo`:
 
-.. code-block:: nim
-
+  ```nim
   echo "Hello ":
     let a = "Wor"
     let b = "ld!"
     a & b
+  ```
 
 For macros this way of calling is very useful; syntax trees of arbitrary
 complexity can be passed to macros with this notation.
@@ -125,7 +125,7 @@ The Syntax Tree
 In order to build a Nim syntax tree one needs to know how Nim source
 code is represented as a syntax tree, and how such a tree needs to
 look like so that the Nim compiler will understand it. The nodes of the
-Nim syntax tree are documented in the `macros `_ module.
+Nim syntax tree are documented in the [macros](macros.html) module.
 But a more interactive way to explore the Nim
 syntax tree is with `macros.treeRepr`, it converts a syntax tree
 into a multi-line string for printing on the console. It can be used
@@ -134,8 +134,7 @@ and for debug printing of generated syntax tree. `dumpTree` is a
 predefined macro that just prints its argument in a tree representation,
 but does nothing else. Here is an example of such a tree representation:
 
-.. code-block:: nim
-
+  ```nim
   dumpTree:
     var mt: MyType = MyType(a:123.456, b:"abcdef")
 
@@ -153,6 +152,7 @@ but does nothing else. Here is an example of such a tree representation:
   #           ExprColonExpr
   #             Ident "b"
   #             StrLit "abcdef"
+  ```
 
 
 Custom Semantic Checking
@@ -166,10 +166,10 @@ macro evaluation should be caught and create a nice error message.
 the checks need to be more complex, arbitrary error messages can
 be created with the `macros.error` proc.
 
-.. code-block:: nim
-
+  ```nim
   macro myAssert(arg: untyped): untyped =
     arg.expectKind nnkInfix
+  ```
 
 
 Generating Code
@@ -187,22 +187,36 @@ tree with calls to `newTree` and `newLit` the macro
 Backticks are used to insert code from `NimNode` symbols into the
 generated expression.
 
-.. code-block:: nim
-    macro a(i) = quote do: let `i` = 0
-    a b
+  ```nim  test = "nim c $1"
+  import std/macros
+  macro a(i) = quote do:
+    let `i` = 0
+
+  a b
+  doAssert b == 0
+  ```
 
 A custom prefix operator can be defined whenever backticks are needed.
 
-.. code-block:: nim
-    macro a(i) = quote("@") do: assert @i == 0
-    let b = 0
-    a b
+  ```nim  test = "nim c $1"
+  import std/macros
+  macro a(i) = quote("@") do:
+    assert @i == 0
+
+  let b = 0
+  a b
+  ```
 
 The injected symbol needs accent quoted when it resolves to a symbol.
 
-.. code-block:: nim
-    macro a(i) = quote("@") do: let `@i` == 0
-    a b
+  ```nim  test = "nim c $1"
+  import std/macros
+  macro a(i) = quote("@") do:
+    let `@i` = 0
+
+  a b
+  doAssert b == 0
+  ```
 
 Make sure to inject only symbols of type `NimNode` into the generated syntax
 tree. You can use `newLit` to convert arbitrary values into
@@ -210,9 +224,7 @@ expressions trees of type `NimNode` so that it is safe to inject
 them into the tree.
 
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   import std/macros
 
   type
@@ -232,26 +244,26 @@ them into the tree.
       echo `mtLit`
 
   myMacro("Hallo")
+  ```
 
 The call to `myMacro` will generate the following code:
 
-.. code-block:: nim
+  ```nim
   echo "Hallo"
   echo MyType(a: 123.456'f64, b: "abcdef")
+  ```
 
 
 Building Your First Macro
 -------------------------
 
 To give a starting point to writing macros we will show now how to
-implement the `myDebug` macro mentioned earlier. The first thing to
+implement the `myAssert` macro mentioned earlier. The first thing to
 do is to build a simple example of the macro usage, and then just
 print the argument. This way it is possible to get an idea of what a
 correct argument should look like.
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   import std/macros
 
   macro myAssert(arg: untyped): untyped =
@@ -261,13 +273,14 @@ correct argument should look like.
   let b = 2
 
   myAssert(a != b)
+  ```
 
-.. code-block::
-
+  ```
   Infix
     Ident "!="
     Ident "a"
     Ident "b"
+  ```
 
 
 From the output, it is possible to see that the argument is an infix
@@ -275,9 +288,7 @@ operator (node kind is "Infix"), as well as that the two operands are
 at index 1 and 2. With this information, the actual macro can be
 written.
 
-.. code-block:: nim
-    :test: "nim c $1"
-
+  ```nim  test = "nim c $1"
   import std/macros
 
   macro myAssert(arg: untyped): untyped =
@@ -298,6 +309,7 @@ written.
 
   myAssert(a != b)
   myAssert(a == b)
+  ```
 
 
 This is the code that will be generated. To debug what the macro
@@ -305,9 +317,10 @@ actually generated, the statement `echo result.repr` can be used, in
 the last line of the macro. It is also the statement that has been
 used to get this output.
 
-.. code-block:: nim
+  ```nim
   if not (a != b):
     raise newException(AssertionDefect, $a & " != " & $b)
+  ```
 
 With Power Comes Responsibility
 -------------------------------
@@ -353,7 +366,7 @@ recommended way. But still `strformat` is a good example for a
 practical use case for a macro that is slightly more complex than the
 `assert` macro.
 
-`Strformat `_
+[Strformat](https://github.com/nim-lang/Nim/blob/devel/lib/pure/strformat.nim)
 
 Ast Pattern Matching
 --------------------
@@ -362,7 +375,7 @@ Ast Pattern Matching is a macro library to aid in writing complex
 macros. This can be seen as a good example of how to repurpose the
 Nim syntax tree with new semantics.
 
-`Ast Pattern Matching `_
+[Ast Pattern Matching](https://github.com/nim-lang/ast-pattern-matching)
 
 OpenGL Sandbox
 --------------
@@ -371,4 +384,4 @@ This project has a working Nim to GLSL compiler written entirely in
 macros. It scans recursively through all used function symbols to
 compile them so that cross library functions can be executed on the GPU.
 
-`OpenGL Sandbox `_
+[OpenGL Sandbox](https://github.com/krux02/opengl-sandbox)
diff --git a/drnim/drnim.nim b/drnim/drnim.nim
index 44eab86259..eb0d89aa28 100644
--- a/drnim/drnim.nim
+++ b/drnim/drnim.nim
@@ -68,6 +68,7 @@ type
   DrnimContext = ref object
     z3: Z3_context
     graph: ModuleGraph
+    idgen: IdGenerator
     facts: seq[(PNode, VersionScope)]
     varVersions: seq[int] # this maps variable IDs to their current version.
     varSyms: seq[PSym] # mirrors 'varVersions'
@@ -80,6 +81,7 @@ type
 
   DrCon = object
     graph: ModuleGraph
+    idgen: IdGenerator
     mapping: Table[string, Z3_ast]
     canonParameterNames: bool
     assumeUniqueness: bool
@@ -301,12 +303,12 @@ proc nodeToZ3(c: var DrCon; n: PNode; scope: VersionScope; vars: var seq[PNode])
 
 proc nodeToDomain(c: var DrCon; n, q: PNode; opAnd: PSym): PNode =
   assert n.kind == nkInfix
-  let opLe = createMagic(c.graph, "<=", mLeI)
+  let opLe = createMagic(c.graph, c.idgen, "<=", mLeI)
   case $n[0]
   of "..":
     result = buildCall(opAnd, buildCall(opLe, n[1], q), buildCall(opLe, q, n[2]))
   of "..<":
-    let opLt = createMagic(c.graph, "<", mLtI)
+    let opLt = createMagic(c.graph, c.idgen, "<", mLtI)
     result = buildCall(opAnd, buildCall(opLe, n[1], q), buildCall(opLt, q, n[2]))
   else:
     notImplemented(n)
@@ -315,7 +317,7 @@ template quantorToZ3(fn) {.dirty.} =
   template ctx: untyped = c.up.z3
 
   var bound = newSeq[Z3_app](n.len-2)
-  let opAnd = createMagic(c.graph, "and", mAnd)
+  let opAnd = createMagic(c.graph, c.idgen, "and", mAnd)
   var known: PNode
   for i in 1..n.len-2:
     let it = n[i]
@@ -333,7 +335,7 @@ template quantorToZ3(fn) {.dirty.} =
 
   var dummy: seq[PNode]
   assert known != nil
-  let x = nodeToZ3(c, buildCall(createMagic(c.graph, "->", mImplies),
+  let x = nodeToZ3(c, buildCall(createMagic(c.graph, c.idgen, "->", mImplies),
                    known, n[^1]), scope, dummy)
   result = fn(ctx, 0, bound.len.cuint, addr(bound[0]), 0, nil, x)
 
@@ -395,8 +397,8 @@ proc nodeToZ3(c: var DrCon; n: PNode; scope: VersionScope; vars: var seq[PNode])
       else:
         notImplemented(n)
     of mHigh:
-      let addOpr = createMagic(c.graph, "+", mAddI)
-      let lenOpr = createMagic(c.graph, "len", mLengthOpenArray)
+      let addOpr = createMagic(c.graph, c.idgen, "+", mAddI)
+      let lenOpr = createMagic(c.graph, c.idgen, "len", mLengthOpenArray)
       let asLenExpr = addOpr.buildCall(lenOpr.buildCall(n[1]), nkIntLit.newIntNode(-1))
       result = rec asLenExpr
     of mLow:
@@ -577,8 +579,8 @@ proc addRangeInfo(c: var DrCon, n: PNode; scope: VersionScope, res: var seq[Z3_a
         res.add nodeToZ3(c, translateEnsures(ensures, n), scope, dummy)
     return
 
-  let x = newTree(nkInfix, newSymNode createMagic(c.graph, "<=", cmpOp), lowBound, n)
-  let y = newTree(nkInfix, newSymNode createMagic(c.graph, "<=", cmpOp), n, highBound)
+  let x = newTree(nkInfix, newSymNode createMagic(c.graph, c.idgen, "<=", cmpOp), lowBound, n)
+  let y = newTree(nkInfix, newSymNode createMagic(c.graph, c.idgen, "<=", cmpOp), n, highBound)
 
   var dummy: seq[PNode]
   res.add nodeToZ3(c, x, scope, dummy)
@@ -677,6 +679,7 @@ proc proofEngine(ctx: DrnimContext; assumptions: seq[(PNode, VersionScope)];
                  toProve: (PNode, VersionScope)): (bool, string) =
   var c: DrCon
   c.graph = ctx.graph
+  c.idgen = ctx.idgen
   c.assumeUniqueness = assumeUniqueness
   c.up = ctx
   result = proofEngineAux(c, assumptions, toProve)
@@ -738,10 +741,11 @@ proc compatibleProps(graph: ModuleGraph; formal, actual: PType): bool {.nimcall.
 
       var c: DrCon
       c.graph = graph
+      c.idgen = graph.idgen
       c.canonParameterNames = true
       try:
         c.up = DrnimContext(z3: setupZ3(), o: initOperators(graph), graph: graph, owner: nil,
-          opImplies: createMagic(graph, "->", mImplies))
+          opImplies: createMagic(graph, c.idgen, "->", mImplies))
         template zero: untyped = VersionScope(0)
         if not frequires.isEmpty:
           result = not arequires.isEmpty and proofEngineAux(c, @[(frequires, zero)], (arequires, zero))[0]
@@ -847,7 +851,7 @@ proc checkLe(c: DrnimContext, a, b: PNode) =
     of tyChar, tyUInt..tyUInt64: cmpOp = mLeU
     else: discard
 
-  let cmp = newTree(nkInfix, newSymNode createMagic(c.graph, "<=", cmpOp), a, b)
+  let cmp = newTree(nkInfix, newSymNode createMagic(c.graph, c.idgen, "<=", cmpOp), a, b)
   cmp.info = a.info
   discard prove(c, cmp)
 
@@ -1026,7 +1030,7 @@ proc traverseAsgn(c: DrnimContext; n: PNode) =
   proc replaceByOldParams(fact, le: PNode): PNode =
     if guards.sameTree(fact, le):
       result = newNodeIT(nkCall, fact.info, fact.typ)
-      result.add newSymNode createMagic(c.graph, "old", mOld)
+      result.add newSymNode createMagic(c.graph, c.idgen, "old", mOld)
       result.add fact
     else:
       result = shallowCopy(fact)
@@ -1176,8 +1180,9 @@ proc strongSemCheck(graph: ModuleGraph; owner: PSym; n: PNode) =
     c.z3 = setupZ3()
     c.o = initOperators(graph)
     c.graph = graph
+    c.idgen = graph.idgen
     c.owner = owner
-    c.opImplies = createMagic(c.graph, "->", mImplies)
+    c.opImplies = createMagic(c.graph, c.idgen, "->", mImplies)
     try:
       traverse(c, n)
       ensuresCheck(c, owner)
@@ -1200,23 +1205,7 @@ proc mainCommand(graph: ModuleGraph) =
   registerPass graph, semPass
   compileProject(graph)
   if conf.errorCounter == 0:
-    let mem =
-      when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
-      else: formatSize(getTotalMem()) & " totmem"
-    let loc = $conf.linesCompiled
-    let build = if isDefined(conf, "danger"): "Dangerous Release"
-                elif isDefined(conf, "release"): "Release"
-                else: "Debug"
-    let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
-    let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName
-    rawMessage(conf, hintSuccessX, [
-      "loc", loc,
-      "sec", sec,
-      "mem", mem,
-      "build", build,
-      "project", project,
-      "output", ""
-      ])
+    genSuccessX(graph.config)
 
 proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
   var p = parseopt.initOptParser(cmd)
diff --git a/drnim/tests/config.nims b/drnim/tests/config.nims
index a6b0e7d88c..346b0b4e66 100644
--- a/drnim/tests/config.nims
+++ b/drnim/tests/config.nims
@@ -3,7 +3,7 @@ switch("path", "$nim/testament/lib") # so we can `import stdtest/foo` in this di
 ## prevent common user config settings to interfere with testament expectations
 ## Indifidual tests can override this if needed to test for these options.
 switch("colors", "off")
-switch("listFullPaths", "off")
+switch("filenames", "canonical")
 switch("excessiveStackTrace", "off")
 
 # we only want to check the marked parts in the tests:
diff --git a/koch.nim b/koch.nim
index bf7fb1e620..089f8e1ab3 100644
--- a/koch.nim
+++ b/koch.nim
@@ -10,8 +10,13 @@
 #
 
 const
-  NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master
-  # examples of possible values: #head, #ea82b54, 1.2.3
+  # examples of possible values for repos: Head, ea82b54
+  NimbleStableCommit = "f8bd7b5fa6ea7a583b411b5959b06e6b5eb23667" # master
+  AtlasStableCommit = "5faec3e9a33afe99a7d22377dd1b45a5391f5504"
+  ChecksumsStableCommit = "025bcca3915a1b9f19878cea12ad68f9884648fc"
+  SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01"
+
+  # examples of possible values for fusion: #head, #ea82b54, 1.2.3
   FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014"
   HeadHash = "#head"
 when not defined(windows):
@@ -34,6 +39,9 @@ import std/[os, strutils, parseopt, osproc]
   # If this fails with: `Error: cannot open file: std/os`, see
   # https://github.com/nim-lang/Nim/pull/14291 for explanation + how to fix.
 
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, syncio]
+
 import tools / kochdocs
 import tools / deps
 
@@ -59,6 +67,7 @@ Options:
   --nim:path               use specified path for nim binary
   --localdocs[:path]       only build local documentations. If a path is not
                            specified (or empty), the default is used.
+  --skipIntegrityCheck     skips integrity check when booting the compiler
 Possible Commands:
   boot [options]           bootstraps with given command line options
   distrohelper [bindir]    helper for distro packagers
@@ -67,6 +76,7 @@ Possible Commands:
                            e.g. nimble)
                            doesn't require network connectivity
   nimble                   builds the Nimble tool
+  atlas                    builds the Atlas tool
   fusion                   installs fusion via Nimble
 
 Boot options:
@@ -76,9 +86,12 @@ Boot options:
   -d:leanCompiler          produce a compiler without JS codegen or
                            documentation generator in order to use less RAM
                            for bootstrapping
+  -d:nimHasLibFFI          adds FFI support for allowing compile-time VM to
+                           interface with native functions (experimental,
+                           requires prior `koch installdeps libffi`)
 
 Commands for core developers:
-  runCI                    runs continuous integration (CI), e.g. from travis
+  runCI                    runs continuous integration (CI), e.g. from Github Actions
   docs [options]           generates the full documentation
   csource -d:danger        builds the C sources for installation
   pdf                      builds the PDF documentation
@@ -89,10 +102,6 @@ Commands for core developers:
   tests [options]          run the testsuite (run a subset of tests by
                            specifying a category, e.g. `tests cat async`)
   temp options             creates a temporary compiler for testing
-  pushcsource              push generated C sources to its repo
-Web options:
-  --googleAnalytics:UA-... add the given google analytics code to the docs. To
-                           build the official docs, use UA-48159761-1
 """
 
 let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed e.g.: `nim c -o:koch_debug koch.nim`
@@ -149,13 +158,27 @@ proc bundleNimbleExe(latest: bool, args: string) =
   let commit = if latest: "HEAD" else: NimbleStableCommit
   cloneDependency(distDir, "https://github.com/nim-lang/nimble.git",
                   commit = commit, allowBundled = true)
+  cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/checksums.git",
+                commit = ChecksumsStableCommit, allowBundled = true) # or copy it from dist?
+  cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/sat.git",
+                commit = SatStableCommit, allowBundled = true)
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim",
-             options = "-d:release --noNimblePath " & args)
+             options = "-d:release -d:nimNimbleBootstrap --noNimblePath " & args)
+
+proc bundleAtlasExe(latest: bool, args: string) =
+  let commit = if latest: "HEAD" else: AtlasStableCommit
+  cloneDependency(distDir, "https://github.com/nim-lang/atlas.git",
+                  commit = commit, allowBundled = true)
+  cloneDependency(distDir / "atlas" / distDir, "https://github.com/nim-lang/sat.git",
+                commit = SatStableCommit, allowBundled = true)
+  # installer.ini expects it under $nim/bin
+  nimCompile("dist/atlas/src/atlas.nim",
+             options = "-d:release --noNimblePath -d:nimAtlasBootstrap " & args)
 
 proc bundleNimsuggest(args: string) =
   nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim",
-                 options = "-d:release -d:danger " & args)
+                 options = "-d:danger " & args)
 
 proc buildVccTool(args: string) =
   let input = "tools/vccexe/vccexe.nim"
@@ -182,8 +205,14 @@ proc bundleWinTools(args: string) =
     nimCompile(r"tools\downloader.nim",
                options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui " & args)
 
+proc bundleChecksums(latest: bool) =
+  let commit = if latest: "HEAD" else: ChecksumsStableCommit
+  cloneDependency(distDir, "https://github.com/nim-lang/checksums.git", commit, allowBundled = true)
+
 proc zip(latest: bool; args: string) =
+  bundleChecksums(latest)
   bundleNimbleExe(latest, args)
+  bundleAtlasExe(latest, args)
   bundleNimsuggest(args)
   bundleNimpretty(args)
   bundleWinTools(args)
@@ -216,13 +245,28 @@ proc buildTools(args: string = "") =
                  options = "-d:release " & args)
   when defined(windows): buildVccTool(args)
   bundleNimpretty(args)
-  nimCompileFold("Compile nimfind", "tools/nimfind.nim",
-                 options = "-d:release " & args)
-  nimCompileFold("Compile testament", "testament/testament.nim",
+  nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args)
+
+  # pre-packages a debug version of nim which can help in many cases investigate issuses
+  # withouth having to rebuild compiler.
+  # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler
+  # `-d:debug` should be changed to a flag that doesn't require re-compiling nim
+  # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces
+  nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options =
+      "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args,
+      outputName = "nim_dbg")
+
+proc testTools(args: string = "") =
+  nimCompileFold("Compile nimgrep", "tools/nimgrep.nim",
                  options = "-d:release " & args)
+  when defined(windows): buildVccTool(args)
+  bundleNimpretty(args)
+  nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args)
 
 proc nsis(latest: bool; args: string) =
+  bundleChecksums(latest)
   bundleNimbleExe(latest, args)
+  bundleAtlasExe(latest, args)
   bundleNimsuggest(args)
   bundleWinTools(args)
   # make sure we have generated the niminst executables:
@@ -242,19 +286,21 @@ proc install(args: string) =
   geninstall()
   exec("sh ./install.sh $#" % args)
 
-when false:
-  proc web(args: string) =
-    nimexec("js tools/dochack/dochack.nim")
-    nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" %
-        [args, VersionAsString])
-
-  proc website(args: string) =
-    nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" %
-        [args, VersionAsString])
-
-  proc pdf(args="") =
-    exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" %
-        [findNim().quoteShell(), args, VersionAsString], additionalPATH=findNim().splitFile.dir)
+proc installDeps(dep: string, commit = "") =
+  # the hashes/urls are version controlled here, so can be changed seamlessly
+  # and tied to a nim release (mimicking git submodules)
+  var commit = commit
+  case dep
+  of "tinyc":
+    if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3"
+    cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit)
+  of "libffi":
+    # technically a nimble package, however to play nicely with --noNimblePath,
+    # let's just clone it wholesale:
+    if commit.len == 0: commit = "bb2bdaf1a29a4bff6fbd8ae4695877cbb3ec783e"
+    cloneDependency(distDir, "https://github.com/Araq/libffi", commit)
+  else: doAssert false, "unsupported: " & dep
+  # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206
 
 # -------------- boot ---------------------------------------------------------
 
@@ -283,9 +329,9 @@ proc thVersion(i: int): string =
 
 template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true"
 
-proc boot(args: string) =
+proc boot(args: string, skipIntegrityCheck: bool) =
   ## bootstrapping is a process that involves 3 steps:
-  ## 1. use csources to produce nim1.exe. This nim1.exe is buggy but
+  ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but
   ## rock solid for building a Nim compiler. It shouldn't be used for anything else.
   ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need.
   ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps.
@@ -298,39 +344,42 @@ proc boot(args: string) =
   let smartNimcache = (if "release" in args or "danger" in args: "nimcache/r_" else: "nimcache/d_") &
                       hostOS & "_" & hostCPU
 
+  if not dirExists("dist/checksums"):
+    bundleChecksums(false)
+
+  let usingLibFFI = "nimHasLibFFI" in args
+  if usingLibFFI and not dirExists("dist/libffi"):
+    installDeps("libffi")
+
   let nimStart = findStartNim().quoteShell()
-  for i in 0..2:
-    # Nim versions < (1, 1) expect Nim's exception type to have a 'raiseId' field for
-    # C++ interop. Later Nim versions do this differently and removed the 'raiseId' field.
-    # Thus we always bootstrap the first iteration with "c" and not with "cpp" as
-    # a workaround.
-    let defaultCommand = if useCpp and i > 0: "cpp" else: "c"
+  let times = 2 - ord(skipIntegrityCheck)
+  for i in 0..times:
+    let defaultCommand = if useCpp: "cpp" else: "c"
     let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: ""
     echo "iteration: ", i+1
     var extraOption = ""
     var nimi = i.thVersion
     if i == 0:
       nimi = nimStart
-      extraOption.add " --skipUserCfg --skipParentCfg"
+      extraOption.add " --skipUserCfg --skipParentCfg -d:nimKochBootstrap"
+
+      # --noNimblePath precludes nimble packages as dependencies to the compiler,
+      # so libffi is not "installed as a nimble package"
+      if usingLibFFI: extraOption.add " --path:./dist"
         # The configs are skipped for bootstrap
         # (1st iteration) to prevent newer flags from breaking bootstrap phase.
       let ret = execCmdEx(nimStart & " --version")
       doAssert ret.exitCode == 0
       let version = ret.output.splitLines[0]
-      # remove these when csources get updated
-      template addLib() =
+      if version.startsWith "Nim Compiler Version 0.20.0":
         extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291
-      if version.startsWith "Nim Compiler Version 0.19.0":
-        extraOption.add " -d:nimBoostrapCsources0_19_0"
-        addLib()
-      elif version.startsWith "Nim Compiler Version 0.20.0": addLib()
 
     # in order to use less memory, we split the build into two steps:
     # --compileOnly produces a $project.json file and does not run GCC/Clang.
     # jsonbuild then uses the $project.json file to build the Nim binary.
-    exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" %
+    exec "$# $# $# --nimcache:$# $# --noNimblePath --compileOnly compiler" / "nim.nim" %
       [nimi, bootOptions, extraOption, smartNimcache, args]
-    exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" %
+    exec "$# jsonscript --noNimblePath --nimcache:$# $# compiler" / "nim.nim" %
       [nimi, smartNimcache, args]
 
     if sameFileContent(output, i.thVersion):
@@ -339,7 +388,9 @@ proc boot(args: string) =
       return
     copyExe(output, (i+1).thVersion)
   copyExe(output, finalDest)
-  when not defined(windows): echo "[Warning] executables are still not equal"
+  when not defined(windows):
+    if not skipIntegrityCheck:
+      echo "[Warning] executables are still not equal"
 
 # -------------- clean --------------------------------------------------------
 
@@ -457,9 +508,12 @@ proc temp(args: string) =
       result[1].add " " & quoteShell(args[i])
       inc i
 
+  if not dirExists("dist/checksums"):
+    bundleChecksums(false)
+
   let d = getAppDir()
-  var output = d / "compiler" / "nim".exe
-  var finalDest = d / "bin" / "nim_temp".exe
+  let output = d / "compiler" / "nim".exe
+  let finalDest = d / "bin" / "nim_temp".exe
   # 125 is the magic number to tell git bisect to skip the current commit.
   var (bootArgs, programArgs) = splitArgs(args)
   if "doc" notin programArgs and
@@ -483,6 +537,22 @@ proc xtemp(cmd: string) =
   finally:
     copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe)
 
+proc icTest(args: string) =
+  temp("")
+  let inp = os.parseCmdLine(args)[0]
+  let content = readFile(inp)
+  let nimExe = getAppDir() / "bin" / "nim_temp".exe
+  var i = 0
+  for fragment in content.split("#!EDIT!#"):
+    let file = inp.replace(".nim", "_temp.nim")
+    writeFile(file, fragment)
+    var cmd = nimExe & " cpp --ic:on -d:nimIcIntegrityChecks --listcmd "
+    if i == 0:
+      cmd.add "-f "
+    cmd.add quoteShell(file)
+    exec(cmd)
+    inc i
+
 proc buildDrNim(args: string) =
   if not dirExists("dist/nimz3"):
     exec("git clone https://github.com/zevv/nimz3.git dist/nimz3")
@@ -510,32 +580,30 @@ proc hostInfo(): string =
   "hostOS: $1, hostCPU: $2, int: $3, float: $4, cpuEndian: $5, cwd: $6" %
     [hostOS, hostCPU, $int.sizeof, $float.sizeof, $cpuEndian, getCurrentDir()]
 
-proc installDeps(dep: string, commit = "") =
-  # the hashes/urls are version controlled here, so can be changed seamlessly
-  # and tied to a nim release (mimicking git submodules)
-  var commit = commit
-  case dep
-  of "tinyc":
-    if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3"
-    cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit)
-  else: doAssert false, "unsupported: " & dep
-  # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206
-
 proc runCI(cmd: string) =
   doAssert cmd.len == 0, cmd # avoid silently ignoring
   echo "runCI: ", cmd
   echo hostInfo()
   # boot without -d:nimHasLibFFI to make sure this still works
-  kochExecFold("Boot in release mode", "boot -d:release -d:nimStrictMode")
+  # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in
+  # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd).
+  kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib")
+
+  when false: # debugging: when you need to run only 1 test in CI, use something like this:
+    execFold("debugging test", "nim r tests/stdlib/tosproc.nim")
+    doAssert false, "debugging only"
 
   ## build nimble early on to enable remainder to depend on it if needed
   kochExecFold("Build Nimble", "nimble")
 
+  execFold("Install smtp", "nimble install smtp -y")
+
   let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_")
   if getEnv("NIM_TEST_PACKAGES", "0") == "1":
-    execFold("Test selected Nimble packages", "nim r testament/testament $# pcat nimble-packages" % batchParam)
+    nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release")
+    execFold("Test selected Nimble packages", "testament $# pcat nimble-packages" % batchParam)
   else:
-    buildTools()
+    testTools()
 
     for a in "zip opengl sdl1 jester@#head".split:
       let buildDeps = "build"/"deps" # xxx factor pending https://github.com/timotheecour/Nim/issues/616
@@ -546,51 +614,35 @@ proc runCI(cmd: string) =
     ## run tests
     execFold("Test nimscript", "nim e tests/test_nimscript.nims")
     when defined(windows):
-      # note: will be over-written below
-      execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament")
+      execFold("Compile tester", "nim c --usenimcache -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament")
 
     # main bottleneck here
     # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`.
     # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343
-    execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam)
+    execFold("Run tester", "nim c -r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam)
 
-    block CT_FFI:
+    block: # nimHasLibFFI:
       when defined(posix): # windows can be handled in future PR's
-        execFold("nimble install -y libffi", "nimble install -y libffi")
-        const nimFFI = "./bin/nim.ctffi"
+        installDeps("libffi")
+        const nimFFI = "bin/nim.ctffi"
         # no need to bootstrap with koch boot (would be slower)
         let backend = if doUseCpp(): "cpp" else: "c"
-        execFold("build with -d:nimHasLibFFI", "nim $1 -d:release -d:nimHasLibFFI -o:$2 compiler/nim.nim" % [backend, nimFFI])
+        execFold("build with -d:nimHasLibFFI", "nim $1 -d:release --noNimblePath -d:nimHasLibFFI --path:./dist -o:$2 compiler/nim.nim" % [backend, nimFFI])
         execFold("test with -d:nimHasLibFFI", "$1 $2 -r testament/testament --nim:$1 r tests/misc/trunner.nim -d:nimTrunnerFfi" % [nimFFI, backend])
 
-    execFold("Run nimdoc tests", "nim c -r nimdoc/tester")
-    execFold("Run rst2html tests", "nim c -r nimdoc/rsttester")
-    execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim")
+    execFold("Run nimdoc tests", "nim r nimdoc/tester")
+    execFold("Run rst2html tests", "nim r nimdoc/rsttester")
+    execFold("Run nimpretty tests", "nim r nimpretty/tester.nim")
     when defined(posix):
-      execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester")
+      # refs #18385, build with -d:release instead of -d:danger for testing
+      # We could also skip building nimsuggest in buildTools, or build it with -d:release
+      # in bundleNimsuggest depending on some environment variable when we are in CI. One advantage
+      # of rebuilding is this won't affect bin/nimsuggest when running runCI locally
+      execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest")
+      execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
 
-proc pushCsources() =
-  if not dirExists("../csources/.git"):
-    quit "[Error] no csources git repository found"
-  csource("-d:danger")
-  let cwd = getCurrentDir()
-  try:
-    copyDir("build/c_code", "../csources/c_code")
-    copyFile("build/build.sh", "../csources/build.sh")
-    copyFile("build/build.bat", "../csources/build.bat")
-    copyFile("build/build64.bat", "../csources/build64.bat")
-    copyFile("build/makefile", "../csources/makefile")
+    kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib")
 
-    setCurrentDir("../csources")
-    for kind, path in walkDir("c_code"):
-      if kind == pcDir:
-        exec("git add " & path / "*.c")
-    exec("git commit -am \"updated csources to version " & NimVersion & "\"")
-    exec("git push origin master")
-    exec("git tag -am \"Version $1\" v$1" % NimVersion)
-    exec("git push origin v$1" % NimVersion)
-  finally:
-    setCurrentDir(cwd)
 
 proc testUnixInstall(cmdLineRest: string) =
   csource("-d:danger" & cmdLineRest)
@@ -616,7 +668,7 @@ proc testUnixInstall(cmdLineRest: string) =
       execCleanPath("./koch tools")
       # check the tests work:
       putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH")
-      execCleanPath("./koch tests --nim:./bin/nim cat megatest", destDir / "bin")
+      execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin")
     else:
       echo "Version check: failure"
   finally:
@@ -644,9 +696,16 @@ proc valgrind(cmd: string) =
   let supp = getAppDir() / "tools" / "nimgrind.supp"
   exec("valgrind --suppressions=" & supp & valcmd)
 
-proc showHelp() =
+proc showHelp(success: bool) =
   quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)),
-                   CompileDate, CompileTime], QuitSuccess)
+                   CompileDate, CompileTime], if success: QuitSuccess else: QuitFailure)
+
+proc branchDone() =
+  let thisBranch = execProcess("git symbolic-ref --short HEAD").strip()
+  if thisBranch != "devel" and thisBranch != "":
+    exec("git checkout devel")
+    exec("git branch -D " & thisBranch)
+    exec("git pull --rebase")
 
 when isMainModule:
   var op = initOptParser()
@@ -654,11 +713,13 @@ when isMainModule:
     latest = false
     localDocsOnly = false
     localDocsOut = ""
+    skipIntegrityCheck = false
   while true:
     op.next()
     case op.kind
     of cmdLongOption, cmdShortOption:
       case normalize(op.key)
+      of "help", "h": showHelp(success = true)
       of "latest": latest = true
       of "stable": latest = false
       of "nim": nimExe = op.val.absolutePath # absolute so still works with changeDir
@@ -666,12 +727,14 @@ when isMainModule:
         localDocsOnly = true
         if op.val.len > 0:
           localDocsOut = op.val.absolutePath
-      else: showHelp()
+      of "skipintegritycheck":
+        skipIntegrityCheck = true
+      else: showHelp(success = false)
     of cmdArgument:
       case normalize(op.key)
-      of "boot": boot(op.cmdLineRest)
+      of "boot": boot(op.cmdLineRest, skipIntegrityCheck)
       of "clean": clean(op.cmdLineRest)
-      of "doc", "docs": buildDocs(op.cmdLineRest, localDocsOnly, localDocsOut)
+      of "doc", "docs": buildDocs(op.cmdLineRest & " --d:nimPreviewSlimSystem " & paCode, localDocsOnly, localDocsOut)
       of "doc0", "docs0":
         # undocumented command for Araq-the-merciful:
         buildDocs(op.cmdLineRest & gaCode)
@@ -691,6 +754,7 @@ when isMainModule:
       of "xtemp": xtemp(op.cmdLineRest)
       of "wintools": bundleWinTools(op.cmdLineRest)
       of "nimble": bundleNimbleExe(latest, op.cmdLineRest)
+      of "atlas": bundleAtlasExe(latest, op.cmdLineRest)
       of "nimsuggest": bundleNimsuggest(op.cmdLineRest)
       # toolsNoNimble is kept for backward compatibility with build scripts
       of "toolsnonimble", "toolsnoexternal":
@@ -698,13 +762,20 @@ when isMainModule:
       of "tools":
         buildTools(op.cmdLineRest)
         bundleNimbleExe(latest, op.cmdLineRest)
-      of "pushcsource", "pushcsources": pushCsources()
+        bundleAtlasExe(latest, op.cmdLineRest)
+      of "checksums":
+        bundleChecksums(latest)
+      of "pushcsource":
+        quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim"
       of "valgrind": valgrind(op.cmdLineRest)
       of "c2nim": bundleC2nim(op.cmdLineRest)
       of "drnim": buildDrNim(op.cmdLineRest)
       of "fusion":
         let suffix = if latest: HeadHash else: FusionStableHash
         exec("nimble install -y fusion@$#" % suffix)
-      else: showHelp()
+      of "ic": icTest(op.cmdLineRest)
+      of "branchdone": branchDone()
+      else: showHelp(success = false)
       break
-    of cmdEnd: break
+    of cmdEnd:
+      showHelp(success = false)
diff --git a/lib/core/hotcodereloading.nim b/lib/core/hotcodereloading.nim
index 73f38402de..3a876885ce 100644
--- a/lib/core/hotcodereloading.nim
+++ b/lib/core/hotcodereloading.nim
@@ -11,7 +11,7 @@
 
 when defined(hotcodereloading):
   import
-    macros
+    std/macros
 
   template beforeCodeReload*(body: untyped) =
     hcrAddEventHandler(true, proc = body) {.executeOnReload.}
diff --git a/lib/core/locks.nim b/lib/core/locks.nim
index bddd6d8641..5237274792 100644
--- a/lib/core/locks.nim
+++ b/lib/core/locks.nim
@@ -18,8 +18,7 @@ when not compileOption("threads") and not defined(nimdoc):
   when false: # fix #12330
     {.error: "Locks requires --threads:on option.".}
 
-const insideRLocksModule = false
-include "system/syslocks"
+import std/private/syslocks
 
 type
   Lock* = SysLock ## Nim lock; whether this is re-entrant
@@ -28,25 +27,30 @@ type
 
 {.push stackTrace: off.}
 
+
+proc `$`*(lock: Lock): string =
+  # workaround bug #14873
+  result = "()"
+
 proc initLock*(lock: var Lock) {.inline.} =
   ## Initializes the given lock.
   when not defined(js):
     initSysLock(lock)
 
-proc deinitLock*(lock: var Lock) {.inline.} =
+proc deinitLock*(lock: Lock) {.inline.} =
   ## Frees the resources associated with the lock.
   deinitSys(lock)
 
-proc tryAcquire*(lock: var Lock): bool =
+proc tryAcquire*(lock: var Lock): bool {.inline.} =
   ## Tries to acquire the given lock. Returns `true` on success.
   result = tryAcquireSys(lock)
 
-proc acquire*(lock: var Lock) =
+proc acquire*(lock: var Lock) {.inline.} =
   ## Acquires the given lock.
   when not defined(js):
     acquireSys(lock)
 
-proc release*(lock: var Lock) =
+proc release*(lock: var Lock) {.inline.} =
   ## Releases the given lock.
   when not defined(js):
     releaseSys(lock)
@@ -56,22 +60,26 @@ proc initCond*(cond: var Cond) {.inline.} =
   ## Initializes the given condition variable.
   initSysCond(cond)
 
-proc deinitCond*(cond: var Cond) {.inline.} =
+proc deinitCond*(cond: Cond) {.inline.} =
   ## Frees the resources associated with the condition variable.
   deinitSysCond(cond)
 
 proc wait*(cond: var Cond, lock: var Lock) {.inline.} =
-  ## waits on the condition variable `cond`.
+  ## Waits on the condition variable `cond`.
   waitSysCond(cond, lock)
 
 proc signal*(cond: var Cond) {.inline.} =
-  ## sends a signal to the condition variable `cond`.
+  ## Sends a signal to the condition variable `cond`.
   signalSysCond(cond)
 
+proc broadcast*(cond: var Cond) {.inline.} =
+  ## Unblocks all threads currently blocked on the
+  ## specified condition variable `cond`.
+  broadcastSysCond(cond)
+
 template withLock*(a: Lock, body: untyped) =
   ## Acquires the given lock, executes the statements in body and
   ## releases the lock after the statements finish executing.
-  mixin acquire, release
   acquire(a)
   {.locks: [a].}:
     try:
diff --git a/lib/core/macrocache.nim b/lib/core/macrocache.nim
index 8fe1fa603f..39999fa113 100644
--- a/lib/core/macrocache.nim
+++ b/lib/core/macrocache.nim
@@ -7,38 +7,238 @@
 #    distribution, for details about the copyright.
 #
 
-## This module provides an API for macros that need to collect compile
-## time information across module boundaries in global variables.
-## Starting with version 0.19 of Nim this is not directly supported anymore
-## as it breaks incremental compilations.
-## Instead the API here needs to be used.
+## This module provides an API for macros to collect compile-time information
+## across module boundaries. It should be used instead of global `{.compileTime.}`
+## variables as those break incremental compilation.
+##
+## The main feature of this module is that if you create `CacheTable`s or
+## any other `Cache` types with the same name in different modules, their
+## content will be shared, meaning that you can fill a `CacheTable` in
+## one module, and iterate over its contents in another.
+
+runnableExamples:
+  import std/macros
+
+  const mcTable = CacheTable"myTable"
+  const mcSeq = CacheSeq"mySeq"
+  const mcCounter = CacheCounter"myCounter"
+
+  static:
+    # add new key "val" with the value `myval`
+    let myval = newLit("hello ic")
+    mcTable["val"] = myval
+    assert mcTable["val"].kind == nnkStrLit
+
+  # Can access the same cache from different static contexts
+  # All the information is retained
+  static:
+    # get value from `mcTable` and add it to `mcSeq`
+    mcSeq.add(mcTable["val"])
+    assert mcSeq.len == 1
+
+  static:
+    assert mcSeq[0].strVal == "hello ic"
+
+    # increase `mcCounter` by 3
+    mcCounter.inc(3)
+    assert mcCounter.value == 3
+
 
 type
   CacheSeq* = distinct string
+    ## Compile-time sequence of `NimNode`s.
   CacheTable* = distinct string
+    ## Compile-time table of key-value pairs.
+    ##
+    ## Keys are `string`s and values are `NimNode`s.
   CacheCounter* = distinct string
+    ## Compile-time counter, uses `int` for storing the count.
 
-proc value*(c: CacheCounter): int {.magic: "NccValue".}
-proc inc*(c: CacheCounter; by = 1) {.magic: "NccInc".}
+proc value*(c: CacheCounter): int {.magic: "NccValue".} =
+  ## Returns the value of a counter `c`.
+  runnableExamples:
+    static:
+      let counter = CacheCounter"valTest"
+      # default value is 0
+      assert counter.value == 0
 
-proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".}
-proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".}
-proc len*(s: CacheSeq): int {.magic: "NcsLen".}
-proc `[]`*(s: CacheSeq; i: int): NimNode {.magic: "NcsAt".}
+      inc counter
+      assert counter.value == 1
+
+proc inc*(c: CacheCounter; by = 1) {.magic: "NccInc".} =
+  ## Increments the counter `c` with the value `by`.
+  runnableExamples:
+    static:
+      let counter = CacheCounter"incTest"
+      inc counter
+      inc counter, 5
+
+      assert counter.value == 6
+
+proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".} =
+  ## Adds `value` to `s`.
+  runnableExamples:
+    import std/macros
+    const mySeq = CacheSeq"addTest"
+
+    static:
+      mySeq.add(newLit(5))
+      mySeq.add(newLit("hello ic"))
+
+      assert mySeq.len == 2
+      assert mySeq[1].strVal == "hello ic"
+
+proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} =
+  ## Adds `value` to `s`.
+  ##
+  ## .. hint:: This doesn't do anything if `value` is already in `s`.
+  runnableExamples:
+    import std/macros
+    const mySeq = CacheSeq"inclTest"
+
+    static:
+      mySeq.incl(newLit(5))
+      mySeq.incl(newLit(5))
+
+      # still one element
+      assert mySeq.len == 1
+
+proc len*(s: CacheSeq): int {.magic: "NcsLen".} =
+  ## Returns the length of `s`.
+  runnableExamples:
+    import std/macros
+
+    const mySeq = CacheSeq"lenTest"
+    static:
+      let val = newLit("helper")
+      mySeq.add(val)
+      assert mySeq.len == 1
+
+      mySeq.add(val)
+      assert mySeq.len == 2
+
+proc `[]`*(s: CacheSeq; i: int): NimNode {.magic: "NcsAt".} =
+  ## Returns the `i`th value from `s`.
+  runnableExamples:
+    import std/macros
+
+    const mySeq = CacheSeq"subTest"
+    static:
+      mySeq.add(newLit(42))
+      assert mySeq[0].intVal == 42
+
+proc `[]`*(s: CacheSeq; i: BackwardsIndex): NimNode =
+  ## Returns the `i`th last value from `s`.
+  runnableExamples:
+    import std/macros
+
+    const mySeq = CacheSeq"backTest"
+    static:
+      mySeq &= newLit(42)
+      mySeq &= newLit(7)
+      assert mySeq[^1].intVal == 7  # Last item
+      assert mySeq[^2].intVal == 42 # Second last item
+  s[s.len - int(i)]
 
 iterator items*(s: CacheSeq): NimNode =
+  ## Iterates over each item in `s`.
+  runnableExamples:
+    import std/macros
+    const myseq = CacheSeq"itemsTest"
+
+    static:
+      myseq.add(newLit(5))
+      myseq.add(newLit(42))
+
+      for val in myseq:
+        # check that all values in `myseq` are int literals
+        assert val.kind == nnkIntLit
+
   for i in 0 ..< len(s): yield s[i]
 
-proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".}
-  ## 'key' has to be unique!
+proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} =
+  ## Inserts a `(key, value)` pair into `t`.
+  ##
+  ## .. warning:: `key` has to be unique! Assigning `value` to a `key` that is already
+  ##   in the table will result in a compiler error.
+  runnableExamples:
+    import std/macros
 
-proc len*(t: CacheTable): int {.magic: "NctLen".}
-proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".}
+    const mcTable = CacheTable"subTest"
+    static:
+      # assign newLit(5) to the key "value"
+      mcTable["value"] = newLit(5)
+
+      # check that we can get the value back
+      assert mcTable["value"].kind == nnkIntLit
+
+proc len*(t: CacheTable): int {.magic: "NctLen".} =
+  ## Returns the number of elements in `t`.
+  runnableExamples:
+    import std/macros
+
+    const dataTable = CacheTable"lenTest"
+    static:
+      dataTable["key"] = newLit(5)
+      assert dataTable.len == 1
+
+proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} =
+  ## Retrieves the `NimNode` value at `t[key]`.
+  runnableExamples:
+    import std/macros
+
+    const mcTable = CacheTable"subTest"
+    static:
+      mcTable["toAdd"] = newStmtList()
+
+      # get the NimNode back
+      assert mcTable["toAdd"].kind == nnkStmtList
+
+proc hasKey*(t: CacheTable; key: string): bool =
+  ## Returns true if `key` is in the table `t`.
+  ##
+  ## See also:
+  ## * [contains proc][contains(CacheTable, string)] for use with the `in` operator
+  runnableExamples:
+    import std/macros
+    const mcTable = CacheTable"hasKeyEx"
+    static:
+      assert not mcTable.hasKey("foo")
+      mcTable["foo"] = newEmptyNode()
+      # Will now be true since we inserted a value
+      assert mcTable.hasKey("foo")
+  discard "Implemented in vmops"
+
+proc contains*(t: CacheTable; key: string): bool {.inline.} =
+  ## Alias of [hasKey][hasKey(CacheTable, string)] for use with the `in` operator.
+  runnableExamples:
+    import std/macros
+    const mcTable = CacheTable"containsEx"
+    static:
+      mcTable["foo"] = newEmptyNode()
+      # Will be true since we gave it a value before
+      assert "foo" in mcTable
+  t.hasKey(key)
 
 proc hasNext(t: CacheTable; iter: int): bool {.magic: "NctHasNext".}
 proc next(t: CacheTable; iter: int): (string, NimNode, int) {.magic: "NctNext".}
 
 iterator pairs*(t: CacheTable): (string, NimNode) =
+  ## Iterates over all `(key, value)` pairs in `t`.
+  runnableExamples:
+    import std/macros
+    const mytabl = CacheTable"values"
+
+    static:
+      mytabl["intVal"] = newLit(5)
+      mytabl["otherVal"] = newLit(6)
+      for key, val in mytabl:
+        # make sure that we actually get the same keys
+        assert key in ["intVal", "otherVal"]
+
+        # all vals are int literals
+        assert val.kind == nnkIntLit
+
   var h = 0
   while hasNext(t, h):
     let (a, b, h2) = next(t, h)
diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 491235d8b8..bd1de9cd7f 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -10,6 +10,10 @@
 include "system/inclrtl"
 import std/private/since
 
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, formatfloat]
+
+
 ## This module contains the interface to the compiler's abstract syntax
 ## tree (`AST`:idx:). Macros operate on this tree.
 ##
@@ -19,6 +23,8 @@ import std/private/since
 
 ## .. include:: ../../doc/astspec.txt
 
+## .. importdoc:: system.nim
+
 # If you look for the implementation of the magic symbol
 # ``{.magic: "Foo".}``, search for `mFoo` and `opcFoo`.
 
@@ -71,21 +77,22 @@ type
     nnkTupleTy, nnkTupleClassTy, nnkTypeClassTy, nnkStaticTy,
     nnkRecList, nnkRecCase, nnkRecWhen,
     nnkRefTy, nnkPtrTy, nnkVarTy,
-    nnkConstTy, nnkMutableTy,
+    nnkConstTy, nnkOutTy,
     nnkDistinctTy,
     nnkProcTy,
     nnkIteratorTy,         # iterator type
-    nnkSharedTy,           # 'shared T'
+    nnkSinkAsgn,
     nnkEnumTy,
     nnkEnumFieldDef,
-    nnkArglist, nnkPattern
+    nnkArgList, nnkPattern
     nnkHiddenTryStmt,
     nnkClosure,
     nnkGotoState,
     nnkState,
     nnkBreakState,
     nnkFuncDef,
-    nnkTupleConstr
+    nnkTupleConstr,
+    nnkError,  ## erroneous AST node
 
   NimNodeKinds* = set[NimNodeKind]
   NimTypeKind* = enum  # some types are no longer used, see ast.nim
@@ -120,6 +127,10 @@ type
 
   TNimSymKinds* {.deprecated.} = set[NimSymKind]
 
+const
+  nnkMutableTy* {.deprecated.} = nnkOutTy
+  nnkSharedTy* {.deprecated.} = nnkSinkAsgn
+
 type
   NimIdent* {.deprecated.} = object of RootObj
     ## Represents a Nim identifier in the AST. **Note**: This is only
@@ -134,8 +145,9 @@ type
 
 const
   nnkLiterals* = {nnkCharLit..nnkNilLit}
+  # see matching set CallNodes below
   nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand,
-                   nnkCallStrLit}
+                   nnkCallStrLit, nnkHiddenCallConv}
   nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit}
 
 {.push warnings: off.}
@@ -177,7 +189,7 @@ template `^^`(n: NimNode, i: untyped): untyped =
 
 proc `[]`*[T, U: Ordinal](n: NimNode, x: HSlice[T, U]): seq[NimNode] =
   ## Slice operation for NimNode.
-  ## Returns a seq of child of `n` who inclusive range [n[x.a], n[x.b]].
+  ## Returns a seq of child of `n` who inclusive range `[n[x.a], n[x.b]]`.
   let xa = n ^^ x.a
   let L = (n ^^ x.b) - xa + 1
   result = newSeq[NimNode](L)
@@ -196,10 +208,9 @@ template `or`*(x, y: NimNode): NimNode =
   ## Evaluate `x` and when it is not an empty node, return
   ## it. Otherwise evaluate to `y`. Can be used to chain several
   ## expressions to get the first expression that is not empty.
-  ##
-  ## .. code-block:: nim
-  ##
+  ##   ```nim
   ##   let node = mightBeEmpty() or mightAlsoBeEmpty() or fallbackNode
+  ##   ```
 
   let arg = x
   if arg != nil and arg.kind != nnkEmpty:
@@ -208,12 +219,12 @@ template `or`*(x, y: NimNode): NimNode =
     y
 
 proc add*(father, child: NimNode): NimNode {.magic: "NAdd", discardable,
-  noSideEffect, locks: 0.}
+  noSideEffect.}
   ## Adds the `child` to the `father` node. Returns the
   ## father node so that calls can be nested.
 
 proc add*(father: NimNode, children: varargs[NimNode]): NimNode {.
-  magic: "NAddMultiple", discardable, noSideEffect, locks: 0.}
+  magic: "NAddMultiple", discardable, noSideEffect.}
   ## Adds each child of `children` to the `father` node.
   ## Returns the `father` node so that calls can be nested.
 
@@ -336,8 +347,7 @@ proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} =
       newLit(x.getTypeImpl.repr)
     let t = """
 object
-  arr: array[0 .. 3, float32]
-"""
+  arr: array[0 .. 3, float32]"""
     doAssert(dumpTypeImpl(a) == t)
     doAssert(dumpTypeImpl(b) == t)
     doAssert(dumpTypeImpl(c) == t)
@@ -390,10 +400,38 @@ proc newNimNode*(kind: NimNodeKind,
   ## produced code crashes. You should ensure that it is set to a node that
   ## you are transforming.
 
-proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.}
-proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.}
+proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.} =
+  ## Creates a new AST node by copying the node `n`. Note that unlike `copyNimTree`,
+  ## child nodes of `n` are not copied.
+  runnableExamples:
+    macro foo(x: typed) =
+      var s = copyNimNode(x)
+      doAssert s.len == 0
+      doAssert s.kind == nnkStmtList
 
-proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign.}
+    foo:
+      let x = 12
+      echo x
+
+proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.} =
+  ## Creates a new AST node by recursively copying the node `n`. Note that
+  ## unlike `copyNimNode`, this copies `n`, the children of `n`, etc.
+  runnableExamples:
+    macro foo(x: typed) =
+      var s = copyNimTree(x)
+      doAssert s.len == 2
+      doAssert s.kind == nnkStmtList
+
+    foo:
+      let x = 12
+      echo x
+
+when defined(nimHasNoReturnError):
+  {.pragma: errorNoReturn, noreturn.}
+else:
+  {.pragma: errorNoReturn.}
+
+proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign, errorNoReturn.}
   ## Writes an error message at compile time. The optional `n: NimNode`
   ## parameter is used as the source for file and line number information in
   ## the compilation error message.
@@ -404,29 +442,29 @@ proc warning*(msg: string, n: NimNode = nil) {.magic: "NWarning", benign.}
 proc hint*(msg: string, n: NimNode = nil) {.magic: "NHint", benign.}
   ## Writes a hint message at compile time.
 
-proc newStrLitNode*(s: string): NimNode {.compileTime, noSideEffect.} =
+proc newStrLitNode*(s: string): NimNode {.noSideEffect.} =
   ## Creates a string literal node from `s`.
   result = newNimNode(nnkStrLit)
   result.strVal = s
 
-proc newCommentStmtNode*(s: string): NimNode {.compileTime, noSideEffect.} =
+proc newCommentStmtNode*(s: string): NimNode {.noSideEffect.} =
   ## Creates a comment statement node.
   result = newNimNode(nnkCommentStmt)
   result.strVal = s
 
-proc newIntLitNode*(i: BiggestInt): NimNode {.compileTime.} =
+proc newIntLitNode*(i: BiggestInt): NimNode =
   ## Creates an int literal node from `i`.
   result = newNimNode(nnkIntLit)
   result.intVal = i
 
-proc newFloatLitNode*(f: BiggestFloat): NimNode {.compileTime.} =
+proc newFloatLitNode*(f: BiggestFloat): NimNode =
   ## Creates a float literal node from `f`.
   result = newNimNode(nnkFloatLit)
   result.floatVal = f
 
 {.push warnings: off.}
 
-proc newIdentNode*(i: NimIdent): NimNode {.compileTime, deprecated.} =
+proc newIdentNode*(i: NimIdent): NimNode {.deprecated: "use ident(string)".} =
   ## Creates an identifier node from `i`.
   result = newNimNode(nnkIdent)
   result.ident = i
@@ -441,9 +479,11 @@ proc ident*(name: string): NimNode {.magic: "StrToIdent", noSideEffect.}
   ## Create a new ident node from a string.
 
 type
-  BindSymRule* = enum    ## specifies how `bindSym` behaves
+  BindSymRule* = enum    ## Specifies how `bindSym` behaves. The difference
+                         ## between open and closed symbols can be found in
+                         ## ``_
     brClosed,            ## only the symbols in current scope are bound
-    brOpen,              ## open wrt overloaded symbols, but may be a single
+    brOpen,              ## open for overloaded symbols, but may be a single
                          ## symbol if not ambiguous (the rules match that of
                          ## binding in generics)
     brForceOpen          ## same as brOpen, but it will always be open even
@@ -462,12 +502,7 @@ proc bindSym*(ident: string | NimNode, rule: BindSymRule = brClosed): NimNode {.
   ## If `rule == brForceOpen` always an `nnkOpenSymChoice` tree is
   ## returned even if the symbol is not ambiguous.
   ##
-  ## Experimental feature:
-  ## use {.experimental: "dynamicBindSym".} to activate it.
-  ## If called from template / regular code, `ident` and `rule` must be
-  ## constant expression / literal value.
-  ## If called from macros / compile time procs / static blocks,
-  ## `ident` and `rule` can be VM computed value.
+  ## See the `manual `_ for more details.
 
 proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {.
   magic: "NGenSym", noSideEffect.}
@@ -475,10 +510,11 @@ proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {.
   ## needs to occur in a declaration context.
 
 proc callsite*(): NimNode {.magic: "NCallSite", benign, deprecated:
-  "Deprecated since v0.18.1; use varargs[untyped] in the macro prototype instead".}
+  "Deprecated since v0.18.1; use `varargs[untyped]` in the macro prototype instead".}
   ## Returns the AST of the invocation expression that invoked this macro.
+  # see https://github.com/nim-lang/RFCs/issues/387 as candidate replacement.
 
-proc toStrLit*(n: NimNode): NimNode {.compileTime.} =
+proc toStrLit*(n: NimNode): NimNode =
   ## Converts the AST `n` to the concrete Nim code and wraps that
   ## in a string literal node.
   return newStrLitNode(repr(n))
@@ -504,46 +540,64 @@ proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.}
 proc copyLineInfo*(arg: NimNode, info: NimNode) {.magic: "NLineInfo", noSideEffect.}
   ## Copy lineinfo from `info`.
 
-proc lineInfoObj*(n: NimNode): LineInfo {.compileTime.} =
+proc setLine(arg: NimNode, line: uint16) {.magic: "NLineInfo", noSideEffect.}
+proc setColumn(arg: NimNode, column: int16) {.magic: "NLineInfo", noSideEffect.}
+proc setFile(arg: NimNode, file: string) {.magic: "NLineInfo", noSideEffect.}
+
+proc setLineInfo*(arg: NimNode, file: string, line: int, column: int) =
+  ## Sets the line info on the NimNode. The file needs to exists, but can be a
+  ## relative path. If you want to attach line info to a block using `quote`
+  ## you'll need to add the line information after the quote block.
+  arg.setFile(file)
+  arg.setLine(line.uint16)
+  arg.setColumn(column.int16)
+
+proc setLineInfo*(arg: NimNode, lineInfo: LineInfo) =
+  ## See `setLineInfo proc<#setLineInfo,NimNode,string,int,int>`_
+  setLineInfo(arg, lineInfo.filename, lineInfo.line, lineInfo.column)
+
+proc lineInfoObj*(n: NimNode): LineInfo =
   ## Returns `LineInfo` of `n`, using absolute path for `filename`.
   result = LineInfo(filename: n.getFile, line: n.getLine, column: n.getColumn)
 
-proc lineInfo*(arg: NimNode): string {.compileTime.} =
+proc lineInfo*(arg: NimNode): string =
   ## Return line info in the form `filepath(line, column)`.
   $arg.lineInfoObj
 
-proc internalParseExpr(s: string): NimNode {.
+proc internalParseExpr(s, filename: string): NimNode {.
   magic: "ParseExprToAst", noSideEffect.}
 
-proc internalParseStmt(s: string): NimNode {.
+proc internalParseStmt(s, filename: string): NimNode {.
   magic: "ParseStmtToAst", noSideEffect.}
 
 proc internalErrorFlag*(): string {.magic: "NError", noSideEffect.}
   ## Some builtins set an error flag. This is then turned into a proper
   ## exception. **Note**: Ordinary application code should not call this.
 
-proc parseExpr*(s: string): NimNode {.noSideEffect, compileTime.} =
+proc parseExpr*(s: string; filename: string = ""): NimNode {.noSideEffect.} =
   ## Compiles the passed string to its AST representation.
   ## Expects a single expression. Raises `ValueError` for parsing errors.
-  result = internalParseExpr(s)
+  ## A filename can be given for more informative errors.
+  result = internalParseExpr(s, filename)
   let x = internalErrorFlag()
   if x.len > 0: raise newException(ValueError, x)
 
-proc parseStmt*(s: string): NimNode {.noSideEffect, compileTime.} =
+proc parseStmt*(s: string; filename: string = ""): NimNode {.noSideEffect.} =
   ## Compiles the passed string to its AST representation.
   ## Expects one or more statements. Raises `ValueError` for parsing errors.
-  result = internalParseStmt(s)
+  ## A filename can be given for more informative errors.
+  result = internalParseStmt(s, filename)
   let x = internalErrorFlag()
   if x.len > 0: raise newException(ValueError, x)
 
 proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.}
   ## Obtains the AST nodes returned from a macro or template invocation.
+  ## See also `genasts.genAst`.
   ## Example:
-  ##
-  ## .. code-block:: nim
-  ##
+  ##   ```nim
   ##   macro FooMacro() =
   ##     var ast = getAst(BarTemplate())
+  ##   ```
 
 proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
   ## Quasi-quoting operator.
@@ -558,6 +612,8 @@ proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
   ##
   ## A custom operator interpolation needs accent quoted (``) whenever it resolves
   ## to a symbol.
+  ##
+  ## See also `genasts `_ which avoids some issues with `quote`.
   runnableExamples:
     macro check(ex: untyped) =
       # this is a simplified version of the check macro from the
@@ -626,25 +682,25 @@ proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
         doAssert y == 3
     bar2()
 
-proc expectKind*(n: NimNode, k: NimNodeKind) {.compileTime.} =
+proc expectKind*(n: NimNode, k: NimNodeKind) =
   ## Checks that `n` is of kind `k`. If this is not the case,
   ## compilation aborts with an error message. This is useful for writing
   ## macros that check the AST that is passed to them.
   if n.kind != k: error("Expected a node of kind " & $k & ", got " & $n.kind, n)
 
-proc expectMinLen*(n: NimNode, min: int) {.compileTime.} =
+proc expectMinLen*(n: NimNode, min: int) =
   ## Checks that `n` has at least `min` children. If this is not the case,
   ## compilation aborts with an error message. This is useful for writing
   ## macros that check its number of arguments.
   if n.len < min: error("Expected a node with at least " & $min & " children, got " & $n.len, n)
 
-proc expectLen*(n: NimNode, len: int) {.compileTime.} =
+proc expectLen*(n: NimNode, len: int) =
   ## Checks that `n` has exactly `len` children. If this is not the case,
   ## compilation aborts with an error message. This is useful for writing
   ## macros that check its number of arguments.
   if n.len != len: error("Expected a node with " & $len & " children, got " & $n.len, n)
 
-proc expectLen*(n: NimNode, min, max: int) {.compileTime.} =
+proc expectLen*(n: NimNode, min, max: int) =
   ## Checks that `n` has a number of children in the range `min..max`.
   ## If this is not the case, compilation aborts with an error message.
   ## This is useful for writing macros that check its number of arguments.
@@ -652,13 +708,12 @@ proc expectLen*(n: NimNode, min, max: int) {.compileTime.} =
     error("Expected a node with " & $min & ".." & $max & " children, got " & $n.len, n)
 
 proc newTree*(kind: NimNodeKind,
-              children: varargs[NimNode]): NimNode {.compileTime.} =
+              children: varargs[NimNode]): NimNode =
   ## Produces a new node with children.
   result = newNimNode(kind)
   result.add(children)
 
-proc newCall*(theProc: NimNode,
-              args: varargs[NimNode]): NimNode {.compileTime.} =
+proc newCall*(theProc: NimNode, args: varargs[NimNode]): NimNode =
   ## Produces a new call node. `theProc` is the proc that is called with
   ## the arguments `args[0..]`.
   result = newNimNode(nnkCall)
@@ -667,7 +722,7 @@ proc newCall*(theProc: NimNode,
 
 {.push warnings: off.}
 
-proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.compileTime, deprecated:
+proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.deprecated:
   "Deprecated since v0.18.1; use 'newCall(string, ...)' or 'newCall(NimNode, ...)' instead".} =
   ## Produces a new call node. `theProc` is the proc that is called with
   ## the arguments `args[0..]`.
@@ -678,128 +733,128 @@ proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.compileTime,
 {.pop.}
 
 proc newCall*(theProc: string,
-              args: varargs[NimNode]): NimNode {.compileTime.} =
+              args: varargs[NimNode]): NimNode =
   ## Produces a new call node. `theProc` is the proc that is called with
   ## the arguments `args[0..]`.
   result = newNimNode(nnkCall)
   result.add(newIdentNode(theProc))
   result.add(args)
 
-proc newLit*(c: char): NimNode {.compileTime.} =
+proc newLit*(c: char): NimNode =
   ## Produces a new character literal node.
   result = newNimNode(nnkCharLit)
   result.intVal = ord(c)
 
-proc newLit*(i: int): NimNode {.compileTime.} =
+proc newLit*(i: int): NimNode =
   ## Produces a new integer literal node.
   result = newNimNode(nnkIntLit)
   result.intVal = i
 
-proc newLit*(i: int8): NimNode {.compileTime.} =
+proc newLit*(i: int8): NimNode =
   ## Produces a new integer literal node.
   result = newNimNode(nnkInt8Lit)
   result.intVal = i
 
-proc newLit*(i: int16): NimNode {.compileTime.} =
+proc newLit*(i: int16): NimNode =
   ## Produces a new integer literal node.
   result = newNimNode(nnkInt16Lit)
   result.intVal = i
 
-proc newLit*(i: int32): NimNode {.compileTime.} =
+proc newLit*(i: int32): NimNode =
   ## Produces a new integer literal node.
   result = newNimNode(nnkInt32Lit)
   result.intVal = i
 
-proc newLit*(i: int64): NimNode {.compileTime.} =
+proc newLit*(i: int64): NimNode =
   ## Produces a new integer literal node.
   result = newNimNode(nnkInt64Lit)
   result.intVal = i
 
-proc newLit*(i: uint): NimNode {.compileTime.} =
+proc newLit*(i: uint): NimNode =
   ## Produces a new unsigned integer literal node.
   result = newNimNode(nnkUIntLit)
   result.intVal = BiggestInt(i)
 
-proc newLit*(i: uint8): NimNode {.compileTime.} =
+proc newLit*(i: uint8): NimNode =
   ## Produces a new unsigned integer literal node.
   result = newNimNode(nnkUInt8Lit)
   result.intVal = BiggestInt(i)
 
-proc newLit*(i: uint16): NimNode {.compileTime.} =
+proc newLit*(i: uint16): NimNode =
   ## Produces a new unsigned integer literal node.
   result = newNimNode(nnkUInt16Lit)
   result.intVal = BiggestInt(i)
 
-proc newLit*(i: uint32): NimNode {.compileTime.} =
+proc newLit*(i: uint32): NimNode =
   ## Produces a new unsigned integer literal node.
   result = newNimNode(nnkUInt32Lit)
   result.intVal = BiggestInt(i)
 
-proc newLit*(i: uint64): NimNode {.compileTime.} =
+proc newLit*(i: uint64): NimNode =
   ## Produces a new unsigned integer literal node.
   result = newNimNode(nnkUInt64Lit)
   result.intVal = BiggestInt(i)
 
-proc newLit*(b: bool): NimNode {.compileTime.} =
+proc newLit*(b: bool): NimNode =
   ## Produces a new boolean literal node.
   result = if b: bindSym"true" else: bindSym"false"
 
-proc newLit*(s: string): NimNode {.compileTime.} =
+proc newLit*(s: string): NimNode =
   ## Produces a new string literal node.
   result = newNimNode(nnkStrLit)
   result.strVal = s
 
 when false:
   # the float type is not really a distinct type as described in https://github.com/nim-lang/Nim/issues/5875
-  proc newLit*(f: float): NimNode {.compileTime.} =
+  proc newLit*(f: float): NimNode =
     ## Produces a new float literal node.
     result = newNimNode(nnkFloatLit)
     result.floatVal = f
 
-proc newLit*(f: float32): NimNode {.compileTime.} =
+proc newLit*(f: float32): NimNode =
   ## Produces a new float literal node.
   result = newNimNode(nnkFloat32Lit)
   result.floatVal = f
 
-proc newLit*(f: float64): NimNode {.compileTime.} =
+proc newLit*(f: float64): NimNode =
   ## Produces a new float literal node.
   result = newNimNode(nnkFloat64Lit)
   result.floatVal = f
 
 when declared(float128):
-  proc newLit*(f: float128): NimNode {.compileTime.} =
+  proc newLit*(f: float128): NimNode =
     ## Produces a new float literal node.
     result = newNimNode(nnkFloat128Lit)
     result.floatVal = f
 
-proc newLit*(arg: enum): NimNode {.compileTime.} =
+proc newLit*(arg: enum): NimNode =
   result = newCall(
     arg.typeof.getTypeInst[1],
     newLit(int(arg))
   )
 
-proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.}
-proc newLit*[T](arg: seq[T]): NimNode {.compileTime.}
-proc newLit*[T](s: set[T]): NimNode {.compileTime.}
-proc newLit*[T: tuple](arg: T): NimNode {.compileTime.}
+proc newLit*[N,T](arg: array[N,T]): NimNode
+proc newLit*[T](arg: seq[T]): NimNode
+proc newLit*[T](s: set[T]): NimNode
+proc newLit*[T: tuple](arg: T): NimNode
 
-proc newLit*(arg: object): NimNode {.compileTime.} =
+proc newLit*(arg: object): NimNode =
   result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1])
   for a, b in arg.fieldPairs:
     result.add nnkExprColonExpr.newTree( newIdentNode(a), newLit(b) )
 
-proc newLit*(arg: ref object): NimNode {.compileTime.} =
+proc newLit*(arg: ref object): NimNode =
   ## produces a new ref type literal node.
   result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1])
   for a, b in fieldPairs(arg[]):
     result.add nnkExprColonExpr.newTree(newIdentNode(a), newLit(b))
 
-proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} =
+proc newLit*[N,T](arg: array[N,T]): NimNode =
   result = nnkBracket.newTree
   for x in arg:
     result.add newLit(x)
 
-proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} =
+proc newLit*[T](arg: seq[T]): NimNode =
   let bracket = nnkBracket.newTree
   for x in arg:
     bracket.add newLit(x)
@@ -812,7 +867,7 @@ proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} =
     var typ = getTypeInst(typeof(arg))[1]
     result = newCall(typ,result)
 
-proc newLit*[T](s: set[T]): NimNode {.compileTime.} =
+proc newLit*[T](s: set[T]): NimNode =
   result = nnkCurly.newTree
   for x in s:
     result.add newLit(x)
@@ -824,7 +879,7 @@ proc newLit*[T](s: set[T]): NimNode {.compileTime.} =
 proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}
   ## See `typetraits.isNamedTuple`
 
-proc newLit*[T: tuple](arg: T): NimNode {.compileTime.} =
+proc newLit*[T: tuple](arg: T): NimNode =
   ## use -d:nimHasWorkaround14720 to restore behavior prior to PR, forcing
   ## a named tuple even when `arg` is unnamed.
   result = nnkTupleConstr.newTree
@@ -835,7 +890,7 @@ proc newLit*[T: tuple](arg: T): NimNode {.compileTime.} =
     for b in arg.fields:
       result.add newLit(b)
 
-proc nestList*(op: NimNode; pack: NimNode): NimNode {.compileTime.} =
+proc nestList*(op: NimNode; pack: NimNode): NimNode =
   ## Nests the list `pack` into a tree of call expressions:
   ## `[a, b, c]` is transformed into `op(a, op(c, d))`.
   ## This is also known as fold expression.
@@ -845,7 +900,7 @@ proc nestList*(op: NimNode; pack: NimNode): NimNode {.compileTime.} =
   for i in countdown(pack.len - 2, 0):
     result = newCall(op, pack[i], result)
 
-proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode {.compileTime.} =
+proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode =
   ## Nests the list `pack` into a tree of call expressions:
   ## `[a, b, c]` is transformed into `op(a, op(c, d))`.
   ## This is also known as fold expression.
@@ -853,6 +908,29 @@ proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode {.compileTime
   for i in countdown(pack.len - 1, 0):
     result = newCall(op, pack[i], result)
 
+proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.}
+  ## Style insensitive comparison.
+
+proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.}
+  ## Style insensitive comparison.  `a` can be an identifier or a
+  ## symbol. `a` may be wrapped in an export marker
+  ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`),
+  ## these nodes will be unwrapped.
+
+proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.}
+  ## Style insensitive comparison.  `b` can be an identifier or a
+  ## symbol. `b` may be wrapped in an export marker
+  ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`),
+  ## these nodes will be unwrapped.
+
+proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.}
+  ## Style insensitive comparison.  `a` and `b` can be an
+  ## identifier or a symbol. Both may be wrapped in an export marker
+  ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`),
+  ## these nodes will be unwrapped.
+
+const collapseSymChoice = not defined(nimLegacyMacrosCollapseSymChoice)
+
 proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indented = false) {.benign.} =
   if level > 0:
     if indented:
@@ -874,37 +952,52 @@ proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indent
     discard # same as nil node in this representation
   of nnkCharLit .. nnkInt64Lit:
     res.add(" " & $n.intVal)
+  of nnkUIntLit .. nnkUInt64Lit:
+    res.add(" " & $cast[uint64](n.intVal))
   of nnkFloatLit .. nnkFloat64Lit:
     res.add(" " & $n.floatVal)
   of nnkStrLit .. nnkTripleStrLit, nnkCommentStmt, nnkIdent, nnkSym:
     res.add(" " & $n.strVal.newLit.repr)
   of nnkNone:
     assert false
+  elif n.kind in {nnkOpenSymChoice, nnkClosedSymChoice} and collapseSymChoice:
+    res.add(" " & $n.len)
+    if n.len > 0:
+      var allSameSymName = true
+      for i in 0.. 0:
+        res.add(" " & n[0].strVal.newLit.repr)
     else:
       res.add(".newTree(")
       for j in 0.. `a.b`
   return newNimNode(nnkDotExpr).add(a, b)
 
-proc newColonExpr*(a, b: NimNode): NimNode {.compileTime.} =
+proc newColonExpr*(a, b: NimNode): NimNode =
   ## Create new colon expression.
   ## newColonExpr(a, b) -> `a: b`
   newNimNode(nnkExprColonExpr).add(a, b)
 
 proc newIdentDefs*(name, kind: NimNode;
-                   default = newEmptyNode()): NimNode {.compileTime.} =
+                   default = newEmptyNode()): NimNode =
   ## Creates a new `nnkIdentDefs` node of a specific kind and value.
   ##
   ## `nnkIdentDefs` need to have at least three children, but they can have
@@ -1081,30 +1183,29 @@ proc newIdentDefs*(name, kind: NimNode;
   ## `let` or `var` blocks may have an empty `kind` node if the
   ## identifier is being assigned a value. Example:
   ##
-  ## .. code-block:: nim
-  ##
+  ##   ```nim
   ##   var varSection = newNimNode(nnkVarSection).add(
   ##     newIdentDefs(ident("a"), ident("string")),
   ##     newIdentDefs(ident("b"), newEmptyNode(), newLit(3)))
   ##   # --> var
   ##   #       a: string
   ##   #       b = 3
+  ##   ```
   ##
   ## If you need to create multiple identifiers you need to use the lower level
   ## `newNimNode`:
-  ##
-  ## .. code-block:: nim
-  ##
+  ##   ```nim
   ##   result = newNimNode(nnkIdentDefs).add(
   ##     ident("a"), ident("b"), ident("c"), ident("string"),
   ##       newStrLitNode("Hello"))
+  ##   ```
   newNimNode(nnkIdentDefs).add(name, kind, default)
 
-proc newNilLit*(): NimNode {.compileTime.} =
+proc newNilLit*(): NimNode =
   ## New nil literal shortcut.
   result = newNimNode(nnkNilLit)
 
-proc last*(node: NimNode): NimNode {.compileTime.} = node[node.len-1]
+proc last*(node: NimNode): NimNode = node[node.len-1]
   ## Return the last item in nodes children. Same as `node[^1]`.
 
 
@@ -1112,10 +1213,10 @@ const
   RoutineNodes* = {nnkProcDef, nnkFuncDef, nnkMethodDef, nnkDo, nnkLambda,
                    nnkIteratorDef, nnkTemplateDef, nnkConverterDef, nnkMacroDef}
   AtomicNodes* = {nnkNone..nnkNilLit}
-  CallNodes* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand,
-    nnkCallStrLit, nnkHiddenCallConv}
+  # see matching set nnkCallKinds above
+  CallNodes* = nnkCallKinds
 
-proc expectKind*(n: NimNode; k: set[NimNodeKind]) {.compileTime.} =
+proc expectKind*(n: NimNode; k: set[NimNodeKind]) =
   ## Checks that `n` is of kind `k`. If this is not the case,
   ## compilation aborts with an error message. This is useful for writing
   ## macros that check the AST that is passed to them.
@@ -1125,7 +1226,7 @@ proc newProc*(name = newEmptyNode();
               params: openArray[NimNode] = [newEmptyNode()];
               body: NimNode = newStmtList();
               procType = nnkProcDef;
-              pragmas: NimNode = newEmptyNode()): NimNode {.compileTime.} =
+              pragmas: NimNode = newEmptyNode()): NimNode =
   ## Shortcut for creating a new proc.
   ##
   ## The `params` array must start with the return type of the proc,
@@ -1142,17 +1243,14 @@ proc newProc*(name = newEmptyNode();
     newEmptyNode(),
     body)
 
-proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]):
-                NimNode {.compileTime.} =
+proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): NimNode =
   ## Constructor for `if` statements.
-  ##
-  ## .. code-block:: nim
-  ##
-  ##    newIfStmt(
-  ##      (Ident, StmtList),
-  ##      ...
-  ##    )
-  ##
+  ##   ```nim
+  ##   newIfStmt(
+  ##     (Ident, StmtList),
+  ##     ...
+  ##   )
+  ##   ```
   result = newNimNode(nnkIfStmt)
   if len(branches) < 1:
     error("If statement must have at least one branch")
@@ -1160,20 +1258,18 @@ proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]):
     result.add(newTree(nnkElifBranch, i.cond, i.body))
 
 proc newEnum*(name: NimNode, fields: openArray[NimNode],
-              public, pure: bool): NimNode {.compileTime.} =
+              public, pure: bool): NimNode =
 
   ## Creates a new enum. `name` must be an ident. Fields are allowed to be
-  ## either idents or EnumFieldDef
-  ##
-  ## .. code-block:: nim
-  ##
-  ##    newEnum(
-  ##      name    = ident("Colors"),
-  ##      fields  = [ident("Blue"), ident("Red")],
-  ##      public  = true, pure = false)
-  ##
-  ##    # type Colors* = Blue Red
+  ## either idents or EnumFieldDef:
+  ##   ```nim
+  ##   newEnum(
+  ##     name    = ident("Colors"),
+  ##     fields  = [ident("Blue"), ident("Red")],
+  ##     public  = true, pure = false)
   ##
+  ##   # type Colors* = Blue Red
+  ##   ```
 
   expectKind name, nnkIdent
   if len(fields) < 1:
@@ -1203,7 +1299,7 @@ proc newEnum*(name: NimNode, fields: openArray[NimNode],
 
   return typeSect
 
-proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}=
+proc copyChildrenTo*(src, dest: NimNode) =
   ## Copy all children from `src` to `dest`.
   for i in 0 ..< src.len:
     dest.add src[i].copyNimTree
@@ -1211,7 +1307,7 @@ proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}=
 template expectRoutine(node: NimNode) =
   expectKind(node, RoutineNodes)
 
-proc name*(someProc: NimNode): NimNode {.compileTime.} =
+proc name*(someProc: NimNode): NimNode =
   someProc.expectRoutine
   result = someProc[0]
   if result.kind == nnkPostfix:
@@ -1222,21 +1318,28 @@ proc name*(someProc: NimNode): NimNode {.compileTime.} =
   elif result.kind == nnkAccQuoted:
     result = result[0]
 
-proc `name=`*(someProc: NimNode; val: NimNode) {.compileTime.} =
+proc `name=`*(someProc: NimNode; val: NimNode) =
   someProc.expectRoutine
   if someProc[0].kind == nnkPostfix:
     someProc[0][1] = val
   else: someProc[0] = val
 
-proc params*(someProc: NimNode): NimNode {.compileTime.} =
-  someProc.expectRoutine
-  result = someProc[3]
-proc `params=`* (someProc: NimNode; params: NimNode) {.compileTime.}=
-  someProc.expectRoutine
-  expectKind(params, nnkFormalParams)
-  someProc[3] = params
+proc params*(someProc: NimNode): NimNode =
+  if someProc.kind == nnkProcTy:
+    someProc[0]
+  else:
+    someProc.expectRoutine
+    someProc[3]
 
-proc pragma*(someProc: NimNode): NimNode {.compileTime.} =
+proc `params=`* (someProc: NimNode; params: NimNode) =
+  expectKind(params, nnkFormalParams)
+  if someProc.kind == nnkProcTy:
+    someProc[0] = params
+  else:
+    someProc.expectRoutine
+    someProc[3] = params
+
+proc pragma*(someProc: NimNode): NimNode =
   ## Get the pragma of a proc type.
   ## These will be expanded.
   if someProc.kind == nnkProcTy:
@@ -1244,7 +1347,7 @@ proc pragma*(someProc: NimNode): NimNode {.compileTime.} =
   else:
     someProc.expectRoutine
     result = someProc[4]
-proc `pragma=`*(someProc: NimNode; val: NimNode) {.compileTime.}=
+proc `pragma=`*(someProc: NimNode; val: NimNode) =
   ## Set the pragma of a proc type.
   expectKind(val, {nnkEmpty, nnkPragma})
   if someProc.kind == nnkProcTy:
@@ -1253,7 +1356,7 @@ proc `pragma=`*(someProc: NimNode; val: NimNode) {.compileTime.}=
     someProc.expectRoutine
     someProc[4] = val
 
-proc addPragma*(someProc, pragma: NimNode) {.compileTime.} =
+proc addPragma*(someProc, pragma: NimNode) =
   ## Adds pragma to routine definition.
   someProc.expectKind(RoutineNodes + {nnkProcTy})
   var pragmaNode = someProc.pragma
@@ -1265,7 +1368,7 @@ proc addPragma*(someProc, pragma: NimNode) {.compileTime.} =
 template badNodeKind(n, f) =
   error("Invalid node kind " & $n.kind & " for macros.`" & $f & "`", n)
 
-proc body*(someProc: NimNode): NimNode {.compileTime.} =
+proc body*(someProc: NimNode): NimNode =
   case someProc.kind:
   of RoutineNodes:
     return someProc[6]
@@ -1276,7 +1379,7 @@ proc body*(someProc: NimNode): NimNode {.compileTime.} =
   else:
     badNodeKind someProc, "body"
 
-proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} =
+proc `body=`*(someProc: NimNode, val: NimNode) =
   case someProc.kind
   of RoutineNodes:
     someProc[6] = val
@@ -1287,9 +1390,17 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} =
   else:
     badNodeKind someProc, "body="
 
-proc basename*(a: NimNode): NimNode {.compileTime, benign.}
+proc basename*(a: NimNode): NimNode =
+  ## Pull an identifier from prefix/postfix expressions.
+  case a.kind
+  of nnkIdent: result = a
+  of nnkPostfix, nnkPrefix: result = a[1]
+  of nnkPragmaExpr: result = basename(a[0])
+  else:
+    error("Do not know how to get basename of (" & treeRepr(a) & ")\n" &
+      repr(a), a)
 
-proc `$`*(node: NimNode): string {.compileTime.} =
+proc `$`*(node: NimNode): string =
   ## Get the string of an identifier node.
   case node.kind
   of nnkPostfix:
@@ -1299,7 +1410,9 @@ proc `$`*(node: NimNode): string {.compileTime.} =
   of nnkOpenSymChoice, nnkClosedSymChoice:
     result = $node[0]
   of nnkAccQuoted:
-    result = $node[0]
+    result = ""
+    for i in 0 ..< node.len:
+      result.add(repr(node[i]))
   else:
     badNodeKind node, "$"
 
@@ -1320,10 +1433,10 @@ iterator children*(n: NimNode): NimNode {.inline.} =
 
 template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} =
   ## Find the first child node matching condition (or nil).
-  ##
-  ## .. code-block:: nim
+  ##   ```nim
   ##   var res = findChild(n, it.kind == nnkPostfix and
-  ##                          it.basename.ident == toNimIdent"foo")
+  ##                          it.basename.ident == ident"foo")
+  ##   ```
   block:
     var res: NimNode
     for it in n.children:
@@ -1332,7 +1445,7 @@ template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} =
         break
     res
 
-proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} =
+proc insert*(a: NimNode; pos: int; b: NimNode) =
   ## Insert node `b` into node `a` at `pos`.
   if len(a)-1 < pos:
     # add some empty nodes first
@@ -1347,17 +1460,7 @@ proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} =
       a[i + 1] = a[i]
     a[pos] = b
 
-proc basename*(a: NimNode): NimNode =
-  ## Pull an identifier from prefix/postfix expressions.
-  case a.kind
-  of nnkIdent: result = a
-  of nnkPostfix, nnkPrefix: result = a[1]
-  of nnkPragmaExpr: result = basename(a[0])
-  else:
-    error("Do not know how to get basename of (" & treeRepr(a) & ")\n" &
-      repr(a), a)
-
-proc `basename=`*(a: NimNode; val: string) {.compileTime.}=
+proc `basename=`*(a: NimNode; val: string) =
   case a.kind
   of nnkIdent:
     a.strVal = val
@@ -1368,64 +1471,40 @@ proc `basename=`*(a: NimNode; val: string) {.compileTime.}=
     error("Do not know how to get basename of (" & treeRepr(a) & ")\n" &
       repr(a), a)
 
-proc postfix*(node: NimNode; op: string): NimNode {.compileTime.} =
+proc postfix*(node: NimNode; op: string): NimNode =
   newNimNode(nnkPostfix).add(ident(op), node)
 
-proc prefix*(node: NimNode; op: string): NimNode {.compileTime.} =
+proc prefix*(node: NimNode; op: string): NimNode =
   newNimNode(nnkPrefix).add(ident(op), node)
 
 proc infix*(a: NimNode; op: string;
-            b: NimNode): NimNode {.compileTime.} =
+            b: NimNode): NimNode =
   newNimNode(nnkInfix).add(ident(op), a, b)
 
-proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {.
-  compileTime.} =
+proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] =
   node.expectKind nnkPostfix
   result = (node[1], $node[0])
 
-proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {.
-  compileTime.} =
+proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] =
   node.expectKind nnkPrefix
   result = (node[1], $node[0])
 
-proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string;
-                                        right: NimNode] {.compileTime.} =
+proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] =
   expectKind(node, nnkInfix)
   result = (node[1], $node[0], node[2])
 
-proc copy*(node: NimNode): NimNode {.compileTime.} =
+proc copy*(node: NimNode): NimNode =
   ## An alias for `copyNimTree<#copyNimTree,NimNode>`_.
   return node.copyNimTree()
 
-proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.}
-  ## Style insensitive comparison.
-
-proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.}
-  ## Style insensitive comparison.  `a` can be an identifier or a
-  ## symbol. `a` may be wrapped in an export marker
-  ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`),
-  ## these nodes will be unwrapped.
-
-proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.}
-  ## Style insensitive comparison.  `b` can be an identifier or a
-  ## symbol. `b` may be wrapped in an export marker
-  ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`),
-  ## these nodes will be unwrapped.
-
-proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.}
-  ## Style insensitive comparison.  `a` and `b` can be an
-  ## identifier or a symbol. Both may be wrapped in an export marker
-  ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`),
-  ## these nodes will be unwrapped.
-
-proc expectIdent*(n: NimNode, name: string) {.compileTime, since: (1,1).} =
+proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} =
   ## Check that `eqIdent(n,name)` holds true. If this is not the
   ## case, compilation aborts with an error message. This is useful
   ## for writing macros that check the AST that is passed to them.
   if not eqIdent(n, name):
     error("Expected identifier to be `" & name & "` here", n)
 
-proc hasArgOfName*(params: NimNode; name: string): bool {.compileTime.}=
+proc hasArgOfName*(params: NimNode; name: string): bool =
   ## Search `nnkFormalParams` for an argument.
   expectKind(params, nnkFormalParams)
   for i in 1.. 1 and typ[1].kind == nnkProcTy:
     return typ[1][1]
   elif typ.typeKind == ntyTypeDesc:
-    let impl = typ[1].getImpl()
-    if impl[0].kind == nnkPragmaExpr:
+    let impl = getImpl(
+      if kind(typ[1]) == nnkBracketExpr: typ[1][0]
+      else: typ[1]
+    )
+    if impl.kind == nnkNilLit:
+      return impl
+    elif impl[0].kind == nnkPragmaExpr:
       return impl[0][1]
     else:
       return impl[0] # handle types which don't have macro at all
@@ -1495,10 +1595,10 @@ proc customPragmaNode(n: NimNode): NimNode =
     let impl = n.getImpl()
     if impl.kind in RoutineNodes:
       return impl.pragma
-    elif impl.kind == nnkIdentDefs and impl[0].kind == nnkPragmaExpr:
+    elif impl.kind in {nnkIdentDefs, nnkConstDef} and impl[0].kind == nnkPragmaExpr:
       return impl[0][1]
     else:
-      let timpl = typ.getImpl()
+      let timpl = getImpl(if typ.kind == nnkBracketExpr: typ[0] else: typ)
       if timpl.len>0 and timpl[0].len>1:
         return timpl[0][1]
       else:
@@ -1506,12 +1606,13 @@ proc customPragmaNode(n: NimNode): NimNode =
 
   if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}:
     let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1])
-    let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0])
-    var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst)
+    var typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0])
+    while typInst.kind in {nnkVarTy, nnkBracketExpr}: typInst = typInst[0]
+    var typDef = getImpl(typInst)
     while typDef != nil:
       typDef.expectKind(nnkTypeDef)
-      let typ = typDef[2]
-      typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy})
+      let typ = typDef[2].extractTypeImpl()
+      if typ.kind notin {nnkRefTy, nnkPtrTy, nnkObjectTy}: break
       let isRef = typ.kind in {nnkRefTy, nnkPtrTy}
       if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X)
         typDef = getImpl(typ[0])
@@ -1522,18 +1623,17 @@ proc customPragmaNode(n: NimNode): NimNode =
         for i in 0.. 0:
           var identDefs = identDefsStack.pop()
-          if identDefs.kind == nnkRecCase:
-            identDefsStack.add(identDefs[0])
-            for i in 1.. 0 and p[0].kind == nnkSym and p[0] == cp:
-      if p.len == 2:
+      if p.len == 2 or (p.len == 3 and p[1].kind == nnkSym and p[1].symKind == nskType):
         result = p[1]
       else:
         let def = p[0].getImpl[3]
@@ -1607,6 +1709,21 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped =
     error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error,
 
 macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped =
+  ## Calls `callee` with `args` unpacked as individual arguments.
+  ## This is useful in 2 cases:
+  ## * when forwarding `varargs[T]` for some typed `T`
+  ## * when forwarding `varargs[untyped]` when `args` can potentially be empty,
+  ##   due to a compiler limitation
+  runnableExamples:
+    template call1(fun: typed; args: varargs[untyped]): untyped =
+      unpackVarargs(fun, args)
+      # when varargsLen(args) > 0: fun(args) else: fun() # this would also work
+    template call2(fun: typed; args: varargs[typed]): untyped =
+      unpackVarargs(fun, args)
+    proc fn1(a = 0, b = 1) = discard (a, b)
+    call1(fn1, 10, 11)
+    call1(fn1) # `args` is empty in this case
+    if false: call2(echo, 10, 11) # would print 1011
   result = newCall(callee)
   for i in 0 ..< args.len:
     result.add args[i]
@@ -1660,33 +1777,34 @@ proc extractDocCommentsAndRunnables*(n: NimNode): NimNode =
   ## runnableExamples in `a`, stopping at the first child that is neither.
   ## Example:
   ##
-  ## .. code-block:: nim
-  ##  import std/macros
-  ##  macro transf(a): untyped =
-  ##    result = quote do:
-  ##      proc fun2*() = discard
-  ##    let header = extractDocCommentsAndRunnables(a.body)
-  ##    # correct usage: rest is appended
-  ##    result.body = header
-  ##    result.body.add quote do: discard # just an example
-  ##    # incorrect usage: nesting inside a nnkStmtList:
-  ##    # result.body = quote do: (`header`; discard)
+  ##   ```nim
+  ##   import std/macros
+  ##   macro transf(a): untyped =
+  ##     result = quote do:
+  ##       proc fun2*() = discard
+  ##     let header = extractDocCommentsAndRunnables(a.body)
+  ##     # correct usage: rest is appended
+  ##     result.body = header
+  ##     result.body.add quote do: discard # just an example
+  ##     # incorrect usage: nesting inside a nnkStmtList:
+  ##     # result.body = quote do: (`header`; discard)
   ##
-  ##  proc fun*() {.transf.} =
-  ##    ## first comment
-  ##    runnableExamples: discard
-  ##    runnableExamples: discard
-  ##    ## last comment
-  ##    discard # first statement after doc comments + runnableExamples
-  ##    ## not docgen'd
+  ##   proc fun*() {.transf.} =
+  ##     ## first comment
+  ##     runnableExamples: discard
+  ##     runnableExamples: discard
+  ##     ## last comment
+  ##     discard # first statement after doc comments + runnableExamples
+  ##     ## not docgen'd
+  ##   ```
 
   result = newStmtList()
   for ni in n:
     case ni.kind
     of nnkCommentStmt:
       result.add ni
-    of nnkCall:
-      if ni[0].kind == nnkIdent and ni[0].strVal == "runnableExamples":
+    of nnkCall, nnkCommand:
+      if ni[0].kind == nnkIdent and ni[0].eqIdent "runnableExamples":
         result.add ni
       else: break
     else: break
diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim
index 27a9c5e60e..8cb0cef055 100644
--- a/lib/core/rlocks.nim
+++ b/lib/core/rlocks.nim
@@ -11,10 +11,12 @@
 
 
 when not compileOption("threads") and not defined(nimdoc):
-  {.error: "Rlocks requires --threads:on option.".}
+  when false:
+    # make rlocks modlue consistent with locks module,
+    # so they can replace each other seamlessly.
+    {.error: "Rlocks requires --threads:on option.".}
 
-const insideRLocksModule = true
-include "system/syslocks"
+import std/private/syslocks
 
 type
   RLock* = SysLock ## Nim lock, re-entrant
@@ -29,27 +31,27 @@ proc initRLock*(lock: var RLock) {.inline.} =
   else:
     initSysLock(lock)
 
-proc deinitRLock*(lock: var RLock) {.inline.} =
+proc deinitRLock*(lock: RLock) {.inline.} =
   ## Frees the resources associated with the lock.
   deinitSys(lock)
 
-proc tryAcquire*(lock: var RLock): bool =
+proc tryAcquire*(lock: var RLock): bool {.inline.} =
   ## Tries to acquire the given lock. Returns `true` on success.
   result = tryAcquireSys(lock)
 
-proc acquire*(lock: var RLock) =
+proc acquire*(lock: var RLock) {.inline.} =
   ## Acquires the given lock.
   acquireSys(lock)
 
-proc release*(lock: var RLock) =
+proc release*(lock: var RLock) {.inline.} =
   ## Releases the given lock.
   releaseSys(lock)
 
-template withRLock*(lock: var RLock, code: untyped): untyped =
+template withRLock*(lock: RLock, code: untyped) =
   ## Acquires the given lock and then executes the code.
-  block:
-    acquire(lock)
-    defer:
-      release(lock)
-    {.locks: [lock].}:
+  acquire(lock)
+  {.locks: [lock].}:
+    try:
       code
+    finally:
+      release(lock)
diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim
index 1c21e10bfa..3c54338364 100644
--- a/lib/core/typeinfo.nim
+++ b/lib/core/typeinfo.nim
@@ -9,16 +9,29 @@
 
 ## This module implements an interface to Nim's `runtime type information`:idx:
 ## (`RTTI`:idx:). See the `marshal `_ module for an example of
-## what this module allows you to do.
+## what this allows you to do.
 ##
-## Note that even though `Any` and its operations hide the nasty low level
-## details from its clients, it remains inherently unsafe! Also, Nim's
-## runtime type information will evolve and may eventually be deprecated.
-## As an alternative approach to programmatically understanding and
-## manipulating types, consider using the `macros `_ package to
-## work with the types' AST representation at compile time. See, for example,
-## the `getTypeImpl proc`_. As an alternative
-## approach to storing arbitrary types at runtime, consider using generics.
+## .. note:: Even though `Any` and its operations hide the nasty low level
+##   details from its users, it remains inherently unsafe! Also, Nim's
+##   runtime type information will evolve and may eventually be deprecated.
+##   As an alternative approach to programmatically understanding and
+##   manipulating types, consider using the `macros `_ module to
+##   work with the types' AST representation at compile time. See for example
+##   the `getTypeImpl proc `_. As an alternative
+##   approach to storing arbitrary types at runtime, consider using generics.
+
+runnableExamples:
+  var x: Any
+
+  var i = 42
+  x = i.toAny
+  assert x.kind == akInt
+  assert x.getInt == 42
+
+  var s = @[1, 2, 3]
+  x = s.toAny
+  assert x.kind == akSequence
+  assert x.len == 3
 
 {.push hints: off.}
 
@@ -27,45 +40,50 @@ include "system/hti.nim"
 
 {.pop.}
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
+
 type
-  AnyKind* = enum      ## what kind of `any` it is
-    akNone = 0,         ## invalid any
-    akBool = 1,         ## any represents a `bool`
-    akChar = 2,         ## any represents a `char`
-    akEnum = 14,        ## any represents an enum
-    akArray = 16,       ## any represents an array
-    akObject = 17,      ## any represents an object
-    akTuple = 18,       ## any represents a tuple
-    akSet = 19,         ## any represents a set
-    akRange = 20,       ## any represents a range
-    akPtr = 21,         ## any represents a ptr
-    akRef = 22,         ## any represents a ref
-    akSequence = 24,    ## any represents a sequence
-    akProc = 25,        ## any represents a proc
-    akPointer = 26,     ## any represents a pointer
-    akString = 28,      ## any represents a string
-    akCString = 29,     ## any represents a cstring
-    akInt = 31,         ## any represents an int
-    akInt8 = 32,        ## any represents an int8
-    akInt16 = 33,       ## any represents an int16
-    akInt32 = 34,       ## any represents an int32
-    akInt64 = 35,       ## any represents an int64
-    akFloat = 36,       ## any represents a float
-    akFloat32 = 37,     ## any represents a float32
-    akFloat64 = 38,     ## any represents a float64
-    akFloat128 = 39,    ## any represents a float128
-    akUInt = 40,        ## any represents an unsigned int
-    akUInt8 = 41,       ## any represents an unsigned int8
-    akUInt16 = 42,      ## any represents an unsigned in16
-    akUInt32 = 43,      ## any represents an unsigned int32
-    akUInt64 = 44,      ## any represents an unsigned int64
+  AnyKind* = enum       ## The kind of `Any`.
+    akNone = 0,         ## invalid
+    akBool = 1,         ## bool
+    akChar = 2,         ## char
+    akEnum = 14,        ## enum
+    akArray = 16,       ## array
+    akObject = 17,      ## object
+    akTuple = 18,       ## tuple
+    akSet = 19,         ## set
+    akRange = 20,       ## range
+    akPtr = 21,         ## ptr
+    akRef = 22,         ## ref
+    akSequence = 24,    ## sequence
+    akProc = 25,        ## proc
+    akPointer = 26,     ## pointer
+    akString = 28,      ## string
+    akCString = 29,     ## cstring
+    akInt = 31,         ## int
+    akInt8 = 32,        ## int8
+    akInt16 = 33,       ## int16
+    akInt32 = 34,       ## int32
+    akInt64 = 35,       ## int64
+    akFloat = 36,       ## float
+    akFloat32 = 37,     ## float32
+    akFloat64 = 38,     ## float64
+    akFloat128 = 39,    ## float128
+    akUInt = 40,        ## uint
+    akUInt8 = 41,       ## uint8
+    akUInt16 = 42,      ## uin16
+    akUInt32 = 43,      ## uint32
+    akUInt64 = 44,      ## uint64
 #    akOpt = 44+18       ## the builtin 'opt' type.
 
-  Any* = object          ## can represent any nim value; NOTE: the wrapped
-                          ## value can be modified with its wrapper! This means
-                          ## that `Any` keeps a non-traced pointer to its
-                          ## wrapped value and **must not** live longer than
-                          ## its wrapped value.
+  Any* = object
+    ## A type that can represent any nim value.
+    ##
+    ## .. danger:: The wrapped value can be modified with its wrapper! This means
+    ##   that `Any` keeps a non-traced pointer to its wrapped value and
+    ##   **must not** live longer than its wrapped value.
     value: pointer
     when defined(js):
       rawType: PNimType
@@ -73,7 +91,7 @@ type
       rawTypePtr: pointer
 
   ppointer = ptr pointer
-  pbyteArray = ptr array[0xffff, int8]
+  pbyteArray = ptr array[0xffff, uint8]
 
 when not defined(gcDestructors):
   type
@@ -84,9 +102,9 @@ when not defined(gcDestructors):
     PGenSeq = ptr TGenericSeq
 
   when defined(gogc):
-    const GenericSeqSize = (3 * sizeof(int))
+    const GenericSeqSize = 3 * sizeof(int)
   else:
-    const GenericSeqSize = (2 * sizeof(int))
+    const GenericSeqSize = 2 * sizeof(int)
 
 else:
   include system/seqs_v2_reimpl
@@ -103,8 +121,7 @@ when not defined(js):
 proc genericAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.}
 
 when not defined(gcDestructors):
-  proc genericShallowAssign(dest, src: pointer, mt: PNimType) {.
-    importCompilerProc.}
+  proc genericShallowAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.}
   proc incrSeq(seq: PGenSeq, elemSize, elemAlign: int): PGenSeq {.importCompilerProc.}
   proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.}
   proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.}
@@ -112,25 +129,27 @@ when not defined(gcDestructors):
 else:
   proc nimNewObj(size, align: int): pointer {.importCompilerProc.}
   proc newSeqPayload(cap, elemSize, elemAlign: int): pointer {.importCompilerProc.}
-  proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {.
+  proc prepareSeqAddUninit(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {.
+    importCompilerProc.}
+  proc zeroNewElements(len: int; p: pointer; addlen, elemSize, elemAlign: int) {.
     importCompilerProc.}
 
-template `+!!`(a, b): untyped = cast[pointer](cast[ByteAddress](a) + b)
+template `+!!`(a, b): untyped = cast[pointer](cast[int](a) + b)
 
 proc getDiscriminant(aa: pointer, n: ptr TNimNode): int =
   assert(n.kind == nkCase)
   var d: int
-  var a = cast[ByteAddress](aa)
+  let a = cast[int](aa)
   case n.typ.size
-  of 1: d = ze(cast[ptr int8](a +% n.offset)[])
-  of 2: d = ze(cast[ptr int16](a +% n.offset)[])
-  of 4: d = int(cast[ptr int32](a +% n.offset)[])
-  of 8: d = int(cast[ptr int64](a +% n.offset)[])
+  of 1: d = int(cast[ptr uint8](a +% n.offset)[])
+  of 2: d = int(cast[ptr uint16](a +% n.offset)[])
+  of 4: d = int(cast[ptr uint32](a +% n.offset)[])
+  of 8: d = int(cast[ptr uint64](a +% n.offset)[])
   else: assert(false)
   return d
 
 proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode =
-  var discr = getDiscriminant(aa, n)
+  let discr = getDiscriminant(aa, n)
   if discr <% n.len:
     result = n.sons[discr]
     if result == nil: result = n.sons[n.len]
@@ -144,17 +163,17 @@ proc newAny(value: pointer, rawType: PNimType): Any {.inline.} =
 
 when declared(system.VarSlot):
   proc toAny*(x: VarSlot): Any {.inline.} =
-    ## Constructs a `Any` object from a variable slot `x`.
+    ## Constructs an `Any` object from a variable slot `x`.
     ## This captures `x`'s address, so `x` can be modified with its
-    ## `Any` wrapper! The client needs to ensure that the wrapper
+    ## `Any` wrapper! The caller needs to ensure that the wrapper
     ## **does not** live longer than `x`!
     ## This is provided for easier reflection capabilities of a debugger.
     result.value = x.address
     result.rawType = x.typ
 
 proc toAny*[T](x: var T): Any {.inline.} =
-  ## Constructs a `Any` object from `x`. This captures `x`'s address, so
-  ## `x` can be modified with its `Any` wrapper! The client needs to ensure
+  ## Constructs an `Any` object from `x`. This captures `x`'s address, so
+  ## `x` can be modified with its `Any` wrapper! The caller needs to ensure
   ## that the wrapper **does not** live longer than `x`!
   newAny(addr(x), cast[PNimType](getTypeInfo(x)))
 
@@ -167,12 +186,12 @@ proc size*(x: Any): int {.inline.} =
   result = x.rawType.size
 
 proc baseTypeKind*(x: Any): AnyKind {.inline.} =
-  ## Gets the base type's kind; `akNone` is returned if `x` has no base type.
+  ## Gets the base type's kind. If `x` has no base type, `akNone` is returned.
   if x.rawType.base != nil:
     result = AnyKind(ord(x.rawType.base.kind))
 
 proc baseTypeSize*(x: Any): int {.inline.} =
-  ## Returns the size of `x`'s basetype.
+  ## Returns the size of `x`'s base type. If `x` has no base type, 0 is returned.
   if x.rawType.base != nil:
     result = x.rawType.base.size
 
@@ -203,7 +222,9 @@ proc extendSeq*(x: Any) =
   when defined(gcDestructors):
     var s = cast[ptr NimSeqV2Reimpl](x.value)
     let elem = x.rawType.base
-    s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAdd(s.len, s.p, 1, elem.size, elem.align))
+    if s.p == nil or s.p.cap < s.len+1:
+      s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAddUninit(s.len, s.p, 1, elem.size, elem.align))
+    zeroNewElements(s.len, s.p, 1, elem.size, elem.align)
     inc s.len
   else:
     var y = cast[ptr PGenSeq](x.value)[]
@@ -301,33 +322,32 @@ proc len*(x: Any): int =
 
 
 proc base*(x: Any): Any =
-  ## Returns base Any (useful for inherited object types).
+  ## Returns the base type of `x` (useful for inherited object types).
   result.rawType = x.rawType.base
   result.value = x.value
 
 
 proc isNil*(x: Any): bool =
-  ## `isNil` for an any `x` that represents a cstring, proc or
+  ## `isNil` for an `x` that represents a cstring, proc or
   ## some pointer type.
-  assert x.rawType.kind in {tyCString, tyRef, tyPtr, tyPointer, tyProc}
+  assert x.rawType.kind in {tyCstring, tyRef, tyPtr, tyPointer, tyProc}
   result = isNil(cast[ppointer](x.value)[])
 
-const
-  pointerLike = when defined(gcDestructors): {tyCString, tyRef, tyPtr, tyPointer, tyProc}
-                else: {tyString, tyCString, tyRef, tyPtr, tyPointer,
-                            tySequence, tyProc}
+const pointerLike =
+  when defined(gcDestructors): {tyCstring, tyRef, tyPtr, tyPointer, tyProc}
+  else: {tyString, tyCstring, tyRef, tyPtr, tyPointer, tySequence, tyProc}
 
 proc getPointer*(x: Any): pointer =
   ## Retrieves the pointer value out of `x`. `x` needs to be of kind
   ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`,
-  ## `akPointer`, `akSequence`.
+  ## `akPointer` or `akSequence`.
   assert x.rawType.kind in pointerLike
   result = cast[ppointer](x.value)[]
 
 proc setPointer*(x: Any, y: pointer) =
   ## Sets the pointer value of `x`. `x` needs to be of kind
   ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`,
-  ## `akPointer`, `akSequence`.
+  ## `akPointer` or `akSequence`.
   assert x.rawType.kind in pointerLike
   if y != nil and x.rawType.kind != tyPointer:
     genericAssign(x.value, y, x.rawType)
@@ -349,15 +369,15 @@ proc fieldsAux(p: pointer, n: ptr TNimNode,
     if m != nil: fieldsAux(p, m, ret)
 
 iterator fields*(x: Any): tuple[name: string, any: Any] =
-  ## Iterates over every active field of the any `x` that represents an object
+  ## Iterates over every active field of `x`. `x` needs to represent an object
   ## or a tuple.
   assert x.rawType.kind in {tyTuple, tyObject}
-  var p = x.value
+  let p = x.value
   var t = x.rawType
   # XXX BUG: does not work yet, however is questionable anyway
   when false:
     if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[]
-  var ret: seq[tuple[name: cstring, any: Any]] = @[]
+  var ret: seq[tuple[name: cstring, any: Any]]
   if t.kind == tyObject:
     while true:
       fieldsAux(p, t.node, ret)
@@ -368,8 +388,7 @@ iterator fields*(x: Any): tuple[name: string, any: Any] =
   for name, any in items(ret):
     yield ($name, any)
 
-proc getFieldNode(p: pointer, n: ptr TNimNode,
-                  name: cstring): ptr TNimNode =
+proc getFieldNode(p: pointer, n: ptr TNimNode, name: cstring): ptr TNimNode =
   case n.kind
   of nkNone: assert(false)
   of nkSlot:
@@ -383,17 +402,17 @@ proc getFieldNode(p: pointer, n: ptr TNimNode,
     if cmpNimIdentifier(n.name, name) == 0:
       result = n
     else:
-      var m = selectBranch(p, n)
+      let m = selectBranch(p, n)
       if m != nil: result = getFieldNode(p, m, name)
 
 proc `[]=`*(x: Any, fieldName: string, value: Any) =
-  ## Sets a field of `x`; `x` represents an object or a tuple.
+  ## Sets a field of `x`. `x` needs to represent an object or a tuple.
   var t = x.rawType
   # XXX BUG: does not work yet, however is questionable anyway
   when false:
     if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[]
   assert x.rawType.kind in {tyTuple, tyObject}
-  var n = getFieldNode(x.value, t.node, fieldName)
+  let n = getFieldNode(x.value, t.node, fieldName)
   if n != nil:
     assert n.typ == value.rawType
     genericAssign(x.value +!! n.offset, value.value, value.rawType)
@@ -401,13 +420,13 @@ proc `[]=`*(x: Any, fieldName: string, value: Any) =
     raise newException(ValueError, "invalid field name: " & fieldName)
 
 proc `[]`*(x: Any, fieldName: string): Any =
-  ## Gets a field of `x`; `x` represents an object or a tuple.
+  ## Gets a field of `x`. `x` needs to represent an object or a tuple.
   var t = x.rawType
   # XXX BUG: does not work yet, however is questionable anyway
   when false:
     if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[]
   assert x.rawType.kind in {tyTuple, tyObject}
-  var n = getFieldNode(x.value, t.node, fieldName)
+  let n = getFieldNode(x.value, t.node, fieldName)
   if n != nil:
     result.value = x.value +!! n.offset
     result.rawType = n.typ
@@ -417,39 +436,39 @@ proc `[]`*(x: Any, fieldName: string): Any =
     raise newException(ValueError, "invalid field name: " & fieldName)
 
 proc `[]`*(x: Any): Any =
-  ## Dereference operation for the any `x` that represents a ptr or a ref.
+  ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref.
   assert x.rawType.kind in {tyRef, tyPtr}
   result.value = cast[ppointer](x.value)[]
   result.rawType = x.rawType.base
 
 proc `[]=`*(x, y: Any) =
-  ## Dereference operation for the any `x` that represents a ptr or a ref.
+  ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref.
   assert x.rawType.kind in {tyRef, tyPtr}
   assert y.rawType == x.rawType.base
   genericAssign(cast[ppointer](x.value)[], y.value, y.rawType)
 
 proc getInt*(x: Any): int =
-  ## Retrieves the int value out of `x`. `x` needs to represent an int.
+  ## Retrieves the `int` value out of `x`. `x` needs to represent an `int`.
   assert skipRange(x.rawType).kind == tyInt
   result = cast[ptr int](x.value)[]
 
 proc getInt8*(x: Any): int8 =
-  ## Retrieves the int8 value out of `x`. `x` needs to represent an int8.
+  ## Retrieves the `int8` value out of `x`. `x` needs to represent an `int8`.
   assert skipRange(x.rawType).kind == tyInt8
   result = cast[ptr int8](x.value)[]
 
 proc getInt16*(x: Any): int16 =
-  ## Retrieves the int16 value out of `x`. `x` needs to represent an int16.
+  ## Retrieves the `int16` value out of `x`. `x` needs to represent an `int16`.
   assert skipRange(x.rawType).kind == tyInt16
   result = cast[ptr int16](x.value)[]
 
 proc getInt32*(x: Any): int32 =
-  ## Retrieves the int32 value out of `x`. `x` needs to represent an int32.
+  ## Retrieves the `int32` value out of `x`. `x` needs to represent an `int32`.
   assert skipRange(x.rawType).kind == tyInt32
   result = cast[ptr int32](x.value)[]
 
 proc getInt64*(x: Any): int64 =
-  ## Retrieves the int64 value out of `x`. `x` needs to represent an int64.
+  ## Retrieves the `int64` value out of `x`. `x` needs to represent an `int64`.
   assert skipRange(x.rawType).kind == tyInt64
   result = cast[ptr int64](x.value)[]
 
@@ -457,7 +476,7 @@ proc getBiggestInt*(x: Any): BiggestInt =
   ## Retrieves the integer value out of `x`. `x` needs to represent
   ## some integer, a bool, a char, an enum or a small enough bit set.
   ## The value might be sign-extended to `BiggestInt`.
-  var t = skipRange(x.rawType)
+  let t = skipRange(x.rawType)
   case t.kind
   of tyInt: result = BiggestInt(cast[ptr int](x.value)[])
   of tyInt8: result = BiggestInt(cast[ptr int8](x.value)[])
@@ -468,8 +487,8 @@ proc getBiggestInt*(x: Any): BiggestInt =
   of tyChar: result = BiggestInt(cast[ptr char](x.value)[])
   of tyEnum, tySet:
     case t.size
-    of 1: result = ze64(cast[ptr int8](x.value)[])
-    of 2: result = ze64(cast[ptr int16](x.value)[])
+    of 1: result = int64(cast[ptr uint8](x.value)[])
+    of 2: result = int64(cast[ptr uint16](x.value)[])
     of 4: result = BiggestInt(cast[ptr int32](x.value)[])
     of 8: result = BiggestInt(cast[ptr int64](x.value)[])
     else: assert false
@@ -482,7 +501,7 @@ proc getBiggestInt*(x: Any): BiggestInt =
 proc setBiggestInt*(x: Any, y: BiggestInt) =
   ## Sets the integer value of `x`. `x` needs to represent
   ## some integer, a bool, a char, an enum or a small enough bit set.
-  var t = skipRange(x.rawType)
+  let t = skipRange(x.rawType)
   case t.kind
   of tyInt: cast[ptr int](x.value)[] = int(y)
   of tyInt8: cast[ptr int8](x.value)[] = int8(y)
@@ -493,8 +512,8 @@ proc setBiggestInt*(x: Any, y: BiggestInt) =
   of tyChar: cast[ptr char](x.value)[] = chr(y.int)
   of tyEnum, tySet:
     case t.size
-    of 1: cast[ptr int8](x.value)[] = toU8(y.int)
-    of 2: cast[ptr int16](x.value)[] = toU16(y.int)
+    of 1: cast[ptr uint8](x.value)[] = uint8(y.int)
+    of 2: cast[ptr uint16](x.value)[] = uint16(y.int)
     of 4: cast[ptr int32](x.value)[] = int32(y)
     of 8: cast[ptr int64](x.value)[] = y
     else: assert false
@@ -505,38 +524,34 @@ proc setBiggestInt*(x: Any, y: BiggestInt) =
   else: assert false
 
 proc getUInt*(x: Any): uint =
-  ## Retrieves the uint value out of `x`, `x` needs to represent an uint.
+  ## Retrieves the `uint` value out of `x`. `x` needs to represent a `uint`.
   assert skipRange(x.rawType).kind == tyUInt
   result = cast[ptr uint](x.value)[]
 
 proc getUInt8*(x: Any): uint8 =
-  ## Retrieves the uint8 value out of `x`, `x` needs to represent an
-  ## uint8.
+  ## Retrieves the `uint8` value out of `x`. `x` needs to represent a `uint8`.
   assert skipRange(x.rawType).kind == tyUInt8
   result = cast[ptr uint8](x.value)[]
 
 proc getUInt16*(x: Any): uint16 =
-  ## Retrieves the uint16 value out of `x`, `x` needs to represent an
-  ## uint16.
+  ## Retrieves the `uint16` value out of `x`. `x` needs to represent a `uint16`.
   assert skipRange(x.rawType).kind == tyUInt16
   result = cast[ptr uint16](x.value)[]
 
 proc getUInt32*(x: Any): uint32 =
-  ## Retrieves the uint32 value out of `x`, `x` needs to represent an
-  ## uint32.
+  ## Retrieves the `uint32` value out of `x`. `x` needs to represent a `uint32`.
   assert skipRange(x.rawType).kind == tyUInt32
   result = cast[ptr uint32](x.value)[]
 
 proc getUInt64*(x: Any): uint64 =
-  ## Retrieves the uint64 value out of `x`, `x` needs to represent an
-  ## uint64.
+  ## Retrieves the `uint64` value out of `x`. `x` needs to represent a `uint64`.
   assert skipRange(x.rawType).kind == tyUInt64
   result = cast[ptr uint64](x.value)[]
 
 proc getBiggestUint*(x: Any): uint64 =
   ## Retrieves the unsigned integer value out of `x`. `x` needs to
   ## represent an unsigned integer.
-  var t = skipRange(x.rawType)
+  let t = skipRange(x.rawType)
   case t.kind
   of tyUInt: result = uint64(cast[ptr uint](x.value)[])
   of tyUInt8: result = uint64(cast[ptr uint8](x.value)[])
@@ -546,9 +561,9 @@ proc getBiggestUint*(x: Any): uint64 =
   else: assert false
 
 proc setBiggestUint*(x: Any; y: uint64) =
-  ## Sets the unsigned integer value of `c`. `c` needs to represent an
+  ## Sets the unsigned integer value of `x`. `x` needs to represent an
   ## unsigned integer.
-  var t = skipRange(x.rawType)
+  let t = skipRange(x.rawType)
   case t.kind:
   of tyUInt: cast[ptr uint](x.value)[] = uint(y)
   of tyUInt8: cast[ptr uint8](x.value)[] = uint8(y)
@@ -558,14 +573,14 @@ proc setBiggestUint*(x: Any; y: uint64) =
   else: assert false
 
 proc getChar*(x: Any): char =
-  ## Retrieves the char value out of `x`. `x` needs to represent a char.
-  var t = skipRange(x.rawType)
+  ## Retrieves the `char` value out of `x`. `x` needs to represent a `char`.
+  let t = skipRange(x.rawType)
   assert t.kind == tyChar
   result = cast[ptr char](x.value)[]
 
 proc getBool*(x: Any): bool =
-  ## Retrieves the bool value out of `x`. `x` needs to represent a bool.
-  var t = skipRange(x.rawType)
+  ## Retrieves the `bool` value out of `x`. `x` needs to represent a `bool`.
+  let t = skipRange(x.rawType)
   assert t.kind == tyBool
   result = cast[ptr bool](x.value)[]
 
@@ -579,10 +594,10 @@ proc getEnumOrdinal*(x: Any, name: string): int =
   ## Gets the enum field ordinal from `name`. `x` needs to represent an enum
   ## but is only used to access the type information. In case of an error
   ## `low(int)` is returned.
-  var typ = skipRange(x.rawType)
+  let typ = skipRange(x.rawType)
   assert typ.kind == tyEnum
-  var n = typ.node
-  var s = n.sons
+  let n = typ.node
+  let s = n.sons
   for i in 0 .. n.len-1:
     if cmpNimIdentifier($s[i].name, name) == 0:
       if ntfEnumHole notin typ.flags:
@@ -595,16 +610,16 @@ proc getEnumField*(x: Any, ordinalValue: int): string =
   ## Gets the enum field name as a string. `x` needs to represent an enum
   ## but is only used to access the type information. The field name of
   ## `ordinalValue` is returned.
-  var typ = skipRange(x.rawType)
+  let typ = skipRange(x.rawType)
   assert typ.kind == tyEnum
-  var e = ordinalValue
+  let e = ordinalValue
   if ntfEnumHole notin typ.flags:
     if e <% typ.node.len:
       return $typ.node.sons[e].name
   else:
     # ugh we need a slow linear search:
-    var n = typ.node
-    var s = n.sons
+    let n = typ.node
+    let s = n.sons
     for i in 0 .. n.len-1:
       if s[i].offset == e: return $s[i].name
   result = $e
@@ -614,17 +629,17 @@ proc getEnumField*(x: Any): string =
   result = getEnumField(x, getBiggestInt(x).int)
 
 proc getFloat*(x: Any): float =
-  ## Retrieves the float value out of `x`. `x` needs to represent an float.
+  ## Retrieves the `float` value out of `x`. `x` needs to represent a `float`.
   assert skipRange(x.rawType).kind == tyFloat
   result = cast[ptr float](x.value)[]
 
 proc getFloat32*(x: Any): float32 =
-  ## Retrieves the float32 value out of `x`. `x` needs to represent an float32.
+  ## Retrieves the `float32` value out of `x`. `x` needs to represent a `float32`.
   assert skipRange(x.rawType).kind == tyFloat32
   result = cast[ptr float32](x.value)[]
 
 proc getFloat64*(x: Any): float64 =
-  ## Retrieves the float64 value out of `x`. `x` needs to represent an float64.
+  ## Retrieves the `float64` value out of `x`. `x` needs to represent a `float64`.
   assert skipRange(x.rawType).kind == tyFloat64
   result = cast[ptr float64](x.value)[]
 
@@ -647,7 +662,7 @@ proc setBiggestFloat*(x: Any, y: BiggestFloat) =
   else: assert false
 
 proc getString*(x: Any): string =
-  ## Retrieves the string value out of `x`. `x` needs to represent a string.
+  ## Retrieves the `string` value out of `x`. `x` needs to represent a `string`.
   assert x.rawType.kind == tyString
   when defined(gcDestructors):
     result = cast[ptr string](x.value)[]
@@ -656,13 +671,13 @@ proc getString*(x: Any): string =
       result = cast[ptr string](x.value)[]
 
 proc setString*(x: Any, y: string) =
-  ## Sets the string value of `x`. `x` needs to represent a string.
+  ## Sets the `string` value of `x`. `x` needs to represent a `string`.
   assert x.rawType.kind == tyString
   cast[ptr string](x.value)[] = y # also correct for gcDestructors
 
 proc getCString*(x: Any): cstring =
-  ## Retrieves the cstring value out of `x`. `x` needs to represent a cstring.
-  assert x.rawType.kind == tyCString
+  ## Retrieves the `cstring` value out of `x`. `x` needs to represent a `cstring`.
+  assert x.rawType.kind == tyCstring
   result = cast[ptr cstring](x.value)[]
 
 proc assign*(x, y: Any) =
@@ -672,34 +687,34 @@ proc assign*(x, y: Any) =
   genericAssign(x.value, y.value, y.rawType)
 
 iterator elements*(x: Any): int =
-  ## Iterates over every element of `x` that represents a Nim bitset.
+  ## Iterates over every element of `x`. `x` needs to represent a `set`.
   assert x.rawType.kind == tySet
-  var typ = x.rawType
-  var p = x.value
+  let typ = x.rawType
+  let p = x.value
   # "typ.slots.len" field is for sets the "first" field
   var u: int64
   case typ.size
-  of 1: u = ze64(cast[ptr int8](p)[])
-  of 2: u = ze64(cast[ptr int16](p)[])
-  of 4: u = ze64(cast[ptr int32](p)[])
+  of 1: u = int64(cast[ptr uint8](p)[])
+  of 2: u = int64(cast[ptr uint16](p)[])
+  of 4: u = int64(cast[ptr uint32](p)[])
   of 8: u = cast[ptr int64](p)[]
   else:
-    var a = cast[pbyteArray](p)
+    let a = cast[pbyteArray](p)
     for i in 0 .. typ.size*8-1:
-      if (ze(a[i div 8]) and (1 shl (i mod 8))) != 0:
-        yield i+typ.node.len
+      if (int(a[i div 8]) and (1 shl (i mod 8))) != 0:
+        yield i + typ.node.len
   if typ.size <= 8:
     for i in 0..sizeof(int64)*8-1:
       if (u and (1'i64 shl int64(i))) != 0'i64:
-        yield i+typ.node.len
+        yield i + typ.node.len
 
 proc inclSetElement*(x: Any, elem: int) =
   ## Includes an element `elem` in `x`. `x` needs to represent a Nim bitset.
   assert x.rawType.kind == tySet
-  var typ = x.rawType
-  var p = x.value
+  let typ = x.rawType
+  let p = x.value
   # "typ.slots.len" field is for sets the "first" field
-  var e = elem - typ.node.len
+  let e = elem - typ.node.len
   case typ.size
   of 1:
     var a = cast[ptr int8](p)
@@ -715,4 +730,4 @@ proc inclSetElement*(x: Any, elem: int) =
     a[] = a[] or (1'i64 shl e)
   else:
     var a = cast[pbyteArray](p)
-    a[e shr 3] = toU8(a[e shr 3] or (1 shl (e and 7)))
+    a[e shr 3] = a[e shr 3] or uint8(1 shl (e and 7))
diff --git a/lib/deprecated/pure/LockFreeHash.nim b/lib/deprecated/pure/LockFreeHash.nim
deleted file mode 100644
index 97e1ef6ad4..0000000000
--- a/lib/deprecated/pure/LockFreeHash.nim
+++ /dev/null
@@ -1,609 +0,0 @@
-#nim c -t:-march=i686 --cpu:amd64 --threads:on -d:release lockfreehash.nim
-
-import math, hashes
-
-#------------------------------------------------------------------------------
-## Memory Utility Functions
-
-proc newHeap*[T](): ptr T =
-  result = cast[ptr T](alloc0(sizeof(T)))
-
-proc copyNew*[T](x: var T): ptr T =
-  var
-    size = sizeof(T)
-    mem = alloc(size)
-  copyMem(mem, x.addr, size)
-  return cast[ptr T](mem)
-
-proc copyTo*[T](val: var T, dest: int) =
-  copyMem(pointer(dest), val.addr, sizeof(T))
-
-proc allocType*[T](): pointer = alloc(sizeof(T))
-
-proc newShared*[T](): ptr T =
-  result = cast[ptr T](allocShared0(sizeof(T)))
-
-proc copyShared*[T](x: var T): ptr T =
-  var
-    size = sizeof(T)
-    mem = allocShared(size)
-  copyMem(mem, x.addr, size)
-  return cast[ptr T](mem)
-
-#------------------------------------------------------------------------------
-## Pointer arithmetic
-
-proc `+`*(p: pointer, i: int): pointer {.inline.} =
-  cast[pointer](cast[int](p) + i)
-
-const
-  minTableSize = 8
-  reProbeLimit = 12
-  minCopyWork = 4096
-  intSize = sizeof(int)
-
-
-
-when sizeof(int) == 4: # 32bit
-  type
-    Raw = range[0..1073741823]
-    ## The range of uint values that can be stored directly in a value slot
-    ## when on a 32 bit platform
-elif sizeof(int) == 8: # 64bit
-  type
-    Raw = range[0'i64..4611686018427387903'i64]
-    ## The range of uint values that can be stored directly in a value slot
-    ## when on a 64 bit platform
-else:
-  {.error: "unsupported platform".}
-
-type
-  Entry = tuple
-    key: int
-    value: int
-
-  EntryArr = ptr array[0..10_000_000, Entry]
-
-  PConcTable[K, V] = ptr object {.pure.}
-    len: int
-    used: int
-    active: int
-    copyIdx: int
-    copyDone: int
-    next: PConcTable[K, V]
-    data: EntryArr
-
-proc setVal[K, V](table: var PConcTable[K, V], key: int, val: int,
-  expVal: int, match: bool): int
-
-#------------------------------------------------------------------------------
-
-# Create a new table
-proc newLFTable*[K, V](size: int = minTableSize): PConcTable[K, V] =
-  let
-    dataLen = max(nextPowerOfTwo(size), minTableSize)
-    dataSize = dataLen*sizeof(Entry)
-    dataMem = allocShared0(dataSize)
-    tableSize = 7 * intSize
-    tableMem = allocShared0(tableSize)
-    table = cast[PConcTable[K, V]](tableMem)
-  table.len = dataLen
-  table.used = 0
-  table.active = 0
-  table.copyIdx = 0
-  table.copyDone = 0
-  table.next = nil
-  table.data = cast[EntryArr](dataMem)
-  result = table
-
-#------------------------------------------------------------------------------
-
-# Delete a table
-proc deleteConcTable[K, V](tbl: PConcTable[K, V]) =
-  deallocShared(tbl.data)
-  deallocShared(tbl)
-
-#------------------------------------------------------------------------------
-
-proc `[]`[K, V](table: var PConcTable[K, V], i: int): var Entry {.inline.} =
-  table.data[i]
-
-#------------------------------------------------------------------------------
-# State flags stored in ptr
-
-
-proc pack[T](x: T): int {.inline.} =
-  result = (cast[int](x) shl 2)
-  #echo("packKey ",cast[int](x) , " -> ", result)
-
-# Pop the flags off returning a 4 byte aligned ptr to our Key or Val
-proc pop(x: int): int {.inline.} =
-  result = x and 0xFFFFFFFC'i32
-
-# Pop the raw value off of our Key or Val
-proc popRaw(x: int): int {.inline.} =
-  result = x shr 2
-
-# Pop the flags off returning a 4 byte aligned ptr to our Key or Val
-proc popPtr[V](x: int): ptr V {.inline.} =
-  result = cast[ptr V](pop(x))
-  #echo("popPtr " & $x & " -> " & $cast[int](result))
-
-# Ghost (sentinel)
-# K or V is no longer valid use new table
-const Ghost = 0xFFFFFFFC
-proc isGhost(x: int): bool {.inline.} =
-  result = x == 0xFFFFFFFC
-
-# Tombstone
-# applied to V = K is dead
-proc isTomb(x: int): bool {.inline.} =
-  result = (x and 0x00000002) != 0
-
-proc setTomb(x: int): int {.inline.} =
-  result = x or 0x00000002
-
-# Prime
-# K or V is in new table copied from old
-proc isPrime(x: int): bool {.inline.} =
-  result = (x and 0x00000001) != 0
-
-proc setPrime(x: int): int {.inline.} =
-  result = x or 0x00000001
-
-#------------------------------------------------------------------------------
-
-##This is for i32 only need to override for i64
-proc hashInt(x: int): int {.inline.} =
-  var h = uint32(x) #shr 2'u32
-  h = h xor (h shr 16'u32)
-  h *= 0x85ebca6b'u32
-  h = h xor (h shr 13'u32)
-  h *= 0xc2b2ae35'u32
-  h = h xor (h shr 16'u32)
-  result = int(h)
-
-#------------------------------------------------------------------------------
-
-proc resize[K, V](self: PConcTable[K, V]): PConcTable[K, V] =
-  var next = atomic_load_n(self.next.addr, ATOMIC_RELAXED)
-  #echo("next = " & $cast[int](next))
-  if next != nil:
-    #echo("A new table already exists, copy in progress")
-    return next
-  var
-    oldLen = atomic_load_n(self.len.addr, ATOMIC_RELAXED)
-    newTable = newLFTable[K, V](oldLen*2)
-    success = atomic_compare_exchange_n(self.next.addr, next.addr, newTable,
-                  false, ATOMIC_RELAXED, ATOMIC_RELAXED)
-  if not success:
-    echo("someone beat us to it! delete table we just created and return his " &
-        $cast[int](next))
-    deleteConcTable(newTable)
-    return next
-  else:
-    echo("Created New Table! " & $cast[int](newTable) & " Size = " & $newTable.len)
-    return newTable
-
-
-#------------------------------------------------------------------------------
-#proc keyEQ[K](key1: ptr K, key2: ptr K): bool {.inline.} =
-proc keyEQ[K](key1: int, key2: int): bool {.inline.} =
-  result = false
-  when K is Raw:
-    if key1 == key2:
-      result = true
-  else:
-    var
-      p1 = popPtr[K](key1)
-      p2 = popPtr[K](key2)
-    if p1 != nil and p2 != nil:
-      if cast[int](p1) == cast[int](p2):
-        return true
-      if p1[] == p2[]:
-        return true
-
-#------------------------------------------------------------------------------
-
-#proc tableFull(self: var PConcTable[K,V]) : bool {.inline.} =
-
-
-#------------------------------------------------------------------------------
-
-proc copySlot[K, V](idx: int, oldTbl: var PConcTable[K, V],
-    newTbl: var PConcTable[K, V]): bool =
-  #echo("Copy idx " & $idx)
-  var
-    oldVal = 0
-    oldkey = 0
-    ok = false
-  result = false
-  #Block the key so no other threads waste time here
-  while not ok:
-    ok = atomic_compare_exchange_n(oldTbl[idx].key.addr, oldKey.addr,
-      setTomb(oldKey), false, ATOMIC_RELAXED, ATOMIC_RELAXED)
-  #echo("oldKey was = " & $oldKey & "  set it to tomb " & $setTomb(oldKey))
-  #Prevent new values from appearing in the old table by priming
-  oldVal = atomic_load_n(oldTbl[idx].value.addr, ATOMIC_RELAXED)
-  while not isPrime(oldVal):
-    var box = if oldVal == 0 or isTomb(oldVal): oldVal.setTomb.setPrime
-      else: oldVal.setPrime
-    if atomic_compare_exchange_n(oldTbl[idx].value.addr, oldVal.addr,
-      box, false, ATOMIC_RELAXED, ATOMIC_RELAXED):
-      if isPrime(box) and isTomb(box):
-        return true
-      oldVal = box
-      break
-  #echo("oldVal was = ", oldVal, "  set it to prime ", box)
-  if isPrime(oldVal) and isTomb(oldVal):
-    #when not (K is Raw):
-    #  deallocShared(popPtr[K](oldKey))
-    return false
-  if isTomb(oldVal):
-    echo("oldVal is Tomb!!!, should not happen")
-  if pop(oldVal) != 0:
-    result = setVal(newTbl, pop(oldKey), pop(oldVal), 0, true) == 0
-  #if result:
-    #echo("Copied a Slot! idx= " & $idx & " key= " & $oldKey & " val= " & $oldVal)
-  #else:
-    #echo("copy slot failed")
-  # Our copy is done so we disable the old slot
-  while not ok:
-    ok = atomic_compare_exchange_n(oldTbl[idx].value.addr, oldVal.addr,
-      oldVal.setTomb.setPrime, false, ATOMIC_RELAXED, ATOMIC_RELAXED)
-  #echo("disabled old slot")
-  #echo"---------------------"
-
-#------------------------------------------------------------------------------
-
-proc promote[K, V](table: var PConcTable[K, V]) =
-  var
-    newData = atomic_load_n(table.next.data.addr, ATOMIC_RELAXED)
-    newLen = atomic_load_n(table.next.len.addr, ATOMIC_RELAXED)
-    newUsed = atomic_load_n(table.next.used.addr, ATOMIC_RELAXED)
-
-  deallocShared(table.data)
-  atomic_store_n(table.data.addr, newData, ATOMIC_RELAXED)
-  atomic_store_n(table.len.addr, newLen, ATOMIC_RELAXED)
-  atomic_store_n(table.used.addr, newUsed, ATOMIC_RELAXED)
-  atomic_store_n(table.copyIdx.addr, 0, ATOMIC_RELAXED)
-  atomic_store_n(table.copyDone.addr, 0, ATOMIC_RELAXED)
-  deallocShared(table.next)
-  atomic_store_n(table.next.addr, nil, ATOMIC_RELAXED)
-  echo("new table swapped!")
-
-#------------------------------------------------------------------------------
-
-proc checkAndPromote[K, V](table: var PConcTable[K, V], workDone: int): bool =
-  var
-    oldLen = atomic_load_n(table.len.addr, ATOMIC_RELAXED)
-    copyDone = atomic_load_n(table.copyDone.addr, ATOMIC_RELAXED)
-    ok: bool
-  result = false
-  if workDone > 0:
-    #echo("len to copy =" & $oldLen)
-    #echo("copyDone + workDone = " & $copyDone & " + " & $workDone)
-    while not ok:
-      ok = atomic_compare_exchange_n(table.copyDone.addr, copyDone.addr,
-        copyDone + workDone, false, ATOMIC_RELAXED, ATOMIC_RELAXED)
-    #if ok: echo("set copyDone")
-    # If the copy is done we can promote this table
-    if copyDone + workDone >= oldLen:
-      # Swap new data
-      #echo("work is done!")
-      table.promote
-      result = true
-
-#------------------------------------------------------------------------------
-
-proc copySlotAndCheck[K, V](table: var PConcTable[K, V], idx: int):
-  PConcTable[K, V] =
-  var
-    newTable = cast[PConcTable[K, V]](atomic_load_n(table.next.addr,
-        ATOMIC_RELAXED))
-  result = newTable
-  if newTable != nil and copySlot(idx, table, newTable):
-    #echo("copied a single slot, idx = " & $idx)
-    if checkAndPromote(table, 1): return table
-
-
-#------------------------------------------------------------------------------
-
-proc helpCopy[K, V](table: var PConcTable[K, V]): PConcTable[K, V] =
-  var
-    newTable = cast[PConcTable[K, V]](atomic_load_n(table.next.addr,
-        ATOMIC_RELAXED))
-  result = newTable
-  if newTable != nil:
-    var
-      oldLen = atomic_load_n(table.len.addr, ATOMIC_RELAXED)
-      copyDone = atomic_load_n(table.copyDone.addr, ATOMIC_RELAXED)
-      copyIdx = 0
-      work = min(oldLen, minCopyWork)
-      #panicStart = -1
-      workDone = 0
-    if copyDone < oldLen:
-      var ok: bool
-      while not ok:
-        ok = atomic_compare_exchange_n(table.copyIdx.addr, copyIdx.addr,
-          copyIdx + work, false, ATOMIC_RELAXED, ATOMIC_RELAXED)
-      #echo("copy idx = ", copyIdx)
-      for i in 0..work-1:
-        var idx = (copyIdx + i) and (oldLen - 1)
-        if copySlot(idx, table, newTable):
-          workDone += 1
-      if workDone > 0:
-        #echo("did work ", workDone, " on thread ", cast[int](myThreadID[pointer]()))
-        if checkAndPromote(table, workDone): return table
-    # In case a thread finished all the work then got stalled before promotion
-    if checkAndPromote(table, 0): return table
-
-
-
-#------------------------------------------------------------------------------
-
-proc setVal[K, V](table: var PConcTable[K, V], key: int, val: int,
-  expVal: int, match: bool): int =
-  #echo("-try set- in table ", " key = ", (popPtr[K](key)[]), " val = ", val)
-  when K is Raw:
-    var idx = hashInt(key)
-  else:
-    var idx = popPtr[K](key)[].hash
-  var
-    nextTable: PConcTable[K, V]
-    probes = 1
-  # spin until we find a key slot or build and jump to next table
-  while true:
-    idx = idx and (table.len - 1)
-    #echo("try set idx = " & $idx & "for" & $key)
-    var
-      probedKey = 0
-      openKey = atomic_compare_exchange_n(table[idx].key.addr, probedKey.addr,
-        key, false, ATOMIC_RELAXED, ATOMIC_RELAXED)
-    if openKey:
-      if val.isTomb:
-        #echo("val was tomb, bail, no reason to set an open slot to tomb")
-        return val
-      #increment used slots
-      #echo("found an open slot, total used = " &
-      #$atomic_add_fetch(table.used.addr, 1, ATOMIC_RELAXED))
-      discard atomic_add_fetch(table.used.addr, 1, ATOMIC_RELAXED)
-      break # We found an open slot
-    #echo("set idx ", idx, " key = ", key, " probed = ", probedKey)
-    if keyEQ[K](probedKey, key):
-      #echo("we found the matching slot")
-      break # We found a matching slot
-    if (not(expVal != 0 and match)) and (probes >= reProbeLimit or key.isTomb):
-      if key.isTomb: echo("Key is Tombstone")
-      #if probes >= reProbeLimit: echo("Too much probing " & $probes)
-      #echo("try to resize")
-      #create next bigger table
-      nextTable = resize(table)
-      #help do some copying
-      #echo("help copy old table to new")
-      nextTable = helpCopy(table)
-      #now setVal in the new table instead
-      #echo("jumping to next table to set val")
-      return setVal(nextTable, key, val, expVal, match)
-    else:
-      idx += 1
-      probes += 1
-  # Done spinning for a new slot
-  var oldVal = atomic_load_n(table[idx].value.addr, ATOMIC_RELAXED)
-  if val == oldVal:
-    #echo("this val is already in the slot")
-    return oldVal
-  nextTable = atomic_load_n(table.next.addr, ATOMIC_SEQ_CST)
-  if nextTable == nil and
-    ((oldVal == 0 and
-    (probes >= reProbeLimit or table.used / table.len > 0.8)) or
-    (isPrime(oldVal))):
-    if table.used / table.len > 0.8: echo("resize because usage ratio = " &
-      $(table.used / table.len))
-    if isPrime(oldVal): echo("old val isPrime, should be a rare mem ordering event")
-    nextTable = resize(table)
-  if nextTable != nil:
-    #echo("tomb old slot then set in new table")
-    nextTable = copySlotAndCheck(table, idx)
-    return setVal(nextTable, key, val, expVal, match)
-  # Finally ready to add new val to table
-  while true:
-    if match and oldVal != expVal:
-      #echo("set failed, no match  oldVal= " & $oldVal & " expVal= " & $expVal)
-      return oldVal
-    if atomic_compare_exchange_n(table[idx].value.addr, oldVal.addr,
-        val, false, ATOMIC_RELEASE, ATOMIC_RELAXED):
-      #echo("val set at table " & $cast[int](table))
-      if expVal != 0:
-        if (oldVal == 0 or isTomb(oldVal)) and not isTomb(val):
-          discard atomic_add_fetch(table.active.addr, 1, ATOMIC_RELAXED)
-        elif not (oldVal == 0 or isTomb(oldVal)) and isTomb(val):
-          discard atomic_add_fetch(table.active.addr, -1, ATOMIC_RELAXED)
-      if oldVal == 0 and expVal != 0:
-        return setTomb(oldVal)
-      else: return oldVal
-    if isPrime(oldVal):
-      nextTable = copySlotAndCheck(table, idx)
-      return setVal(nextTable, key, val, expVal, match)
-
-#------------------------------------------------------------------------------
-
-proc getVal[K, V](table: var PConcTable[K, V], key: int): int =
-  #echo("-try get-  key = " & $key)
-  when K is Raw:
-    var idx = hashInt(key)
-  else:
-    var idx = popPtr[K](key)[].hash
-    #echo("get idx ", idx)
-  var
-    probes = 0
-    val: int
-  while true:
-    idx = idx and (table.len - 1)
-    var
-      newTable: PConcTable[K, V] # = atomic_load_n(table.next.addr, ATOMIC_ACQUIRE)
-      probedKey = atomic_load_n(table[idx].key.addr, ATOMIC_SEQ_CST)
-    if keyEQ[K](probedKey, key):
-      #echo("found key after ", probes+1)
-      val = atomic_load_n(table[idx].value.addr, ATOMIC_ACQUIRE)
-      if not isPrime(val):
-        if isTomb(val):
-          #echo("val was tomb but not prime")
-          return 0
-        else:
-          #echo("-GotIt- idx = ", idx, " key = ", key, " val ", val )
-          return val
-      else:
-        newTable = copySlotAndCheck(table, idx)
-        return getVal(newTable, key)
-    else:
-      #echo("probe ", probes, " idx = ", idx, " key = ", key, " found ", probedKey )
-      if probes >= reProbeLimit*4 or key.isTomb:
-        if newTable == nil:
-          #echo("too many probes and no new table ", key, "  ", idx )
-          return 0
-        else:
-          newTable = helpCopy(table)
-          return getVal(newTable, key)
-      idx += 1
-      probes += 1
-
-#------------------------------------------------------------------------------
-
-#proc set*(table: var PConcTable[Raw,Raw], key: Raw, val: Raw) =
-#  discard setVal(table, pack(key), pack(key), 0, false)
-
-#proc set*[V](table: var PConcTable[Raw,V], key: Raw, val: ptr V) =
-#  discard setVal(table, pack(key), cast[int](val), 0, false)
-
-proc set*[K, V](table: var PConcTable[K, V], key: var K, val: var V) =
-  when not (K is Raw):
-    var newKey = cast[int](copyShared(key))
-  else:
-    var newKey = pack(key)
-  when not (V is Raw):
-    var newVal = cast[int](copyShared(val))
-  else:
-    var newVal = pack(val)
-  var oldPtr = pop(setVal(table, newKey, newVal, 0, false))
-    #echo("oldPtr = ", cast[int](oldPtr), " newPtr = ", cast[int](newPtr))
-  when not (V is Raw):
-    if newVal != oldPtr and oldPtr != 0:
-      deallocShared(cast[ptr V](oldPtr))
-
-
-
-proc get*[K, V](table: var PConcTable[K, V], key: var K): V =
-  when not (V is Raw):
-    when not (K is Raw):
-      return popPtr[V](getVal(table, cast[int](key.addr)))[]
-    else:
-      return popPtr[V](getVal(table, pack(key)))[]
-  else:
-    when not (K is Raw):
-      return popRaw(getVal(table, cast[int](key.addr)))
-    else:
-      return popRaw(getVal(table, pack(key)))
-
-
-
-
-
-
-
-
-
-
-
-#proc `[]`[K,V](table: var PConcTable[K,V], key: K): PEntry[K,V] {.inline.} =
-#  getVal(table, key)
-
-#proc `[]=`[K,V](table: var PConcTable[K,V], key: K, val: V): PEntry[K,V] {.inline.} =
-#  setVal(table, key, val)
-
-
-
-
-
-
-#Tests ----------------------------
-when not defined(testing) and isMainModule:
-  import locks, times, mersenne
-
-  const
-    numTests = 100000
-    numThreads = 10
-
-
-
-  type
-    TestObj = tuple
-      thr: int
-      f0: int
-      f1: int
-
-    Data = tuple[k: string, v: TestObj]
-    PDataArr = array[0..numTests-1, Data]
-    Dict = PConcTable[string, TestObj]
-
-  var
-    thr: array[0..numThreads-1, Thread[Dict]]
-
-    table = newLFTable[string, TestObj](8)
-    rand = newMersenneTwister(2525)
-
-  proc createSampleData(len: int): PDataArr =
-    #result = cast[PDataArr](allocShared0(sizeof(Data)*numTests))
-    for i in 0..len-1:
-      result[i].k = "mark" & $(i+1)
-      #echo("mark" & $(i+1), " ", hash("mark" & $(i+1)))
-      result[i].v.thr = 0
-      result[i].v.f0 = i+1
-      result[i].v.f1 = 0
-      #echo("key = " & $(i+1) & " Val ptr = " & $cast[int](result[i].v.addr))
-
-
-
-  proc threadProc(tp: Dict) {.thread.} =
-    var t = cpuTime();
-    for i in 1..numTests:
-      var key = "mark" & $(i)
-      var got = table.get(key)
-      got.thr = cast[int](myThreadID[pointer]())
-      got.f1 = got.f1 + 1
-      table.set(key, got)
-    t = cpuTime() - t
-    echo t
-
-
-  var testData = createSampleData(numTests)
-
-  for i in 0..numTests-1:
-    table.set(testData[i].k, testData[i].v)
-
-  var i = 0
-  while i < numThreads:
-    createThread(thr[i], threadProc, table)
-    i += 1
-
-  joinThreads(thr)
-
-
-
-
-
-  var fails = 0
-
-  for i in 0..numTests-1:
-    var got = table.get(testData[i].k)
-    if got.f0 != i+1 or got.f1 != numThreads:
-      fails += 1
-      echo(got)
-
-  echo("Failed read or write = ", fails)
-
-
-  #for i in 1..numTests:
-  #  echo(i, " = ", hashInt(i) and 8191)
-
-  deleteConcTable(table)
diff --git a/lib/deprecated/pure/events.nim b/lib/deprecated/pure/events.nim
deleted file mode 100644
index 218cb45d57..0000000000
--- a/lib/deprecated/pure/events.nim
+++ /dev/null
@@ -1,102 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2011 Alexander Mitchell-Robinson
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## :Author: Alexander Mitchell-Robinson
-##
-## Unstable API.
-##
-## This module implements an event system that is not dependent on external
-## graphical toolkits. It was originally called `NimEE` because
-## it was inspired by Python's PyEE module. There are two ways you can use
-## events: one is a python-inspired way; the other is more of a C-style way.
-##
-## .. code-block:: Nim
-##    var ee = initEventEmitter()
-##    var genericargs: EventArgs
-##    proc handleevent(e: EventArgs) =
-##        echo("Handled!")
-##
-##    # Python way
-##    ee.on("EventName", handleevent)
-##    ee.emit("EventName", genericargs)
-##
-##    # C/Java way
-##    # Declare a type
-##    type
-##        SomeObject = object of RootObj
-##            SomeEvent: EventHandler
-##    var myobj: SomeObject
-##    myobj.SomeEvent = initEventHandler("SomeEvent")
-##    myobj.SomeEvent.addHandler(handleevent)
-##    ee.emit(myobj.SomeEvent, genericargs)
-
-type
-  EventArgs* = object of RootObj ## Base object for event arguments that are passed to callback functions.
-  EventHandler* = tuple[name: string, handlers: seq[proc(e: EventArgs) {.closure.}]] ## An eventhandler for an event.
-
-type
-  EventEmitter* = object ## An object that fires events and holds event handlers for an object.
-    s: seq[EventHandler]
-  EventError* = object of ValueError
-
-proc initEventHandler*(name: string): EventHandler =
-  ## Initializes an EventHandler with the specified name and returns it.
-  result.handlers = @[]
-  result.name = name
-
-proc addHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) =
-  ## Adds the callback to the specified event handler.
-  handler.handlers.add(fn)
-
-proc removeHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) =
-  ## Removes the callback from the specified event handler.
-  for i in countup(0, len(handler.handlers)-1):
-    if fn == handler.handlers[i]:
-      handler.handlers.del(i)
-      break
-
-proc containsHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}): bool =
-  ## Checks if a callback is registered to this event handler.
-  return handler.handlers.contains(fn)
-
-
-proc clearHandlers*(handler: var EventHandler) =
-  ## Clears all of the callbacks from the event handler.
-  setLen(handler.handlers, 0)
-
-proc getEventHandler(emitter: var EventEmitter, event: string): int =
-  for k in 0..high(emitter.s):
-    if emitter.s[k].name == event: return k
-  return -1
-
-proc on*(emitter: var EventEmitter, event: string, fn: proc(e: EventArgs) {.closure.}) =
-  ## Assigns a event handler with the specified callback. If the event
-  ## doesn't exist, it will be created.
-  var i = getEventHandler(emitter, event)
-  if i < 0:
-    var eh = initEventHandler(event)
-    addHandler(eh, fn)
-    emitter.s.add(eh)
-  else:
-    addHandler(emitter.s[i], fn)
-
-proc emit*(emitter: var EventEmitter, eventhandler: var EventHandler,
-           args: EventArgs) =
-  ## Fires an event handler with specified event arguments.
-  for fn in items(eventhandler.handlers): fn(args)
-
-proc emit*(emitter: var EventEmitter, event: string, args: EventArgs) =
-  ## Fires an event handler with specified event arguments.
-  var i = getEventHandler(emitter, event)
-  if i >= 0:
-    emit(emitter, emitter.s[i], args)
-
-proc initEventEmitter*(): EventEmitter =
-  ## Creates and returns a new EventEmitter.
-  result.s = @[]
diff --git a/lib/deprecated/pure/future.nim b/lib/deprecated/pure/future.nim
new file mode 100644
index 0000000000..0e06161f20
--- /dev/null
+++ b/lib/deprecated/pure/future.nim
@@ -0,0 +1,6 @@
+## This module is a deprecated alias for the `sugar` module. Deprecated since 0.19.0.
+
+{.deprecated: "Use the new 'sugar' module instead".}
+
+import std/sugar
+export sugar
diff --git a/lib/pure/mersenne.nim b/lib/deprecated/pure/mersenne.nim
similarity index 94%
rename from lib/pure/mersenne.nim
rename to lib/deprecated/pure/mersenne.nim
index 6778e2d627..37c5085b1d 100644
--- a/lib/pure/mersenne.nim
+++ b/lib/deprecated/pure/mersenne.nim
@@ -2,24 +2,20 @@
 #
 #            Nim's Runtime Library
 #        (c) Copyright 2015 Nim Contributors
-#
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
-#
-
 ## The [Mersenne Twister](https://en.wikipedia.org/wiki/Mersenne_Twister)
 ## random number generator.
-##
-## **Note:** The procs in this module work at compile-time.
+## .. note:: The procs in this module work at compile-time.
+
+{.deprecated: "use `std/random` instead".}
 
 runnableExamples:
   var rand = newMersenneTwister(uint32.high)  ## must be "var"
   doAssert rand.getNum() != rand.getNum()  ## pseudorandom number
-
 ## See also
 ## ========
 ## * `random module`_ for Nim's standard random number generator
-
 type
   MersenneTwister* = object
     ## The Mersenne Twister.
@@ -35,6 +31,7 @@ proc newMersenneTwister*(seed: uint32): MersenneTwister =
                                       (result.mt[i-1] shr 30'u32)) + i)
 
 proc generateNumbers(m: var MersenneTwister) =
+
   for i in 0..623:
     var y = (m.mt[i] and 0x80000000'u32) +
             (m.mt[(i+1) mod 624] and 0x7fffffff'u32)
@@ -48,7 +45,6 @@ proc getNum*(m: var MersenneTwister): uint32 =
     generateNumbers(m)
   result = m.mt[m.index]
   m.index = (m.index + 1) mod m.mt.len
-
   result = result xor (result shr 11'u32)
   result = result xor ((result shl 7'u32) and 0x9d2c5680'u32)
   result = result xor ((result shl 15'u32) and 0xefc60000'u32)
diff --git a/lib/deprecated/pure/ospaths.nim b/lib/deprecated/pure/ospaths.nim
index 6c7fe4fb30..43fcb17ccf 100644
--- a/lib/deprecated/pure/ospaths.nim
+++ b/lib/deprecated/pure/ospaths.nim
@@ -7,10 +7,11 @@
 #    distribution, for details about the copyright.
 #
 
-## This module is deprecated, `import os` instead.
-{.deprecated: "import os.nim instead".}
+## This module is deprecated since 0.20.0, `import std/os` instead.
 
-import os
+{.deprecated: "use `std/os` instead".}
+
+import std/os
 export ReadEnvEffect, WriteEnvEffect, ReadDirEffect, WriteDirEffect, OSErrorCode,
   doslikeFileSystem, CurDir, ParDir, DirSep, AltSep, PathSep, FileSystemCaseSensitive,
   ExeExt, ScriptExt, DynlibFormat, ExtSep, joinPath, `/`, splitPath, parentDir,
diff --git a/lib/pure/oswalkdir.nim b/lib/deprecated/pure/oswalkdir.nim
similarity index 67%
rename from lib/pure/oswalkdir.nim
rename to lib/deprecated/pure/oswalkdir.nim
index 866f9ed702..57a2cb81d7 100644
--- a/lib/pure/oswalkdir.nim
+++ b/lib/deprecated/pure/oswalkdir.nim
@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## This module is deprecated, `import os` instead.
-{.deprecated: "import os.nim instead".}
-import os
+## This module is deprecated, `import std/os` instead.
+{.deprecated: "import 'std/os' instead".}
+import std/os
 export PathComponent, walkDir, walkDirRec
diff --git a/lib/deprecated/pure/parseopt2.nim b/lib/deprecated/pure/parseopt2.nim
deleted file mode 100644
index a6d5de06df..0000000000
--- a/lib/deprecated/pure/parseopt2.nim
+++ /dev/null
@@ -1,155 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module provides the standard Nim command line parser.
-## It supports one convenience iterator over all command line options and some
-## lower-level features.
-##
-## Supported syntax:
-##
-## 1. short options - `-abcd`, where a, b, c, d are names
-## 2. long option - `--foo:bar`, `--foo=bar` or `--foo`
-## 3. argument - everything else
-
-{.deprecated: "Use the 'parseopt' module instead".}
-{.push debugger: off.}
-
-include "system/inclrtl"
-
-import
-  os, strutils
-
-type
-  CmdLineKind* = enum         ## the detected command line token
-    cmdEnd,                   ## end of command line reached
-    cmdArgument,              ## argument detected
-    cmdLongOption,            ## a long option `--option` detected
-    cmdShortOption            ## a short option `-c` detected
-  OptParser* =
-      object of RootObj ## this object implements the command line parser
-    cmd: seq[string]
-    pos: int
-    remainingShortOptions: string
-    kind*: CmdLineKind        ## the detected command line token
-    key*, val*: string        ## key and value pair; `key` is the option
-                              ## or the argument, `value` is not "" if
-                              ## the option was given a value
-
-proc initOptParser*(cmdline: seq[string]): OptParser {.rtl.} =
-  ## Initializes option parses with cmdline. cmdline should not contain
-  ## argument 0 - program name.
-  ## If cmdline.len == 0 default to current command line arguments.
-  result.remainingShortOptions = ""
-  when not defined(createNimRtl):
-    if cmdline.len == 0:
-      result.cmd = commandLineParams()
-      return
-  else:
-    assert cmdline != nil, "Cannot determine command line arguments."
-
-  result.cmd = @cmdline
-
-when not defined(createNimRtl):
-  proc initOptParser*(): OptParser =
-    ## Initializes option parser from current command line arguments.
-    return initOptParser(commandLineParams())
-
-proc next*(p: var OptParser) {.rtl, extern: "npo2$1".}
-
-proc nextOption(p: var OptParser, token: string, allowEmpty: bool) =
-  for splitchar in [':', '=']:
-    if splitchar in token:
-      let pos = token.find(splitchar)
-      p.key = token[0..pos-1]
-      p.val = token[pos+1..token.len-1]
-      return
-
-  p.key = token
-  if allowEmpty:
-    p.val = ""
-  else:
-    p.remainingShortOptions = token[0..token.len-1]
-    p.next()
-
-proc next(p: var OptParser) =
-  if p.remainingShortOptions.len != 0:
-    p.kind = cmdShortOption
-    p.key = p.remainingShortOptions[0..0]
-    p.val = ""
-    p.remainingShortOptions = p.remainingShortOptions[1..p.remainingShortOptions.len-1]
-    return
-
-  if p.pos >= p.cmd.len:
-    p.kind = cmdEnd
-    return
-
-  let token = p.cmd[p.pos]
-  p.pos += 1
-
-  if token.startsWith("--"):
-    p.kind = cmdLongOption
-    nextOption(p, token[2..token.len-1], allowEmpty=true)
-  elif token.startsWith("-"):
-    p.kind = cmdShortOption
-    nextOption(p, token[1..token.len-1], allowEmpty=true)
-  else:
-    p.kind = cmdArgument
-    p.key = token
-    p.val = ""
-
-proc cmdLineRest*(p: OptParser): string {.rtl, extern: "npo2$1".} =
-  ## Returns the part of command line string that has not been parsed yet,
-  ## properly quoted.
-  return p.cmd[p.pos..p.cmd.len-1].quoteShellCommand
-
-type
-  GetoptResult* = tuple[kind: CmdLineKind, key, val: string]
-
-iterator getopt*(p: var OptParser): GetoptResult =
-  ## This is an convenience iterator for iterating over the given OptParser object.
-  ## Example:
-  ##
-  ## .. code-block:: nim
-  ##   var p = initOptParser("--left --debug:3 -l=4 -r:2")
-  ##   for kind, key, val in p.getopt():
-  ##     case kind
-  ##     of cmdArgument:
-  ##       filename = key
-  ##     of cmdLongOption, cmdShortOption:
-  ##       case key
-  ##       of "help", "h": writeHelp()
-  ##       of "version", "v": writeVersion()
-  ##     of cmdEnd: assert(false) # cannot happen
-  ##   if filename == "":
-  ##     # no filename has been given, so we show the help:
-  ##     writeHelp()
-  p.pos = 0
-  while true:
-    next(p)
-    if p.kind == cmdEnd: break
-    yield (p.kind, p.key, p.val)
-
-when declared(paramCount):
-  iterator getopt*(): GetoptResult =
-    ## This is an convenience iterator for iterating over the command line arguments.
-    ## This create a new OptParser object.
-    ## See above for a more detailed example
-    ##
-    ## .. code-block:: nim
-    ##   for kind, key, val in getopt():
-    ##     # this will iterate over all arguments passed to the cmdline.
-    ##     continue
-    ##
-    var p = initOptParser()
-    while true:
-      next(p)
-      if p.kind == cmdEnd: break
-      yield (p.kind, p.key, p.val)
-
-{.pop.}
diff --git a/lib/deprecated/pure/securehash.nim b/lib/deprecated/pure/securehash.nim
deleted file mode 100644
index 2f4530d889..0000000000
--- a/lib/deprecated/pure/securehash.nim
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-## This module is a deprecated alias for the `sha1` module.
-{.deprecated.}
-
-include "../std/sha1"
diff --git a/lib/deprecated/pure/sharedstrings.nim b/lib/deprecated/pure/sharedstrings.nim
deleted file mode 100644
index 29cb0fcc25..0000000000
--- a/lib/deprecated/pure/sharedstrings.nim
+++ /dev/null
@@ -1,152 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Shared string support for Nim.
-
-type
-  UncheckedCharArray = UncheckedArray[char]
-
-type
-  Buffer = ptr object
-    refcount: int
-    capacity, realLen: int
-    data: UncheckedCharArray
-
-  SharedString* = object ## A string that can be shared. Slicing is O(1).
-    buffer: Buffer
-    first, len: int
-
-proc decRef(b: Buffer) {.inline.} =
-  if atomicDec(b.refcount) <= 0:
-    deallocShared(b)
-
-proc incRef(b: Buffer) {.inline.} =
-  atomicInc(b.refcount)
-
-{.experimental.}
-
-proc `=destroy`*(s: SharedString) =
-  #echo "destroyed"
-  if not s.buffer.isNil:
-    decRef(s.buffer)
-
-when false:
-  proc `=`*(dest: var SharedString; src: SharedString) =
-    incRef(src.buffer)
-    if not dest.buffer.isNil:
-      decRef(dest.buffer)
-    dest.buffer = src.buffer
-    dest.first = src.first
-    dest.len = src.len
-
-proc len*(s: SharedString): int = s.len
-
-proc `[]`*(s: SharedString; i: Natural): char =
-  if i < s.len: result = s.buffer.data[i+s.first]
-  else: raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1))
-
-proc `[]=`*(s: var SharedString; i: Natural; value: char) =
-  if i < s.len: s.buffer.data[i+s.first] = value
-  else: raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1))
-
-proc `[]`*(s: SharedString; ab: HSlice[int, int]): SharedString =
-  #incRef(src.buffer)
-  if ab.a < s.len:
-    result.buffer = s.buffer
-    result.first = ab.a
-    result.len = min(s.len, ab.b - ab.a + 1)
-  # else: produce empty string ;-)
-
-proc newBuffer(cap, len: int): Buffer =
-  assert cap >= len
-  result = cast[Buffer](allocShared0(sizeof(int)*3 + cap))
-  result.refcount = 0
-  result.capacity = cap
-  result.realLen = len
-
-proc newSharedString*(len: Natural): SharedString =
-  if len != 0:
-    # optimization: Don't have an underlying buffer when 'len == 0'
-    result.buffer = newBuffer(len, len)
-  result.first = 0
-  result.len = len
-
-proc newSharedString*(s: string): SharedString =
-  let len = s.len
-  if len != 0:
-    # optimization: Don't have an underlying buffer when 'len == 0'
-    result.buffer = newBuffer(len, len)
-    copyMem(addr result.buffer.data[0], cstring(s), s.len)
-  result.first = 0
-  result.len = len
-
-when declared(atomicLoadN):
-  template load(x): untyped = atomicLoadN(addr x, ATOMIC_SEQ_CST)
-else:
-  # XXX Fixme
-  template load(x): untyped = x
-
-proc add*(s: var SharedString; t: cstring; len: Natural) =
-  if len == 0: return
-  let newLen = s.len + len
-  if s.buffer.isNil:
-    s.buffer = newBuffer(len, len)
-    copyMem(addr s.buffer.data[0], t, len)
-    s.len = len
-  elif newLen >= s.buffer.capacity or s.first != 0 or
-      s.len != s.buffer.realLen or load(s.buffer.refcount) > 1:
-    let oldBuf = s.buffer
-    s.buffer = newBuffer(max(s.buffer.capacity * 3 div 2, newLen), newLen)
-    copyMem(addr s.buffer.data[0], addr oldBuf.data[s.first], s.len)
-    copyMem(addr s.buffer.data[s.len], t, len)
-    decRef(oldBuf)
-  else:
-    copyMem(addr s.buffer.data[s.len], t, len)
-    s.buffer.realLen += len
-    s.len += len
-
-proc add*(s: var SharedString; t: string) =
-  s.add(t.cstring, t.len)
-
-proc rawData*(s: var SharedString): pointer =
-  if s.buffer.isNil: result = nil
-  else: result = addr s.buffer.data[s.first]
-
-proc add*(s: var SharedString; t: SharedString) =
-  if t.buffer.isNil: return
-  s.add(cast[cstring](addr s.buffer.data[s.first]), t.len)
-
-proc `$`*(s: SharedString): string =
-  result = newString(s.len)
-  if s.len > 0:
-    copyMem(addr result[0], addr s.buffer.data[s.first], s.len)
-
-proc `==`*(s: SharedString; t: string): bool =
-  if s.buffer.isNil: result = t.len == 0
-  else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first],
-                                             cstring(t), t.len)
-
-proc `==`*(s, t: SharedString): bool =
-  if s.buffer.isNil: result = t.len == 0
-  else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first],
-                                             addr t.buffer.data[t.first], t.len)
-
-iterator items*(s: SharedString): char =
-  let buf = s.buffer.data
-  let x = s.first
-  if buf != nil:
-    for i in 0..`_
 
+runnableExamples:
+  assert diffInt(
+    [0, 1, 2, 3, 4, 5, 6, 7, 8],
+    [-1, 1, 2, 3, 4, 5, 666, 7, 42]) ==
+    @[Item(startA: 0, startB: 0, deletedA: 1, insertedB: 1),
+      Item(startA: 6, startB: 6, deletedA: 1, insertedB: 1),
+      Item(startA: 8, startB: 8, deletedA: 1, insertedB: 1)]
+
+runnableExamples:
+  # 2 samples of text (from "The Call of Cthulhu" by Lovecraft)
+  let txt0 = """
+abc
+def ghi
+jkl2"""
+  let txt1 = """
+bacx
+abc
+def ghi
+jkl"""
+  assert diffText(txt0, txt1) ==
+    @[Item(startA: 0, startB: 0, deletedA: 0, insertedB: 1),
+      Item(startA: 2, startB: 3, deletedA: 1, insertedB: 1)]
+
 # code owner: Arne Döring
 #
 # This is based on C# code written by Matthias Hertel, http://www.mathertel.de
@@ -40,7 +43,10 @@
 # "An O(ND) Difference Algorithm and its Variations" by Eugene Myers
 # Algorithmica Vol. 1 No. 2, 1986, p 251.
 
-import tables, strutils
+import std/[tables, strutils]
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 type
   Item* = object    ## An Item in the list of differences.
@@ -288,9 +294,9 @@ proc diffInt*(arrayA, arrayB: openArray[int]): seq[Item] =
   var dataB = newDiffData(@arrayB, arrayB.len)
 
   let max = dataA.len + dataB.len + 1
-  ## vector for the (0,0) to (x,y) search
+  # vector for the (0,0) to (x,y) search
   var downVector = newSeq[int](2 * max + 2)
-  ## vector for the (u,v) to (N,M) search
+  # vector for the (u,v) to (N,M) search
   var upVector = newSeq[int](2 * max + 2)
 
   lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector)
@@ -309,7 +315,7 @@ proc diffText*(textA, textB: string): seq[Item] =
   ## `textB` B-version of the text (usually the new one)
   ##
   ## Returns a seq of Items that describe the differences.
-
+  # See also `gitutils.diffStrings`.
   # prepare the input-text and convert to comparable numbers.
   var h = initTable[string, int]()  # TextA.len + TextB.len  <- probably wrong initial size
   # The A-Version of the data (original data) to be compared.
@@ -321,9 +327,9 @@ proc diffText*(textA, textB: string): seq[Item] =
   h.clear # free up hashtable memory (maybe)
 
   let max = dataA.len + dataB.len + 1
-  ## vector for the (0,0) to (x,y) search
+  # vector for the (0,0) to (x,y) search
   var downVector = newSeq[int](2 * max + 2)
-  ## vector for the (u,v) to (N,M) search
+  # vector for the (u,v) to (N,M) search
   var upVector = newSeq[int](2 * max + 2)
 
   lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector)
diff --git a/lib/genode/alloc.nim b/lib/genode/alloc.nim
index a21a3ad7bd..24fb9954ef 100644
--- a/lib/genode/alloc.nim
+++ b/lib/genode/alloc.nim
@@ -15,20 +15,20 @@ when not defined(genode):
   {.error: "Genode only module".}
 
 when not declared(GenodeEnv):
-  include genode/env
+  import genode/env
 
-type DataspaceCapability {.
-  importcpp: "Genode::Dataspace_capability", pure.} = object
+type RamDataspaceCapability {.
+  importcpp: "Genode::Ram_dataspace_capability", pure.} = object
 
 type
   Map = object
     attachment: pointer
     size: int
-    ds: DataspaceCapability
+    ds: RamDataspaceCapability
 
   SlabMeta = object
     next: ptr MapSlab
-    ds: DataspaceCapability
+    ds: RamDataspaceCapability
 
   MapSlab = object
     meta: SlabMeta
@@ -45,11 +45,11 @@ proc capsAvail(env: GenodeEnv): int {.
   ## Return the number of available capabilities.
   ## Each dataspace allocation consumes a capability.
 
-proc allocDataspace(env: GenodeEnv; size: int): DataspaceCapability {.
+proc allocDataspace(env: GenodeEnv; size: int): RamDataspaceCapability {.
   importcpp: "#->pd().alloc(@)".}
   ## Allocate a dataspace and its capability.
 
-proc attachDataspace(env: GenodeEnv; ds: DataspaceCapability): pointer {.
+proc attachDataspace(env: GenodeEnv; ds: RamDataspaceCapability): pointer {.
   importcpp: "#->rm().attach(@)".}
   ## Attach a dataspace into the component address-space.
 
@@ -57,7 +57,7 @@ proc detachAddress(env: GenodeEnv; p: pointer) {.
   importcpp: "#->rm().detach(@)".}
   ## Detach a dataspace from the component address-space.
 
-proc freeDataspace(env: GenodeEnv; ds: DataspaceCapability) {.
+proc freeDataspace(env: GenodeEnv; ds: RamDataspaceCapability) {.
   importcpp: "#->pd().free(@)".}
   ## Free a dataspace.
 
@@ -111,7 +111,7 @@ proc osDeallocPages(p: pointer; size: int) =
       if m.attachment == p:
         if m.size != size:
           echo "cannot partially detach dataspace"
-          quit -1
+          rawQuit -1
         runtimeEnv.detachAddress m.attachment
         runtimeEnv.freeDataspace m.ds
         m[] = Map()
diff --git a/lib/genode/constructibles.nim b/lib/genode/constructibles.nim
new file mode 100644
index 0000000000..3a4a646e08
--- /dev/null
+++ b/lib/genode/constructibles.nim
@@ -0,0 +1,21 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2022 Emery Hemingway
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+type Constructible*[T] {.
+  importcpp: "Genode::Constructible",
+  header: "", byref, pure.} = object
+
+proc construct*[T](x: Constructible[T]) {.importcpp.}
+  ## Construct a constructible C++ object.
+
+proc destruct*[T](x: Constructible[T]) {.importcpp.}
+  ## Destruct a constructible C++ object.
+
+proc constructed*[T](x: Constructible[T]): bool {.importcpp.}
+  ## Test if an object is constructed.
diff --git a/lib/genode/entrypoints.nim b/lib/genode/entrypoints.nim
new file mode 100644
index 0000000000..0bf5e0e0ec
--- /dev/null
+++ b/lib/genode/entrypoints.nim
@@ -0,0 +1,22 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2022 Emery Hemingway
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## See `Genode Foundations - Entrypoint `
+## for a description of Entrypoints.
+
+type
+  EntrypointObj {.
+    importcpp: "Genode::Entrypoint",
+    header: "",
+    pure.} = object
+  Entrypoint* = ptr EntrypointObj
+    ## Opaque Entrypoint object.
+
+proc ep*(env: GenodeEnv): Entrypoint {.importcpp: "(&#->ep())".}
+  ## Access the entrypoint associated with `env`.
diff --git a/lib/genode/env.nim b/lib/genode/env.nim
index ef4a258830..babe2a8a0e 100644
--- a/lib/genode/env.nim
+++ b/lib/genode/env.nim
@@ -17,13 +17,13 @@
 #
 
 when not defined(genode):
-  {.error: "Genode only include".}
+  {.error: "Genode only module".}
 
 type
-  GenodeEnvObj {.importcpp: "Genode::Env", header: "", pure.} = object
-  GenodeEnvPtr = ptr GenodeEnvObj
+  GenodeEnvObj* {.importcpp: "Genode::Env", header: "", pure.} = object
+  GenodeEnvPtr* = ptr GenodeEnvObj
 
-const runtimeEnvSym = "nim_runtime_env"
+const runtimeEnvSym* = "nim_runtime_env"
 
 when not defined(nimscript):
-  var runtimeEnv {.importcpp: runtimeEnvSym.}: GenodeEnvPtr
+  var runtimeEnv* {.importcpp: runtimeEnvSym.}: GenodeEnvPtr
diff --git a/lib/genode/signals.nim b/lib/genode/signals.nim
new file mode 100644
index 0000000000..7d18757303
--- /dev/null
+++ b/lib/genode/signals.nim
@@ -0,0 +1,77 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2022 Emery Hemingway
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## See `Genode Foundations - Asynchronous notifications `
+## for a description of Genode signals.
+
+when not defined(genode) or defined(nimdoc):
+  {.error: "Genode only module".}
+
+import ./entrypoints, ./constructibles
+
+export ep # Entrypoint accessor on GenodeEnv
+
+type
+  SignalContextCapability* {.
+    importcpp: "Genode::Signal_context_capability",
+    header: "", pure.} = object
+    ## Capability to an asynchronous signal context.
+
+proc isValid*(cap: SignalContextCapability): bool {.importcpp: "#.valid()".}
+  ## Call the Genode core to check if this `SignalContextCapability` is valid.
+  # TODO: RpcEffect
+
+type
+  HandlerProc = proc () {.closure, gcsafe.}
+
+  SignalHandlerBase {.
+    importcpp: "Nim::SignalHandler",
+    header: "genode_cpp/signals.h",
+    pure.} = object
+
+  SignalHandlerCpp = Constructible[SignalHandlerBase]
+
+  SignalHandlerObj = object
+    cpp: SignalHandlerCpp
+    cb: HandlerProc
+      ## Signal handling procedure called during dispatch.
+
+  SignalHandler* = ref SignalHandlerObj
+    ## Nim object enclosing a Genode signal handler.
+
+proc construct(cpp: SignalHandlerCpp; ep: Entrypoint; sh: SignalHandler) {.importcpp.}
+
+proc cap(cpp: SignalHandlerCpp): SignalContextCapability {.importcpp: "#->cap()".}
+
+proc newSignalHandler*(ep: Entrypoint; cb: HandlerProc): SignalHandler =
+  ## Create a new signal handler. A label is recommended for
+  ## debugging purposes. A signal handler will not be garbage
+  ## collected until after it has been dissolved.
+  result = SignalHandler(cb: cb)
+  result.cpp.construct(ep, result)
+  GCref result
+
+proc dissolve*(sig: SignalHandler) =
+  ## Dissolve signal dispatcher from entrypoint.
+  # TODO: =destroy?
+  destruct sig.cpp
+  sig.cb = nil # lose the callback
+  GCunref sig
+
+proc cap*(sig: SignalHandler): SignalContextCapability =
+  ## Signal context capability. Can be delegated to external components.
+  sig.cpp.cap
+
+proc submit*(cap: SignalContextCapability) {.
+    importcpp: "Genode::Signal_transmitter(#).submit()".}
+  ## Submit a signal to a context capability.
+
+proc nimHandleSignal(p: pointer) {.exportc.} =
+  ## C symbol invoked by entrypoint during signal dispatch.
+  cast[SignalHandler](p).cb()
diff --git a/lib/genode_cpp/signals.h b/lib/genode_cpp/signals.h
new file mode 100644
index 0000000000..fa3975d386
--- /dev/null
+++ b/lib/genode_cpp/signals.h
@@ -0,0 +1,39 @@
+/*
+ *
+ *           Nim's Runtime Library
+ *       (c) Copyright 2022 Emery Hemingway
+ *
+ *   See the file "copying.txt", included in this
+ *   distribution, for details about the copyright.
+ *
+ */
+
+#ifndef _NIM_SIGNALS_H_
+#define _NIM_SIGNALS_H_
+
+#include 
+#include 
+#include 
+
+// Symbol for calling back into Nim
+extern "C" void nimHandleSignal(void *arg);
+
+namespace Nim { struct SignalHandler; }
+
+struct Nim::SignalHandler
+{
+	// Pointer to the Nim handler object.
+	void *arg;
+
+	void handle_signal() {
+		Libc::with_libc([this] () { nimHandleSignal(arg); }); }
+
+	Genode::Signal_handler handler;
+
+	SignalHandler(Genode::Entrypoint *ep, void *arg)
+	: arg(arg), handler(*ep, *this, &SignalHandler::handle_signal) { }
+
+	Genode::Signal_context_capability cap() { return handler; }
+};
+
+#endif
diff --git a/lib/genode_cpp/syslocks.h b/lib/genode_cpp/syslocks.h
index 8ba39abc21..b5d5ae6943 100644
--- a/lib/genode_cpp/syslocks.h
+++ b/lib/genode_cpp/syslocks.h
@@ -13,7 +13,7 @@
 
 /* Genode includes */
 #include 
-#include 
+#include 
 
 namespace Nim {
 	struct SysLock;
@@ -22,15 +22,14 @@ namespace Nim {
 
 struct Nim::SysLock
 {
-	Genode::Lock _lock_a, _lock_b;
+	Genode::Mutex _mutex_a, _mutex_b;
 	bool         _locked;
 
 	void acquireSys()
 	{
-		_lock_a.lock();
+		Genode::Mutex::Guard guard(_mutex_a);
 		_locked = true;
-		_lock_a.unlock();
-		_lock_b.lock();
+		_mutex_b.acquire();
 	}
 
 	bool tryAcquireSys()
@@ -38,23 +37,22 @@ struct Nim::SysLock
 		if (_locked)
 			return false;
 
-		_lock_a.lock();
+		Genode::Mutex::Guard guard(_mutex_a);
+
 		if (_locked) {
-			_lock_a.unlock();
 			return false;
 		} else {
 			_locked = true;
-			_lock_b.lock();
-			_lock_a.unlock();
+			_mutex_b.acquire();
 			return true;
 		}
 	}
 
 	void releaseSys()
 	{
+		Genode::Mutex::Guard guard(_mutex_a);
 		_locked = false;
-		_lock_a.unlock();
-		_lock_b.unlock();
+		_mutex_b.release();
 	}
 };
 
@@ -73,6 +71,11 @@ struct Nim::SysCond
 	{
 		_semaphore.up();
 	}
+
+	void broadcastSysCond()
+	{
+		_semaphore.up();
+	}
 };
 
 #endif
diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim
deleted file mode 100644
index 242ea1b0d8..0000000000
--- a/lib/impure/db_mysql.nim
+++ /dev/null
@@ -1,424 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## A higher level `mySQL`:idx: database wrapper. The same interface is
-## implemented for other databases too.
-##
-## See also: `db_odbc `_, `db_sqlite `_,
-## `db_postgres `_.
-##
-## Parameter substitution
-## ======================
-##
-## All `db_*` modules support the same form of parameter substitution.
-## That is, using the `?` (question mark) to signify the place where a
-## value should be placed. For example:
-##
-## .. code-block:: Nim
-##     sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)"
-##
-##
-## Examples
-## ========
-##
-## Opening a connection to a database
-## ----------------------------------
-##
-## .. code-block:: Nim
-##     import std/db_mysql
-##     let db = open("localhost", "user", "password", "dbname")
-##     db.close()
-##
-## Creating a table
-## ----------------
-##
-## .. code-block:: Nim
-##      db.exec(sql"DROP TABLE IF EXISTS myTable")
-##      db.exec(sql("""CREATE TABLE myTable (
-##                       id integer,
-##                       name varchar(50) not null)"""))
-##
-## Inserting data
-## --------------
-##
-## .. code-block:: Nim
-##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
-##             "Dominik")
-##
-## Larger example
-## --------------
-##
-## .. code-block:: Nim
-##
-##  import std/[db_mysql, math]
-##
-##  let theDb = open("localhost", "nim", "nim", "test")
-##
-##  theDb.exec(sql"Drop table if exists myTestTbl")
-##  theDb.exec(sql("create table myTestTbl (" &
-##      " Id    INT(11)     NOT NULL AUTO_INCREMENT PRIMARY KEY, " &
-##      " Name  VARCHAR(50) NOT NULL, " &
-##      " i     INT(11), " &
-##      " f     DECIMAL(18,10))"))
-##
-##  theDb.exec(sql"START TRANSACTION")
-##  for i in 1..1000:
-##    theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
-##          "Item#" & $i, i, sqrt(i.float))
-##  theDb.exec(sql"COMMIT")
-##
-##  for x in theDb.fastRows(sql"select * from myTestTbl"):
-##    echo x
-##
-##  let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
-##          "Item#1001", 1001, sqrt(1001.0))
-##  echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id)
-##
-##  theDb.close()
-
-
-import strutils, mysql
-
-import db_common
-export db_common
-
-import std/private/since
-
-type
-  DbConn* = distinct PMySQL ## encapsulates a database connection
-  Row* = seq[string]   ## a row of a dataset. NULL database values will be
-                       ## converted to nil.
-  InstantRow* = object ## a handle that can be used to get a row's
-                       ## column text on demand
-    row: cstringArray
-    len: int
-
-proc dbError*(db: DbConn) {.noreturn.} =
-  ## raises a DbError exception.
-  var e: ref DbError
-  new(e)
-  e.msg = $mysql.error(PMySQL db)
-  raise e
-
-when false:
-  proc dbQueryOpt*(db: DbConn, query: string, args: varargs[string, `$`]) =
-    var stmt = mysql_stmt_init(db)
-    if stmt == nil: dbError(db)
-    if mysql_stmt_prepare(stmt, query, len(query)) != 0:
-      dbError(db)
-    var
-      binding: seq[MYSQL_BIND]
-    discard mysql_stmt_close(stmt)
-
-proc dbQuote*(s: string): string =
-  ## DB quotes the string.
-  result = newStringOfCap(s.len + 2)
-  result.add "'"
-  for c in items(s):
-    # see https://cheatsheetseries.owasp.org/cheatsheets/SQL_Injection_Prevention_Cheat_Sheet.html#mysql-escaping
-    case c
-    of '\0': result.add "\\0"
-    of '\b': result.add "\\b"
-    of '\t': result.add "\\t"
-    of '\l': result.add "\\n"
-    of '\r': result.add "\\r"
-    of '\x1a': result.add "\\Z"
-    of '"': result.add "\\\""
-    of '\'': result.add "\\'"
-    of '\\': result.add "\\\\"
-    of '_': result.add "\\_"
-    else: result.add c
-  add(result, '\'')
-
-proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
-  result = ""
-  var a = 0
-  for c in items(string(formatstr)):
-    if c == '?':
-      add(result, dbQuote(args[a]))
-      inc(a)
-    else:
-      add(result, c)
-
-proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
-  tags: [ReadDbEffect, WriteDbEffect].} =
-  ## tries to execute the query and returns true if successful, false otherwise.
-  var q = dbFormat(query, args)
-  return mysql.realQuery(PMySQL db, q, q.len) == 0'i32
-
-proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) =
-  var q = dbFormat(query, args)
-  if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db)
-
-proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-  tags: [ReadDbEffect, WriteDbEffect].} =
-  ## executes the query and raises EDB if not successful.
-  var q = dbFormat(query, args)
-  if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db)
-
-proc newRow(L: int): Row =
-  newSeq(result, L)
-  for i in 0..L-1: result[i] = ""
-
-proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) =
-  if row != nil:
-    while mysql.fetchRow(sqlres) != nil: discard
-  mysql.freeResult(sqlres)
-
-iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## executes the query and iterates over the result dataset.
-  ##
-  ## This is very fast, but potentially dangerous.  Use this iterator only
-  ## if you require **ALL** the rows.
-  ##
-  ## Breaking the fastRows() iterator during a loop will cause the next
-  ## database query to raise an [EDb] exception `Commands out of sync`.
-  rawExec(db, query, args)
-  var sqlres = mysql.useResult(PMySQL db)
-  if sqlres != nil:
-    var
-      L = int(mysql.numFields(sqlres))
-      row: cstringArray
-      result: Row
-      backup: Row
-    newSeq(result, L)
-    while true:
-      row = mysql.fetchRow(sqlres)
-      if row == nil: break
-      for i in 0..L-1:
-        setLen(result[i], 0)
-        result[i].add row[i]
-      yield result
-    properFreeResult(sqlres, row)
-
-iterator instantRows*(db: DbConn, query: SqlQuery,
-                      args: varargs[string, `$`]): InstantRow
-                      {.tags: [ReadDbEffect].} =
-  ## Same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the iterator body.
-  rawExec(db, query, args)
-  var sqlres = mysql.useResult(PMySQL db)
-  if sqlres != nil:
-    let L = int(mysql.numFields(sqlres))
-    var row: cstringArray
-    while true:
-      row = mysql.fetchRow(sqlres)
-      if row == nil: break
-      yield InstantRow(row: row, len: L)
-    properFreeResult(sqlres, row)
-
-proc setTypeName(t: var DbType; f: PFIELD) =
-  t.name = $f.name
-  t.maxReprLen = Natural(f.max_length)
-  if (NOT_NULL_FLAG and f.flags) != 0: t.notNull = true
-  case f.ftype
-  of TYPE_DECIMAL:
-    t.kind = dbDecimal
-  of TYPE_TINY:
-    t.kind = dbInt
-    t.size = 1
-  of TYPE_SHORT:
-    t.kind = dbInt
-    t.size = 2
-  of TYPE_LONG:
-    t.kind = dbInt
-    t.size = 4
-  of TYPE_FLOAT:
-    t.kind = dbFloat
-    t.size = 4
-  of TYPE_DOUBLE:
-    t.kind = dbFloat
-    t.size = 8
-  of TYPE_NULL:
-    t.kind = dbNull
-  of TYPE_TIMESTAMP:
-    t.kind = dbTimestamp
-  of TYPE_LONGLONG:
-    t.kind = dbInt
-    t.size = 8
-  of TYPE_INT24:
-    t.kind = dbInt
-    t.size = 3
-  of TYPE_DATE:
-    t.kind = dbDate
-  of TYPE_TIME:
-    t.kind = dbTime
-  of TYPE_DATETIME:
-    t.kind = dbDatetime
-  of TYPE_YEAR:
-    t.kind = dbDate
-  of TYPE_NEWDATE:
-    t.kind = dbDate
-  of TYPE_VARCHAR, TYPE_VAR_STRING, TYPE_STRING:
-    t.kind = dbVarchar
-  of TYPE_BIT:
-    t.kind = dbBit
-  of TYPE_NEWDECIMAL:
-    t.kind = dbDecimal
-  of TYPE_ENUM: t.kind = dbEnum
-  of TYPE_SET: t.kind = dbSet
-  of TYPE_TINY_BLOB, TYPE_MEDIUM_BLOB, TYPE_LONG_BLOB,
-     TYPE_BLOB: t.kind = dbBlob
-  of TYPE_GEOMETRY:
-    t.kind = dbGeometry
-
-proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) =
-  setLen(columns, L)
-  for i in 0..`_, `db_sqlite `_,
-## `db_mysql `_.
-##
-## Parameter substitution
-## ======================
-##
-## All `db_*` modules support the same form of parameter substitution.
-## That is, using the `?` (question mark) to signify the place where a
-## value should be placed. For example:
-##
-## .. code-block:: Nim
-##     sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)"
-##
-##
-## Examples
-## ========
-##
-## Opening a connection to a database
-## ----------------------------------
-##
-## .. code-block:: Nim
-##     import std/db_odbc
-##     var db = open("localhost", "user", "password", "dbname")
-##     db.close()
-##
-## Creating a table
-## ----------------
-##
-## .. code-block:: Nim
-##      db.exec(sql"DROP TABLE IF EXISTS myTable")
-##      db.exec(sql("""CREATE TABLE myTable (
-##                       id integer,
-##                       name varchar(50) not null)"""))
-##
-## Inserting data
-## --------------
-##
-## .. code-block:: Nim
-##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
-##             "Andreas")
-##
-## Large example
-## -------------
-##
-## .. code-block:: Nim
-##
-##  import std/[db_odbc, math]
-##
-##  var theDb = open("localhost", "nim", "nim", "test")
-##
-##  theDb.exec(sql"Drop table if exists myTestTbl")
-##  theDb.exec(sql("create table myTestTbl (" &
-##      " Id    INT(11)     NOT NULL AUTO_INCREMENT PRIMARY KEY, " &
-##      " Name  VARCHAR(50) NOT NULL, " &
-##      " i     INT(11), " &
-##      " f     DECIMAL(18,10))"))
-##
-##  theDb.exec(sql"START TRANSACTION")
-##  for i in 1..1000:
-##    theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
-##          "Item#" & $i, i, sqrt(i.float))
-##  theDb.exec(sql"COMMIT")
-##
-##  for x in theDb.fastRows(sql"select * from myTestTbl"):
-##    echo x
-##
-##  let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
-##          "Item#1001", 1001, sqrt(1001.0))
-##  echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id)
-##
-##  theDb.close()
-
-import strutils, odbcsql
-import db_common
-export db_common
-
-import std/private/since
-
-type
-  OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt]
-  DbConn* = OdbcConnTyp    ## encapsulates a database connection
-  Row* = seq[string]   ## a row of a dataset. NULL database values will be
-                       ## converted to nil.
-  InstantRow* = tuple[row: seq[string], len: int]  ## a handle that can be
-                                                    ## used to get a row's
-                                                    ## column text on demand
-
-var
-  buf: array[0..4096, char]
-
-proc properFreeResult(hType: int, sqlres: var SqlHandle) {.
-          tags: [WriteDbEffect], raises: [].} =
-  try:
-    discard SQLFreeHandle(hType.TSqlSmallInt, sqlres)
-    sqlres = nil
-  except: discard
-
-proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {.
-          tags: [ReadDbEffect], raises: [].} =
-  ## Returns ODBC error information
-  var
-    sqlState: array[0..512, char]
-    nativeErr: array[0..512, char]
-    errMsg: array[0..512, char]
-    retSz: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0
-  try:
-    sqlState[0] = '\0'
-    nativeErr[0] = '\0'
-    errMsg[0] = '\0'
-    res = SQLErr(db.env, db.hDb, db.stmt,
-              cast[PSQLCHAR](sqlState.addr),
-              cast[PSQLCHAR](nativeErr.addr),
-              cast[PSQLCHAR](errMsg.addr),
-              511.TSqlSmallInt, retSz.addr)
-  except:
-    discard
-  return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg))
-
-proc dbError*(db: var DbConn) {.
-          tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} =
-  ## Raises an `[DbError]` exception with ODBC error information
-  var
-    e: ref DbError
-    ss, ne, msg: string = ""
-    isAnError = false
-    res: int = 0
-    prevSs = ""
-  while true:
-    prevSs = ss
-    (res, ss, ne, msg) = db.getErrInfo()
-    if prevSs == ss:
-      break
-    # sqlState of 00000 is not an error
-    elif ss == "00000":
-      break
-    elif ss == "01000":
-      echo "\nWarning: ", ss, " ", msg
-      continue
-    else:
-      isAnError = true
-      echo "\nError: ", ss, " ", msg
-  if isAnError:
-    new(e)
-    e.msg = "ODBC Error"
-    if db.stmt != nil:
-      properFreeResult(SQL_HANDLE_STMT, db.stmt)
-    properFreeResult(SQL_HANDLE_DBC, db.hDb)
-    properFreeResult(SQL_HANDLE_ENV, db.env)
-    raise e
-
-proc sqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} =
-  ## Wrapper that raises [EDb] if `resVal` is neither SQL_SUCCESS or SQL_NO_DATA
-  if resVal notIn [SQL_SUCCESS, SQL_NO_DATA]: dbError(db)
-
-proc sqlGetDBMS(db: var DbConn): string {.
-        tags: [ReadDbEffect, WriteDbEffect], raises: [] .} =
-  ## Returns the ODBC SQL_DBMS_NAME string
-  const
-    SQL_DBMS_NAME = 17.SqlUSmallInt
-  var
-    sz: TSqlSmallInt = 0
-  buf[0] = '\0'
-  try:
-    db.sqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr),
-                        4095.TSqlSmallInt, sz.addr))
-  except: discard
-  return $(addr buf)
-
-proc dbQuote*(s: string): string {.noSideEffect.} =
-  ## DB quotes the string.
-  result = "'"
-  for c in items(s):
-    if c == '\'': add(result, "''")
-    else: add(result, c)
-  add(result, '\'')
-
-proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {.
-                  noSideEffect.} =
-  ## Replace any `?` placeholders with `args`,
-  ## and quotes the arguments
-  result = ""
-  var a = 0
-  for c in items(string(formatstr)):
-    if c == '?':
-      add(result, dbQuote(args[a]))
-      inc(a)
-    else:
-      add(result, c)
-
-proc prepareFetch(db: var DbConn, query: SqlQuery,
-                args: varargs[string, `$`]): TSqlSmallInt {.
-                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  # Prepare a statement, execute it and fetch the data to the driver
-  # ready for retrieval of the data
-  # Used internally by iterators and retrieval procs
-  # requires calling
-  #      properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  # when finished
-  db.sqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt))
-  var q = dbFormat(query, args)
-  db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
-  db.sqlCheck(SQLExecute(db.stmt))
-  result = SQLFetch(db.stmt)
-  db.sqlCheck(result)
-
-proc prepareFetchDirect(db: var DbConn, query: SqlQuery,
-                args: varargs[string, `$`]) {.
-                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  # Prepare a statement, execute it and fetch the data to the driver
-  # ready for retrieval of the data
-  # Used internally by iterators and retrieval procs
-  # requires calling
-  #      properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  # when finished
-  db.sqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt))
-  var q = dbFormat(query, args)
-  db.sqlCheck(SQLExecDirect(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
-  db.sqlCheck(SQLFetch(db.stmt))
-
-proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
-  tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
-  ## Tries to execute the query and returns true if successful, false otherwise.
-  var
-    res:TSqlSmallInt = -1
-  try:
-    db.prepareFetchDirect(query, args)
-    var
-      rCnt:TSqlLen = -1
-    res = SQLRowCount(db.stmt, rCnt)
-    properFreeResult(SQL_HANDLE_STMT, db.stmt)
-    if res != SQL_SUCCESS: dbError(db)
-  except: discard
-  return res == SQL_SUCCESS
-
-proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-            tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  db.prepareFetchDirect(query, args)
-
-proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-            tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Executes the query and raises EDB if not successful.
-  db.prepareFetchDirect(query, args)
-  properFreeResult(SQL_HANDLE_STMT, db.stmt)
-
-proc newRow(L: int): Row {.noSideEFfect.} =
-  newSeq(result, L)
-  for i in 0..L-1: result[i] = ""
-
-iterator fastRows*(db: var DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.
-                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Executes the query and iterates over the result dataset.
-  ##
-  ## This is very fast, but potentially dangerous.  Use this iterator only
-  ## if you require **ALL** the rows.
-  ##
-  ## Breaking the fastRows() iterator during a loop may cause a driver error
-  ## for subsequent queries
-  ##
-  ## Rows are retrieved from the server at each iteration.
-  var
-    rowRes: Row
-    sz: TSqlLen = 0
-    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0
-  res = db.prepareFetch(query, args)
-  if res == SQL_NO_DATA:
-    discard
-  elif res == SQL_SUCCESS:
-    res = SQLNumResultCols(db.stmt, cCnt)
-    rowRes = newRow(cCnt)
-    rowRes.setLen(max(cCnt,0))
-    while res == SQL_SUCCESS:
-      for colId in 1..cCnt:
-        buf[0] = '\0'
-        db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                                 cast[cstring](buf.addr), 4095, sz.addr))
-        rowRes[colId-1] = $(addr buf)
-      yield rowRes
-      res = SQLFetch(db.stmt)
-  properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  db.sqlCheck(res)
-
-iterator instantRows*(db: var DbConn, query: SqlQuery,
-                      args: varargs[string, `$`]): InstantRow
-                {.tags: [ReadDbEffect, WriteDbEffect].} =
-  ## Same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the iterator body.
-  var
-    rowRes: Row = @[]
-    sz: TSqlLen = 0
-    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0
-  res = db.prepareFetch(query, args)
-  if res == SQL_NO_DATA:
-    discard
-  elif res == SQL_SUCCESS:
-    res = SQLNumResultCols(db.stmt, cCnt)
-    rowRes = newRow(cCnt)
-    rowRes.setLen(max(cCnt,0))
-    while res == SQL_SUCCESS:
-      for colId in 1..cCnt:
-        buf[0] = '\0'
-        db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                                 cast[cstring](buf.addr), 4095, sz.addr))
-        rowRes[colId-1] = $(addr buf)
-      yield (row: rowRes, len: cCnt.int)
-      res = SQLFetch(db.stmt)
-  properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  db.sqlCheck(res)
-
-proc `[]`*(row: InstantRow, col: int): string {.inline.} =
-  ## Returns text for given column of the row
-  $row.row[col]
-
-proc unsafeColumnAt*(row: InstantRow, index: int): cstring {.inline.} =
-  ## Return cstring of given column of the row
-  row.row[index]
-
-proc len*(row: InstantRow): int {.inline.} =
-  ## Returns number of columns in the row
-  row.len
-
-proc getRow*(db: var DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.
-          tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Retrieves a single row. If the query doesn't return any rows, this proc
-  ## will return a Row with empty strings for each column.
-  var
-    rowRes: Row
-    sz: TSqlLen = 0
-    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0
-  res = db.prepareFetch(query, args)
-  if res == SQL_NO_DATA:
-    result = @[]
-  elif res == SQL_SUCCESS:
-    res = SQLNumResultCols(db.stmt, cCnt)
-    rowRes = newRow(cCnt)
-    rowRes.setLen(max(cCnt,0))
-    for colId in 1..cCnt:
-      buf[0] = '\0'
-      db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                               cast[cstring](buf.addr), 4095, sz.addr))
-      rowRes[colId-1] = $(addr buf)
-    res = SQLFetch(db.stmt)
-    result = rowRes
-  properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  db.sqlCheck(res)
-
-proc getAllRows*(db: var DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.
-           tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} =
-  ## Executes the query and returns the whole result dataset.
-  var
-    rows: seq[Row] = @[]
-    rowRes: Row
-    sz: TSqlLen = 0
-    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0
-  res = db.prepareFetch(query, args)
-  if res == SQL_NO_DATA:
-    result = @[]
-  elif res == SQL_SUCCESS:
-    res = SQLNumResultCols(db.stmt, cCnt)
-    rowRes = newRow(cCnt)
-    rowRes.setLen(max(cCnt,0))
-    while res == SQL_SUCCESS:
-      for colId in 1..cCnt:
-        buf[0] = '\0'
-        db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                                 cast[cstring](buf.addr), 4095, sz.addr))
-        rowRes[colId-1] = $(addr buf)
-      rows.add(rowRes)
-      res = SQLFetch(db.stmt)
-    result = rows
-  properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  db.sqlCheck(res)
-
-iterator rows*(db: var DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.
-         tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Same as `fastRows`, but slower and safe.
-  ##
-  ## This retrieves ALL rows into memory before
-  ## iterating through the rows.
-  ## Large dataset queries will impact on memory usage.
-  for r in items(getAllRows(db, query, args)): yield r
-
-proc getValue*(db: var DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.
-           tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
-  ## Executes the query and returns the first column of the first row of the
-  ## result dataset. Returns "" if the dataset contains no rows or the database
-  ## value is NULL.
-  result = ""
-  try:
-    result = getRow(db, query, args)[0]
-  except: discard
-
-proc tryInsertId*(db: var DbConn, query: SqlQuery,
-                  args: varargs[string, `$`]): int64 {.
-            tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
-  ## Executes the query (typically "INSERT") and returns the
-  ## generated ID for the row or -1 in case of an error.
-  if not tryExec(db, query, args):
-    result = -1'i64
-  else:
-    result = -1'i64
-    try:
-      case sqlGetDBMS(db).toLower():
-      of "postgresql":
-        result = getValue(db, sql"SELECT LASTVAL();", []).parseInt
-      of "mysql":
-        result = getValue(db, sql"SELECT LAST_INSERT_ID();", []).parseInt
-      of "sqlite":
-        result = getValue(db, sql"SELECT LAST_INSERT_ROWID();", []).parseInt
-      of "microsoft sql server":
-        result = getValue(db, sql"SELECT SCOPE_IDENTITY();", []).parseInt
-      of "oracle":
-        result = getValue(db, sql"SELECT id.currval FROM DUAL;", []).parseInt
-      else: result = -1'i64
-    except: discard
-
-proc insertId*(db: var DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.
-         tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Executes the query (typically "INSERT") and returns the
-  ## generated ID for the row.
-  result = tryInsertID(db, query, args)
-  if result < 0: dbError(db)
-
-proc tryInsert*(db: var DbConn, query: SqlQuery,pkName: string,
-                args: varargs[string, `$`]): int64
-               {.tags: [ReadDbEffect, WriteDbEffect], raises: [], since: (1, 3).} =
-  ## same as tryInsertID
-  tryInsertID(db, query, args)
-
-proc insert*(db: var DbConn, query: SqlQuery, pkName: string,
-             args: varargs[string, `$`]): int64 
-            {.tags: [ReadDbEffect, WriteDbEffect], since: (1, 3).} =
-  ## same as insertId
-  result = tryInsert(db, query,pkName, args)
-  if result < 0: dbError(db)
-
-proc execAffectedRows*(db: var DbConn, query: SqlQuery,
-                       args: varargs[string, `$`]): int64 {.
-             tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Runs the query (typically "UPDATE") and returns the
-  ## number of affected rows
-  result = -1
-  db.sqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt.SqlHandle))
-  var q = dbFormat(query, args)
-  db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
-  rawExec(db, query, args)
-  var rCnt:TSqlLen = -1
-  db.sqlCheck(SQLRowCount(db.hDb, rCnt))
-  properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  result = rCnt.int64
-
-proc close*(db: var DbConn) {.
-      tags: [WriteDbEffect], raises: [].} =
-  ## Closes the database connection.
-  if db.hDb != nil:
-    try:
-      var res = SQLDisconnect(db.hDb)
-      if db.stmt != nil:
-        res = SQLFreeHandle(SQL_HANDLE_STMT, db.stmt)
-      res = SQLFreeHandle(SQL_HANDLE_DBC, db.hDb)
-      res = SQLFreeHandle(SQL_HANDLE_ENV, db.env)
-      db = (hDb: nil, env: nil, stmt: nil)
-    except:
-      discard
-
-proc open*(connection, user, password, database: string): DbConn {.
-  tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Opens a database connection.
-  ##
-  ## Raises `EDb` if the connection could not be established.
-  ##
-  ## Currently the database parameter is ignored,
-  ## but included to match `open()` in the other db_xxxxx library modules.
-  var
-    val = SQL_OV_ODBC3
-    resLen = 0
-  result = (hDb: nil, env: nil, stmt: nil)
-  # allocate environment handle
-  var res = SQLAllocHandle(SQL_HANDLE_ENV, result.env, result.env)
-  if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.")
-  res = SQLSetEnvAttr(result.env,
-                      SQL_ATTR_ODBC_VERSION.TSqlInteger,
-                      cast[SqlPointer](val), resLen.TSqlInteger)
-  if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.")
-  # allocate hDb handle
-  res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb)
-  if res != SQL_SUCCESS: dbError("Error: unable to allocate connection handle.")
-
-  # Connect: connection = dsn str,
-  res = SQLConnect(result.hDb,
-                  connection.PSQLCHAR , connection.len.TSqlSmallInt,
-                  user.PSQLCHAR, user.len.TSqlSmallInt,
-                  password.PSQLCHAR, password.len.TSqlSmallInt)
-  if res != SQL_SUCCESS:
-    result.dbError()
-
-proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
-  ## Currently not implemented for ODBC.
-  ##
-  ## Sets the encoding of a database connection, returns true for
-  ## success, false for failure.
-  ##result = set_character_set(connection, encoding) == 0
-  dbError("setEncoding() is currently not implemented by the db_odbc module")
diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim
deleted file mode 100644
index 034a948526..0000000000
--- a/lib/impure/db_postgres.nim
+++ /dev/null
@@ -1,590 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## A higher level `PostgreSQL`:idx: database wrapper. This interface
-## is implemented for other databases also.
-##
-## See also: `db_odbc `_, `db_sqlite `_,
-## `db_mysql `_.
-##
-## Parameter substitution
-## ======================
-##
-## All `db_*` modules support the same form of parameter substitution.
-## That is, using the `?` (question mark) to signify the place where a
-## value should be placed. For example:
-##
-## .. code-block:: Nim
-##     sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)"
-##
-## **Note**: There are two approaches to parameter substitution support by
-## this module.
-##
-## 1. `SqlQuery` using `?, ?, ?, ...` (same as all the `db_*` modules)
-##
-## 2. `SqlPrepared` using `$1, $2, $3, ...`
-##
-## .. code-block:: Nim
-##   prepare(db, "myExampleInsert",
-##           sql"""INSERT INTO myTable
-##                 (colA, colB, colC)
-##                 VALUES ($1, $2, $3)""",
-##           3)
-##
-##
-## Unix Socket
-## ===========
-##
-## Using Unix sockets instead of TCP connection can
-## `improve performance up to 30% ~ 175% for some operations `_.
-##
-## To use Unix sockets with `db_postgres`, change the server address to the socket file path:
-##
-## .. code-block:: Nim
-##   import std/db_postgres ## Change "localhost" or "127.0.0.1" to the socket file path
-##   let db = db_postgres.open("/run/postgresql", "user", "password", "database")
-##   echo db.getAllRows(sql"SELECT version();")
-##   db.close()
-##
-## The socket file path is operating system specific and distribution specific,
-## additional configuration may or may not be needed on your `postgresql.conf`.
-## The Postgres server must be on the same computer and only works for Unix-like operating systems.
-##
-##
-## Examples
-## ========
-##
-## Opening a connection to a database
-## ----------------------------------
-##
-## .. code-block:: Nim
-##     import std/db_postgres
-##     let db = open("localhost", "user", "password", "dbname")
-##     db.close()
-##
-## Creating a table
-## ----------------
-##
-## .. code-block:: Nim
-##      db.exec(sql"DROP TABLE IF EXISTS myTable")
-##      db.exec(sql("""CREATE TABLE myTable (
-##                       id integer,
-##                       name varchar(50) not null)"""))
-##
-## Inserting data
-## --------------
-##
-## .. code-block:: Nim
-##     db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)",
-##             "Dominik")
-import strutils, postgres
-
-import db_common
-export db_common
-
-import std/private/since
-
-type
-  DbConn* = PPGconn    ## encapsulates a database connection
-  Row* = seq[string]   ## a row of a dataset. NULL database values will be
-                       ## converted to nil.
-  InstantRow* = object ## a handle that can be
-    res: PPGresult     ## used to get a row's
-    line: int          ## column text on demand
-  SqlPrepared* = distinct string ## a identifier for the prepared queries
-
-proc dbError*(db: DbConn) {.noreturn.} =
-  ## raises a DbError exception.
-  var e: ref DbError
-  new(e)
-  e.msg = $pqErrorMessage(db)
-  raise e
-
-proc dbQuote*(s: string): string =
-  ## DB quotes the string.
-  result = "'"
-  for c in items(s):
-    if c == '\'': add(result, "''")
-    else: add(result, c)
-  add(result, '\'')
-
-proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
-  result = ""
-  var a = 0
-  if args.len > 0 and not string(formatstr).contains("?"):
-    dbError("""parameter substitution expects "?" """)
-  if args.len == 0:
-    return string(formatstr)
-  else:
-    for c in items(string(formatstr)):
-      if c == '?':
-        add(result, dbQuote(args[a]))
-        inc(a)
-      else:
-        add(result, c)
-
-proc tryExec*(db: DbConn, query: SqlQuery,
-              args: varargs[string, `$`]): bool {.tags: [ReadDbEffect, WriteDbEffect].} =
-  ## tries to execute the query and returns true if successful, false otherwise.
-  var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil,
-                        nil, nil, 0)
-  result = pqresultStatus(res) == PGRES_COMMAND_OK
-  pqclear(res)
-
-proc tryExec*(db: DbConn, stmtName: SqlPrepared,
-              args: varargs[string, `$`]): bool {.tags: [
-              ReadDbEffect, WriteDbEffect].} =
-  ## tries to execute the query and returns true if successful, false otherwise.
-  var arr = allocCStringArray(args)
-  var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
-                           nil, nil, 0)
-  deallocCStringArray(arr)
-  result = pqresultStatus(res) == PGRES_COMMAND_OK
-  pqclear(res)
-
-proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-  tags: [ReadDbEffect, WriteDbEffect].} =
-  ## executes the query and raises EDB if not successful.
-  var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil,
-                        nil, nil, 0)
-  if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db)
-  pqclear(res)
-
-proc exec*(db: DbConn, stmtName: SqlPrepared,
-          args: varargs[string]) {.tags: [ReadDbEffect, WriteDbEffect].} =
-  var arr = allocCStringArray(args)
-  var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
-                           nil, nil, 0)
-  deallocCStringArray(arr)
-  if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db)
-  pqclear(res)
-
-proc newRow(L: int): Row =
-  newSeq(result, L)
-  for i in 0..L-1: result[i] = ""
-
-proc setupQuery(db: DbConn, query: SqlQuery,
-                args: varargs[string]): PPGresult =
-  result = pqexec(db, dbFormat(query, args))
-  if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db)
-
-proc setupQuery(db: DbConn, stmtName: SqlPrepared,
-                 args: varargs[string]): PPGresult =
-  var arr = allocCStringArray(args)
-  result = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
-                          nil, nil, 0)
-  deallocCStringArray(arr)
-  if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db)
-
-proc prepare*(db: DbConn; stmtName: string, query: SqlQuery;
-              nParams: int): SqlPrepared =
-  ## Creates a new `SqlPrepared` statement. Parameter substitution is done
-  ## via `$1`, `$2`, `$3`, etc.
-  if nParams > 0 and not string(query).contains("$1"):
-    dbError("parameter substitution expects \"$1\"")
-  var res = pqprepare(db, stmtName, query.string, int32(nParams), nil)
-  if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db)
-  return SqlPrepared(stmtName)
-
-proc setRow(res: PPGresult, r: var Row, line, cols: int32) =
-  for col in 0'i32..cols-1:
-    setLen(r[col], 0)
-    let x = pqgetvalue(res, line, col)
-    if x.isNil:
-      r[col] = ""
-    else:
-      add(r[col], x)
-
-iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## executes the query and iterates over the result dataset. This is very
-  ## fast, but potentially dangerous: If the for-loop-body executes another
-  ## query, the results can be undefined. For Postgres it is safe though.
-  var res = setupQuery(db, query, args)
-  var L = pqnfields(res)
-  var result = newRow(L)
-  for i in 0'i32..pqntuples(res)-1:
-    setRow(res, result, i, L)
-    yield result
-  pqclear(res)
-
-iterator fastRows*(db: DbConn, stmtName: SqlPrepared,
-                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## executes the prepared query and iterates over the result dataset.
-  var res = setupQuery(db, stmtName, args)
-  var L = pqNfields(res)
-  var result = newRow(L)
-  for i in 0'i32..pqNtuples(res)-1:
-    setRow(res, result, i, L)
-    yield result
-  pqClear(res)
-
-iterator instantRows*(db: DbConn, query: SqlQuery,
-                      args: varargs[string, `$`]): InstantRow
-                      {.tags: [ReadDbEffect].} =
-  ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within iterator body.
-  var res = setupQuery(db, query, args)
-  for i in 0'i32..pqNtuples(res)-1:
-    yield InstantRow(res: res, line: i)
-  pqClear(res)
-
-iterator instantRows*(db: DbConn, stmtName: SqlPrepared,
-                      args: varargs[string, `$`]): InstantRow
-                      {.tags: [ReadDbEffect].} =
-  ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within iterator body.
-  var res = setupQuery(db, stmtName, args)
-  for i in 0'i32..pqNtuples(res)-1:
-    yield InstantRow(res: res, line: i)
-  pqClear(res)
-
-proc getColumnType(res: PPGresult, col: int) : DbType =
-  ## returns DbType for given column in the row
-  ## defined in pg_type.h file in the postgres source code
-  ## Wire representation for types: http://www.npgsql.org/dev/types.html
-  var oid = pqftype(res, int32(col))
-  ## The integer returned is the internal OID number of the type
-  case oid
-  of 16: return DbType(kind: DbTypeKind.dbBool, name: "bool")
-  of 17: return DbType(kind: DbTypeKind.dbBlob, name: "bytea")
-
-  of 21:   return DbType(kind: DbTypeKind.dbInt, name: "int2", size: 2)
-  of 23:   return DbType(kind: DbTypeKind.dbInt, name: "int4", size: 4)
-  of 20:   return DbType(kind: DbTypeKind.dbInt, name: "int8", size: 8)
-  of 1560: return DbType(kind: DbTypeKind.dbBit, name: "bit")
-  of 1562: return DbType(kind: DbTypeKind.dbInt, name: "varbit")
-
-  of 18:   return DbType(kind: DbTypeKind.dbFixedChar, name: "char")
-  of 19:   return DbType(kind: DbTypeKind.dbFixedChar, name: "name")
-  of 1042: return DbType(kind: DbTypeKind.dbFixedChar, name: "bpchar")
-
-  of 25:   return DbType(kind: DbTypeKind.dbVarchar, name: "text")
-  of 1043: return DbType(kind: DbTypeKind.dbVarChar, name: "varchar")
-  of 2275: return DbType(kind: DbTypeKind.dbVarchar, name: "cstring")
-
-  of 700: return DbType(kind: DbTypeKind.dbFloat, name: "float4")
-  of 701: return DbType(kind: DbTypeKind.dbFloat, name: "float8")
-
-  of 790:  return DbType(kind: DbTypeKind.dbDecimal, name: "money")
-  of 1700: return DbType(kind: DbTypeKind.dbDecimal, name: "numeric")
-
-  of 704:  return DbType(kind: DbTypeKind.dbTimeInterval, name: "tinterval")
-  of 702:  return DbType(kind: DbTypeKind.dbTimestamp, name: "abstime")
-  of 703:  return DbType(kind: DbTypeKind.dbTimeInterval, name: "reltime")
-  of 1082: return DbType(kind: DbTypeKind.dbDate, name: "date")
-  of 1083: return DbType(kind: DbTypeKind.dbTime, name: "time")
-  of 1114: return DbType(kind: DbTypeKind.dbTimestamp, name: "timestamp")
-  of 1184: return DbType(kind: DbTypeKind.dbTimestamp, name: "timestamptz")
-  of 1186: return DbType(kind: DbTypeKind.dbTimeInterval, name: "interval")
-  of 1266: return DbType(kind: DbTypeKind.dbTime, name: "timetz")
-
-  of 114:  return DbType(kind: DbTypeKind.dbJson, name: "json")
-  of 142:  return DbType(kind: DbTypeKind.dbXml, name: "xml")
-  of 3802: return DbType(kind: DbTypeKind.dbJson, name: "jsonb")
-
-  of 600: return DbType(kind: DbTypeKind.dbPoint, name: "point")
-  of 601: return DbType(kind: DbTypeKind.dbLseg, name: "lseg")
-  of 602: return DbType(kind: DbTypeKind.dbPath, name: "path")
-  of 603: return DbType(kind: DbTypeKind.dbBox, name: "box")
-  of 604: return DbType(kind: DbTypeKind.dbPolygon, name: "polygon")
-  of 628: return DbType(kind: DbTypeKind.dbLine, name: "line")
-  of 718: return DbType(kind: DbTypeKind.dbCircle, name: "circle")
-
-  of 650: return DbType(kind: DbTypeKind.dbInet, name: "cidr")
-  of 829: return DbType(kind: DbTypeKind.dbMacAddress, name: "macaddr")
-  of 869: return DbType(kind: DbTypeKind.dbInet, name: "inet")
-
-  of 2950: return DbType(kind: DbTypeKind.dbVarchar, name: "uuid")
-  of 3614: return DbType(kind: DbTypeKind.dbVarchar, name: "tsvector")
-  of 3615: return DbType(kind: DbTypeKind.dbVarchar, name: "tsquery")
-  of 2970: return DbType(kind: DbTypeKind.dbVarchar, name: "txid_snapshot")
-
-  of 27:   return DbType(kind: DbTypeKind.dbComposite, name: "tid")
-  of 1790: return DbType(kind: DbTypeKind.dbComposite, name: "refcursor")
-  of 2249: return DbType(kind: DbTypeKind.dbComposite, name: "record")
-  of 3904: return DbType(kind: DbTypeKind.dbComposite, name: "int4range")
-  of 3906: return DbType(kind: DbTypeKind.dbComposite, name: "numrange")
-  of 3908: return DbType(kind: DbTypeKind.dbComposite, name: "tsrange")
-  of 3910: return DbType(kind: DbTypeKind.dbComposite, name: "tstzrange")
-  of 3912: return DbType(kind: DbTypeKind.dbComposite, name: "daterange")
-  of 3926: return DbType(kind: DbTypeKind.dbComposite, name: "int8range")
-
-  of 22:   return DbType(kind: DbTypeKind.dbArray, name: "int2vector")
-  of 30:   return DbType(kind: DbTypeKind.dbArray, name: "oidvector")
-  of 143:  return DbType(kind: DbTypeKind.dbArray, name: "xml[]")
-  of 199:  return DbType(kind: DbTypeKind.dbArray, name: "json[]")
-  of 629:  return DbType(kind: DbTypeKind.dbArray, name: "line[]")
-  of 651:  return DbType(kind: DbTypeKind.dbArray, name: "cidr[]")
-  of 719:  return DbType(kind: DbTypeKind.dbArray, name: "circle[]")
-  of 791:  return DbType(kind: DbTypeKind.dbArray, name: "money[]")
-  of 1000: return DbType(kind: DbTypeKind.dbArray, name: "bool[]")
-  of 1001: return DbType(kind: DbTypeKind.dbArray, name: "bytea[]")
-  of 1002: return DbType(kind: DbTypeKind.dbArray, name: "char[]")
-  of 1003: return DbType(kind: DbTypeKind.dbArray, name: "name[]")
-  of 1005: return DbType(kind: DbTypeKind.dbArray, name: "int2[]")
-  of 1006: return DbType(kind: DbTypeKind.dbArray, name: "int2vector[]")
-  of 1007: return DbType(kind: DbTypeKind.dbArray, name: "int4[]")
-  of 1008: return DbType(kind: DbTypeKind.dbArray, name: "regproc[]")
-  of 1009: return DbType(kind: DbTypeKind.dbArray, name: "text[]")
-  of 1028: return DbType(kind: DbTypeKind.dbArray, name: "oid[]")
-  of 1010: return DbType(kind: DbTypeKind.dbArray, name: "tid[]")
-  of 1011: return DbType(kind: DbTypeKind.dbArray, name: "xid[]")
-  of 1012: return DbType(kind: DbTypeKind.dbArray, name: "cid[]")
-  of 1013: return DbType(kind: DbTypeKind.dbArray, name: "oidvector[]")
-  of 1014: return DbType(kind: DbTypeKind.dbArray, name: "bpchar[]")
-  of 1015: return DbType(kind: DbTypeKind.dbArray, name: "varchar[]")
-  of 1016: return DbType(kind: DbTypeKind.dbArray, name: "int8[]")
-  of 1017: return DbType(kind: DbTypeKind.dbArray, name: "point[]")
-  of 1018: return DbType(kind: DbTypeKind.dbArray, name: "lseg[]")
-  of 1019: return DbType(kind: DbTypeKind.dbArray, name: "path[]")
-  of 1020: return DbType(kind: DbTypeKind.dbArray, name: "box[]")
-  of 1021: return DbType(kind: DbTypeKind.dbArray, name: "float4[]")
-  of 1022: return DbType(kind: DbTypeKind.dbArray, name: "float8[]")
-  of 1023: return DbType(kind: DbTypeKind.dbArray, name: "abstime[]")
-  of 1024: return DbType(kind: DbTypeKind.dbArray, name: "reltime[]")
-  of 1025: return DbType(kind: DbTypeKind.dbArray, name: "tinterval[]")
-  of 1027: return DbType(kind: DbTypeKind.dbArray, name: "polygon[]")
-  of 1040: return DbType(kind: DbTypeKind.dbArray, name: "macaddr[]")
-  of 1041: return DbType(kind: DbTypeKind.dbArray, name: "inet[]")
-  of 1263: return DbType(kind: DbTypeKind.dbArray, name: "cstring[]")
-  of 1115: return DbType(kind: DbTypeKind.dbArray, name: "timestamp[]")
-  of 1182: return DbType(kind: DbTypeKind.dbArray, name: "date[]")
-  of 1183: return DbType(kind: DbTypeKind.dbArray, name: "time[]")
-  of 1185: return DbType(kind: DbTypeKind.dbArray, name: "timestamptz[]")
-  of 1187: return DbType(kind: DbTypeKind.dbArray, name: "interval[]")
-  of 1231: return DbType(kind: DbTypeKind.dbArray, name: "numeric[]")
-  of 1270: return DbType(kind: DbTypeKind.dbArray, name: "timetz[]")
-  of 1561: return DbType(kind: DbTypeKind.dbArray, name: "bit[]")
-  of 1563: return DbType(kind: DbTypeKind.dbArray, name: "varbit[]")
-  of 2201: return DbType(kind: DbTypeKind.dbArray, name: "refcursor[]")
-  of 2951: return DbType(kind: DbTypeKind.dbArray, name: "uuid[]")
-  of 3643: return DbType(kind: DbTypeKind.dbArray, name: "tsvector[]")
-  of 3645: return DbType(kind: DbTypeKind.dbArray, name: "tsquery[]")
-  of 3807: return DbType(kind: DbTypeKind.dbArray, name: "jsonb[]")
-  of 2949: return DbType(kind: DbTypeKind.dbArray, name: "txid_snapshot[]")
-  of 3905: return DbType(kind: DbTypeKind.dbArray, name: "int4range[]")
-  of 3907: return DbType(kind: DbTypeKind.dbArray, name: "numrange[]")
-  of 3909: return DbType(kind: DbTypeKind.dbArray, name: "tsrange[]")
-  of 3911: return DbType(kind: DbTypeKind.dbArray, name: "tstzrange[]")
-  of 3913: return DbType(kind: DbTypeKind.dbArray, name: "daterange[]")
-  of 3927: return DbType(kind: DbTypeKind.dbArray, name: "int8range[]")
-  of 2287: return DbType(kind: DbTypeKind.dbArray, name: "record[]")
-
-  of 705:  return DbType(kind: DbTypeKind.dbUnknown, name: "unknown")
-  else: return DbType(kind: DbTypeKind.dbUnknown, name: $oid) ## Query the system table pg_type to determine exactly which type is referenced.
-
-proc setColumnInfo(columns: var DbColumns; res: PPGresult; L: int32) =
-  setLen(columns, L)
-  for i in 0'i32.. 0:
-    setRow(res, result, 0, L)
-  pqclear(res)
-
-proc getRow*(db: DbConn, stmtName: SqlPrepared,
-             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  var res = setupQuery(db, stmtName, args)
-  var L = pqNfields(res)
-  result = newRow(L)
-  if pqntuples(res) > 0:
-    setRow(res, result, 0, L)
-  pqClear(res)
-
-proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.
-                 tags: [ReadDbEffect].} =
-  ## executes the query and returns the whole result dataset.
-  result = @[]
-  for r in fastRows(db, query, args):
-    result.add(r)
-
-proc getAllRows*(db: DbConn, stmtName: SqlPrepared,
-                 args: varargs[string, `$`]): seq[Row] {.tags:
-                 [ReadDbEffect].} =
-  ## executes the prepared query and returns the whole result dataset.
-  result = @[]
-  for r in fastRows(db, stmtName, args):
-    result.add(r)
-
-iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## same as `fastRows`, but slower and safe.
-  for r in items(getAllRows(db, query, args)): yield r
-
-iterator rows*(db: DbConn, stmtName: SqlPrepared,
-               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## same as `fastRows`, but slower and safe.
-  for r in items(getAllRows(db, stmtName, args)): yield r
-
-proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.
-               tags: [ReadDbEffect].} =
-  ## executes the query and returns the first column of the first row of the
-  ## result dataset. Returns "" if the dataset contains no rows or the database
-  ## value is NULL.
-  var res = setupQuery(db, query, args)
-  if pqntuples(res) > 0:
-    var x = pqgetvalue(res, 0, 0)
-    result = if isNil(x): "" else: $x
-  else:
-    result = ""
-
-proc getValue*(db: DbConn, stmtName: SqlPrepared,
-               args: varargs[string, `$`]): string {.
-               tags: [ReadDbEffect].} =
-  ## executes the query and returns the first column of the first row of the
-  ## result dataset. Returns "" if the dataset contains no rows or the database
-  ## value is NULL.
-  var res = setupQuery(db, stmtName, args)
-  if pqntuples(res) > 0:
-    var x = pqgetvalue(res, 0, 0)
-    result = if isNil(x): "" else: $x
-  else:
-    result = ""
-
-proc tryInsertID*(db: DbConn, query: SqlQuery,
-                  args: varargs[string, `$`]): int64 {.
-                  tags: [WriteDbEffect].}=
-  ## executes the query (typically "INSERT") and returns the
-  ## generated ID for the row or -1 in case of an error. For Postgre this adds
-  ## `RETURNING id` to the query, so it only works if your primary key is
-  ## named `id`.
-  var x = pqgetvalue(setupQuery(db, SqlQuery(string(query) & " RETURNING id"),
-    args), 0, 0)
-  if not isNil(x):
-    result = parseBiggestInt($x)
-  else:
-    result = -1
-
-proc insertID*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.
-               tags: [WriteDbEffect].} =
-  ## executes the query (typically "INSERT") and returns the
-  ## generated ID for the row. For Postgre this adds
-  ## `RETURNING id` to the query, so it only works if your primary key is
-  ## named `id`.
-  result = tryInsertID(db, query, args)
-  if result < 0: dbError(db)
-
-proc tryInsert*(db: DbConn, query: SqlQuery,pkName: string,
-                args: varargs[string, `$`]): int64
-               {.tags: [WriteDbEffect], since: (1, 3).}=
-  ## executes the query (typically "INSERT") and returns the
-  ## generated ID for the row or -1 in case of an error.
-  var x = pqgetvalue(setupQuery(db, SqlQuery(string(query) & " RETURNING " & pkName),
-    args), 0, 0)
-  if not isNil(x):
-    result = parseBiggestInt($x)
-  else:
-    result = -1
-
-proc insert*(db: DbConn, query: SqlQuery, pkName: string,
-             args: varargs[string, `$`]): int64
-            {.tags: [WriteDbEffect], since: (1, 3).} =
-  ## executes the query (typically "INSERT") and returns the
-  ## generated ID
-  result = tryInsertID(db, query, args)
-  if result < 0: dbError(db)
-
-proc execAffectedRows*(db: DbConn, query: SqlQuery,
-                       args: varargs[string, `$`]): int64 {.tags: [
-                       ReadDbEffect, WriteDbEffect].} =
-  ## executes the query (typically "UPDATE") and returns the
-  ## number of affected rows.
-  var q = dbFormat(query, args)
-  var res = pqExec(db, q)
-  if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db)
-  result = parseBiggestInt($pqcmdTuples(res))
-  pqclear(res)
-
-proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared,
-                       args: varargs[string, `$`]): int64 {.tags: [
-                       ReadDbEffect, WriteDbEffect].} =
-  ## executes the query (typically "UPDATE") and returns the
-  ## number of affected rows.
-  var arr = allocCStringArray(args)
-  var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
-                           nil, nil, 0)
-  deallocCStringArray(arr)
-  if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db)
-  result = parseBiggestInt($pqcmdTuples(res))
-  pqclear(res)
-
-proc close*(db: DbConn) {.tags: [DbEffect].} =
-  ## closes the database connection.
-  if db != nil: pqfinish(db)
-
-proc open*(connection, user, password, database: string): DbConn {.
-  tags: [DbEffect].} =
-  ## opens a database connection. Raises `EDb` if the connection could not
-  ## be established.
-  ##
-  ## Clients can also use Postgres keyword/value connection strings to
-  ## connect.
-  ##
-  ## Example:
-  ##
-  ## .. code-block:: nim
-  ##
-  ##      con = open("", "", "", "host=localhost port=5432 dbname=mydb")
-  ##
-  ## See http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
-  ## for more information.
-  let
-    colonPos = connection.find(':')
-    host = if colonPos < 0: connection
-           else: substr(connection, 0, colonPos-1)
-    port = if colonPos < 0: ""
-           else: substr(connection, colonPos+1)
-  result = pqsetdbLogin(host, port, nil, nil, database, user, password)
-  if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil
-
-proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [DbEffect].} =
-  ## sets the encoding of a database connection, returns true for
-  ## success, false for failure.
-  return pqsetClientEncoding(connection, encoding) == 0
-
-
-# Tests are in ../../tests/untestable/tpostgres.
diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim
deleted file mode 100644
index 8324079602..0000000000
--- a/lib/impure/db_sqlite.nim
+++ /dev/null
@@ -1,950 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## A higher level `SQLite`:idx: database wrapper. This interface
-## is implemented for other databases too.
-##
-## Basic usage
-## ===========
-##
-## The basic flow of using this module is:
-##
-## 1. Open database connection
-## 2. Execute SQL query
-## 3. Close database connection
-##
-## Parameter substitution
-## ----------------------
-##
-## All `db_*` modules support the same form of parameter substitution.
-## That is, using the `?` (question mark) to signify the place where a
-## value should be placed. For example:
-##
-## .. code-block:: Nim
-##
-##    sql"INSERT INTO my_table (colA, colB, colC) VALUES (?, ?, ?)"
-##
-## Opening a connection to a database
-## ----------------------------------
-##
-## .. code-block:: Nim
-##
-##    import std/db_sqlite
-##
-##    # user, password, database name can be empty.
-##    # These params are not used on db_sqlite module.
-##    let db = open("mytest.db", "", "", "")
-##    db.close()
-##
-## Creating a table
-## ----------------
-##
-## .. code-block:: Nim
-##
-##    db.exec(sql"DROP TABLE IF EXISTS my_table")
-##    db.exec(sql"""CREATE TABLE my_table (
-##                     id   INTEGER,
-##                     name VARCHAR(50) NOT NULL
-##                  )""")
-##
-## Inserting data
-## --------------
-##
-## .. code-block:: Nim
-##
-##    db.exec(sql"INSERT INTO my_table (id, name) VALUES (0, ?)",
-##            "Jack")
-##
-## Larger example
-## --------------
-##
-## .. code-block:: nim
-##
-##    import std/[db_sqlite, math]
-##
-##    let db = open("mytest.db", "", "", "")
-##
-##    db.exec(sql"DROP TABLE IF EXISTS my_table")
-##    db.exec(sql"""CREATE TABLE my_table (
-##                     id    INTEGER PRIMARY KEY,
-##                     name  VARCHAR(50) NOT NULL,
-##                     i     INT(11),
-##                     f     DECIMAL(18, 10)
-##                  )""")
-##
-##    db.exec(sql"BEGIN")
-##    for i in 1..1000:
-##      db.exec(sql"INSERT INTO my_table (name, i, f) VALUES (?, ?, ?)",
-##              "Item#" & $i, i, sqrt(i.float))
-##    db.exec(sql"COMMIT")
-##
-##    for x in db.fastRows(sql"SELECT * FROM my_table"):
-##      echo x
-##
-##    let id = db.tryInsertId(sql"""INSERT INTO my_table (name, i, f)
-##                                  VALUES (?, ?, ?)""",
-##                            "Item#1001", 1001, sqrt(1001.0))
-##    echo "Inserted item: ", db.getValue(sql"SELECT name FROM my_table WHERE id=?", id)
-##
-##    db.close()
-##
-## Storing binary data example
-##----------------------------
-##
-## .. code-block:: nim
-##
-##   import std/random
-##
-##   ## Generate random float datas
-##   var orig = newSeq[float64](150)
-##   randomize()
-##   for x in orig.mitems:
-##     x = rand(1.0)/10.0
-##
-##   let db = open("mysqlite.db", "", "", "")
-##   block: ## Create database
-##     ## Binary datas needs to be of type BLOB in SQLite
-##     let createTableStr = sql"""CREATE TABLE test(
-##       id INTEGER NOT NULL PRIMARY KEY,
-##       data BLOB
-##     )
-##     """
-##     db.exec(createTableStr)
-##
-##   block: ## Insert data
-##     var id = 1
-##     ## Data needs to be converted to seq[byte] to be interpreted as binary by bindParams
-##     var dbuf = newSeq[byte](orig.len*sizeof(float64))
-##     copyMem(unsafeAddr(dbuf[0]), unsafeAddr(orig[0]), dbuf.len)
-##
-##     ## Use prepared statement to insert binary data into database
-##     var insertStmt = db.prepare("INSERT INTO test (id, data) VALUES (?, ?)")
-##     insertStmt.bindParams(id, dbuf)
-##     let bres = db.tryExec(insertStmt)
-##     ## Check insert
-##     doAssert(bres)
-##     # Destroy statement
-##     finalize(insertStmt)
-##
-##   block: ## Use getValue to select data
-##     var dataTest = db.getValue(sql"SELECT data FROM test WHERE id = ?", 1)
-##     ## Calculate sequence size from buffer size
-##     let seqSize = int(dataTest.len*sizeof(byte)/sizeof(float64))
-##     ## Copy binary string data in dataTest into a seq
-##     var res: seq[float64] = newSeq[float64](seqSize)
-##     copyMem(unsafeAddr(res[0]), addr(dataTest[0]), dataTest.len)
-##
-##     ## Check datas obtained is identical
-##     doAssert res == orig
-##
-##   db.close()
-##
-##
-## Note
-## ====
-## This module does not implement any ORM features such as mapping the types from the schema.
-## Instead, a `seq[string]` is returned for each row.
-##
-## The reasoning is as follows:
-## 1. it's close to what many DBs offer natively (char**)
-## 2. it hides the number of types that the DB supports
-##    (int? int64? decimal up to 10 places? geo coords?)
-## 3. it's convenient when all you do is to forward the data to somewhere else (echo, log, put the data into a new query)
-##
-## See also
-## ========
-##
-## * `db_odbc module `_ for ODBC database wrapper
-## * `db_mysql module `_ for MySQL database wrapper
-## * `db_postgres module `_ for PostgreSQL database wrapper
-
-{.experimental: "codeReordering".}
-
-import sqlite3, macros
-
-import db_common
-export db_common
-
-import std/private/since
-
-type
-  DbConn* = PSqlite3  ## Encapsulates a database connection.
-  Row* = seq[string]  ## A row of a dataset. `NULL` database values will be
-                      ## converted to an empty string.
-  InstantRow* = PStmt ## A handle that can be used to get a row's column
-                      ## text on demand.
-  SqlPrepared* = distinct PStmt ## a identifier for the prepared queries
-
-proc dbError*(db: DbConn) {.noreturn.} =
-  ## Raises a `DbError` exception.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##    if not db.tryExec(sql"SELECT * FROM not_exist_table"):
-  ##      dbError(db)
-  ##    db.close()
-  var e: ref DbError
-  new(e)
-  e.msg = $sqlite3.errmsg(db)
-  raise e
-
-proc dbQuote*(s: string): string =
-  ## Escapes the `'` (single quote) char to `''`.
-  ## Because single quote is used for defining `VARCHAR` in SQL.
-  runnableExamples:
-    doAssert dbQuote("'") == "''''"
-    doAssert dbQuote("A Foobar's pen.") == "'A Foobar''s pen.'"
-
-  result = "'"
-  for c in items(s):
-    if c == '\'': add(result, "''")
-    else: add(result, c)
-  add(result, '\'')
-
-proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
-  result = ""
-  var a = 0
-  for c in items(string(formatstr)):
-    if c == '?':
-      add(result, dbQuote(args[a]))
-      inc(a)
-    else:
-      add(result, c)
-
-proc prepare*(db: DbConn; q: string): SqlPrepared {.since: (1, 3).} =
-  ## Creates a new `SqlPrepared` statement.
-  if prepare_v2(db, q, q.len.cint,result.PStmt, nil) != SQLITE_OK:
-    discard finalize(result.PStmt)
-    dbError(db)
-
-proc tryExec*(db: DbConn, query: SqlQuery,
-              args: varargs[string, `$`]): bool {.
-              tags: [ReadDbEffect, WriteDbEffect].} =
-  ## Tries to execute the query and returns `true` if successful, `false` otherwise.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##    if not db.tryExec(sql"SELECT * FROM my_table"):
-  ##      dbError(db)
-  ##    db.close()
-  assert(not db.isNil, "Database not connected.")
-  var q = dbFormat(query, args)
-  var stmt: sqlite3.PStmt
-  if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK:
-    let x = step(stmt)
-    if x in {SQLITE_DONE, SQLITE_ROW}:
-      result = finalize(stmt) == SQLITE_OK
-    else:
-      discard finalize(stmt)
-      result = false
-
-proc tryExec*(db: DbConn, stmtName: SqlPrepared): bool {.
-              tags: [ReadDbEffect, WriteDbEffect].} =
-    let x = step(stmtName.PStmt)
-    if x in {SQLITE_DONE, SQLITE_ROW}:
-      result = true
-    else:
-      discard finalize(stmtName.PStmt)
-      result = false
-
-proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`])  {.
-  tags: [ReadDbEffect, WriteDbEffect].} =
-  ## Executes the query and raises a `DbError` exception if not successful.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##    try:
-  ##      db.exec(sql"INSERT INTO my_table (id, name) VALUES (?, ?)",
-  ##              1, "item#1")
-  ##    except:
-  ##      stderr.writeLine(getCurrentExceptionMsg())
-  ##    finally:
-  ##      db.close()
-  if not tryExec(db, query, args): dbError(db)
-
-proc newRow(L: int): Row =
-  newSeq(result, L)
-  for i in 0..L-1: result[i] = ""
-
-proc setupQuery(db: DbConn, query: SqlQuery,
-                args: varargs[string]): PStmt =
-  assert(not db.isNil, "Database not connected.")
-  var q = dbFormat(query, args)
-  if prepare_v2(db, q, q.len.cint, result, nil) != SQLITE_OK: dbError(db)
-
-proc setupQuery(db: DbConn, stmtName: SqlPrepared): SqlPrepared {.since: (1, 3).} =
-  assert(not db.isNil, "Database not connected.")
-  result = stmtName
-
-proc setRow(stmt: PStmt, r: var Row, cols: cint) =
-  for col in 0'i32..cols-1:
-    let cb = column_bytes(stmt, col)
-    setLen(r[col], cb) # set capacity
-    if column_type(stmt, col) == SQLITE_BLOB:
-      copyMem(addr(r[col][0]), column_blob(stmt, col), cb)
-    else:
-      setLen(r[col], 0)
-      let x = column_text(stmt, col)
-      if not isNil(x): add(r[col], x)
-
-iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## Executes the query and iterates over the result dataset.
-  ##
-  ## This is very fast, but potentially dangerous. Use this iterator only
-  ## if you require **ALL** the rows.
-  ##
-  ## **Note:** Breaking the `fastRows()` iterator during a loop will cause the
-  ## next database query to raise a `DbError` exception `unable to close due
-  ## to ...`.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    for row in db.fastRows(sql"SELECT id, name FROM my_table"):
-  ##      echo row
-  ##
-  ##    # Output:
-  ##    # @["1", "item#1"]
-  ##    # @["2", "item#2"]
-  ##
-  ##    db.close()
-  var stmt = setupQuery(db, query, args)
-  var L = (column_count(stmt))
-  var result = newRow(L)
-  try:
-    while step(stmt) == SQLITE_ROW:
-      setRow(stmt, result, L)
-      yield result
-  finally:
-    if finalize(stmt) != SQLITE_OK: dbError(db)
-
-iterator fastRows*(db: DbConn, stmtName: SqlPrepared): Row
-                  {.tags: [ReadDbEffect,WriteDbEffect], since: (1, 3).} =
-  discard setupQuery(db, stmtName)
-  var L = (column_count(stmtName.PStmt))
-  var result = newRow(L)
-  try:
-    while step(stmtName.PStmt) == SQLITE_ROW:
-      setRow(stmtName.PStmt, result, L)
-      yield result
-  except:
-    dbError(db)
-
-iterator instantRows*(db: DbConn, query: SqlQuery,
-                      args: varargs[string, `$`]): InstantRow
-                      {.tags: [ReadDbEffect].} =
-  ## Similar to `fastRows iterator <#fastRows.i,DbConn,SqlQuery,varargs[string,]>`_
-  ## but returns a handle that can be used to get column text
-  ## on demand using `[]`. Returned handle is valid only within the iterator body.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    for row in db.instantRows(sql"SELECT * FROM my_table"):
-  ##      echo "id:" & row[0]
-  ##      echo "name:" & row[1]
-  ##      echo "length:" & $len(row)
-  ##
-  ##    # Output:
-  ##    # id:1
-  ##    # name:item#1
-  ##    # length:2
-  ##    # id:2
-  ##    # name:item#2
-  ##    # length:2
-  ##
-  ##    db.close()
-  var stmt = setupQuery(db, query, args)
-  try:
-    while step(stmt) == SQLITE_ROW:
-      yield stmt
-  finally:
-    if finalize(stmt) != SQLITE_OK: dbError(db)
-
-iterator instantRows*(db: DbConn, stmtName: SqlPrepared): InstantRow
-                      {.tags: [ReadDbEffect,WriteDbEffect], since: (1, 3).} =
-  var stmt = setupQuery(db, stmtName).PStmt
-  try:
-    while step(stmt) == SQLITE_ROW:
-      yield stmt
-  except:
-    dbError(db)
-
-proc toTypeKind(t: var DbType; x: int32) =
-  case x
-  of SQLITE_INTEGER:
-    t.kind = dbInt
-    t.size = 8
-  of SQLITE_FLOAT:
-    t.kind = dbFloat
-    t.size = 8
-  of SQLITE_BLOB: t.kind = dbBlob
-  of SQLITE_NULL: t.kind = dbNull
-  of SQLITE_TEXT: t.kind = dbVarchar
-  else: t.kind = dbUnknown
-
-proc setColumns(columns: var DbColumns; x: PStmt) =
-  let L = column_count(x)
-  setLen(columns, L)
-  for i in 0'i32 ..< L:
-    columns[i].name = $column_name(x, i)
-    columns[i].typ.name = $column_decltype(x, i)
-    toTypeKind(columns[i].typ, column_type(x, i))
-    columns[i].tableName = $column_table_name(x, i)
-
-iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery,
-                      args: varargs[string, `$`]): InstantRow
-                      {.tags: [ReadDbEffect].} =
-  ## Similar to `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_,
-  ## but sets information about columns to `columns`.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    var columns: DbColumns
-  ##    for row in db.instantRows(columns, sql"SELECT * FROM my_table"):
-  ##      discard
-  ##    echo columns[0]
-  ##
-  ##    # Output:
-  ##    # (name: "id", tableName: "my_table", typ: (kind: dbNull,
-  ##    # notNull: false, name: "INTEGER", size: 0, maxReprLen: 0, precision: 0,
-  ##    # scale: 0, min: 0, max: 0, validValues: @[]), primaryKey: false,
-  ##    # foreignKey: false)
-  ##
-  ##    db.close()
-  var stmt = setupQuery(db, query, args)
-  setColumns(columns, stmt)
-  try:
-    while step(stmt) == SQLITE_ROW:
-      yield stmt
-  finally:
-    if finalize(stmt) != SQLITE_OK: dbError(db)
-
-proc `[]`*(row: InstantRow, col: int32): string {.inline.} =
-  ## Returns text for given column of the row.
-  ##
-  ## See also:
-  ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_
-  ##   example code
-  $column_text(row, col)
-
-proc unsafeColumnAt*(row: InstantRow, index: int32): cstring {.inline.} =
-  ## Returns cstring for given column of the row.
-  ##
-  ## See also:
-  ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_
-  ##   example code
-  column_text(row, index)
-
-proc len*(row: InstantRow): int32 {.inline.} =
-  ## Returns number of columns in a row.
-  ##
-  ## See also:
-  ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_
-  ##   example code
-  column_count(row)
-
-proc getRow*(db: DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## Retrieves a single row. If the query doesn't return any rows, this proc
-  ## will return a `Row` with empty strings for each column.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    doAssert db.getRow(sql"SELECT id, name FROM my_table"
-  ##                       ) == Row(@["1", "item#1"])
-  ##    doAssert db.getRow(sql"SELECT id, name FROM my_table WHERE id = ?",
-  ##                       2) == Row(@["2", "item#2"])
-  ##
-  ##    # Returns empty.
-  ##    doAssert db.getRow(sql"INSERT INTO my_table (id, name) VALUES (?, ?)",
-  ##                       3, "item#3") == @[]
-  ##    doAssert db.getRow(sql"DELETE FROM my_table WHERE id = ?", 3) == @[]
-  ##    doAssert db.getRow(sql"UPDATE my_table SET name = 'ITEM#1' WHERE id = ?",
-  ##                       1) == @[]
-  ##    db.close()
-  var stmt = setupQuery(db, query, args)
-  var L = (column_count(stmt))
-  result = newRow(L)
-  if step(stmt) == SQLITE_ROW:
-    setRow(stmt, result, L)
-  if finalize(stmt) != SQLITE_OK: dbError(db)
-
-proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} =
-  ## Executes the query and returns the whole result dataset.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    doAssert db.getAllRows(sql"SELECT id, name FROM my_table") == @[Row(@["1", "item#1"]), Row(@["2", "item#2"])]
-  ##    db.close()
-  result = @[]
-  for r in fastRows(db, query, args):
-    result.add(r)
-
-proc getAllRows*(db: DbConn, stmtName: SqlPrepared): seq[Row]
-                {.tags: [ReadDbEffect,WriteDbEffect], since: (1, 3).} =
-  result = @[]
-  for r in fastRows(db, stmtName):
-    result.add(r)
-
-iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## Similar to `fastRows iterator <#fastRows.i,DbConn,SqlQuery,varargs[string,]>`_,
-  ## but slower and safe.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    for row in db.rows(sql"SELECT id, name FROM my_table"):
-  ##      echo row
-  ##
-  ##    ## Output:
-  ##    ## @["1", "item#1"]
-  ##    ## @["2", "item#2"]
-  ##
-  ##    db.close()
-  for r in fastRows(db, query, args): yield r
-
-iterator rows*(db: DbConn, stmtName: SqlPrepared): Row
-              {.tags: [ReadDbEffect,WriteDbEffect], since: (1, 3).} =
-  for r in fastRows(db, stmtName): yield r
-
-proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} =
-  ## Executes the query and returns the first column of the first row of the
-  ## result dataset. Returns `""` if the dataset contains no rows or the database
-  ## value is `NULL`.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    doAssert db.getValue(sql"SELECT name FROM my_table WHERE id = ?",
-  ##                         2) == "item#2"
-  ##    doAssert db.getValue(sql"SELECT id, name FROM my_table") == "1"
-  ##    doAssert db.getValue(sql"SELECT name, id FROM my_table") == "item#1"
-  ##
-  ##    db.close()
-  var stmt = setupQuery(db, query, args)
-  if step(stmt) == SQLITE_ROW:
-    let cb = column_bytes(stmt, 0)
-    if cb == 0:
-      result = ""
-    else:
-      if column_type(stmt, 0) == SQLITE_BLOB:
-        result.setLen(cb)
-        copyMem(addr(result[0]), column_blob(stmt, 0), cb)
-      else:
-        result = newStringOfCap(cb)
-        add(result, column_text(stmt, 0))
-  else:
-    result = ""
-  if finalize(stmt) != SQLITE_OK: dbError(db)
-
-proc getValue*(db: DbConn,  stmtName: SqlPrepared): string
-              {.tags: [ReadDbEffect,WriteDbEffect], since: (1, 3).} =
-  var stmt = setupQuery(db, stmtName).PStmt
-  if step(stmt) == SQLITE_ROW:
-    let cb = column_bytes(stmt, 0)
-    if cb == 0:
-      result = ""
-    else:
-      if column_type(stmt, 0) == SQLITE_BLOB:
-        result.setLen(cb)
-        copyMem(addr(result[0]), column_blob(stmt, 0), cb)
-      else:
-        result = newStringOfCap(cb)
-        add(result, column_text(stmt, 0))
-  else:
-    result = ""
-
-proc tryInsertID*(db: DbConn, query: SqlQuery,
-                  args: varargs[string, `$`]): int64
-                  {.tags: [WriteDbEffect], raises: [].} =
-  ## Executes the query (typically "INSERT") and returns the
-  ## generated ID for the row or -1 in case of an error.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##    db.exec(sql"CREATE TABLE my_table (id INTEGER, name VARCHAR(50) NOT NULL)")
-  ##
-  ##    doAssert db.tryInsertID(sql"INSERT INTO not_exist_table (id, name) VALUES (?, ?)",
-  ##                            1, "item#1") == -1
-  ##    db.close()
-  assert(not db.isNil, "Database not connected.")
-  var q = dbFormat(query, args)
-  var stmt: sqlite3.PStmt
-  result = -1
-  if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK:
-    if step(stmt) == SQLITE_DONE:
-      result = last_insert_rowid(db)
-    if finalize(stmt) != SQLITE_OK:
-      result = -1
-  else:
-    discard finalize(stmt)
-
-proc insertID*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} =
-  ## Executes the query (typically "INSERT") and returns the
-  ## generated ID for the row.
-  ##
-  ## Raises a `DbError` exception when failed to insert row.
-  ## For Postgre this adds `RETURNING id` to the query, so it only works
-  ## if your primary key is named `id`.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##    db.exec(sql"CREATE TABLE my_table (id INTEGER, name VARCHAR(50) NOT NULL)")
-  ##
-  ##    for i in 0..2:
-  ##      let id = db.insertID(sql"INSERT INTO my_table (id, name) VALUES (?, ?)", i, "item#" & $i)
-  ##      echo "LoopIndex = ", i, ", InsertID = ", id
-  ##
-  ##    # Output:
-  ##    # LoopIndex = 0, InsertID = 1
-  ##    # LoopIndex = 1, InsertID = 2
-  ##    # LoopIndex = 2, InsertID = 3
-  ##
-  ##    db.close()
-  result = tryInsertID(db, query, args)
-  if result < 0: dbError(db)
-
-proc tryInsert*(db: DbConn, query: SqlQuery, pkName: string,
-                args: varargs[string, `$`]): int64
-               {.tags: [WriteDbEffect], raises: [], since: (1, 3).} =
-  ## same as tryInsertID
-  tryInsertID(db, query, args)
-
-proc insert*(db: DbConn, query: SqlQuery, pkName: string,
-             args: varargs[string, `$`]): int64
-            {.tags: [WriteDbEffect], since: (1, 3).} =
-  ## same as insertId
-  result = tryInsert(db, query,pkName, args)
-  if result < 0: dbError(db)
-
-proc execAffectedRows*(db: DbConn, query: SqlQuery,
-                       args: varargs[string, `$`]): int64 {.
-                       tags: [ReadDbEffect, WriteDbEffect].} =
-  ## Executes the query (typically "UPDATE") and returns the
-  ## number of affected rows.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##
-  ##    # Records of my_table:
-  ##    # | id | name     |
-  ##    # |----|----------|
-  ##    # |  1 | item#1   |
-  ##    # |  2 | item#2   |
-  ##
-  ##    doAssert db.execAffectedRows(sql"UPDATE my_table SET name = 'TEST'") == 2
-  ##
-  ##    db.close()
-  exec(db, query, args)
-  result = changes(db)
-
-proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared): int64
-                      {.tags: [ReadDbEffect, WriteDbEffect],since: (1, 3).} =
-  exec(db, stmtName)
-  result = changes(db)
-
-proc close*(db: DbConn) {.tags: [DbEffect].} =
-  ## Closes the database connection.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    let db = open("mytest.db", "", "", "")
-  ##    db.close()
-  if sqlite3.close(db) != SQLITE_OK: dbError(db)
-
-proc open*(connection, user, password, database: string): DbConn {.
-  tags: [DbEffect].} =
-  ## Opens a database connection. Raises a `DbError` exception if the connection
-  ## could not be established.
-  ##
-  ## **Note:** Only the `connection` parameter is used for `sqlite`.
-  ##
-  ## **Examples:**
-  ##
-  ## .. code-block:: Nim
-  ##
-  ##    try:
-  ##      let db = open("mytest.db", "", "", "")
-  ##      ## do something...
-  ##      ## db.getAllRows(sql"SELECT * FROM my_table")
-  ##      db.close()
-  ##    except:
-  ##      stderr.writeLine(getCurrentExceptionMsg())
-  var db: DbConn
-  if sqlite3.open(connection, db) == SQLITE_OK:
-    result = db
-  else:
-    dbError(db)
-
-proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [DbEffect].} =
-  ## Sets the encoding of a database connection, returns `true` for
-  ## success, `false` for failure.
-  ##
-  ## **Note:** The encoding cannot be changed once it's been set.
-  ## According to SQLite3 documentation, any attempt to change
-  ## the encoding after the database is created will be silently
-  ## ignored.
-  exec(connection, sql"PRAGMA encoding = ?", [encoding])
-  result = connection.getValue(sql"PRAGMA encoding") == encoding
-
-proc finalize*(sqlPrepared:SqlPrepared) {.discardable, since: (1, 3).} =
-  discard finalize(sqlPrepared.PStmt)
-
-template dbBindParamError*(paramIdx: int, val: varargs[untyped]) =
-  ## Raises a `DbError` exception.
-  var e: ref DbError
-  new(e)
-  e.msg = "error binding param in position " & $paramIdx
-  raise e
-
-proc bindParam*(ps: SqlPrepared, paramIdx: int, val: int32) {.since: (1, 3).} =
-  ## Binds a int32  to the specified paramIndex.
-  if bind_int(ps.PStmt, paramIdx.int32, val) != SQLITE_OK:
-    dbBindParamError(paramIdx, val)
-
-proc bindParam*(ps: SqlPrepared, paramIdx: int, val: int64) {.since: (1, 3).} =
-  ## Binds a int64  to the specified paramIndex.
-  if bind_int64(ps.PStmt, paramIdx.int32, val) != SQLITE_OK:
-    dbBindParamError(paramIdx, val)
-
-proc bindParam*(ps: SqlPrepared, paramIdx: int, val: int) {.since: (1, 3).} =
-  ## Binds a int  to the specified paramIndex.
-  when sizeof(int) == 8:
-    bindParam(ps, paramIdx, val.int64)
-  else:
-    bindParam(ps, paramIdx, val.int32)
-
-proc bindParam*(ps: SqlPrepared, paramIdx: int, val: float64) {.since: (1, 3).} =
-  ## Binds a 64bit float to the specified paramIndex.
-  if bind_double(ps.PStmt, paramIdx.int32, val) != SQLITE_OK:
-    dbBindParamError(paramIdx, val)
-
-proc bindNull*(ps: SqlPrepared, paramIdx: int) {.since: (1, 3).} =
-  ## Sets the bindparam at the specified paramIndex to null
-  ## (default behaviour by sqlite).
-  if bind_null(ps.PStmt, paramIdx.int32) != SQLITE_OK:
-    dbBindParamError(paramIdx)
-
-proc bindParam*(ps: SqlPrepared, paramIdx: int, val: string, copy = true) {.since: (1, 3).} =
-  ## Binds a string to the specified paramIndex.
-  ## if copy is true then SQLite makes its own private copy of the data immediately
-  if bind_text(ps.PStmt, paramIdx.int32, val.cstring, val.len.int32, if copy: SQLITE_TRANSIENT else: SQLITE_STATIC) != SQLITE_OK:
-    dbBindParamError(paramIdx, val)
-
-proc bindParam*(ps: SqlPrepared, paramIdx: int,val: openArray[byte], copy = true) {.since: (1, 3).} =
-  ## binds a blob to the specified paramIndex.
-  ## if copy is true then SQLite makes its own private copy of the data immediately
-  let len = val.len
-  if bind_blob(ps.PStmt, paramIdx.int32, val[0].unsafeAddr, len.int32, if copy: SQLITE_TRANSIENT else: SQLITE_STATIC) != SQLITE_OK:
-    dbBindParamError(paramIdx, val)
-
-macro bindParams*(ps: SqlPrepared, params: varargs[untyped]): untyped {.since: (1, 3).} =
-  let bindParam = bindSym("bindParam", brOpen)
-  let bindNull = bindSym("bindNull")
-  let preparedStatement = genSym()
-  result = newStmtList()
-  # Store `ps` in a temporary variable. This prevents `ps` from being evaluated every call.
-  result.add newNimNode(nnkLetSection).add(newIdentDefs(preparedStatement, newEmptyNode(), ps))
-  for idx, param in params:
-    if param.kind != nnkNilLit:
-      result.add newCall(bindParam, preparedStatement, newIntLitNode idx + 1, param)
-    else:
-      result.add newCall(bindNull, preparedStatement, newIntLitNode idx + 1)
-
-macro untypedLen(args: varargs[untyped]): int =
-  newLit(args.len)
-
-template exec*(db: DbConn, stmtName: SqlPrepared,
-          args: varargs[typed]): untyped =
-  when untypedLen(args) > 0:
-    if reset(stmtName.PStmt) != SQLITE_OK:
-      dbError(db)
-    if clear_bindings(stmtName.PStmt) != SQLITE_OK:
-      dbError(db)
-    stmtName.bindParams(args)
-  if not tryExec(db, stmtName): dbError(db)
-
-when not defined(testing) and isMainModule:
-  var db = open(":memory:", "", "", "")
-  exec(db, sql"create table tbl1(one varchar(10), two smallint)", [])
-  exec(db, sql"insert into tbl1 values('hello!',10)", [])
-  exec(db, sql"insert into tbl1 values('goodbye', 20)", [])
-  var p1 = db.prepare "create table tbl2(one varchar(10), two smallint)"
-  exec(db, p1)
-  finalize(p1)
-  var p2 = db.prepare "insert into tbl2 values('hello!',10)"
-  exec(db, p2)
-  finalize(p2)
-  var p3 = db.prepare "insert into tbl2 values('goodbye', 20)"
-  exec(db, p3)
-  finalize(p3)
-  #db.query("create table tbl1(one varchar(10), two smallint)")
-  #db.query("insert into tbl1 values('hello!',10)")
-  #db.query("insert into tbl1 values('goodbye', 20)")
-  for r in db.rows(sql"select * from tbl1", []):
-    echo(r[0], r[1])
-  for r in db.instantRows(sql"select * from tbl1", []):
-    echo(r[0], r[1])
-  var p4 =  db.prepare "select * from tbl2"
-  for r in db.rows(p4):
-    echo(r[0], r[1])
-  finalize(p4)
-  var i5 = 0
-  var p5 =  db.prepare "select * from tbl2"
-  for r in db.instantRows(p5):
-    inc i5
-    echo(r[0], r[1])
-  assert i5 == 2
-  finalize(p5)
-
-  for r in db.rows(sql"select * from tbl2", []):
-    echo(r[0], r[1])
-  for r in db.instantRows(sql"select * from tbl2", []):
-    echo(r[0], r[1])
-  var p6 = db.prepare "select * from tbl2 where one = ? "
-  p6.bindParams("goodbye")
-  var rowsP3 = 0
-  for r in db.rows(p6):
-    rowsP3 = 1
-    echo(r[0], r[1])
-  assert rowsP3 == 1
-  finalize(p6)
-
-  var p7 = db.prepare "select * from tbl2 where two=?"
-  p7.bindParams(20'i32)
-  when sizeof(int) == 4:
-    p7.bindParams(20)
-  var rowsP = 0
-  for r in db.rows(p7):
-    rowsP = 1
-    echo(r[0], r[1])
-  assert rowsP == 1
-  finalize(p7)
-
-  exec(db, sql"CREATE TABLE photos(ID INTEGER PRIMARY KEY AUTOINCREMENT, photo BLOB)")
-  var p8 = db.prepare "INSERT INTO photos (ID,PHOTO) VALUES (?,?)"
-  var d = "abcdefghijklmnopqrstuvwxyz"
-  p8.bindParams(1'i32, "abcdefghijklmnopqrstuvwxyz")
-  exec(db, p8)
-  finalize(p8)
-  var p10 = db.prepare "INSERT INTO photos (ID,PHOTO) VALUES (?,?)"
-  p10.bindParams(2'i32,nil)
-  exec(db, p10)
-  exec( db, p10, 3, nil)
-  finalize(p10)
-  for r in db.rows(sql"select * from photos where ID = 1", []):
-    assert r[1].len == d.len
-    assert r[1] == d
-  var i6 = 0
-  for r in db.rows(sql"select * from photos where ID = 3", []):
-    i6 = 1
-  assert i6 == 1
-  var p9 = db.prepare("select * from photos where PHOTO is ?")
-  p9.bindParams(nil)
-  var rowsP2 = 0
-  for r in db.rows(p9):
-    rowsP2 = 1
-    echo(r[0], repr r[1])
-  assert rowsP2 == 1
-  finalize(p9)
-
-  db_sqlite.close(db)
diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim
index 7b2d7d3ee1..39d238055d 100644
--- a/lib/impure/nre.nim
+++ b/lib/impure/nre.nim
@@ -20,15 +20,17 @@ when defined(js):
 ## search the internet for a wide variety of third-party documentation and
 ## tools.
 ##
-## **Note**: If you love `sequtils.toSeq` we have bad news for you. This
-## library doesn't work with it due to documented compiler limitations. As
-## a workaround, use this:
-##
-## .. code-block:: nim
-##
-##    import std/nre except toSeq
-##
-##
+## .. warning:: If you love `sequtils.toSeq` we have bad news for you. This
+##   library doesn't work with it due to documented compiler limitations. As
+##   a workaround, use this:
+runnableExamples:
+  # either `import std/nre except toSeq` or fully qualify `sequtils.toSeq`:
+  import std/sequtils
+  iterator iota(n: int): int =
+    for i in 0..`_
     ##
     ## `captureCount: int`
-    ##     the number of captures that the pattern has.
+    ## :   the number of captures that the pattern has.
     ##
     ## `captureNameId: Table[string, int]`
-    ##     a table from the capture names to their numeric id.
+    ## :   a table from the capture names to their numeric id.
     ##
     ##
     ## Options
@@ -145,7 +145,7 @@ type
     ## `DOLLAR_ENDONLY`, `FIRSTLINE`, `NO_AUTO_CAPTURE`,
     ## `JAVASCRIPT_COMPAT`, `U`, `NO_STUDY`. In other PCRE wrappers, you
     ## will need to pass these as separate flags to PCRE.
-    pattern*: string  ## not nil
+    pattern*: string
     pcreObj: ptr pcre.Pcre  ## not nil
     pcreExtra: ptr pcre.ExtraData  ## nil
 
@@ -156,49 +156,39 @@ type
     ## execution. On failure, it is none, on success, it is some.
     ##
     ## `pattern: Regex`
-    ##     the pattern that is being matched
+    ## :   the pattern that is being matched
     ##
     ## `str: string`
-    ##     the string that was matched against
+    ## :   the string that was matched against
     ##
     ## `captures[]: string`
-    ##     the string value of whatever was captured at that id. If the value
+    ## :   the string value of whatever was captured at that id. If the value
     ##     is invalid, then behavior is undefined. If the id is `-1`, then
     ##     the whole match is returned. If the given capture was not matched,
-    ##     `nil` is returned.
-    ##
-    ##     -  `"abc".match(re"(\w)").get.captures[0] == "a"`
-    ##     -  `"abc".match(re"(?\w)").get.captures["letter"] == "a"`
-    ##     -  `"abc".match(re"(\w)\w").get.captures[-1] == "ab"`
+    ##     `nil` is returned. See examples for `match`.
     ##
     ## `captureBounds[]: HSlice[int, int]`
-    ##     gets the bounds of the given capture according to the same rules as
+    ## :   gets the bounds of the given capture according to the same rules as
     ##     the above. If the capture is not filled, then `None` is returned.
-    ##     The bounds are both inclusive.
-    ##
-    ##     -  `"abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0`
-    ##     -  `0 in "abc".match(re"(\w)").get.captureBounds == true`
-    ##     -  `"abc".match(re"").get.captureBounds[-1] == 0 .. -1`
-    ##     -  `"abc".match(re"abc").get.captureBounds[-1] == 0 .. 2`
+    ##     The bounds are both inclusive.  See examples for `match`.
     ##
     ## `match: string`
-    ##     the full text of the match.
+    ## :   the full text of the match.
     ##
     ## `matchBounds: HSlice[int, int]`
-    ##     the bounds of the match, as in `captureBounds[]`
+    ## :   the bounds of the match, as in `captureBounds[]`
     ##
     ## `(captureBounds|captures).toTable`
-    ##     returns a table with each named capture as a key.
+    ## :   returns a table with each named capture as a key.
     ##
     ## `(captureBounds|captures).toSeq`
-    ##     returns all the captures by their number.
+    ## :   returns all the captures by their number.
     ##
     ## `$: string`
-    ##     same as `match`
+    ## :   same as `match`
     pattern*: Regex  ## The regex doing the matching.
                      ## Not nil.
     str*: string  ## The string that was matched against.
-                  ## Not nil.
     pcreMatchBounds: seq[HSlice[cint, cint]] ## First item is the bounds of the match
                                             ## Other items are the captures
                                             ## `a` is inclusive start, `b` is exclusive end
@@ -226,22 +216,12 @@ type
     ## for whatever reason. The message contains the error
     ## code.
 
-runnableExamples:
-  # This MUST be kept in sync with the examples in RegexMatch
-  doAssert "abc".match(re"(\w)").get.captures[0] == "a"
-  doAssert "abc".match(re"(?\w)").get.captures["letter"] == "a"
-  doAssert "abc".match(re"(\w)\w").get.captures[-1] == "ab"
-
-  doAssert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0
-  doAssert 0 in "abc".match(re"(\w)").get.captureBounds == true
-  doAssert "abc".match(re"").get.captureBounds[-1] == 0 .. -1
-  doAssert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2
-
-
 proc destroyRegex(pattern: Regex) =
+  `=destroy`(pattern.pattern)
   pcre.free_substring(cast[cstring](pattern.pcreObj))
   if pattern.pcreExtra != nil:
     pcre.free_study(pattern.pcreExtra)
+  `=destroy`(pattern.captureNameToId)
 
 proc getinfo[T](pattern: Regex, opt: cint): T =
   let retcode = pcre.fullinfo(pattern.pcreObj, pattern.pcreExtra, opt, addr result)
@@ -308,7 +288,7 @@ proc matchesCrLf(pattern: Regex): bool =
   let newlineFlags = flags and (pcre.NEWLINE_CRLF or
                                 pcre.NEWLINE_ANY or
                                 pcre.NEWLINE_ANYCRLF)
-  if newLineFlags > 0u32:
+  if newlineFlags > 0u32:
     return true
 
   # get flags from build config
@@ -540,24 +520,33 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt
 proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] =
   ## Like `find(...)<#find,string,Regex,int>`_, but anchored to the start of the
   ## string.
-  ##
   runnableExamples:
-    doAssert "foo".match(re"f").isSome
-    doAssert "foo".match(re"o").isNone
+    assert "foo".match(re"f").isSome
+    assert "foo".match(re"o").isNone
 
+    assert "abc".match(re"(\w)").get.captures[0] == "a"
+    assert "abc".match(re"(?\w)").get.captures["letter"] == "a"
+    assert "abc".match(re"(\w)\w").get.captures[-1] == "ab"
+
+    assert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0
+    assert 0 in "abc".match(re"(\w)").get.captureBounds
+    assert "abc".match(re"").get.captureBounds[-1] == 0 .. -1
+    assert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2
   return str.matchImpl(pattern, start, endpos, pcre.ANCHORED)
 
 iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch =
   ## Works the same as `find(...)<#find,string,Regex,int>`_, but finds every
-  ## non-overlapping match. `"2222".find(re"22")` is `"22", "22"`, not
-  ## `"22", "22", "22"`.
-  ##
+  ## non-overlapping match:
+  runnableExamples:
+    import std/sugar
+    assert collect(for a in "2222".findIter(re"22"): a.match) == @["22", "22"]
+     # not @["22", "22", "22"]
   ## Arguments are the same as `find(...)<#find,string,Regex,int>`_
   ##
   ## Variants:
   ##
   ## -  `proc findAll(...)` returns a `seq[string]`
-  # see pcredemo for explanation
+  # see pcredemo for explanation => https://www.pcre.org/original/doc/html/pcredemo.html
   let matchesCrLf = pattern.matchesCrLf()
   let unicode = uint32(getinfo[culong](pattern, pcre.INFO_OPTIONS) and
     pcre.UTF8) > 0u32
@@ -578,7 +567,7 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R
       # either the end of the input or the string
       # cannot be split here - we also need to bail
       # if we've never matched and we've already tried to...
-      if offset >= strlen or neverMatched:
+      if flags == 0 or offset >= strlen or neverMatched: # All matches found
         break
 
       if matchesCrLf and offset < (str.len - 1) and
@@ -594,7 +583,6 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R
     else:
       neverMatched = false
       offset = match.get.matchBounds.b + 1
-
       yield match.get
 
 proc find*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] =
@@ -602,11 +590,11 @@ proc find*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[Re
   ## positions.
   ##
   ## `start`
-  ##     The start point at which to start matching. `|abc` is `0`;
+  ## :   The start point at which to start matching. `|abc` is `0`;
   ##     `a|bc` is `1`
   ##
   ## `endpos`
-  ##     The maximum index for a match; `int.high` means the end of the
+  ## :   The maximum index for a match; `int.high` means the end of the
   ##     string, otherwise it’s an inclusive upper bound.
   return str.matchImpl(pattern, start, endpos, 0)
 
@@ -619,11 +607,10 @@ proc contains*(str: string, pattern: Regex, start = 0, endpos = int.high): bool
   ## Determine if the string contains the given pattern between the end and
   ## start positions:
   ## This function is equivalent to `isSome(str.find(pattern, start, endpos))`.
-  ##
   runnableExamples:
-    doAssert "abc".contains(re"bc")
-    doAssert not "abc".contains(re"cd")
-    doAssert not "abc".contains(re"a", start = 1)
+    assert "abc".contains(re"bc")
+    assert not "abc".contains(re"cd")
+    assert not "abc".contains(re"a", start = 1)
 
   return isSome(str.find(pattern, start, endpos))
 
@@ -635,16 +622,16 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string]
   ##
   runnableExamples:
     # -  If the match is zero-width, then the string is still split:
-    doAssert "123".split(re"") == @["1", "2", "3"]
+    assert "123".split(re"") == @["1", "2", "3"]
 
     # -  If the pattern has a capture in it, it is added after the string
     #    split:
-    doAssert "12".split(re"(\d)") == @["", "1", "", "2", ""]
+    assert "12".split(re"(\d)") == @["", "1", "", "2", ""]
 
     # -  If `maxsplit != -1`, then the string will only be split
     #    `maxsplit - 1` times. This means that there will be `maxsplit`
     #    strings in the output seq.
-    doAssert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"]
+    assert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"]
 
   result = @[]
   var lastIdx = start
@@ -715,8 +702,7 @@ proc replace*(str: string, pattern: Regex,
   ## each match and the return value is the replacement value.
   ##
   ## If `subproc` is a `proc (string): string`, then it is executed with the
-  ## full text of the match and and the return value is the replacement
-  ## value.
+  ## full text of the match and the return value is the replacement value.
   ##
   ## If `subproc` is a string, the syntax is as follows:
   ##
@@ -747,9 +733,9 @@ proc escapeRe*(str: string): string {.gcsafe.} =
   ##
   ## Escaped char: `\ + * ? [ ^ ] $ ( ) { } = ! < > | : -`
   runnableExamples:
-    doAssert escapeRe("fly+wind") == "fly\\+wind"
-    doAssert escapeRe("!") == "\\!"
-    doAssert escapeRe("nim*") == "nim\\*"
+    assert escapeRe("fly+wind") == "fly\\+wind"
+    assert escapeRe("!") == "\\!"
+    assert escapeRe("nim*") == "nim\\*"
 
   #([\\+*?[^\]$(){}=!<>|:-])
   const SpecialCharMatcher = {'\\', '+', '*', '?', '[', '^', ']', '$', '(',
diff --git a/lib/impure/nre/private/util.nim b/lib/impure/nre/private/util.nim
index d227dcba3b..ed84207766 100644
--- a/lib/impure/nre/private/util.nim
+++ b/lib/impure/nre/private/util.nim
@@ -1,5 +1,5 @@
 ## INTERNAL FILE FOR USE ONLY BY nre.nim.
-import tables
+import std/tables
 
 const Ident = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'}
 const StartIdent = Ident - {'0'..'9'}
diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim
index c580b89d1f..f4fc26380b 100644
--- a/lib/impure/rdstdin.nim
+++ b/lib/impure/rdstdin.nim
@@ -22,11 +22,16 @@ runnableExamples("-r:off"):
     if line.len > 0: echo line
   echo "exiting"
 
+
 when defined(windows):
+  when defined(nimPreviewSlimSystem):
+    import std/syncio
+
   proc readLineFromStdin*(prompt: string): string {.
                           tags: [ReadIOEffect, WriteIOEffect].} =
     ## Reads a line from stdin.
     stdout.write(prompt)
+    stdout.flushFile()
     result = readLine(stdin)
 
   proc readLineFromStdin*(prompt: string, line: var string): bool {.
@@ -50,17 +55,7 @@ elif defined(genode):
     stdin.readLine(line)
 
 else:
-  import linenoise
-
-  proc readLineFromStdin*(prompt: string): string {.
-                          tags: [ReadIOEffect, WriteIOEffect].} =
-    var buffer = linenoise.readLine(prompt)
-    if isNil(buffer):
-      raise newException(IOError, "Linenoise returned nil")
-    result = $buffer
-    if result.len > 0:
-      historyAdd(buffer)
-    linenoise.free(buffer)
+  import std/linenoise
 
   proc readLineFromStdin*(prompt: string, line: var string): bool {.
                           tags: [ReadIOEffect, WriteIOEffect].} =
@@ -73,3 +68,7 @@ else:
       historyAdd(buffer)
     linenoise.free(buffer)
     result = true
+
+  proc readLineFromStdin*(prompt: string): string {.inline.} =
+    if not readLineFromStdin(prompt, result):
+      raise newException(IOError, "Linenoise returned nil")
diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 0c96876b91..053c6ab555 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -17,6 +17,10 @@ when defined(js):
 ## C library. This means that your application will depend on the PCRE
 ## library's licence when using this module, which should not be a problem
 ## though.
+##
+## .. note:: There are also alternative nimble packages such as [tinyre](https://github.com/khchen/tinyre)
+##   and [regex](https://github.com/nitely/nim-regex).
+##
 ## PCRE's licence follows:
 ##
 ## .. include:: ../../doc/regexprs.txt
@@ -32,7 +36,10 @@ runnableExamples:
     # can't match start of string since we're starting at 1
 
 import
-  pcre, strutils, rtarrays
+  std/[pcre, strutils, rtarrays]
+
+when defined(nimPreviewSlimSystem):
+  import std/syncio
 
 const
   MaxSubpatterns* = 20
@@ -41,12 +48,12 @@ const
 
 type
   RegexFlag* = enum     ## options for regular expressions
-    reIgnoreCase = 0,    ## do caseless matching
-    reMultiLine = 1,     ## `^` and `$` match newlines within data
-    reDotAll = 2,        ## `.` matches anything including NL
-    reExtended = 3,      ## ignore whitespace and `#` comments
-    reStudy = 4          ## study the expression (may be omitted if the
-                         ## expression will be used only once)
+    reIgnoreCase = 0,   ## do caseless matching
+    reMultiLine = 1,    ## `^` and `$` match newlines within data
+    reDotAll = 2,       ## `.` matches anything including NL
+    reExtended = 3,     ## ignore whitespace and `#` comments
+    reStudy = 4         ## study the expression (may be omitted if the
+                        ## expression will be used only once)
 
   RegexDesc = object
     h: ptr Pcre
@@ -58,10 +65,16 @@ type
     ## is raised if the pattern is no valid regular expression.
 
 when defined(gcDestructors):
-  proc `=destroy`(x: var RegexDesc) =
-    pcre.free_substring(cast[cstring](x.h))
-    if not isNil(x.e):
-      pcre.free_study(x.e)
+  when defined(nimAllowNonVarDestructor):
+    proc `=destroy`(x: RegexDesc) =
+      pcre.free_substring(cast[cstring](x.h))
+      if not isNil(x.e):
+        pcre.free_study(x.e)
+  else:
+    proc `=destroy`(x: var RegexDesc) =
+      pcre.free_substring(cast[cstring](x.h))
+      if not isNil(x.e):
+        pcre.free_study(x.e)
 
 proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} =
   var e: ref RegexError
@@ -141,6 +154,10 @@ proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string],
     else: matches[i-1] = ""
   return rawMatches[1] - rawMatches[0]
 
+const MaxReBufSize* = high(cint)
+  ## Maximum PCRE (API 1) buffer start/size equal to `high(cint)`, which even
+  ## for 64-bit systems can be either 2`31`:sup:-1 or 2`63`:sup:-1.
+
 proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string],
                  start = 0, bufSize: int): tuple[first, last: int] =
   ## returns the starting position and end position of `pattern` in `buf`
@@ -148,6 +165,9 @@ proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string],
   ## and the captured
   ## substrings in the array `matches`. If it does not match, nothing
   ## is written into `matches` and `(-1,0)` is returned.
+  ##
+  ## Note: The memory for `matches` needs to be allocated before this function is
+  ## called, otherwise it will just remain empty.
   var
     rtarray = initRtArray[cint]((matches.len+1)*3)
     rawMatches = rtarray.getRawData
@@ -167,16 +187,27 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string],
   ## and the captured substrings in the array `matches`.
   ## If it does not match, nothing
   ## is written into `matches` and `(-1,0)` is returned.
-  result = findBounds(cstring(s), pattern, matches, start, s.len)
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
+  runnableExamples:
+    var matches = newSeq[string](1)
+    let (first, last) = findBounds("Hello World", re"(W\w+)", matches)
+    doAssert first == 6
+    doAssert last == 10
+    doAssert matches[0] == "World"
+  result = findBounds(cstring(s), pattern, matches,
+      min(start, MaxReBufSize), min(s.len, MaxReBufSize))
 
 proc findBounds*(buf: cstring, pattern: Regex,
                  matches: var openArray[tuple[first, last: int]],
-                 start = 0, bufSize = 0): tuple[first, last: int] =
+                 start = 0, bufSize: int): tuple[first, last: int] =
   ## returns the starting position and end position of `pattern` in `buf`
   ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated),
   ## and the captured substrings in the array `matches`.
   ## If it does not match, nothing is written into `matches` and
   ## `(-1,0)` is returned.
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   var
     rtarray = initRtArray[cint]((matches.len+1)*3)
     rawMatches = rtarray.getRawData
@@ -197,7 +228,28 @@ proc findBounds*(s: string, pattern: Regex,
   ## and the captured substrings in the array `matches`.
   ## If it does not match, nothing is written into `matches` and
   ## `(-1,0)` is returned.
-  result = findBounds(cstring(s), pattern, matches, start, s.len)
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
+  runnableExamples:
+    var matches = newSeq[tuple[first, last: int]](1)
+    let (first, last) = findBounds("Hello World", re"(\w+)", matches)
+    doAssert first == 0
+    doAssert last == 4
+    doAssert matches[0] == (0, 4)
+  result = findBounds(cstring(s), pattern, matches,
+      min(start, MaxReBufSize), min(s.len, MaxReBufSize))
+
+proc findBoundsImpl(buf: cstring, pattern: Regex,
+                    start = 0, bufSize = 0, flags = 0): tuple[first, last: int] =
+  var rtarray = initRtArray[cint](3)
+  let rawMatches = rtarray.getRawData
+  let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags.int32,
+                cast[ptr cint](rawMatches), 3)
+
+  if res < 0'i32:
+    result = (-1, 0)
+  else:
+    result = (int(rawMatches[0]), int(rawMatches[1]-1))
 
 proc findBounds*(buf: cstring, pattern: Regex,
                  start = 0, bufSize: int): tuple[first, last: int] =
@@ -220,7 +272,8 @@ proc findBounds*(s: string, pattern: Regex,
   ## Note: there is a speed improvement if the matches do not need to be captured.
   runnableExamples:
     assert findBounds("01234abc89", re"abc") == (5,7)
-  result = findBounds(cstring(s), pattern, start, s.len)
+  result = findBounds(cstring(s), pattern,
+      min(start, MaxReBufSize), min(s.len, MaxReBufSize))
 
 proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint): cint =
   var
@@ -236,6 +289,8 @@ proc matchLen*(s: string, pattern: Regex, matches: var openArray[string],
   ## the same as `match`, but it returns the length of the match,
   ## if there is no match, `-1` is returned. Note that a match length
   ## of zero can happen.
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED)
 
 proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string],
@@ -243,6 +298,8 @@ proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string],
   ## the same as `match`, but it returns the length of the match,
   ## if there is no match, `-1` is returned. Note that a match length
   ## of zero can happen.
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   return matchOrFind(buf, pattern, matches, start.cint, bufSize.cint, pcre.ANCHORED)
 
 proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} =
@@ -273,6 +330,7 @@ proc match*(s: string, pattern: Regex, matches: var openArray[string],
   ## match, nothing is written into `matches` and `false` is
   ## returned.
   ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   runnableExamples:
     import std/sequtils
     var matches: array[2, string]
@@ -287,14 +345,18 @@ proc match*(buf: cstring, pattern: Regex, matches: var openArray[string],
   ## match, nothing is written into `matches` and `false` is
   ## returned.
   ## `buf` has length `bufSize` (not necessarily `'\0'` terminated).
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   result = matchLen(buf, pattern, matches, start, bufSize) != -1
 
 proc find*(buf: cstring, pattern: Regex, matches: var openArray[string],
-           start = 0, bufSize = 0): int =
+           start = 0, bufSize: int): int =
   ## returns the starting position of `pattern` in `buf` and the captured
   ## substrings in the array `matches`. If it does not match, nothing
   ## is written into `matches` and `-1` is returned.
   ## `buf` has length `bufSize` (not necessarily `'\0'` terminated).
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   var
     rtarray = initRtArray[cint]((matches.len+1)*3)
     rawMatches = rtarray.getRawData
@@ -313,6 +375,8 @@ proc find*(s: string, pattern: Regex, matches: var openArray[string],
   ## returns the starting position of `pattern` in `s` and the captured
   ## substrings in the array `matches`. If it does not match, nothing
   ## is written into `matches` and `-1` is returned.
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   result = find(cstring(s), pattern, matches, start, s.len)
 
 proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int =
@@ -382,7 +446,7 @@ iterator findAll*(buf: cstring, pattern: Regex, start = 0, bufSize: int): string
 
 proc findAll*(s: string, pattern: Regex, start = 0): seq[string] {.inline.} =
   ## returns all matching `substrings` of `s` that match `pattern`.
-  ## If it does not match, @[] is returned.
+  ## If it does not match, `@[]` is returned.
   result = @[]
   for x in findAll(s, pattern, start): result.add x
 
@@ -396,7 +460,7 @@ template `=~` *(s: string, pattern: Regex): untyped =
       elif line =~ re"\s*(\#.*)": # matches a comment
         # note that the implicit `matches` array is different from 1st branch
         result = $(matches[0],)
-      else: doAssert false
+      else: raiseAssert "unreachable"
       doAssert not declared(matches)
     doAssert parse("NAME = LENA") == """("NAME", "LENA")"""
     doAssert parse("   # comment ... ") == """("# comment ... ",)"""
@@ -414,6 +478,8 @@ proc contains*(s: string, pattern: Regex, start = 0): bool {.inline.} =
 proc contains*(s: string, pattern: Regex, matches: var openArray[string],
               start = 0): bool {.inline.} =
   ## same as `find(s, pattern, matches, start) >= 0`
+  ##
+  ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty.
   return find(s, pattern, matches, start) >= 0
 
 proc startsWith*(s: string, prefix: Regex): bool {.inline.} =
@@ -433,12 +499,16 @@ proc replace*(s: string, sub: Regex, by = ""): string =
     doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)", "?") == "?; ?"
   result = ""
   var prev = 0
+  var flags = int32(0)
   while prev < s.len:
-    var match = findBounds(s, sub, prev)
+    var match = findBoundsImpl(s.cstring, sub, prev, s.len, flags)
+    flags = 0
     if match.first < 0: break
     add(result, substr(s, prev, match.first-1))
     add(result, by)
-    if match.last + 1 == prev: break
+    if match.first > match.last:
+      # 0-len match
+      flags = pcre.NOTEMPTY_ATSTART
     prev = match.last + 1
   add(result, substr(s, prev))
 
@@ -499,19 +569,22 @@ iterator split*(s: string, sep: Regex; maxsplit = -1): string =
       @["", "this", "is", "an", "example", ""]
   var last = 0
   var splits = maxsplit
-  var x: int
+  var x = -1
+  if len(s) == 0:
+    last = 1
+  if matchLen(s, sep, 0) == 0:
+    x = 0
   while last <= len(s):
     var first = last
     var sepLen = 1
+    if x == 0:
+      inc(last)
     while last < len(s):
       x = matchLen(s, sep, last)
       if x >= 0:
         sepLen = x
         break
       inc(last)
-    if x == 0:
-      if last >= len(s): break
-      inc last
     if splits == 0: last = len(s)
     yield substr(s, first, last-1)
     if splits == 0: break
diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim
index 73af232b4c..b7ebc905fe 100644
--- a/lib/js/asyncjs.nim
+++ b/lib/js/asyncjs.nim
@@ -17,37 +17,42 @@
 ##
 ## This is roughly equivalent to the `async` keyword in JavaScript code.
 ##
-## .. code-block:: nim
-##  proc loadGame(name: string): Future[Game] {.async.} =
-##    # code
+##   ```nim
+##   proc loadGame(name: string): Future[Game] {.async.} =
+##     # code
+##   ```
 ##
 ## should be equivalent to
 ##
-## .. code-block:: javascript
+##   ```javascript
 ##   async function loadGame(name) {
 ##     // code
 ##   }
+##   ```
 ##
 ## A call to an asynchronous procedure usually needs `await` to wait for
 ## the completion of the `Future`.
 ##
-## .. code-block:: nim
+##   ```nim
 ##   var game = await loadGame(name)
+##   ```
 ##
 ## Often, you might work with callback-based API-s. You can wrap them with
 ## asynchronous procedures using promises and `newPromise`:
 ##
-## .. code-block:: nim
+##   ```nim
 ##   proc loadGame(name: string): Future[Game] =
 ##     var promise = newPromise() do (resolve: proc(response: Game)):
 ##       cbBasedLoadGame(name) do (game: Game):
 ##         resolve(game)
 ##     return promise
+##   ```
 ##
 ## Forward definitions work properly, you just need to always add the `{.async.}` pragma:
 ##
-## .. code-block:: nim
+##   ```nim
 ##   proc loadGame(name: string): Future[Game] {.async.}
+##   ```
 ##
 ## JavaScript compatibility
 ## ========================
@@ -57,20 +62,21 @@
 ## If you need to use this module with older versions of JavaScript, you can
 ## use a tool that backports the resulting JavaScript code, as babel.
 
-# xxx code-block:: javascript above gives `LanguageXNotSupported` warning.
+# xxx code: javascript above gives `LanguageXNotSupported` warning.
 
 when not defined(js) and not defined(nimsuggest):
   {.fatal: "Module asyncjs is designed to be used with the JavaScript backend.".}
 
 import std/jsffi
 import std/macros
+import std/private/since
 
 type
   Future*[T] = ref object
     future*: T
   ## Wraps the return type of an asynchronous procedure.
 
-  PromiseJs* {.importcpp: "Promise".} = ref object
+  PromiseJs* {.importjs: "Promise".} = ref object
   ## A JavaScript Promise.
 
 
@@ -84,6 +90,8 @@ proc replaceReturn(node: var NimNode) =
       node[z] = nnkReturnStmt.newTree(value)
     elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result":
       node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(jsResolve, son[1]))
+    elif son.kind in RoutineNodes:
+      discard
     else:
       replaceReturn(son)
     inc z
@@ -94,10 +102,18 @@ proc isFutureVoid(node: NimNode): bool =
            node[1].kind == nnkIdent and $node[1] == "void"
 
 proc generateJsasync(arg: NimNode): NimNode =
-  if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}:
+  if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo, nnkProcTy}:
       error("Cannot transform this node kind into an async proc." &
             " proc/method definition or lambda node expected.")
 
+  # Transform type X = proc (): something {.async.}
+  # into      type X = proc (): Future[something]
+  if arg.kind == nnkProcTy:
+    result = arg
+    if arg[0][0].kind == nnkEmpty:
+      result[0][0] = quote do: Future[void]
+    return result
+
   result = arg
   var isVoid = false
   let jsResolve = ident("jsResolve")
@@ -113,7 +129,7 @@ proc generateJsasync(arg: NimNode): NimNode =
 
   if len(code) > 0:
     var awaitFunction = quote:
-      proc await[T](f: Future[T]): T {.importcpp: "(await #)", used.}
+      proc await[T](f: Future[T]): T {.importjs: "(await #)", used.}
     result.body.add(awaitFunction)
 
     var resolve: NimNode
@@ -150,108 +166,104 @@ macro async*(arg: untyped): untyped =
   else:
     result = generateJsasync(arg)
 
-proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importcpp: "(new Promise(#))".}
+proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importjs: "(new Promise(#))".}
   ## A helper for wrapping callback-based functions
   ## into promises and async procedures.
 
-proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importcpp: "(new Promise(#))".}
+proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importjs: "(new Promise(#))".}
   ## A helper for wrapping callback-based functions
   ## into promises and async procedures.
 
-template typeOrVoid[T](a: T): type =
-  # xxx this is useful, make it public in std/typetraits in future work
-  T
-
 template maybeFuture(T): untyped =
   # avoids `Future[Future[T]]`
   when T is Future: T
   else: Future[T]
 
-when defined(nimExperimentalAsyncjsThen):
-  import std/private/since
-  since (1, 5, 1):
-    #[
-    TODO:
-    * map `Promise.all()`
-    * proc toString*(a: Error): cstring {.importjs: "#.toString()".}
 
-    Note:
-    We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible
-    in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options
-    and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho
-    ]#
+since (1, 5, 1):
+  #[
+  TODO:
+  * map `Promise.all()`
+  * proc toString*(a: Error): cstring {.importjs: "#.toString()".}
 
-    type Error*  {.importjs: "Error".} = ref object of JsRoot
-      ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error
-      message*: cstring
-      name*: cstring
+  Note:
+  We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible
+  in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options
+  and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho
+  ]#
 
-    type OnReject* = proc(reason: Error)
+  type Error*  {.importjs: "Error".} = ref object of JsRoot
+    ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error
+    message*: cstring
+    name*: cstring
 
-    proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto =
-      ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then
-      ## Returns a `Future` from the return type of `onSuccess(T.default)`.
-      runnableExamples("-d:nimExperimentalAsyncjsThen"):
-        from std/sugar import `=>`
+  type OnReject* = proc(reason: Error)
 
-        proc fn(n: int): Future[int] {.async.} =
-          if n >= 7: raise newException(ValueError, "foobar: " & $n)
-          else: result = n * 2
+  proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto =
+    ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then
+    ## Returns a `Future` from the return type of `onSuccess(T.default)`.
+    runnableExamples("-r:off"):
+      from std/sugar import `=>`
 
-        proc asyncFact(n: int): Future[int] {.async.} =
-          if n > 0: result = n * await asyncFact(n-1)
-          else: result = 1
+      proc fn(n: int): Future[int] {.async.} =
+        if n >= 7: raise newException(ValueError, "foobar: " & $n)
+        else: result = n * 2
 
-        proc main() {.async.} =
-          block: # then
-            assert asyncFact(3).await == 3*2
-            assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2
-            let x1 = await fn(3)
-            assert x1 == 3 * 2
-            let x2 = await fn(4)
-              .then((a: int) => a.float)
-              .then((a: float) => $a)
-            assert x2 == "8.0"
+      proc asyncFact(n: int): Future[int] {.async.} =
+        if n > 0: result = n * await asyncFact(n-1)
+        else: result = 1
 
-          block: # then with `onReject` callback
-            var witness = 1
-            await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3))
-            assert witness == 2
-            await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3))
-            assert witness == 3
+      proc main() {.async.} =
+        block: # then
+          assert asyncFact(3).await == 3*2
+          assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2
+          let x1 = await fn(3)
+          assert x1 == 3 * 2
+          let x2 = await fn(4)
+            .then((a: int) => a.float)
+            .then((a: float) => $a)
+          assert x2 == "8.0"
 
-      template impl(call): untyped =
-        when typeOrVoid(call) is void:
-          var ret: Future[void]
-        else:
-          var ret = default(maybeFuture(typeof(call)))
-        typeof(ret)
-      when T is void:
-        type A = impl(onSuccess())
+        block: # then with `onReject` callback
+          var witness = 1
+          await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3))
+          assert witness == 2
+          await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3))
+          assert witness == 3
+
+    template impl(call): untyped =
+      # see D20210421T014713
+      when typeof(block: call) is void:
+        var ret: Future[void]
       else:
-        type A = impl(onSuccess(default(T)))
-      var ret: A
-      asm "`ret` = `future`.then(`onSuccess`, `onReject`)"
-      return ret
+        var ret = default(maybeFuture(typeof(call)))
+      typeof(ret)
+    when T is void:
+      type A = impl(onSuccess())
+    else:
+      type A = impl(onSuccess(default(T)))
+    var ret: A
+    {.emit: "`ret` = `future`.then(`onSuccess`, `onReject`)".}
+    return ret
 
-    proc catch*[T](future: Future[T], onReject: OnReject): Future[void] =
-      ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch
-      runnableExamples("-d:nimExperimentalAsyncjsThen"):
-        from std/sugar import `=>`
-        from std/strutils import contains
+  proc catch*[T](future: Future[T], onReject: OnReject): Future[void] =
+    ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch
+    runnableExamples("-r:off"):
+      from std/sugar import `=>`
+      from std/strutils import contains
 
-        proc fn(n: int): Future[int] {.async.} =
-          if n >= 7: raise newException(ValueError, "foobar: " & $n)
-          else: result = n * 2
+      proc fn(n: int): Future[int] {.async.} =
+        if n >= 7: raise newException(ValueError, "foobar: " & $n)
+        else: result = n * 2
 
-        proc main() {.async.} =
-          var reason: Error
-          await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work
-          assert reason == nil
-          await fn(7).catch((r: Error) => (reason = r))
-          assert reason != nil
-          assert  "foobar: 7" in $reason.message
+      proc main() {.async.} =
+        var reason: Error
+        await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work
+        assert reason == nil
+        await fn(7).catch((r: Error) => (reason = r))
+        assert reason != nil
+        assert  "foobar: 7" in $reason.message
 
-        discard main()
+      discard main()
 
-      asm "`result` = `future`.catch(`onReject`)"
+    {.emit: "`result` = `future`.catch(`onReject`)".}
diff --git a/lib/js/dom.nim b/lib/js/dom.nim
index a133f1f69a..be2a34db1f 100644
--- a/lib/js/dom.nim
+++ b/lib/js/dom.nim
@@ -9,6 +9,37 @@
 
 ## Declaration of the Document Object Model for the `JavaScript backend
 ## `_.
+##
+##
+## Document Ready
+## --------------
+##
+## * Basic example of a document ready:
+runnableExamples"-b:js -r:off":
+  proc example(e: Event) = echo "Document is ready"
+  document.addEventListener("DOMContentLoaded", example)  # You can also use "load" event.
+## * This example runs 5 seconds after the document ready:
+runnableExamples"-b:js -r:off":
+  proc example() = echo "5 seconds after document ready"
+  proc domReady(e: Event) = discard setTimeout(example, 5_000) # Document is ready.
+  document.addEventListener("DOMContentLoaded", domReady)
+## Document onUnload
+## -----------------
+##
+## * Simple example of how to implement code that runs when the page unloads:
+runnableExamples"-b:js -r:off":
+  proc example(e: Event) = echo "Document is unloaded"
+  document.addEventListener("unload", example)  # You can also use "beforeunload".
+## Document Autorefresh
+## --------------------
+##
+## * Minimal example of a document autorefresh:
+runnableExamples"-b:js -r:off":
+  proc example() = window.location.reload()
+  discard setTimeout(example, 5_000)
+## - For more examples, see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener
+
+
 import std/private/since
 when not defined(js):
   {.error: "This module only works on the JavaScript platform".}
@@ -17,8 +48,7 @@ const
   DomApiVersion* = 3 ## the version of DOM API we try to follow. No guarantees though.
 
 type
-  EventTarget* = ref EventTargetObj
-  EventTargetObj {.importc.} = object of RootObj
+  EventTarget* {.importc.} = ref object of RootObj
     onabort*: proc (event: Event) {.closure.}
     onblur*: proc (event: Event) {.closure.}
     onchange*: proc (event: Event) {.closure.}
@@ -129,8 +159,7 @@ type
 
   Storage* {.importc.} = ref object
 
-  Window* = ref WindowObj
-  WindowObj {.importc.} = object of EventTargetObj
+  Window* {.importc.} = ref object of EventTarget
     document*: Document
     event*: Event
     history*: History
@@ -159,11 +188,9 @@ type
     sessionStorage*: Storage
     parent*: Window
 
-  Frame* = ref FrameObj
-  FrameObj {.importc.} = object of WindowObj
+  Frame* {.importc.} = ref object of Window
 
-  ClassList* = ref ClassListObj
-  ClassListObj {.importc.} = object of RootObj
+  ClassList* {.importc.} = ref object of RootObj
 
   NodeType* = enum
     ElementNode = 1,
@@ -179,8 +206,7 @@ type
     DocumentFragmentNode,
     NotationNode
 
-  Node* = ref NodeObj
-  NodeObj {.importc.} = object of EventTargetObj
+  Node* {.importc.} = ref object of EventTarget
     attributes*: seq[Node]
     childNodes*: seq[Node]
     children*: seq[Node]
@@ -204,8 +230,7 @@ type
     parentElement*: Element
     isConnected*: bool
 
-  Document* = ref DocumentObj
-  DocumentObj {.importc.} = object of NodeObj
+  Document* {.importc.} = ref object of Node
     activeElement*: Element
     documentElement*: Element
     alinkColor*: cstring
@@ -216,11 +241,13 @@ type
     defaultCharset*: cstring
     fgColor*: cstring
     head*: Element
+    hidden*: bool
     lastModified*: cstring
     linkColor*: cstring
     referrer*: cstring
     title*: cstring
     URL*: cstring
+    visibilityState*: cstring
     vlinkColor*: cstring
     anchors*: seq[AnchorElement]
     forms*: seq[FormElement]
@@ -230,8 +257,7 @@ type
     links*: seq[LinkElement]
     fonts*: FontFaceSet
 
-  Element* = ref ElementObj
-  ElementObj {.importc.} = object of NodeObj
+  Element* {.importc.} = ref object of Node
     className*: cstring
     classList*: ClassList
     checked*: bool
@@ -252,8 +278,7 @@ type
     offsetLeft*: int
     offsetTop*: int
 
-  ValidityState* = ref ValidityStateObj ## see `docs`_
-  ValidityStateObj {.importc.} = object
+  ValidityState* {.importc.} = ref object ## see `docs`_
     badInput*: bool
     customError*: bool
     patternMismatch*: bool
@@ -266,25 +291,21 @@ type
     valid*: bool
     valueMissing*: bool
 
-  Blob* = ref BlobObj ## see `docs`_
-  BlobObj {.importc.} = object of RootObj
+  Blob* {.importc.} = ref object of RootObj ## see `docs`_
     size*: int
     `type`*: cstring
 
-  File* = ref FileObj ## see `docs`_
-  FileObj {.importc.} = object of Blob
+  File* {.importc.} = ref object of Blob ## see `docs`_
     lastModified*: int
     name*: cstring
 
-  TextAreaElement* = ref TextAreaElementObj ## see `docs`_
-  TextAreaElementObj {.importc.} = object of Element
+  TextAreaElement* {.importc.} = ref object of Element ## see `docs`_
     value*: cstring
     selectionStart*, selectionEnd*: int
     selectionDirection*: cstring
     rows*, cols*: int
 
-  InputElement* = ref InputElementObj ## see `docs`_
-  InputElementObj {.importc.} = object of Element
+  InputElement* {.importc.} = ref object of Element ## see `docs`_
     # Properties related to the parent form
     formAction*: cstring
     formEncType*: cstring
@@ -336,15 +357,13 @@ type
     valueAsDate*: cstring
     valueAsNumber*: float
 
-  LinkElement* = ref LinkObj
-  LinkObj {.importc.} = object of ElementObj
+  LinkElement* {.importc.} = ref object of Element
     target*: cstring
     text*: cstring
     x*: int
     y*: int
 
-  EmbedElement* = ref EmbedObj
-  EmbedObj {.importc.} = object of ElementObj
+  EmbedElement* {.importc.} = ref object of Element
     height*: int
     hspace*: int
     src*: cstring
@@ -352,21 +371,18 @@ type
     `type`*: cstring
     vspace*: int
 
-  AnchorElement* = ref AnchorObj
-  AnchorObj {.importc.} = object of ElementObj
+  AnchorElement* {.importc.} = ref object of Element
     text*: cstring
     x*, y*: int
 
-  OptionElement* = ref OptionObj
-  OptionObj {.importc.} = object of ElementObj
+  OptionElement* {.importc.} = ref object of Element
     defaultSelected*: bool
     selected*: bool
     selectedIndex*: int
     text*: cstring
     value*: cstring
 
-  FormElement* = ref FormObj ## see `docs`_
-  FormObj {.importc.} = object of ElementObj
+  FormElement* {.importc.} = ref object of Element ## see `docs`_
     acceptCharset*: cstring
     action*: cstring
     autocomplete*: cstring
@@ -378,8 +394,7 @@ type
     noValidate*: bool
     target*: cstring
 
-  ImageElement* = ref ImageObj
-  ImageObj {.importc.} = object of ElementObj
+  ImageElement* {.importc.} = ref object of Element
     border*: int
     complete*: bool
     height*: int
@@ -389,8 +404,7 @@ type
     vspace*: int
     width*: int
 
-  Style* = ref StyleObj
-  StyleObj {.importc.} = object of RootObj
+  Style* {.importc.} = ref object of RootObj
     alignContent*: cstring
     alignItems*: cstring
     alignSelf*: cstring
@@ -766,8 +780,7 @@ type
     AtTarget,
     BubblingPhase
 
-  Event* = ref EventObj ## see `docs`_
-  EventObj {.importc.} = object of RootObj
+  Event* {.importc.} = ref object of RootObj ## see `docs`_
     bubbles*: bool
     cancelBubble*: bool
     cancelable*: bool
@@ -779,13 +792,11 @@ type
     `type`*: cstring
     isTrusted*: bool
 
-  UIEvent* = ref UIEventObj ## see `docs`_
-  UIEventObj {.importc.} = object of Event
+  UIEvent* {.importc.} = ref object of Event ## see `docs`_
     detail*: int64
     view*: Window
 
-  KeyboardEvent* = ref KeyboardEventObj ## see `docs`_
-  KeyboardEventObj {.importc.} = object of UIEvent
+  KeyboardEvent* {.importc.} = ref object of UIEvent ## see `docs`_
     altKey*, ctrlKey*, metaKey*, shiftKey*: bool
     code*: cstring
     isComposing*: bool
@@ -1147,8 +1158,7 @@ type
     FourthButton = 8,
     FifthButton = 16
 
-  MouseEvent* = ref MouseEventObj ## see `docs`_
-  MouseEventObj {.importc.} = object of UIEvent
+  MouseEvent* {.importc.} = ref object of UIEvent ## see `docs`_
     altKey*, ctrlKey*, metaKey*, shiftKey*: bool
     button*: int
     buttons*: int
@@ -1165,13 +1175,11 @@ type
     File = "file",
     String = "string"
 
-  DataTransferItem* = ref DataTransferItemObj ## see `docs`_
-  DataTransferItemObj {.importc.} = object of RootObj
+  DataTransferItem* {.importc.} = ref object of RootObj ## see `docs`_
     kind*: cstring
     `type`*: cstring
 
-  DataTransfer* = ref DataTransferObj ## see `docs`_
-  DataTransferObj {.importc.} = object of RootObj
+  DataTransfer* {.importc.} = ref object of RootObj ## see `docs`_
     dropEffect*: cstring
     effectAllowed*: cstring
     files*: seq[Element]
@@ -1213,8 +1221,7 @@ type
     ## see `docs`_
     clipboardData*: DataTransfer
 
-  StorageEvent* = ref StorageEventObj ## see `docs`_
-  StorageEventObj {.importc.} = object of Event
+  StorageEvent* {.importc.} = ref object of Event ## see `docs`_
     key*: cstring
     newValue*, oldValue*: cstring
     storageArea*: Storage
@@ -1223,8 +1230,7 @@ type
   TouchList* {.importc.} = ref object of RootObj
     length*: int
 
-  Touch* = ref TouchObj
-  TouchObj {.importc.} = object of RootObj
+  Touch* {.importc.} = ref object of RootObj
     identifier*: int
     screenX*, screenY*, clientX*, clientY*, pageX*, pageY*: int
     target*: Element
@@ -1232,12 +1238,10 @@ type
     rotationAngle*: int
     force*: float
 
-  TouchEvent* = ref TouchEventObj
-  TouchEventObj {.importc.} = object of UIEvent
+  TouchEvent* {.importc.} = ref object of UIEvent
     changedTouches*, targetTouches*, touches*: seq[Touch]
 
-  Location* = ref LocationObj
-  LocationObj {.importc.} = object of RootObj
+  Location* {.importc.} = ref object of RootObj
     hash*: cstring
     host*: cstring
     hostname*: cstring
@@ -1248,12 +1252,10 @@ type
     search*: cstring
     origin*: cstring
 
-  History* = ref HistoryObj
-  HistoryObj {.importc.} = object of RootObj
+  History* {.importc.} = ref object of RootObj
     length*: int
 
-  Navigator* = ref NavigatorObj
-  NavigatorObj {.importc.} = object of RootObj
+  Navigator* {.importc.} = ref object of RootObj
     appCodeName*: cstring
     appName*: cstring
     appVersion*: cstring
@@ -1291,8 +1293,7 @@ type
   ToolBar* = LocationBar
   StatusBar* = LocationBar
 
-  Screen = ref ScreenObj
-  ScreenObj {.importc.} = object of RootObj
+  Screen* {.importc.} = ref object of RootObj
     availHeight*: int
     availWidth*: int
     colorDepth*: int
@@ -1301,7 +1302,7 @@ type
     width*: int
 
   TimeOut* {.importc.} = ref object of RootObj
-  Interval* {.importc.} = object of RootObj
+  Interval* {.importc.} = ref object of RootObj
 
   AddEventListenerOptions* = object
     capture*: bool
@@ -1317,41 +1318,46 @@ type
     ready*: FontFaceSetReady
     onloadingdone*: proc(event: Event)
 
+  ScrollIntoViewOptions* = object
+    behavior*: cstring
+    `block`*: cstring
+    inline*: cstring
+
+  MediaQueryList* {.importc.} = ref object of EventTarget
+    matches*: bool
+    media*: cstring
+
 since (1, 3):
   type
     DomParser* = ref object
       ## DOM Parser object (defined on browser only, may not be on NodeJS).
       ## * https://developer.mozilla.org/en-US/docs/Web/API/DOMParser
       ##
-      ## .. code-block:: nim
+      ##   ```nim
       ##   let prsr = newDomParser()
       ##   discard prsr.parseFromString("Hello World".cstring, "text/html".cstring)
+      ##   ```
 
-    DomException* = ref DOMExceptionObj
+    DomException* {.importc.} = ref object
       ## The DOMException interface represents an abnormal event (called an exception)
       ## which occurs as a result of calling a method or accessing a property of a web API.
       ## Each exception has a name, which is a short "CamelCase" style string identifying
       ## the error or abnormal condition.
       ## https://developer.mozilla.org/en-US/docs/Web/API/DOMException
 
-    DOMExceptionObj {.importc.} = object
-
-    FileReader* = ref FileReaderObj
+    FileReader* {.importc.} = ref object of EventTarget
       ## The FileReader object lets web applications asynchronously read the contents of files
       ## (or raw data buffers) stored on the user's computer, using File or Blob objects to specify
       ## the file or data to read.
       ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader
 
-    FileReaderObj {.importc.} = object of EventTargetObj
-
     FileReaderState* = distinct range[0'u16..2'u16]
     RootNodeOptions* = object of RootObj
       composed*: bool
     DocumentOrShadowRoot* {.importc.} = object of RootObj
       activeElement*: Element
       # styleSheets*: StyleSheetList
-    ShadowRoot* = ref ShadowRootObj
-    ShadowRootObj {.importc.} = object of DocumentOrShadowRoot
+    ShadowRoot* {.importc.} = ref object of DocumentOrShadowRoot
       delegatesFocus*: bool
       host*: Element
       innerHTML*: cstring
@@ -1360,8 +1366,7 @@ since (1, 3):
       mode*: cstring
       delegatesFocus*: bool
 
-    HTMLSlotElement* = ref HTMLSlotElementObj
-    HTMLSlotElementObj {.importc.} = object of RootObj
+    HTMLSlotElement* {.importc.} = ref object of RootObj
       name*: cstring
     SlotOptions* = object of RootObj
       flatten*: bool
@@ -1379,6 +1384,9 @@ proc `class=`*(n: Node; v: cstring) {.importcpp: "#.className = #", nodecl.}
 proc value*(n: Node): cstring {.importcpp: "#.value", nodecl.}
 proc `value=`*(n: Node; v: cstring) {.importcpp: "#.value = #", nodecl.}
 
+proc checked*(n: Node): bool {.importcpp: "#.checked", nodecl.}
+proc `checked=`*(n: Node; v: bool) {.importcpp: "#.checked = #", nodecl.}
+
 proc `disabled=`*(n: Node; v: bool) {.importcpp: "#.disabled = #", nodecl.}
 
 when defined(nodejs):
@@ -1415,7 +1423,7 @@ when defined(nodejs):
         parent.childNodes[i] = newNode
         return
       inc i
-    doAssert false, "old node not in node list"
+    raiseAssert "old node not in node list"
 
   proc removeChild*(parent, child: Node) =
     child.parentNode = nil
@@ -1425,7 +1433,7 @@ when defined(nodejs):
         parent.childNodes.delete(i)
         return
       inc i
-    doAssert false, "old node not in node list"
+    raiseAssert "old node not in node list"
 
   proc insertBefore*(parent, newNode, before: Node) =
     appendChild(parent, newNode)
@@ -1437,7 +1445,7 @@ when defined(nodejs):
         parent.childNodes[i-1] = newNode
         return
       inc i
-    #doAssert false, "before not in node list"
+    #raiseAssert "before not in node list"
 
   proc createElement*(d: Document, identifier: cstring): Element =
     new(result)
@@ -1473,6 +1481,8 @@ else:
 
 proc setTimeout*(action: proc(); ms: int): TimeOut {.importc, nodecl.}
 proc clearTimeout*(t: TimeOut) {.importc, nodecl.}
+proc setInterval*(action: proc(); ms: int): Interval {.importc, nodecl.}
+proc clearInterval*(i: Interval) {.importc, nodecl.}
 
 {.push importcpp.}
 
@@ -1486,17 +1496,19 @@ proc removeEventListener*(et: EventTarget; ev: cstring; cb: proc(ev: Event))
 proc alert*(w: Window, msg: cstring)
 proc back*(w: Window)
 proc blur*(w: Window)
-proc clearInterval*(w: Window, interval: ref Interval)
-proc clearTimeout*(w: Window, timeout: ref TimeOut)
+proc clearInterval*(w: Window, interval: Interval)
+proc clearTimeout*(w: Window, timeout: TimeOut)
 proc close*(w: Window)
 proc confirm*(w: Window, msg: cstring): bool
 proc disableExternalCapture*(w: Window)
 proc enableExternalCapture*(w: Window)
 proc find*(w: Window, text: cstring, caseSensitive = false,
-           backwards = false)
+           backwards = false): bool
 proc focus*(w: Window)
 proc forward*(w: Window)
-proc getComputedStyle*(w: Window, e: Node, pe:Node = nil): Style
+proc getComputedStyle*(w: Window, e: Node, pe: Node = nil): Style
+  ## .. warning:: The returned Style may or may not be read-only at run-time in the browser. getComputedStyle is performance costly.
+
 proc handleEvent*(w: Window, e: Event)
 proc home*(w: Window)
 proc moveBy*(w: Window, x, y: int)
@@ -1510,13 +1522,14 @@ proc resizeTo*(w: Window, x, y: int)
 proc routeEvent*(w: Window, event: Event)
 proc scrollBy*(w: Window, x, y: int)
 proc scrollTo*(w: Window, x, y: int)
-proc setInterval*(w: Window, code: cstring, pause: int): ref Interval
-proc setInterval*(w: Window, function: proc (), pause: int): ref Interval
-proc setTimeout*(w: Window, code: cstring, pause: int): ref TimeOut
-proc setTimeout*(w: Window, function: proc (), pause: int): ref Interval
+proc setInterval*(w: Window, code: cstring, pause: int): Interval
+proc setInterval*(w: Window, function: proc (), pause: int): Interval
+proc setTimeout*(w: Window, code: cstring, pause: int): TimeOut
+proc setTimeout*(w: Window, function: proc (), pause: int): Interval
 proc stop*(w: Window)
 proc requestAnimationFrame*(w: Window, function: proc (time: float)): int
 proc cancelAnimationFrame*(w: Window, id: int)
+proc matchMedia*(w: Window, mediaQueryString: cstring): MediaQueryList
 
 # Node "methods"
 proc appendData*(n: Node, data: cstring)
@@ -1533,6 +1546,7 @@ proc removeAttribute*(n: Node, attr: cstring)
 proc removeAttributeNode*(n, attr: Node)
 proc replaceData*(n: Node, start, len: int, text: cstring)
 proc scrollIntoView*(n: Node)
+proc scrollIntoView*(n: Node, options: ScrollIntoViewOptions)
 proc setAttribute*(n: Node, name, value: cstring)
 proc setAttributeNode*(n: Node, attr: Node)
 proc querySelector*(n: Node, selectors: cstring): Element
@@ -1641,7 +1655,7 @@ proc item*(list: TouchList, i: int): Touch
 proc clearData*(dt: DataTransfer, format: cstring)
 proc getData*(dt: DataTransfer, format: cstring): cstring
 proc setData*(dt: DataTransfer, format: cstring, data: cstring)
-proc setDragImage*(dt: DataTransfer, img: Element, xOffset: int64, yOffset: int64)
+proc setDragImage*(dt: DataTransfer, img: Element, xOffset: int, yOffset: int)
 
 # DataTransferItem "methods"
 proc getAsFile*(dti: DataTransferItem): File
@@ -1668,7 +1682,6 @@ proc `$`*(s: Selection): string = $(s.toString())
 # Storage "methods"
 proc getItem*(s: Storage, key: cstring): cstring
 proc setItem*(s: Storage, key, value: cstring)
-proc hasItem*(s: Storage, key: cstring): bool
 proc clear*(s: Storage)
 proc removeItem*(s: Storage, key: cstring)
 
@@ -1756,5 +1769,73 @@ since (1, 3):
     ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsBinaryString
   proc readAsDataURL*(f: FileReader, b: Blob) {.importcpp: "#.readAsDataURL(#)".}
     ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL
-  proc readAsText*(f: FileReader, b: Blob, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".}
+  proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".}
     ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText
+
+since (1, 5):
+  proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] {.importcpp.}
+
+
+since (1, 7):
+
+  proc insertAdjacentText*(self: Node; position, data: cstring) {.importjs: "#.$1(#, #)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentText
+
+  proc insertAdjacentElement*(self: Node; position: cstring; element: Node) {.importjs: "#.$1(#, #)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentElement
+
+  proc insertAdjacentHTML*(self: Node; position, html: cstring) {.importjs: "#.$1(#, #)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML
+
+  proc after*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/after
+
+  proc before*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/before
+
+  proc append*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/append
+
+  proc closest*(self: Node; cssSelector: cstring): Node {.importjs: "#.$1(#)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/closest
+
+  proc hasAttributeNS*(self: Node; namespace, localName: cstring): bool {.importjs: "(#.$1(#, #) || false)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/hasAttributeNS
+
+  proc removeAttributeNS*(self: Node; namespace, attributeName: cstring) {.importjs: "#.$1(#, #)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/removeAttributeNS
+
+  proc hasPointerCapture*(self: Node; pointerId: SomeNumber): bool {.importjs: "(#.$1(#) || false)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/hasPointerCapture
+
+  proc releasePointerCapture*(self: Node; pointerId: SomeNumber) {.importjs: "#.$1(#)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/releasePointerCapture
+
+  proc requestPointerLock*(self: Node) {.importjs: "#.$1()".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/requestPointerLock
+
+  proc replaceChildren*(self: Node; replacements: Node) {.importjs: "#.$1(@)", varargs.}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/replaceChildren
+
+  proc replaceWith*(self: Node; replacements: Node) {.importjs: "#.$1(@)", varargs.}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/replaceWith
+
+  proc scrollIntoViewIfNeeded*(self: Node; centerIfNeeded: bool) {.importjs: "#.$1(#)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoViewIfNeeded
+
+  proc setHTML*(self: Node; html: cstring) {.importjs: "#.$1(#)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/setHTML
+
+  proc toggleAttribute*(self: Node; name: cstring; force = false): bool {.importjs: "(#.$1(#, #) || false)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/toggleAttribute
+
+  proc matches*(self: Node; cssSelector: cstring): bool {.importjs: "(#.$1(#) || false)".}
+    ## https://developer.mozilla.org/en-US/docs/Web/API/Element/matches
+
+
+since (2, 1):
+  type VisualViewport* {.importc.} = ref object of EventTarget
+    offsetLeft*, offsetTop*, pageLeft*, pageTop*, width*, height*, scale*: float
+    onResize*, onScroll*: proc (event: Event) {.closure.}
+
+  func visualViewport*(self: Window): VisualViewport {.importjs: "#.$1", nodecl.}
diff --git a/lib/js/dom_extensions.nim b/lib/js/dom_extensions.nim
deleted file mode 100644
index f7d37f4bff..0000000000
--- a/lib/js/dom_extensions.nim
+++ /dev/null
@@ -1,5 +0,0 @@
-import std/dom
-
-{.push importcpp.}
-proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element]
-{.pop.}
\ No newline at end of file
diff --git a/lib/js/jsconsole.nim b/lib/js/jsconsole.nim
index bf43adddd4..e74127334e 100644
--- a/lib/js/jsconsole.nim
+++ b/lib/js/jsconsole.nim
@@ -118,6 +118,7 @@ since (1, 5):
 
   func timeStamp*(console: Console; label: cstring) {.importcpp.}
     ## https://developer.mozilla.org/en-US/docs/Web/API/Console/timeStamp
+    ##
     ## ..warning:: non-standard
 
 
diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim
index 693e507991..be353875c8 100644
--- a/lib/js/jscore.nim
+++ b/lib/js/jscore.nim
@@ -13,7 +13,7 @@
 ## specific requirements and solely targets JavaScript, you should be using
 ## the relevant functions in the `math`, `json`, and `times` stdlib
 ## modules instead.
-import std/private/since
+import std/private/[since, jsutils]
 
 when not defined(js):
   {.error: "This module only works on the JavaScript platform".}
@@ -74,9 +74,16 @@ proc parse*(d: DateLib, s: cstring): int {.importcpp.}
 proc newDate*(): DateTime {.
   importcpp: "new Date()".}
 
-proc newDate*(date: int|int64|string): DateTime {.
+proc newDate*(date: int|string): DateTime {.
   importcpp: "new Date(#)".}
 
+whenJsNoBigInt64:
+  proc newDate*(date: int64): DateTime {.
+    importcpp: "new Date(#)".}
+do:
+  proc newDate*(date: int64): DateTime {.
+    importcpp: "new Date(Number(#))".}
+
 proc newDate*(year, month, day, hours, minutes,
              seconds, milliseconds: int): DateTime {.
   importcpp: "new Date(#,#,#,#,#,#,#)".}
@@ -88,20 +95,27 @@ proc getMilliseconds*(d: DateTime): int {.importcpp.}
 proc getMinutes*(d: DateTime): int {.importcpp.}
 proc getMonth*(d: DateTime): int {.importcpp.}
 proc getSeconds*(d: DateTime): int {.importcpp.}
-proc getYear*(d: DateTime): int {.importcpp.}
 proc getTime*(d: DateTime): int {.importcpp.}
-proc toString*(d: DateTime): cstring {.importcpp.}
+proc getTimezoneOffset*(d: DateTime): int {.importcpp.}
 proc getUTCDate*(d: DateTime): int {.importcpp.}
+proc getUTCDay*(d: DateTime): int {.importcpp.}
 proc getUTCFullYear*(d: DateTime): int {.importcpp.}
 proc getUTCHours*(d: DateTime): int {.importcpp.}
 proc getUTCMilliseconds*(d: DateTime): int {.importcpp.}
 proc getUTCMinutes*(d: DateTime): int {.importcpp.}
 proc getUTCMonth*(d: DateTime): int {.importcpp.}
 proc getUTCSeconds*(d: DateTime): int {.importcpp.}
-proc getUTCDay*(d: DateTime): int {.importcpp.}
-proc getTimezoneOffset*(d: DateTime): int {.importcpp.}
+proc getYear*(d: DateTime): int {.importcpp.}
+
 proc setFullYear*(d: DateTime, year: int) {.importcpp.}
 
+func toDateString*(d: DateTime): cstring {.importcpp.}
+func toISOString*(d: DateTime): cstring {.importcpp.}
+func toJSON*(d: DateTime): cstring {.importcpp.}
+proc toString*(d: DateTime): cstring {.importcpp.}
+func toTimeString*(d: DateTime): cstring {.importcpp.}
+func toUTCString*(d: DateTime): cstring {.importcpp.}
+
 #JSON library
 proc stringify*(l: JsonLib, s: JsRoot): cstring {.importcpp.}
 proc parse*(l: JsonLib, s: cstring): JsRoot {.importcpp.}
@@ -110,3 +124,30 @@ proc parse*(l: JsonLib, s: cstring): JsRoot {.importcpp.}
 since (1, 5):
   func debugger*() {.importjs: "debugger@".}
     ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger
+
+  func copyWithin*[T](self: openArray[T]; target: int): seq[T] {.importjs: "#.copyWithin(#)".}
+  func copyWithin*[T](self: openArray[T]; target, start: int): seq[T] {.importjs: "#.copyWithin(#, #)".}
+  func copyWithin*[T](self: openArray[T]; target, start, ends: int): seq[T] {.importjs: "#.copyWithin(#, #, #)".} =
+    ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin
+    ## `copyWithin` uses shallow copy.
+    runnableExamples:
+      assert ['a', 'b', 'c', 'd', 'e'].copyWithin(0, 3, 4) == @['d', 'b', 'c', 'd', 'e']
+      assert ['a', 'b', 'c', 'd', 'e'].copyWithin(1, 3) == @['a', 'd', 'e', 'd', 'e']
+      assert [1, 2, 3, 4, 5].copyWithin(-2) == @[1, 2, 3, 1, 2]
+      assert [1, 2, 3, 4, 5].copyWithin(0, 3) == @[4, 5, 3, 4, 5]
+      assert [1, 2, 3, 4, 5].copyWithin(0, 3, 4) == @[4, 2, 3, 4, 5]
+      assert [1, 2, 3, 4, 5].copyWithin(-2, -3, -1) == @[1, 2, 3, 3, 4]
+
+
+since (1, 7):
+  func shift*[T](self: seq[T]): T {.importjs: "#.$1()".} =
+    ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift
+    runnableExamples:
+      var arrai = @[1, 2, 3]
+      assert arrai.shift() == 1
+      assert arrai == @[2, 3]
+
+  func queueMicrotask*(function: proc) {.importjs: "$1(#)".} =
+    ## * https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask
+    ## * https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide
+    runnableExamples"-r:off": queueMicrotask(proc() = echo "Microtask")
diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim
index 0734d891a6..d50d58ae51 100644
--- a/lib/js/jsffi.nim
+++ b/lib/js/jsffi.nim
@@ -21,7 +21,7 @@ runnableExamples:
   var document {.importc, nodecl.}: JsObject
   var console {.importc, nodecl.}: JsObject
   # import the "$" function
-  proc jq(selector: JsObject): JsObject {.importcpp: "$$(#)".}
+  proc jq(selector: JsObject): JsObject {.importjs: "$$(#)".}
 
   # Use jQuery to make the following code run, after the document is ready.
   # This uses an experimental `.()` operator for `JsObject`, to emit
@@ -64,13 +64,13 @@ proc validJsName(name: string): bool =
     if chr notin {'A'..'Z','a'..'z','_','$','0'..'9'}:
       return false
 
-template mangleJsName(name: cstring): cstring =
+template mangleJsName(name: string): string =
   inc nameCounter
   "mangledName" & $nameCounter
 
 # only values that can be mapped 1 to 1 with cstring should be keys: they have an injective function with cstring
 
-proc toJsKey*[T: SomeInteger](text: cstring, t: type T): T {.importcpp: "parseInt(#)".}
+proc toJsKey*[T: SomeInteger](text: cstring, t: type T): T {.importjs: "parseInt(#)".}
 
 proc toJsKey*[T: enum](text: cstring, t: type T): T =
   T(text.toJsKey(int))
@@ -78,7 +78,7 @@ proc toJsKey*[T: enum](text: cstring, t: type T): T =
 proc toJsKey*(text: cstring, t: type cstring): cstring =
   text
 
-proc toJsKey*[T: SomeFloat](text: cstring, t: type T): T {.importcpp: "parseFloat(#)".}
+proc toJsKey*[T: SomeFloat](text: cstring, t: type T): T {.importjs: "parseFloat(#)".}
 
 type
   JsKey* = concept a, type T
@@ -103,10 +103,10 @@ var
   jsFilename* {.importc: "__filename", nodecl.}: cstring
     ## JavaScript's __filename pseudo-variable.
 
-proc isNull*[T](x: T): bool {.noSideEffect, importcpp: "(# === null)".}
+proc isNull*[T](x: T): bool {.noSideEffect, importjs: "(# === null)".}
   ## Checks if a value is exactly null.
 
-proc isUndefined*[T](x: T): bool {.noSideEffect, importcpp: "(# === undefined)".}
+proc isUndefined*[T](x: T): bool {.noSideEffect, importjs: "(# === undefined)".}
   ## Checks if a value is exactly undefined.
 
 # Exceptions
@@ -121,35 +121,35 @@ type
   JsURIError* {.importc: "URIError".} = object of JsError
 
 # New
-proc newJsObject*: JsObject {.importcpp: "{@}".}
+proc newJsObject*: JsObject {.importjs: "{@}".}
   ## Creates a new empty JsObject.
 
-proc newJsAssoc*[K: JsKey, V]: JsAssoc[K, V] {.importcpp: "{@}".}
+proc newJsAssoc*[K: JsKey, V]: JsAssoc[K, V] {.importjs: "{@}".}
   ## Creates a new empty JsAssoc with key type `K` and value type `V`.
 
 # Checks
 proc hasOwnProperty*(x: JsObject, prop: cstring): bool
-  {.importcpp: "#.hasOwnProperty(#)".}
+  {.importjs: "#.hasOwnProperty(#)".}
   ## Checks, whether `x` has a property of name `prop`.
 
-proc jsTypeOf*(x: JsObject): cstring {.importcpp: "typeof(#)".}
+proc jsTypeOf*(x: JsObject): cstring {.importjs: "typeof(#)".}
   ## Returns the name of the JsObject's JavaScript type as a cstring.
 
-proc jsNew*(x: auto): JsObject {.importcpp: "(new #)".}
+proc jsNew*(x: auto): JsObject {.importjs: "(new #)".}
   ## Turns a regular function call into an invocation of the
   ## JavaScript's `new` operator.
 
-proc jsDelete*(x: auto): JsObject {.importcpp: "(delete #)".}
+proc jsDelete*(x: auto): JsObject {.importjs: "(delete #)".}
   ## JavaScript's `delete` operator.
 
 proc require*(module: cstring): JsObject {.importc.}
   ## JavaScript's `require` function.
 
 # Conversion to and from JsObject
-proc to*(x: JsObject, T: typedesc): T {.importcpp: "(#)".}
+proc to*(x: JsObject, T: typedesc): T {.importjs: "(#)".}
   ## Converts a JsObject `x` to type `T`.
 
-proc toJs*[T](val: T): JsObject {.importcpp: "(#)".}
+proc toJs*[T](val: T): JsObject {.importjs: "(#)".}
   ## Converts a value of any type to type JsObject.
 
 template toJs*(s: string): JsObject = cstring(s).toJs
@@ -160,50 +160,50 @@ macro jsFromAst*(n: untyped): untyped =
     result = newProc(procType = nnkDo, body = result)
   return quote: toJs(`result`)
 
-proc `&`*(a, b: cstring): cstring {.importcpp: "(# + #)".}
+proc `&`*(a, b: cstring): cstring {.importjs: "(# + #)".}
   ## Concatenation operator for JavaScript strings.
 
-proc `+`  *(x, y: JsObject): JsObject {.importcpp: "(# + #)".}
-proc `-`  *(x, y: JsObject): JsObject {.importcpp: "(# - #)".}
-proc `*`  *(x, y: JsObject): JsObject {.importcpp: "(# * #)".}
-proc `/`  *(x, y: JsObject): JsObject {.importcpp: "(# / #)".}
-proc `%`  *(x, y: JsObject): JsObject {.importcpp: "(# % #)".}
-proc `+=` *(x, y: JsObject): JsObject {.importcpp: "(# += #)", discardable.}
-proc `-=` *(x, y: JsObject): JsObject {.importcpp: "(# -= #)", discardable.}
-proc `*=` *(x, y: JsObject): JsObject {.importcpp: "(# *= #)", discardable.}
-proc `/=` *(x, y: JsObject): JsObject {.importcpp: "(# /= #)", discardable.}
-proc `%=` *(x, y: JsObject): JsObject {.importcpp: "(# %= #)", discardable.}
-proc `++` *(x:    JsObject): JsObject {.importcpp: "(++#)".}
-proc `--` *(x:    JsObject): JsObject {.importcpp: "(--#)".}
-proc `>`  *(x, y: JsObject): JsObject {.importcpp: "(# > #)".}
-proc `<`  *(x, y: JsObject): JsObject {.importcpp: "(# < #)".}
-proc `>=` *(x, y: JsObject): JsObject {.importcpp: "(# >= #)".}
-proc `<=` *(x, y: JsObject): JsObject {.importcpp: "(# <= #)".}
-proc `**` *(x, y: JsObject): JsObject {.importcpp: "((#) ** #)".}
+proc `+`*(x, y: JsObject): JsObject {.importjs: "(# + #)".}
+proc `-`*(x, y: JsObject): JsObject {.importjs: "(# - #)".}
+proc `*`*(x, y: JsObject): JsObject {.importjs: "(# * #)".}
+proc `/`*(x, y: JsObject): JsObject {.importjs: "(# / #)".}
+proc `%`*(x, y: JsObject): JsObject {.importjs: "(# % #)".}
+proc `+=`*(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.}
+proc `-=`*(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.}
+proc `*=`*(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.}
+proc `/=`*(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.}
+proc `%=`*(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.}
+proc `++`*(x:    JsObject): JsObject {.importjs: "(++#)".}
+proc `--`*(x:    JsObject): JsObject {.importjs: "(--#)".}
+proc `>`*(x, y: JsObject): JsObject {.importjs: "(# > #)".}
+proc `<`*(x, y: JsObject): JsObject {.importjs: "(# < #)".}
+proc `>=`*(x, y: JsObject): JsObject {.importjs: "(# >= #)".}
+proc `<=`*(x, y: JsObject): JsObject {.importjs: "(# <= #)".}
+proc `**`*(x, y: JsObject): JsObject {.importjs: "((#) ** #)".}
   # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812
-proc `and`*(x, y: JsObject): JsObject {.importcpp: "(# && #)".}
-proc `or` *(x, y: JsObject): JsObject {.importcpp: "(# || #)".}
-proc `not`*(x:    JsObject): JsObject {.importcpp: "(!#)".}
-proc `in` *(x, y: JsObject): JsObject {.importcpp: "(# in #)".}
+proc `and`*(x, y: JsObject): JsObject {.importjs: "(# && #)".}
+proc `or`*(x, y: JsObject): JsObject {.importjs: "(# || #)".}
+proc `not`*(x:    JsObject): JsObject {.importjs: "(!#)".}
+proc `in`*(x, y: JsObject): JsObject {.importjs: "(# in #)".}
 
-proc `[]`*(obj: JsObject, field: cstring): JsObject {.importcpp: getImpl.}
+proc `[]`*(obj: JsObject, field: cstring): JsObject {.importjs: getImpl.}
   ## Returns the value of a property of name `field` from a JsObject `obj`.
 
-proc `[]`*(obj: JsObject, field: int): JsObject {.importcpp: getImpl.}
+proc `[]`*(obj: JsObject, field: int): JsObject {.importjs: getImpl.}
   ## Returns the value of a property of name `field` from a JsObject `obj`.
 
-proc `[]=`*[T](obj: JsObject, field: cstring, val: T) {.importcpp: setImpl.}
+proc `[]=`*[T](obj: JsObject, field: cstring, val: T) {.importjs: setImpl.}
   ## Sets the value of a property of name `field` in a JsObject `obj` to `v`.
 
-proc `[]=`*[T](obj: JsObject, field: int, val: T) {.importcpp: setImpl.}
+proc `[]=`*[T](obj: JsObject, field: int, val: T) {.importjs: setImpl.}
   ## Sets the value of a property of name `field` in a JsObject `obj` to `v`.
 
 proc `[]`*[K: JsKey, V](obj: JsAssoc[K, V], field: K): V
-  {.importcpp: getImpl.}
+  {.importjs: getImpl.}
   ## Returns the value of a property of name `field` from a JsAssoc `obj`.
 
 proc `[]=`*[K: JsKey, V](obj: JsAssoc[K, V], field: K, val: V)
-  {.importcpp: setImpl.}
+  {.importjs: setImpl.}
   ## Sets the value of a property of name `field` in a JsAssoc `obj` to `v`.
 
 proc `[]`*[V](obj: JsAssoc[cstring, V], field: string): V =
@@ -212,7 +212,7 @@ proc `[]`*[V](obj: JsAssoc[cstring, V], field: string): V =
 proc `[]=`*[V](obj: JsAssoc[cstring, V], field: string, val: V) =
   obj[cstring(field)] = val
 
-proc `==`*(x, y: JsRoot): bool {.importcpp: "(# === #)".}
+proc `==`*(x, y: JsRoot): bool {.importjs: "(# === #)".}
   ## Compares two JsObjects or JsAssocs. Be careful though, as this is comparison
   ## like in JavaScript, so if your JsObjects are in fact JavaScript Objects,
   ## and not strings or numbers, this is a *comparison of references*.
@@ -227,36 +227,40 @@ macro `.`*(obj: JsObject, field: untyped): JsObject =
     assert obj.a.to(int) == 20
   if validJsName($field):
     let importString = "#." & $field
+    let helperName = genSym(nskProc, "helper")
     result = quote do:
-      proc helper(o: JsObject): JsObject
-        {.importcpp: `importString`, gensym.}
-      helper(`obj`)
+      proc `helperName`(o: JsObject): JsObject
+        {.importjs: `importString`.}
+      `helperName`(`obj`)
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName($field)
+      mangledNames[$field] = mangleJsName($field)
     let importString = "#." & mangledNames[$field]
+    let helperName = genSym(nskProc, "helper")
     result = quote do:
-      proc helper(o: JsObject): JsObject
-        {.importcpp: `importString`, gensym.}
-      helper(`obj`)
+      proc `helperName`(o: JsObject): JsObject
+        {.importjs: `importString`.}
+      `helperName`(`obj`)
 
 macro `.=`*(obj: JsObject, field, value: untyped): untyped =
   ## Experimental dot accessor (set) for type JsObject.
   ## Sets the value of a property of name `field` in a JsObject `x` to `value`.
   if validJsName($field):
     let importString = "#." & $field & " = #"
+    let helperName = genSym(nskProc, "helper")
     result = quote do:
-      proc helper(o: JsObject, v: auto)
-        {.importcpp: `importString`, gensym.}
-      helper(`obj`, `value`)
+      proc `helperName`(o: JsObject, v: auto)
+        {.importjs: `importString`.}
+      `helperName`(`obj`, `value`)
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName($field)
+      mangledNames[$field] = mangleJsName($field)
     let importString = "#." & mangledNames[$field] & " = #"
+    let helperName = genSym(nskProc, "helper")
     result = quote do:
-      proc helper(o: JsObject, v: auto)
-        {.importcpp: `importString`, gensym.}
-      helper(`obj`, `value`)
+      proc `helperName`(o: JsObject, v: auto)
+        {.importjs: `importString`.}
+      `helperName`(`obj`, `value`)
 
 macro `.()`*(obj: JsObject,
              field: untyped,
@@ -268,26 +272,26 @@ macro `.()`*(obj: JsObject,
   ## so be careful when using this.)
   ##
   ## Example:
-  ##
-  ## .. code-block:: nim
-  ##
-  ##  # Let's get back to the console example:
-  ##  var console {.importc, nodecl.}: JsObject
-  ##  let res = console.log("I return undefined!")
-  ##  console.log(res) # This prints undefined, as console.log always returns
-  ##                   # undefined. Thus one has to be careful, when using
-  ##                   # JsObject calls.
+  ##   ```nim
+  ##   # Let's get back to the console example:
+  ##   var console {.importc, nodecl.}: JsObject
+  ##   let res = console.log("I return undefined!")
+  ##   console.log(res) # This prints undefined, as console.log always returns
+  ##                    # undefined. Thus one has to be careful, when using
+  ##                    # JsObject calls.
+  ##   ```
   var importString: string
   if validJsName($field):
     importString = "#." & $field & "(@)"
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName($field)
+      mangledNames[$field] = mangleJsName($field)
     importString = "#." & mangledNames[$field] & "(@)"
-  result = quote:
-    proc helper(o: JsObject): JsObject
-      {.importcpp: `importString`, gensym, discardable.}
-    helper(`obj`)
+  let helperName = genSym(nskProc, "helper")
+  result = quote do:
+    proc `helperName`(o: JsObject): JsObject
+      {.importjs: `importString`, discardable.}
+    `helperName`(`obj`)
   for idx in 0 ..< args.len:
     let paramName = newIdentNode("param" & $idx)
     result[0][3].add newIdentDefs(paramName, newIdentNode("JsObject"))
@@ -302,12 +306,13 @@ macro `.`*[K: cstring, V](obj: JsAssoc[K, V],
     importString = "#." & $field
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName($field)
+      mangledNames[$field] = mangleJsName($field)
     importString = "#." & mangledNames[$field]
+  let helperName = genSym(nskProc, "helper")
   result = quote do:
-    proc helper(o: type(`obj`)): `obj`.V
-      {.importcpp: `importString`, gensym.}
-    helper(`obj`)
+    proc `helperName`(o: type(`obj`)): `obj`.V
+      {.importjs: `importString`.}
+    `helperName`(`obj`)
 
 macro `.=`*[K: cstring, V](obj: JsAssoc[K, V],
                                     field: untyped,
@@ -319,12 +324,13 @@ macro `.=`*[K: cstring, V](obj: JsAssoc[K, V],
     importString = "#." & $field & " = #"
   else:
     if not mangledNames.hasKey($field):
-      mangledNames[$field] = $mangleJsName($field)
+      mangledNames[$field] = mangleJsName($field)
     importString = "#." & mangledNames[$field] & " = #"
+  let helperName = genSym(nskProc, "helper")
   result = quote do:
-    proc helper(o: type(`obj`), v: `obj`.V)
-      {.importcpp: `importString`, gensym.}
-    helper(`obj`, `value`)
+    proc `helperName`(o: type(`obj`), v: `obj`.V)
+      {.importjs: `importString`.}
+    `helperName`(`obj`, `value`)
 
 macro `.()`*[K: cstring, V: proc](obj: JsAssoc[K, V],
                                            field: untyped,
@@ -348,8 +354,8 @@ iterator pairs*(obj: JsObject): (cstring, JsObject) =
   var k: cstring
   var v: JsObject
   {.emit: "for (var `k` in `obj`) {".}
-  {.emit: "  if (!`obj`.hasOwnProperty(`k`)) continue;".}
-  {.emit: "  `v`=`obj`[`k`];".}
+  {.emit: "  if (!`obj`.hasOwnProperty(`k`)) { continue; }".}
+  {.emit: "  `v` = `obj`[`k`];".}
   yield (k, v)
   {.emit: "}".}
 
@@ -357,8 +363,8 @@ iterator items*(obj: JsObject): JsObject =
   ## Yields the `values` of each field in a JsObject, wrapped into a JsObject.
   var v: JsObject
   {.emit: "for (var k in `obj`) {".}
-  {.emit: "  if (!`obj`.hasOwnProperty(k)) continue;".}
-  {.emit: "  `v`=`obj`[k];".}
+  {.emit: "  if (!`obj`.hasOwnProperty(k)) { continue; }".}
+  {.emit: "  `v` = `obj`[k];".}
   yield v
   {.emit: "}".}
 
@@ -366,7 +372,7 @@ iterator keys*(obj: JsObject): cstring =
   ## Yields the `names` of each field in a JsObject.
   var k: cstring
   {.emit: "for (var `k` in `obj`) {".}
-  {.emit: "  if (!`obj`.hasOwnProperty(`k`)) continue;".}
+  {.emit: "  if (!`obj`.hasOwnProperty(`k`)) { continue; }".}
   yield k
   {.emit: "}".}
 
@@ -376,8 +382,8 @@ iterator pairs*[K: JsKey, V](assoc: JsAssoc[K, V]): (K,V) =
   var k: cstring
   var v: V
   {.emit: "for (var `k` in `assoc`) {".}
-  {.emit: "  if (!`assoc`.hasOwnProperty(`k`)) continue;".}
-  {.emit: "  `v`=`assoc`[`k`];".}
+  {.emit: "  if (!`assoc`.hasOwnProperty(`k`)) { continue; }".}
+  {.emit: "  `v` = `assoc`[`k`];".}
   yield (k.toJsKey(K), v)
   {.emit: "}".}
 
@@ -385,8 +391,8 @@ iterator items*[K, V](assoc: JsAssoc[K, V]): V =
   ## Yields the `values` in a JsAssoc.
   var v: V
   {.emit: "for (var k in `assoc`) {".}
-  {.emit: "  if (!`assoc`.hasOwnProperty(k)) continue;".}
-  {.emit: "  `v`=`assoc`[k];".}
+  {.emit: "  if (!`assoc`.hasOwnProperty(k)) { continue; }".}
+  {.emit: "  `v` = `assoc`[k];".}
   yield v
   {.emit: "}".}
 
@@ -394,7 +400,7 @@ iterator keys*[K: JsKey, V](assoc: JsAssoc[K, V]): K =
   ## Yields the `keys` in a JsAssoc.
   var k: cstring
   {.emit: "for (var `k` in `assoc`) {".}
-  {.emit: "  if (!`assoc`.hasOwnProperty(`k`)) continue;".}
+  {.emit: "  if (!`assoc`.hasOwnProperty(`k`)) { continue; }".}
   yield k.toJsKey(K)
   {.emit: "}".}
 
@@ -407,21 +413,20 @@ macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto =
   ##
   ## Example:
   ##
-  ## .. code-block:: nim
+  ##   ```nim
+  ##   # Let's say we have a type with a ton of fields, where some fields do not
+  ##   # need to be set, and we do not want those fields to be set to `nil`:
+  ##   type
+  ##     ExtremelyHugeType = ref object
+  ##       a, b, c, d, e, f, g: int
+  ##       h, i, j, k, l: cstring
+  ##       # And even more fields ...
   ##
-  ##  # Let's say we have a type with a ton of fields, where some fields do not
-  ##  # need to be set, and we do not want those fields to be set to `nil`:
-  ##  type
-  ##    ExtremelyHugeType = ref object
-  ##      a, b, c, d, e, f, g: int
-  ##      h, i, j, k, l: cstring
-  ##      # And even more fields ...
-  ##
-  ##  let obj = ExtremelyHugeType{ a: 1, k: "foo".cstring, d: 42 }
-  ##
-  ##  # This generates roughly the same JavaScript as:
-  ##  {.emit: "var obj = {a: 1, k: "foo", d: 42};".}
+  ##   let obj = ExtremelyHugeType{ a: 1, k: "foo".cstring, d: 42 }
   ##
+  ##   # This generates roughly the same JavaScript as:
+  ##   {.emit: "var obj = {a: 1, k: "foo", d: 42};".}
+  ##   ```
   let a = ident"a"
   var body = quote do:
     var `a` {.noinit.}: `typ`
@@ -463,7 +468,7 @@ proc replaceSyms(n: NimNode): NimNode =
     for i in 0..= 201112 && !defined __STDC_NO_THREADS__
+#if defined _WIN32
+#  if defined _MSC_VER || defined __BORLANDC__
+#    define NIM_THREADVAR __declspec(thread)
+#  else
+#    define NIM_THREADVAR __thread
+#  endif
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__
 #  define NIM_THREADVAR _Thread_local
 #elif defined _WIN32 && ( \
        defined _MSC_VER || \
        defined __ICL || \
-       defined __DMC__ || \
        defined __BORLANDC__ )
 #  define NIM_THREADVAR __declspec(thread)
 #elif defined(__TINYC__) || defined(__GENODE__)
@@ -148,8 +140,7 @@ __AVR__
 #endif
 
 /* --------------- how int64 constants should be declared: ----------- */
-#if defined(__GNUC__) || defined(__LCC__) || \
-    defined(__POCC__) || defined(__DMC__) || defined(_MSC_VER)
+#if defined(__GNUC__) || defined(_MSC_VER)
 #  define IL64(x) x##LL
 #else /* works only without LL */
 #  define IL64(x) ((NI64)x)
@@ -187,12 +178,13 @@ __AVR__
 #  define N_THISCALL_PTR(rettype, name) rettype (__thiscall *name)
 #  define N_SAFECALL_PTR(rettype, name) rettype (__stdcall *name)
 
-#  ifdef __cplusplus
-#    define N_LIB_EXPORT  NIM_EXTERNC __declspec(dllexport)
+#  ifdef __EMSCRIPTEN__
+#    define N_LIB_EXPORT  NIM_EXTERNC __declspec(dllexport) __attribute__((used))
+#    define N_LIB_EXPORT_VAR  __declspec(dllexport) __attribute__((used))
 #  else
 #    define N_LIB_EXPORT  NIM_EXTERNC __declspec(dllexport)
+#    define N_LIB_EXPORT_VAR  __declspec(dllexport)
 #  endif
-#  define N_LIB_EXPORT_VAR  __declspec(dllexport)
 #  define N_LIB_IMPORT  extern __declspec(dllimport)
 #else
 #  define N_LIB_PRIVATE __attribute__((visibility("hidden")))
@@ -221,8 +213,13 @@ __AVR__
 #    define N_FASTCALL_PTR(rettype, name) rettype (*name)
 #    define N_SAFECALL_PTR(rettype, name) rettype (*name)
 #  endif
-#  define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default")))
-#  define N_LIB_EXPORT_VAR  __attribute__((visibility("default")))
+#  ifdef __EMSCRIPTEN__
+#    define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default"), used))
+#    define N_LIB_EXPORT_VAR  __attribute__((visibility("default"), used))
+#  else
+#    define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default")))
+#    define N_LIB_EXPORT_VAR  __attribute__((visibility("default")))
+#  endif
 #  define N_LIB_IMPORT  extern
 #endif
 
@@ -240,8 +237,7 @@ __AVR__
 
 #define N_NOINLINE_PTR(rettype, name) rettype (*name)
 
-#if defined(__BORLANDC__) || defined(__WATCOMC__) || \
-    defined(__POCC__) || defined(_MSC_VER) || defined(WIN32) || defined(_WIN32)
+#if defined(__BORLANDC__) || defined(_MSC_VER) || defined(WIN32) || defined(_WIN32)
 /* these compilers have a fastcall so use it: */
 #  ifdef __TINYC__
 #    define N_NIMCALL(rettype, name) rettype __attribute((__fastcall)) name
@@ -289,8 +285,7 @@ __AVR__
 #endif
 
 /* Known compiler with stdint.h that doesn't fit the general pattern? */
-#if defined(__LCC__) || defined(__DMC__) || defined(__POCC__) || \
-  defined(__AVR__) || (defined(__cplusplus) && (__cplusplus < 201103))
+#if defined(__AVR__) || (defined(__cplusplus) && (__cplusplus < 201103))
 #  define HAVE_STDINT_H
 #endif
 
@@ -325,6 +320,9 @@ typedef unsigned char NIM_BOOL; // best effort
 #endif
 
 NIM_STATIC_ASSERT(sizeof(NIM_BOOL) == 1, ""); // check whether really needed
+NIM_STATIC_ASSERT(CHAR_BIT == 8, "");
+  // fail fast for (rare) environments where this doesn't hold, as some implicit
+  // assumptions would need revisiting (e.g. `uint8` or https://github.com/nim-lang/Nim/pull/18505)
 
 #define NIM_TRUE true
 #define NIM_FALSE false
@@ -347,8 +345,7 @@ NIM_STATIC_ASSERT(sizeof(NIM_BOOL) == 1, ""); // check whether really needed
                               the generated code does not rely on it anymore */
 #endif
 
-#if defined(__BORLANDC__) || defined(__DMC__) \
-   || defined(__WATCOMC__) || defined(_MSC_VER)
+#if defined(__BORLANDC__) || defined(_MSC_VER)
 typedef signed char NI8;
 typedef signed short int NI16;
 typedef signed int NI32;
@@ -474,7 +471,9 @@ typedef char* NCSTRING;
   } name = {{length, (NI) ((NU)length | NIM_STRLIT_FLAG)}, str}
 
 /* declared size of a sequence/variable length array: */
-#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
+#if defined(__cplusplus) && defined(__clang__)
+#  define SEQ_DECL_SIZE 1
+#elif defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
 #  define SEQ_DECL_SIZE /* empty is correct! */
 #else
 #  define SEQ_DECL_SIZE 1000000
@@ -543,7 +542,7 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); }
 #endif
 
 // Test to see if Nim and the C compiler agree on the size of a pointer.
-NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, "");
+NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, "Pointer size mismatch between Nim and C/C++ backend. You probably need to setup the backend compiler for target CPU.");
 
 #ifdef USE_NIM_NAMESPACE
 }
diff --git a/lib/nimhcr.nim b/lib/nimhcr.nim
index 42e3266ae8..e87bb24139 100644
--- a/lib/nimhcr.nim
+++ b/lib/nimhcr.nim
@@ -200,6 +200,9 @@ batchable: false
 #     block. Perhaps something can be done about this - some way of re-allocating
 #     the state and transferring the old...
 
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 when not defined(js) and (defined(hotcodereloading) or
                           defined(createNimHcr) or
                           defined(testNimHcr)):
@@ -209,14 +212,14 @@ when not defined(js) and (defined(hotcodereloading) or
              else: "so"
   type
     HcrProcGetter* = proc (libHandle: pointer, procName: cstring): pointer {.nimcall.}
-    HcrGcMarkerProc = proc () {.nimcall.}
+    HcrGcMarkerProc = proc () {.nimcall, raises: [].}
     HcrModuleInitializer* = proc () {.nimcall.}
 
 when defined(createNimHcr):
   when system.appType != "lib":
     {.error: "This file has to be compiled as a library!".}
 
-  import os, tables, sets, times, strutils, reservedmem, dynlib
+  import std/[os, tables, sets, times, strutils, reservedmem, dynlib]
 
   template trace(args: varargs[untyped]) =
     when defined(testNimHcr) or defined(traceHcr):
@@ -302,7 +305,7 @@ when defined(createNimHcr):
       hash: string
       gen: int
       lastModification: Time
-      handlers: seq[tuple[isBefore: bool, cb: proc ()]]
+      handlers: seq[tuple[isBefore: bool, cb: proc () {.nimcall.}]]
 
   proc newModuleDesc(): ModuleDesc =
     result.procs = initTable[string, ProcSym]()
@@ -487,7 +490,7 @@ when defined(createNimHcr):
           recursiveDiscovery(modules[curr].imports)
           allModulesOrderedByDFS.add(curr)
           continue
-      loadDll(curr)
+      loadDll(curr.cstring)
       # first load all dependencies of the current module and init it after that
       recursiveDiscovery(modules[curr].imports)
 
@@ -497,20 +500,20 @@ when defined(createNimHcr):
   proc initModules() =
     # first init the pointers to hcr functions and also do the registering of typeinfo globals
     for curr in modulesToInit:
-      initHcrData(curr)
-      initTypeInfoGlobals(curr)
+      initHcrData(curr.cstring)
+      initTypeInfoGlobals(curr.cstring)
     # for now system always gets fully inited before any other module (including when reloading)
-    initPointerData(system)
-    initGlobalScope(system)
+    initPointerData(system.cstring)
+    initGlobalScope(system.cstring)
     # proceed with the DatInit calls - for all modules - including the main one!
     for curr in allModulesOrderedByDFS:
       if curr != system:
-        initPointerData(curr)
+        initPointerData(curr.cstring)
     mainDatInit()
     # execute top-level code (in global scope)
     for curr in modulesToInit:
       if curr != system:
-        initGlobalScope(curr)
+        initGlobalScope(curr.cstring)
     # cleanup old symbols which are gone now
     for curr in modulesToInit:
       cleanupSymbols(curr)
@@ -554,8 +557,12 @@ when defined(createNimHcr):
     # Future versions of NIMHCR won't use the GC, because all globals and the
     # metadata needed to access them will be placed in shared memory, so they
     # can be manipulated from external programs without reloading.
-    GC_disable()
-    defer: GC_enable()
+    when declared(GC_disable):
+      GC_disable()
+      defer: GC_enable()
+    elif declared(GC_disableOrc):
+      GC_disableOrc()
+      defer: GC_enableOrc()
 
     inc(generation)
     trace "HCR RELOADING: ", generation
@@ -595,7 +602,7 @@ when defined(createNimHcr):
       hashToModuleMap.del(modules[name].hash)
       modules.del(name)
 
-  proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} =
+  proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.} =
     modules[currentModule].handlers.add(
       (isBefore: isBefore, cb: cb))
 
@@ -606,7 +613,7 @@ when defined(createNimHcr):
   proc hcrGeneration*(): int {.nimhcr.} =
     generation
 
-  proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.} =
+  proc hcrMarkGlobals*() {.compilerproc, exportc, dynlib, nimcall, gcsafe.} =
     # This is gcsafe, because it will be registered
     # only in the GC of the main thread.
     {.gcsafe.}:
@@ -646,19 +653,19 @@ elif defined(hotcodereloading) or defined(testNimHcr):
 
     proc hcrPerformCodeReload*() {.nimhcr.}
 
-    proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.}
+    proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.}
 
-    proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.}
+    proc hcrMarkGlobals*() {.raises: [], nimhcr, nimcall, gcsafe.}
 
     when declared(nimRegisterGlobalMarker):
-      nimRegisterGlobalMarker(hcrMarkGlobals)
+      nimRegisterGlobalMarker(cast[GlobalMarkerProc](hcrMarkGlobals))
 
   else:
     proc hcrHasModuleChanged*(moduleHash: string): bool =
       # TODO
       false
 
-    proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) =
+    proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) =
       # TODO
       discard
 
diff --git a/lib/nimrtl.nim b/lib/nimrtl.nim
index 301ffb43b9..a2fb6ce600 100644
--- a/lib/nimrtl.nim
+++ b/lib/nimrtl.nim
@@ -26,6 +26,7 @@ batchable: false
 ## * unicode
 ## * pegs
 ## * ropes
+## * cstrutils
 ##
 
 when system.appType != "lib":
@@ -35,5 +36,5 @@ when not defined(createNimRtl):
   {.error: "This file has to be compiled with '-d:createNimRtl'".}
 
 import
-  parseutils, strutils, parseopt, parsecfg, strtabs, unicode, pegs, ropes,
-  os, osproc, times, cstrutils
+  std/[parseutils, strutils, parseopt, parsecfg, strtabs, unicode, pegs, ropes,
+  os, osproc, times, cstrutils]
diff --git a/lib/nintendoswitch/switch_memory.nim b/lib/nintendoswitch/switch_memory.nim
deleted file mode 100644
index f34bd363a1..0000000000
--- a/lib/nintendoswitch/switch_memory.nim
+++ /dev/null
@@ -1,36 +0,0 @@
-## All of these library headers and source can be found in the github repo
-## https://github.com/switchbrew/libnx.
-
-const virtMemHeader = ""
-const svcHeader = ""
-const mallocHeader = ""
-
-## Aligns a block of memory with request `size` to `bytes` size. For
-## example, a request of memalign(0x1000, 0x1001) == 0x2000 bytes allocated
-proc memalign*(bytes: csize, size: csize): pointer {.importc: "memalign",
-    header: mallocHeader.}
-
-# Should be required, but not needed now because of how
-# svcUnmapMemory frees all memory
-#proc free*(address: pointer) {.importc: "free",
-#    header: mallocHeader.}
-
-## Maps a memaligned block of memory from `src_addr` to `dst_addr`. The
-## Nintendo Switch requires this call in order to make use of memory, otherwise
-## an invalid memory access occurs.
-proc svcMapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {.
-    importc: "svcMapMemory", header: svcHeader.}
-
-## Unmaps (frees) all memory from both `dst_addr` and `src_addr`. **Must** be called
-## whenever svcMapMemory is used. The Switch will expect all memory to be allocated
-## before gfxExit() calls ()
-proc svcUnmapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {.
-    importc: "svcUnmapMemory", header: svcHeader.}
-
-proc virtmemReserveMap*(size: csize): pointer {.importc: "virtmemReserveMap",
-    header: virtMemHeader.}
-
-# Should be required, but not needed now because of how
-# svcUnmapMemory frees all memory
-#proc virtmemFreeMap*(address: pointer; size: csize) {.importc: "virtmemFreeMap",
-#    header: virtMemHeader.}
diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim
new file mode 100644
index 0000000000..0a41d85b53
--- /dev/null
+++ b/lib/packages/docutils/dochelpers.nim
@@ -0,0 +1,298 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2021 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Integration helpers between ``docgen.nim`` and ``rst.nim``.
+##
+## Function `toLangSymbol(linkText)`_ produces a signature `docLink` of
+## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_
+## matches it with `generated`, produced from `PNode` by ``docgen.rst``.
+
+import rstast
+import std/strutils
+
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, syncio]
+
+
+type
+  LangSymbol* = object       ## symbol signature in Nim
+    symKind*: string           ## "proc", "const", "type", etc
+    symTypeKind*: string       ## ""|enum|object|tuple -
+                               ## valid only when `symKind == "type"`
+    name*: string              ## plain symbol name without any parameters
+    generics*: string          ## generic parameters (without brackets)
+    isGroup*: bool             ## is LangSymbol a group with overloads?
+    # the following fields are valid iff `isGroup` == false
+    # (always false when parsed by `toLangSymbol` because link like foo_
+    # can point to just a single symbol foo, e.g. proc).
+    parametersProvided*: bool  ## to disambiguate `proc f`_ and `proc f()`_
+    parameters*: seq[tuple[name: string, `type`: string]]
+                               ## name-type seq, e.g. for proc
+    outType*: string           ## result type, e.g. for proc
+
+proc `$`*(s: LangSymbol): string =  # for debug
+  ("(symkind=$1, symTypeKind=$2, name=$3, generics=$4, isGroup=$5, " &
+   "parametersProvided=$6, parameters=$7, outType=$8)") % [
+      s.symKind, s.symTypeKind , s.name, s.generics, $s.isGroup,
+      $s.parametersProvided, $s.parameters, s.outType]
+
+func nimIdentBackticksNormalize*(s: string): string =
+  ## Normalizes the string `s` as a Nim identifier.
+  ##
+  ## Unlike `nimIdentNormalize` removes spaces and backticks.
+  ##
+  ## .. Warning:: No checking (e.g. that identifiers cannot start from
+  ##    digits or '_', or that number of backticks is even) is performed.
+  runnableExamples:
+    doAssert nimIdentBackticksNormalize("Foo_bar") == "Foobar"
+    doAssert nimIdentBackticksNormalize("FoO BAr") == "Foobar"
+    doAssert nimIdentBackticksNormalize("`Foo BAR`") == "Foobar"
+    doAssert nimIdentBackticksNormalize("` Foo BAR `") == "Foobar"
+    # not a valid identifier:
+    doAssert nimIdentBackticksNormalize("`_x_y`") == "_xy"
+  result = newString(s.len)
+  var firstChar = true
+  var j = 0
+  for i in 0..len(s) - 1:
+    if s[i] in {'A'..'Z'}:
+      if not firstChar:  # to lowercase
+        result[j] = chr(ord(s[i]) + (ord('a') - ord('A')))
+      else:
+        result[j] = s[i]
+        firstChar = false
+      inc j
+    elif s[i] notin {'_', ' ', '`'}:
+      result[j] = s[i]
+      inc j
+      firstChar = false
+    elif s[i] == '_' and firstChar:
+      result[j] = '_'
+      inc j
+      firstChar = false
+    else: discard  # just omit '`' or ' '
+  if j != s.len: setLen(result, j)
+
+proc langSymbolGroup*(kind: string, name: string): LangSymbol =
+  if kind notin ["proc", "func", "macro", "method", "iterator",
+                 "template", "converter"]:
+    raise newException(ValueError, "unknown symbol kind $1" % [kind])
+  result = LangSymbol(symKind: kind, name: name, isGroup: true)
+
+proc toLangSymbol*(linkText: PRstNode): LangSymbol =
+  ## Parses `linkText` into a more structured form using a state machine.
+  ##
+  ## This proc is designed to allow link syntax with operators even
+  ## without escaped backticks inside:
+  ##   
+  ##     `proc *`_
+  ##     `proc []`_
+  ##
+  ## This proc should be kept in sync with the `renderTypes` proc from
+  ## ``compiler/typesrenderer.nim``.
+  template fail(msg: string) =
+    raise newException(ValueError, msg)
+  if linkText.kind notin {rnRstRef, rnInner}:
+    fail("toLangSymbol: wrong input kind " & $linkText.kind)
+
+  const NimDefs = ["proc", "func", "macro", "method", "iterator",
+                   "template", "converter", "const", "type", "var",
+                   "enum", "object", "tuple", "module"]
+  template resolveSymKind(x: string) =
+    if x in ["enum", "object", "tuple"]:
+      result.symKind = "type"
+      result.symTypeKind = x
+    else:
+      result.symKind = x
+  type
+    State = enum
+      inBeginning
+      afterSymKind
+      beforeSymbolName  # auxiliary state to catch situations like `proc []`_ after space
+      atSymbolName
+      afterSymbolName
+      genericsPar
+      parameterName
+      parameterType
+      outType
+  var state = inBeginning
+  var curIdent = ""
+  template flushIdent() =
+    if curIdent != "":
+      case state
+      of inBeginning:  fail("incorrect state inBeginning")
+      of afterSymKind:  resolveSymKind curIdent
+      of beforeSymbolName:  fail("incorrect state beforeSymbolName")
+      of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize
+      of afterSymbolName: fail("incorrect state afterSymbolName")
+      of genericsPar: result.generics = curIdent
+      of parameterName: result.parameters.add (curIdent, "")
+      of parameterType:
+        for a in countdown(result.parameters.len - 1, 0):
+          if result.parameters[a].`type` == "":
+            result.parameters[a].`type` = curIdent
+      of outType: result.outType = curIdent
+      curIdent = ""
+  var parens = 0
+  let L = linkText.sons.len
+  template s(i: int): string = linkText.sons[i].text
+  var i = 0
+  template nextState =
+    case s(i)
+    of " ":
+      if state == afterSymKind:
+        flushIdent
+        state = beforeSymbolName
+    of "`":
+      curIdent.add "`"
+      inc i
+      while i < L:  # add contents between ` ` as a whole
+        curIdent.add s(i)
+        if s(i) == "`":
+          break
+        inc i
+      curIdent = curIdent.nimIdentBackticksNormalize
+      if state in {inBeginning, afterSymKind, beforeSymbolName}:
+        state = atSymbolName
+        flushIdent
+        state = afterSymbolName
+    of "[":
+      if state notin {inBeginning, afterSymKind, beforeSymbolName}:
+        inc parens
+      if state in {inBeginning, afterSymKind, beforeSymbolName}:
+        state = atSymbolName
+        curIdent.add s(i)
+      elif state in {atSymbolName, afterSymbolName} and parens == 1:
+        flushIdent
+        state = genericsPar
+        curIdent.add s(i)
+      else: curIdent.add s(i)
+    of "]":
+      if state notin {inBeginning, afterSymKind, beforeSymbolName, atSymbolName}:
+        dec parens
+      if state == genericsPar and parens == 0:
+        curIdent.add s(i)
+        flushIdent
+      else: curIdent.add s(i)
+    of "(":
+      inc parens
+      if state in {inBeginning, afterSymKind, beforeSymbolName}:
+        result.parametersProvided = true
+        state = atSymbolName
+        flushIdent
+        state = parameterName
+      elif state in {atSymbolName, afterSymbolName, genericsPar} and parens == 1:
+        result.parametersProvided = true
+        flushIdent
+        state = parameterName
+      else: curIdent.add s(i)
+    of ")":
+      dec parens
+      if state in {parameterName, parameterType} and parens == 0:
+        flushIdent
+        state = outType
+      else: curIdent.add s(i)
+    of "{":  # remove pragmas
+      while i < L:
+        if s(i) == "}":
+          break
+        inc i
+    of ",", ";":
+      if state in {parameterName, parameterType} and parens == 1:
+        flushIdent
+        state = parameterName
+      else: curIdent.add s(i)
+    of "*":  # skip export symbol
+      if state == atSymbolName:
+        flushIdent
+        state = afterSymbolName
+      elif state == afterSymbolName:
+        discard
+      else: curIdent.add "*"
+    of ":":
+      if state == outType: discard
+      elif state == parameterName:
+        flushIdent
+        state = parameterType
+      else: curIdent.add ":"
+    else:
+      let isPostfixSymKind = i > 0 and i == L - 1 and
+          result.symKind == "" and s(i) in NimDefs
+      if isPostfixSymKind:  # for links like `foo proc`_
+        resolveSymKind s(i)
+      else:
+        case state
+        of inBeginning:
+          if s(i) in NimDefs:
+            state = afterSymKind
+          else:
+            state = atSymbolName
+          curIdent.add s(i)
+        of afterSymKind, beforeSymbolName:
+          state = atSymbolName
+          curIdent.add s(i)
+        of parameterType:
+          case s(i)
+          of "ref": curIdent.add "ref."
+          of "ptr": curIdent.add "ptr."
+          of "var": discard
+          else: curIdent.add s(i).nimIdentBackticksNormalize
+        of atSymbolName:
+          curIdent.add s(i)
+        else:
+          curIdent.add s(i).nimIdentBackticksNormalize
+  while i < L:
+    nextState
+    inc i
+  if state == afterSymKind:  # treat `type`_ as link to symbol `type`
+    state = atSymbolName
+  flushIdent
+  result.isGroup = false
+
+proc match*(generated: LangSymbol, docLink: LangSymbol): bool =
+  ## Returns true if `generated` can be a target for `docLink`.
+  ## If `generated` is an overload group then only `symKind` and `name`
+  ## are compared for success.
+  result = true
+  if docLink.symKind != "":
+    if generated.symKind == "proc":
+      result = docLink.symKind in ["proc", "func"]
+    else:
+      result = generated.symKind == docLink.symKind
+      if result and docLink.symKind == "type" and docLink.symTypeKind != "":
+        result = generated.symTypeKind == docLink.symTypeKind
+    if not result: return
+  result = generated.name == docLink.name
+  if not result: return
+  if generated.isGroup:
+    # if `()` were added then it's not a reference to the whole group:
+    return not docLink.parametersProvided
+  if docLink.generics != "":
+    result = generated.generics == docLink.generics
+    if not result: return
+  if docLink.outType != "":
+    result = generated.outType == docLink.outType
+    if not result: return
+  if docLink.parametersProvided:
+    result = generated.parameters.len == docLink.parameters.len
+    if not result: return
+    var onlyType = false
+    for i in 0 ..< generated.parameters.len:
+      let g = generated.parameters[i]
+      let d = docLink.parameters[i]
+      if i == 0:
+        if g.`type` == d.name:
+          onlyType = true  # only types, not names, are provided in `docLink`
+      if onlyType:
+        result = g.`type` == d.name
+      else:
+        if d.`type` != "":
+          result = g.`type` == d.`type`
+          if not result: return
+        result = g.name == d.name
+      if not result: return
diff --git a/lib/packages/docutils/docutils.nimble b/lib/packages/docutils/docutils.nimble
deleted file mode 100644
index f1683c5152..0000000000
--- a/lib/packages/docutils/docutils.nimble
+++ /dev/null
@@ -1,4 +0,0 @@
-version       = "0.10.0"
-author        = "Andreas Rumpf"
-description   = "Nim's reStructuredText processor."
-license       = "MIT"
diff --git a/lib/packages/docutils/docutils.nimble.old b/lib/packages/docutils/docutils.nimble.old
new file mode 100644
index 0000000000..f97c3bdde1
--- /dev/null
+++ b/lib/packages/docutils/docutils.nimble.old
@@ -0,0 +1,7 @@
+# xxx disabled this as this isn't really a nimble package and it affects logic
+# used to compute canonical imports, refs https://github.com/nim-lang/Nim/pull/16999#issuecomment-805442914
+
+version       = "0.10.0"
+author        = "Andreas Rumpf"
+description   = "Nim's reStructuredText processor."
+license       = "MIT"
diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim
index 8c91e0a8ed..f8376f46c3 100644
--- a/lib/packages/docutils/highlite.nim
+++ b/lib/packages/docutils/highlite.nim
@@ -11,11 +11,9 @@
 ## Currently only few languages are supported, other languages may be added.
 ## The interface supports one language nested in another.
 ##
-## **Note:** Import `packages/docutils/highlite` to use this module
-##
 ## You can use this to build your own syntax highlighting, check this example:
 ##
-## .. code::nim
+##   ```Nim
 ##   let code = """for x in $int.high: echo x.ord mod 2 == 0"""
 ##   var toknizr: GeneralTokenizer
 ##   initGeneralTokenizer(toknizr, code)
@@ -33,21 +31,43 @@
 ##     else:
 ##       echo toknizr.kind # All the kinds of tokens can be processed here.
 ##       echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1)
+##   ```
 ##
 ## The proc `getSourceLanguage` can get the language `enum` from a string:
-##
-## .. code::nim
+##   ```Nim
 ##   for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l)
+##   ```
 ##
+## There is also a `Cmd` pseudo-language supported, which is a simple generic
+## shell/cmdline tokenizer (UNIX shell/Powershell/Windows Command):
+## no escaping, no programming language constructs besides variable definition
+## at the beginning of line. It supports these operators:
+##   ```Cmd
+##   &  &&  |  ||  (  )  ''  ""  ;  # for comments
+##   ```
+##
+## Instead of escaping always use quotes like here
+## `nimgrep --ext:'nim|nims' file.name`:cmd: shows how to input ``|``.
+## Any argument that contains ``.`` or ``/`` or ``\`` will be treated
+## as a file or directory.
+##
+## In addition to `Cmd` there is also `Console` language for
+## displaying interactive sessions.
+## Lines with a command should start with ``$``, other lines are considered
+## as program output.
 
 import
-  strutils
-from algorithm import binarySearch
+  std/strutils
+from std/algorithm import binarySearch
+
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, syncio]
+
 
 type
   SourceLanguage* = enum
     langNone, langNim, langCpp, langCsharp, langC, langJava,
-    langYaml, langPython
+    langYaml, langPython, langCmd, langConsole
   TokenClass* = enum
     gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber,
     gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit,
@@ -55,7 +75,7 @@ type
     gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression,
     gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler,
     gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel,
-    gtReference, gtOther
+    gtReference, gtPrompt, gtProgramOutput, gtProgram, gtOption, gtOther
   GeneralTokenizer* = object of RootObj
     kind*: TokenClass
     start*, length*: int
@@ -66,14 +86,20 @@ type
 
 const
   sourceLanguageToStr*: array[SourceLanguage, string] = ["none",
-    "Nim", "C++", "C#", "C", "Java", "Yaml", "Python"]
+    "Nim", "C++", "C#", "C", "Java", "Yaml", "Python", "Cmd", "Console"]
+  sourceLanguageToAlpha*: array[SourceLanguage, string] = ["none",
+    "Nim", "cpp", "csharp", "C", "Java", "Yaml", "Python", "Cmd", "Console"]
+    ## list of languages spelled with alpabetic characters
   tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace",
     "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber",
     "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit",
     "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment",
     "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData",
     "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink",
-    "Label", "Reference", "Other"]
+    "Label", "Reference", "Prompt", "ProgramOutput",
+    # start from lower-case if there is a corresponding RST role (see rst.nim)
+    "program", "option",
+    "Other"]
 
   # The following list comes from doc/keywords.txt, make sure it is
   # synchronized with this array by running the module itself as a test case.
@@ -90,9 +116,11 @@ const
     "xor", "yield"]
 
 proc getSourceLanguage*(name: string): SourceLanguage =
-  for i in countup(succ(low(SourceLanguage)), high(SourceLanguage)):
+  for i in succ(low(SourceLanguage)) .. high(SourceLanguage):
     if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0:
       return i
+    if cmpIgnoreStyle(name, sourceLanguageToAlpha[i]) == 0:
+      return i
   result = langNone
 
 proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) =
@@ -102,9 +130,7 @@ proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) =
   g.length = 0
   g.state = low(TokenClass)
   g.lang = low(SourceLanguage)
-  var pos = 0                     # skip initial whitespace:
-  while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos)
-  g.pos = pos
+  g.pos = 0
 
 proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: string) =
   initGeneralTokenizer(g, cstring(buf))
@@ -165,9 +191,6 @@ const
 proc isKeyword(x: openArray[string], y: string): int =
   binarySearch(x, y)
 
-proc isKeywordIgnoreCase(x: openArray[string], y: string): int =
-  binarySearch(x, y, cmpIgnoreCase)
-
 proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) =
   const
     hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'}
@@ -177,31 +200,33 @@ proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) =
   var pos = g.pos
   g.start = g.pos
   if g.state == gtStringLit:
-    g.kind = gtStringLit
-    while true:
+    if g.buf[pos] == '\\':
+      g.kind = gtEscapeSequence
+      inc(pos)
       case g.buf[pos]
-      of '\\':
-        g.kind = gtEscapeSequence
+      of 'x', 'X':
         inc(pos)
-        case g.buf[pos]
-        of 'x', 'X':
-          inc(pos)
-          if g.buf[pos] in hexChars: inc(pos)
-          if g.buf[pos] in hexChars: inc(pos)
-        of '0'..'9':
-          while g.buf[pos] in {'0'..'9'}: inc(pos)
-        of '\0':
-          g.state = gtNone
-        else: inc(pos)
-        break
-      of '\0', '\r', '\n':
+        if g.buf[pos] in hexChars: inc(pos)
+        if g.buf[pos] in hexChars: inc(pos)
+      of '0'..'9':
+        while g.buf[pos] in {'0'..'9'}: inc(pos)
+      of '\0':
         g.state = gtNone
-        break
-      of '\"':
-        inc(pos)
-        g.state = gtNone
-        break
       else: inc(pos)
+    else:
+      g.kind = gtStringLit
+      while true:
+        case g.buf[pos]
+        of '\\':
+          break
+        of '\0', '\r', '\n':
+          g.state = gtNone
+          break
+        of '\"':
+          inc(pos)
+          g.state = gtNone
+          break
+        else: inc(pos)
   else:
     case g.buf[pos]
     of ' ', '\t'..'\r':
@@ -299,17 +324,18 @@ proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) =
       pos = nimNumber(g, pos)
     of '\'':
       inc(pos)
-      g.kind = gtCharLit
-      while true:
-        case g.buf[pos]
-        of '\0', '\r', '\n':
-          break
-        of '\'':
-          inc(pos)
-          break
-        of '\\':
-          inc(pos, 2)
-        else: inc(pos)
+      if g.kind != gtPunctuation:
+        g.kind = gtCharLit
+        while true:
+          case g.buf[pos]
+          of '\0', '\r', '\n':
+            break
+          of '\'':
+            inc(pos)
+            break
+          of '\\':
+            inc(pos, 2)
+          else: inc(pos)
     of '\"':
       inc(pos)
       if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'):
@@ -473,6 +499,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string],
           of '\0':
             break
           else: inc(pos)
+      else:
+        g.kind = gtOperator
+        while g.buf[pos] in OpChars: inc(pos)
     of '#':
       inc(pos)
       if hasPreprocessor in flags:
@@ -900,6 +929,74 @@ proc pythonNextToken(g: var GeneralTokenizer) =
       "with", "yield"]
   nimNextToken(g, keywords)
 
+proc cmdNextToken(g: var GeneralTokenizer, dollarPrompt = false) =
+  var pos = g.pos
+  g.start = g.pos
+  if g.state == low(TokenClass):
+    g.state = if dollarPrompt: gtPrompt else: gtProgram
+  case g.buf[pos]
+  of ' ', '\t'..'\r':
+    g.kind = gtWhitespace
+    while g.buf[pos] in {' ', '\t'..'\r'}:
+      if g.buf[pos] == '\n':
+        g.state = if dollarPrompt: gtPrompt else: gtProgram
+      inc(pos)
+  of '\'', '"':
+    g.kind = gtOption
+    let q = g.buf[pos]
+    inc(pos)
+    while g.buf[pos] notin {q, '\0'}:
+      inc(pos)
+    if g.buf[pos] == q: inc(pos)
+  of '#':
+    g.kind = gtComment
+    while g.buf[pos] notin {'\n', '\0'}:
+      inc(pos)
+  of '&', '|':
+    g.kind = gtOperator
+    inc(pos)
+    if g.buf[pos] == g.buf[pos-1]: inc(pos)
+    g.state = gtProgram
+  of '(':
+    g.kind = gtOperator
+    g.state = gtProgram
+    inc(pos)
+  of ')':
+    g.kind = gtOperator
+    inc(pos)
+  of ';':
+    g.state = gtProgram
+    g.kind = gtOperator
+    inc(pos)
+  of '\0': g.kind = gtEof
+  elif dollarPrompt and g.state == gtPrompt:
+    if g.buf[pos] == '$' and g.buf[pos+1] in {' ', '\t'}:
+      g.kind = gtPrompt
+      inc pos, 2
+      g.state = gtProgram
+    else:
+      g.kind = gtProgramOutput
+      while g.buf[pos] notin {'\n', '\0'}:
+        inc(pos)
+  else:
+    if g.state == gtProgram:
+      g.kind = gtProgram
+      g.state = gtOption
+    else:
+      g.kind = gtOption
+    while g.buf[pos] notin {' ', '\t'..'\r', '&', '|', '(', ')', '\'', '"', '\0'}:
+      if g.buf[pos] == ';' and g.buf[pos+1] == ' ':
+        # (check space because ';' can be used inside arguments in Win bat)
+        break
+      if g.kind == gtOption and g.buf[pos] in {'/', '\\', '.'}:
+        g.kind = gtIdentifier  # for file/dir name
+      elif g.kind == gtProgram and g.buf[pos] == '=':
+        g.kind = gtIdentifier  # for env variable setting at beginning of line
+        g.state = gtProgram
+      inc(pos)
+  g.length = pos - g.pos
+  g.pos = pos
+
 proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   g.lang = lang
   case lang
@@ -911,6 +1008,20 @@ proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   of langJava: javaNextToken(g)
   of langYaml: yamlNextToken(g)
   of langPython: pythonNextToken(g)
+  of langCmd: cmdNextToken(g)
+  of langConsole: cmdNextToken(g, dollarPrompt=true)
+
+proc tokenize*(text: string, lang: SourceLanguage): seq[(string, TokenClass)] =
+  var g: GeneralTokenizer
+  initGeneralTokenizer(g, text)
+  var prevPos = 0
+  while true:
+    getNextToken(g, lang)
+    if g.kind == gtEof:
+      break
+    var s = text[prevPos ..< g.pos]
+    result.add (s, g.kind)
+    prevPos = g.pos
 
 when isMainModule:
   var keywords: seq[string]
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index b5eef76105..9af163e157 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -7,156 +7,46 @@
 #    distribution, for details about the copyright.
 #
 
-## ==================================
-##                rst
-## ==================================
+## This module implements a `reStructuredText`:idx: (RST) and
+## `Markdown`:idx: parser.
+## User's manual on supported markup syntax and command line usage can be
+## found in [Nim-flavored Markdown and reStructuredText](markdown_rst.html).
 ##
-## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-## Nim-flavored reStructuredText
-## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+## * See also [Nim DocGen Tools Guide](docgen.html) for handling of
+##   ``.nim`` files.
+## * See also [packages/docutils/rstgen module](rstgen.html) to know how to
+##   generate HTML or Latex strings (for embedding them into custom documents).
 ##
-## This module implements a `reStructuredText`:idx: (RST) parser.
-## A large subset is implemented with some limitations_ and
-## `Nim-specific features`_.
-## A few `extra features`_ of the `Markdown`:idx: syntax are
-## also supported.
-##
-## Nim can output the result to HTML [#html]_ or Latex [#latex]_.
-##
-## .. [#html] commands ``nim doc`` for ``*.nim`` files and
-##    ``nim rst2html`` for ``*.rst`` files
-##
-## .. [#latex] command ``nim rst2tex`` for ``*.rst``.
-##
-## If you are new to RST please consider reading the following:
-##
-## 1) a short `quick introduction`_
-## 2) an `RST reference`_: a comprehensive cheatsheet for RST
-## 3) a more formal 50-page `RST specification`_.
-##
-## Features
-## --------
-##
-## Supported standard RST features:
-##
-## * body elements
-##   + sections
-##   + transitions
-##   + paragraphs
-##   + bullet lists using \+, \*, \-
-##   + enumerated lists using arabic numerals or alphabet
-##     characters:  1. ... 2. ... *or* a. ... b. ... *or* A. ... B. ...
-##   + footnotes (including manually numbered, auto-numbered, auto-numbered
-##     with label, and auto-symbol footnotes) and citations
-##   + definition lists
-##   + field lists
-##   + option lists
-##   + indented literal blocks
-##   + simple tables
-##   + directives (see official documentation in `RST directives list`_):
-##     - ``image``, ``figure`` for including images and videos
-##     - ``code``
-##     - ``contents`` (table of contents), ``container``, ``raw``
-##     - ``include``
-##     - admonitions: "attention", "caution", "danger", "error", "hint",
-##       "important", "note", "tip", "warning", "admonition"
-##     - substitution definitions: `replace` and `image`
-##   + comments
-## * inline markup
-##   + *emphasis*, **strong emphasis**,
-##     ``inline literals``, hyperlink references (including embedded URI),
-##     substitution references, standalone hyperlinks,
-##     internal links (inline and outline)
-##   + \`interpreted text\` with roles ``:literal:``, ``:strong:``,
-##     ``emphasis``, ``:sub:``/``:subscript:``, ``:sup:``/``:supscript:``
-##     (see `RST roles list`_ for description).
-##   + inline internal targets
-##
-## .. _`Nim-specific features`:
-##
-## Additional Nim-specific features:
-##
-## * directives: ``code-block`` [cmp:Sphinx]_, ``title``,
-##   ``index`` [cmp:Sphinx]_
-##
-## * ***triple emphasis*** (bold and italic) using \*\*\*
-## * ``:idx:`` role for \`interpreted text\` to include the link to this
-##   text into an index (example: `Nim index`_).
-##
-## .. [cmp:Sphinx] similar but different from the directives of
-##    Python `Sphinx directives`_ extensions
-##
-## .. _`extra features`:
-##
-## Optional additional features, turned on by ``options: RstParseOption`` in
-## `rstParse proc <#rstParse,string,string,int,int,bool,RstParseOptions,FindFileHandler,MsgHandler>`_:
-##
-## * emoji / smiley symbols
-## * Markdown tables
-## * Markdown code blocks
-## * Markdown links
-## * Markdown headlines
-## * using ``1`` as auto-enumerator in enumerated lists like RST ``#``
-##   (auto-enumerator ``1`` can not be used with ``#`` in the same list)
-##
-## .. Note:: By default Nim has ``roSupportMarkdown`` and
-##    ``roSupportRawDirective`` turned **on**.
-##
-## .. warning:: Using Nim-specific features can cause other RST implementations
-##   to fail on your document.
-##
-## Limitations
-## -----------
-##
-## * no Unicode support in character width calculations
-## * body elements
-##   - no roman numerals in enumerated lists
-##   - no quoted literal blocks
-##   - no doctest blocks
-##   - no grid tables
-##   - some directives are missing (check official `RST directives list`_):
-##     ``parsed-literal``, ``sidebar``, ``topic``, ``math``, ``rubric``,
-##     ``epigraph``, ``highlights``, ``pull-quote``, ``compound``,
-##     ``table``, ``csv-table``, ``list-table``, ``section-numbering``,
-##     ``header``, ``footer``, ``meta``, ``class``
-##     - no ``role`` directives and no custom interpreted text roles
-##     - some standard roles are not supported (check `RST roles list`_)
-## * inline markup
-##   - no simple-inline-markup
-##   - no embedded aliases
-##
-## Usage
-## -----
-##
-## See `Nim DocGen Tools Guide `_ for the details about
-## ``nim doc``, ``nim rst2html`` and ``nim rst2tex`` commands.
-##
-## See `packages/docutils/rstgen module `_ to know how to
-## generate HTML or Latex strings to embed them into your documents.
-##
-## .. Tip:: Import ``packages/docutils/rst`` to use this module
-##    programmatically.
-##
-## .. _quick introduction: https://docutils.sourceforge.io/docs/user/rst/quickstart.html
-## .. _RST reference: https://docutils.sourceforge.io/docs/user/rst/quickref.html
-## .. _RST specification: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html
-## .. _RST directives list: https://docutils.sourceforge.io/docs/ref/rst/directives.html
-## .. _RST roles list: https://docutils.sourceforge.io/docs/ref/rst/roles.html
-## .. _Nim index: https://nim-lang.org/docs/theindex.html
-## .. _Sphinx directives: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html
+## Choice between Markdown and RST as well as optional additional features are
+## turned on by passing ``options:`` [RstParseOptions] to [proc rstParse].
 
 import
-  os, strutils, rstast, std/enumutils, algorithm, lists, sequtils,
-  std/private/miscdollars
+  std/[os, strutils, enumutils, algorithm, lists, sequtils,
+  tables, strscans]
+import dochelpers, rstidx, rstast
+import std/private/miscdollars
+from highlite import SourceLanguage, getSourceLanguage
+
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, syncio]
+
 
 type
   RstParseOption* = enum     ## options for the RST parser
-    roSkipPounds,             ## skip ``#`` at line beginning (documentation
-                              ## embedded in Nim comments)
     roSupportSmilies,         ## make the RST parser support smilies like ``:)``
     roSupportRawDirective,    ## support the ``raw`` directive (don't support
                               ## it for sandboxing)
-    roSupportMarkdown         ## support additional features of Markdown
+    roSupportMarkdown,        ## support additional features of Markdown
+    roPreferMarkdown,         ## parse as Markdown (keeping RST as "extension"
+                              ## to Markdown) -- implies `roSupportMarkdown`
+    roNimFile                 ## set for Nim files where default interpreted
+                              ## text role should be :nim:
+    roSandboxDisabled         ## this option enables certain options
+                              ## (e.g. raw, include, importdoc)
+                              ## which are disabled by default as they can
+                              ## enable users to read arbitrary data and
+                              ## perform XSS if the parser is used in a web
+                              ## app.
 
   RstParseOptions* = set[RstParseOption]
 
@@ -165,28 +55,41 @@ type
     mcWarning = "Warning",
     mcError = "Error"
 
+  # keep the order in sync with compiler/docgen.nim and compiler/lineinfos.nim:
   MsgKind* = enum          ## the possible messages
     meCannotOpenFile = "cannot open '$1'",
     meExpected = "'$1' expected",
+    meMissingClosing = "$1",
     meGridTableNotImplemented = "grid table is not implemented",
     meMarkdownIllformedTable = "illformed delimiter row of a Markdown table",
+    meIllformedTable = "Illformed table: $1",
     meNewSectionExpected = "new section expected $1",
     meGeneralParseError = "general parse error",
     meInvalidDirective = "invalid directive: '$1'",
+    meInvalidField = "invalid field: $1",
     meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1",
     mwRedefinitionOfLabel = "redefinition of label '$1'",
     mwUnknownSubstitution = "unknown substitution '$1'",
+    mwAmbiguousLink = "ambiguous doc link $1",
+    mwBrokenLink = "broken link '$1'",
     mwUnsupportedLanguage = "language '$1' not supported",
     mwUnsupportedField = "field '$1' not supported",
-    mwRstStyle = "RST style: $1"
+    mwRstStyle = "RST style: $1",
+    mwUnusedImportdoc = "importdoc for '$1' is not used",
+    meSandboxedDirective = "disabled directive: '$1'",
 
   MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind,
                        arg: string) {.closure, gcsafe.} ## what to do in case of an error
   FindFileHandler* = proc (filename: string): string {.closure, gcsafe.}
+  FindRefFileHandler* =
+    proc (targetRelPath: string):
+         tuple[targetPath: string, linkRelPath: string] {.closure, gcsafe.}
+    ## returns where .html or .idx file should be found by its relative path;
+    ## `linkRelPath` is a prefix to be added before a link anchor from such file
 
 proc rstnodeToRefname*(n: PRstNode): string
 proc addNodes*(n: PRstNode): string
-proc getFieldValue*(n: PRstNode, fieldname: string): string
+proc getFieldValue*(n: PRstNode, fieldname: string): string {.gcsafe.}
 proc getArgument*(n: PRstNode): string
 
 # ----------------------------- scanner part --------------------------------
@@ -233,6 +136,10 @@ const
     ":geek:": "icon_e_geek",
     ":ugeek:": "icon_e_ugeek"
   }
+  SandboxDirAllowlist = [
+    "image", "code", "code-block", "admonition", "attention", "caution",
+    "container", "contents", "danger", "default-role", "error", "figure",
+    "hint", "important", "index", "note", "role", "tip", "title", "warning"]
 
 type
   TokType = enum
@@ -253,8 +160,8 @@ type
     buf*: cstring
     bufpos*: int
     line*, col*, baseIndent*: int
-    skipPounds*: bool
     adornmentLine*: bool
+    escapeNext*: bool
 
 proc getThing(L: var Lexer, tok: var Token, s: set[char]) =
   tok.kind = tkWord
@@ -292,10 +199,18 @@ proc getPunctAdornment(L: var Lexer, tok: var Token) =
   tok.col = L.col
   var pos = L.bufpos
   let c = L.buf[pos]
-  while true:
+  if not L.escapeNext and (c != '\\' or L.adornmentLine):
+    while true:
+      tok.symbol.add(L.buf[pos])
+      inc pos
+      if L.buf[pos] != c: break
+  elif L.escapeNext:
     tok.symbol.add(L.buf[pos])
     inc pos
-    if L.buf[pos] != c: break
+  else:  # not L.escapeNext and c == '\\' and not L.adornmentLine
+    tok.symbol.add '\\'
+    inc pos
+    L.escapeNext = true
   inc L.col, pos - L.bufpos
   L.bufpos = pos
   if tok.symbol == "\\": tok.kind = tkPunct
@@ -317,9 +232,6 @@ proc getIndentAux(L: var Lexer, start: int): int =
     else: inc pos
   elif L.buf[pos] == '\n':
     inc pos
-  if L.skipPounds:
-    if L.buf[pos] == '#': inc pos
-    if L.buf[pos] == '#': inc pos
   while true:
     case L.buf[pos]
     of ' ', '\v', '\f':
@@ -384,30 +296,19 @@ proc rawGetTok(L: var Lexer, tok: var Token) =
       inc L.col
   tok.col = max(tok.col - L.baseIndent, 0)
 
-proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int =
+proc getTokens(buffer: string, tokens: var TokenSeq) =
   var L: Lexer
   var length = tokens.len
   L.buf = cstring(buffer)
   L.line = 0                  # skip UTF-8 BOM
   if L.buf[0] == '\xEF' and L.buf[1] == '\xBB' and L.buf[2] == '\xBF':
     inc L.bufpos, 3
-  L.skipPounds = skipPounds
-  if skipPounds:
-    if L.buf[L.bufpos] == '#':
-      inc L.bufpos
-      inc result
-    if L.buf[L.bufpos] == '#':
-      inc L.bufpos
-      inc result
-    L.baseIndent = 0
-    while L.buf[L.bufpos] == ' ':
-      inc L.bufpos
-      inc L.baseIndent
-      inc result
   while true:
     inc length
     setLen(tokens, length)
+    let toEscape = L.escapeNext
     rawGetTok(L, tokens[length - 1])
+    if toEscape: L.escapeNext = false
     if tokens[length - 1].kind == tkEof: break
   if tokens[0].kind == tkWhite:
     # BUGFIX
@@ -421,14 +322,54 @@ type
     line: int            # the last line of this style occurrence
                          # (for error message)
     hasPeers: bool       # has headings on the same level of hierarchy?
+  LiteralBlockKind = enum  # RST-style literal blocks after `::`
+    lbNone,
+    lbIndentedLiteralBlock,
+    lbQuotedLiteralBlock
   LevelMap = seq[LevelInfo]   # Saves for each possible title adornment
                               # style its level in the current document.
+  SubstitutionKind = enum
+    rstSubstitution = "substitution",
+    hyperlinkAlias = "hyperlink alias",
+    implicitHyperlinkAlias = "implicitly-generated hyperlink alias"
   Substitution = object
+    kind*: SubstitutionKind
     key*: string
     value*: PRstNode
-  AnchorSubst = tuple
-    mainAnchor: string
-    aliases: seq[string]
+    info*: TLineInfo   # place where the substitution was defined
+  AnchorRule = enum
+    arInternalRst,  ## For automatically generated RST anchors (from
+                    ## headings, footnotes, inline internal targets):
+                    ## case-insensitive, 1-space-significant (by RST spec)
+    arExternalRst,  ## For external .nim doc comments or .rst/.md
+    arNim   ## For anchors generated by ``docgen.nim``: Nim-style case
+            ## sensitivity, etc. (see `proc normalizeNimName`_ for details)
+    arHyperlink,  ## For links with manually set anchors in
+                  ## form `text `_
+  RstAnchorKind = enum
+    manualDirectiveAnchor = "manual directive anchor",
+    manualInlineAnchor = "manual inline anchor",
+    footnoteAnchor = "footnote anchor",
+    headlineAnchor = "implicitly-generated headline anchor"
+  AnchorSubst = object
+    info: TLineInfo         # the file where the anchor was defined
+    priority: int
+    case kind: range[arInternalRst .. arNim]
+    of arInternalRst:
+      anchorType: RstAnchorKind
+      target: PRstNode
+    of arExternalRst:
+      anchorTypeExt: RstAnchorKind
+      refnameExt: string
+    of arNim:
+      module: FileIndex     # anchor's module (generally not the same as file)
+      tooltip: string       # displayed tooltip for Nim-generated anchors
+      langSym: LangSymbol
+      refname: string     # A reference name that will be inserted directly
+                          # into HTML/Latex.
+      external: bool
+  AnchorSubstTable = Table[string, seq[AnchorSubst]]
+                         # use `seq` to account for duplicate anchors
   FootnoteType = enum
     fnManualNumber,     # manually numbered footnote like [3]
     fnAutoNumber,       # auto-numbered footnote [#]
@@ -439,43 +380,71 @@ type
     kind: FootnoteType  # discriminator
     number: int         # valid for fnManualNumber (always) and fnAutoNumber,
                         # fnAutoNumberLabel after resolveSubs is called
-    autoNumIdx: int     # order of occurence: fnAutoNumber, fnAutoNumberLabel
-    autoSymIdx: int     # order of occurence: fnAutoSymbol
+    autoNumIdx: int     # order of occurrence: fnAutoNumber, fnAutoNumberLabel
+    autoSymIdx: int     # order of occurrence: fnAutoSymbol
     label: string       # valid for fnAutoNumberLabel
-
-  SharedState = object
-    options: RstParseOptions    # parsing options
+  RstFileTable* = object
+    filenameToIdx*: Table[string, FileIndex]
+    idxToFilename*: seq[string]
+  ImportdocInfo = object
+    used: bool             # was this import used?
+    fromInfo: TLineInfo    # place of `.. importdoc::` directive
+    idxPath: string        # full path to ``.idx`` file
+    linkRelPath: string    # prefix before target anchor
+    title: string          # document title obtained from ``.idx``
+  RstSharedState = object
+    options*: RstParseOptions   # parsing options
     hLevels: LevelMap           # hierarchy of heading styles
     hTitleCnt: int              # =0 if no title, =1 if only main title,
                                 # =2 if both title and subtitle are present
     hCurLevel: int              # current section level
+    currRole: string            # current interpreted text role
+    currRoleKind: RstNodeKind   # ... and its node kind
     subs: seq[Substitution]     # substitutions
-    refs: seq[Substitution]     # references
-    anchors: seq[AnchorSubst]   # internal target substitutions
-    lineFootnoteNum: seq[int]     # footnote line, auto numbers .. [#]
-    lineFootnoteNumRef: seq[int]  # footnote line, their reference [#]_
-    lineFootnoteSym: seq[int]     # footnote line, auto symbols .. [*]
-    lineFootnoteSymRef: seq[int]  # footnote line, their reference [*]_
+    refs*: seq[Substitution]    # references
+    anchors*: AnchorSubstTable
+                                # internal target substitutions
+    lineFootnoteNum: seq[TLineInfo]     # footnote line, auto numbers .. [#]
+    lineFootnoteNumRef: seq[TLineInfo]  # footnote line, their reference [#]_
+    currFootnoteNumRef: int             # ... their counter for `resolveSubs`
+    lineFootnoteSym: seq[TLineInfo]     # footnote line, auto symbols .. [*]
+    lineFootnoteSymRef: seq[TLineInfo]  # footnote line, their reference [*]_
+    currFootnoteSymRef: int             # ... their counter for `resolveSubs`
     footnotes: seq[FootnoteSubst] # correspondence b/w footnote label,
                                   # number, order of occurrence
     msgHandler: MsgHandler      # How to handle errors.
-    findFile: FindFileHandler   # How to find files.
+    findFile: FindFileHandler   # How to find files for include.
+    findRefFile: FindRefFileHandler
+                                # How to find files imported by importdoc.
+    filenames*: RstFileTable    # map file name <-> FileIndex (for storing
+                                # file names for warnings after 1st stage)
+    currFileIdx*: FileIndex     # current index in `filenames`
+    tocPart*: seq[PRstNode]     # all the headings of a document
+    hasToc*: bool
+    idxImports*: Table[string, ImportdocInfo]
+                                # map `importdoc`ed filename -> it's info
+    nimFileImported*: bool      # Was any ``.nim`` module `importdoc`ed ?
 
-  PSharedState = ref SharedState
+  PRstSharedState* = ref RstSharedState
+  ManualAnchor = object
+    alias: string     # a (short) name that can substitute the `anchor`
+    anchor: string    # anchor = id = refname
+    info: TLineInfo
   RstParser = object of RootObj
     idx*: int
     tok*: TokenSeq
-    s*: PSharedState
+    s*: PRstSharedState
     indentStack*: seq[int]
-    filename*: string
     line*, col*: int            ## initial line/column of whole text or
                                 ## documenation fragment that will be added
                                 ## in case of error/warning reporting to
                                 ## (relative) line/column of the token.
-    hasToc*: bool
-    curAnchor*: string          # variable to track latest anchor in s.anchors
+    curAnchors*: seq[ManualAnchor]
+                                ## seq to accumulate aliases for anchors:
+                                ## because RST can have >1 alias per 1 anchor
 
   EParseError* = object of ValueError
+  SectionParser = proc (p: var RstParser): PRstNode {.nimcall, gcsafe.}
 
 const
   LineRstInit* = 1  ## Initial line number for standalone RST text
@@ -509,36 +478,141 @@ proc defaultFindFile*(filename: string): string =
   if fileExists(filename): result = filename
   else: result = ""
 
-proc newSharedState(options: RstParseOptions,
-                    findFile: FindFileHandler,
-                    msgHandler: MsgHandler): PSharedState =
-  new(result)
-  result.subs = @[]
-  result.refs = @[]
-  result.options = options
-  result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler
-  result.findFile = if not isNil(findFile): findFile else: defaultFindFile
+proc defaultFindRefFile*(filename: string): (string, string) =
+  (filename, "")
+
+proc defaultRole(options: RstParseOptions): string =
+  if roNimFile in options: "nim" else: "literal"
+
+proc whichRoleAux(sym: string): RstNodeKind =
+  let r = sym.toLowerAscii
+  case r
+  of "idx": result = rnIdx
+  of "literal": result = rnInlineLiteral
+  of "strong": result = rnStrongEmphasis
+  of "emphasis": result = rnEmphasis
+  of "sub", "subscript": result = rnSub
+  of "sup", "superscript": result = rnSup
+  # literal and code are the same in our implementation
+  of "code": result = rnInlineLiteral
+  of "program", "option", "tok": result = rnCodeFragment
+  # c++ currently can be spelled only as cpp, c# only as csharp
+  elif getSourceLanguage(r) != langNone:
+    result = rnInlineCode
+  else:  # unknown role
+    result = rnUnknownRole
+
+proc len(filenames: RstFileTable): int = filenames.idxToFilename.len
+
+proc addFilename*(s: PRstSharedState, file1: string): FileIndex =
+  ## Returns index of filename, adding it if it has not been used before
+  let nextIdx = s.filenames.len.FileIndex
+  result = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx)
+  if result == nextIdx:
+    s.filenames.filenameToIdx[file1] = result
+    s.filenames.idxToFilename.add file1
+
+proc setCurrFilename*(s: PRstSharedState, file1: string) =
+  s.currFileIdx = addFilename(s, file1)
+
+proc getFilename(filenames: RstFileTable, fid: FileIndex): string =
+  doAssert(0 <= fid.int and fid.int < filenames.len,
+      "incorrect FileIndex $1 (range 0..$2)" % [
+        $fid.int, $(filenames.len - 1)])
+  result = filenames.idxToFilename[fid.int]
+
+proc getFilename(s: PRstSharedState, subst: AnchorSubst): string =
+  getFilename(s.filenames, subst.info.fileIndex)
+
+proc getModule(s: PRstSharedState, subst: AnchorSubst): string =
+  result = getFilename(s.filenames, subst.module)
+
+proc currFilename(s: PRstSharedState): string =
+  getFilename(s.filenames, s.currFileIdx)
+
+proc newRstSharedState*(options: RstParseOptions,
+                        filename: string,
+                        findFile: FindFileHandler,
+                        findRefFile: FindRefFileHandler,
+                        msgHandler: MsgHandler,
+                        hasToc: bool): PRstSharedState =
+  let r = defaultRole(options)
+  result = PRstSharedState(
+      currRole: r,
+      currRoleKind: whichRoleAux(r),
+      options: options,
+      msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler,
+      findFile: if not isNil(findFile): findFile else: defaultFindFile,
+      findRefFile:
+        if not isNil(findRefFile): findRefFile
+        else: defaultFindRefFile,
+      hasToc: hasToc
+  )
+  setCurrFilename(result, filename)
 
 proc curLine(p: RstParser): int = p.line + currentTok(p).line
 
 proc findRelativeFile(p: RstParser; filename: string): string =
-  result = p.filename.splitFile.dir / filename
+  result = p.s.currFilename.splitFile.dir / filename
   if not fileExists(result):
     result = p.s.findFile(filename)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) =
-  p.s.msgHandler(p.filename, curLine(p),
+  p.s.msgHandler(p.s.currFilename, curLine(p),
                              p.col + currentTok(p).col, msgKind, arg)
 
+proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) =
+  s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg)
+
+proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string;
+                line, col: int) =
+  s.msgHandler(s.currFilename, line, col, msgKind, arg)
+
+proc rstMessage(s: PRstSharedState, filename: string, msgKind: MsgKind,
+                arg: string) =
+  s.msgHandler(filename, LineRstInit, ColRstInit, msgKind, arg)
+
+proc rstMessage*(filenames: RstFileTable, f: MsgHandler,
+                 info: TLineInfo, msgKind: MsgKind, arg: string) =
+  ## Print warnings using `info`, i.e. in 2nd-pass warnings for
+  ## footnotes/substitutions/references or from ``rstgen.nim``.
+  let file = getFilename(filenames, info.fileIndex)
+  f(file, info.line.int, info.col.int, msgKind, arg)
+
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) =
-  p.s.msgHandler(p.filename, p.line + line,
+  p.s.msgHandler(p.s.currFilename, p.line + line,
                              p.col + col, msgKind, arg)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind) =
-  p.s.msgHandler(p.filename, curLine(p),
+  p.s.msgHandler(p.s.currFilename, curLine(p),
                              p.col + currentTok(p).col, msgKind,
                              currentTok(p).symbol)
 
+# Functions `isPureRst` & `stopOrWarn` address differences between
+# Markdown and RST:
+# * Markdown always tries to continue working. If it is really impossible
+#   to parse a markup element, its proc just returns `nil` and parsing
+#   continues for it as for normal text paragraph.
+#   The downside is that real mistakes/typos are often silently ignored.
+#   The same applies to legacy `RstMarkdown` mode for nimforum.
+# * RST really signals errors. The downside is that it's more intrusive -
+#   the user must escape special syntax with \ explicitly.
+#
+# TODO: we need to apply this strategy to all markup elements eventually.
+
+func isPureRst(p: RstParser): bool = roSupportMarkdown notin p.s.options
+func isRst(p: RstParser): bool = roPreferMarkdown notin p.s.options
+func isMd(p: RstParser): bool = roPreferMarkdown in p.s.options
+func isMd(s: PRstSharedState): bool = roPreferMarkdown in s.options
+
+proc stopOrWarn(p: RstParser, errorType: MsgKind, arg: string) =
+  let realMsgKind = if isPureRst(p): errorType else: mwRstStyle
+  rstMessage(p, realMsgKind, arg)
+
+proc stopOrWarn(p: RstParser, errorType: MsgKind, arg: string, line, col: int) =
+  let realMsgKind = if isPureRst(p): errorType else: mwRstStyle
+  rstMessage(p, realMsgKind, arg, line, col)
+
 proc currInd(p: RstParser): int =
   result = p.indentStack[high(p.indentStack)]
 
@@ -548,17 +622,78 @@ proc pushInd(p: var RstParser, ind: int) =
 proc popInd(p: var RstParser) =
   if p.indentStack.len > 1: setLen(p.indentStack, p.indentStack.len - 1)
 
-proc initParser(p: var RstParser, sharedState: PSharedState) =
+# Working with indentation in rst.nim
+# -----------------------------------
+#
+# Every line break has an associated tkIndent.
+# The tokenizer writes back the first column of next non-blank line
+# in all preceeding tkIndent tokens to the `ival` field of tkIndent.
+#
+# RST document is separated into body elements (B.E.), every of which
+# has a dedicated handler proc (or block of logic when B.E. is a block quote)
+# that should follow the next rule:
+#   Every B.E. handler proc should finish at tkIndent (newline)
+#   after its B.E. finishes.
+#   Then its callers (which is `parseSection` or another B.E. handler)
+#   check for tkIndent ival (without necessity to advance `p.idx`)
+#   and decide themselves whether they continue processing or also stop.
+#
+# An example::
+#
+#   L    RST text fragment                  indentation
+#     +--------------------+
+#   1 |                    | <- (empty line at the start of file) no tokens
+#   2 |First paragraph.    | <- tkIndent has ival=0, and next tkWord has col=0
+#   3 |                    | <- tkIndent has ival=0
+#   4 |* bullet item and   | <- tkIndent has ival=0, and next tkPunct has col=0
+#   5 |  its continuation  | <- tkIndent has ival=2, and next tkWord has col=2
+#   6 |                    | <- tkIndent has ival=4
+#   7 |    Block quote     | <- tkIndent has ival=4, and next tkWord has col=4
+#   8 |                    | <- tkIndent has ival=0
+#   9 |                    | <- tkIndent has ival=0
+#   10|Final paragraph     | <- tkIndent has ival=0, and tkWord has col=0
+#     +--------------------+
+#    C:01234
+#
+# Here parser starts with initial `indentStack=[0]` and then calls the
+# 1st `parseSection`:
+#
+#   - `parseSection` calls `parseParagraph` and "First paragraph" is parsed
+#   - bullet list handler is started at reaching ``*`` (L4 C0), it
+#     starts bullet item logic (L4 C2), which calls `pushInd(p, ind=2)`,
+#     then calls `parseSection` (2nd call, nested) which parses
+#     paragraph "bullet list and its continuation" and then starts
+#     a block quote logic (L7 C4).
+#     The block quote logic calls calls `pushInd(p, ind=4)` and
+#     calls `parseSection` again, so a (simplified) sequence of calls now is::
+#
+#       parseSection -> parseBulletList ->
+#         parseSection (+block quote logic) -> parseSection
+#
+#     3rd `parseSection` finishes, block quote logic calls `popInd(p)`,
+#     it returns to bullet item logic, which sees that next tkIndent has
+#     ival=0 and stops there since the required indentation for a bullet item
+#     is 2 and 0<2; the bullet item logic calls `popInd(p)`.
+#     Then bullet list handler checks that next tkWord (L10 C0) has the
+#     right indentation but does not have ``*`` so stops at tkIndent (L10).
+#   - 1st `parseSection` invocation calls `parseParagraph` and the
+#     "Final paragraph" is parsed.
+#
+# If a B.E. handler has advanced `p.idx` past tkIndent to check
+# whether it should continue its processing or not, and decided not to,
+# then this B.E. handler should step back (e.g. do `dec p.idx`).
+
+proc initParser(p: var RstParser, sharedState: PRstSharedState) =
   p.indentStack = @[0]
   p.tok = @[]
   p.idx = 0
-  p.filename = ""
-  p.hasToc = false
   p.col = ColRstInit
   p.line = LineRstInit
   p.s = sharedState
 
 proc addNodesAux(n: PRstNode, result: var string) =
+  if n == nil:
+    return
   if n.kind == rnLeaf:
     result.add(n.text)
   else:
@@ -567,6 +702,11 @@ proc addNodesAux(n: PRstNode, result: var string) =
 proc addNodes(n: PRstNode): string =
   n.addNodesAux(result)
 
+proc linkName(n: PRstNode): string =
+  ## Returns a normalized reference name, see:
+  ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
+  n.addNodes.toLowerAscii
+
 proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) =
   template special(s) =
     if b:
@@ -622,26 +762,37 @@ proc rstnodeToRefname(n: PRstNode): string =
   var b = false
   rstnodeToRefnameAux(n, result, b)
 
-proc findSub(p: var RstParser, n: PRstNode): int =
+proc findSub(s: PRstSharedState, n: PRstNode): int =
   var key = addNodes(n)
   # the spec says: if no exact match, try one without case distinction:
-  for i in countup(0, high(p.s.subs)):
-    if key == p.s.subs[i].key:
+  for i in countup(0, high(s.subs)):
+    if key == s.subs[i].key:
       return i
-  for i in countup(0, high(p.s.subs)):
-    if cmpIgnoreStyle(key, p.s.subs[i].key) == 0:
+  for i in countup(0, high(s.subs)):
+    if cmpIgnoreStyle(key, s.subs[i].key) == 0:
       return i
   result = -1
 
+proc lineInfo(p: RstParser, iTok: int): TLineInfo =
+  result.col = int16(p.col + p.tok[iTok].col)
+  result.line = uint16(p.line + p.tok[iTok].line)
+  result.fileIndex = p.s.currFileIdx
+
+proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx)
+# TODO: we need this simplification because we don't preserve exact starting
+# token of currently parsed element:
+proc prevLineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx-1)
+
 proc setSub(p: var RstParser, key: string, value: PRstNode) =
   var length = p.s.subs.len
   for i in 0 ..< length:
     if key == p.s.subs[i].key:
       p.s.subs[i].value = value
       return
-  p.s.subs.add(Substitution(key: key, value: value))
+  p.s.subs.add(Substitution(key: key, value: value, info: prevLineInfo(p)))
 
-proc setRef(p: var RstParser, key: string, value: PRstNode) =
+proc setRef(p: var RstParser, key: string, value: PRstNode,
+            refType: SubstitutionKind) =
   var length = p.s.refs.len
   for i in 0 ..< length:
     if key == p.s.refs[i].key:
@@ -649,37 +800,127 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) =
         rstMessage(p, mwRedefinitionOfLabel, key)
       p.s.refs[i].value = value
       return
-  p.s.refs.add(Substitution(key: key, value: value))
+  p.s.refs.add(Substitution(kind: refType, key: key, value: value,
+                            info: prevLineInfo(p)))
 
-proc findRef(p: var RstParser, key: string): PRstNode =
-  for i in countup(0, high(p.s.refs)):
-    if key == p.s.refs[i].key:
-      return p.s.refs[i].value
+proc findRef(s: PRstSharedState, key: string): seq[Substitution] =
+  for i in countup(0, high(s.refs)):
+    if key == s.refs[i].key:
+      result.add s.refs[i]
 
-proc addAnchor(p: var RstParser, refn: string, reset: bool) =
-  ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor``
-  if p.curAnchor == "":
-    p.s.anchors.add (refn, @[refn])
-  else:
-    p.s.anchors[^1].mainAnchor = refn
-    p.s.anchors[^1].aliases.add refn
-  if reset:
-    p.curAnchor = ""
-  else:
-    p.curAnchor = refn
+# Ambiguity in links: we don't follow procedure of removing implicit targets
+# defined in https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#implicit-hyperlink-targets
+# Instead we just give explicit links a higher priority than to implicit ones
+# and report ambiguities as warnings. Hopefully it is easy to remove
+# ambiguities manually. Nim auto-generated links from ``docgen.nim``
+# have lowest priority: 1 (for procs) and below for other symbol types.
 
-proc findMainAnchor(p: RstParser, refn: string): string =
-  for subst in p.s.anchors:
-    if subst.mainAnchor == refn:  # no need to rename
-      result = subst.mainAnchor
-      break
-    var toLeave = false
-    for anchor in subst.aliases:
-      if anchor == refn:  # this anchor will be named as mainAnchor
-        result = subst.mainAnchor
-        toLeave = true
-    if toLeave:
-      break
+proc refPriority(k: SubstitutionKind): int =
+  case k
+  of rstSubstitution: result = 8
+  of hyperlinkAlias: result = 7
+  of implicitHyperlinkAlias: result = 2
+
+proc internalRefPriority(k: RstAnchorKind): int =
+  case k
+  of manualDirectiveAnchor: result = 6
+  of manualInlineAnchor: result = 5
+  of footnoteAnchor: result = 4
+  of headlineAnchor: result = 3
+
+proc `$`(subst: AnchorSubst): string =  # for debug
+  let s =
+    case subst.kind
+    of arInternalRst: "type=" & $subst.anchorType
+    of arExternalRst: "type=" & $subst.anchorTypeExt
+    of arNim: "langsym=" & $subst.langSym
+  result = "(kind=$1, priority=$2, $3)" % [$subst.kind, $subst.priority, s]
+
+proc addAnchorRst(p: var RstParser, name: string, target: PRstNode,
+                  anchorType: RstAnchorKind) =
+  ## Associates node `target` (which has field `anchor`) with an
+  ## alias `name` and updates the corresponding aliases in `p.curAnchors`.
+  let prio = internalRefPriority(anchorType)
+  for a in p.curAnchors:
+    p.s.anchors.mgetOrPut(a.alias, newSeq[AnchorSubst]()).add(
+        AnchorSubst(kind: arInternalRst, target: target, priority: prio,
+                    info: a.info, anchorType: manualDirectiveAnchor))
+  if name != "":
+    p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add(
+        AnchorSubst(kind: arInternalRst, target: target, priority: prio,
+                    info: prevLineInfo(p), anchorType: anchorType))
+  p.curAnchors.setLen 0
+
+proc addAnchorExtRst(s: var PRstSharedState, key: string, refn: string,
+                  anchorType: RstAnchorKind, info: TLineInfo) =
+  let name = key.toLowerAscii
+  let prio = internalRefPriority(anchorType)
+  s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add(
+      AnchorSubst(kind: arExternalRst, refnameExt: refn, priority: prio,
+                  info: info,
+                  anchorTypeExt: anchorType))
+
+proc addAnchorNim*(s: var PRstSharedState, external: bool, refn: string, tooltip: string,
+                   langSym: LangSymbol, priority: int,
+                   info: TLineInfo, module: FileIndex) =
+  ## Adds an anchor `refn`, which follows
+  ## the rule `arNim` (i.e. a symbol in ``*.nim`` file)
+  s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add(
+      AnchorSubst(kind: arNim, external: external, refname: refn, langSym: langSym,
+                  tooltip: tooltip, priority: priority,
+                  info: info))
+
+proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode,
+                       info: TLineInfo):
+                      seq[AnchorSubst] =
+  var langSym: LangSymbol
+  try:
+    langSym = toLangSymbol(signature)
+  except ValueError:  # parsing failed, not a Nim symbol
+    return
+  let substitutions = s.anchors.getOrDefault(langSym.name,
+                                             newSeq[AnchorSubst]())
+  if substitutions.len == 0:
+    return
+  # logic to select only groups instead of concrete symbols
+  # with overloads, note that the same symbol can be defined
+  # in multiple modules and `importdoc`ed:
+  type GroupKey = tuple[symKind: string, origModule: string]
+  # map (symKind, file) (like "proc", "os.nim") -> found symbols/groups:
+  var found: Table[GroupKey, seq[AnchorSubst]]
+  for subst in substitutions:
+    if subst.kind == arNim:
+      if match(subst.langSym, langSym):
+        let key: GroupKey = (subst.langSym.symKind, getModule(s, subst))
+        found.mgetOrPut(key, newSeq[AnchorSubst]()).add subst
+  for key, sList in found:
+    if sList.len == 1:
+      result.add sList[0]
+    else:  # > 1, there are overloads, potential ambiguity in this `symKind`
+      if langSym.parametersProvided:
+        # there are non-group signatures, select only them
+        for s in sList:
+          if not s.langSym.isGroup:
+            result.add s
+      else:  # when there are many overloads a link like foo_ points to all
+             # of them, so selecting the group
+        var foundGroup = false
+        for s in sList:
+          if s.langSym.isGroup:
+            result.add s
+            foundGroup = true
+            break
+        doAssert(foundGroup,
+                 "docgen has not generated the group for $1 (file $2)" % [
+                 langSym.name, getModule(s, sList[0]) ])
+
+proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo):
+                      seq[AnchorSubst] =
+  let name = linkText.toLowerAscii
+  let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]())
+  for s in substitutions:
+    if s.kind in {arInternalRst, arExternalRst}:
+      result.add s
 
 proc addFootnoteNumManual(p: var RstParser, num: int) =
   ## add manually-numbered footnote
@@ -693,7 +934,7 @@ proc addFootnoteNumAuto(p: var RstParser, label: string) =
   ## add auto-numbered footnote.
   ## Empty label [#] means it'll be resolved by the occurrence.
   if label == "":  # simple auto-numbered [#]
-    p.s.lineFootnoteNum.add curLine(p)
+    p.s.lineFootnoteNum.add lineInfo(p)
     p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label))
   else:           # auto-numbered with label [#label]
     for fnote in p.s.footnotes:
@@ -703,31 +944,39 @@ proc addFootnoteNumAuto(p: var RstParser, label: string) =
     p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label))
 
 proc addFootnoteSymAuto(p: var RstParser) =
-  p.s.lineFootnoteSym.add curLine(p)
+  p.s.lineFootnoteSym.add lineInfo(p)
   p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, ""))
 
-proc orderFootnotes(p: var RstParser) =
+proc orderFootnotes(s: PRstSharedState) =
   ## numerate auto-numbered footnotes taking into account that all
   ## manually numbered ones always have preference.
-  ## Save the result back to p.s.footnotes.
+  ## Save the result back to `s.footnotes`.
 
   # Report an error if found any mismatch in number of automatic footnotes
-  proc listFootnotes(lines: seq[int]): string =
+  proc listFootnotes(locations: seq[TLineInfo]): string =
+    var lines: seq[string]
+    for info in locations:
+      if s.filenames.len > 1:
+        let file = getFilename(s.filenames, info.fileIndex)
+        lines.add file & ":"
+      else:  # no need to add file name here if there is only 1
+        lines.add ""
+      lines[^1].add $info.line
     result.add $lines.len & " (lines " & join(lines, ", ") & ")"
-  if p.s.lineFootnoteNum.len != p.s.lineFootnoteNumRef.len:
-    rstMessage(p, meFootnoteMismatch,
-      "$1 != $2" % [listFootnotes(p.s.lineFootnoteNum),
-                    listFootnotes(p.s.lineFootnoteNumRef)] &
+  if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len:
+    rstMessage(s, meFootnoteMismatch,
+      "$1 != $2" % [listFootnotes(s.lineFootnoteNum),
+                    listFootnotes(s.lineFootnoteNumRef)] &
         " for auto-numbered footnotes")
-  if p.s.lineFootnoteSym.len != p.s.lineFootnoteSymRef.len:
-    rstMessage(p, meFootnoteMismatch,
-      "$1 != $2" % [listFootnotes(p.s.lineFootnoteSym),
-                    listFootnotes(p.s.lineFootnoteSymRef)] &
+  if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len:
+    rstMessage(s, meFootnoteMismatch,
+      "$1 != $2" % [listFootnotes(s.lineFootnoteSym),
+                    listFootnotes(s.lineFootnoteSymRef)] &
         " for auto-symbol footnotes")
 
   var result: seq[FootnoteSubst]
   var manuallyN, autoN, autoSymbol: seq[FootnoteSubst]
-  for fs in p.s.footnotes:
+  for fs in s.footnotes:
     if fs.kind == fnManualNumber: manuallyN.add fs
     elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs
     else: autoSymbol.add fs
@@ -774,41 +1023,73 @@ proc orderFootnotes(p: var RstParser) =
     let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols)
     result.add((fs.kind, -1, -1, fs.autoSymIdx, label))
 
-  p.s.footnotes = result
+  s.footnotes = result
 
-proc getFootnoteNum(p: var RstParser, label: string): int =
+proc getFootnoteNum(s: PRstSharedState, label: string): int =
   ## get number from label. Must be called after `orderFootnotes`.
   result = -1
-  for fnote in p.s.footnotes:
+  for fnote in s.footnotes:
     if fnote.label == label:
       return fnote.number
 
-proc getFootnoteNum(p: var RstParser, order: int): int =
+proc getFootnoteNum(s: PRstSharedState, order: int): int =
   ## get number from occurrence. Must be called after `orderFootnotes`.
   result = -1
-  for fnote in p.s.footnotes:
+  for fnote in s.footnotes:
     if fnote.autoNumIdx == order:
       return fnote.number
 
-proc getAutoSymbol(p: var RstParser, order: int): string =
+proc getAutoSymbol(s: PRstSharedState, order: int): string =
   ## get symbol from occurrence of auto-symbol footnote.
   result = "???"
-  for fnote in p.s.footnotes:
+  for fnote in s.footnotes:
     if fnote.autoSymIdx == order:
       return fnote.label
 
 proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode =
   ## create node and consume the current anchor
   result = newRstNode(kind)
-  if p.curAnchor != "":
-    result.anchor = p.curAnchor
-    p.curAnchor = ""
+  if p.curAnchors.len > 0:
+    result.anchor = p.curAnchors[0].anchor
+    addAnchorRst(p, "", result, manualDirectiveAnchor)
 
 template newLeaf(s: string): PRstNode = newRstLeaf(s)
 
 proc newLeaf(p: var RstParser): PRstNode =
   result = newLeaf(currentTok(p).symbol)
 
+proc validRefnamePunct(x: string): bool =
+  ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
+  x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
+
+func getRefnameIdx(p: RstParser, startIdx: int): int =
+  ## Gets last token index of a refname ("word" in RST terminology):
+  ##
+  ##   reference names are single words consisting of alphanumerics plus
+  ##   isolated (no two adjacent) internal hyphens, underscores, periods,
+  ##   colons and plus signs; no whitespace or other characters are allowed.
+  ##
+  ## Refnames are used for:
+  ## - reference names
+  ## - role names
+  ## - directive names
+  ## - footnote labels
+  ##
+  # TODO: use this func in all other relevant places
+  var j = startIdx
+  if p.tok[j].kind == tkWord:
+    inc j
+    while p.tok[j].kind == tkPunct and validRefnamePunct(p.tok[j].symbol) and
+        p.tok[j+1].kind == tkWord:
+      inc j, 2
+  result = j - 1
+
+func getRefname(p: RstParser, startIdx: int): (string, int) =
+  let lastIdx = getRefnameIdx(p, startIdx)
+  result[1] = lastIdx
+  for j in startIdx..lastIdx:
+    result[0].add p.tok[j].symbol
+
 proc getReferenceName(p: var RstParser, endStr: string): PRstNode =
   var res = newRstNode(rnInner)
   while true:
@@ -837,54 +1118,82 @@ proc expect(p: var RstParser, tok: string) =
   if currentTok(p).symbol == tok: inc p.idx
   else: rstMessage(p, meExpected, tok)
 
-proc isInlineMarkupEnd(p: RstParser, markup: string): bool =
+proc inlineMarkdownEnd(p: RstParser): bool =
+  result = prevTok(p).kind notin {tkIndent, tkWhite}
+  ## (For a special case of ` we don't allow spaces surrounding it
+  ## unlike original Markdown because this behavior confusing/useless)
+
+proc inlineRstEnd(p: RstParser): bool =
   # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules
-  result = currentTok(p).symbol == markup
-  if not result: return
   # Rule 2:
   result = prevTok(p).kind notin {tkIndent, tkWhite}
   if not result: return
   # Rule 7:
   result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or
-      markup in ["``", "`"] and nextTok(p).kind in {tkIndent, tkWhite, tkWord, tkEof} or
       nextTok(p).symbol[0] in
       {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'}
-  if not result: return
-  # Rule 4:
-  if p.idx > 0:
-    if markup != "``" and prevTok(p).symbol == "\\":
-      result = false
 
-proc isInlineMarkupStart(p: RstParser, markup: string): bool =
-  # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules
-  var d: char
-  if markup != "_`":
+proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool =
+  if exact:
     result = currentTok(p).symbol == markup
-  else:  # _` is a 2 token case
-    result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`"
+  else:
+    result = currentTok(p).symbol.endsWith markup
+    if (not result) and markup == "``":
+      # check that escaping may have splitted `` to 2 tokens ` and `
+      result = currentTok(p).symbol == "`" and prevTok(p).symbol == "`"
   if not result: return
-  # Rule 6:
+  # surroundings check
+  if markup in ["_", "__"]:
+    result = inlineRstEnd(p)
+  else:
+    if roPreferMarkdown in p.s.options: result = inlineMarkdownEnd(p)
+    else: result = inlineRstEnd(p)
+
+proc rstRuleSurround(p: RstParser): bool =
+  result = true
+  # Rules 4 & 5:
+  if p.idx > 0:
+    var d: char
+    var c = prevTok(p).symbol[0]
+    case c
+    of '\'', '\"': d = c
+    of '(': d = ')'
+    of '[': d = ']'
+    of '{': d = '}'
+    of '<': d = '>'
+    else: d = '\0'
+    if d != '\0': result = nextTok(p).symbol[0] != d
+
+proc inlineMarkdownStart(p: RstParser): bool =
+  result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof}
+  if not result: return
+  # this rst rule is really nice, let us use it in Markdown mode too.
+  result = rstRuleSurround(p)
+
+proc inlineRstStart(p: RstParser): bool =
+  ## rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules
+  # Rule 6
   result = p.idx == 0 or prevTok(p).kind in {tkIndent, tkWhite} or
-      (markup in ["``", "`"] and prevTok(p).kind in {tkIndent, tkWhite, tkWord}) or
       prevTok(p).symbol[0] in {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'}
   if not result: return
   # Rule 1:
   result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof}
   if not result: return
-  # Rules 4 & 5:
-  if p.idx > 0:
-    if prevTok(p).symbol == "\\":
-      result = false
-    else:
-      var c = prevTok(p).symbol[0]
-      case c
-      of '\'', '\"': d = c
-      of '(': d = ')'
-      of '[': d = ']'
-      of '{': d = '}'
-      of '<': d = '>'
-      else: d = '\0'
-      if d != '\0': result = nextTok(p).symbol[0] != d
+  result = rstRuleSurround(p)
+
+proc isInlineMarkupStart(p: RstParser, markup: string): bool =
+  if markup != "_`":
+    result = currentTok(p).symbol == markup
+  else:  # _` is a 2 token case
+    result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`"
+  if not result: return
+  # surroundings check
+  if markup in ["_", "__", "[", "|"]:
+    # Note: we require space/punctuation even before [markdown link](...)
+    result = inlineRstStart(p)
+  else:
+    if roPreferMarkdown in p.s.options: result = inlineMarkdownStart(p)
+    else: result = inlineRstStart(p)
 
 proc match(p: RstParser, start: int, expr: string): bool =
   # regular expressions are:
@@ -905,7 +1214,10 @@ proc match(p: RstParser, start: int, expr: string): bool =
   var last = expr.len - 1
   while i <= last:
     case expr[i]
-    of 'w': result = p.tok[j].kind == tkWord
+    of 'w':
+      let lastIdx = getRefnameIdx(p, j)
+      result = lastIdx >= j
+      if result: j = lastIdx
     of ' ': result = p.tok[j].kind == tkWhite
     of 'i': result = p.tok[j].kind == tkIndent
     of 'I': result = p.tok[j].kind in {tkIndent, tkEof}
@@ -939,7 +1251,20 @@ proc match(p: RstParser, start: int, expr: string): bool =
     inc i
   result = true
 
-proc fixupEmbeddedRef(n, a, b: PRstNode) =
+proc safeProtocol*(linkStr: var string): string =
+  # Returns link's protocol and, if it's not safe, clears `linkStr`
+  result = ""
+  if scanf(linkStr, "$w:", result):
+    # if it has a protocol at all, ensure that it's not 'javascript:' or worse:
+    if cmpIgnoreCase(result, "http") == 0 or
+        cmpIgnoreCase(result, "https") == 0 or
+        cmpIgnoreCase(result, "ftp") == 0:
+      discard "it's fine"
+    else:
+      linkStr = ""
+
+proc fixupEmbeddedRef(p: var RstParser, n, a, b: PRstNode): bool =
+  # Returns `true` if the link belongs to an allowed protocol
   var sep = - 1
   for i in countdown(n.len - 2, 0):
     if n.sons[i].text == "<":
@@ -947,54 +1272,89 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
       break
   var incr = if sep > 0 and n.sons[sep - 1].text[0] == ' ': 2 else: 1
   for i in countup(0, sep - incr): a.add(n.sons[i])
-  for i in countup(sep + 1, n.len - 2): b.add(n.sons[i])
+  var linkStr = ""
+  for i in countup(sep + 1, n.len - 2): linkStr.add(n.sons[i].addNodes)
+  if linkStr != "":
+    let protocol = safeProtocol(linkStr)
+    result = linkStr != ""
+    if not result:
+      rstMessage(p, mwBrokenLink, protocol,
+                 p.tok[p.idx-3].line, p.tok[p.idx-3].col)
+  b.add newLeaf(linkStr)
+
+proc whichRole(p: RstParser, sym: string): RstNodeKind =
+  result = whichRoleAux(sym)
+  if result == rnUnknownRole:
+    rstMessage(p, mwUnsupportedLanguage, sym)
+
+proc toInlineCode(n: PRstNode, language: string): PRstNode =
+  ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
+  result = newRstNode(rnInlineCode, info=n.info)
+  let args = newRstNode(rnDirArg)
+  var lang = language
+  if language == "cpp": lang = "c++"
+  elif language == "csharp": lang = "c#"
+  args.add newLeaf(lang)
+  result.add args
+  result.add PRstNode(nil)
+  var lb = newRstNode(rnLiteralBlock)
+  var s: string
+  for i in n.sons:
+    assert i.kind == rnLeaf
+    s.add i.text
+  lb.add newLeaf(s)
+  result.add lb
+
+proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode =
+  let newN = newRstNode(rnInner, n.sons)
+  let newSons = @[newN, newLeaf(roleName)]
+  result = newRstNode(kind, newSons)
 
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
+  ## Finalizes node `n` that was tentatively determined as interpreted text.
   var newKind = n.kind
   var newSons = n.sons
-  if isInlineMarkupEnd(p, "_") or isInlineMarkupEnd(p, "__"):
+
+  proc finalizeInterpreted(node: PRstNode, newKind: RstNodeKind,
+                           newSons: seq[PRstNode], roleName: string):
+                          PRstNode {.nimcall.} =
+    # fixes interpreted text (`x` or `y`:role:) to proper internal AST format
+    if newKind in {rnUnknownRole, rnCodeFragment}:
+      result = node.toOtherRole(newKind, roleName)
+    elif newKind == rnInlineCode:
+      result = node.toInlineCode(language=roleName)
+    else:
+      result = newRstNode(newKind, newSons)
+
+  if isInlineMarkupEnd(p, "_", exact=true) or
+      isInlineMarkupEnd(p, "__", exact=true):
     inc p.idx
     if p.tok[p.idx-2].symbol == "`" and p.tok[p.idx-3].symbol == ">":
       var a = newRstNode(rnInner)
       var b = newRstNode(rnInner)
-      fixupEmbeddedRef(n, a, b)
-      if a.len == 0:
-        newKind = rnStandaloneHyperlink
-        newSons = @[b]
-      else:
-        newKind = rnHyperlink
-        newSons = @[a, b]
-        setRef(p, rstnodeToRefname(a), b)
-    elif n.kind == rnInterpretedText:
-      newKind = rnRef
-    else:
-      newKind = rnRef
-      newSons = @[n]
-    result = newRstNode(newKind, newSons)
+      if fixupEmbeddedRef(p, n, a, b):
+        if a.len == 0:  # e.g. ``_
+          newKind = rnStandaloneHyperlink
+          newSons = @[b]
+        else:  # e.g. `link title `_
+          newKind = rnHyperlink
+          newSons = @[a, b]
+          setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias)
+      else:  # include as plain text, not a link
+        newKind = rnInner
+        newSons = n.sons
+      result = newRstNode(newKind, newSons)
+    else:  # some link that will be resolved in `resolveSubs`
+      newKind = rnRstRef
+      result = newRstNode(newKind, sons=newSons, info=n.info)
   elif match(p, p.idx, ":w:"):
     # a role:
-    if nextTok(p).symbol == "idx":
-      newKind = rnIdx
-    elif nextTok(p).symbol == "literal":
-      newKind = rnInlineLiteral
-    elif nextTok(p).symbol == "strong":
-      newKind = rnStrongEmphasis
-    elif nextTok(p).symbol == "emphasis":
-      newKind = rnEmphasis
-    elif nextTok(p).symbol == "sub" or
-        nextTok(p).symbol == "subscript":
-      newKind = rnSub
-    elif nextTok(p).symbol == "sup" or
-        nextTok(p).symbol == "supscript":
-      newKind = rnSup
-    else:
-      newKind = rnGeneralRole
-      let newN = newRstNode(rnInner, n.sons)
-      newSons = @[newN, newLeaf(nextTok(p).symbol)]
-    inc p.idx, 3
-    result = newRstNode(newKind, newSons)
-  else:  # no change
-    result = n
+    let (roleName, lastIdx) = getRefname(p, p.idx+1)
+    newKind = whichRole(p, roleName)
+    result = n.finalizeInterpreted(newKind, newSons, roleName)
+    p.idx = lastIdx + 2
+  else:
+    result = n.finalizeInterpreted(p.s.currRoleKind, newSons, p.s.currRole)
 
 proc matchVerbatim(p: RstParser, start: int, expr: string): int =
   result = start
@@ -1015,49 +1375,78 @@ proc parseSmiley(p: var RstParser): PRstNode =
       result.text = val
       return
 
-proc validRefnamePunct(x: string): bool =
-  ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
-  x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
-
 proc isUrl(p: RstParser, i: int): bool =
   result = p.tok[i+1].symbol == ":" and p.tok[i+2].symbol == "//" and
     p.tok[i+3].kind == tkWord and
     p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"]
 
+proc checkParen(token: Token, parensStack: var seq[char]): bool {.inline.} =
+  ## Returns `true` iff `token` is a closing parenthesis for some
+  ## previous opening parenthesis saved in `parensStack`.
+  ## This is according Markdown balanced parentheses rule
+  ## (https://spec.commonmark.org/0.29/#link-destination)
+  ## to allow links like
+  ## https://en.wikipedia.org/wiki/APL_(programming_language),
+  ## we use it for RST also.
+  result = false
+  if token.kind == tkPunct:
+    let c = token.symbol[0]
+    if c in {'(', '[', '{'}:  # push
+      parensStack.add c
+    elif c in {')', ']', '}'}:  # try pop
+      # a case like ([) inside a link is allowed and [ is also `pop`ed:
+      for i in countdown(parensStack.len - 1, 0):
+        if (parensStack[i] == '(' and c == ')' or
+            parensStack[i] == '[' and c == ']' or
+            parensStack[i] == '{' and c == '}'):
+          parensStack.setLen i
+          result = true
+          break
+
+proc parseUrl(p: var RstParser): PRstNode =
+  ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#standalone-hyperlinks
+  result = newRstNode(rnStandaloneHyperlink)
+  var lastIdx = p.idx
+  var closedParenIdx = p.idx - 1  # for balanced parens rule
+  var parensStack: seq[char]
+  while p.tok[lastIdx].kind in {tkWord, tkPunct, tkOther}:
+    let isClosing = checkParen(p.tok[lastIdx], parensStack)
+    if isClosing:
+      closedParenIdx = lastIdx
+    inc lastIdx
+  dec lastIdx
+  # standalone URL can not end with punctuation in RST
+  while lastIdx > closedParenIdx and p.tok[lastIdx].kind == tkPunct and
+      p.tok[lastIdx].symbol != "/":
+    dec lastIdx
+  var s = ""
+  for i in p.idx .. lastIdx: s.add p.tok[i].symbol
+  result.add s
+  p.idx = lastIdx + 1
+
 proc parseWordOrRef(p: var RstParser, father: PRstNode) =
   ## Parses a normal word or may be a reference or URL.
   if nextTok(p).kind != tkPunct:  # <- main path, a normal word
     father.add newLeaf(p)
     inc p.idx
   elif isUrl(p, p.idx):           # URL http://something
-    var n = newRstNode(rnStandaloneHyperlink)
-    while true:
-      case currentTok(p).kind
-      of tkWord, tkAdornment, tkOther: discard
-      of tkPunct:
-        if nextTok(p).kind notin {tkWord, tkAdornment, tkOther, tkPunct}:
-          break
-      else: break
-      n.add(newLeaf(p))
-      inc p.idx
-    father.add(n)
+    father.add parseUrl(p)
   else:
     # check for reference (probably, long one like some.ref.with.dots_ )
     var saveIdx = p.idx
-    var isRef = false
+    var reference: PRstNode = nil
     inc p.idx
     while currentTok(p).kind in {tkWord, tkPunct}:
       if currentTok(p).kind == tkPunct:
-        if isInlineMarkupEnd(p, "_"):
-          isRef = true
+        if isInlineMarkupEnd(p, "_", exact=true):
+          reference = newRstNode(rnRstRef, info=lineInfo(p, saveIdx))
           break
         if not validRefnamePunct(currentTok(p).symbol):
           break
       inc p.idx
-    if isRef:
-      let r = newRstNode(rnRef)
-      for i in saveIdx..p.idx-1: r.add newLeaf(p.tok[i].symbol)
-      father.add r
+    if reference != nil:
+      for i in saveIdx..p.idx-1: reference.add newLeaf(p.tok[i].symbol)
+      father.add reference
       inc p.idx  # skip final _
     else:  # 1 normal word
       father.add newLeaf(p.tok[saveIdx].symbol)
@@ -1065,10 +1454,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) =
 
 proc parseBackslash(p: var RstParser, father: PRstNode) =
   assert(currentTok(p).kind == tkPunct)
-  if currentTok(p).symbol == "\\\\":
-    father.add newLeaf("\\")
-    inc p.idx
-  elif currentTok(p).symbol == "\\":
+  if currentTok(p).symbol == "\\":
     # XXX: Unicode?
     inc p.idx
     if currentTok(p).kind != tkWhite: father.add(newLeaf(p))
@@ -1086,14 +1472,39 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
   while true:
     case currentTok(p).kind
     of tkPunct:
-      if isInlineMarkupEnd(p, postfix):
+      if isInlineMarkupEnd(p, postfix, exact=false):
+        let l = currentTok(p).symbol.len
+        if l > postfix.len:
+          # handle cases like *emphasis with stars****. (It's valid RST!)
+          father.add newLeaf(currentTok(p).symbol[0 ..< l - postfix.len])
+        elif postfix == "``" and currentTok(p).symbol == "`" and
+            prevTok(p).symbol == "`":
+          # handle cases like ``literal\`` - delete ` already added after \
+          father.sons.setLen(father.sons.len - 1)
         inc p.idx
         break
-      elif interpretBackslash:
-        parseBackslash(p, father)
       else:
-        father.add(newLeaf(p))
-        inc p.idx
+        if postfix == "`":
+          if currentTok(p).symbol == "\\":
+            if nextTok(p).symbol == "\\":
+              father.add newLeaf("\\")
+              father.add newLeaf("\\")
+              inc p.idx, 2
+            elif nextTok(p).symbol == "`":  # escape `
+              father.add newLeaf("`")
+              inc p.idx, 2
+            else:
+              father.add newLeaf("\\")
+              inc p.idx
+          else:
+            father.add(newLeaf(p))
+            inc p.idx
+        else:
+          if interpretBackslash:
+            parseBackslash(p, father)
+          else:
+            father.add(newLeaf(p))
+            inc p.idx
     of tkAdornment, tkWord, tkOther:
       father.add(newLeaf(p))
       inc p.idx
@@ -1108,62 +1519,185 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
       inc p.idx
     else: rstMessage(p, meExpected, postfix, line, col)
 
+proc parseMarkdownCodeblockFields(p: var RstParser): PRstNode =
+  ## Parses additional (after language string) code block parameters
+  ## in a format *suggested* in the `CommonMark Spec`_ with handling of `"`.
+  if currentTok(p).kind == tkIndent:
+    result = nil
+  else:
+    result = newRstNode(rnFieldList)
+  while currentTok(p).kind != tkIndent:
+    if currentTok(p).kind == tkWhite:
+      inc p.idx
+    else:
+      let field = newRstNode(rnField)
+      var fieldName = ""
+      while currentTok(p).kind notin {tkWhite, tkIndent, tkEof} and
+            currentTok(p).symbol != "=":
+        fieldName.add currentTok(p).symbol
+        inc p.idx
+      field.add(newRstNode(rnFieldName, @[newLeaf(fieldName)]))
+      if currentTok(p).kind == tkWhite: inc p.idx
+      let fieldBody = newRstNode(rnFieldBody)
+      if currentTok(p).symbol == "=":
+        inc p.idx
+        if currentTok(p).kind == tkWhite: inc p.idx
+        var fieldValue = ""
+        if currentTok(p).symbol == "\"":
+          while true:
+            fieldValue.add currentTok(p).symbol
+            inc p.idx
+            if currentTok(p).kind == tkEof:
+              rstMessage(p, meExpected, "\"")
+            elif currentTok(p).symbol == "\"":
+              fieldValue.add "\""
+              inc p.idx
+              break
+        else:
+          while currentTok(p).kind notin {tkWhite, tkIndent, tkEof}:
+            fieldValue.add currentTok(p).symbol
+            inc p.idx
+        fieldBody.add newLeaf(fieldValue)
+      field.add(fieldBody)
+      result.add(field)
+
+proc mayLoadFile(p: RstParser, result: var PRstNode) =
+  var filename = strip(getFieldValue(result, "file"),
+                       chars = Whitespace + {'"'})
+  if filename != "":
+    if roSandboxDisabled notin p.s.options:
+      let tok = p.tok[p.idx-2]
+      rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col)
+    var path = p.findRelativeFile(filename)
+    if path == "": rstMessage(p, meCannotOpenFile, filename)
+    var n = newRstNode(rnLiteralBlock)
+    n.add newLeaf(readFile(path))
+    result.sons[2] = n
+
+proc defaultCodeLangNim(p: RstParser, result: var PRstNode) =
+  # Create a field block if the input block didn't have any.
+  if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList)
+  assert result.sons[1].kind == rnFieldList
+  # Hook the extra field and specify the Nim language as value.
+  var extraNode = newRstNode(rnField, info=lineInfo(p))
+  extraNode.add(newRstNode(rnFieldName))
+  extraNode.add(newRstNode(rnFieldBody))
+  extraNode.sons[0].add newLeaf("default-language")
+  extraNode.sons[1].add newLeaf("Nim")
+  result.sons[1].add(extraNode)
+
 proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
+  result = newRstNodeA(p, rnCodeBlock)
+  result.sons.setLen(3)
+  let line = curLine(p)
+  let baseCol = currentTok(p).col
+  let baseSym = currentTok(p).symbol  # usually just ```
+  inc p.idx
+  result.info = lineInfo(p)
   var args = newRstNode(rnDirArg)
   if currentTok(p).kind == tkWord:
     args.add(newLeaf(p))
     inc p.idx
+    result.sons[1] = parseMarkdownCodeblockFields(p)
+    mayLoadFile(p, result)
   else:
     args = nil
   var n = newLeaf("")
   while true:
-    case currentTok(p).kind
-    of tkEof:
-      rstMessage(p, meExpected, "```")
+    if currentTok(p).kind == tkEof:
+      rstMessage(p, meMissingClosing,
+                 "$1 (started at line $2)" % [baseSym, $line])
       break
-    of tkPunct, tkAdornment:
-      if currentTok(p).symbol == "```":
-        inc p.idx
-        break
-      else:
-        n.text.add(currentTok(p).symbol)
-        inc p.idx
+    elif nextTok(p).kind in {tkPunct, tkAdornment} and
+         nextTok(p).symbol[0] == baseSym[0] and
+         nextTok(p).symbol.len >= baseSym.len:
+      inc p.idx, 2
+      break
+    elif currentTok(p).kind == tkIndent:
+      n.text.add "\n"
+      if currentTok(p).ival > baseCol:
+        n.text.add " ".repeat(currentTok(p).ival - baseCol)
+      elif currentTok(p).ival < baseCol:
+        rstMessage(p, mwRstStyle,
+                   "unexpected de-indentation in Markdown code block")
+      inc p.idx
     else:
       n.text.add(currentTok(p).symbol)
       inc p.idx
-  var lb = newRstNode(rnLiteralBlock)
-  lb.add(n)
-  result = newRstNodeA(p, rnCodeBlock)
-  result.add(args)
-  result.add(PRstNode(nil))
-  result.add(lb)
+  result.sons[0] = args
+  if result.sons[2] == nil:
+    var lb = newRstNode(rnLiteralBlock)
+    lb.add(n)
+    result.sons[2] = lb
+  if result.sons[0].isNil and roNimFile in p.s.options:
+    defaultCodeLangNim(p, result)
 
 proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool =
-  result = true
-  var desc, link = ""
+  # Parses Markdown link. If it's Pandoc auto-link then its second
+  # son (target) will be in tokenized format (rnInner with leafs).
+  var desc = newRstNode(rnInner)
   var i = p.idx
 
+  var parensStack: seq[char]
   template parse(endToken, dest) =
+    parensStack.setLen 0
     inc i # skip begin token
     while true:
-      if p.tok[i].kind in {tkEof, tkIndent}: return false
-      if p.tok[i].symbol == endToken: break
-      dest.add p.tok[i].symbol
+      if p.tok[i].kind == tkEof: return false
+      if p.tok[i].kind == tkIndent and p.tok[i+1].kind == tkIndent:
+        return false
+      let isClosing = checkParen(p.tok[i], parensStack)
+      if p.tok[i].symbol == endToken and not isClosing:
+        break
+      let symbol = if p.tok[i].kind == tkIndent: " " else: p.tok[i].symbol
+      when dest is string: dest.add symbol
+      else: dest.add newLeaf(symbol)
       inc i
     inc i # skip end token
 
   parse("]", desc)
-  if p.tok[i].symbol != "(": return false
-  parse(")", link)
-  let child = newRstNode(rnHyperlink)
-  child.add desc
-  child.add link
-  # only commit if we detected no syntax error:
-  father.add child
-  p.idx = i
-  result = true
+  if p.tok[i].symbol == "(":
+    var link = ""
+    let linkIdx = i + 1
+    parse(")", link)
+    # only commit if we detected no syntax error:
+    let protocol = safeProtocol(link)
+    if link == "":
+      result = false
+      rstMessage(p, mwBrokenLink, protocol,
+                 p.tok[linkIdx].line, p.tok[linkIdx].col)
+    else:
+      let child = newRstNode(rnHyperlink)
+      child.add newLeaf(desc.addNodes)
+      child.add link
+      father.add child
+      p.idx = i
+      result = true
+  elif roPreferMarkdown in p.s.options:
+    # Use Pandoc's implicit_header_references extension
+    var n = newRstNode(rnPandocRef)
+    if p.tok[i].symbol == "[":
+      var link = newRstNode(rnInner)
+      let targetIdx = i + 1
+      parse("]", link)
+      n.add desc
+      if link.len != 0:  # [description][target]
+        n.add link
+        n.info = lineInfo(p, targetIdx)
+      else:              # [description=target][]
+        n.add desc
+        n.info = lineInfo(p, p.idx + 1)
+    else:                # [description=target]
+      n.add desc
+      n.add desc  # target is the same as description
+      n.info = lineInfo(p, p.idx + 1)
+    father.add n
+    p.idx = i
+    result = true
+  else:
+    result = false
 
-proc getFootnoteType(label: PRstNode): (FootnoteType, int) =
+proc getRstFootnoteType(label: PRstNode): (FootnoteType, int) =
   if label.sons.len >= 1 and label.sons[0].kind == rnLeaf and
       label.sons[0].text == "#":
     if label.sons.len == 1:
@@ -1176,12 +1710,23 @@ proc getFootnoteType(label: PRstNode): (FootnoteType, int) =
   elif label.len == 1 and label.sons[0].kind == rnLeaf:
     try:
       result = (fnManualNumber, parseInt(label.sons[0].text))
-    except:
+    except ValueError:
       result = (fnCitation, -1)
   else:
     result = (fnCitation, -1)
 
-proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
+proc getMdFootnoteType(label: PRstNode): (FootnoteType, int) =
+  try:
+    result = (fnManualNumber, parseInt(label.sons[0].text))
+  except ValueError:
+    result = (fnAutoNumberLabel, -1)
+
+proc getFootnoteType(s: PRstSharedState, label: PRstNode): (FootnoteType, int) =
+  ## Returns footnote/citation type and manual number (if present).
+  if isMd(s): getMdFootnoteType(label)
+  else: getRstFootnoteType(label)
+
+proc parseRstFootnoteName(p: var RstParser, reference: bool): PRstNode =
   ## parse footnote/citation label. Precondition: start at `[`.
   ## Label text should be valid ref. name symbol, otherwise nil is returned.
   var i = p.idx + 1
@@ -1211,8 +1756,57 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
     inc i
   p.idx = i
 
+proc isMdFootnoteName(p: RstParser, reference: bool): bool =
+  ## Pandoc Markdown footnote extension.
+  let j = p.idx
+  result = p.tok[j].symbol == "[" and p.tok[j+1].symbol == "^" and
+           p.tok[j+2].kind == tkWord
+
+proc parseMdFootnoteName(p: var RstParser, reference: bool): PRstNode =
+  if isMdFootnoteName(p, reference):
+    result = newRstNode(rnInner)
+    var j = p.idx + 2
+    while p.tok[j].kind in {tkWord, tkOther} or
+        validRefnamePunct(p.tok[j].symbol):
+      result.add newLeaf(p.tok[j].symbol)
+      inc j
+    if j == p.idx + 2:
+      return nil
+    if p.tok[j].symbol == "]":
+      if reference:
+        p.idx = j + 1  # skip ]
+      else:
+        if p.tok[j+1].symbol == ":":
+          p.idx = j + 2  # skip ]:
+        else:
+          result = nil
+    else:
+      result = nil
+  else:
+    result = nil
+
+proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
+  if isMd(p): parseMdFootnoteName(p, reference)
+  else:
+    if isInlineMarkupStart(p, "["): parseRstFootnoteName(p, reference)
+    else: nil
+
+proc isMarkdownCodeBlock(p: RstParser, idx: int): bool =
+  let tok = p.tok[idx]
+  template allowedSymbol: bool =
+    (tok.symbol[0] == '`' or
+      roPreferMarkdown in p.s.options and tok.symbol[0] == '~')
+  result = (roSupportMarkdown in p.s.options and
+            tok.kind in {tkPunct, tkAdornment} and
+            allowedSymbol and
+            tok.symbol.len >= 3)
+
+proc isMarkdownCodeBlock(p: RstParser): bool =
+  isMarkdownCodeBlock(p, p.idx)
+
 proc parseInline(p: var RstParser, father: PRstNode) =
   var n: PRstNode  # to be used in `if` condition
+  let saveIdx = p.idx
   case currentTok(p).kind
   of tkPunct:
     if isInlineMarkupStart(p, "***"):
@@ -1231,43 +1825,55 @@ proc parseInline(p: var RstParser, father: PRstNode) =
       var n = newRstNode(rnInlineTarget)
       inc p.idx
       parseUntil(p, n, "`", false)
-      let refn = rstnodeToRefname(n)
-      p.s.anchors.add (refn, @[refn])
+      n.anchor = rstnodeToRefname(n)
+      addAnchorRst(p, name = linkName(n), target = n,
+                   anchorType=manualInlineAnchor)
       father.add(n)
-    elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
-      inc p.idx
+    elif isMarkdownCodeBlock(p):
       father.add(parseMarkdownCodeblock(p))
     elif isInlineMarkupStart(p, "``"):
       var n = newRstNode(rnInlineLiteral)
       parseUntil(p, n, "``", false)
       father.add(n)
+    elif match(p, p.idx, ":w:") and
+        (var lastIdx = getRefnameIdx(p, p.idx + 1);
+         p.tok[lastIdx+2].symbol == "`"):
+      let (roleName, _) = getRefname(p, p.idx+1)
+      let k = whichRole(p, roleName)
+      var n = newRstNode(k)
+      p.idx = lastIdx + 2
+      if k == rnInlineCode:
+        n = n.toInlineCode(language=roleName)
+      parseUntil(p, n, "`", false) # bug #17260
+      if k in {rnUnknownRole, rnCodeFragment}:
+        n = n.toOtherRole(k, roleName)
+      father.add(n)
     elif isInlineMarkupStart(p, "`"):
-      var n = newRstNode(rnInterpretedText)
-      parseUntil(p, n, "`", true)
+      var n = newRstNode(rnInterpretedText, info=lineInfo(p, p.idx+1))
+      parseUntil(p, n, "`", false) # bug #17260
       n = parsePostfix(p, n)
       father.add(n)
     elif isInlineMarkupStart(p, "|"):
-      var n = newRstNode(rnSubstitutionReferences)
+      var n = newRstNode(rnSubstitutionReferences, info=lineInfo(p, p.idx+1))
       parseUntil(p, n, "|", false)
       father.add(n)
+    elif currentTok(p).symbol == "[" and nextTok(p).symbol != "[" and
+         (n = parseFootnoteName(p, reference=true); n != nil):
+      var nn = newRstNode(rnFootnoteRef)
+      nn.info = lineInfo(p, saveIdx+1)
+      nn.add n
+      let (fnType, _) = getFootnoteType(p.s, n)
+      case fnType
+      of fnAutoSymbol:
+        p.s.lineFootnoteSymRef.add lineInfo(p)
+      of fnAutoNumber:
+        p.s.lineFootnoteNumRef.add lineInfo(p)
+      else: discard
+      father.add(nn)
     elif roSupportMarkdown in p.s.options and
         currentTok(p).symbol == "[" and nextTok(p).symbol != "[" and
         parseMarkdownLink(p, father):
       discard "parseMarkdownLink already processed it"
-    elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and
-         (n = parseFootnoteName(p, reference=true); n != nil):
-      var nn = newRstNode(rnFootnoteRef)
-      nn.add n
-      let (fnType, _) = getFootnoteType(n)
-      case fnType
-      of fnAutoSymbol:
-        p.s.lineFootnoteSymRef.add curLine(p)
-        nn.order = p.s.lineFootnoteSymRef.len
-      of fnAutoNumber:
-        p.s.lineFootnoteNumRef.add curLine(p)
-        nn.order = p.s.lineFootnoteNumRef.len
-      else: discard
-      father.add(nn)
     else:
       if roSupportSmilies in p.s.options:
         let n = parseSmiley(p)
@@ -1283,8 +1889,7 @@ proc parseInline(p: var RstParser, father: PRstNode) =
         return
     parseWordOrRef(p, father)
   of tkAdornment, tkOther, tkWhite:
-    if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
-      inc p.idx
+    if isMarkdownCodeBlock(p):
       father.add(parseMarkdownCodeblock(p))
       return
     if roSupportSmilies in p.s.options:
@@ -1297,44 +1902,39 @@ proc parseInline(p: var RstParser, father: PRstNode) =
   else: discard
 
 proc getDirective(p: var RstParser): string =
-  if currentTok(p).kind == tkWhite and nextTok(p).kind == tkWord:
-    var j = p.idx
-    inc p.idx
-    result = currentTok(p).symbol
-    inc p.idx
-    while currentTok(p).kind in {tkWord, tkPunct, tkAdornment, tkOther}:
-      if currentTok(p).symbol == "::": break
-      result.add(currentTok(p).symbol)
-      inc p.idx
-    if currentTok(p).kind == tkWhite: inc p.idx
-    if currentTok(p).symbol == "::":
-      inc p.idx
-      if currentTok(p).kind == tkWhite: inc p.idx
-    else:
-      p.idx = j               # set back
-      result = ""             # error
-  else:
-    result = ""
-  result = result.toLowerAscii()
+  result = ""
+  if currentTok(p).kind == tkWhite:
+    let (name, lastIdx) = getRefname(p, p.idx + 1)
+    let afterIdx = lastIdx + 1
+    if name.len > 0:
+      if p.tok[afterIdx].symbol == "::":
+        result = name
+        p.idx = afterIdx + 1
+        if currentTok(p).kind == tkWhite:
+          inc p.idx
+        elif currentTok(p).kind != tkIndent:
+          rstMessage(p, mwRstStyle,
+              "whitespace or newline expected after directive " & name)
+        result = result.toLowerAscii()
+      elif p.tok[afterIdx].symbol == ":":
+        rstMessage(p, mwRstStyle,
+            "double colon :: may be missing at end of '" & name & "'",
+            p.tok[afterIdx].line, p.tok[afterIdx].col)
+      elif p.tok[afterIdx].kind == tkPunct and p.tok[afterIdx].symbol[0] == ':':
+        rstMessage(p, mwRstStyle,
+            "too many colons for a directive (should be ::)",
+            p.tok[afterIdx].line, p.tok[afterIdx].col)
 
-proc parseComment(p: var RstParser): PRstNode =
-  case currentTok(p).kind
-  of tkIndent, tkEof:
-    if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent:
-      inc p.idx              # empty comment
-    else:
-      var indent = currentTok(p).ival
-      while true:
-        case currentTok(p).kind
-        of tkEof:
-          break
-        of tkIndent:
-          if currentTok(p).ival < indent: break
-        else:
-          discard
-        inc p.idx
+proc parseComment(p: var RstParser, col: int): PRstNode =
+  if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent:
+    inc p.idx              # empty comment
   else:
-    while currentTok(p).kind notin {tkIndent, tkEof}: inc p.idx
+    while currentTok(p).kind != tkEof:
+      if currentTok(p).kind == tkIndent and currentTok(p).ival > col or
+         currentTok(p).kind != tkIndent and currentTok(p).col > col:
+        inc p.idx
+      else:
+        break
   result = nil
 
 proc parseLine(p: var RstParser, father: PRstNode) =
@@ -1350,22 +1950,103 @@ proc parseUntilNewline(p: var RstParser, father: PRstNode) =
     of tkEof, tkIndent: break
 
 proc parseSection(p: var RstParser, result: PRstNode) {.gcsafe.}
+
+proc tokenAfterNewline(p: RstParser, start: int): int =
+  result = start
+  while true:
+    case p.tok[result].kind
+    of tkEof:
+      break
+    of tkIndent:
+      inc result
+      break
+    else: inc result
+
+proc tokenAfterNewline(p: RstParser): int {.inline.} =
+  result = tokenAfterNewline(p, p.idx)
+
+proc getWrappableIndent(p: RstParser): int =
+  ## Gets baseline indentation for bodies of field lists and directives.
+  ## Handles situations like this (with possible de-indent in [case.3])::
+  ##
+  ##   :field:   definition                                          [case.1]
+  ##
+  ##   currInd   currentTok(p).col
+  ##   |         |
+  ##   v         v
+  ##
+  ##   .. Note:: defItem:                                            [case.2]
+  ##                 definition
+  ##
+  ##                 ^
+  ##                 |
+  ##                 nextIndent
+  ##
+  ##   .. Note:: - point1                                            [case.3]
+  ##       - point 2
+  ##
+  ##       ^
+  ##       |
+  ##       nextIndent
+  if currentTok(p).kind == tkIndent:
+    result = currentTok(p).ival
+  else:
+    var nextIndent = p.tok[tokenAfterNewline(p)-1].ival
+    if nextIndent <= currInd(p):          # parse only this line     [case.1]
+      result = currentTok(p).col
+    elif nextIndent >= currentTok(p).col: # may be a definition list [case.2]
+      result = currentTok(p).col
+    else:
+      result = nextIndent                 # allow parsing next lines [case.3]
+
+proc getMdBlockIndent(p: RstParser): int =
+  ## Markdown version of `getWrappableIndent`.
+  if currentTok(p).kind == tkIndent:
+    result = currentTok(p).ival
+  else:
+    var nextIndent = p.tok[tokenAfterNewline(p)-1].ival
+    # TODO: Markdown-compliant definition should allow nextIndent == currInd(p):
+    if nextIndent <= currInd(p):           # parse only this line
+      result = currentTok(p).col
+    else:
+      result = nextIndent                 # allow parsing next lines [case.3]
+
+proc indFollows(p: RstParser): bool =
+  result = currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p)
+
+proc parseBlockContent(p: var RstParser, father: var PRstNode,
+                       contentParser: SectionParser): bool {.gcsafe.} =
+  ## parse the final content part of explicit markup blocks (directives,
+  ## footnotes, etc). Returns true if succeeded.
+  if currentTok(p).kind != tkIndent or indFollows(p):
+    let blockIndent = getWrappableIndent(p)
+    pushInd(p, blockIndent)
+    let content = contentParser(p)
+    popInd(p)
+    father.add content
+    result = true
+
+proc parseSectionWrapper(p: var RstParser): PRstNode =
+  result = newRstNode(rnInner)
+  parseSection(p, result)
+  while result.kind == rnInner and result.len == 1:
+    result = result.sons[0]
+
 proc parseField(p: var RstParser): PRstNode =
   ## Returns a parsed rnField node.
   ##
   ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody.
-  result = newRstNode(rnField)
+  result = newRstNode(rnField, info=lineInfo(p))
   var col = currentTok(p).col
   var fieldname = newRstNode(rnFieldName)
   parseUntil(p, fieldname, ":", false)
   var fieldbody = newRstNode(rnFieldBody)
-  if currentTok(p).kind != tkIndent: parseLine(p, fieldbody)
-  if currentTok(p).kind == tkIndent:
-    var indent = currentTok(p).ival
-    if indent > col:
-      pushInd(p, indent)
-      parseSection(p, fieldbody)
-      popInd(p)
+  if currentTok(p).kind == tkWhite: inc p.idx
+  let indent = getWrappableIndent(p)
+  if indent > col:
+    pushInd(p, indent)
+    parseSection(p, fieldbody)
+    popInd(p)
   result.add(fieldname)
   result.add(fieldbody)
 
@@ -1424,7 +2105,7 @@ proc parseLiteralBlock(p: var RstParser): PRstNode =
   var n = newLeaf("")
   if currentTok(p).kind == tkIndent:
     var indent = currentTok(p).ival
-    inc p.idx
+    while currentTok(p).kind == tkIndent: inc p.idx  # skip blank lines
     while true:
       case currentTok(p).kind
       of tkEof:
@@ -1445,6 +2126,44 @@ proc parseLiteralBlock(p: var RstParser): PRstNode =
       inc p.idx
   result.add(n)
 
+proc parseQuotedLiteralBlock(p: var RstParser): PRstNode =
+  result = newRstNodeA(p, rnLiteralBlock)
+  var n = newLeaf("")
+  if currentTok(p).kind == tkIndent:
+    var indent = currInd(p)
+    while currentTok(p).kind == tkIndent: inc p.idx  # skip blank lines
+    var quoteSym = currentTok(p).symbol[0]
+    while true:
+      case currentTok(p).kind
+      of tkEof:
+        break
+      of tkIndent:
+        if currentTok(p).ival < indent:
+          break
+        elif currentTok(p).ival == indent:
+          if nextTok(p).kind == tkPunct and nextTok(p).symbol[0] == quoteSym:
+            n.text.add("\n")
+            inc p.idx
+          elif nextTok(p).kind == tkIndent:
+            break
+          else:
+            rstMessage(p, mwRstStyle, "no newline after quoted literal block")
+            break
+        else:
+          rstMessage(p, mwRstStyle,
+                     "unexpected indentation in quoted literal block")
+          break
+      else:
+        n.text.add(currentTok(p).symbol)
+        inc p.idx
+  result.add(n)
+
+proc parseRstLiteralBlock(p: var RstParser, kind: LiteralBlockKind): PRstNode =
+  if kind == lbIndentedLiteralBlock:
+    result = parseLiteralBlock(p)
+  else:
+    result = parseQuotedLiteralBlock(p)
+
 proc getLevel(p: var RstParser, c: char, hasOverline: bool): int =
   ## Returns (preliminary) heading level corresponding to `c` and
   ## `hasOverline`. If level does not exist, add it first.
@@ -1457,38 +2176,27 @@ proc getLevel(p: var RstParser, c: char, hasOverline: bool): int =
                             line: curLine(p), hasPeers: false)
   result = p.s.hLevels.len - 1
 
-proc countTitles(p: var RstParser, n: PRstNode) =
-  ## Fill `p.s.hTitleCnt`
+proc countTitles(s: PRstSharedState, n: PRstNode) =
+  ## Fill `s.hTitleCnt`
+  if n == nil: return
   for node in n.sons:
     if node != nil:
       if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}:
         break
       if node.kind == rnOverline:
-        if p.s.hLevels[p.s.hTitleCnt].hasPeers:
+        if s.hLevels[s.hTitleCnt].hasPeers:
           break
-        inc p.s.hTitleCnt
-        if p.s.hTitleCnt >= 2:
+        inc s.hTitleCnt
+        if s.hTitleCnt >= 2:
           break
 
-proc tokenAfterNewline(p: RstParser, start: int): int =
-  result = start
-  while true:
-    case p.tok[result].kind
-    of tkEof:
-      break
-    of tkIndent:
-      inc result
-      break
-    else: inc result
-
-proc tokenAfterNewline(p: RstParser): int {.inline.} =
-  result = tokenAfterNewline(p, p.idx)
-
 proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool =
   ## check that underline/overline length is enough for the heading.
   ## No support for Unicode.
   if p.tok[adornmentIdx].symbol in ["::", "..", "|"]:
     return false
+  if isMarkdownCodeBlock(p, adornmentIdx):
+    return false
   var headlineLen = 0
   var failure = ""
   if p.idx < adornmentIdx:  # check for underline
@@ -1508,17 +2216,20 @@ proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool =
     while p.tok[i].kind notin {tkEof, tkIndent}:
       headlineLen += p.tok[i].symbol.len
       inc i
-    result = p.tok[adornmentIdx].symbol.len >= headlineLen and
-         headlineLen != 0
-    if result:
-      result = result and p.tok[i].kind == tkIndent and
-         p.tok[i+1].kind == tkAdornment and
-         p.tok[i+1].symbol == p.tok[adornmentIdx].symbol
-      if not result:
-        failure = "(underline '" & p.tok[i+1].symbol & "' does not match " &
-            "overline '" & p.tok[adornmentIdx].symbol & "')"
-    else:
-      failure = "(overline '" & p.tok[adornmentIdx].symbol & "' is too short)"
+    if p.tok[i].kind == tkIndent and
+       p.tok[i+1].kind == tkAdornment and
+       p.tok[i+1].symbol[0] == p.tok[adornmentIdx].symbol[0]:
+      result = p.tok[adornmentIdx].symbol.len >= headlineLen and
+           headlineLen != 0
+      if result:
+        result = p.tok[i+1].symbol == p.tok[adornmentIdx].symbol
+        if not result:
+          failure = "(underline '" & p.tok[i+1].symbol & "' does not match " &
+              "overline '" & p.tok[adornmentIdx].symbol & "')"
+      else:
+        failure = "(overline '" & p.tok[adornmentIdx].symbol & "' is too short)"
+    else:  # it's not overline/underline section, not reporting error
+      return false
   if not result:
     rstMessage(p, meNewSectionExpected, failure)
 
@@ -1528,6 +2239,33 @@ proc isLineBlock(p: RstParser): bool =
       p.tok[j].col > currentTok(p).col or
       p.tok[j].symbol == "\n"
 
+proc isMarkdownBlockQuote(p: RstParser): bool =
+  result = currentTok(p).symbol[0] == '>'
+
+proc whichRstLiteralBlock(p: RstParser): LiteralBlockKind =
+  ## Checks that the following tokens are either Indented Literal Block or
+  ## Quoted Literal Block (which is not quite the same as Markdown quote block).
+  ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks
+  if currentTok(p).symbol == "::" and nextTok(p).kind == tkIndent:
+    if currInd(p) > nextTok(p).ival:
+      result = lbNone
+    if currInd(p) < nextTok(p).ival:
+      result = lbIndentedLiteralBlock
+    elif currInd(p) == nextTok(p).ival:
+      var i = p.idx + 1
+      while p.tok[i].kind == tkIndent: inc i
+      const validQuotingCharacters = {
+          '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-',
+          '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^',
+          '_', '`', '{', '|', '}', '~'}
+      if p.tok[i].kind in {tkPunct, tkAdornment} and
+          p.tok[i].symbol[0] in validQuotingCharacters:
+        result = lbQuotedLiteralBlock
+      else:
+        result = lbNone
+  else:
+    result = lbNone
+
 proc predNL(p: RstParser): bool =
   result = true
   if p.idx > 0:
@@ -1540,6 +2278,39 @@ proc isDefList(p: RstParser): bool =
       p.tok[j].kind in {tkWord, tkOther, tkPunct} and
       p.tok[j - 2].symbol != "::"
 
+proc `$`(t: Token): string =  # for debugging only
+  result = "(" & $t.kind & " line=" & $t.line & " col=" & $t.col
+  if t.kind == tkIndent: result = result & " ival=" & $t.ival & ")"
+  else: result = result & " symbol=" & t.symbol & ")"
+
+proc skipNewlines(p: RstParser, j: int): int =
+  result = j
+  while p.tok[result].kind != tkEof and p.tok[result].kind == tkIndent:
+    inc result  # skip blank lines
+
+proc skipNewlines(p: var RstParser) =
+  p.idx = skipNewlines(p, p.idx)
+
+const maxMdRelInd = 3  ## In Markdown: maximum indentation that does not yet
+                       ## make the indented block a code
+
+proc isMdRelInd(outerInd, nestedInd: int): bool =
+  result = outerInd <= nestedInd and nestedInd <= outerInd + maxMdRelInd
+
+proc isMdDefBody(p: RstParser, j: int, termCol: int): bool =
+  let defCol = p.tok[j].col
+  result = p.tok[j].symbol == ":" and
+    isMdRelInd(termCol, defCol) and
+    p.tok[j+1].kind == tkWhite and
+    p.tok[j+2].kind in {tkWord, tkOther, tkPunct}
+
+proc isMdDefListItem(p: RstParser, idx: int): bool =
+  var j = tokenAfterNewline(p, idx)
+  j = skipNewlines(p, j)
+  let termCol = p.tok[j].col
+  result = isMdRelInd(currInd(p), termCol) and
+      isMdDefBody(p, j, termCol)
+
 proc isOptionList(p: RstParser): bool =
   result = match(p, p.idx, "-w") or match(p, p.idx, "--w") or
            match(p, p.idx, "/w") or match(p, p.idx, "//w")
@@ -1566,11 +2337,12 @@ proc findPipe(p: RstParser, start: int): bool =
 proc whichSection(p: RstParser): RstNodeKind =
   if currentTok(p).kind in {tkAdornment, tkPunct}:
     # for punctuation sequences that can be both tkAdornment and tkPunct
-    if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
+    if isMarkdownCodeBlock(p):
       return rnCodeBlock
-    elif currentTok(p).symbol == "::":
+    elif isRst(p) and currentTok(p).symbol == "::":
       return rnLiteralBlock
-    elif currentTok(p).symbol == ".." and predNL(p):
+    elif currentTok(p).symbol == ".."  and
+       nextTok(p).kind in {tkWhite, tkIndent}:
      return rnDirective
   case currentTok(p).kind
   of tkAdornment:
@@ -1582,34 +2354,42 @@ proc whichSection(p: RstParser): RstNodeKind =
     elif match(p, p.idx + 1, " a"): result = rnTable
     elif currentTok(p).symbol == "|" and isLineBlock(p):
       result = rnLineBlock
-    elif match(p, p.idx + 1, "i") and isAdornmentHeadline(p, p.idx):
+    elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p):
+      result = rnMarkdownBlockQuote
+    elif (match(p, p.idx + 1, "i") and not match(p, p.idx + 2, "I")) and
+         isAdornmentHeadline(p, p.idx):
       result = rnOverline
     else:
-      result = rnLeaf
+      result = rnParagraph
   of tkPunct:
     if isMarkdownHeadline(p):
       result = rnMarkdownHeadline
     elif roSupportMarkdown in p.s.options and predNL(p) and
         match(p, p.idx, "| w") and findPipe(p, p.idx+3):
       result = rnMarkdownTable
+    elif isMd(p) and isMdFootnoteName(p, reference=false):
+      result = rnFootnote
     elif currentTok(p).symbol == "|" and isLineBlock(p):
       result = rnLineBlock
+    elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p):
+      result = rnMarkdownBlockQuote
     elif match(p, tokenAfterNewline(p), "aI") and
         isAdornmentHeadline(p, tokenAfterNewline(p)):
       result = rnHeadline
-    elif predNL(p) and
-        currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite:
+    elif currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite:
       result = rnBulletList
-    elif match(p, p.idx, ":w:") and predNL(p):
+    elif match(p, p.idx, ":w:E"):
       # (currentTok(p).symbol == ":")
       result = rnFieldList
     elif match(p, p.idx, "(e) ") or match(p, p.idx, "e) ") or
          match(p, p.idx, "e. "):
       result = rnEnumList
-    elif isDefList(p):
-      result = rnDefList
     elif isOptionList(p):
       result = rnOptionList
+    elif isRst(p) and isDefList(p):
+      result = rnDefList
+    elif isMd(p) and isMdDefListItem(p, p.idx):
+      result = rnMdDefList
     else:
       result = rnParagraph
   of tkWord, tkOther, tkWhite:
@@ -1618,7 +2398,9 @@ proc whichSection(p: RstParser): RstNodeKind =
       if isAdornmentHeadline(p, tokIdx): result = rnHeadline
       else: result = rnParagraph
     elif match(p, p.idx, "e) ") or match(p, p.idx, "e. "): result = rnEnumList
-    elif isDefList(p): result = rnDefList
+    elif isRst(p) and isDefList(p): result = rnDefList
+    elif isMd(p) and isMdDefListItem(p, p.idx):
+      result = rnMdDefList
     else: result = rnParagraph
   else: result = rnLeaf
 
@@ -1648,14 +2430,112 @@ proc parseLineBlock(p: var RstParser): PRstNode =
       else:
         break
 
+proc parseDoc(p: var RstParser): PRstNode {.gcsafe.}
+
+proc getQuoteSymbol(p: RstParser, idx: int): tuple[sym: string, depth: int, tokens: int] =
+  result = ("", 0, 0)
+  var i = idx
+  result.sym &= p.tok[i].symbol
+  result.depth += p.tok[i].symbol.len
+  inc result.tokens
+  inc i
+  while p.tok[i].kind == tkWhite and i+1 < p.tok.len and
+        p.tok[i+1].kind == tkPunct and p.tok[i+1].symbol[0] == '>':
+    result.sym &= p.tok[i].symbol
+    result.sym &= p.tok[i+1].symbol
+    result.depth += p.tok[i+1].symbol.len
+    inc result.tokens, 2
+    inc i, 2
+
+proc parseMarkdownQuoteSegment(p: var RstParser, curSym: string, col: int):
+                              PRstNode =
+  ## We define *segment* as a group of lines that starts with exactly the
+  ## same quote symbol. If the following lines don't contain any `>` (*lazy*
+  ## continuation) they considered as continuation of the current segment.
+  var q: RstParser  # to delete `>` at a start of line and then parse normally
+  initParser(q, p.s)
+  q.col = p.col
+  q.line = p.line
+  var minCol = int.high  # minimum colum num in the segment
+  while true:  # move tokens of segment from `p` to `q` skipping `curSym`
+    case currentTok(p).kind
+    of tkEof:
+      break
+    of tkIndent:
+      if nextTok(p).kind in {tkIndent, tkEof}:
+        break
+      else:
+        if nextTok(p).symbol[0] == '>':
+          var (quoteSym, _, quoteTokens) = getQuoteSymbol(p, p.idx + 1)
+          if quoteSym == curSym:  # the segment continues
+            var iTok = tokenAfterNewline(p, p.idx+1)
+            if p.tok[iTok].kind notin {tkEof, tkIndent} and
+                p.tok[iTok].symbol[0] != '>':
+              rstMessage(p, mwRstStyle,
+                  "two or more quoted lines are followed by unquoted line " &
+                  $(curLine(p) + 1))
+              break
+            q.tok.add currentTok(p)
+            var ival = currentTok(p).ival + quoteSym.len
+            inc p.idx, (1 + quoteTokens)  # skip newline and > > >
+            if currentTok(p).kind == tkWhite:
+              ival += currentTok(p).symbol.len
+              inc p.idx
+            # fix up previous `tkIndent`s to ival (as if >>> were not there)
+            var j = q.tok.len - 1
+            while j >= 0 and q.tok[j].kind == tkIndent:
+              q.tok[j].ival = ival
+              dec j
+          else:  # next segment started
+            break
+        elif currentTok(p).ival < col:
+          break
+        else:  # the segment continues, a case like:
+               # > beginning
+               # continuation
+          q.tok.add currentTok(p)
+          inc p.idx
+    else:
+      if currentTok(p).col < minCol: minCol = currentTok(p).col
+      q.tok.add currentTok(p)
+      inc p.idx
+  q.indentStack = @[minCol]
+  # if initial indentation `minCol` is > 0 then final newlines
+  # should be omitted so that parseDoc could advance to the end of tokens:
+  var j = q.tok.len - 1
+  while q.tok[j].kind == tkIndent: dec j
+  q.tok.setLen (j+1)
+  q.tok.add Token(kind: tkEof, line: currentTok(p).line)
+  result = parseDoc(q)
+
+proc parseMarkdownBlockQuote(p: var RstParser): PRstNode =
+  var (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx)
+  let col = currentTok(p).col
+  result = newRstNodeA(p, rnMarkdownBlockQuote)
+  inc p.idx, quoteTokens  # skip first >
+  while true:
+    var item = newRstNode(rnMarkdownBlockQuoteItem)
+    item.quotationDepth = quotationDepth
+    if currentTok(p).kind == tkWhite: inc p.idx
+    item.add parseMarkdownQuoteSegment(p, curSym, col)
+    result.add(item)
+    if currentTok(p).kind == tkIndent and currentTok(p).ival == col and
+        nextTok(p).kind != tkEof and nextTok(p).symbol[0] == '>':
+      (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx + 1)
+      inc p.idx, (1 + quoteTokens)  # skip newline and > > >
+    else:
+      break
+
 proc parseParagraph(p: var RstParser, result: PRstNode) =
   while true:
     case currentTok(p).kind
     of tkIndent:
       if nextTok(p).kind == tkIndent:
         inc p.idx
-        break
-      elif currentTok(p).ival == currInd(p):
+        break  # blank line breaks paragraph for both Md & Rst
+      elif currentTok(p).ival == currInd(p) or (
+          isMd(p) and currentTok(p).ival > currInd(p)):
+          # (Md allows adding additional indentation inside paragraphs)
         inc p.idx
         case whichSection(p)
         of rnParagraph, rnLeaf, rnHeadline, rnMarkdownHeadline,
@@ -1663,16 +2543,20 @@ proc parseParagraph(p: var RstParser, result: PRstNode) =
           result.add newLeaf(" ")
         of rnLineBlock:
           result.addIfNotNil(parseLineBlock(p))
-        else: break
+        of rnMarkdownBlockQuote:
+          result.addIfNotNil(parseMarkdownBlockQuote(p))
+        else:
+          dec p.idx  # allow subsequent block to be parsed as another section
+          break
       else:
         break
     of tkPunct:
-      if currentTok(p).symbol == "::" and
-          nextTok(p).kind == tkIndent and
-          currInd(p) < nextTok(p).ival:
+      if isRst(p) and (
+          let literalBlockKind = whichRstLiteralBlock(p);
+          literalBlockKind != lbNone):
         result.add newLeaf(":")
         inc p.idx            # skip '::'
-        result.add(parseLiteralBlock(p))
+        result.add(parseRstLiteralBlock(p, literalBlockKind))
         break
       else:
         parseInline(p, result)
@@ -1712,7 +2596,8 @@ proc parseHeadline(p: var RstParser): PRstNode =
     result.level = getLevel(p, c, hasOverline=false)
     checkHeadingHierarchy(p, result.level)
     p.s.hCurLevel = result.level
-  addAnchor(p, rstnodeToRefname(result), reset=true)
+  addAnchorRst(p, linkName(result), result, anchorType=headlineAnchor)
+  p.s.tocPart.add result
 
 proc parseOverline(p: var RstParser): PRstNode =
   var c = currentTok(p).symbol[0]
@@ -1734,87 +2619,208 @@ proc parseOverline(p: var RstParser): PRstNode =
   if currentTok(p).kind == tkAdornment:
     inc p.idx
     if currentTok(p).kind == tkIndent: inc p.idx
-  addAnchor(p, rstnodeToRefname(result), reset=true)
+  addAnchorRst(p, linkName(result), result, anchorType=headlineAnchor)
+  p.s.tocPart.add result
+
+proc fixHeadlines(s: PRstSharedState) =
+  # Fix up section levels depending on presence of a title and subtitle:
+  for n in s.tocPart:
+    if n.kind in {rnHeadline, rnOverline}:
+      if s.hTitleCnt == 2:
+        if n.level == 1:    # it's the subtitle
+          n.level = 0
+        elif n.level >= 2:  # normal sections, start numbering from 1
+          n.level -= 1
+      elif s.hTitleCnt == 0:
+        n.level += 1
+  # Set headline anchors:
+  for iHeading in 0 .. s.tocPart.high:
+    let n: PRstNode = s.tocPart[iHeading]
+    if n.level >= 1:
+      n.anchor = rstnodeToRefname(n)
+      # Fix anchors for uniqueness if `.. contents::` is present
+      if s.hasToc:
+        # Find the last higher level section for unique reference name
+        var sectionPrefix = ""
+        for i in countdown(iHeading - 1, 0):
+          if s.tocPart[i].level >= 1 and s.tocPart[i].level < n.level:
+            sectionPrefix = rstnodeToRefname(s.tocPart[i]) & "-"
+            break
+        if sectionPrefix != "":
+          n.anchor = sectionPrefix & n.anchor
+  s.tocPart.setLen 0
 
 type
-  IntSeq = seq[int]
-  ColumnLimits = tuple
+  ColSpec = object
+    start, stop: int
+  RstCols = seq[ColSpec]
+  ColumnLimits = tuple  # for Markdown
     first, last: int
   ColSeq = seq[ColumnLimits]
 
-proc tokEnd(p: RstParser): int =
-  result = currentTok(p).col + currentTok(p).symbol.len - 1
+proc tokStart(p: RstParser, idx: int): int =
+  result = p.tok[idx].col
 
-proc getColumns(p: var RstParser, cols: var IntSeq) =
+proc tokStart(p: RstParser): int =
+  result = tokStart(p, p.idx)
+
+proc tokEnd(p: RstParser, idx: int): int =
+  result = p.tok[idx].col + p.tok[idx].symbol.len - 1
+
+proc tokEnd(p: RstParser): int =
+  result = tokEnd(p, p.idx)
+
+proc getColumns(p: RstParser, cols: var RstCols, startIdx: int): int =
+  # Fills table column specification (or separator) `cols` and returns
+  # the next parser index after it.
   var L = 0
+  result = startIdx
   while true:
     inc L
     setLen(cols, L)
-    cols[L - 1] = tokEnd(p)
-    assert(currentTok(p).kind == tkAdornment)
-    inc p.idx
-    if currentTok(p).kind != tkWhite: break
-    inc p.idx
-    if currentTok(p).kind != tkAdornment: break
-  if currentTok(p).kind == tkIndent: inc p.idx
-  # last column has no limit:
-  cols[L - 1] = 32000
+    cols[L - 1].start = tokStart(p, result)
+    cols[L - 1].stop = tokEnd(p, result)
+    assert(p.tok[result].kind == tkAdornment)
+    inc result
+    if p.tok[result].kind != tkWhite: break
+    inc result
+    if p.tok[result].kind != tkAdornment: break
+  if p.tok[result].kind == tkIndent: inc result
 
-proc parseDoc(p: var RstParser): PRstNode {.gcsafe.}
+proc checkColumns(p: RstParser, cols: RstCols) =
+  var i = p.idx
+  if p.tok[i].symbol[0] != '=':
+    stopOrWarn(p, meIllformedTable,
+               "only tables with `=` columns specification are allowed")
+  for col in 0 ..< cols.len:
+    if tokEnd(p, i) != cols[col].stop:
+      stopOrWarn(p, meIllformedTable,
+                 "end of table column #$1 should end at position $2" % [
+                   $(col+1), $(cols[col].stop+ColRstOffset)],
+                 p.tok[i].line, tokEnd(p, i))
+    inc i
+    if col == cols.len - 1:
+      if p.tok[i].kind == tkWhite:
+        inc i
+      if p.tok[i].kind notin {tkIndent, tkEof}:
+        stopOrWarn(p, meIllformedTable, "extraneous column specification")
+    elif p.tok[i].kind == tkWhite:
+      inc i
+    else:
+      stopOrWarn(p, meIllformedTable,
+                 "no enough table columns", p.tok[i].line, p.tok[i].col)
+
+proc getSpans(p: RstParser, nextLine: int,
+              cols: RstCols, unitedCols: RstCols): seq[int] =
+  ## Calculates how many columns a joined cell occupies.
+  if unitedCols.len > 0:
+    result = newSeq[int](unitedCols.len)
+    var
+      iCell = 0
+      jCell = 0
+      uCell = 0
+    while jCell < cols.len:
+      if cols[jCell].stop < unitedCols[uCell].stop:
+        inc jCell
+      elif cols[jCell].stop == unitedCols[uCell].stop:
+        result[uCell] = jCell - iCell + 1
+        iCell = jCell + 1
+        jCell = jCell + 1
+        inc uCell
+      else:
+        rstMessage(p, meIllformedTable,
+                   "spanning underline does not match main table columns",
+                   p.tok[nextLine].line, p.tok[nextLine].col)
+
+proc parseSimpleTableRow(p: var RstParser, cols: RstCols, colChar: char): PRstNode =
+  ## Parses 1 row in RST simple table.
+  # Consider that columns may be spanning (united by using underline like ----):
+  let nextLine = tokenAfterNewline(p)
+  var unitedCols: RstCols
+  var afterSpan: int
+  if p.tok[nextLine].kind == tkAdornment and p.tok[nextLine].symbol[0] == '-':
+    afterSpan = getColumns(p, unitedCols, nextLine)
+    if unitedCols == cols and p.tok[nextLine].symbol[0] == colChar:
+      # legacy rst.nim compat.: allow punctuation like `----` in main boundaries
+      afterSpan = nextLine
+      unitedCols.setLen 0
+  else:
+    afterSpan = nextLine
+  template colEnd(i): int =
+    if i == cols.len - 1: high(int)  # last column has no limit
+    elif unitedCols.len > 0: unitedCols[i].stop else: cols[i].stop
+  template colStart(i): int =
+    if unitedCols.len > 0: unitedCols[i].start else: cols[i].start
+  var row = newSeq[string](if unitedCols.len > 0: unitedCols.len else: cols.len)
+  var spans: seq[int] = getSpans(p, nextLine, cols, unitedCols)
+
+  let line = currentTok(p).line
+  # Iterate over the lines a single cell may span:
+  while true:
+    var nCell = 0
+    # distribute tokens between cells in the current line:
+    while currentTok(p).kind notin {tkIndent, tkEof}:
+      if tokEnd(p) <= colEnd(nCell):
+        if tokStart(p) < colStart(nCell):
+          if currentTok(p).kind != tkWhite:
+            stopOrWarn(p, meIllformedTable,
+                       "this word crosses table column from the left")
+            row[nCell].add(currentTok(p).symbol)
+        else:
+          row[nCell].add(currentTok(p).symbol)
+        inc p.idx
+      else:
+        if tokStart(p) < colEnd(nCell) and currentTok(p).kind != tkWhite:
+          stopOrWarn(p, meIllformedTable,
+                     "this word crosses table column from the right")
+          row[nCell].add(currentTok(p).symbol)
+          inc p.idx
+        inc nCell
+    if currentTok(p).kind == tkIndent: inc p.idx
+    if tokEnd(p) <= colEnd(0): break
+    # Continued current cells because the 1st column is empty.
+    if currentTok(p).kind in {tkEof, tkAdornment}:
+      break
+    for nCell in countup(1, high(row)): row[nCell].add('\n')
+  result = newRstNode(rnTableRow)
+  var q: RstParser
+  for uCell in 0 ..< row.len:
+    initParser(q, p.s)
+    q.col = colStart(uCell)
+    q.line = line - 1
+    getTokens(row[uCell], q.tok)
+    let cell = newRstNode(rnTableDataCell)
+    cell.span = if spans.len == 0: 0 else: spans[uCell]
+    cell.add(parseDoc(q))
+    result.add(cell)
+  if afterSpan > p.idx:
+    p.idx = afterSpan
 
 proc parseSimpleTable(p: var RstParser): PRstNode =
-  var
-    cols: IntSeq
-    row: seq[string]
-    i, last, line: int
-    c: char
-    q: RstParser
-    a, b: PRstNode
+  var cols: RstCols
   result = newRstNodeA(p, rnTable)
-  cols = @[]
-  row = @[]
-  a = nil
-  c = currentTok(p).symbol[0]
+  let startIdx = getColumns(p, cols, p.idx)
+  let colChar = currentTok(p).symbol[0]
+  checkColumns(p, cols)
+  p.idx = startIdx
+  result.colCount = cols.len
   while true:
     if currentTok(p).kind == tkAdornment:
-      last = tokenAfterNewline(p)
-      if p.tok[last].kind in {tkEof, tkIndent}:
+      checkColumns(p, cols)
+      p.idx = tokenAfterNewline(p)
+      if currentTok(p).kind in {tkEof, tkIndent}:
         # skip last adornment line:
-        p.idx = last
         break
-      getColumns(p, cols)
-      setLen(row, cols.len)
-      if a != nil:
-        for j in 0 ..< a.len:  # fix rnTableDataCell -> rnTableHeaderCell
-          a.sons[j] = newRstNode(rnTableHeaderCell, a.sons[j].sons)
+      if result.sons.len > 0: result.sons[^1].endsHeader = true
+      # fix rnTableDataCell -> rnTableHeaderCell for previous table rows:
+      for nRow in 0 ..< result.sons.len:
+        for nCell in 0 ..< result.sons[nRow].len:
+          template cell: PRstNode = result.sons[nRow].sons[nCell]
+          cell = PRstNode(kind: rnTableHeaderCell, sons: cell.sons,
+                          span: cell.span, anchor: cell.anchor)
     if currentTok(p).kind == tkEof: break
-    for j in countup(0, high(row)): row[j] = ""
-    # the following while loop iterates over the lines a single cell may span:
-    line = currentTok(p).line
-    while true:
-      i = 0
-      while currentTok(p).kind notin {tkIndent, tkEof}:
-        if tokEnd(p) <= cols[i]:
-          row[i].add(currentTok(p).symbol)
-          inc p.idx
-        else:
-          if currentTok(p).kind == tkWhite: inc p.idx
-          inc i
-      if currentTok(p).kind == tkIndent: inc p.idx
-      if tokEnd(p) <= cols[0]: break
-      if currentTok(p).kind in {tkEof, tkAdornment}: break
-      for j in countup(1, high(row)): row[j].add('\n')
-    a = newRstNode(rnTableRow)
-    for j in countup(0, high(row)):
-      initParser(q, p.s)
-      q.col = cols[j]
-      q.line = line - 1
-      q.filename = p.filename
-      q.col += getTokens(row[j], false, q.tok)
-      b = newRstNode(rnTableDataCell)
-      b.add(parseDoc(q))
-      a.add(b)
-    result.add(a)
+    let tabRow = parseSimpleTableRow(p, cols, colChar)
+    result.add tabRow
 
 proc readTableRow(p: var RstParser): ColSeq =
   if currentTok(p).symbol == "|": inc p.idx
@@ -1847,29 +2853,28 @@ proc isValidDelimiterRow(p: var RstParser, colNum: int): bool =
 proc parseMarkdownTable(p: var RstParser): PRstNode =
   var
     row: ColSeq
-    colNum: int
     a, b: PRstNode
     q: RstParser
   result = newRstNodeA(p, rnMarkdownTable)
 
   proc parseRow(p: var RstParser, cellKind: RstNodeKind, result: PRstNode) =
     row = readTableRow(p)
-    if colNum == 0: colNum = row.len # table header
-    elif row.len < colNum: row.setLen(colNum)
+    if result.colCount == 0: result.colCount = row.len # table header
+    elif row.len < result.colCount: row.setLen(result.colCount)
     a = newRstNode(rnTableRow)
-    for j in 0 ..< colNum:
+    for j in 0 ..< result.colCount:
       b = newRstNode(cellKind)
       initParser(q, p.s)
       q.col = p.col
       q.line = currentTok(p).line - 1
-      q.filename = p.filename
-      q.col += getTokens(getColContents(p, row[j]), false, q.tok)
+      getTokens(getColContents(p, row[j]), q.tok)
       b.add(parseDoc(q))
       a.add(b)
     result.add(a)
 
   parseRow(p, rnTableHeaderCell, result)
-  if not isValidDelimiterRow(p, colNum): rstMessage(p, meMarkdownIllformedTable)
+  if not isValidDelimiterRow(p, result.colCount):
+    rstMessage(p, meMarkdownIllformedTable)
   while predNL(p) and currentTok(p).symbol == "|":
     parseRow(p, rnTableDataCell, result)
 
@@ -1901,8 +2906,10 @@ proc parseBulletList(p: var RstParser): PRstNode =
 
 proc parseOptionList(p: var RstParser): PRstNode =
   result = newRstNodeA(p, rnOptionList)
+  let col = currentTok(p).col
+  var order = 1
   while true:
-    if isOptionList(p):
+    if currentTok(p).col == col and isOptionList(p):
       var a = newRstNode(rnOptionGroup)
       var b = newRstNode(rnDescription)
       var c = newRstNode(rnOptionListItem)
@@ -1920,10 +2927,42 @@ proc parseOptionList(p: var RstParser): PRstNode =
         popInd(p)
       else:
         parseLine(p, b)
-      if currentTok(p).kind == tkIndent: inc p.idx
+      while currentTok(p).kind == tkIndent: inc p.idx
       c.add(a)
       c.add(b)
+      c.order = order; inc order
       result.add(c)
+    else:
+      if currentTok(p).kind != tkEof: dec p.idx  # back to tkIndent
+      break
+
+proc parseMdDefinitionList(p: var RstParser): PRstNode =
+  ## Parses (Pandoc/kramdown/PHPextra) Markdown definition lists.
+  result = newRstNodeA(p, rnMdDefList)
+  let termCol = currentTok(p).col
+  while true:
+    var item = newRstNode(rnDefItem)
+    var term = newRstNode(rnDefName)
+    parseLine(p, term)
+    skipNewlines(p)
+    inc p.idx, 2  # skip ":" and space
+    item.add(term)
+    while true:
+      var def = newRstNode(rnDefBody)
+      let indent = getMdBlockIndent(p)
+      pushInd(p, indent)
+      parseSection(p, def)
+      popInd(p)
+      item.add(def)
+      let j = skipNewlines(p, p.idx)
+      if isMdDefBody(p, j, termCol):  # parse next definition body
+        p.idx = j + 2  # skip ":" and space
+      else:
+        break
+    result.add(item)
+    let j = skipNewlines(p, p.idx)
+    if p.tok[j].col == termCol and isMdDefListItem(p, j):
+      p.idx = j  # parse next item
     else:
       break
 
@@ -1935,6 +2974,8 @@ proc parseDefinitionList(p: var RstParser): PRstNode =
     var col = currentTok(p).col
     result = newRstNodeA(p, rnDefList)
     while true:
+      if isOptionList(p):
+        break  # option list has priority over def.list
       j = p.idx
       var a = newRstNode(rnDefName)
       parseLine(p, a)
@@ -1991,11 +3032,10 @@ proc parseEnumList(p: var RstParser): PRstNode =
         let n = p.line + p.tok[j].line
         let msg = "\n" & """
           not enough indentation on line $2
-              (should be at column $3 if it's a continuation of enum. list),
+            (should be at column $3 if it's a continuation of enum. list),
           or no blank line after line $1 (if it should be the next paragraph),
           or no escaping \ at the beginning of line $1
-              (if lines $1..$2 are a normal paragraph, not enum. list)""".
-          unindent(8)
+            (if lines $1..$2 are a normal paragraph, not enum. list)""".dedent
         let c = p.col + requiredIndent + ColRstOffset
         rstMessage(p, mwRstStyle, msg % [$(n-1), $n, $c],
                    p.tok[j].line, p.tok[j].col)
@@ -2032,13 +3072,13 @@ proc parseEnumList(p: var RstParser): PRstNode =
       let enumerator = p.tok[p.idx + 1 + wildIndex[w]].symbol
       # check that it's in sequence: enumerator == next(prevEnum)
       if "n" in wildcards[w]:  # arabic numeral
-        let prevEnumI = try: parseInt(prevEnum) except: 1
+        let prevEnumI = try: parseInt(prevEnum) except ValueError: 1
         if enumerator in autoEnums:
           if prevAE != "" and enumerator != prevAE:
             break
           prevAE = enumerator
           curEnum = prevEnumI + 1
-        else: curEnum = (try: parseInt(enumerator) except: 1)
+        else: curEnum = (try: parseInt(enumerator) except ValueError: 1)
         if curEnum - prevEnumI != 1:
           break
         prevEnum = enumerator
@@ -2053,6 +3093,57 @@ proc parseEnumList(p: var RstParser): PRstNode =
     else:
       break
 
+proc prefix(ftnType: FootnoteType): string =
+  case ftnType
+  of fnManualNumber: result = "footnote-"
+  of fnAutoNumber: result = "footnoteauto-"
+  of fnAutoNumberLabel: result = "footnote-"
+  of fnAutoSymbol: result = "footnotesym-"
+  of fnCitation: result = "citation-"
+
+proc parseFootnote(p: var RstParser): PRstNode {.gcsafe.} =
+  ## Parses footnotes and citations, always returns 2 sons:
+  ##
+  ## 1) footnote label, always containing rnInner with 1 or more sons
+  ## 2) footnote body, which may be nil
+  var label: PRstNode
+  if isRst(p):
+    inc p.idx  # skip space after `..`
+  label = parseFootnoteName(p, reference=false)
+  if label == nil:
+    if isRst(p):
+      dec p.idx
+    return nil
+  result = newRstNode(rnFootnote)
+  result.add label
+  let (fnType, i) = getFootnoteType(p.s, label)
+  var name = ""
+  var anchor = fnType.prefix
+  case fnType
+  of fnManualNumber:
+    addFootnoteNumManual(p, i)
+    anchor.add $i
+  of fnAutoNumber, fnAutoNumberLabel:
+    name = rstnodeToRefname(label)
+    addFootnoteNumAuto(p, name)
+    if fnType == fnAutoNumberLabel:
+      anchor.add name
+    else:  # fnAutoNumber
+      result.order = p.s.lineFootnoteNum.len
+      anchor.add $result.order
+  of fnAutoSymbol:
+    addFootnoteSymAuto(p)
+    result.order = p.s.lineFootnoteSym.len
+    anchor.add $p.s.lineFootnoteSym.len
+  of fnCitation:
+    anchor.add rstnodeToRefname(label)
+  addAnchorRst(p, anchor, target = result, anchorType = footnoteAnchor)
+  result.anchor = anchor
+  if currentTok(p).kind == tkWhite: inc p.idx
+  discard parseBlockContent(p, result, parseSectionWrapper)
+  if result.len < 2:
+    result.add nil
+
 proc sonKind(father: PRstNode, i: int): RstNodeKind =
   result = rnLeaf
   if i < father.len: result = father.sons[i].kind
@@ -2066,11 +3157,19 @@ proc parseSection(p: var RstParser, result: PRstNode) =
       if currInd(p) == currentTok(p).ival:
         inc p.idx
       elif currentTok(p).ival > currInd(p):
-        pushInd(p, currentTok(p).ival)
-        var a = newRstNodeA(p, rnBlockQuote)
-        parseSection(p, a)
-        result.add(a)
-        popInd(p)
+        if roPreferMarkdown in p.s.options:  # Markdown => normal paragraphs
+          if currentTok(p).ival - currInd(p) >= 4:
+            result.add parseLiteralBlock(p)
+          else:
+            pushInd(p, currentTok(p).ival)
+            parseSection(p, result)
+            popInd(p)
+        else:  # RST mode => block quotes
+          pushInd(p, currentTok(p).ival)
+          var a = newRstNodeA(p, rnBlockQuote)
+          parseSection(p, a)
+          result.add(a)
+          popInd(p)
       else:
         while currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent:
           inc p.idx  # skip blank lines
@@ -2085,11 +3184,14 @@ proc parseSection(p: var RstParser, result: PRstNode) =
       a = parseLiteralBlock(p)
     of rnBulletList: a = parseBulletList(p)
     of rnLineBlock: a = parseLineBlock(p)
+    of rnMarkdownBlockQuote: a = parseMarkdownBlockQuote(p)
     of rnDirective: a = parseDotDot(p)
+    of rnFootnote: a = parseFootnote(p)
     of rnEnumList: a = parseEnumList(p)
     of rnLeaf: rstMessage(p, meNewSectionExpected, "(syntax error)")
     of rnParagraph: discard
     of rnDefList: a = parseDefinitionList(p)
+    of rnMdDefList: a = parseMdDefinitionList(p)
     of rnFieldList:
       if p.idx > 0: dec p.idx
       a = parseFields(p)
@@ -2110,15 +3212,6 @@ proc parseSection(p: var RstParser, result: PRstNode) =
     result.sons[0] = newRstNode(rnInner, result.sons[0].sons,
                                 anchor=result.sons[0].anchor)
 
-proc parseSectionWrapper(p: var RstParser): PRstNode =
-  result = newRstNode(rnInner)
-  parseSection(p, result)
-  while result.kind == rnInner and result.len == 1:
-    result = result.sons[0]
-
-proc `$`(t: Token): string =
-  result = $t.kind & ' ' & t.symbol
-
 proc parseDoc(p: var RstParser): PRstNode =
   result = parseSectionWrapper(p)
   if currentTok(p).kind != tkEof:
@@ -2128,7 +3221,6 @@ type
   DirFlag = enum
     hasArg, hasOptions, argIsFile, argIsWord
   DirFlags = set[DirFlag]
-  SectionParser = proc (p: var RstParser): PRstNode {.nimcall.}
 
 proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode =
   ## Parses arguments and options for a directive block.
@@ -2141,6 +3233,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode
   ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to
   ## check them before accessing.
   result = newRstNodeA(p, k)
+  if k == rnCodeBlock: result.info = lineInfo(p)
   var args: PRstNode = nil
   var options: PRstNode = nil
   if hasArg in flags:
@@ -2163,28 +3256,13 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode
       parseLine(p, args)
   result.add(args)
   if hasOptions in flags:
-    if currentTok(p).kind == tkIndent and currentTok(p).ival >= 3 and
+    if currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) and
         nextTok(p).symbol == ":":
+      pushInd(p, currentTok(p).ival)
       options = parseFields(p)
+      popInd(p)
   result.add(options)
 
-proc indFollows(p: RstParser): bool =
-  result = currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p)
-
-proc parseBlockContent(p: var RstParser, father: var PRstNode,
-                       contentParser: SectionParser): bool =
-  ## parse the final content part of explicit markup blocks (directives,
-  ## footnotes, etc). Returns true if succeeded.
-  if currentTok(p).kind != tkIndent or indFollows(p):
-    var nextIndent = p.tok[tokenAfterNewline(p)-1].ival
-    if nextIndent <= currInd(p):  # parse only this line
-      nextIndent = currentTok(p).col
-    pushInd(p, nextIndent)
-    var content = contentParser(p)
-    popInd(p)
-    father.add content
-    result = true
-
 proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags,
                     contentParser: SectionParser): PRstNode =
   ## A helper proc that does main work for specific directive procs.
@@ -2265,15 +3343,16 @@ proc dirInclude(p: var RstParser): PRstNode =
 
       var q: RstParser
       initParser(q, p.s)
-      q.filename = path
-      q.col += getTokens(
-        inputString[startPosition..endPosition].strip(),
-        false,
+      let saveFileIdx = p.s.currFileIdx
+      setCurrFilename(p.s, path)
+      getTokens(
+        inputString[startPosition..endPosition],
         q.tok)
       # workaround a GCC bug; more like the interior pointer bug?
       #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0:
       #  InternalError("Too many binary zeros in include file")
       result = parseDoc(q)
+      p.s.currFileIdx = saveFileIdx
 
 proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
   ## Parses a code block.
@@ -2291,28 +3370,14 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
   ##
   ## As an extension this proc will process the ``file`` extension field and if
   ## present will replace the code block with the contents of the referenced
-  ## file.
+  ## file. This behaviour is disabled in sandboxed mode and can be re-enabled
+  ## with the `roSandboxDisabled` flag.
   result = parseDirective(p, rnCodeBlock, {hasArg, hasOptions}, parseLiteralBlock)
-  var filename = strip(getFieldValue(result, "file"))
-  if filename != "":
-    var path = p.findRelativeFile(filename)
-    if path == "": rstMessage(p, meCannotOpenFile, filename)
-    var n = newRstNode(rnLiteralBlock)
-    n.add newLeaf(readFile(path))
-    result.sons[2] = n
+  mayLoadFile(p, result)
 
   # Extend the field block if we are using our custom Nim extension.
   if nimExtension:
-    # Create a field block if the input block didn't have any.
-    if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList)
-    assert result.sons[1].kind == rnFieldList
-    # Hook the extra field and specify the Nim language as value.
-    var extraNode = newRstNode(rnField)
-    extraNode.add(newRstNode(rnFieldName))
-    extraNode.add(newRstNode(rnFieldBody))
-    extraNode.sons[0].add newLeaf("default-language")
-    extraNode.sons[1].add newLeaf("Nim")
-    result.sons[1].add(extraNode)
+    defaultCodeLangNim(p, result)
 
 proc dirContainer(p: var RstParser): PRstNode =
   result = parseDirective(p, rnContainer, {hasArg}, parseSectionWrapper)
@@ -2330,6 +3395,7 @@ proc dirTitle(p: var RstParser): PRstNode =
 
 proc dirContents(p: var RstParser): PRstNode =
   result = parseDirective(p, rnContents, {hasArg}, nil)
+  p.s.hasToc = true
 
 proc dirIndex(p: var RstParser): PRstNode =
   result = parseDirective(p, rnIndex, {}, parseSectionWrapper)
@@ -2340,6 +3406,18 @@ proc dirAdmonition(p: var RstParser, d: string): PRstNode =
 
 proc dirDefaultRole(p: var RstParser): PRstNode =
   result = parseDirective(p, rnDefaultRole, {hasArg}, nil)
+  if result.sons[0].len == 0: p.s.currRole = defaultRole(p.s.options)
+  else:
+    assert result.sons[0].sons[0].kind == rnLeaf
+    p.s.currRole = result.sons[0].sons[0].text
+  p.s.currRoleKind = whichRole(p, p.s.currRole)
+
+proc dirRole(p: var RstParser): PRstNode =
+  result = parseDirective(p, rnDirective, {hasArg, hasOptions}, nil)
+  # just check that language is supported, TODO: real role association
+  let lang = getFieldValue(result, "language").strip
+  if lang != "" and getSourceLanguage(lang) == langNone:
+    rstMessage(p, mwUnsupportedLanguage, lang)
 
 proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind,
                contentParser: SectionParser) =
@@ -2376,19 +3454,36 @@ proc dirRaw(p: var RstParser): PRstNode =
   else:
     dirRawAux(p, result, rnRaw, parseSectionWrapper)
 
+proc dirImportdoc(p: var RstParser): PRstNode =
+  result = parseDirective(p, rnDirective, {}, parseLiteralBlock)
+  assert result.sons[2].kind == rnLiteralBlock
+  assert result.sons[2].sons[0].kind == rnLeaf
+  let filenames: seq[string] = split(result.sons[2].sons[0].text, seps = {','})
+  proc rmSpaces(s: string): string = s.split.join("")
+  for origFilename in filenames:
+    p.s.idxImports[origFilename.rmSpaces] = ImportdocInfo(fromInfo: lineInfo(p))
+
 proc selectDir(p: var RstParser, d: string): PRstNode =
   result = nil
+  let tok = p.tok[p.idx-2] # report on directive in ".. directive::"
+  if roSandboxDisabled notin p.s.options:
+    if d notin SandboxDirAllowlist:
+      rstMessage(p, meSandboxedDirective, d, tok.line, tok.col)
+
   case d
   of "admonition", "attention", "caution": result = dirAdmonition(p, d)
   of "code": result = dirCodeBlock(p)
   of "code-block": result = dirCodeBlock(p, nimExtension = true)
   of "container": result = dirContainer(p)
   of "contents": result = dirContents(p)
-  of "danger", "error": result = dirAdmonition(p, d)
+  of "danger": result = dirAdmonition(p, d)
+  of "default-role": result = dirDefaultRole(p)
+  of "error": result = dirAdmonition(p, d)
   of "figure": result = dirFigure(p)
   of "hint": result = dirAdmonition(p, d)
   of "image": result = dirImage(p)
   of "important": result = dirAdmonition(p, d)
+  of "importdoc": result = dirImportdoc(p)
   of "include": result = dirInclude(p)
   of "index": result = dirIndex(p)
   of "note": result = dirAdmonition(p, d)
@@ -2397,62 +3492,13 @@ proc selectDir(p: var RstParser, d: string): PRstNode =
       result = dirRaw(p)
     else:
       rstMessage(p, meInvalidDirective, d)
+  of "role": result = dirRole(p)
   of "tip": result = dirAdmonition(p, d)
   of "title": result = dirTitle(p)
   of "warning": result = dirAdmonition(p, d)
-  of "default-role": result = dirDefaultRole(p)
   else:
-    let tok = p.tok[p.idx-2]  # report on directive in ".. directive::"
     rstMessage(p, meInvalidDirective, d, tok.line, tok.col)
 
-proc prefix(ftnType: FootnoteType): string =
-  case ftnType
-  of fnManualNumber: result = "footnote-"
-  of fnAutoNumber: result = "footnoteauto-"
-  of fnAutoNumberLabel: result = "footnote-"
-  of fnAutoSymbol: result = "footnotesym-"
-  of fnCitation: result = "citation-"
-
-proc parseFootnote(p: var RstParser): PRstNode =
-  ## Parses footnotes and citations, always returns 2 sons:
-  ##
-  ## 1) footnote label, always containing rnInner with 1 or more sons
-  ## 2) footnote body, which may be nil
-  inc p.idx
-  let label = parseFootnoteName(p, reference=false)
-  if label == nil:
-    dec p.idx
-    return nil
-  result = newRstNode(rnFootnote)
-  result.add label
-  let (fnType, i) = getFootnoteType(label)
-  var name = ""
-  var anchor = fnType.prefix
-  case fnType
-  of fnManualNumber:
-    addFootnoteNumManual(p, i)
-    anchor.add $i
-  of fnAutoNumber, fnAutoNumberLabel:
-    name = rstnodeToRefname(label)
-    addFootnoteNumAuto(p, name)
-    if fnType == fnAutoNumberLabel:
-      anchor.add name
-    else:  # fnAutoNumber
-      result.order = p.s.lineFootnoteNum.len
-      anchor.add $result.order
-  of fnAutoSymbol:
-    addFootnoteSymAuto(p)
-    result.order = p.s.lineFootnoteSym.len
-    anchor.add $p.s.lineFootnoteSym.len
-  of fnCitation:
-    anchor.add rstnodeToRefname(label)
-  addAnchor(p, anchor, reset=true)
-  result.anchor = anchor
-  if currentTok(p).kind == tkWhite: inc p.idx
-  discard parseBlockContent(p, result, parseSectionWrapper)
-  if result.len < 2:
-    result.add nil
-
 proc parseDotDot(p: var RstParser): PRstNode =
   # parse "explicit markup blocks"
   result = nil
@@ -2467,13 +3513,24 @@ proc parseDotDot(p: var RstParser): PRstNode =
   elif match(p, p.idx, " _"):
     # hyperlink target:
     inc p.idx, 2
-    var a = getReferenceName(p, ":")
+    var ending = ":"
+    if currentTok(p).symbol == "`":
+      inc p.idx
+      ending = "`"
+    var a = getReferenceName(p, ending)
+    if ending == "`":
+      if currentTok(p).symbol == ":":
+        inc p.idx
+      else:
+        rstMessage(p, meExpected, ":")
     if currentTok(p).kind == tkWhite: inc p.idx
     var b = untilEol(p)
     if len(b) == 0:  # set internal anchor
-      addAnchor(p, rstnodeToRefname(a), reset=false)
+      p.curAnchors.add ManualAnchor(
+        alias: linkName(a), anchor: rstnodeToRefname(a), info: prevLineInfo(p)
+      )
     else:  # external hyperlink
-      setRef(p, rstnodeToRefname(a), b)
+      setRef(p, rstnodeToRefname(a), b, refType=hyperlinkAlias)
   elif match(p, p.idx, " |"):
     # substitution definitions:
     inc p.idx, 2
@@ -2494,9 +3551,218 @@ proc parseDotDot(p: var RstParser): PRstNode =
       (n = parseFootnote(p); n != nil):
     result = n
   else:
-    result = parseComment(p)
+    result = parseComment(p, col)
 
-proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
+proc rstParsePass1*(fragment: string,
+                    line, column: int,
+                    sharedState: PRstSharedState): PRstNode =
+  ## Parses an RST `fragment`.
+  ## The result should be further processed by
+  ## preparePass2_ and resolveSubs_ (which is pass 2).
+  var p: RstParser
+  initParser(p, sharedState)
+  p.line = line
+  p.col = column
+  getTokens(fragment, p.tok)
+  result = parseDoc(p)
+
+proc extractLinkEnd(x: string): string =
+  ## From links like `path/to/file.html#/%` extract `file.html#/%`.
+  let i = find(x, '#')
+  let last =
+    if i >= 0: i
+    else: x.len - 1
+  let j = rfind(x, '/', start=0, last=last)
+  if j >= 0:
+    result = x[j+1 .. ^1]
+  else:
+    result = x
+
+proc loadIdxFile(s: var PRstSharedState, origFilename: string) =
+  doAssert roSandboxDisabled in s.options
+  var info: TLineInfo
+  info.fileIndex = addFilename(s, origFilename)
+  var (dir, basename, ext) = origFilename.splitFile
+  if ext notin [".md", ".rst", ".nim", ""]:
+    rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo,
+               meCannotOpenFile, origFilename & ": unknown extension")
+  let idxFilename = dir / basename & ".idx"
+  let (idxPath, linkRelPath) = s.findRefFile(idxFilename)
+  s.idxImports[origFilename].linkRelPath = linkRelPath
+  var
+    fileEntries: seq[IndexEntry]
+    title: IndexEntry
+  try:
+    (fileEntries, title) = parseIdxFile(idxPath)
+  except IOError:
+    rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo,
+               meCannotOpenFile, idxPath)
+  except ValueError as e:
+    s.msgHandler(idxPath, LineRstInit, ColRstInit, meInvalidField, e.msg)
+
+  var isMarkup = false  # for sanity check to avoid mixing .md <-> .nim
+  for entry in fileEntries:
+    # Though target .idx already has inside it the path to HTML relative
+    # project's root, we won't rely on it and use `linkRelPath` instead.
+    let refn = extractLinkEnd(entry.link)
+    # select either markup (rst/md) or Nim cases:
+    if entry.kind in {ieMarkupTitle, ieNimTitle}:
+      s.idxImports[origFilename].title = entry.keyword
+    case entry.kind
+    of ieIdxRole, ieHeading, ieMarkupTitle:
+      if ext == ".nim" and entry.kind == ieMarkupTitle:
+        rstMessage(s, idxPath, meInvalidField,
+                   $ieMarkupTitle & " in supposedly .nim-derived file")
+      if entry.kind == ieMarkupTitle:
+        isMarkup = true
+      info.line = entry.line.uint16
+      addAnchorExtRst(s, key = entry.keyword, refn = refn,
+                      anchorType = headlineAnchor, info=info)
+    of ieNim, ieNimGroup, ieNimTitle:
+      if ext in [".md", ".rst"] or isMarkup:
+        rstMessage(s, idxPath, meInvalidField,
+                   $entry.kind & " in supposedly markup-derived file")
+      s.nimFileImported = true
+      var langSym: LangSymbol
+      if entry.kind in {ieNim, ieNimTitle}:
+        var q: RstParser
+        initParser(q, s)
+        info.line = entry.line.uint16
+        setLen(q.tok, 0)
+        q.idx = 0
+        getTokens(entry.linkTitle, q.tok)
+        var sons = newSeq[PRstNode](q.tok.len)
+        for i in 0 ..< q.tok.len: sons[i] = newLeaf(q.tok[i].symbol)
+        let linkTitle = newRstNode(rnInner, sons)
+        langSym = linkTitle.toLangSymbol
+      else:  # entry.kind == ieNimGroup
+        langSym = langSymbolGroup(kind=entry.linkTitle, name=entry.keyword)
+      addAnchorNim(s, external = true, refn = refn, tooltip = entry.linkDesc,
+                   langSym = langSym, priority = -4, # lowest
+                   info = info, module = info.fileIndex)
+  doAssert s.idxImports[origFilename].title != ""
+
+proc preparePass2*(s: var PRstSharedState, mainNode: PRstNode, importdoc = true) =
+  ## Records titles in node `mainNode` and orders footnotes.
+  countTitles(s, mainNode)
+  fixHeadlines(s)
+  orderFootnotes(s)
+  if importdoc:
+    for origFilename in s.idxImports.keys:
+      loadIdxFile(s, origFilename)
+
+proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode =
+  # Associate this link alias with its target and change node kind to
+  # rnHyperlink or rnInternalRef appropriately.
+  var desc, alias: PRstNode
+  if n.kind == rnPandocRef:  # link like [desc][alias]
+    desc = n.sons[0]
+    alias = n.sons[1]
+  else:  # n.kind == rnRstRef, link like `desc=alias`_
+    desc = n
+    alias = n
+  type LinkDef = object
+    ar: AnchorRule
+    priority: int
+    tooltip: string
+    target: PRstNode
+    info: TLineInfo
+    externFilename: string
+      # when external anchor: origin filename where anchor was defined
+    isTitle: bool
+  proc cmp(x, y: LinkDef): int =
+    result = cmp(x.priority, y.priority)
+    if result == 0:
+      result = cmp(x.target, y.target)
+  var foundLinks: seq[LinkDef]
+  let refn = rstnodeToRefname(alias)
+  var hyperlinks = findRef(s, refn)
+  for y in hyperlinks:
+    foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind),
+                           target: y.value, info: y.info,
+                           tooltip: "(" & $y.kind & ")")
+  let substRst = findMainAnchorRst(s, alias.addNodes, n.info)
+  template getExternFilename(subst: AnchorSubst): string =
+    if subst.kind == arExternalRst or
+        (subst.kind == arNim and subst.external):
+      getFilename(s, subst)
+    else: ""
+  for subst in substRst:
+    var refname, fullRefname: string
+    if subst.kind == arInternalRst:
+      refname = subst.target.anchor
+      fullRefname = refname
+    else:  # arExternalRst
+      refname = subst.refnameExt
+      fullRefname = s.idxImports[getFilename(s, subst)].linkRelPath &
+                      "/" & refname
+    let anchorType =
+      if subst.kind == arInternalRst: subst.anchorType
+      else: subst.anchorTypeExt  # arExternalRst
+    foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority,
+                           target: newLeaf(fullRefname),
+                           info: subst.info,
+                           externFilename: getExternFilename(subst),
+                           isTitle: isDocumentationTitle(refname),
+                           tooltip: "(" & $anchorType & ")")
+  # find anchors automatically generated from Nim symbols
+  if roNimFile in s.options or s.nimFileImported:
+    let substNim = findMainAnchorNim(s, signature=alias, n.info)
+    for subst in substNim:
+      let fullRefname =
+        if subst.external:
+          s.idxImports[getFilename(s, subst)].linkRelPath &
+              "/" & subst.refname
+        else: subst.refname
+      foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority,
+                             target: newLeaf(fullRefname),
+                             externFilename: getExternFilename(subst),
+                             isTitle: isDocumentationTitle(subst.refname),
+                             info: subst.info, tooltip: subst.tooltip)
+  foundLinks.sort(cmp = cmp, order = Descending)
+  let aliasStr = addNodes(alias)
+  if foundLinks.len >= 1:
+    if foundLinks[0].externFilename != "":
+      s.idxImports[foundLinks[0].externFilename].used = true
+    let kind = if foundLinks[0].ar in {arHyperlink, arExternalRst}: rnHyperlink
+               elif foundLinks[0].ar == arNim:
+                 if foundLinks[0].externFilename == "": rnNimdocRef
+                 else: rnHyperlink
+               else: rnInternalRef
+    result = newRstNode(kind)
+    let documentName =  # filename without ext for `.nim`, title for `.md`
+      if foundLinks[0].ar == arNim:
+        changeFileExt(foundLinks[0].externFilename.extractFilename, "")
+      elif foundLinks[0].externFilename != "":
+        s.idxImports[foundLinks[0].externFilename].title
+      else: foundLinks[0].externFilename.extractFilename
+    let linkText =
+      if foundLinks[0].externFilename != "":
+        if foundLinks[0].isTitle: newLeaf(addNodes(desc))
+        else: newLeaf(documentName & ": " & addNodes(desc))
+      else:
+        newRstNode(rnInner, desc.sons)
+    result.sons = @[linkText, foundLinks[0].target]
+    if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip
+    if foundLinks.len > 1:  # report ambiguous link
+      var targets = newSeq[string]()
+      for l in foundLinks:
+        var t = "    "
+        if s.filenames.len > 1:
+          t.add getFilename(s.filenames, l.info.fileIndex)
+        let n = l.info.line
+        let c = l.info.col + ColRstOffset
+        t.add "($1, $2): $3" % [$n, $c, l.tooltip]
+        targets.add t
+      rstMessage(s.filenames, s.msgHandler, n.info, mwAmbiguousLink,
+                 "`$1`\n  clash:\n$2" % [
+                   aliasStr, targets.join("\n")])
+  else:  # nothing found
+    result = n
+    rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, aliasStr)
+
+proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
+  ## Makes pass 2 of RST parsing.
   ## Resolves substitutions and anchor aliases, groups footnotes.
   ## Takes input node `n` and returns the same node with recursive
   ## substitutions in `n.sons` to `result`.
@@ -2504,94 +3770,75 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
   if n == nil: return
   case n.kind
   of rnSubstitutionReferences:
-    var x = findSub(p, n)
+    var x = findSub(s, n)
     if x >= 0:
-      result = p.s.subs[x].value
+      result = s.subs[x].value
     else:
       var key = addNodes(n)
       var e = getEnv(key)
       if e != "": result = newLeaf(e)
-      else: rstMessage(p, mwUnknownSubstitution, key)
-  of rnHeadline, rnOverline:
-    # fix up section levels depending on presence of a title and subtitle
-    if p.s.hTitleCnt == 2:
-      if n.level == 1:    # it's the subtitle
-        n.level = 0
-      elif n.level >= 2:  # normal sections
-        n.level -= 1
-    elif p.s.hTitleCnt == 0:
-      n.level += 1
-  of rnRef:
-    let refn = rstnodeToRefname(n)
-    var y = findRef(p, refn)
-    if y != nil:
-      result = newRstNode(rnHyperlink)
-      let text = newRstNode(rnInner, n.sons)
-      result.sons = @[text, y]
-    else:
-      let s = findMainAnchor(p, refn)
-      if s != "":
-        result = newRstNode(rnInternalRef)
-        let text = newRstNode(rnInner, n.sons)
-        result.sons = @[text,        # visible text of reference
-                        newLeaf(s)]  # link itself
+      else: rstMessage(s.filenames, s.msgHandler, n.info,
+                       mwUnknownSubstitution, key)
+  of rnRstRef, rnPandocRef:
+    result = resolveLink(s, n)
   of rnFootnote:
-    var (fnType, num) = getFootnoteType(n.sons[0])
+    var (fnType, num) = getFootnoteType(s, n.sons[0])
     case fnType
     of fnManualNumber, fnCitation:
       discard "no need to alter fixed text"
     of fnAutoNumberLabel, fnAutoNumber:
       if fnType == fnAutoNumberLabel:
         let labelR = rstnodeToRefname(n.sons[0])
-        num = getFootnoteNum(p, labelR)
+        num = getFootnoteNum(s, labelR)
       else:
-        num = getFootnoteNum(p, n.order)
+        num = getFootnoteNum(s, n.order)
       var nn = newRstNode(rnInner)
       nn.add newLeaf($num)
       result.sons[0] = nn
     of fnAutoSymbol:
-      let sym = getAutoSymbol(p, n.order)
+      let sym = getAutoSymbol(s, n.order)
       n.sons[0].sons[0].text = sym
-    n.sons[1] = resolveSubs(p, n.sons[1])
+    n.sons[1] = resolveSubs(s, n.sons[1])
   of rnFootnoteRef:
-    var (fnType, num) = getFootnoteType(n.sons[0])
+    var (fnType, num) = getFootnoteType(s, n.sons[0])
     template addLabel(number: int | string) =
       var nn = newRstNode(rnInner)
       nn.add newLeaf($number)
       result.add(nn)
     var refn = fnType.prefix
     # create new rnFootnoteRef, add final label, and finalize target refn:
-    result = newRstNode(rnFootnoteRef)
+    result = newRstNode(rnFootnoteRef, info = n.info)
     case fnType
     of fnManualNumber:
       addLabel num
       refn.add $num
     of fnAutoNumber:
-      addLabel getFootnoteNum(p, n.order)
-      refn.add $n.order
+      inc s.currFootnoteNumRef
+      addLabel getFootnoteNum(s, s.currFootnoteNumRef)
+      refn.add $s.currFootnoteNumRef
     of fnAutoNumberLabel:
-      addLabel getFootnoteNum(p, rstnodeToRefname(n))
+      addLabel getFootnoteNum(s, rstnodeToRefname(n))
       refn.add rstnodeToRefname(n)
     of fnAutoSymbol:
-      addLabel getAutoSymbol(p, n.order)
-      refn.add $n.order
+      inc s.currFootnoteSymRef
+      addLabel getAutoSymbol(s, s.currFootnoteSymRef)
+      refn.add $s.currFootnoteSymRef
     of fnCitation:
       result.add n.sons[0]
       refn.add rstnodeToRefname(n)
-    let s = findMainAnchor(p, refn)
-    if s != "":
-      result.add newLeaf(s)     # add link
+    # TODO: correctly report ambiguities
+    let anchorInfo = findMainAnchorRst(s, refn, n.info)
+    if anchorInfo.len != 0:
+      result.add newLeaf(anchorInfo[0].target.anchor)  # add link
     else:
-      rstMessage(p, mwUnknownSubstitution, refn)
+      rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
       result.add newLeaf(refn)  # add link
   of rnLeaf:
     discard
-  of rnContents:
-    p.hasToc = true
   else:
     var regroup = false
     for i in 0 ..< n.len:
-      n.sons[i] = resolveSubs(p, n.sons[i])
+      n.sons[i] = resolveSubs(s, n.sons[i])
       if n.sons[i] != nil and n.sons[i].kind == rnFootnote:
         regroup = true
     if regroup:  # group footnotes together into rnFootnoteGroup
@@ -2609,18 +3856,28 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
           inc i
       result.sons = newSons
 
+proc completePass2*(s: PRstSharedState) =
+  for (filename, importdocInfo) in s.idxImports.pairs:
+    if not importdocInfo.used:
+      rstMessage(s.filenames, s.msgHandler, importdocInfo.fromInfo,
+                 mwUnusedImportdoc, filename)
+
 proc rstParse*(text, filename: string,
-               line, column: int, hasToc: var bool,
+               line, column: int,
                options: RstParseOptions,
                findFile: FindFileHandler = nil,
-               msgHandler: MsgHandler = nil): PRstNode =
-  var p: RstParser
-  initParser(p, newSharedState(options, findFile, msgHandler))
-  p.filename = filename
-  p.line = line
-  p.col = column + getTokens(text, roSkipPounds in options, p.tok)
-  let unresolved = parseDoc(p)
-  countTitles(p, unresolved)
-  orderFootnotes(p)
-  result = resolveSubs(p, unresolved)
-  hasToc = p.hasToc
+               findRefFile: FindRefFileHandler = nil,
+               msgHandler: MsgHandler = nil):
+              tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] =
+  ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`,
+  ## note that 2nd tuple element should be fed to `initRstGenerator`
+  ## argument `filenames` (it is being filled here at least with `filename`
+  ## and possibly with other files from RST ``.. include::`` statement).
+  var sharedState = newRstSharedState(options, filename, findFile, findRefFile,
+                                      msgHandler, hasToc=false)
+  let unresolved = rstParsePass1(text, line, column, sharedState)
+  preparePass2(sharedState, unresolved)
+  result.node = resolveSubs(sharedState, unresolved)
+  completePass2(sharedState)
+  result.filenames = sharedState.filenames
+  result.hasToc = sharedState.hasToc
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index c68df7daa1..2bbb0d0b83 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -8,10 +8,12 @@
 #
 
 ## This module implements an AST for the `reStructuredText`:idx: parser.
-##
-## **Note:** Import ``packages/docutils/rstast`` to use this module
 
-import strutils, json
+import std/[strutils, json]
+
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
 
 type
   RstNodeKind* = enum        ## the possible node kinds of an PRstNode
@@ -25,7 +27,7 @@ type
     rnBulletItem,             # a bullet item
     rnEnumList,               # an enumerated list
     rnEnumItem,               # an enumerated item
-    rnDefList,                # a definition list
+    rnDefList, rnMdDefList,   # a definition list (RST/Markdown)
     rnDefItem,                # an item of a definition list consisting of ...
     rnDefName,                # ... a name part ...
     rnDefBody,                # ... and a body part ...
@@ -34,7 +36,10 @@ type
     rnFieldName,              # consisting of a field name ...
     rnFieldBody,              # ... and a field body
     rnOptionList, rnOptionListItem, rnOptionGroup, rnOption, rnOptionString,
-    rnOptionArgument, rnDescription, rnLiteralBlock, rnQuotedLiteralBlock,
+    rnOptionArgument, rnDescription, rnLiteralBlock,
+    rnMarkdownBlockQuote,     # a quote starting from punctuation like >>>
+    rnMarkdownBlockQuoteItem, # a quotation block, quote lines starting with
+                              # the same number of chars
     rnLineBlock,              # the | thingie
     rnLineBlockItem,          # a son of rnLineBlock - one line inside it.
                               # When `RstNode` lineIndent="\n" the line's empty
@@ -44,7 +49,11 @@ type
     rnCitation,               # similar to footnote, so use rnFootnote instead
     rnFootnoteGroup,          # footnote group - exists for a purely stylistic
                               # reason: to display a few footnotes as 1 block
-    rnStandaloneHyperlink, rnHyperlink, rnRef, rnInternalRef, rnFootnoteRef,
+    rnStandaloneHyperlink, rnHyperlink,
+    rnRstRef,                 # RST reference like `section name`_
+    rnPandocRef,              # Pandoc Markdown reference like [section name]
+    rnInternalRef, rnFootnoteRef,
+    rnNimdocRef,              # reference to automatically generated Nim symbol
     rnDirective,              # a general directive
     rnDirArg,                 # a directive argument (for some directives).
                               # here are directives that are not rnDirective:
@@ -57,12 +66,17 @@ type
                               #     * `file#id `_
                               #     * `file#id '_
     rnSubstitutionDef,        # a definition of a substitution
-    rnGeneralRole,            # Inline markup:
+    # Inline markup:
+    rnInlineCode,             # interpreted text with code in a known language
+    rnCodeFragment,           # inline code for highlighting with the specified
+                              # class (which cannot be inferred from context)
+    rnUnknownRole,            # interpreted text with an unknown role
     rnSub, rnSup, rnIdx,
     rnEmphasis,               # "*"
     rnStrongEmphasis,         # "**"
     rnTripleEmphasis,         # "***"
-    rnInterpretedText,        # "`"
+    rnInterpretedText,        # "`" an auxiliary role for parsing that will
+                              # be converted into other kinds like rnInlineCode
     rnInlineLiteral,          # "``"
     rnInlineTarget,           # "_`target`"
     rnSubstitutionReferences, # "|"
@@ -71,6 +85,11 @@ type
     rnLeaf                    # a leaf; the node's text field contains the
                               # leaf val
 
+  FileIndex* = distinct int32
+  TLineInfo* = object
+    line*: uint16
+    col*: int16
+    fileIndex*: FileIndex
 
   PRstNode* = ref RstNode    ## an RST node
   RstNodeSeq* = seq[PRstNode]
@@ -89,21 +108,42 @@ type
       level*: int             ## level of headings starting from 1 (main
                               ## chapter) to larger ones (minor sub-sections)
                               ## level=0 means it's document title or subtitle
-    of rnFootnote, rnCitation, rnFootnoteRef:
+    of rnFootnote, rnCitation, rnOptionListItem:
       order*: int             ## footnote order (for auto-symbol footnotes and
                               ## auto-numbered ones without a label)
+    of rnMarkdownBlockQuoteItem:
+      quotationDepth*: int    ## number of characters in line prefix
+    of rnRstRef, rnPandocRef, rnSubstitutionReferences,
+        rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef:
+      info*: TLineInfo        ## To have line/column info for warnings at
+                              ## nodes that are post-processed after parsing
+    of rnNimdocRef:
+      tooltip*: string
+    of rnTable, rnGridTable, rnMarkdownTable:
+      colCount*: int          ## Number of (not-united) cells in the table
+    of rnTableRow:
+      endsHeader*: bool       ## Is last row in the header of table?
+    of rnTableHeaderCell, rnTableDataCell:
+      span*: int              ## Number of table columns that the cell occupies
     else:
       discard
     anchor*: string           ## anchor, internal link target
                               ## (aka HTML id tag, aka Latex label/hypertarget)
     sons*: RstNodeSeq        ## the node's sons
 
+proc `==`*(a, b: FileIndex): bool {.borrow.}
+
 proc len*(n: PRstNode): int =
   result = len(n.sons)
 
 proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[],
                  anchor = ""): PRstNode =
+  result = PRstNode(kind: kind, sons: sons, anchor: anchor)
+
+proc newRstNode*(kind: RstNodeKind, info: TLineInfo,
+                 sons: seq[PRstNode] = @[]): PRstNode =
   result = PRstNode(kind: kind, sons: sons)
+  result.info = info
 
 proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} =
   assert kind in {rnLeaf, rnSmiley}
@@ -244,7 +284,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) =
     inc(d.indent, 2)
     renderRstSons(d, n, result)
     dec(d.indent, 2)
-  of rnRef:
+  of rnRstRef:
     result.add("`")
     renderRstSons(d, n, result)
     result.add("`_")
@@ -254,7 +294,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) =
     result.add(" <")
     renderRstToRst(d, n.sons[1], result)
     result.add(">`_")
-  of rnGeneralRole:
+  of rnUnknownRole:
     result.add('`')
     renderRstToRst(d, n.sons[0],result)
     result.add("`:")
@@ -337,19 +377,35 @@ proc renderRstToJsonNode(node: PRstNode): JsonNode =
 
 proc renderRstToJson*(node: PRstNode): string =
   ## Writes the given RST node as JSON that is in the form
-  ## ::
-  ##   {
-  ##     "kind":string node.kind,
-  ##     "text":optional string node.text,
-  ##     "level":optional int node.level,
-  ##     "sons":optional node array
-  ##   }
+  ##
+  ##     {
+  ##       "kind":string node.kind,
+  ##       "text":optional string node.text,
+  ##       "level":optional int node.level,
+  ##       "sons":optional node array
+  ##     }
   renderRstToJsonNode(node).pretty
 
-proc renderRstToStr*(node: PRstNode, indent=0): string =
-  ## Writes the parsed RST `node` into a compact string
+proc renderRstToText*(node: PRstNode): string =
+  ## minimal text representation of markup node
+  const code = {rnCodeFragment, rnInterpretedText, rnInlineLiteral, rnInlineCode}
+  if node == nil:
+    return ""
+  case node.kind
+  of rnLeaf, rnSmiley:
+    result.add node.text
+  else:
+    if node.kind in code: result.add "`"
+    for i in 0 ..< node.sons.len:
+      if node.kind in {rnInlineCode, rnCodeBlock} and i == 0:
+        continue  # omit language specifier
+      result.add renderRstToText(node.sons[i])
+    if node.kind in code: result.add "`"
+
+proc treeRepr*(node: PRstNode, indent=0): string =
+  ## Writes the parsed RST `node` into an AST tree with compact string
   ## representation in the format (one line per every sub-node):
-  ## ``indent - kind - text - level - order - anchor (if non-zero)``
+  ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)``
   ## (suitable for debugging of RST parsing).
   if node == nil:
     result.add " ".repeat(indent) & "[nil]\n"
@@ -357,21 +413,32 @@ proc renderRstToStr*(node: PRstNode, indent=0): string =
   result.add " ".repeat(indent) & $node.kind
   case node.kind
   of rnLeaf, rnSmiley:
-    result.add (if node.text == "": "" else: "\t'" & node.text & "'")
+    result.add (if node.text == "": "" else: "  '" & node.text & "'")
   of rnEnumList:
-    result.add "\tlabelFmt=" & node.labelFmt
+    result.add "  labelFmt=" & node.labelFmt
   of rnLineBlockItem:
     var txt: string
-    if node.lineIndent == "\n": txt = "\t(blank line)"
-    else: txt = "\tlineIndent=" & $node.lineIndent.len
+    if node.lineIndent == "\n": txt = "  (blank line)"
+    else: txt = "  lineIndent=" & $node.lineIndent.len
     result.add txt
+  of rnAdmonition:
+    result.add "  adType=" & node.adType
   of rnHeadline, rnOverline, rnMarkdownHeadline:
-    result.add "\tlevel=" & $node.level
-  of rnFootnote, rnCitation, rnFootnoteRef:
-    result.add (if node.order == 0:   "" else: "\torder=" & $node.order)
+    result.add "  level=" & $node.level
+  of rnFootnote, rnCitation, rnOptionListItem:
+    result.add (if node.order == 0:   "" else: "  order=" & $node.order)
+  of rnMarkdownBlockQuoteItem:
+    result.add "  quotationDepth=" & $node.quotationDepth
+  of rnTable, rnGridTable, rnMarkdownTable:
+    result.add "  colCount=" & $node.colCount
+  of rnTableHeaderCell, rnTableDataCell:
+    if node.span > 0:
+      result.add "  span=" & $node.span
+  of rnTableRow:
+    if node.endsHeader: result.add "  endsHeader"
   else:
     discard
-  result.add (if node.anchor == "": "" else: "\tanchor='" & node.anchor & "'")
+  result.add (if node.anchor == "": "" else: "  anchor='" & node.anchor & "'")
   result.add "\n"
   for son in node.sons:
-    result.add renderRstToStr(son, indent=indent+2)
+    result.add treeRepr(son, indent=indent+2)
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index c52a0fdccd..7fc0ac03a8 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -37,12 +37,16 @@
 ##
 ## * The same goes for footnotes/citations links: they point to themselves.
 ##   No backreferences are generated since finding all references of a footnote
-##   can be done by simply searching for [footnoteName].
-##
-## .. Tip: Import ``packages/docutils/rstgen`` to use this module
+##   can be done by simply searching for ``[footnoteName]``.
+
+import std/[strutils, os, hashes, strtabs, tables, sequtils,
+  algorithm, parseutils, strbasics]
+
+import rstast, rst, rstidx, highlite
+
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, syncio, formatfloat]
 
-import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils,
-  algorithm, parseutils
 
 import ../../std/private/since
 
@@ -55,26 +59,26 @@ type
     outHtml,            # output is HTML
     outLatex            # output is Latex
 
-  TocEntry = object
-    n*: PRstNode
-    refname*, header*: string
-
   MetaEnum* = enum
-    metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion
+    metaNone, metaTitleRaw, metaTitle, metaSubtitle, metaAuthor, metaVersion
+
+  EscapeMode* = enum  # in Latex text inside options [] and URLs is
+                      # escaped slightly differently than in normal text
+    emText, emOption, emUrl  # emText is currently used for code also
 
   RstGenerator* = object of RootObj
     target*: OutputTarget
     config*: StringTableRef
     splitAfter*: int          # split too long entries in the TOC
     listingCounter*: int
-    tocPart*: seq[TocEntry]
+    tocPart*: seq[PRstNode]   # headings for Table of Contents
     hasToc*: bool
     theIndex: string # Contents of the index file to be dumped at the end.
-    options*: RstParseOptions
     findFile*: FindFileHandler
     msgHandler*: MsgHandler
     outDir*: string      ## output directory, initialized by docgen.nim
     destFile*: string    ## output (HTML) file, initialized by docgen.nim
+    filenames*: RstFileTable
     filename*: string         ## source Nim or Rst file
     meta*: array[MetaEnum, string]
     currentSection: string ## \
@@ -85,7 +89,9 @@ type
     ## for hyperlinks. See renderIndexTerm proc for details.
     id*: int               ## A counter useful for generating IDs.
     onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int;
-                          content: string)
+                          content: string) {.gcsafe.}
+    escMode*: EscapeMode
+    curQuotationDepth: int
 
   PDoc = var RstGenerator ## Alias to type less.
 
@@ -109,9 +115,10 @@ proc init(p: var CodeBlockParams) =
 
 proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
                        config: StringTableRef, filename: string,
-                       options: RstParseOptions,
                        findFile: FindFileHandler = nil,
-                       msgHandler: MsgHandler = nil) =
+                       msgHandler: MsgHandler = nil,
+                       filenames = default(RstFileTable),
+                       hasToc = false) =
   ## Initializes a ``RstGenerator``.
   ##
   ## You need to call this before using a ``RstGenerator`` with any other
@@ -147,22 +154,25 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
   ##
   ## Example:
   ##
-  ## .. code-block:: nim
-  ##
+  ##   ```nim
   ##   import packages/docutils/rstgen
   ##
   ##   var gen: RstGenerator
   ##   gen.initRstGenerator(outHtml, defaultConfig(), "filename", {})
+  ##   ```
   g.config = config
   g.target = target
   g.tocPart = @[]
+  g.hasToc = hasToc
   g.filename = filename
+  g.filenames = filenames
   g.splitAfter = 20
   g.theIndex = ""
-  g.options = options
   g.findFile = findFile
   g.currentSection = ""
   g.id = 0
+  g.escMode = emText
+  g.curQuotationDepth = 0
   let fileParts = filename.splitFile
   if fileParts.ext == ".nim":
     g.currentSection = "Module " & fileParts.name
@@ -181,7 +191,9 @@ proc writeIndexFile*(g: var RstGenerator, outfile: string) =
   ## If the index is empty the file won't be created.
   if g.theIndex.len > 0: writeFile(outfile, g.theIndex)
 
-proc addXmlChar(dest: var string, c: char) =
+proc addHtmlChar(dest: var string, c: char) =
+  # Escapes HTML characters. Note that single quote ' is not escaped as
+  # ' -- unlike XML (for standards pre HTML5 it was even forbidden).
   case c
   of '&': add(dest, "&")
   of '<': add(dest, "<")
@@ -189,35 +201,36 @@ proc addXmlChar(dest: var string, c: char) =
   of '\"': add(dest, """)
   else: add(dest, c)
 
-proc addRtfChar(dest: var string, c: char) =
+proc addTexChar(dest: var string, c: char, escMode: EscapeMode) =
+  ## Escapes 10 special Latex characters and sometimes ` and [, ].
+  ## TODO: @ is always a normal symbol (besides the header), am I wrong?
+  ## All escapes that need to work in text and code blocks (`emText` mode)
+  ## should start from \ (to be compatible with fancyvrb/fvextra).
   case c
-  of '{': add(dest, "\\{")
-  of '}': add(dest, "\\}")
-  of '\\': add(dest, "\\\\")
+  of '_', '&', '#', '%': add(dest, "\\" & c)
+  # commands \label and \pageref don't accept \$ by some reason but OK with $:
+  of '$': (if escMode == emUrl: add(dest, c) else: add(dest, "\\" & c))
+  # \~ and \^ have a special meaning unless they are followed by {}
+  of '~', '^': add(dest, "\\" & c & "{}")
+  # Latex loves to substitute ` to opening quote, even in texttt mode!
+  of '`': add(dest, "\\textasciigrave{}")
+  # add {} to avoid gobbling up space by \textbackslash
+  of '\\': add(dest, "\\textbackslash{}")
+  # Using { and } in URL in Latex: https://tex.stackexchange.com/a/469175
+  of '{':
+    add(dest, if escMode == emUrl: "\\%7B" else: "\\{")
+  of '}':
+    add(dest, if escMode == emUrl: "\\%7D" else: "\\}")
+  of ']':
+    # escape ] inside an optional argument in e.g. \section[static[T]]{..
+    add(dest, if escMode == emOption: "\\text{]}" else: "]")
   else: add(dest, c)
 
-proc addTexChar(dest: var string, c: char) =
-  case c
-  of '_': add(dest, "\\_")
-  of '{': add(dest, "\\symbol{123}")
-  of '}': add(dest, "\\symbol{125}")
-  of '[': add(dest, "\\symbol{91}")
-  of ']': add(dest, "\\symbol{93}")
-  of '\\': add(dest, "\\symbol{92}")
-  of '$': add(dest, "\\$")
-  of '&': add(dest, "\\&")
-  of '#': add(dest, "\\#")
-  of '%': add(dest, "\\%")
-  of '~': add(dest, "\\symbol{126}")
-  of '@': add(dest, "\\symbol{64}")
-  of '^': add(dest, "\\symbol{94}")
-  of '`': add(dest, "\\symbol{96}")
-  else: add(dest, c)
-
-proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} =
+proc escChar*(target: OutputTarget, dest: var string,
+              c: char, escMode: EscapeMode) {.inline.} =
   case target
-  of outHtml:  addXmlChar(dest, c)
-  of outLatex: addTexChar(dest, c)
+  of outHtml:  addHtmlChar(dest, c)
+  of outLatex: addTexChar(dest, c, escMode)
 
 proc addSplitter(target: OutputTarget; dest: var string) {.inline.} =
   case target
@@ -236,7 +249,7 @@ proc nextSplitPoint*(s: string, start: int): int =
     inc(result)
   dec(result)                 # last valid index
 
-proc esc*(target: OutputTarget, s: string, splitAfter = -1): string =
+proc esc*(target: OutputTarget, s: string, splitAfter = -1, escMode = emText): string =
   ## Escapes the HTML.
   result = ""
   if splitAfter >= 0:
@@ -247,11 +260,11 @@ proc esc*(target: OutputTarget, s: string, splitAfter = -1): string =
       #if (splitter != " ") or (partLen + k - j + 1 > splitAfter):
       partLen = 0
       addSplitter(target, result)
-      for i in countup(j, k): escChar(target, result, s[i])
+      for i in countup(j, k): escChar(target, result, s[i], escMode)
       inc(partLen, k - j + 1)
       j = k + 1
   else:
-    for i in countup(0, len(s) - 1): escChar(target, result, s[i])
+    for i in countup(0, len(s) - 1): escChar(target, result, s[i], escMode)
 
 
 proc disp(target: OutputTarget, xml, tex: string): string =
@@ -271,19 +284,18 @@ proc dispA(target: OutputTarget, dest: var string,
 proc `or`(x, y: string): string {.inline.} =
   result = if x.len == 0: y else: x
 
-proc renderRstToOut*(d: var RstGenerator, n: PRstNode, result: var string)
+proc renderRstToOut*(d: var RstGenerator, n: PRstNode, result: var string) {.gcsafe.}
   ## Writes into ``result`` the rst ast ``n`` using the ``d`` configuration.
   ##
   ## Before using this proc you need to initialise a ``RstGenerator`` with
   ## ``initRstGenerator`` and parse a rst file with ``rstParse`` from the
   ## `packages/docutils/rst module `_. Example:
-  ##
-  ## .. code-block:: nim
-  ##
+  ##   ```nim
   ##   # ...configure gen and rst vars...
   ##   var generatedHtml = ""
   ##   renderRstToOut(gen, rst, generatedHtml)
   ##   echo generatedHtml
+  ##   ```
 
 proc renderAux(d: PDoc, n: PRstNode, result: var string) =
   for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], result)
@@ -310,31 +322,8 @@ proc renderAux(d: PDoc, n: PRstNode, html, tex: string, result: var string) =
 
 # ---------------- index handling --------------------------------------------
 
-proc quoteIndexColumn(text: string): string =
-  ## Returns a safe version of `text` for serialization to the ``.idx`` file.
-  ##
-  ## The returned version can be put without worries in a line based tab
-  ## separated column text file. The following character sequence replacements
-  ## will be performed for that goal:
-  ##
-  ## * ``"\\"`` => ``"\\\\"``
-  ## * ``"\n"`` => ``"\\n"``
-  ## * ``"\t"`` => ``"\\t"``
-  result = newStringOfCap(text.len + 3)
-  for c in text:
-    case c
-    of '\\': result.add "\\"
-    of '\L': result.add "\\n"
-    of '\C': discard
-    of '\t': result.add "\\t"
-    else: result.add c
-
-proc unquoteIndexColumn(text: string): string =
-  ## Returns the unquoted version generated by ``quoteIndexColumn``.
-  result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\"))
-
-proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string,
-                   linkTitle, linkDesc = "") =
+proc setIndexTerm*(d: var RstGenerator; k: IndexEntryKind, htmlFile, id, term: string,
+                   linkTitle, linkDesc = "", line = 0) =
   ## Adds a `term` to the index using the specified hyperlink identifier.
   ##
   ## A new entry will be added to the index using the format
@@ -357,21 +346,8 @@ proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string,
   ## <#writeIndexFile,RstGenerator,string>`_. The purpose of the index is
   ## documented in the `docgen tools guide
   ## `_.
-  var
-    entry = term
-    isTitle = false
-  entry.add('\t')
-  entry.add(htmlFile)
-  if id.len > 0:
-    entry.add('#')
-    entry.add(id)
-  else:
-    isTitle = true
-  if linkTitle.len > 0 or linkDesc.len > 0:
-    entry.add('\t' & linkTitle.quoteIndexColumn)
-    entry.add('\t' & linkDesc.quoteIndexColumn)
-  entry.add("\n")
-
+  let (entry, isTitle) = formatIndexEntry(k, htmlFile, id, term,
+                                          linkTitle, linkDesc, line)
   if isTitle: d.theIndex.insert(entry)
   else: d.theIndex.add(entry)
 
@@ -384,6 +360,15 @@ proc hash(n: PRstNode): int =
       result = result !& hash(n.sons[i])
     result = !$result
 
+proc htmlFileRelPath(d: PDoc): string =
+  if d.outDir.len == 0:
+    # /foo/bar/zoo.nim -> zoo.html
+    changeFileExt(extractFilename(d.filename), HtmlExt)
+  else: # d is initialized in docgen.nim
+    # outDir   = /foo              -\
+    # destFile = /foo/bar/zoo.html -|-> bar/zoo.html
+    d.destFile.relativePath(d.outDir, '/')
+
 proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) =
   ## Renders the string decorated within \`foobar\`\:idx\: markers.
   ##
@@ -400,18 +385,13 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) =
 
   var term = ""
   renderAux(d, n, term)
-  setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection)
-  dispA(d.target, result, "$2", "$2\\label{$1}",
+  setIndexTerm(d, ieIdxRole,
+  htmlFileRelPath(d), id, term, d.currentSection)
+  dispA(d.target, result, "$2", "\\nimindexterm{$1}{$2}",
         [id, term])
 
 type
-  IndexEntry = object
-    keyword: string
-    link: string
-    linkTitle: string ## contains a prettier text for the href
-    linkDesc: string ## the title attribute of the final href
-
-  IndexedDocs = Table[IndexEntry, seq[IndexEntry]] ## \
+  IndexedDocs* = Table[IndexEntry, seq[IndexEntry]] ## \
     ## Contains the index sequences for doc types.
     ##
     ## The key is a *fake* IndexEntry which will contain the title of the
@@ -421,21 +401,6 @@ type
     ## The value indexed by this IndexEntry is a sequence with the real index
     ## entries found in the ``.idx`` file.
 
-proc cmp(a, b: IndexEntry): int =
-  ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`.
-  result = cmpIgnoreStyle(a.keyword, b.keyword)
-  if result == 0:
-    result = cmpIgnoreStyle(a.link, b.link)
-
-proc hash(x: IndexEntry): Hash =
-  ## Returns the hash for the combined fields of the type.
-  ##
-  ## The hash is computed as the chained hash of the individual string hashes.
-  result = x.keyword.hash !& x.link.hash
-  result = result !& x.linkTitle.hash
-  result = result !& x.linkDesc.hash
-  result = !$result
-
 when defined(gcDestructors):
   template `<-`(a, b: var IndexEntry) = a = move(b)
 else:
@@ -444,6 +409,7 @@ else:
     shallowCopy a.link, b.link
     shallowCopy a.linkTitle, b.linkTitle
     shallowCopy a.linkDesc, b.linkDesc
+    shallowCopy a.module, b.module
 
 proc sortIndex(a: var openArray[IndexEntry]) =
   # we use shellsort here; fast and simple
@@ -483,16 +449,20 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string =
   result = "
"
   var i = 0
   while i < symbols.len:
-    let keyword = symbols[i].keyword
+    let keyword = esc(outHtml, symbols[i].keyword)
     let cleanedKeyword = keyword.escapeLink
     result.addf("
$1:
    \n",
                 [keyword, cleanedKeyword])
     var j = i
-    while j < symbols.len and keyword == symbols[j].keyword:
+    while j < symbols.len and symbols[i].keyword == symbols[j].keyword:
       let
         url = symbols[j].link.escapeLink
-        text = if symbols[j].linkTitle.len > 0: symbols[j].linkTitle else: url
-        desc = if symbols[j].linkDesc.len > 0: symbols[j].linkDesc else: ""
+        module = symbols[j].module
+        text =
+          if symbols[j].linkTitle.len > 0:
+            esc(outHtml, module & ": " & symbols[j].linkTitle)
+          else: url
+        desc = symbols[j].linkDesc
       if desc.len > 0:
         result.addf("""
  • $2
    • 
@@ -506,13 +476,6 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string =
     i = j
   result.add("
")
 
-proc isDocumentationTitle(hyperlink: string): bool =
-  ## Returns true if the hyperlink is actually a documentation title.
-  ##
-  ## Documentation titles lack the hash. See `mergeIndexes()
-  ## <#mergeIndexes,string>`_ for a more detailed explanation.
-  result = hyperlink.find('#') < 0
-
 proc stripTocLevel(s: string): tuple[level: int, text: string] =
   ## Returns the *level* of the toc along with the text without it.
   for c in 0 ..< s.len:
@@ -546,17 +509,15 @@ proc generateDocumentationToc(entries: seq[IndexEntry]): string =
     level = 1
   levels.newSeq(entries.len)
   for entry in entries:
-    let (rawLevel, rawText) = stripTocLevel(entry.linkTitle or entry.keyword)
+    let (rawLevel, rawText) = stripTocLevel(entry.linkTitle)
     if rawLevel < 1:
       # This is a normal symbol, push it *inside* one level from the last one.
       levels[L].level = level + 1
-      # Also, ignore the linkTitle and use directly the keyword.
-      levels[L].text = entry.keyword
     else:
       # The level did change, update the level indicator.
       level = rawLevel
       levels[L].level = rawLevel
-      levels[L].text = rawText
+    levels[L].text = rawText
     inc L
 
   # Now generate hierarchical lists based on the precalculated levels.
@@ -587,7 +548,7 @@ proc generateDocumentationIndex(docs: IndexedDocs): string =
   for title in titles:
     let tocList = generateDocumentationToc(docs.getOrDefault(title))
     result.add("\n")
+      title.link & "\">" & title.linkTitle & "\n" & tocList & "\n")
 
 proc generateDocumentationJumps(docs: IndexedDocs): string =
   ## Returns a plain list of hyperlinks to documentation TOCs in HTML.
@@ -599,7 +560,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string =
 
   var chunks: seq[string] = @[]
   for title in titles:
-    chunks.add("" & title.keyword & "")
+    chunks.add("" & title.linkTitle & "")
 
   result.add(chunks.join(", ") & ".
")
 
@@ -613,7 +574,7 @@ proc generateModuleJumps(modules: seq[string]): string =
 
   result.add(chunks.join(", ") & ".
")
 
-proc readIndexDir(dir: string):
+proc readIndexDir*(dir: string):
     tuple[modules: seq[string], symbols: seq[IndexEntry], docs: IndexedDocs] =
   ## Walks `dir` reading ``.idx`` files converting them in IndexEntry items.
   ##
@@ -628,39 +589,12 @@ proc readIndexDir(dir: string):
   # Scan index files and build the list of symbols.
   for path in walkDirRec(dir):
     if path.endsWith(IndexExt):
-      var
-        fileEntries: seq[IndexEntry]
-        title: IndexEntry
-        f = 0
-      newSeq(fileEntries, 500)
-      setLen(fileEntries, 0)
-      for line in lines(path):
-        let s = line.find('\t')
-        if s < 0: continue
-        setLen(fileEntries, f+1)
-        fileEntries[f].keyword = line.substr(0, s-1)
-        fileEntries[f].link = line.substr(s+1)
-        # See if we detect a title, a link without a `#foobar` trailing part.
-        if title.keyword.len == 0 and fileEntries[f].link.isDocumentationTitle:
-          title.keyword = fileEntries[f].keyword
-          title.link = fileEntries[f].link
-
-        if fileEntries[f].link.find('\t') > 0:
-          let extraCols = fileEntries[f].link.split('\t')
-          fileEntries[f].link = extraCols[0]
-          assert extraCols.len == 3
-          fileEntries[f].linkTitle = extraCols[1].unquoteIndexColumn
-          fileEntries[f].linkDesc = extraCols[2].unquoteIndexColumn
-        else:
-          fileEntries[f].linkTitle = ""
-          fileEntries[f].linkDesc = ""
-        inc f
+      var (fileEntries, title) = parseIdxFile(path)
       # Depending on type add this to the list of symbols or table of APIs.
-      if title.keyword.len == 0:
-        for i in 0 ..< f:
-          # Don't add to symbols TOC entries (they start with a whitespace).
-          let toc = fileEntries[i].linkTitle
-          if toc.len > 0 and toc[0] == ' ':
+
+      if title.kind == ieNimTitle:
+        for i in 0 ..< fileEntries.len:
+          if fileEntries[i].kind != ieNim:
             continue
           # Ok, non TOC entry, add it.
           setLen(result.symbols, L + 1)
@@ -670,16 +604,22 @@ proc readIndexDir(dir: string):
           var x = fileEntries[0].link
           let i = find(x, '#')
           if i > 0:
-            x = x.substr(0, i-1)
+            x.setLen(i)
           if i != 0:
             # don't add entries starting with '#'
             result.modules.add(x.changeFileExt(""))
       else:
         # Generate the symbolic anchor for index quickjumps.
-        title.linkTitle = "doc_toc_" & $result.docs.len
+        title.aux = "doc_toc_" & $result.docs.len
         result.docs[title] = fileEntries
 
-  sort(result.modules, system.cmp)
+      for i in 0 ..< fileEntries.len:
+        if fileEntries[i].kind != ieIdxRole:
+          continue
+
+        setLen(result.symbols, L + 1)
+        result.symbols[L] = fileEntries[i]
+        inc L
 
 proc mergeIndexes*(dir: string): string =
   ## Merges all index files in `dir` and returns the generated index as HTML.
@@ -710,6 +650,7 @@ proc mergeIndexes*(dir: string): string =
   ## Returns the merged and sorted indices into a single HTML block which can
   ## be further embedded into nimdoc templates.
   var (modules, symbols, docs) = readIndexDir(dir)
+  sort(modules, system.cmp)
 
   result = ""
   # Generate a quick jump list of documents.
@@ -737,67 +678,32 @@ proc mergeIndexes*(dir: string): string =
 
 # ----------------------------------------------------------------------------
 
-proc stripTocHtml(s: string): string =
-  ## Ugly quick hack to remove HTML tags from TOC titles.
-  ##
-  ## A TocEntry.header field already contains rendered HTML tags. Instead of
-  ## implementing a proper version of renderRstToOut() which recursively
-  ## renders an rst tree to plain text, we simply remove text found between
-  ## angled brackets. Given the limited possibilities of rst inside TOC titles
-  ## this should be enough.
-  result = s
-  var first = result.find('<')
-  while first >= 0:
-    let last = result.find('>', first)
-    if last < 0:
-      # Abort, since we didn't found a closing angled bracket.
-      return
-    result.delete(first, last)
-    first = result.find('<', first)
-
 proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
   var tmp = ""
   for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
   d.currentSection = tmp
-  # Find the last higher level section for unique reference name
-  var sectionPrefix = ""
-  for i in countdown(d.tocPart.high, 0):
-    let n2 = d.tocPart[i].n
-    if n2.level < n.level:
-      sectionPrefix = rstnodeToRefname(n2) & "-"
-      break
-  var refname = sectionPrefix & rstnodeToRefname(n)
+  var tocName = esc(d.target, renderRstToText(n), escMode = emOption)
+    # for Latex: simple text without commands that may break TOC/hyperref
   if d.hasToc:
-    var length = len(d.tocPart)
-    setLen(d.tocPart, length + 1)
-    d.tocPart[length].refname = refname
-    d.tocPart[length].n = n
-    d.tocPart[length].header = tmp
-
+    d.tocPart.add n
     dispA(d.target, result, "\n$3", "\\rsth$4{$3}$2\n",
-      [$n.level, refname.idS, tmp, $chr(n.level - 1 + ord('A')), refname])
+      "$2 href=\"#$5\">$3", "\\rsth$4[$6]{$3}$2\n",
+      [$n.level, n.anchor.idS, tmp,
+       $chr(n.level - 1 + ord('A')), n.anchor, tocName])
   else:
     dispA(d.target, result, "\n$3",
-                            "\\rsth$4{$3}$2\n", [
-        $n.level, refname.idS, tmp,
-        $chr(n.level - 1 + ord('A'))])
+                            "\\rsth$4[$5]{$3}$2\n", [
+        $n.level, n.anchor.idS, tmp,
+        $chr(n.level - 1 + ord('A')), tocName])
 
   # Generate index entry using spaces to indicate TOC level for the output HTML.
   assert n.level >= 0
-  let
-    htmlFileRelPath = if d.outDir.len == 0:
-                        # /foo/bar/zoo.nim -> zoo.html
-                        changeFileExt(extractFilename(d.filename), HtmlExt)
-                      else: # d is initialized in docgen.nim
-                        # outDir   = /foo              -\
-                        # destFile = /foo/bar/zoo.html -|-> bar/zoo.html
-                        d.destFile.relativePath(d.outDir, '/')
-  setIndexTerm(d, htmlFileRelPath, refname, tmp.stripTocHtml,
-    spaces(max(0, n.level)) & tmp)
+  setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor,
+               term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp)
 
 proc renderOverline(d: PDoc, n: PRstNode, result: var string) =
   if n.level == 0 and d.meta[metaTitle].len == 0:
+    d.meta[metaTitleRaw] = n.addNodes
     for i in countup(0, len(n)-1):
       renderRstToOut(d, n.sons[i], d.meta[metaTitle])
     d.currentSection = d.meta[metaTitle]
@@ -809,21 +715,25 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) =
     var tmp = ""
     for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
     d.currentSection = tmp
+    var tocName = esc(d.target, renderRstToText(n), escMode=emOption)
     dispA(d.target, result, "
$3
",
-                   "\\rstov$4{$3}$2\n", [$n.level,
-        rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A'))])
+                   "\\rstov$4[$5]{$3}$2\n", [$n.level,
+                   n.anchor.idS, tmp, $chr(n.level - 1 + ord('A')), tocName])
+    setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor,
+                 term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp)
 
-
-proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) =
+proc renderTocEntry(d: PDoc, n: PRstNode, result: var string) =
+  var header = ""
+  for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], header)
   dispA(d.target, result,
     "
  • $2
    • \n",
-    "\\item\\label{$1_toc} $2\\ref{$1}\n", [e.refname, e.header])
+    "\\item\\label{$1_toc} $2\\ref{$1}\n", [n.anchor, header])
 
 proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int,
                        result: var string) =
   var tmp = ""
   while j <= high(d.tocPart):
-    var a = abs(d.tocPart[j].n.level)
+    var a = abs(d.tocPart[j].level)
     if a == lvl:
       renderTocEntry(d, d.tocPart[j], tmp)
       inc(j)
@@ -877,7 +787,9 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) =
     htmlOut = ""
 
   # support for `:target:` links for images:
-  var target = esc(d.target, getFieldValue(n, "target").strip())
+  var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl)
+  discard safeProtocol(target)
+
   if target.len > 0:
     # `htmlOut` needs to be of the following format for link to work for images:
     # 
@@ -898,6 +810,25 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) =
     "\\includegraphics{$1}",
     [d.config.getOrDefault"doc.smiley_format" % n.text])
 
+proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int =
+  template err(msg: string) =
+    rstMessage(d.filenames, d.msgHandler, n.info, meInvalidField, msg)
+  let value = n.getFieldValue
+  var number: int
+  let nChars = parseInt(value, number)
+  if nChars == 0:
+    if value.len == 0:
+      # use a good default value:
+      result = 1
+    else:
+      err("field $1 requires an integer, but '$2' was given" %
+          [fieldName, value])
+  elif nChars < value.len:
+    err("extra arguments were given to $1: '$2'" %
+        [fieldName, value[nChars..^1]])
+  else:
+    result = number
+
 proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
   ## Parses useful fields which can appear before a code block.
   ##
@@ -907,9 +838,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
   of "number-lines":
     params.numberLines = true
     # See if the field has a parameter specifying a different line than 1.
-    var number: int
-    if parseInt(n.getFieldValue, number) > 0:
-      params.startLine = number
+    params.startLine = getField1Int(d, n, "number-lines")
   of "file", "filename":
     # The ``file`` option is a Nim extension to the official spec, it acts
     # like it would for other directives like ``raw`` or ``cvs-table``. This
@@ -927,14 +856,13 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
       # consider whether `$docCmd` should be appended here too
       params.testCmd = unescape(params.testCmd)
   of "status", "exitcode":
-    var status: int
-    if parseInt(n.getFieldValue, status) > 0:
-      params.status = status
+    params.status = getField1Int(d, n, n.getArgument)
   of "default-language":
     params.langStr = n.getFieldValue.strip
     params.lang = params.langStr.getSourceLanguage
   else:
-    d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument)
+    rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedField,
+               n.getArgument)
 
 proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
   ## Iterates over all code block fields and returns processed params.
@@ -944,8 +872,7 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
   result.init
   if n.isNil:
     return
-  assert n.kind == rnCodeBlock
-  assert(not n.sons[2].isNil)
+  assert n.kind in {rnCodeBlock, rnInlineCode}
 
   # Parse the field list for rendering parameters if there are any.
   if not n.sons[1].isNil:
@@ -982,15 +909,34 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string,
     result.beginTable.add($line & "\n")
     line.inc
     codeLines.dec
-  result.beginTable.add("" & (
+  result.beginTable.add("
    " & ( d.config.getOrDefault"doc.listing_start" % [id, sourceLanguageToStr[params.lang], idStr])) result.endTable = (d.config.getOrDefault"doc.listing_end" % id) & "" & ( d.config.getOrDefault"doc.listing_button" % id) -proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = - ## Renders a code block, appending it to `result`. +proc renderCodeLang*(result: var string, lang: SourceLanguage, code: string, + target: OutputTarget) = + var g: GeneralTokenizer + initGeneralTokenizer(g, code) + while true: + getNextToken(g, lang) + case g.kind + of gtEof: break + of gtNone, gtWhitespace: + add(result, substr(code, g.start, g.length + g.start - 1)) + else: + dispA(target, result, "$1", "\\span$2{$1}", [ + esc(target, substr(code, g.start, g.length+g.start-1)), + tokenClassToStr[g.kind]]) + deinitGeneralTokenizer(g) + +proc renderNimCode*(result: var string, code: string, target: OutputTarget) = + renderCodeLang(result, langNim, code, target) + +proc renderCode(d: PDoc, n: PRstNode, result: var string) {.gcsafe.} = + ## Renders a code (code block or inline code), appending it to `result`. ## ## If the code block uses the ``number-lines`` option, a table will be ## generated with two columns, the first being a list of numbers and the @@ -999,38 +945,40 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = ## may also come from the parser through the internal ``default-language`` ## option to differentiate between a plain code block and Nim's code block ## extension. - assert n.kind == rnCodeBlock - if n.sons[2] == nil: return + assert n.kind in {rnCodeBlock, rnInlineCode} var params = d.parseCodeBlockParams(n) + if n.sons[2] == nil: return var m = n.sons[2].sons[0] assert m.kind == rnLeaf if params.testCmd.len > 0 and d.onTestSnippet != nil: d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text) - let (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text, + var blockStart, blockEnd: string + case d.target + of outHtml: + if n.kind == rnCodeBlock: + (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text, n.anchor.idS) - dispA(d.target, result, blockStart, - "\\begin{rstpre}\n" & n.anchor.idS & "\n", []) + else: # rnInlineCode + blockStart = "" + blockEnd = "" + of outLatex: + if n.kind == rnCodeBlock: + blockStart = "\n\n" & n.anchor.idS & "\\begin{rstpre}\n" + blockEnd = "\n\\end{rstpre}\n\n" + else: # rnInlineCode + blockStart = "\\rstcode{" + blockEnd = "}" + dispA(d.target, result, blockStart, blockStart, []) if params.lang == langNone: - if len(params.langStr) > 0: - d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr) - for letter in m.text: escChar(d.target, result, letter) + if len(params.langStr) > 0 and params.langStr.toLowerAscii != "none": + rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedLanguage, + params.langStr) + for letter in m.text: escChar(d.target, result, letter, emText) else: - var g: GeneralTokenizer - initGeneralTokenizer(g, m.text) - while true: - getNextToken(g, params.lang) - case g.kind - of gtEof: break - of gtNone, gtWhitespace: - add(result, substr(m.text, g.start, g.length + g.start - 1)) - else: - dispA(d.target, result, "$1", "\\span$2{$1}", [ - esc(d.target, substr(m.text, g.start, g.length+g.start-1)), - tokenClassToStr[g.kind]]) - deinitGeneralTokenizer(g) - dispA(d.target, result, blockEnd, "\n\\end{rstpre}\n") + renderCodeLang(result, params.lang, m.text, d.target) + dispA(d.target, result, blockEnd, blockEnd) proc renderContainer(d: PDoc, n: PRstNode, result: var string) = var tmp = "" @@ -1041,11 +989,6 @@ proc renderContainer(d: PDoc, n: PRstNode, result: var string) = else: dispA(d.target, result, "
    $2
    ", "$2", [arg, tmp]) -proc texColumns(n: PRstNode): string = - result = "" - let nColumns = if n.sons.len > 0: len(n.sons[0]) else: 1 - for i in countup(1, nColumns): add(result, "|X") - proc renderField(d: PDoc, n: PRstNode, result: var string) = var b = false if d.target == outLatex: @@ -1121,7 +1064,7 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = case n.adType of "hint", "note", "tip": htmlCls = "admonition-info"; texSz = "\\normalsize"; texColor = "green" - of "attention", "admonition", "important", "warning": + of "attention", "admonition", "important", "warning", "caution": htmlCls = "admonition-warning"; texSz = "\\large"; texColor = "orange" of "danger", "error": htmlCls = "admonition-error"; texSz = "\\Large"; texColor = "red" @@ -1131,30 +1074,68 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = renderAux(d, n, htmlHead & "" & txt & ":    \n" & "$1\n", - "\n\n\\begin{mdframed}[linecolor=" & texColor & "]$2\n" & + "\n\n\\begin{rstadmonition}[borderline west={0.2em}{0pt}{" & + texColor & "}]$2\n" & "{" & texSz & "\\color{" & texColor & "}{\\textbf{" & txt & ":}}} " & - "$1\n\\end{mdframed}\n", + "$1\n\\end{rstadmonition}\n", result) +proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, + external: bool, nimdoc = false, tooltip="") = + var linkStr = "" + block: + let mode = d.escMode + d.escMode = emUrl + renderRstToOut(d, link, linkStr) + d.escMode = mode + discard safeProtocol(linkStr) + var textStr = "" + renderRstToOut(d, text, textStr) + let nimDocStr = if nimdoc: " nimdoc" else: "" + var tooltipStr = "" + if tooltip != "": + tooltipStr = """ title="$1"""" % [ esc(d.target, tooltip) ] + if external: + dispA(d.target, result, + "$1", + "\\href{$2}{$1}", [textStr, linkStr, nimDocStr, tooltipStr]) + else: + dispA(d.target, result, + "$1", + "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", + [textStr, linkStr, nimDocStr, tooltipStr]) + +proc traverseForIndex*(d: PDoc, n: PRstNode) = + ## A version of [renderRstToOut] that only fills entries for ``.idx`` files. + var discarded: string + if n == nil: return + case n.kind + of rnIdx: renderIndexTerm(d, n, discarded) + of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, discarded) + of rnOverline: renderOverline(d, n, discarded) + else: + for i in 0 ..< len(n): + traverseForIndex(d, n.sons[i]) + proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return case n.kind of rnInner: renderAux(d, n, result) of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, result) of rnOverline: renderOverline(d, n, result) - of rnTransition: renderAux(d, n, "\n", "\\hrule$2\n", result) - of rnParagraph: renderAux(d, n, "$1

    \n", "$2\n$1\n\n", result) + of rnTransition: renderAux(d, n, "\n", "\n\n\\vspace{0.6em}\\hrule$2\n", result) + of rnParagraph: renderAux(d, n, "$1

    \n", "\n\n$2\n$1\n\n", result) of rnBulletList: renderAux(d, n, "$1\n", "\\begin{itemize}\n$2\n$1\\end{itemize}\n", result) of rnBulletItem, rnEnumItem: renderAux(d, n, "$1\n", "\\item $2$1\n", result) of rnEnumList: renderEnumList(d, n, result) - of rnDefList: + of rnDefList, rnMdDefList: renderAux(d, n, "$1\n", "\\begin{description}\n$2\n$1\\end{description}\n", result) of rnDefItem: renderAux(d, n, result) - of rnDefName: renderAux(d, n, "$1\n", "$2\\item[$1] ", result) + of rnDefName: renderAux(d, n, "$1\n", "$2\\item[$1]\\ ", result) of rnDefBody: renderAux(d, n, "$1\n", "$2\n$1\n", result) of rnFieldList: var tmp = "" @@ -1178,21 +1159,51 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnIndex: renderRstToOut(d, n.sons[2], result) of rnOptionList: - renderAux(d, n, "$1", - "\\begin{description}\n$2\n$1\\end{description}\n", result) + renderAux(d, n, "$1", + "\\begin{rstoptlist}$2\n$1\\end{rstoptlist}", result) of rnOptionListItem: - renderAux(d, n, "$1\n", "$1", result) + var addclass = if n.order mod 2 == 1: " odd" else: "" + renderAux(d, n, + "
    $1
    \n", + "$1", result) of rnOptionGroup: - renderAux(d, n, "$1", "\\item[$1]", result) + renderAux(d, n, + "
    " & + "$1
    ", + "\\item[\\rstcodeitem{\\spanoption{$1}}]", result) of rnDescription: - renderAux(d, n, "$1\n", " $1\n", result) + renderAux(d, n, "
    $1
    ", + " $1\n", result) of rnOption, rnOptionString, rnOptionArgument: - doAssert false, "renderRstToOut" + raiseAssert "renderRstToOut" of rnLiteralBlock: renderAux(d, n, "$1\n", - "\\begin{rstpre}\n$2\n$1\n\\end{rstpre}\n", result) - of rnQuotedLiteralBlock: - doAssert false, "renderRstToOut" + "\n\n$2\\begin{rstpre}\n$1\n\\end{rstpre}\n\n", result) + of rnMarkdownBlockQuote: + d.curQuotationDepth = 1 + var tmp = "" + renderAux(d, n, "$1", "$1", tmp) + let itemEnding = + if d.target == outHtml: "" else: "\\end{rstquote}" + tmp.add itemEnding.repeat(d.curQuotationDepth - 1) + dispA(d.target, result, + "$1\n", + "\n\\begin{rstquote}\n$2\n$1\\end{rstquote}\n", [tmp, n.anchor.idS]) + of rnMarkdownBlockQuoteItem: + let addQuotationDepth = n.quotationDepth - d.curQuotationDepth + var itemPrefix: string # start or ending (quotation grey bar on the left) + if addQuotationDepth >= 0: + let s = + if d.target == outHtml: "
    " + else: "\\begin{rstquote}" + itemPrefix = s.repeat(addQuotationDepth) + else: + let s = + if d.target == outHtml: "
    " + else: "\\end{rstquote}" + itemPrefix = s.repeat(-addQuotationDepth) + renderAux(d, n, itemPrefix & "

    $1

    ", itemPrefix & "\n$1", result) + d.curQuotationDepth = n.quotationDepth of rnLineBlock: if n.sons.len == 1 and n.sons[0].lineIndent == "\n": # whole line block is one empty line, no need to add extra spacing @@ -1217,25 +1228,50 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnTable, rnGridTable, rnMarkdownTable: renderAux(d, n, "$1", - "\\begin{table}\n$2\n\\begin{rsttab}{" & - texColumns(n) & "|}\n\\hline\n$1\\end{rsttab}\\end{table}", result) + "\n$2\n\\begin{rsttab}{" & + "L".repeat(n.colCount) & "}\n\\toprule\n$1" & + "\\addlinespace[0.1em]\\bottomrule\n\\end{rsttab}", result) of rnTableRow: if len(n) >= 1: - if d.target == outLatex: - #var tmp = "" - renderRstToOut(d, n.sons[0], result) - for i in countup(1, len(n) - 1): - result.add(" & ") - renderRstToOut(d, n.sons[i], result) - result.add("\\\\\n\\hline\n") - else: + case d.target + of outHtml: result.add("") renderAux(d, n, result) result.add("\n") - of rnTableDataCell: - renderAux(d, n, "$1", "$1", result) - of rnTableHeaderCell: - renderAux(d, n, "$1", "\\textbf{$1}", result) + of outLatex: + if n.sons[0].kind == rnTableHeaderCell: + result.add "\\rowcolor{gray!15} " + var spanLines: seq[(int, int)] + var nCell = 0 + for uCell in 0 .. n.len - 1: + renderRstToOut(d, n.sons[uCell], result) + if n.sons[uCell].span > 0: + spanLines.add (nCell + 1, nCell + n.sons[uCell].span) + nCell += n.sons[uCell].span + else: + nCell += 1 + if uCell != n.len - 1: + result.add(" & ") + result.add("\\\\") + if n.endsHeader: result.add("\\midrule\n") + for (start, stop) in spanLines: + result.add("\\cmidrule(lr){$1-$2}" % [$start, $stop]) + result.add("\n") + of rnTableHeaderCell, rnTableDataCell: + case d.target + of outHtml: + let tag = if n.kind == rnTableHeaderCell: "th" else: "td" + var spanSpec: string + if n.span <= 1: spanSpec = "" + else: + spanSpec = " colspan=\"" & $n.span & "\" style=\"text-align: center\"" + renderAux(d, n, "<$1$2>$$1" % [tag, spanSpec], "", result) + of outLatex: + let text = if n.kind == rnTableHeaderCell: "\\textbf{$1}" else: "$1" + var latexStr: string + if n.span <= 1: latexStr = text + else: latexStr = "\\multicolumn{" & $n.span & "}{c}{" & text & "}" + renderAux(d, n, "", latexStr, result) of rnFootnoteGroup: renderAux(d, n, "
    " & @@ -1254,22 +1290,19 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = "   $1\n\n", "\\item[\\textsuperscript{[$3]}]$2 $1\n", [body, n.anchor.idS, mark, n.anchor]) - of rnRef: - var tmp = "" - renderAux(d, n, tmp) - dispA(d.target, result, - "$1", - "$1\\ref{$2}", [tmp, rstnodeToRefname(n)]) + of rnPandocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnRstRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=false) of rnStandaloneHyperlink: - renderAux(d, n, - "$1", - "\\href{$1}{$1}", result) + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true) of rnInternalRef: - var tmp = "" - renderAux(d, n.sons[0], tmp) - dispA(d.target, result, - "$1", - "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [tmp, n.sons[1].text]) + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnNimdocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false, + nimdoc=true, tooltip=n.tooltip) + of rnHyperlink: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true) of rnFootnoteRef: var tmp = "[" renderAux(d, n.sons[0], tmp) @@ -1279,14 +1312,6 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = "$1", "\\textsuperscript{\\hyperlink{$2}{\\textbf{$1}}}", [tmp, n.sons[1].text]) - of rnHyperlink: - var tmp0 = "" - var tmp1 = "" - renderRstToOut(d, n.sons[0], tmp0) - renderRstToOut(d, n.sons[1], tmp1) - dispA(d.target, result, - "$1", - "\\href{$2}{$1}", [tmp0, tmp1]) of rnDirArg, rnRaw: renderAux(d, n, result) of rnRawHtml: if d.target != outLatex and not lastSon(n).isNil: @@ -1296,19 +1321,28 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = result.add addNodes(lastSon(n)) of rnImage, rnFigure: renderImage(d, n, result) - of rnCodeBlock: renderCodeBlock(d, n, result) + of rnCodeBlock, rnInlineCode: renderCode(d, n, result) of rnContainer: renderContainer(d, n, result) of rnSubstitutionReferences, rnSubstitutionDef: renderAux(d, n, "|$1|", "|$1|", result) of rnDirective: renderAux(d, n, "", "", result) - of rnGeneralRole: + of rnUnknownRole, rnCodeFragment: var tmp0 = "" var tmp1 = "" renderRstToOut(d, n.sons[0], tmp0) renderRstToOut(d, n.sons[1], tmp1) - dispA(d.target, result, "$1", "\\span$2{$1}", - [tmp0, tmp1]) + var class = tmp1 + # don't allow missing role break latex compilation: + if d.target == outLatex and n.kind == rnUnknownRole: class = "Other" + if n.kind == rnCodeFragment: + dispA(d.target, result, + "" & + "$1", + "\\rstcode{\\span$2{$1}}", [tmp0, class]) + else: # rnUnknownRole, not necessarily code/monospace font + dispA(d.target, result, "$1", "\\span$2{$1}", + [tmp0, class]) of rnSub: renderAux(d, n, "$1", "\\rstsub{$1}", result) of rnSup: renderAux(d, n, "$1", "\\rstsup{$1}", result) of rnEmphasis: renderAux(d, n, "$1", "\\emph{$1}", result) @@ -1322,7 +1356,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnInlineLiteral, rnInterpretedText: renderAux(d, n, "$1", - "\\texttt{$1}", result) + "\\rstcode{$1}", result) of rnInlineTarget: var tmp = "" renderAux(d, n, tmp) @@ -1331,12 +1365,13 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = "\\label{$2}\\hypertarget{$2}{$1}", [tmp, rstnodeToRefname(n)]) of rnSmiley: renderSmiley(d, n, result) - of rnLeaf: result.add(esc(d.target, n.text)) + of rnLeaf: result.add(esc(d.target, n.text, escMode=d.escMode)) of rnContents: d.hasToc = true of rnDefaultRole: discard of rnTitle: d.meta[metaTitle] = "" renderRstToOut(d, n.sons[0], d.meta[metaTitle]) + d.meta[metaTitleRaw] = n.sons[0].addNodes # ----------------------------------------------------------------------------- @@ -1477,7 +1512,7 @@ $content proc rstToHtml*(s: string, options: RstParseOptions, config: StringTableRef, - msgHandler: MsgHandler = rst.defaultMsgHandler): string = + msgHandler: MsgHandler = rst.defaultMsgHandler): string {.gcsafe.} = ## Converts an input rst string into embeddable HTML. ## ## This convenience proc parses any input string using rst markup (it doesn't @@ -1487,12 +1522,13 @@ proc rstToHtml*(s: string, options: RstParseOptions, ## work. For an explanation of the ``config`` parameter see the ## ``initRstGenerator`` proc. Example: ## - ## .. code-block:: nim + ## ```nim ## import packages/docutils/rstgen, strtabs ## ## echo rstToHtml("*Hello* **world**!", {}, ## newStringTable(modeStyleInsensitive)) ## # --> Hello world! + ## ``` ## ## If you need to allow the rst ``include`` directive or tweak the generated ## output you have to create your own ``RstGenerator`` with @@ -1501,25 +1537,30 @@ proc rstToHtml*(s: string, options: RstParseOptions, proc myFindFile(filename: string): string = # we don't find any files in online mode: result = "" + proc myFindRefFile(filename: string): (string, string) = + result = ("", "") const filen = "input" + let (rst, filenames, t) = rstParse(s, filen, + line=LineRstInit, column=ColRstInit, + options, myFindFile, myFindRefFile, msgHandler) var d: RstGenerator - initRstGenerator(d, outHtml, config, filen, options, myFindFile, msgHandler) - var dummyHasToc = false - var rst = rstParse(s, filen, line=LineRstInit, column=ColRstInit, - dummyHasToc, options, myFindFile, msgHandler) + initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, + filenames, hasToc = t) result = "" renderRstToOut(d, rst, result) + strbasics.strip(result) proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} = ## Convenience proc for `renderRstToOut` and `initRstGenerator`. runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}""" if rstSource.len == 0: return - var option: bool + let (rst, filenames, t) = rstParse(rstSource, "", + line=LineRstInit, column=ColRstInit, + options) var rstGenera: RstGenerator - rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", options) - rstGenera.renderRstToOut( - rstParse(rstSource, "", line=LineRstInit, column=ColRstInit, - option, options), - result) + rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", + filenames=filenames, hasToc = t) + rstGenera.renderRstToOut(rst, result) + strbasics.strip(result) diff --git a/lib/packages/docutils/rstidx.nim b/lib/packages/docutils/rstidx.nim new file mode 100644 index 0000000000..1472d28fd7 --- /dev/null +++ b/lib/packages/docutils/rstidx.nim @@ -0,0 +1,141 @@ +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## Nim `idx`:idx: file format related definitions. + +import std/[strutils, syncio, hashes] +from std/os import splitFile + +type + IndexEntryKind* = enum ## discriminator tag + ieMarkupTitle = "markupTitle" + ## RST/Markdown title, text in `keyword` + + ## HTML text in `linkTitle` + ieNimTitle = "nimTitle" + ## Nim title + ieHeading = "heading" ## RST/Markdown markup heading, escaped + ieIdxRole = "idx" ## RST :idx: definition, escaped + ieNim = "nim" ## Nim symbol, unescaped + ieNimGroup = "nimgrp" ## Nim overload group, unescaped + IndexEntry* = object + kind*: IndexEntryKind ## 0. + keyword*: string ## 1. + link*: string ## 2. + linkTitle*: string ## 3. contains a prettier text for the href + linkDesc*: string ## 4. the title attribute of the final href + line*: int ## 5. + module*: string ## origin file, NOT a field in ``.idx`` file + aux*: string ## auxuliary field, NOT a field in ``.idx`` file + +proc isDocumentationTitle*(hyperlink: string): bool = + ## Returns true if the hyperlink is actually a documentation title. + ## + ## Documentation titles lack the hash. See `mergeIndexes() + ## <#mergeIndexes,string>`_ for a more detailed explanation. + result = hyperlink.find('#') < 0 + +proc `$`*(e: IndexEntry): string = + """("$1", "$2", "$3", "$4", $5)""" % [ + e.keyword, e.link, e.linkTitle, e.linkDesc, $e.line] + +proc quoteIndexColumn(text: string): string = + ## Returns a safe version of `text` for serialization to the ``.idx`` file. + ## + ## The returned version can be put without worries in a line based tab + ## separated column text file. The following character sequence replacements + ## will be performed for that goal: + ## + ## * ``"\\"`` => ``"\\\\"`` + ## * ``"\n"`` => ``"\\n"`` + ## * ``"\t"`` => ``"\\t"`` + result = newStringOfCap(text.len + 3) + for c in text: + case c + of '\\': result.add "\\" + of '\L': result.add "\\n" + of '\C': discard + of '\t': result.add "\\t" + else: result.add c + +proc unquoteIndexColumn*(text: string): string = + ## Returns the unquoted version generated by ``quoteIndexColumn``. + result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) + +proc formatIndexEntry*(kind: IndexEntryKind; htmlFile, id, term, linkTitle, + linkDesc: string, line: int): + tuple[entry: string, isTitle: bool] = + result.entry = $kind + result.entry.add('\t') + result.entry.add term + result.entry.add('\t') + result.entry.add(htmlFile) + if id.len > 0: + result.entry.add('#') + result.entry.add(id) + result.isTitle = false + else: + result.isTitle = true + result.entry.add('\t' & linkTitle.quoteIndexColumn) + result.entry.add('\t' & linkDesc.quoteIndexColumn) + result.entry.add('\t' & $line) + result.entry.add("\n") + +proc parseIndexEntryKind(s: string): IndexEntryKind = + result = case s: + of "nim": ieNim + of "nimgrp": ieNimGroup + of "heading": ieHeading + of "idx": ieIdxRole + of "nimTitle": ieNimTitle + of "markupTitle": ieMarkupTitle + else: raise newException(ValueError, "unknown index entry value $1" % [s]) + +proc parseIdxFile*(path: string): + tuple[fileEntries: seq[IndexEntry], title: IndexEntry] = + var + f = 0 + newSeq(result.fileEntries, 500) + setLen(result.fileEntries, 0) + let (_, base, _) = path.splitFile + for line in lines(path): + let s = line.find('\t') + if s < 0: continue + setLen(result.fileEntries, f+1) + let cols = line.split('\t') + result.fileEntries[f].kind = parseIndexEntryKind(cols[0]) + result.fileEntries[f].keyword = cols[1] + result.fileEntries[f].link = cols[2] + if result.fileEntries[f].kind == ieIdxRole: + result.fileEntries[f].module = base + else: + if result.title.keyword.len == 0: + result.fileEntries[f].module = base + else: + result.fileEntries[f].module = result.title.keyword + + result.fileEntries[f].linkTitle = cols[3].unquoteIndexColumn + result.fileEntries[f].linkDesc = cols[4].unquoteIndexColumn + result.fileEntries[f].line = parseInt(cols[5]) + + if result.fileEntries[f].kind in {ieNimTitle, ieMarkupTitle}: + result.title = result.fileEntries[f] + inc f + +proc cmp*(a, b: IndexEntry): int = + ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. + result = cmpIgnoreStyle(a.keyword, b.keyword) + if result == 0: + result = cmpIgnoreStyle(a.link, b.link) + +proc hash*(x: IndexEntry): Hash = + ## Returns the hash for the combined fields of the type. + ## + ## The hash is computed as the chained hash of the individual string hashes. + result = x.keyword.hash !& x.link.hash + result = result !& x.linkTitle.hash + result = result !& x.linkDesc.hash + result = !$result diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim index 1f105ecac2..007488354a 100644 --- a/lib/posix/epoll.nim +++ b/lib/posix/epoll.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -from posix import SocketHandle +from std/posix import SocketHandle const EPOLLIN* = 0x00000001 @@ -33,19 +33,28 @@ const EPOLL_CTL_DEL* = 2 # Remove a file descriptor from the interface. EPOLL_CTL_MOD* = 3 # Change file descriptor epoll_event structure. +# https://github.com/torvalds/linux/blob/ff6992735ade75aae3e35d16b17da1008d753d28/include/uapi/linux/eventpoll.h#L77 +when defined(linux) and defined(amd64): + {.pragma: epollPacked, packed.} +else: + {.pragma: epollPacked.} + type - EpollData* {.importc: "union epoll_data", - header: "", pure, final.} = object # TODO: This is actually a union. + EpollData* {.importc: "epoll_data_t", + header: "", pure, final, union.} = object + `ptr`* {.importc: "ptr".}: pointer + fd* {.importc: "fd".}: cint + u32* {.importc: "u32".}: uint32 u64* {.importc: "u64".}: uint64 - EpollEvent* {.importc: "struct epoll_event", header: "", pure, final.} = object + EpollEvent* {.importc: "struct epoll_event", header: "", pure, final, epollPacked.} = object events*: uint32 # Epoll events data*: EpollData # User data variable proc epoll_create*(size: cint): cint {.importc: "epoll_create", header: "".} ## Creates an epoll instance. Returns an fd for the new instance. - ## + ## ## The "size" parameter is a hint specifying the number of file ## descriptors to be associated with the new instance. The fd ## returned by epoll_create() should be closed with close(). @@ -59,7 +68,7 @@ proc epoll_ctl*(epfd: cint; op: cint; fd: cint | SocketHandle; event: ptr EpollE importc: "epoll_ctl", header: "".} ## Manipulate an epoll instance "epfd". Returns `0` in case of success, ## `-1` in case of error (the "errno" variable will contain the specific error code). - ## + ## ## The "op" parameter is one of the `EPOLL_CTL_*` ## constants defined above. The "fd" parameter is the target of the ## operation. The "event" parameter describes which events the caller diff --git a/lib/posix/inotify.nim b/lib/posix/inotify.nim index db698c59ce..7bc1504e5c 100644 --- a/lib/posix/inotify.nim +++ b/lib/posix/inotify.nim @@ -7,11 +7,15 @@ # distribution, for details about the copyright. # +when defined(nimPreviewSlimSystem): + import std/syncio + # Get the platform-dependent flags. # Structure describing an inotify event. type InotifyEvent* {.pure, final, importc: "struct inotify_event", - header: "".} = object ## An Inotify event. + header: "", + completeStruct.} = object ## An Inotify event. wd* {.importc: "wd".}: FileHandle ## Watch descriptor. mask* {.importc: "mask".}: uint32 ## Watch mask. cookie* {.importc: "cookie".}: uint32 ## Cookie to synchronize two events. @@ -75,11 +79,11 @@ proc inotify_rm_watch*(fd: cint; wd: cint): cint {.cdecl, iterator inotify_events*(evs: pointer, n: int): ptr InotifyEvent = ## Abstract the packed buffer interface to yield event object pointers. - ## - ## .. code-block:: Nim + ## ```Nim ## var evs = newSeq[byte](8192) # Already did inotify_init+add_watch ## while (let n = read(fd, evs[0].addr, 8192); n) > 0: # read forever ## for e in inotify_events(evs[0].addr, n): echo e[].len # echo name lens + ## ``` var ev: ptr InotifyEvent = cast[ptr InotifyEvent](evs) var n = n while n > 0: diff --git a/lib/posix/kqueue.nim b/lib/posix/kqueue.nim index c83ae33ea3..2450cdb424 100644 --- a/lib/posix/kqueue.nim +++ b/lib/posix/kqueue.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -from posix import Timespec +from std/posix import Timespec when defined(macosx) or defined(freebsd) or defined(openbsd) or defined(dragonfly): diff --git a/lib/posix/linux.nim b/lib/posix/linux.nim index 5ce9bf2fb7..29fd4288d0 100644 --- a/lib/posix/linux.nim +++ b/lib/posix/linux.nim @@ -1,4 +1,4 @@ -import posix +import std/posix ## Flags of `clone` syscall. ## See `clone syscall manual @@ -29,9 +29,9 @@ const CLONE_NEWPID* = 0x20000000'i32 CLONE_NEWNET* = 0x40000000'i32 CLONE_IO* = 0x80000000'i32 - CLONE_STOPPED* {.deprecated.} = 0x02000000'i32 -# fn should be of type proc (a2: pointer): void {.cdecl.} + +# fn should be of type proc (a2: pointer) {.cdecl.} proc clone*(fn: pointer; child_stack: pointer; flags: cint; arg: pointer; ptid: ptr Pid; tls: pointer; ctid: ptr Pid): cint {.importc, header: "".} diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 520d0caa0f..fbe945df33 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -37,6 +37,9 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} +when defined(nimPreviewSlimSystem): + import std/syncio + # TODO these constants don't seem to be fetched from a header file for unknown # platforms - where do they come from and why are they here? when false: @@ -151,11 +154,13 @@ proc htons*(a1: uint16): uint16 {.importc, header: "".} proc ntohl*(a1: uint32): uint32 {.importc, header: "".} proc ntohs*(a1: uint16): uint16 {.importc, header: "".} -proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} -proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} -proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {. +when not defined(zephyr): + proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} + proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} + +proc inet_ntop*(a1: cint, a2: pointer | ptr InAddr | ptr In6Addr, a3: cstring, a4: int32): cstring {. importc:"(char *)$1", header: "".} -proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {. +proc inet_pton*(a1: cint, a2: cstring, a3: pointer | ptr InAddr | ptr In6Addr): cint {. importc, header: "".} var @@ -183,11 +188,36 @@ proc dlsym*(a1: pointer, a2: cstring): pointer {.importc, header: "", s proc creat*(a1: cstring, a2: Mode): cint {.importc, header: "", sideEffect.} proc fcntl*(a1: cint | SocketHandle, a2: cint): cint {.varargs, importc, header: "", sideEffect.} -proc open*(a1: cstring, a2: cint): cint {.varargs, importc, header: "", sideEffect.} +proc openImpl(a1: cstring, a2: cint): cint {.varargs, importc: "open", header: "", sideEffect.} +proc open*(a1: cstring, a2: cint, mode: Mode | cint = 0.Mode): cint {.inline.} = + # prevents bug #17888 + openImpl(a1, a2, mode) + proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. importc, header: "".} -proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. - importc, header: "".} + +proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} +when defined(osx): # 2001 POSIX evidently does not concern Apple + type FStore {.importc: "fstore_t", header: "", bycopy.} = object + fst_flags: uint32 ## IN: flags word + fst_posmode: cint ## IN: indicates offset field + fst_offset, ## IN: start of the region + fst_length, ## IN: size of the region + fst_bytesalloc: Off ## OUT: number of bytes allocated + var F_PEOFPOSMODE {.importc, header: "".}: cint + var F_ALLOCATEALL {.importc, header: "".}: uint32 + var F_PREALLOCATE {.importc, header: "".}: cint + proc posix_fallocate*(a1: cint, a2, a3: Off): cint = + var fst = FStore(fst_flags: F_ALLOCATEALL, fst_posmode: F_PEOFPOSMODE, + fst_offset: a2, fst_length: a3) + # Must also call ftruncate to match what POSIX does. Unlike posix_fallocate, + # this can shrink files. Could guard w/getFileSize, but caller likely knows + # present size & has no good reason to call this unless it is growing. + if fcntl(a1, F_PREALLOCATE, fst.addr) != cint(-1): ftruncate(a1, a2 + a3) + else: cint(-1) +else: + proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. + importc, header: "".} when not defined(haiku) and not defined(openbsd): proc fmtmsg*(a1: int, a2: cstring, a3: cint, @@ -240,26 +270,52 @@ proc setlocale*(a1: cint, a2: cstring): cstring {. proc strfmon*(a1: cstring, a2: int, a3: cstring): int {.varargs, importc, header: "".} -when not defined(nintendoswitch): - proc mq_close*(a1: Mqd): cint {.importc, header: "".} - proc mq_getattr*(a1: Mqd, a2: ptr MqAttr): cint {. - importc, header: "".} - proc mq_notify*(a1: Mqd, a2: ptr SigEvent): cint {. - importc, header: "".} - proc mq_open*(a1: cstring, a2: cint): Mqd {. - varargs, importc, header: "".} - proc mq_receive*(a1: Mqd, a2: cstring, a3: int, a4: var int): int {. - importc, header: "".} - proc mq_send*(a1: Mqd, a2: cstring, a3: int, a4: int): cint {. - importc, header: "".} - proc mq_setattr*(a1: Mqd, a2, a3: ptr MqAttr): cint {. +when not (defined(nintendoswitch) or defined(macos) or defined(macosx)): + proc mq_notify*(mqdes: Mqd, event: ptr SigEvent): cint {. importc, header: "".} - proc mq_timedreceive*(a1: Mqd, a2: cstring, a3: int, a4: int, - a5: ptr Timespec): int {.importc, header: "".} - proc mq_timedsend*(a1: Mqd, a2: cstring, a3: int, a4: int, - a5: ptr Timespec): cint {.importc, header: "".} - proc mq_unlink*(a1: cstring): cint {.importc, header: "".} + proc mq_open*(name: cstring, flags: cint): Mqd {. + varargs, importc, header: "".} + + proc mq_close*(mqdes: Mqd): cint {.importc, header: "".} + + proc mq_receive*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: var cuint + ): int {.importc, header: "".} + + proc mq_timedreceive*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint, + timeout: ptr Timespec + ): int {.importc, header: "".} + + proc mq_send*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint + ): cint {.importc, header: "".} + + proc mq_timedsend*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint, + timeout: ptr Timespec + ): cint {.importc, header: "".} + + proc mq_getattr*(mqdes: Mqd, attribute: ptr MqAttr): cint {. + importc, header: "".} + + proc mq_setattr*(mqdes: Mqd, newAttribute, oldAttribute: ptr MqAttr): cint {. + importc, header: "".} + + proc mq_unlink*(mqdes: cstring): cint {.importc, header: "".} proc getpwnam*(a1: cstring): ptr Passwd {.importc, header: "".} @@ -447,7 +503,7 @@ proc pthread_spin_unlock*(a1: ptr Pthread_spinlock): cint {. proc pthread_testcancel*() {.importc, header: "".} -proc exitnow*(code: int): void {.importc: "_exit", header: "".} +proc exitnow*(code: int) {.importc: "_exit", header: "".} proc access*(a1: cstring, a2: cint): cint {.importc, header: "".} proc alarm*(a1: cint): cint {.importc, header: "".} proc chdir*(a1: cstring): cint {.importc, header: "".} @@ -476,7 +532,6 @@ proc fpathconf*(a1, a2: cint): int {.importc, header: "".} proc fsync*(a1: cint): cint {.importc, header: "".} ## synchronize a file's buffer cache to the storage device -proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "", sideEffect.} proc getuid*(): Uid {.importc, header: "", sideEffect.} ## returns the real user ID of the calling process @@ -529,7 +584,8 @@ proc pread*(a1: cint, a2: pointer, a3: int, a4: Off): int {. proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {. importc, header: "".} proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "".} -proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "".} +when not defined(nintendoswitch): + proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "".} proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", header: "", varargs, tags: [WriteIOEffect].} ## A system call for device-specific input/output operations and other @@ -548,7 +604,10 @@ proc setsid*(): Pid {.importc, header: "".} proc setuid*(a1: Uid): cint {.importc, header: "".} proc sleep*(a1: cint): cint {.importc, header: "".} proc swab*(a1, a2: pointer, a3: int) {.importc, header: "".} -proc symlink*(a1, a2: cstring): cint {.importc, header: "".} +when not defined(nintendoswitch): + proc symlink*(a1, a2: cstring): cint {.importc, header: "".} +else: + proc symlink*(a1, a2: cstring): cint = -1 proc sync*() {.importc, header: "".} proc sysconf*(a1: cint): int {.importc, header: "".} proc tcgetpgrp*(a1: cint): Pid {.importc, header: "".} @@ -766,6 +825,13 @@ else: proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, a3: var Timespec): cint {.importc, header: "".} +when defined(sunos) or defined(solaris): + # The following compile time flag is needed on Illumos/Solaris to use the POSIX + # `sigwait` implementation. See the documentation here: + # https://docs.oracle.com/cd/E19455-01/806-5257/6je9h033k/index.html + # https://www.illumos.org/man/2/sigwait + {.passc: "-D_POSIX_PTHREAD_SEMANTICS".} + proc sigwait*(a1: var Sigset, a2: var cint): cint {. importc, header: "".} proc sigwaitinfo*(a1: var Sigset, a2: var SigInfo): cint {. @@ -878,19 +944,9 @@ proc CMSG_NXTHDR*(mhdr: ptr Tmsghdr, cmsg: ptr Tcmsghdr): ptr Tcmsghdr {. proc CMSG_FIRSTHDR*(mhdr: ptr Tmsghdr): ptr Tcmsghdr {. importc, header: "".} -{.push warning[deprecated]: off.} -proc CMSG_SPACE*(len: csize): csize {. - importc, header: "", deprecated: "argument `len` should be of type `csize_t`".} -{.pop.} - proc CMSG_SPACE*(len: csize_t): csize_t {. importc, header: "".} -{.push warning[deprecated]: off.} -proc CMSG_LEN*(len: csize): csize {. - importc, header: "", deprecated: "argument `len` should be of type `csize_t`".} -{.pop.} - proc CMSG_LEN*(len: csize_t): csize_t {. importc, header: "".} @@ -902,7 +958,7 @@ proc `==`*(x, y: SocketHandle): bool {.borrow.} proc accept*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen): SocketHandle {. importc, header: "", sideEffect.} -when defined(linux) or defined(bsd): +when defined(linux) or defined(bsd) or defined(nuttx): proc accept4*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen, flags: cint): SocketHandle {.importc, header: "".} @@ -1000,7 +1056,7 @@ proc endhostent*() {.importc, header: "".} proc endnetent*() {.importc, header: "".} proc endprotoent*() {.importc, header: "".} proc endservent*() {.importc, header: "".} -proc freeaddrinfo*(a1: ptr AddrInfo) {.importc, header: "".} +proc freeAddrInfo*(a1: ptr AddrInfo) {.importc: "freeaddrinfo", header: "".} proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "".} @@ -1087,11 +1143,11 @@ template onSignal*(signals: varargs[cint], body: untyped) = ## scope. ## ## Example: - ## - ## .. code-block:: + ## ```Nim ## from std/posix import SIGINT, SIGTERM, onSignal ## onSignal(SIGINT, SIGTERM): ## echo "bye from signal ", sig + ## ``` for s in signals: handle_signal(s, @@ -1108,12 +1164,12 @@ type ## The getrlimit() and setrlimit() system calls get and set resource limits respectively. ## Each resource has an associated soft and hard limit, as defined by the RLimit structure -proc setrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "setrlimit",header: "".} +proc setrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "setrlimit", header: "".} ## The setrlimit() system calls sets resource limits. -proc getrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "getrlimit",header: "".} +proc getrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "getrlimit", header: "".} ## The getrlimit() system call gets resource limits. when defined(nimHasStyleChecks): diff --git a/lib/posix/posix_freertos_consts.nim b/lib/posix/posix_freertos_consts.nim index efe87aacfc..0f0fc0aae4 100644 --- a/lib/posix/posix_freertos_consts.nim +++ b/lib/posix/posix_freertos_consts.nim @@ -363,6 +363,8 @@ var SEM_FAILED* {.importc: "SEM_FAILED", header: "".}: pointer # # var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "".}: cint +var FD_MAX* {.importc: "CONFIG_LWIP_MAX_SOCKETS", header: "".}: cint + # var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint @@ -498,3 +500,7 @@ const F_TEST* = cint(3) const F_TLOCK* = cint(2) const F_ULOCK* = cint(0) +# +const SEEK_SET* = cint(0) +const SEEK_CUR* = cint(1) +const SEEK_END* = cint(2) diff --git a/lib/posix/posix_haiku.nim b/lib/posix/posix_haiku.nim index d626b21060..32a6d24e26 100644 --- a/lib/posix/posix_haiku.nim +++ b/lib/posix/posix_haiku.nim @@ -304,7 +304,7 @@ type Stack* {.importc: "stack_t", header: "", final, pure.} = object ## stack_t ss_sp*: pointer ## Stack base or pointer. - ss_size*: csize ## Stack size. + ss_size*: csize_t ## Stack size. ss_flags*: cint ## Flags. SigInfo* {.importc: "siginfo_t", @@ -404,7 +404,7 @@ type IOVec* {.importc: "struct iovec", pure, final, header: "".} = object ## struct iovec iov_base*: pointer ## Base address of a memory region for input or output. - iov_len*: csize ## The size of the memory pointed to by iov_base. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. Tmsghdr* {.importc: "struct msghdr", pure, final, header: "".} = object ## struct msghdr diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim index 7f6a589f08..8d11c507d7 100644 --- a/lib/posix/posix_linux_amd64.nim +++ b/lib/posix/posix_linux_amd64.nim @@ -47,7 +47,7 @@ type d_ino*: Ino d_off*: Off d_reclen*: cushort - d_type*: int8 # cuchar really! + d_type*: int8 # uint8 really! d_name*: array[256, cchar] Tflock* {.importc: "struct flock", final, pure, @@ -168,7 +168,7 @@ type Pthread_key* {.importc: "pthread_key_t", header: "".} = cuint Pthread_mutex* {.importc: "pthread_mutex_t", header: "", pure, final.} = object - abi: array[48 div sizeof(clong), clong] + abi: array[40 div sizeof(clong), clong] Pthread_mutexattr* {.importc: "pthread_mutexattr_t", header: "", pure, final.} = object abi: array[4 div sizeof(cint), cint] @@ -309,7 +309,7 @@ type sa_mask*: Sigset ## Set of signals to be blocked during execution of ## the signal handling function. sa_flags*: cint ## Special flags. - sa_sigaction*: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} + sa_restorer: proc() {.noconv.} ## not intended for application use. Stack* {.importc: "stack_t", header: "", final, pure.} = object ## stack_t @@ -325,17 +325,23 @@ type SigInfo* {.importc: "siginfo_t", header: "", final, pure.} = object ## siginfo_t si_signo*: cint ## Signal number. - si_code*: cint ## Signal code. si_errno*: cint ## If non-zero, an errno value associated with ## this signal, as defined in . + si_code*: cint ## Signal code. si_pid*: Pid ## Sending process ID. si_uid*: Uid ## Real user ID of sending process. si_addr*: pointer ## Address of faulting instruction. si_status*: cint ## Exit value or signal. si_band*: int ## Band event for SIGPOLL. si_value*: SigVal ## Signal value. - pad {.importc: "_pad"}: array[128 - 56, uint8] + pad {.importc: "_pad".}: array[128 - 56, uint8] +template sa_sigaction*(v: Sigaction): proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} = + cast[proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.}](v.sa_handler) +proc `sa_sigaction=`*(v: var Sigaction, x: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.}) = + v.sa_handler = cast[proc (x: cint) {.noconv.}](x) + +type Nl_item* {.importc: "nl_item", header: "".} = cint Nl_catd* {.importc: "nl_catd", header: "".} = pointer @@ -428,8 +434,8 @@ type header: "", pure, final.} = object ## struct sockaddr_storage ss_family*: TSa_Family ## Address family. - ss_padding: array[128 - sizeof(cshort) - sizeof(culong), char] - ss_align: clong + ss_padding {.importc: "__ss_padding".}: array[128 - sizeof(cshort) - sizeof(culong), char] + ss_align {.importc: "__ss_align".}: clong Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "".} = object ## struct if_nameindex diff --git a/lib/posix/posix_linux_amd64_consts.nim b/lib/posix/posix_linux_amd64_consts.nim index f3230f71d6..fbe8d0666d 100644 --- a/lib/posix/posix_linux_amd64_consts.nim +++ b/lib/posix/posix_linux_amd64_consts.nim @@ -453,6 +453,7 @@ const MAP_POPULATE* = cint(32768) # const RLIMIT_NOFILE* = cint(7) +const RLIMIT_STACK* = cint(3) # const FD_SETSIZE* = cint(1024) @@ -464,6 +465,7 @@ const MSG_EOR* = cint(128) const MSG_OOB* = cint(1) const SCM_RIGHTS* = cint(1) const SO_ACCEPTCONN* = cint(30) +const SO_BINDTODEVICE* = cint(25) const SO_BROADCAST* = cint(6) const SO_DEBUG* = cint(1) const SO_DONTROUTE* = cint(5) diff --git a/lib/posix/posix_macos_amd64.nim b/lib/posix/posix_macos_amd64.nim index 2e68af3308..d6b5834cb6 100644 --- a/lib/posix/posix_macos_amd64.nim +++ b/lib/posix/posix_macos_amd64.nim @@ -108,15 +108,6 @@ type p_sign_posn*: char thousands_sep*: cstring - Mqd* {.importc: "mqd_t", header: "", final, pure.} = object - MqAttr* {.importc: "struct mq_attr", - header: "", - final, pure.} = object ## message queue attribute - mq_flags*: int ## Message queue flags. - mq_maxmsg*: int ## Maximum number of messages. - mq_msgsize*: int ## Maximum message size. - mq_curmsgs*: int ## Number of messages currently queued. - Passwd* {.importc: "struct passwd", header: "", final, pure.} = object ## struct passwd pw_name*: cstring ## User's login name. @@ -375,6 +366,18 @@ when hasSpawnH: Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", header: "", final, pure.} = object + +when not defined(macos) and not defined(macosx): # freebsd + type + Mqd* {.importc: "mqd_t", header: "", final, pure.} = object + MqAttr* {.importc: "struct mq_attr", + header: "", + final, pure.} = object ## message queue attribute + mq_flags*: int ## Message queue flags. + mq_maxmsg*: int ## Maximum number of messages. + mq_msgsize*: int ## Maximum message size. + mq_curmsgs*: int ## Number of messages currently queued. + when defined(linux): # from sys/un.h const Sockaddr_un_path_length* = 108 diff --git a/lib/posix/posix_nintendoswitch.nim b/lib/posix/posix_nintendoswitch.nim index 44e4314379..b66563695c 100644 --- a/lib/posix/posix_nintendoswitch.nim +++ b/lib/posix/posix_nintendoswitch.nim @@ -33,7 +33,7 @@ type Dirent* {.importc: "struct dirent", header: "", final, pure.} = object ## dirent_t struct d_ino*: Ino - d_type*: int8 # cuchar really! + d_type*: int8 # uint8 really! d_name*: array[256, cchar] Tflock* {.importc: "struct flock", final, pure, @@ -286,7 +286,7 @@ type header: "", final, pure.} = object ## stack_t ss_sp*: pointer ## Stack base or pointer. ss_flags*: cint ## Flags. - ss_size*: csize ## Stack size. + ss_size*: csize_t ## Stack size. SigInfo* {.importc: "siginfo_t", header: "", final, pure.} = object ## siginfo_t @@ -321,7 +321,7 @@ type aio_lio_opcode*: cint ## Operation to be performed. aio_reqprio*: cint ## Request priority offset. aio_buf*: pointer ## Location of buffer. - aio_nbytes*: csize ## Length of transfer. + aio_nbytes*: csize_t ## Length of transfer. aio_sigevent*: SigEvent ## Signal number and value. next_prio: pointer abs_prio: cint @@ -347,20 +347,20 @@ type SockAddr* {.importc: "struct sockaddr", header: "", pure, final.} = object ## struct sockaddr - sa_len: cuchar + sa_len: uint8 sa_family*: TSa_Family ## Address family. sa_data*: array[14, char] ## Socket address (variable-length data). Sockaddr_storage* {.importc: "struct sockaddr_storage", header: "", pure, final.} = object ## struct sockaddr_storage - ss_len: cuchar + ss_len: uint8 ss_family*: TSa_Family ## Address family. - ss_padding1: array[64 - sizeof(cuchar) - sizeof(cshort), char] + ss_padding1: array[64 - sizeof(uint8) - sizeof(cshort), char] ss_align: clonglong ss_padding2: array[ - 128 - sizeof(cuchar) - sizeof(cshort) - - (64 - sizeof(cuchar) - sizeof(cshort)) - 64, char] + 128 - sizeof(uint8) - sizeof(cshort) - + (64 - sizeof(uint8) - sizeof(cshort)) - 64, char] Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "".} = object ## struct if_nameindex @@ -378,15 +378,15 @@ type msg_name*: pointer ## Optional address. msg_namelen*: SockLen ## Size of address. msg_iov*: ptr IOVec ## Scatter/gather array. - msg_iovlen*: csize ## Members in msg_iov. + msg_iovlen*: csize_t ## Members in msg_iov. msg_control*: pointer ## Ancillary data; see below. - msg_controllen*: csize ## Ancillary data buffer len. + msg_controllen*: csize_t ## Ancillary data buffer len. msg_flags*: cint ## Flags on received message. Tcmsghdr* {.importc: "struct cmsghdr", pure, final, header: "".} = object ## struct cmsghdr - cmsg_len*: csize ## Data byte count, including the cmsghdr. + cmsg_len*: csize_t ## Data byte count, including the cmsghdr. cmsg_level*: cint ## Originating protocol. cmsg_type*: cint ## Protocol-specific type. diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 6584bfab2e..ea8731405d 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -10,7 +10,7 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} -when defined(freertos): +when defined(freertos) or defined(zephyr): const hasSpawnH = false # should exist for every Posix system nowadays hasAioH = false @@ -391,9 +391,33 @@ when hasSpawnH: header: "", final, pure.} = object when defined(linux): + const Sockaddr_max_length* = 255 # from sys/un.h const Sockaddr_un_path_length* = 108 +elif defined(zephyr): + when defined(net_ipv6): + const Sockaddr_max_length* = 24 + elif defined(net_raw): + const Sockaddr_max_length* = 20 + elif defined(net_ipv4): + const Sockaddr_max_length* = 8 + else: + const Sockaddr_max_length* = 255 # just for compilation purposes + + const Sockaddr_un_path_length* = Sockaddr_max_length + # Zephyr is heavily customizable so it's easy to get to a state + # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. + when defined(net_ipv4) or defined(net_ipv6) or defined(net_raw): + {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", + Sockaddr_max_length, + ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", + " Check that Nim and Zephyr IPv4/IPv6 settings match.", + " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} +elif defined(freertos) or defined(lwip): + const Sockaddr_max_length* = 14 + const Sockaddr_un_path_length* = 108 else: + const Sockaddr_max_length* = 255 # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html # this is >=92 const Sockaddr_un_path_length* = 92 @@ -408,48 +432,50 @@ when defined(lwip): pure, final.} = object ## struct sockaddr sa_len*: uint8 ## Address family. sa_family*: TSa_Family ## Address family. - sa_data*: array[0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..Sockaddr_max_length-sizeof(uint8)-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + s2_len*: uint8 ## Address family. + ss_family*: TSa_Family ## Address family. + s2_data1*: array[2, char] ## Address family. + s2_data2*: array[3, uint32] ## Address family. + when defined(lwip6) or defined(net_ipv6): + s2_data3*: array[3, uint32] ## Address family. +elif defined(zephyr): + type + SockAddr* {.importc: "struct sockaddr", header: "", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr) == ", sizeof(Sockaddr), ",\"struct size mismatch\" );"].} + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr_storage) == ", sizeof(Sockaddr_storage), ",\"struct size mismatch\" );"].} else: type SockAddr* {.importc: "struct sockaddr", header: "", pure, final.} = object ## struct sockaddr sa_family*: TSa_Family ## Address family. - sa_data*: array[0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. type Sockaddr_un* {.importc: "struct sockaddr_un", header: "", pure, final.} = object ## struct sockaddr_un sun_family*: TSa_Family ## Address family. - sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + sun_path*: array[0..Sockaddr_un_path_length-sizeof(TSa_Family), char] ## Socket path -when defined(lwip): - when not defined(lwip6): - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - s2_len*: uint8 ## Address family. - ss_family*: TSa_Family ## Address family. - s2_data1*: array[2, char] ## Address family. - s2_data2*: array[3, uint32] ## Address family. - else: - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - s2_len*: uint8 ## Address family. - ss_family*: TSa_Family ## Address family. - s2_data1*: array[2, char] ## Address family. - s2_data2*: array[3, uint32] ## Address family. - s2_data3*: array[3, uint32] ## Address family. -else: - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - ss_family*: TSa_Family ## Address family. - type Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "".} = object ## struct if_nameindex @@ -494,6 +520,7 @@ type header: "".} = object ## struct in_addr s_addr*: InAddrScalar + # TODO: Fixme for FreeRTOS/LwIP, these are incorrect Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, header: "".} = object ## struct sockaddr_in sin_family*: TSa_Family ## AF_INET. @@ -577,7 +604,12 @@ when not defined(lwip): events*: cshort ## The input event flags (see below). revents*: cshort ## The output event flags (see below). - Tnfds* {.importc: "nfds_t", header: "".} = cint + when defined(zephyr): + type + Tnfds* = distinct cint + else: + type + Tnfds* {.importc: "nfds_t", header: "".} = cint var errno* {.importc, header: "".}: cint ## error variable @@ -608,10 +640,13 @@ when defined(linux) or defined(nimdoc): ## or UDP packets. (Requires Linux kernel > 3.9) else: const SO_REUSEPORT* = cint(15) +elif defined(nuttx): + # Not supported, use SO_REUSEADDR to avoid compilation errors. + var SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "".}: cint else: var SO_REUSEPORT* {.importc, header: "".}: cint -when defined(linux) or defined(bsd): +when defined(linux) or defined(bsd) or defined(nuttx): var SOCK_CLOEXEC* {.importc, header: "".}: cint when defined(macosx): @@ -625,7 +660,7 @@ elif defined(solaris): # Solaris doesn't have MSG_NOSIGNAL const MSG_NOSIGNAL* = 0'i32 -elif defined(freertos) or defined(lwip): +elif defined(zephyr) or defined(freertos) or defined(lwip): # LwIP/FreeRTOS doesn't have MSG_NOSIGNAL const MSG_NOSIGNAL* = 0x20'i32 @@ -640,14 +675,14 @@ when defined(haiku): when hasSpawnH: when defined(linux): - # better be safe than sorry; Linux has this flag, macosx doesn't, don't - # know about the other OSes + # better be safe than sorry; Linux has this flag, macosx and NuttX don't, + # don't know about the other OSes - # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we + # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we # can't get the magic number from spawn.h const POSIX_SPAWN_USEVFORK* = cint(0x40) else: - # macosx lacks this, so we define the constant to be 0 to not affect + # macosx and NuttX lack this, so we define the constant to be 0 to not affect # OR'ing of flags: const POSIX_SPAWN_USEVFORK* = cint(0) diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index f43407b406..d346b4150d 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -99,7 +99,10 @@ var EXDEV* {.importc: "EXDEV", header: "".}: cint # var F_DUPFD* {.importc: "F_DUPFD", header: "".}: cint -var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "".}: cint +when defined(nuttx): + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD_CLOEXEC", header: "".}: cint +else: + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "".}: cint var F_GETFD* {.importc: "F_GETFD", header: "".}: cint var F_SETFD* {.importc: "F_SETFD", header: "".}: cint var F_GETFL* {.importc: "F_GETFL", header: "".}: cint @@ -127,6 +130,11 @@ var O_RDONLY* {.importc: "O_RDONLY", header: "".}: cint var O_RDWR* {.importc: "O_RDWR", header: "".}: cint var O_WRONLY* {.importc: "O_WRONLY", header: "".}: cint var O_CLOEXEC* {.importc: "O_CLOEXEC", header: "".}: cint +when defined(nuttx): + var O_DIRECT* {.importc: "O_DIRECT", header: "".}: cint + var O_PATH* {.importc: "O_PATH", header: "".}: cint + var O_NOATIME* {.importc: "O_NOATIME", header: "".}: cint + var O_TMPFILE* {.importc: "O_TMPFILE", header: "".}: cint var POSIX_FADV_NORMAL* {.importc: "POSIX_FADV_NORMAL", header: "".}: cint var POSIX_FADV_SEQUENTIAL* {.importc: "POSIX_FADV_SEQUENTIAL", header: "".}: cint var POSIX_FADV_RANDOM* {.importc: "POSIX_FADV_RANDOM", header: "".}: cint @@ -459,9 +467,13 @@ var POSIX_TYPED_MEM_MAP_ALLOCATABLE* {.importc: "POSIX_TYPED_MEM_MAP_ALLOCATABLE # var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "".}: cint +var RLIMIT_STACK* {.importc: "RLIMIT_STACK", header: "".}: cint # var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint +when defined(zephyr): + # Zephyr specific hardcoded value + var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "".}: cint # var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "".}: cint @@ -470,6 +482,7 @@ var MSG_EOR* {.importc: "MSG_EOR", header: "".}: cint var MSG_OOB* {.importc: "MSG_OOB", header: "".}: cint var SCM_RIGHTS* {.importc: "SCM_RIGHTS", header: "".}: cint var SO_ACCEPTCONN* {.importc: "SO_ACCEPTCONN", header: "".}: cint +var SO_BINDTODEVICE* {.importc: "SO_BINDTODEVICE", header: "".}: cint var SO_BROADCAST* {.importc: "SO_BROADCAST", header: "".}: cint var SO_DEBUG* {.importc: "SO_DEBUG", header: "".}: cint var SO_DONTROUTE* {.importc: "SO_DONTROUTE", header: "".}: cint @@ -487,10 +500,15 @@ var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "".}: cint var SO_TYPE* {.importc: "SO_TYPE", header: "".}: cint var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "".}: cint var SOCK_RAW* {.importc: "SOCK_RAW", header: "".}: cint -var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "".}: cint +when defined(zephyr): + const SOCK_SEQPACKET* = cint(5) + var SOMAXCONN* {.importc: "CONFIG_NET_SOCKETS_POLL_MAX", header: "".}: cint +else: + var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "".}: cint + var SOMAXCONN* {.importc: "SOMAXCONN", header: "".}: cint + var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "".}: cint var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "".}: cint -var SOMAXCONN* {.importc: "SOMAXCONN", header: "".}: cint var MSG_PEEK* {.importc: "MSG_PEEK", header: "".}: cint var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "".}: cint var MSG_WAITALL* {.importc: "MSG_WAITALL", header: "".}: cint @@ -734,3 +752,6 @@ var SEEK_SET* {.importc: "SEEK_SET", header: "".}: cint var SEEK_CUR* {.importc: "SEEK_CUR", header: "".}: cint var SEEK_END* {.importc: "SEEK_END", header: "".}: cint +# +when defined(nuttx): + var NEPOLL_MAX* {.importc: "CONFIG_FS_NEPOLL_DESCRIPTORS", header: "".}: cint diff --git a/lib/posix/posix_utils.nim b/lib/posix/posix_utils.nim index aeec73a451..0c668246fc 100644 --- a/lib/posix/posix_utils.nim +++ b/lib/posix/posix_utils.nim @@ -7,18 +7,21 @@ # ## A set of helpers for the POSIX module. -## Raw interfaces are in the other posix*.nim files. +## Raw interfaces are in the other ``posix*.nim`` files. # Where possible, contribute OS-independent procs in `os `_ instead. -import posix, parsecfg, os +import std/[posix, parsecfg, os] import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + type Uname* = object sysname*, nodename*, release*, version*, machine*: string template charArrayToString(input: typed): string = - $cstring(addr input) + $cast[cstring](addr input) proc uname*(): Uname = ## Provides system information in a `Uname` struct with sysname, nodename, @@ -31,7 +34,7 @@ proc uname*(): Uname = var u: Utsname if uname(u) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) result.sysname = charArrayToString u.sysname result.nodename = charArrayToString u.nodename @@ -40,44 +43,45 @@ proc uname*(): Uname = result.machine = charArrayToString u.machine proc fsync*(fd: int) = - ## synchronize a file's buffer cache to the storage device - if fsync(fd.cint) != 0: - raise newException(OSError, $strerror(errno)) + ## synchronize a file's buffer cache to the storage device + if fsync(fd.cint) != 0: + raiseOSError(OSErrorCode(errno)) proc stat*(path: string): Stat = ## Returns file status in a `Stat` structure if stat(path.cstring, result) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryLock*(a1: pointer, a2: int) = ## Locks pages starting from a1 for a1 bytes and prevent them from being swapped. if mlock(a1, a2) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryLockAll*(flags: int) = ## Locks all memory for the running process to prevent swapping. ## - ## example:: - ## + ## example: + ## ```nim ## memoryLockAll(MCL_CURRENT or MCL_FUTURE) + ## ``` if mlockall(flags.cint) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryUnlock*(a1: pointer, a2: int) = ## Unlock pages starting from a1 for a1 bytes and allow them to be swapped. if munlock(a1, a2) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryUnlockAll*() = ## Unlocks all memory for the running process to allow swapping. if munlockall() != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc sendSignal*(pid: Pid, signal: int) = ## Sends a signal to a running process by calling `kill`. ## Raise exception in case of failure e.g. process not running. if kill(pid, signal.cint) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc mkstemp*(prefix: string, suffix=""): (string, File) = ## Creates a unique temporary file from a prefix string. A six-character string @@ -86,7 +90,7 @@ proc mkstemp*(prefix: string, suffix=""): (string, File) = ## Returns the filename and a file opened in r/w mode. var tmpl = cstring(prefix & "XXXXXX" & suffix) let fd = - if len(suffix)==0: + if len(suffix) == 0: when declared(mkostemp): mkostemp(tmpl, O_CLOEXEC) else: @@ -99,14 +103,14 @@ proc mkstemp*(prefix: string, suffix=""): (string, File) = var f: File if open(f, fd, fmReadWrite): return ($tmpl, f) - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc mkdtemp*(prefix: string): string = ## Creates a unique temporary directory from a prefix string. Adds a six chars suffix. ## The directory is created with permissions 0700. Returns the directory name. var tmpl = cstring(prefix & "XXXXXX") if mkdtemp(tmpl) == nil: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) return $tmpl proc osReleaseFile*(): Config {.since: (1, 5).} = diff --git a/lib/posix/termios.nim b/lib/posix/termios.nim index b03ce74f3a..7fb6bb81c6 100644 --- a/lib/posix/termios.nim +++ b/lib/posix/termios.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -import posix +import std/posix type Speed* = cuint @@ -237,8 +237,11 @@ proc tcFlow*(fd: cint; action: cint): cint {.importc: "tcflow", header: "".} # Get process group ID for session leader for controlling terminal FD. -# Window size ioctl. Should work on on any Unix that xterm has been ported to. -var TIOCGWINSZ*{.importc, header: "".}: culong +# Window size ioctl. Solaris based systems have an uncommen place for this. +when defined(solaris) or defined(sunos): + var TIOCGWINSZ*{.importc, header: "".}: culong +else: + var TIOCGWINSZ*{.importc, header: "".}: culong when defined(nimHasStyleChecks): {.push styleChecks: off.} diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index a32df65bb4..b12ed7cddb 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -42,6 +42,12 @@ runnableExamples: ## * `sequtils module`_ for working with the built-in seq type ## * `tables module`_ for sorting tables +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + + type SortOrder* = enum Descending, Ascending @@ -129,43 +135,29 @@ proc reverse*[T](a: var openArray[T]) = # the max is needed, since a.high is -1 if a is empty reverse(a, 0, max(0, a.high)) -proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] = - ## Returns the reverse of the slice `a[first..last]`. - ## - ## If an invalid range is passed, it raises `IndexDefect`. +proc reversed*[T](a: openArray[T]): seq[T] {.inline.} = + ## Returns the elements of `a` in reverse order. ## ## **See also:** - ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice ## * `reverse proc<#reverse,openArray[T]>`_ runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = a.reversed(1, 3) - assert b == @[4, 3, 2] - assert last >= first - 1 - var i = last - first - var x = first.int - result = newSeq[T](i + 1) - while i >= 0: - result[i] = a[x] - dec(i) - inc(x) + assert [10, 11, 12].reversed == @[12, 11, 10] + assert seq[string].default.reversed == @[] + let n = a.len + result.setLen(n) + for i in 0..`_ reverse a slice - ## * `reverse proc<#reverse,openArray[T]>`_ - runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = reversed(a) - assert b == @[6, 5, 4, 3, 2, 1] - reversed(a, 0, a.high) +proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] + {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} = + reversed(toOpenArray(a, first, last)) + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} proc binarySearch*[T, K](a: openArray[T], key: K, - cmp: proc (x: T, y: K): int {.closure.}): int = + cmp: proc (x: T, y: K): int {.closure.}): int {.effectsOf: cmp.} = ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. ## Assumes that `a` is sorted according to `cmp`. ## @@ -227,7 +219,7 @@ const onlySafeCode = true proc lowerBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is not less than ## (i.e. greater or equal to) `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -277,7 +269,7 @@ proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) ## * `upperBound proc<#upperBound,openArray[T],T>`_ proc upperBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is greater than ## `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -334,12 +326,12 @@ template `<-`(a, b) = else: copyMem(addr(a), addr(b), sizeof(T)) -proc merge[T](a, b: var openArray[T], lo, m, hi: int, - cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = +proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, + cmp: proc (x, y: T): int {.closure.}, order: SortOrder) {.effectsOf: cmp.} = # Optimization: If max(left) <= min(right) there is nothing to do! # 1 2 3 4 ## 5 6 7 8 # -> O(n) for sorted arrays. - # On random data this saves up to 40% of merge calls. + # On random data this saves up to 40% of mergeAlt calls. if cmp(a[m], a[m+1]) * order <= 0: return var j = lo # copy a[j..m] into b: @@ -375,7 +367,7 @@ proc merge[T](a, b: var openArray[T], lo, m, hi: int, func sort*[T](a: var openArray[T], cmp: proc (x, y: T): int {.closure.}, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Default Nim sort (an implementation of merge sort). The sorting ## is guaranteed to be stable (that is, equal elements stay in the same order) ## and the worst case is guaranteed to be O(n log n). @@ -387,22 +379,22 @@ func sort*[T](a: var openArray[T], ## `cmp`, you may use `system.cmp` or instead call the overloaded ## version of `sort`, which uses `system.cmp`. ## - ## .. code-block:: nim - ## - ## sort(myIntArray, system.cmp[int]) - ## # do not use cmp[string] here as we want to use the specialized - ## # overload: - ## sort(myStrArray, system.cmp) + ## ```nim + ## sort(myIntArray, system.cmp[int]) + ## # do not use cmp[string] here as we want to use the specialized + ## # overload: + ## sort(myStrArray, system.cmp) + ## ``` ## ## You can inline adhoc comparison procs with the `do notation - ## `_. Example: - ## - ## .. code-block:: nim + ## `_. Example: ## + ## ```nim ## people.sort do (x, y: Person) -> int: ## result = cmp(x.surname, y.surname) ## if result == 0: ## result = cmp(x.name, y.name) + ## ``` ## ## **See also:** ## * `sort proc<#sort,openArray[T]>`_ @@ -422,7 +414,7 @@ func sort*[T](a: var openArray[T], while s < n: var m = n-1-s while m >= 0: - merge(a, b, max(m-s+1, 0), m, m+s, cmp, order) + mergeAlt(a, b, max(m-s+1, 0), m, m+s, cmp, order) dec(m, s*2) s = s*2 @@ -437,7 +429,7 @@ proc sort*[T](a: var openArray[T], order = SortOrder.Ascending) = sort[T](a, ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): seq[T] = + order = SortOrder.Ascending): seq[T] {.effectsOf: cmp.} = ## Returns `a` sorted by `cmp` in the specified `order`. ## ## **See also:** @@ -514,7 +506,7 @@ template sortedByIt*(seq1, op: untyped): untyped = func isSorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): bool = + order = SortOrder.Ascending): bool {.effectsOf: cmp.} = ## Checks to see whether `a` is already sorted in `order` ## using `cmp` for the comparison. The parameters are identical ## to `sort`. Requires O(n) time. @@ -559,11 +551,103 @@ proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = assert isSorted(e) == false isSorted(a, system.cmp[T], order) +proc merge*[T]( + result: var seq[T], + x, y: openArray[T], cmp: proc(x, y: T): int {.closure.} +) {.since: (1, 5, 1), effectsOf: cmp.} = + ## Merges two sorted `openArray`. `x` and `y` are assumed to be sorted. + ## If you do not wish to provide your own `cmp`, + ## you may use `system.cmp` or instead call the overloaded + ## version of `merge`, which uses `system.cmp`. + ## + ## .. note:: The original data of `result` is not cleared, + ## new data is appended to `result`. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T]>`_ + runnableExamples: + let x = @[1, 3, 6] + let y = @[2, 3, 4] + + block: + var merged = @[7] # new data is appended to merged sequence + merged.merge(x, y, system.cmp[int]) + assert merged == @[7, 1, 2, 3, 3, 4, 6] + + block: + var merged = @[7] # if you only want new data, clear merged sequence first + merged.setLen(0) + merged.merge(x, y, system.cmp[int]) + assert merged.isSorted + assert merged == @[1, 2, 3, 3, 4, 6] + + import std/sugar + + var res: seq[(int, int)] + res.merge([(1, 1)], [(1, 2)], (a, b) => a[0] - b[0]) + assert res == @[(1, 1), (1, 2)] + + assert seq[int].default.dup(merge([1, 3], [2, 4])) == @[1, 2, 3, 4] + + let + sizeX = x.len + sizeY = y.len + oldLen = result.len + + result.setLen(oldLen + sizeX + sizeY) + + var + ix = 0 + iy = 0 + i = oldLen + + while true: + if ix == sizeX: + while iy < sizeY: + result[i] = y[iy] + inc i + inc iy + return + + if iy == sizeY: + while ix < sizeX: + result[i] = x[ix] + inc i + inc ix + return + + let itemX = x[ix] + let itemY = y[iy] + + if cmp(itemX, itemY) > 0: # to have a stable sort + result[i] = itemY + inc iy + else: + result[i] = itemX + inc ix + + inc i + +proc merge*[T](result: var seq[T], x, y: openArray[T]) {.inline, since: (1, 5, 1).} = + ## Shortcut version of `merge` that uses `system.cmp[T]` as the comparison function. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T],proc(T,T)>`_ + runnableExamples: + let x = [5, 10, 15, 20, 25] + let y = [50, 40, 30, 20, 10].sorted + + var merged: seq[int] + merged.merge(x, y) + assert merged.isSorted + assert merged == @[5, 10, 10, 15, 20, 20, 25, 30, 40, 50] + merge(result, x, y, system.cmp) + proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = ## Produces the Cartesian product of the array. ## Every element of the result is a combination of one element from each seq in `x`, ## with the ith element coming from `x[i]`. - ## + ## ## .. warning:: complexity may explode. runnableExamples: assert product(@[@[1], @[2]]) == @[@[1, 2]] @@ -741,10 +825,10 @@ proc rotateLeft*[T](arg: var openArray[T]; slice: HSlice[int, int]; ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. ## ## `slice` - ## The indices of the element range that should be rotated. + ## : The indices of the element range that should be rotated. ## ## `dist` - ## The distance in amount of elements that the data should be rotated. + ## : The distance in amount of elements that the data should be rotated. ## Can be negative, can be any number. ## ## **See also:** @@ -792,10 +876,10 @@ proc rotatedLeft*[T](arg: openArray[T]; slice: HSlice[int, int], ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. ## ## `slice` - ## The indices of the element range that should be rotated. + ## : The indices of the element range that should be rotated. ## ## `dist` - ## The distance in amount of elements that the data should be rotated. + ## : The distance in amount of elements that the data should be rotated. ## Can be negative, can be any number. ## ## **See also:** diff --git a/lib/pure/async.nim b/lib/pure/async.nim index 97b29f81d2..e4d8d41c3d 100644 --- a/lib/pure/async.nim +++ b/lib/pure/async.nim @@ -1,6 +1,9 @@ +## Exports [asyncmacro](asyncmacro.html) and [asyncfutures](asyncfutures.html) for native backends, +## and [asyncjs](asyncjs.html) on the JS backend. + when defined(js): - import asyncjs - export asyncjs + import std/asyncjs + export asyncjs else: - import asyncmacro, asyncfutures - export asyncmacro, asyncfutures + import std/[asyncmacro, asyncfutures] + export asyncmacro, asyncfutures diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 55a20270d9..126db7a7f2 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -41,13 +41,13 @@ ## requested amount of data is read **or** an exception occurs. ## ## Code to read some data from a socket may look something like this: -## -## .. code-block::nim -## var future = socket.recv(100) -## future.addCallback( -## proc () = -## echo(future.read) -## ) +## ```Nim +## var future = socket.recv(100) +## future.addCallback( +## proc () = +## echo(future.read) +## ) +## ``` ## ## All asynchronous functions returning a `Future` will not block. They ## will not however return immediately. An asynchronous function will have @@ -61,8 +61,8 @@ ## callback on this future which will be called once the future completes. ## All the callback does is write the data stored in the future to `stdout`. ## The `read` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. +## completes with an error for you (if it did, it will simply raise the +## error), if there is no error, however, it returns the value of the future. ## ## Asynchronous procedures ## ======================= @@ -98,40 +98,71 @@ ## `await`. The following section shows different ways that you can handle ## exceptions in async procs. ## +## .. caution:: +## Procedures marked {.async.} do not support mutable parameters such +## as `var int`. References such as `ref int` should be used instead. +## ## Handling Exceptions ## ------------------- ## -## The most reliable way to handle exceptions is to use `yield` on a future +## You can handle exceptions in the same way as in ordinary Nim code; +## by using the try statement: +## +## ```Nim +## try: +## let data = await sock.recv(100) +## echo("Received ", data) +## except: +## # Handle exception +## ``` +## +## An alternative approach to handling exceptions is to use `yield` on a future ## then check the future's `failed` property. For example: ## -## .. code-block:: Nim -## var future = sock.recv(100) -## yield future -## if future.failed: -## # Handle exception -## -## The `async` procedures also offer limited support for the try statement. -## -## .. code-block:: Nim -## try: -## let data = await sock.recv(100) -## echo("Received ", data) -## except: -## # Handle exception -## -## Unfortunately the semantics of the try statement may not always be correct, -## and occasionally the compilation may fail altogether. -## As such it is better to use the former style when possible. +## ```Nim +## var future = sock.recv(100) +## yield future +## if future.failed: +## # Handle exception +## ``` ## ## ## Discarding futures ## ================== ## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the `asyncCheck` procedure instead of the `discard` keyword. Note -## however that this does not wait for completion, and you should use -## `waitFor` for that purpose. +## Futures should **never** be discarded directly because they may contain +## errors. If you do not care for the result of a Future then you should use +## the `asyncCheck` procedure instead of the `discard` keyword. Note that this +## does not wait for completion, and you should use `waitFor` or `await` for that purpose. +## +## .. note:: `await` also checks if the future fails, so you can safely discard +## its result. +## +## Handling futures +## ================ +## +## There are many different operations that apply to a future. +## The three primary high-level operations are `asyncCheck`, +## `waitFor`, and `await`. +## +## * `asyncCheck`: Raises an exception if the future fails. It neither waits +## for the future to finish nor returns the result of the future. +## * `waitFor`: Polls the event loop and blocks the current thread until the +## future finishes. This is often used to call an async procedure from a +## synchronous context and should never be used in an `async` proc. +## * `await`: Pauses execution in the current async procedure until the future +## finishes. While the current procedure is paused, other async procedures will +## continue running. Should be used instead of `waitFor` in an async +## procedure. +## +## Here is a handy quick reference chart showing their high-level differences: +## ============== ===================== ======================= +## Procedure Context Blocking +## ============== ===================== ======================= +## `asyncCheck` non-async and async non-blocking +## `waitFor` non-async blocks current thread +## `await` async suspends current proc +## ============== ===================== ======================= ## ## Examples ## ======== @@ -165,19 +196,49 @@ ## ================ ## ## * The effect system (`raises: []`) does not work with async procedures. +## * Mutable parameters are not supported by async procedures. +## +## +## Multiple async backend support +## ============================== +## +## Thanks to its powerful macro support, Nim allows ``async``/``await`` to be +## implemented in libraries with only minimal support from the language - as +## such, multiple ``async`` libraries exist, including ``asyncdispatch`` and +## ``chronos``, and more may come to be developed in the future. +## +## Libraries built on top of async/await may wish to support multiple async +## backends - the best way to do so is to create separate modules for each backend +## that may be imported side-by-side. +## +## An alternative way is to select backend using a global compile flag - this +## method makes it difficult to compose applications that use both backends as may +## happen with transitive dependencies, but may be appropriate in some cases - +## libraries choosing this path should call the flag `asyncBackend`, allowing +## applications to choose the backend with `-d:asyncBackend=`. +## +## Known `async` backends include: +## +## * `-d:asyncBackend=none`: disable `async` support completely +## * `-d:asyncBackend=asyncdispatch`: https://nim-lang.org/docs/asyncdispatch.html +## * `-d:asyncBackend=chronos`: https://github.com/status-im/nim-chronos/ +## +## ``none`` can be used when a library supports both a synchronous and +## asynchronous API, to disable the latter. -import os, tables, strutils, times, heapqueue, options, asyncstreams -import options, math, std/monotimes -import asyncfutures except callSoon +import std/[os, tables, strutils, times, heapqueue, options, asyncstreams] +import std/[math, monotimes] +import std/asyncfutures except callSoon -import nativesockets, net, deques +import std/[nativesockets, net, deques] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] export Port, SocketFlag export asyncfutures except callSoon export asyncstreams -#{.injectStmt: newGcInvariant().} - # TODO: Check if yielded future is nil and throw a more meaningful exception type @@ -220,6 +281,8 @@ proc adjustTimeout( result = max(nextTimer.get(), 0) result = min(pollTimeout, result) +proc runOnce(timeout: int): bool {.gcsafe.} + proc callSoon*(cbproc: proc () {.gcsafe.}) {.gcsafe.} ## Schedule `cbproc` to be called as soon as possible. ## The callback is called when control returns to the event loop. @@ -239,7 +302,7 @@ template implementSetInheritable() {.dirty.} = fd.FileHandle.setInheritable(inheritable) when defined(windows) or defined(nimdoc): - import winlean, sets, hashes + import std/[winlean, sets, hashes] type CompletionKey = ULONG_PTR @@ -329,7 +392,7 @@ when defined(windows) or defined(nimdoc): let p = getGlobalDispatcher() p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, @@ -470,7 +533,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dealloc dataBuf.buf dataBuf.buf = nil @@ -488,7 +551,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -540,7 +603,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dataBuf.buf = nil ) @@ -556,7 +619,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -604,7 +667,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() # We don't deallocate `ol` here because even though this completed @@ -639,7 +702,7 @@ when defined(windows) or defined(nimdoc): if errcode == OSErrorCode(-1): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, @@ -649,7 +712,7 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() # We don't deallocate `ol` here because even though this completed @@ -683,7 +746,7 @@ when defined(windows) or defined(nimdoc): else: # datagram sockets don't have disconnection, # so we can just raise an exception - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, @@ -694,7 +757,7 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. if bytesReceived != 0: @@ -707,7 +770,7 @@ when defined(windows) or defined(nimdoc): proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, inheritable = defined(nimInheritHandles)): - owned(Future[tuple[address: string, client: AsyncFD]]) = + owned(Future[tuple[address: string, client: AsyncFD]]) {.gcsafe.} = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. @@ -745,7 +808,7 @@ when defined(windows) or defined(nimdoc): else: retFuture.complete(newAcceptFut.read) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) template completeAccept() {.dirty.} = var listenSock = socket @@ -774,7 +837,7 @@ when defined(windows) or defined(nimdoc): var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) {.gcsafe.} = if not retFuture.finished: if errcode == OSErrorCode(-1): completeAccept() @@ -1024,7 +1087,7 @@ when defined(windows) or defined(nimdoc): raiseOSError(osLastError()) var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.DWORD + var flags = WT_EXECUTEINWAITTHREAD.DWORD or WT_EXECUTEONLYONCE.DWORD proc proccb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = closeWaitable(hProcess) @@ -1103,11 +1166,14 @@ when defined(windows) or defined(nimdoc): initAll() else: - import selectors - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + import std/selectors + from std/posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, MSG_NOSIGNAL when declared(posix.accept4): - from posix import accept4, SOCK_CLOEXEC + from std/posix import accept4, SOCK_CLOEXEC + when defined(genode): + import genode/env # get the implicit Genode env + import genode/signals const InitCallbackListSize = 4 # initial size of callbacks sequence, @@ -1126,6 +1192,8 @@ else: PDispatcher* = ref object of PDispatcherBase selector: Selector[AsyncData] + when defined(genode): + signalHandler: SignalHandler proc `==`*(x, y: AsyncFD): bool {.borrow.} proc `==`*(x, y: AsyncEvent): bool {.borrow.} @@ -1141,9 +1209,22 @@ else: result.selector = newSelector[AsyncData]() result.timers.clear() result.callbacks = initDeque[proc () {.closure, gcsafe.}](InitDelayedCallbackListSize) + when defined(genode): + let entrypoint = ep(cast[GenodeEnv](runtimeEnv)) + result.signalHandler = newSignalHandler(entrypoint): + discard runOnce(0) var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + when defined(nuttx): + import std/exitprocs + + proc cleanDispatcher() {.noconv.} = + gDisp = nil + + proc addFinalyzer() = + addExitProc(cleanDispatcher) + proc setGlobalDispatcher*(disp: owned PDispatcher) = if not gDisp.isNil: assert gDisp.callbacks.len == 0 @@ -1153,6 +1234,8 @@ else: proc getGlobalDispatcher*(): PDispatcher = if gDisp.isNil: setGlobalDispatcher(newDispatcher()) + when defined(nuttx): + addFinalyzer() result = gDisp proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = @@ -1310,10 +1393,11 @@ else: ValueError, "Expecting async operations to stop when fd has closed." ) - - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + when defined(genode): + if timeout == 0: return raise newException(ValueError, "No handles or timers registered in dispatcher.") @@ -1384,7 +1468,7 @@ else: if flags.isDisconnectionError(lastError): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -1413,7 +1497,7 @@ else: if flags.isDisconnectionError(lastError): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1479,7 +1563,7 @@ else: let lastError = osLastError() if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1505,7 +1589,7 @@ else: let lastError = osLastError() if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false else: @@ -1518,7 +1602,7 @@ else: owned(Future[tuple[address: string, client: AsyncFD]]) = var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = + proc cb(sock: AsyncFD): bool {.gcsafe.} = result = true var sockAddress: Sockaddr_storage var addrLen = sizeof(sockAddress).SockLen @@ -1546,7 +1630,7 @@ else: if flags.isDisconnectionError(lastError): return false else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: try: let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) @@ -1675,9 +1759,11 @@ when defined(windows) or defined(nimdoc): proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): + const SO_UPDATE_CONNECT_CONTEXT = 0x7010 + socket.SocketHandle.setSockOptInt(SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, 1) # 15022 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = connectEx(socket.SocketHandle, addrInfo.ai_addr, @@ -1695,7 +1781,7 @@ when defined(windows) or defined(nimdoc): # With ERROR_IO_PENDING `ol` will be deallocated in `poll`, # and the future will be completed/failed there, too. GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) = let retFuture = newFuture[void]("doConnect") @@ -1712,7 +1798,7 @@ else: # interrupted, keep waiting return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true let ret = connect(socket.SocketHandle, @@ -1726,7 +1812,7 @@ else: if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(socket, cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, protocol: Protocol = IPPROTO_RAW) = @@ -1765,7 +1851,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, curAddrInfo = curAddrInfo.ai_next if curAddrInfo == nil: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds() if lastException != nil: @@ -1781,7 +1867,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, try: curFd = createAsyncNativeSocket(domain, sockType, protocol) except: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) closeUnusedFds() raise getCurrentException() when defined(windows): @@ -1791,7 +1877,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo curAddrInfo = curAddrInfo.ai_next else: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds(ord(domain)) retFuture.complete(curFd) @@ -1908,7 +1994,8 @@ proc send*(socket: AsyncFD, data: string, return retFuture # -- Await Macro -include asyncmacro +import std/asyncmacro +export asyncmacro proc readAll*(future: FutureStream[string]): owned(Future[string]) {.async.} = ## Returns a future that will complete when all the string data from the @@ -1945,17 +2032,34 @@ proc activeDescriptors*(): int {.inline.} = result = getGlobalDispatcher().selector.count when defined(posix): - import posix + import std/posix -when defined(linux) or defined(windows) or defined(macosx) or defined(bsd): +when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is - ## supported on the following OSes: Windows, Linux, OSX, BSD. + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. when defined(windows): result = 16_700_000 + elif defined(zephyr) or defined(freertos): + result = FD_MAX else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: raiseOSError(osLastError()) result = int(fdLim.rlim_cur) - 1 + +when defined(genode): + proc scheduleCallbacks*(): bool {.discardable.} = + ## *Genode only.* + ## Schedule callback processing and return immediately. + ## Returns `false` if there is nothing to schedule. + ## RPC servers should call this to dispatch `callSoon` + ## bodies after retiring an RPC to its client. + ## This is effectively a non-blocking `poll(…)` and is + ## equivalent to scheduling a momentary no-op timeout + ## but faster and with less overhead. + let dis = getGlobalDispatcher() + result = dis.callbacks.len > 0 + if result: submit(dis.signalHandler.cap) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c51b338e33..0f65043420 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -9,27 +9,33 @@ ## This module implements asynchronous file reading and writing. ## -## .. code-block:: Nim -## import std/[asyncfile, asyncdispatch, os] +## ```Nim +## import std/[asyncfile, asyncdispatch, os] ## -## proc main() {.async.} = -## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) -## await file.write("test") -## file.setFilePos(0) -## let data = await file.readAll() -## doAssert data == "test" -## file.close() +## proc main() {.async.} = +## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) +## await file.write("test") +## file.setFilePos(0) +## let data = await file.readAll() +## doAssert data == "test" +## file.close() ## -## waitFor main() +## waitFor main() +## ``` -import asyncdispatch, os +import std/[asyncdispatch, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + when defined(windows) or defined(nimdoc): + import std/widestrs # TODO: Fix duplication introduced by PR #4683. when defined(windows) or defined(nimdoc): - import winlean + import std/winlean else: - import posix + import std/posix type AsyncFile* = ref object @@ -96,14 +102,9 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) - when useWinUnicode: - let fd = createFileW(newWideCString(filename), desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) - else: - let fd = createFileA(filename, desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) + let fd = createFileW(newWideCString(filename), desiredAccess, + FILE_SHARE_READ, + nil, creationDisposition, flags, 0) if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) @@ -145,7 +146,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) ol.offset = DWORD(f.offset and 0xffffffff) ol.offsetHigh = DWORD(f.offset shr 32) @@ -161,7 +162,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = # This happens in Windows Server 2003 retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -172,7 +173,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -185,7 +186,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -202,10 +203,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = proc read*(f: AsyncFile, size: int): Future[string] = ## Read `size` bytes from the specified file asynchronously starting at - ## the current position of the file pointer. + ## the current position of the file pointer. `size` should be greater than zero. ## ## If the file pointer is past the end of the file then an empty string is ## returned. + assert size > 0 var retFuture = newFuture[string]("asyncfile.read") when defined(windows) or defined(nimdoc): @@ -226,7 +228,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -249,7 +251,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # This happens in Windows Server 2003 retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -260,7 +262,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -277,7 +279,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -299,6 +301,8 @@ proc readLine*(f: AsyncFile): Future[string] {.async.} = result = "" while true: var c = await read(f, 1) + if c.len == 0: + break if c[0] == '\c': c = await read(f, 1) break @@ -346,7 +350,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = assert bytesCount == size.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) # passing -1 here should work according to MSDN, but doesn't. For more # information see @@ -363,14 +367,14 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == size.int32 retFuture.complete() @@ -385,7 +389,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -419,7 +423,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = assert bytesCount == data.len.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -438,14 +442,14 @@ proc write*(f: AsyncFile, data: string): Future[void] = dealloc buffer buffer = nil GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == data.len.int32 retFuture.complete() @@ -466,7 +470,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim deleted file mode 100644 index 0ee45785da..0000000000 --- a/lib/pure/asyncftpclient.nim +++ /dev/null @@ -1,448 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements an asynchronous FTP client. It allows you to connect -## to an FTP server and perform operations on it such as for example: -## -## * The upload of new files. -## * The removal of existing files. -## * Download of files. -## * Changing of files' permissions. -## * Navigation through the FTP server's directories. -## -## Connecting to an FTP server -## =========================== -## -## In order to begin any sort of transfer of files you must first -## connect to an FTP server. You can do so with the `connect` procedure. -## -## .. code-block::nim -## import std/[asyncdispatch, asyncftpclient] -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## echo("Connected") -## waitFor(main()) -## -## A new `main` async procedure must be declared to allow the use of the -## `await` keyword. The connection will complete asynchronously and the -## client will be connected after the `await ftp.connect()` call. -## -## Uploading a new file -## ==================== -## -## After a connection is made you can use the `store` procedure to upload -## a new file to the FTP server. Make sure to check you are in the correct -## working directory before you do so with the `pwd` procedure, you can also -## instead specify an absolute path. -## -## .. code-block::nim -## import std/[asyncdispatch, asyncftpclient] -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## let currentDir = await ftp.pwd() -## assert currentDir == "/home/user/" -## await ftp.store("file.txt", "file.txt") -## echo("File finished uploading") -## waitFor(main()) -## -## Checking the progress of a file transfer -## ======================================== -## -## The progress of either a file upload or a file download can be checked -## by specifying a `onProgressChanged` procedure to the `store` or -## `retrFile` procedures. -## -## Procs that take an `onProgressChanged` callback will call this every -## `progressInterval` milliseconds. -## -## .. code-block::nim -## import std/[asyncdispatch, asyncftpclient] -## -## proc onProgressChanged(total, progress: BiggestInt, -## speed: float) {.async.} = -## echo("Uploaded ", progress, " of ", total, " bytes") -## echo("Current speed: ", speed, " kb/s") -## -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test", progressInterval = 500) -## await ftp.connect() -## await ftp.store("file.txt", "/home/user/file.txt", onProgressChanged) -## echo("File finished uploading") -## waitFor(main()) - - -import asyncdispatch, asyncnet, nativesockets, strutils, parseutils, os, times -from net import BufferSize - -type - AsyncFtpClient* = ref object - csock*: AsyncSocket - dsock*: AsyncSocket - user*, pass*: string - address*: string - port*: Port - progressInterval: int - jobInProgress*: bool - job*: FtpJob - dsockConnected*: bool - - FtpJobType* = enum - JRetrText, JRetr, JStore - - FtpJob = ref object - prc: proc (ftp: AsyncFtpClient, async: bool): bool {.nimcall, gcsafe.} - case typ*: FtpJobType - of JRetrText: - lines: string - of JRetr, JStore: - file: File - filename: string - total: BiggestInt # In bytes. - progress: BiggestInt # In bytes. - oneSecond: BiggestInt # Bytes transferred in one second. - lastProgressReport: float # Time - toStore: string # Data left to upload (Only used with async) - - FtpEventType* = enum - EvTransferProgress, EvLines, EvRetr, EvStore - - FtpEvent* = object ## Event - filename*: string - case typ*: FtpEventType - of EvLines: - lines*: string ## Lines that have been transferred. - of EvRetr, EvStore: ## Retr/Store operation finished. - nil - of EvTransferProgress: - bytesTotal*: BiggestInt ## Bytes total. - bytesFinished*: BiggestInt ## Bytes transferred. - speed*: BiggestInt ## Speed in bytes/s - currentJob*: FtpJobType ## The current job being performed. - - ReplyError* = object of IOError - - ProgressChangedProc* = - proc (total, progress: BiggestInt, speed: float): - Future[void] {.closure, gcsafe.} - -const multiLineLimit = 10000 - -proc expectReply(ftp: AsyncFtpClient): Future[string] {.async.} = - var line = await ftp.csock.recvLine() - result = line - var count = 0 - while line.len > 3 and line[3] == '-': - ## Multi-line reply. - line = await ftp.csock.recvLine() - result.add("\n" & line) - count.inc() - if count >= multiLineLimit: - raise newException(ReplyError, "Reached maximum multi-line reply count.") - -proc send*(ftp: AsyncFtpClient, m: string): Future[string] {.async.} = - ## Send a message to the server, and wait for a primary reply. - ## `\c\L` is added for you. - ## - ## You need to make sure that the message `m` doesn't contain any newline - ## characters. Failing to do so will raise `AssertionDefect`. - ## - ## **Note:** The server may return multiple lines of coded replies. - doAssert(not m.contains({'\c', '\L'}), "message shouldn't contain any newline characters") - await ftp.csock.send(m & "\c\L") - return await ftp.expectReply() - -proc assertReply(received: string, expected: varargs[string]) = - for i in items(expected): - if received.startsWith(i): return - raise newException(ReplyError, - "Expected reply '$1' got: $2" % - [expected.join("' or '"), received]) - -proc pasv(ftp: AsyncFtpClient) {.async.} = - ## Negotiate a data connection. - ftp.dsock = newAsyncSocket() - - var pasvMsg = (await ftp.send("PASV")).strip - assertReply(pasvMsg, "227") - var betweenParens = captureBetween(pasvMsg, '(', ')') - var nums = betweenParens.split(',') - var ip = nums[0 .. ^3] - var port = nums[^2 .. ^1] - var properPort = port[0].parseInt()*256+port[1].parseInt() - await ftp.dsock.connect(ip.join("."), Port(properPort)) - ftp.dsockConnected = true - -proc normalizePathSep(path: string): string = - return replace(path, '\\', '/') - -proc connect*(ftp: AsyncFtpClient) {.async.} = - ## Connect to the FTP server specified by `ftp`. - await ftp.csock.connect(ftp.address, ftp.port) - - var reply = await ftp.expectReply() - if reply.startsWith("120"): - # 120 Service ready in nnn minutes. - # We wait until we receive 220. - reply = await ftp.expectReply() - - # Handle 220 messages from the server - assertReply(reply, "220") - - if ftp.user != "": - assertReply(await(ftp.send("USER " & ftp.user)), "230", "331") - - if ftp.pass != "": - assertReply(await(ftp.send("PASS " & ftp.pass)), "230") - -proc pwd*(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Returns the current working directory. - let wd = await ftp.send("PWD") - assertReply wd, "257" - return wd.captureBetween('"') # " - -proc cd*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Changes the current directory on the remote FTP server to `dir`. - assertReply(await(ftp.send("CWD " & dir.normalizePathSep)), "250") - -proc cdup*(ftp: AsyncFtpClient) {.async.} = - ## Changes the current directory to the parent of the current directory. - assertReply(await(ftp.send("CDUP")), "200") - -proc getLines(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Downloads text data in ASCII mode - result = "" - assert ftp.dsockConnected - while ftp.dsockConnected: - let r = await ftp.dsock.recvLine() - if r == "": - ftp.dsockConnected = false - else: - result.add(r & "\n") - - assertReply(await(ftp.expectReply()), "226") - -proc listDirs*(ftp: AsyncFtpClient, dir = ""): Future[seq[string]] {.async.} = - ## Returns a list of filenames in the given directory. If `dir` is "", - ## the current directory is used. If `async` is true, this - ## function will return immediately and it will be your job to - ## use asyncdispatch's `poll` to progress this operation. - await ftp.pasv() - - assertReply(await(ftp.send("NLST " & dir.normalizePathSep)), ["125", "150"]) - - result = splitLines(await ftp.getLines()) - -proc fileExists*(ftp: AsyncFtpClient, file: string): Future[bool] {.async.} = - ## Determines whether `file` exists. - var files = await ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc createDir*(ftp: AsyncFtpClient, dir: string, recursive = false){.async.} = - ## Creates a directory `dir`. If `recursive` is true, the topmost - ## subdirectory of `dir` will be created first, following the secondmost... - ## etc. this allows you to give a full path as the `dir` without worrying - ## about subdirectories not existing. - if not recursive: - assertReply(await(ftp.send("MKD " & dir.normalizePathSep)), "257") - else: - var reply = "" - var previousDirs = "" - for p in split(dir, {os.DirSep, os.AltSep}): - if p != "": - previousDirs.add(p) - reply = await ftp.send("MKD " & previousDirs) - previousDirs.add('/') - assertReply reply, "257" - -proc chmod*(ftp: AsyncFtpClient, path: string, - permissions: set[FilePermission]) {.async.} = - ## Changes permission of `path` to `permissions`. - var userOctal = 0 - var groupOctal = 0 - var otherOctal = 0 - for i in items(permissions): - case i - of fpUserExec: userOctal.inc(1) - of fpUserWrite: userOctal.inc(2) - of fpUserRead: userOctal.inc(4) - of fpGroupExec: groupOctal.inc(1) - of fpGroupWrite: groupOctal.inc(2) - of fpGroupRead: groupOctal.inc(4) - of fpOthersExec: otherOctal.inc(1) - of fpOthersWrite: otherOctal.inc(2) - of fpOthersRead: otherOctal.inc(4) - - var perm = $userOctal & $groupOctal & $otherOctal - assertReply(await(ftp.send("SITE CHMOD " & perm & - " " & path.normalizePathSep)), "200") - -proc list*(ftp: AsyncFtpClient, dir = ""): Future[string] {.async.} = - ## Lists all files in `dir`. If `dir` is `""`, uses the current - ## working directory. - await ftp.pasv() - - let reply = await ftp.send("LIST" & " " & dir.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc retrText*(ftp: AsyncFtpClient, file: string): Future[string] {.async.} = - ## Retrieves `file`. File must be ASCII text. - await ftp.pasv() - let reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(ftp.progressInterval) - var dataFut = ftp.dsock.recv(BufferSize) - while ftp.dsockConnected: - await dataFut or countdownFut - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, - progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(ftp.progressInterval) - - if dataFut.finished: - let data = dataFut.read - if data != "": - progress.inc(data.len) - progressInSecond.inc(data.len) - file.write(data) - dataFut = ftp.dsock.recv(BufferSize) - else: - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - -proc defaultOnProgressChanged*(total, progress: BiggestInt, - speed: float): Future[void] {.nimcall, gcsafe.} = - ## Default FTP `onProgressChanged` handler. Does nothing. - result = newFuture[void]() - #echo(total, " ", progress, " ", speed) - result.complete() - -proc retrFile*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Downloads `file` and saves it to `dest`. - ## The `EvRetr` event is passed to the specified `handleEvent` function - ## when the download is finished. The event's `filename` field will be equal - ## to `file`. - var destFile = open(dest, mode = fmWrite) - await ftp.pasv() - var reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply reply, ["125", "150"] - if {'(', ')'} notin reply: - raise newException(ReplyError, "Reply has no file size.") - var fileSize: BiggestInt - if reply.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(ReplyError, "Reply has no file size.") - - await getFile(ftp, destFile, fileSize, onProgressChanged) - destFile.close() - -proc doUpload(ftp: AsyncFtpClient, file: File, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - - let total = file.getFileSize() - var data = newString(4000) - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(ftp.progressInterval) - var sendFut: Future[void] = nil - while ftp.dsockConnected: - if sendFut == nil or sendFut.finished: - # TODO: Async file reading. - let len = file.readBuffer(addr(data[0]), 4000) - setLen(data, len) - if len == 0: - # File finished uploading. - ftp.dsock.close() - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - else: - progress.inc(len) - progressInSecond.inc(len) - sendFut = ftp.dsock.send(data) - - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(ftp.progressInterval) - - await countdownFut or sendFut - -proc store*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Uploads `file` to `dest` on the remote FTP server. Usage of this - ## function asynchronously is recommended to view the progress of - ## the download. - ## The `EvStore` event is passed to the specified `handleEvent` function - ## when the upload is finished, and the `filename` field will be - ## equal to `file`. - var destFile = open(file) - await ftp.pasv() - - let reply = await ftp.send("STOR " & dest.normalizePathSep) - assertReply reply, ["125", "150"] - - await doUpload(ftp, destFile, onProgressChanged) - -proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} = - ## Rename a file or directory on the remote FTP Server from current name - ## `name_from` to new name `name_to` - assertReply(await ftp.send("RNFR " & nameFrom), "350") - assertReply(await ftp.send("RNTO " & nameTo), "250") - -proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} = - ## Delete a file `filename` on the remote FTP server - assertReply(await ftp.send("DELE " & filename), "250") - -proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Delete a directory `dir` on the remote FTP server - assertReply(await ftp.send("RMD " & dir), "250") - -proc newAsyncFtpClient*(address: string, port = Port(21), - user, pass = "", progressInterval: int = 1000): AsyncFtpClient = - ## Creates a new `AsyncFtpClient` object. - new result - result.user = user - result.pass = pass - result.address = address - result.port = port - result.progressInterval = progressInterval - result.dsockConnected = false - result.csock = newAsyncSocket() - -when not defined(testing) and isMainModule: - var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") - proc main(ftp: AsyncFtpClient) {.async.} = - await ftp.connect() - echo await ftp.pwd() - echo await ftp.listDirs() - await ftp.store("payload.jpg", "payload.jpg") - await ftp.retrFile("payload.jpg", "payload2.jpg") - await ftp.rename("payload.jpg", "payload_renamed.jpg") - await ftp.store("payload.jpg", "payload_remove.jpg") - await ftp.removeFile("payload_remove.jpg") - await ftp.createDir("deleteme") - await ftp.removeDir("deleteme") - echo("Finished") - - waitFor main(ftp) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index c0c5c3f07f..29ebf8f89b 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -7,10 +7,14 @@ # distribution, for details about the copyright. # -import os, tables, strutils, times, heapqueue, options, deques, cstrutils +import std/[os, tables, strutils, times, heapqueue, options, deques, cstrutils, typetraits] import system/stacktraces +when defined(nimPreviewSlimSystem): + import std/objectdollar # for StackTraceEntry + import std/assertions + # TODO: This shouldn't need to be included, but should ideally be exported. type CallbackFunc = proc () {.closure, gcsafe.} @@ -25,7 +29,7 @@ type finished: bool error*: ref Exception ## Stored exception errorStackTrace*: string - when not defined(release): + when not defined(release) or defined(futureLogging): stackTrace: seq[StackTraceEntry] ## For debugging purposes only. id: int fromProc: string @@ -47,7 +51,7 @@ const NimAsyncContinueSuffix* = "NimAsyncContinue" ## For internal usage. Do not use. when isFutureLoggingEnabled: - import hashes + import std/hashes type FutureInfo* = object stackTrace*: seq[StackTraceEntry] @@ -93,7 +97,7 @@ proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) = ## Change current implementation of `callSoon`. This is normally called when dispatcher from `asyncdispatcher` is initialized. callSoonProc = p -proc callSoon*(cbproc: proc ()) = +proc callSoon*(cbproc: proc () {.gcsafe.}) = ## Call `cbproc` "soon". ## ## If async dispatcher is running, `cbproc` will be executed during next dispatcher tick. @@ -189,24 +193,22 @@ proc add(callbacks: var CallbackList, function: CallbackFunc) = last = last.next last.next = newCallback -proc complete*[T](future: Future[T], val: T) = - ## Completes `future` with value `val`. +proc completeImpl[T, U](future: Future[T], val: sink U, isVoid: static bool) = #assert(not future.finished, "Future already finished, cannot finish twice.") checkFinished(future) assert(future.error == nil) - future.value = val + when not isVoid: + future.value = val future.finished = true future.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(future) -proc complete*(future: Future[void]) = - ## Completes a void `future`. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - future.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) +proc complete*[T](future: Future[T], val: sink T) = + ## Completes `future` with value `val`. + completeImpl(future, val, false) + +proc complete*(future: Future[void], val = Future[void].default) = + completeImpl(future, (), true) proc complete*[T](future: FutureVar[T]) = ## Completes a `FutureVar`. @@ -217,7 +219,7 @@ proc complete*[T](future: FutureVar[T]) = fut.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(Future[T](future)) -proc complete*[T](future: FutureVar[T], val: T) = +proc complete*[T](future: FutureVar[T], val: sink T) = ## Completes a `FutureVar` with value `val`. ## ## Any previously stored value will be overwritten. @@ -227,7 +229,7 @@ proc complete*[T](future: FutureVar[T], val: T) = fut.finished = true fut.value = val fut.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) + when isFutureLoggingEnabled: logFutureFinish(fut) proc fail*[T](future: Future[T], error: ref Exception) = ## Completes `future` with `error`. @@ -280,19 +282,33 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then `cb` will be called immediately. future.callback = proc () = cb(future) +template getFilenameProcname(entry: StackTraceEntry): (string, string) = + when compiles(entry.filenameStr) and compiles(entry.procnameStr): + # We can't rely on "entry.filename" and "entry.procname" still being valid + # cstring pointers, because the "string.data" buffers they pointed to might + # be already garbage collected (this entry being a non-shallow copy, + # "entry.filename" no longer points to "entry.filenameStr.data", but to the + # buffer of the original object). + (entry.filenameStr, entry.procnameStr) + else: + ($entry.filename, $entry.procname) + proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. + + let (filename, procname) = getFilenameProcname(entry) + result = "" - if entry.procname == cstring"processPendingCallbacks": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + if procname == "processPendingCallbacks": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Executes pending callbacks" - elif entry.procname == cstring"poll": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + elif procname == "poll": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Processes asynchronous completion events" - if entry.procname.endsWith(NimAsyncContinueSuffix): - if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0: + if procname.endsWith(NimAsyncContinueSuffix): + if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0: return "Resumes an async procedure" proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = @@ -305,33 +321,29 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = # Find longest filename & line number combo for alignment purposes. var longestLeft = 0 for entry in entries: - if entry.procname.isNil: continue + let (filename, procname) = getFilenameProcname(entry) - let left = $entry.filename & $entry.line - if left.len > longestLeft: - longestLeft = left.len + if procname == "": continue + + let leftLen = filename.len + len($entry.line) + if leftLen > longestLeft: + longestLeft = leftLen - var indent = 2 # Format the entries. for entry in entries: - if entry.procname.isNil: - if entry.line == reraisedFromBegin: - result.add(spaces(indent) & "#[\n") - indent.inc(2) - elif entry.line == reraisedFromEnd: - indent.dec(2) - result.add(spaces(indent) & "]#\n") - continue + let (filename, procname) = getFilenameProcname(entry) - let left = "$#($#)" % [$entry.filename, $entry.line] - result.add((spaces(indent) & "$#$# $#\n") % [ + if procname == "" and entry.line == reraisedFromBegin: + break + + let left = "$#($#)" % [filename, $entry.line] + result.add((spaces(2) & "$# $#\n") % [ left, - spaces(longestLeft - left.len + 2), - $entry.procname + procname ]) let hint = getHint(entry) if hint.len > 0: - result.add(spaces(indent+2) & "## " & hint & "\n") + result.add(spaces(4) & "## " & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): @@ -351,34 +363,38 @@ proc injectStacktrace[T](future: Future[T]) = newMsg.add($entries) newMsg.add("Exception message: " & exceptionMsg & "\n") - newMsg.add("Exception type:") # # For debugging purposes + # newMsg.add("Exception type:") # for entry in getStackTraceEntries(future.error): # newMsg.add "\n" & $entry future.error.msg = newMsg -proc read*[T](future: Future[T] | FutureVar[T]): T = - ## Retrieves the value of `future`. Future must be finished otherwise - ## this function will fail with a `ValueError` exception. - ## - ## If the result of the future is an error then that error will be raised. - {.push hint[ConvFromXtoItselfNotNeeded]: off.} +template readImpl(future, T) = when future is Future[T]: - let fut = future + let fut {.cursor.} = future else: - let fut = Future[T](future) - {.pop.} + let fut {.cursor.} = Future[T](future) if fut.finished: if fut.error != nil: injectStacktrace(fut) raise fut.error when T isnot void: - result = fut.value + result = distinctBase(future).value else: # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") +proc read*[T](future: Future[T] | FutureVar[T]): lent T = + ## Retrieves the value of `future`. Future must be finished otherwise + ## this function will fail with a `ValueError` exception. + ## + ## If the result of the future is an error then that error will be raised. + readImpl(future, T) + +proc read*(future: Future[void] | FutureVar[void]) = + readImpl(future, void) + proc readError*[T](future: Future[T]): ref Exception = ## Retrieves the exception stored in `future`. ## diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 38be4ceac6..39e945d5e6 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -15,20 +15,20 @@ ## instead of allowing users to connect directly to this server. runnableExamples("-r:off"): - # This example will create an HTTP server on port 8080. The server will - # respond to all requests with a `200 OK` response code and "Hello World" + # This example will create an HTTP server on an automatically chosen port. + # It will respond to all requests with a `200 OK` response code and "Hello World" # as the response body. import std/asyncdispatch proc main {.async.} = - const port = 8080 var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = echo (req.reqMethod, req.url, req.headers) let headers = {"Content-type": "text/plain; charset=utf-8"} await req.respond(Http200, "Hello World", headers.newHttpHeaders()) - echo "test this with: curl localhost:" & $port & "/" - server.listen(Port(port)) + server.listen(Port(0)) # or Port(8080) to hardcode the standard HTTP port. + let port = server.getPort + echo "test this with: curl localhost:" & $port.uint16 & "/" while true: if server.shouldAcceptRequest(): await server.acceptRequest(cb) @@ -39,10 +39,14 @@ runnableExamples("-r:off"): waitFor main() -import asyncnet, asyncdispatch, parseutils, uri, strutils -import httpcore +import std/[asyncnet, asyncdispatch, parseutils, uri, strutils] +import std/httpcore +from std/nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + export httpcore except parseHeader const @@ -70,21 +74,18 @@ type maxBody: int ## The maximum content-length that will be read for the body. maxFDs: int -func getSocket*(a: AsyncHttpServer): AsyncSocket {.since: (1, 5, 1).} = - ## Returns the `AsyncHttpServer`s internal `AsyncSocket` instance. - ## - ## Useful for identifying what port the AsyncHttpServer is bound to, if it - ## was chosen automatically. +proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} = + ## Returns the port `self` was bound to. + ## + ## Useful for identifying what port `self` is bound to, if it + ## was chosen automatically, for example via `listen(Port(0))`. runnableExamples: - from std/asyncdispatch import Port - from std/asyncnet import getFd - from std/nativesockets import getLocalAddr, AF_INET + from std/nativesockets import Port let server = newAsyncHttpServer() - server.listen(Port(0)) # Socket is not bound until this point - let port = getLocalAddr(server.getSocket.getFd, AF_INET)[1] - doAssert uint16(port) > 0 + server.listen(Port(0)) + assert server.getPort.uint16 > 0 server.close() - a.socket + result = getLocalAddr(self.socket)[1] proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = @@ -109,16 +110,16 @@ proc respond*(req: Request, code: HttpCode, content: string, ## This procedure will **not** close the client socket. ## ## Example: - ## - ## .. code-block::nim - ## import std/json - ## proc handler(req: Request) {.async.} = - ## if req.url.path == "/hello-world": - ## let msg = %* {"message": "Hello World"} - ## let headers = newHttpHeaders([("Content-Type","application/json")]) - ## await req.respond(Http200, $msg, headers) - ## else: - ## await req.respond(Http404, "Not Found") + ## ```Nim + ## import std/json + ## proc handler(req: Request) {.async.} = + ## if req.url.path == "/hello-world": + ## let msg = %* {"message": "Hello World"} + ## let headers = newHttpHeaders([("Content-Type","application/json")]) + ## await req.respond(Http200, $msg, headers) + ## else: + ## await req.respond(Http404, "Not Found") + ## ``` var msg = "HTTP/1.1 " & $code & "\c\L" if headers != nil: @@ -157,7 +158,7 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") -func hasChunkedEncoding(request: Request): bool = +func hasChunkedEncoding(request: Request): bool = ## Searches for a chunked transfer encoding const transferEncoding = "Transfer-Encoding" @@ -172,7 +173,7 @@ proc processRequest( server: AsyncHttpServer, req: FutureVar[Request], client: AsyncSocket, - address: string, + address: sink string, lineFut: FutureVar[string], callback: proc (request: Request): Future[void] {.closure, gcsafe.}, ): Future[bool] {.async.} = @@ -186,7 +187,10 @@ proc processRequest( # \n request.headers.clear() request.body = "" - request.hostname.shallowCopy(address) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + request.hostname = address + else: + request.hostname.shallowCopy(address) assert client != nil request.client = client @@ -296,7 +300,7 @@ proc processRequest( while true: lineFut.mget.setLen(0) lineFut.clean() - + # The encoding format alternates between specifying a number of bytes to read # and the data to be read, of the previously specified size if sizeOrData mod 2 == 0: @@ -364,7 +368,9 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string let retry = await processRequest( server, request, client, address, lineFut, callback ) - if not retry: break + if not retry: + client.close() + break const nimMaxDescriptorsFallback* {.intdefine.} = 16_000 ## fallback value for \ @@ -372,17 +378,18 @@ const ## This can be set on the command line during compilation ## via `-d:nimMaxDescriptorsFallback=N` -proc listen*(server: AsyncHttpServer; port: Port; address = "") = +proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET) = ## Listen to the given port and address. when declared(maxDescriptors): server.maxFDs = try: maxDescriptors() except: nimMaxDescriptorsFallback else: server.maxFDs = nimMaxDescriptorsFallback - server.socket = newAsyncSocket() + server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) - if server.reusePort: - server.socket.setSockOpt(OptReusePort, true) + when not defined(nuttx): + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() @@ -404,7 +411,8 @@ proc acceptRequest*(server: AsyncHttpServer, proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure, gcsafe.}, address = ""; - assumedDescriptorsPerRequest = -1) {.async.} = + assumedDescriptorsPerRequest = -1; + domain = AF_INET) {.async.} = ## Starts the process of listening for incoming HTTP connections on the ## specified address and port. ## @@ -417,7 +425,7 @@ proc serve*(server: AsyncHttpServer, port: Port, ## ## You should prefer to call `acceptRequest` instead with a custom server ## loop so that you're in control over the error handling and logging. - listen server, port, address + listen server, port, address, domain while true: if shouldAcceptRequest(server, assumedDescriptorsPerRequest): var (address, client) = await server.socket.acceptAddr() diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 3097ac104e..d4e72c28a6 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -7,10 +7,14 @@ # distribution, for details about the copyright. # -## `asyncdispatch` module depends on the `asyncmacro` module to work properly. +## Implements the `async` and `multisync` macros for `asyncdispatch`. -import macros, strutils, asyncfutures +import std/[macros, strutils, asyncfutures] +type + Context = ref object + inTry: int + hasRet: bool # TODO: Ref https://github.com/nim-lang/Nim/issues/5617 # TODO: Add more line infos @@ -21,9 +25,8 @@ proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimN template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = bind finished - var nameIterVar = iteratorNameSym - proc identName {.closure.} = + proc identName {.closure, stackTrace: off.} = try: if not nameIterVar.finished: var next = nameIterVar() @@ -35,14 +38,11 @@ template createCb(retFutureSym, iteratorNameSym, if next == nil: if not retFutureSym.finished: - let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " & - "`nil` Future?" + let msg = "Async procedure ($1) yielded `nil`, are you await'ing a `nil` Future?" raise newException(AssertionDefect, msg % strName) else: {.gcsafe.}: - {.push hint[ConvFromXtoItselfNotNeeded]: off.} next.addCallback cast[proc() {.closure, gcsafe.}](identName) - {.pop.} except: futureVarCompletions if retFutureSym.finished: @@ -53,8 +53,7 @@ template createCb(retFutureSym, iteratorNameSym, retFutureSym.fail(getCurrentException()) identName() -proc createFutureVarCompletions(futureVarIdents: seq[NimNode], - fromNode: NimNode): NimNode {.compileTime.} = +proc createFutureVarCompletions(futureVarIdents: seq[NimNode], fromNode: NimNode): NimNode = result = newNimNode(nnkStmtList, fromNode) # Add calls to complete each FutureVar parameter. for ident in futureVarIdents: @@ -69,10 +68,7 @@ proc createFutureVarCompletions(futureVarIdents: seq[NimNode], ) ) -proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, - futureVarIdents: seq[NimNode]): NimNode {.compileTime.} = - #echo(node.treeRepr) +proc processBody(ctx: Context; node, needsCompletionSym, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): NimNode = result = node case node.kind of nnkReturnStmt: @@ -81,33 +77,57 @@ proc processBody(node, retFutureSym: NimNode, # As I've painfully found out, the order here really DOES matter. result.add createFutureVarCompletions(futureVarIdents, node) + ctx.hasRet = true if node[0].kind == nnkEmpty: - if not subTypeIsVoid: - result.add newCall(newIdentNode("complete"), retFutureSym, - newIdentNode("result")) + if ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, newIdentNode("result")) else: - result.add newCall(newIdentNode("complete"), retFutureSym) + result.add newAssignment(needsCompletionSym, newLit(true)) else: - let x = node[0].processBody(retFutureSym, subTypeIsVoid, - futureVarIdents) + let x = processBody(ctx, node[0], needsCompletionSym, retFutureSym, futureVarIdents) if x.kind == nnkYieldStmt: result.add x + elif ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, x) else: - result.add newCall(newIdentNode("complete"), retFutureSym, x) + result.add newAssignment(newIdentNode("result"), x) + result.add newAssignment(needsCompletionSym, newLit(true)) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of RoutineNodes-{nnkTemplateDef}: # skip all the nested procedure definitions return - else: discard - - for i in 0 ..< result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, - futureVarIdents) + of nnkTryStmt: + if result[^1].kind == nnkFinally: + inc ctx.inTry + result[0] = processBody(ctx, result[0], needsCompletionSym, retFutureSym, futureVarIdents) + dec ctx.inTry + for i in 1 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + if ctx.inTry == 0 and ctx.hasRet: + let finallyNode = copyNimNode(result[^1]) + let stmtNode = newNimNode(nnkStmtList) + for child in result[^1]: + stmtNode.add child + stmtNode.add newIfStmt( + ( needsCompletionSym, + newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, + newIdentNode("result") + ) + ) + ) + finallyNode.add stmtNode + result[^1] = finallyNode + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) # echo result.repr -proc getName(node: NimNode): string {.compileTime.} = +proc getName(node: NimNode): string = case node.kind of nnkPostfix: return node[1].strVal @@ -118,7 +138,7 @@ proc getName(node: NimNode): string {.compileTime.} = else: error("Unknown name.", node) -proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = +proc getFutureVarIdents(params: NimNode): seq[NimNode] = result = @[] for i in 1 ..< len(params): expectKind(params[i], nnkIdentDefs) @@ -130,7 +150,7 @@ proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = proc isInvalidReturnType(typeName: string): bool = return typeName notin ["Future"] #, "FutureStream"] -proc verifyReturnType(typeName: string, node: NimNode = nil) {.compileTime.} = +proc verifyReturnType(typeName: string, node: NimNode = nil) = if typeName.isInvalidReturnType: error("Expected return type of 'Future' got '$1'" % typeName, node) @@ -140,19 +160,35 @@ template await*(f: typed): untyped {.used.} = error "await expects Future[T], got " & $typeof(f) template await*[T](f: Future[T]): auto {.used.} = - var internalTmpFuture: FutureBase = f - yield internalTmpFuture - (cast[typeof(f)](internalTmpFuture)).read() + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template yieldFuture {.redefine.} = yield FutureBase() -proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = + when compiles(yieldFuture): + var internalTmpFuture: FutureBase = f + yield internalTmpFuture + (cast[typeof(f)](internalTmpFuture)).read() + else: + macro errorAsync(futureError: Future[T]) = + error( + "Can only 'await' inside a proc marked as 'async'. Use " & + "'waitFor' when calling an 'async' proc in a non-async scope instead", + futureError) + errorAsync(f) + +proc asyncSingleProc(prc: NimNode): NimNode = ## This macro transforms a single procedure into a closure iterator. ## The `async` macro supports a stmtList holding multiple async procedures. if prc.kind == nnkProcTy: result = prc if prc[0][0].kind == nnkEmpty: - result[0][0] = parseExpr("Future[void]") + result[0][0] = quote do: Future[void] return result + if prc.kind in RoutineNodes and prc.name.kind != nnkEmpty: + # Only non anonymous functions need/can have stack trace disabled + prc.addPragma(nnkExprColonExpr.newTree(ident"stackTrace", ident"off")) + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.", prc) @@ -184,11 +220,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = else: verifyReturnType(repr(returnType), returnType) - let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].eqIdent("void")) - let futureVarIdents = getFutureVarIdents(prc.params) - var outerProcBody = newNimNode(nnkStmtList, prc.body) # Extract the documentation comment from the original procedure declaration. @@ -215,43 +247,42 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> {.pop.} # -> # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") - var procBody = prc.body.processBody(retFutureSym, subtypeIsVoid, - futureVarIdents) + var iteratorNameSym = genSym(nskIterator, $prcName & " (Async)") + var needsCompletionSym = genSym(nskVar, "needsCompletion") + var ctx = Context() + var procBody = processBody(ctx, prc.body, needsCompletionSym, retFutureSym, futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: # fix #13899, defer should not escape its original scope - procBody = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) - + let blockStmt = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) + procBody = newStmtList() + let resultIdent = ident"result" + procBody.add quote do: + # Check whether there is an implicit return + when typeof(`blockStmt`) is void: + `blockStmt` + else: + `resultIdent` = `blockStmt` procBody.add(createFutureVarCompletions(futureVarIdents, nil)) + procBody.insert(0): quote do: + {.push warning[resultshadowed]: off.} + when `subRetType` isnot void: + var `resultIdent`: `subRetType` + else: + var `resultIdent`: Future[void] + {.pop.} - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + var `needsCompletionSym` = false + procBody.add quote do: + complete(`retFutureSym`, `resultIdent`) - procBody.insert(1, newNimNode(nnkVarSection, prc.body).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - - var closureIterator = newProc(iteratorNameSym, [parseExpr("owned(FutureBase)")], + var closureIterator = newProc(iteratorNameSym, [quote do: owned(FutureBase)], procBody, nnkIteratorDef) closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom = prc.body) closureIterator.addPragma(newIdentNode("closure")) # If proc has an explicit gcsafe pragma, we add it to iterator as well. - if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == - "gcsafe") != nil: + if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == "gcsafe") != nil: closureIterator.addPragma(newIdentNode("gcsafe")) outerProcBody.add(closureIterator) @@ -260,21 +291,20 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # friendlier stack traces: var cbName = genSym(nskProc, prcName & NimAsyncContinueSuffix) var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prcName), - cbName, - createFutureVarCompletions(futureVarIdents, nil)) + newStrLitNode(prcName), + cbName, + createFutureVarCompletions(futureVarIdents, nil) + ) outerProcBody.add procCb # -> return retFuture outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) result = prc - - if subtypeIsVoid: - # Add discardable pragma. - if returnType.kind == nnkEmpty: - # Add Future[void] - result.params[0] = parseExpr("owned(Future[void])") + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # xxx consider removing `owned`? it's inconsistent with non-void case + result.params[0] = quote do: owned(Future[void]) # based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 if procBody.kind != nnkEmpty: @@ -282,10 +312,6 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = `outerProcBody` result.body = body2 - #echo(treeRepr(result)) - #if prcName == "recvLineInto": - # echo(toStrLit(result)) - macro async*(prc: untyped): untyped = ## Macro which processes async procedures into the appropriate ## iterators and yield statements. @@ -301,8 +327,8 @@ macro async*(prc: untyped): untyped = proc splitParamType(paramType: NimNode, async: bool): NimNode = result = paramType if paramType.kind == nnkInfix and paramType[0].strVal in ["|", "or"]: - let firstAsync = "async" in paramType[1].strVal.normalize - let secondAsync = "async" in paramType[2].strVal.normalize + let firstAsync = "async" in paramType[1].toStrLit().strVal.normalize + let secondAsync = "async" in paramType[2].toStrLit().strVal.normalize if firstAsync: result = paramType[if async: 1 else: 2] @@ -355,9 +381,3 @@ macro multisync*(prc: untyped): untyped = result = newStmtList() result.add(asyncSingleProc(asyncPrc)) result.add(sync) - # echo result.repr - -# overload for await as a fallback handler, based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 -# template await*(f: typed): untyped = - # static: - # error "await only available within {.async.}" diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index f36c427de5..d8dc7a798b 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -65,8 +65,7 @@ ## ## The following example demonstrates a simple chat server. ## -## .. code-block::nim -## +## ```Nim ## import std/[asyncnet, asyncdispatch] ## ## var clients {.threadvar.}: seq[AsyncSocket] @@ -93,19 +92,25 @@ ## ## asyncCheck serve() ## runForever() -## +## ``` import std/private/since -import asyncdispatch, nativesockets, net, os + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +import std/[asyncdispatch, nativesockets, net, os] export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when defineSsl: - import openssl + import std/openssl type # TODO: I would prefer to just do: @@ -178,11 +183,12 @@ proc getLocalAddr*(socket: AsyncSocket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: AsyncSocket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: AsyncSocket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc newAsyncSocket*(domain, sockType, protocol: cint, buffered = true, @@ -224,7 +230,7 @@ when defineSsl: let len = bioCtrlPending(socket.bioOut) if len > 0: var data = newString(len) - let read = bioRead(socket.bioOut, addr data[0], len) + let read = bioRead(socket.bioOut, cast[cstring](addr data[0]), len) assert read != 0 if read < 0: raiseSSLError() @@ -242,7 +248,7 @@ when defineSsl: var data = await recv(socket.fd.AsyncFD, BufferSize, flags) let length = len(data) if length > 0: - let ret = bioWrite(socket.bioIn, addr data[0], length.cint) + let ret = bioWrite(socket.bioIn, cast[cstring](addr data[0]), length.cint) if ret < 0: raiseSSLError() elif length == 0: @@ -259,14 +265,17 @@ when defineSsl: ErrClearError() # Call the desired operation. opResult = op - + let err = + if opResult < 0: + getSslError(socket, opResult.cint) + else: + SSL_ERROR_NONE # Send any remaining pending SSL data. await sendPendingSslData(socket, flags) # If the operation failed, try to see if SSL has some data to read # or write. if opResult < 0: - let err = getSslError(socket, opResult.cint) let fut = appeaseSsl(socket, flags, err.cint) yield fut if not fut.read(): @@ -397,7 +406,8 @@ proc recv*(socket: AsyncSocket, size: int, ## to be read then the future will complete with a value of `""`. if socket.isBuffered: result = newString(size) - shallow(result) + when not defined(nimSeqsV2): + shallow(result) let originalBufPos = socket.currPos if socket.bufLen == 0: @@ -453,7 +463,7 @@ proc send*(socket: AsyncSocket, data: string, when defineSsl: var copy = data sslLoop(socket, flags, - sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) + sslWrite(socket.sslHandle, cast[cstring](addr copy[0]), copy.len.cint)) await sendPendingSslData(socket, flags) else: await send(socket.fd.AsyncFD, data, flags) @@ -646,11 +656,16 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) + freeAddrInfo(aiList) raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) -when defined(posix): +proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = + ## Determines whether an AsyncSocket has data buffered. + # xxx dedup with std/net + s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen + +when defined(posix) and not useNimNetLite: proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) = ## Binds Unix socket to `path`. @@ -667,7 +682,7 @@ when defined(posix): elif ret == EINTR: return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true var socketAddr = makeUnixAddr(path) @@ -681,7 +696,7 @@ when defined(posix): if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(AsyncFD(socket.fd), cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) proc bindUnix*(socket: AsyncSocket, path: string) {. tags: [ReadIOEffect].} = @@ -731,6 +746,11 @@ proc close*(socket: AsyncSocket) = raiseSSLError() when defineSsl: + proc sslHandle*(self: AsyncSocket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns ## `socket` into an SSL socket. @@ -847,7 +867,7 @@ proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string, it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: if lastException != nil: diff --git a/lib/pure/asyncstreams.nim b/lib/pure/asyncstreams.nim index 083c6f0eab..c97b98d55c 100644 --- a/lib/pure/asyncstreams.nim +++ b/lib/pure/asyncstreams.nim @@ -9,9 +9,12 @@ ## Unstable API. -import asyncfutures +import std/asyncfutures -import deques +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/deques type FutureStream*[T] = ref object ## Special future that acts as diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index bf196b54de..591d22cc00 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -16,50 +16,48 @@ ## Each Base64 digit represents exactly 6 bits of data. Three 8-bit ## bytes (i.e., a total of 24 bits) can therefore be represented by ## four 6-bit Base64 digits. -## -## Basic usage -## =========== -## + +##[ +# Basic usage ## Encoding data -## ------------- -## -## .. code-block::nim -## import std/base64 -## let encoded = encode("Hello World") -## assert encoded == "SGVsbG8gV29ybGQ=" +]## + +runnableExamples: + let encoded = encode("Hello World") + assert encoded == "SGVsbG8gV29ybGQ=" + ## ## Apart from strings you can also encode lists of integers or characters: ## -## .. code-block::nim -## import std/base64 -## let encodedInts = encode([1,2,3]) -## assert encodedInts == "AQID" -## let encodedChars = encode(['h','e','y']) -## assert encodedChars == "aGV5" -## -## + +runnableExamples: + let encodedInts = encode([1'u8,2,3]) + assert encodedInts == "AQID" + let encodedChars = encode(['h','e','y']) + assert encodedChars == "aGV5" + +##[ ## Decoding data -## ------------- -## -## .. code-block::nim -## import std/base64 -## let decoded = decode("SGVsbG8gV29ybGQ=") -## assert decoded == "Hello World" -## +]## + +runnableExamples: + let decoded = decode("SGVsbG8gV29ybGQ=") + assert decoded == "Hello World" + +##[ ## URL Safe Base64 -## --------------- -## -## .. code-block::nim -## import std/base64 -## doAssert encode("c\xf7>", safe = true) == "Y_c-" -## doAssert encode("c\xf7>", safe = false) == "Y/c+" -## +]## + +runnableExamples: + assert encode("c\xf7>", safe = true) == "Y_c-" + assert encode("c\xf7>", safe = false) == "Y/c+" + ## See also ## ======== ## ## * `hashes module`_ for efficient computations of hash values for diverse Nim types -## * `md5 module`_ implements the MD5 checksum algorithm -## * `sha1 module`_ implements a sha1 encoder and decoder +## * `md5 module`_ for the MD5 checksum algorithm +## * `sha1 module`_ for the SHA-1 checksum algorithm template cbBase(a, b): untyped = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', @@ -68,14 +66,10 @@ template cbBase(a, b): untyped = [ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', a, b] -let +const cb64 = cbBase('+', '/') cb64safe = cbBase('-', '_') -const - cb64VM = cbBase('+', '/') - cb64safeVM = cbBase('-', '_') - const invalidChar = 255 @@ -86,10 +80,13 @@ template encodeInternal(s, alphabet: typed): untyped = result.setLen(encodeSize(s.len)) + let + padding = s.len mod 3 + inputEnds = s.len - padding + var inputIndex = 0 outputIndex = 0 - inputEnds = s.len - s.len mod 3 n: uint32 b: uint32 @@ -115,7 +112,6 @@ template encodeInternal(s, alphabet: typed): untyped = outputChar(n shr 6) outputChar(n shr 0) - var padding = s.len mod 3 if padding == 1: inputByte(b shl 16) outputChar(n shr 18) @@ -134,21 +130,14 @@ template encodeInternal(s, alphabet: typed): untyped = result.setLen(outputIndex) template encodeImpl() {.dirty.} = - when nimVM: - block: - let lookupTableVM = if safe: cb64safeVM else: cb64VM - encodeInternal(s, lookupTableVM) + if safe: + encodeInternal(s, cb64safe) else: - block: - let lookupTable = if safe: unsafeAddr(cb64safe) else: unsafeAddr(cb64) - encodeInternal(s, lookupTable) + encodeInternal(s, cb64) -proc encode*[T: SomeInteger|char](s: openArray[T], safe = false): string = +proc encode*[T: byte|char](s: openArray[T], safe = false): string = ## Encodes `s` into base64 representation. ## - ## This procedure encodes an openarray (array or sequence) of either integers - ## or characters. - ## ## If `safe` is `true` then it will encode using the ## URL-Safe and Filesystem-safe standard alphabet characters, ## which substitutes `-` instead of `+` and `_` instead of `/`. @@ -156,48 +145,55 @@ proc encode*[T: SomeInteger|char](s: openArray[T], safe = false): string = ## * https://tools.ietf.org/html/rfc4648#page-7 ## ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string - ## * `decode proc<#decode,string>`_ for decoding a string - runnableExamples: - assert encode(['n', 'i', 'm']) == "bmlt" - assert encode(@['n', 'i', 'm']) == "bmlt" - assert encode([1, 2, 3, 4, 5]) == "AQIDBAU=" - encodeImpl() - -proc encode*(s: string, safe = false): string = - ## Encodes `s` into base64 representation. - ## - ## This procedure encodes a string. - ## - ## If `safe` is `true` then it will encode using the - ## URL-Safe and Filesystem-safe standard alphabet characters, - ## which substitutes `-` instead of `+` and `_` instead of `/`. - ## * https://en.wikipedia.org/wiki/Base64#URL_applications - ## * https://tools.ietf.org/html/rfc4648#page-7 - ## - ## **See also:** - ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: assert encode("Hello World") == "SGVsbG8gV29ybGQ=" + assert encode(['n', 'i', 'm']) == "bmlt" + assert encode(@['n', 'i', 'm']) == "bmlt" + assert encode([1'u8, 2, 3, 4, 5]) == "AQIDBAU=" encodeImpl() -proc encodeMime*(s: string, lineLen = 75, newLine = "\r\n"): string = +proc encode*[T: SomeInteger and not byte](s: openArray[T], safe = false): string + {.deprecated: "use `byte` or `char` instead".} = + encodeImpl() + +proc encodeMime*(s: string, lineLen = 75.Positive, newLine = "\r\n", + safe = false): string = ## Encodes `s` into base64 representation as lines. ## Used in email MIME format, use `lineLen` and `newline`. ## ## This procedure encodes a string according to MIME spec. ## + ## If `safe` is `true` then it will encode using the + ## URL-Safe and Filesystem-safe standard alphabet characters, + ## which substitutes `-` instead of `+` and `_` instead of `/`. + ## * https://en.wikipedia.org/wiki/Base64#URL_applications + ## * https://tools.ietf.org/html/rfc4648#page-7 + ## ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string + ## * `encode proc<#encode,openArray[T]>`_ for encoding an openArray ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: assert encodeMime("Hello World", 4, "\n") == "SGVs\nbG8g\nV29y\nbGQ=" - result = newStringOfCap(encodeSize(s.len)) - for i, c in encode(s): - if i != 0 and (i mod lineLen == 0): - result.add(newLine) - result.add(c) + template cpy(l, src, idx) = + b = l + while i < b: + result[i] = src[idx] + inc i + inc idx + + if s.len == 0: return + let e = encode(s, safe) + if e.len <= lineLen or newLine.len == 0: + return e + result = newString(e.len + newLine.len * ((e.len div lineLen) - int(e.len mod lineLen == 0))) + var i, j, k, b: int + let nd = e.len - lineLen + while j < nd: + cpy(i + lineLen, e, j) + cpy(i + newLine.len, newLine, k) + k = 0 + cpy(result.len, e, j) proc initDecodeTable*(): array[256, char] = # computes a decode table at compile time @@ -220,7 +216,6 @@ proc decode*(s: string): string = ## ## **See also:** ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray - ## * `encode proc<#encode,string>`_ for encoding a string runnableExamples: assert decode("SGVsbG8gV29ybGQ=") == "Hello World" assert decode(" SGVsbG8gV29ybGQ=") == "Hello World" @@ -249,7 +244,7 @@ proc decode*(s: string): string = inputLen = s.len inputEnds = 0 # strip trailing characters - while s[inputLen - 1] in {'\n', '\r', ' ', '='}: + while inputLen > 0 and s[inputLen - 1] in {'\n', '\r', ' ', '='}: dec inputLen # hot loop: read 4 characters at at time inputEnds = inputLen - 4 diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 57e3c79896..0d3351ee51 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -25,8 +25,9 @@ ## At this time only `fastLog2`, `firstSetBit`, `countLeadingZeroBits` and `countTrailingZeroBits` ## may return undefined and/or platform dependent values if given invalid input. -import macros +import std/macros import std/private/since +from std/private/bitops_utils import forwardImpl, castToUnsigned func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. @@ -58,38 +59,16 @@ macro bitxor*[T: SomeInteger](x, y: T; z: varargs[T]): T = for extra in z: result = newCall(fn, result, extra) -const useBuiltins = not defined(noIntrinsicsBitOpts) -const noUndefined = defined(noUndefinedBitOpts) -const useGCC_builtins = (defined(gcc) or defined(llvm_gcc) or - defined(clang)) and useBuiltins -const useICC_builtins = defined(icc) and useBuiltins -const useVCC_builtins = defined(vcc) and useBuiltins -const arch64 = sizeof(int) == 8 -const useBuiltinsRotate = (defined(amd64) or defined(i386)) and - (defined(gcc) or defined(clang) or defined(vcc) or - (defined(icl) and not defined(cpp))) and useBuiltins - -template toUnsigned(x: int8): uint8 = cast[uint8](x) -template toUnsigned(x: int16): uint16 = cast[uint16](x) -template toUnsigned(x: int32): uint32 = cast[uint32](x) -template toUnsigned(x: int64): uint64 = cast[uint64](x) -template toUnsigned(x: int): uint = cast[uint](x) - -template forwardImpl(impl, arg) {.dirty.} = - when sizeof(x) <= 4: - when x is SomeSignedInt: - impl(cast[uint32](x.int32)) - else: - impl(x.uint32) - else: - when x is SomeSignedInt: - impl(cast[uint64](x.int64)) - else: - impl(x.uint64) type BitsRange*[T] = range[0..sizeof(T)*8-1] ## A range with all bit positions for type `T`. +template typeMasked[T: SomeInteger](x: T): T = + when defined(js): + T(x and ((0xffffffff_ffffffff'u shr (64 - sizeof(T) * 8)))) + else: + x + func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = ## Returns an extracted (and shifted) slice of bits from `v`. runnableExamples: @@ -99,8 +78,8 @@ func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned - (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T + uv = v.castToUnsigned + ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = ## Mutates `v` into an extracted (and shifted) slice of bits from `v`. @@ -111,8 +90,8 @@ proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned - v = (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T + uv = v.castToUnsigned + v = ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = ## Creates a bitmask based on a slice of bits. @@ -122,11 +101,8 @@ func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = let upmost = sizeof(T) * 8 - 1 - bitmask = when T is SomeUnsignedInt: - bitnot(0.T) - else: - bitnot(0.T).toUnsigned - (bitmask shl (upmost - slice.b + slice.a) shr (upmost - slice.b)).T + bitmask = bitnot(0.T).castToUnsigned + ((bitmask shl (upmost - slice.b + slice.a)).typeMasked shr (upmost - slice.b)).T proc masked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = ## Returns `v`, with only the `1` bits from `mask` matching those of @@ -436,13 +412,11 @@ func fastlog2Nim(x: uint64): int {.inline.} = v = v or v shr 32 result = lookup[(v * 0x03F6EAF2CD271461'u64) shr 58].int -# sets.nim cannot import bitops, but bitops can use include -# system/sets to eliminate code duplication. sets.nim defines -# countBits32 and countBits64. import system/countbits_impl -template countSetBitsNim(n: uint32): int = countBits32(n) -template countSetBitsNim(n: uint64): int = countBits64(n) +const useBuiltinsRotate = (defined(amd64) or defined(i386)) and + (defined(gcc) or defined(clang) or defined(vcc) or + (defined(icl) and not defined(cpp))) and useBuiltins template parityImpl[T](value: T): int = # formula id from: https://graphics.stanford.edu/%7Eseander/bithacks.html#ParityParallel @@ -459,11 +433,6 @@ template parityImpl[T](value: T): int = when useGCC_builtins: - # Returns the number of set 1-bits in value. - proc builtin_popcount(x: cuint): cint {.importc: "__builtin_popcount", cdecl.} - proc builtin_popcountll(x: culonglong): cint {. - importc: "__builtin_popcountll", cdecl.} - # Returns the bit parity in value proc builtin_parity(x: cuint): cint {.importc: "__builtin_parity", cdecl.} proc builtin_parityll(x: culonglong): cint {.importc: "__builtin_parityll", cdecl.} @@ -481,51 +450,34 @@ when useGCC_builtins: proc builtin_ctzll(x: culonglong): cint {.importc: "__builtin_ctzll", cdecl.} elif useVCC_builtins: - # Counts the number of one bits (population count) in a 16-, 32-, or 64-byte unsigned integer. - func builtin_popcnt16(a2: uint16): uint16 {. - importc: "__popcnt16", header: "".} - func builtin_popcnt32(a2: uint32): uint32 {. - importc: "__popcnt", header: "".} - func builtin_popcnt64(a2: uint64): uint64 {. - importc: "__popcnt64", header: "".} - # Search the mask data from most significant bit (MSB) to least significant bit (LSB) for a set bit (1). - func bitScanReverse(index: ptr culong, mask: culong): cuchar {. + func bitScanReverse(index: ptr culong, mask: culong): uint8 {. importc: "_BitScanReverse", header: "".} - func bitScanReverse64(index: ptr culong, mask: uint64): cuchar {. + func bitScanReverse64(index: ptr culong, mask: uint64): uint8 {. importc: "_BitScanReverse64", header: "".} # Search the mask data from least significant bit (LSB) to the most significant bit (MSB) for a set bit (1). - func bitScanForward(index: ptr culong, mask: culong): cuchar {. + func bitScanForward(index: ptr culong, mask: culong): uint8 {. importc: "_BitScanForward", header: "".} - func bitScanForward64(index: ptr culong, mask: uint64): cuchar {. + func bitScanForward64(index: ptr culong, mask: uint64): uint8 {. importc: "_BitScanForward64", header: "".} template vcc_scan_impl(fnc: untyped; v: untyped): int = - var index: culong + var index {.inject.}: culong = 0 discard fnc(index.addr, v) index.int elif useICC_builtins: - - # Intel compiler intrinsics: http://fulla.fnal.gov/intel/compiler_c/main_cls/intref_cls/common/intref_allia_misc.htm - # see also: https://software.intel.com/en-us/node/523362 - # Count the number of bits set to 1 in an integer a, and return that count in dst. - func builtin_popcnt32(a: cint): cint {. - importc: "_popcnt", header: "".} - func builtin_popcnt64(a: uint64): cint {. - importc: "_popcnt64", header: "".} - # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined. - func bitScanForward(p: ptr uint32, b: uint32): cuchar {. + func bitScanForward(p: ptr uint32, b: uint32): uint8 {. importc: "_BitScanForward", header: "".} - func bitScanForward64(p: ptr uint32, b: uint64): cuchar {. + func bitScanForward64(p: ptr uint32, b: uint64): uint8 {. importc: "_BitScanForward64", header: "".} # Returns the number of leading 0-bits in x, starting at the most significant bit position. If x is 0, the result is undefined. - func bitScanReverse(p: ptr uint32, b: uint32): cuchar {. + func bitScanReverse(p: ptr uint32, b: uint32): uint8 {. importc: "_BitScanReverse", header: "".} - func bitScanReverse64(p: ptr uint32, b: uint64): cuchar {. + func bitScanReverse64(p: ptr uint32, b: uint64): uint8 {. importc: "_BitScanReverse64", header: "".} template icc_scan_impl(fnc: untyped; v: untyped): int = @@ -533,37 +485,13 @@ elif useICC_builtins: discard fnc(index.addr, v) index.int - func countSetBits*(x: SomeInteger): int {.inline.} = ## Counts the set bits in an integer (also called `Hamming weight`:idx:). runnableExamples: doAssert countSetBits(0b0000_0011'u8) == 2 doAssert countSetBits(0b1010_1010'u8) == 4 - # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. - # like GCC and MSVC - when x is SomeSignedInt: - let x = x.toUnsigned - when nimvm: - result = forwardImpl(countSetBitsNim, x) - else: - when useGCC_builtins: - when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int - else: result = builtin_popcountll(x.culonglong).int - elif useVCC_builtins: - when sizeof(x) <= 2: result = builtin_popcnt16(x.uint16).int - elif sizeof(x) <= 4: result = builtin_popcnt32(x.uint32).int - elif arch64: result = builtin_popcnt64(x.uint64).int - else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).uint32).int + - builtin_popcnt32((x.uint64 shr 32'u64).uint32).int - elif useICC_builtins: - when sizeof(x) <= 4: result = builtin_popcnt32(x.cint).int - elif arch64: result = builtin_popcnt64(x.uint64).int - else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).cint).int + - builtin_popcnt32((x.uint64 shr 32'u64).cint).int - else: - when sizeof(x) <= 4: result = countSetBitsNim(x.uint32) - else: result = countSetBitsNim(x.uint64) + result = countSetBitsImpl(x) func popcount*(x: SomeInteger): int {.inline.} = ## Alias for `countSetBits <#countSetBits,SomeInteger>`_ (Hamming weight). @@ -580,8 +508,7 @@ func parityBits*(x: SomeInteger): int {.inline.} = # Can be used a base if creating ASM version. # https://stackoverflow.com/questions/21617970/how-to-check-if-value-has-even-parity-of-bits-or-odd - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: result = forwardImpl(parityImpl, x) else: @@ -604,8 +531,7 @@ func firstSetBit*(x: SomeInteger): int {.inline.} = doAssert firstSetBit(0b0000_1111'u8) == 1 # GCC builtin 'builtin_ffs' already handle zero input. - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: when noUndefined: if x == 0: @@ -647,8 +573,7 @@ func fastLog2*(x: SomeInteger): int {.inline.} = doAssert fastLog2(0b0000_1000'u8) == 3 doAssert fastLog2(0b0000_1111'u8) == 3 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return -1 @@ -690,8 +615,7 @@ func countLeadingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countLeadingZeroBits(0b0000_1000'u8) == 4 doAssert countLeadingZeroBits(0b0000_1111'u8) == 4 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -719,8 +643,7 @@ func countTrailingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countTrailingZeroBits(0b0000_1000'u8) == 3 doAssert countTrailingZeroBits(0b0000_1111'u8) == 0 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -737,7 +660,7 @@ when useBuiltinsRotate: when defined(gcc): # GCC was tested until version 4.8.1 and intrinsics were present. Not tested # in previous versions. - func builtin_rotl8(value: cuchar, shift: cint): cuchar + func builtin_rotl8(value: uint8, shift: cint): uint8 {.importc: "__rolb", header: "".} func builtin_rotl16(value: cushort, shift: cint): cushort {.importc: "__rolw", header: "".} @@ -747,7 +670,7 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: cint): culonglong {.importc: "__rolq", header: "".} - func builtin_rotr8(value: cuchar, shift: cint): cuchar + func builtin_rotr8(value: uint8, shift: cint): uint8 {.importc: "__rorb", header: "".} func builtin_rotr16(value: cushort, shift: cint): cushort {.importc: "__rorw", header: "".} @@ -763,7 +686,7 @@ when useBuiltinsRotate: # https://releases.llvm.org/8.0.0/tools/clang/docs/ReleaseNotes.html#non-comprehensive-list-of-changes-in-this-release # https://releases.llvm.org/8.0.0/tools/clang/docs/LanguageExtensions.html#builtin-rotateleft # source for correct declarations: https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/Builtins.def - func builtin_rotl8(value: cuchar, shift: cuchar): cuchar + func builtin_rotl8(value: uint8, shift: uint8): uint8 {.importc: "__builtin_rotateleft8", nodecl.} func builtin_rotl16(value: cushort, shift: cushort): cushort {.importc: "__builtin_rotateleft16", nodecl.} @@ -773,7 +696,7 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: culonglong): culonglong {.importc: "__builtin_rotateleft64", nodecl.} - func builtin_rotr8(value: cuchar, shift: cuchar): cuchar + func builtin_rotr8(value: uint8, shift: uint8): uint8 {.importc: "__builtin_rotateright8", nodecl.} func builtin_rotr16(value: cushort, shift: cushort): cushort {.importc: "__builtin_rotateright16", nodecl.} @@ -789,9 +712,9 @@ when useBuiltinsRotate: # https://docs.microsoft.com/en-us/cpp/intrinsics/rotl8-rotl16?view=msvc-160 # https://docs.microsoft.com/en-us/cpp/intrinsics/rotr8-rotr16?view=msvc-160 # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/rotl-rotl64-rotr-rotr64?view=msvc-160 - func builtin_rotl8(value: cuchar, shift: cuchar): cuchar + func builtin_rotl8(value: uint8, shift: uint8): uint8 {.importc: "_rotl8", header: "".} - func builtin_rotl16(value: cushort, shift: cuchar): cushort + func builtin_rotl16(value: cushort, shift: uint8): cushort {.importc: "_rotl16", header: "".} func builtin_rotl32(value: cuint, shift: cint): cuint {.importc: "_rotl", header: "".} @@ -799,9 +722,9 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: cint): culonglong {.importc: "_rotl64", header: "".} - func builtin_rotr8(value: cuchar, shift: cuchar): cuchar + func builtin_rotr8(value: uint8, shift: uint8): uint8 {.importc: "_rotr8", header: "".} - func builtin_rotr16(value: cushort, shift: cuchar): cushort + func builtin_rotr16(value: cushort, shift: uint8): cushort {.importc: "_rotr16", header: "".} func builtin_rotr32(value: cuint, shift: cint): cuint {.importc: "_rotr", header: "".} @@ -811,7 +734,7 @@ when useBuiltinsRotate: elif defined(icl): # Tested on Intel(R) C++ Intel(R) 64 Compiler Classic Version 2021.1.2 Build # 20201208_000000 x64 and x86. Not tested in previous versions. - func builtin_rotl8(value: cuchar, shift: cint): cuchar + func builtin_rotl8(value: uint8, shift: cint): uint8 {.importc: "__rolb", header: "".} func builtin_rotl16(value: cushort, shift: cint): cushort {.importc: "__rolw", header: "".} @@ -821,7 +744,7 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: cint): culonglong {.importc: "__rolq", header: "".} - func builtin_rotr8(value: cuchar, shift: cint): cuchar + func builtin_rotr8(value: uint8, shift: cint): uint8 {.importc: "__rorb", header: "".} func builtin_rotr16(value: cushort, shift: cint): cushort {.importc: "__rorw", header: "".} @@ -844,11 +767,13 @@ func rotr[T: SomeUnsignedInt](value: T, rot: int32): T {.inline.} = let rot = rot and mask (value shr rot) or (value shl ((-rot) and mask)) -func shiftTypeToImpl(size: static int, shift: int): auto {.inline.} = +func shiftTypeTo(size: static int, shift: int): auto {.inline.} = + ## Returns the `shift` for the rotation according to the compiler and the + ## `size`. when (defined(vcc) and (size in [4, 8])) or defined(gcc) or defined(icl): cint(shift) elif (defined(vcc) and (size in [1, 2])) or (defined(clang) and size == 1): - cuchar(shift) + uint8(shift) elif defined(clang): when size == 2: cushort(shift) @@ -857,118 +782,59 @@ func shiftTypeToImpl(size: static int, shift: int): auto {.inline.} = elif size == 8: culonglong(shift) -template shiftTypeTo[T](value: T, shift: int): auto = - ## Returns the `shift` for the rotation according to the compiler and the size - ## of the` value`. - shiftTypeToImpl(sizeof(value), shift) - -func rotateLeftBits*(value: uint8, shift: range[0..8]): uint8 {.inline.} = - ## Left-rotate bits in a 8-bits value. +func rotateLeftBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Left-rotate bits in a `value`. runnableExamples: doAssert rotateLeftBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 - - when nimvm: - rotl(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotl8(value.cuchar, shiftTypeTo(value, shift)).uint8 - else: - rotl(value, shift.int32) - -func rotateLeftBits*(value: uint16, shift: range[0..16]): uint16 {.inline.} = - ## Left-rotate bits in a 16-bits value. - runnableExamples: doAssert rotateLeftBits(0b00111100_11000011'u16, 8) == 0b11000011_00111100'u16 - - when nimvm: - rotl(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotl16(value.cushort, shiftTypeTo(value, shift)).uint16 - else: - rotl(value, shift.int32) - -func rotateLeftBits*(value: uint32, shift: range[0..32]): uint32 {.inline.} = - ## Left-rotate bits in a 32-bits value. - runnableExamples: doAssert rotateLeftBits(0b0000111111110000_1111000000001111'u32, 16) == 0b1111000000001111_0000111111110000'u32 - - when nimvm: - rotl(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotl32(value.cuint, shiftTypeTo(value, shift)).uint32 - else: - rotl(value, shift.int32) - -func rotateLeftBits*(value: uint64, shift: range[0..64]): uint64 {.inline.} = - ## Left-rotate bits in a 64-bits value. - runnableExamples: doAssert rotateLeftBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 - when nimvm: rotl(value, shift.int32) else: - when useBuiltinsRotate and defined(amd64): - builtin_rotl64(value.culonglong, shiftTypeTo(value, shift)).uint64 + when useBuiltinsRotate: + const size = sizeof(T) + when size == 1: + builtin_rotl8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotl16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotl32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotl64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotl(value, shift.int32) else: rotl(value, shift.int32) -func rotateRightBits*(value: uint8, shift: range[0..8]): uint8 {.inline.} = - ## Right-rotate bits in a 8-bits value. +func rotateRightBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Right-rotate bits in a `value`. runnableExamples: doAssert rotateRightBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 - - when nimvm: - rotr(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotr8(value.cuchar, shiftTypeTo(value, shift)).uint8 - else: - rotr(value, shift.int32) - -func rotateRightBits*(value: uint16, shift: range[0..16]): uint16 {.inline.} = - ## Right-rotate bits in a 16-bits value. - runnableExamples: doAssert rotateRightBits(0b00111100_11000011'u16, 8) == 0b11000011_00111100'u16 - - when nimvm: - rotr(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotr16(value.cushort, shiftTypeTo(value, shift)).uint16 - else: - rotr(value, shift.int32) - -func rotateRightBits*(value: uint32, shift: range[0..32]): uint32 {.inline.} = - ## Right-rotate bits in a 32-bits value. - runnableExamples: doAssert rotateRightBits(0b0000111111110000_1111000000001111'u32, 16) == 0b1111000000001111_0000111111110000'u32 - + doAssert rotateRightBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == + 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 when nimvm: rotr(value, shift.int32) else: when useBuiltinsRotate: - builtin_rotr32(value.cuint, shiftTypeTo(value, shift)).uint32 - else: - rotr(value, shift.int32) - -func rotateRightBits*(value: uint64, shift: range[0..64]): uint64 {.inline.} = - ## Right-rotate bits in a 64-bits value. - runnableExamples: - doAssert rotateRightBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == - 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 - - when nimvm: - rotr(value, shift.int32) - else: - when useBuiltinsRotate and defined(amd64): - builtin_rotr64(value.culonglong, shiftTypeTo(value, shift)).uint64 + const size = sizeof(T) + when size == 1: + builtin_rotr8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotr16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotr32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotr64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotr(value, shift.int32) else: rotr(value, shift.int32) diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index 08f5208d29..59e2078dfc 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -12,17 +12,22 @@ ## ## Unstable API. -import std/private/since +import std/private/since # used by the deprecated `openDefaultBrowser()` -import strutils +import std/strutils + +when defined(nimPreviewSlimSystem): + import std/assertions when defined(windows): - import winlean - from os import absolutePath + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + from std/os import absolutePath else: - import os + import std/os when not defined(osx): - import osproc + import std/osproc const osOpenCmd* = when defined(macos) or defined(macosx) or defined(windows): "open" else: "xdg-open" ## \ @@ -35,15 +40,17 @@ proc prepare(s: string): string = else: result = "file://" & absolutePath(s) -proc openDefaultBrowserImpl(url: string) = +proc openDefaultBrowserRaw(url: string) = + ## note the url argument should be alreadly prepared, i.e. the url is passed "AS IS" + when defined(windows): var o = newWideCString(osOpenCmd) - var u = newWideCString(prepare url) + var u = newWideCString(url) discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) elif defined(macosx): - discard execShellCmd(osOpenCmd & " " & quoteShell(prepare url)) + discard execShellCmd(osOpenCmd & " " & quoteShell(url)) else: - var u = quoteShell(prepare url) + var u = quoteShell(url) if execShellCmd(osOpenCmd & " " & u) == 0: return for b in getEnv("BROWSER").split(PathSep): try: @@ -64,26 +71,42 @@ proc openDefaultBrowser*(url: string) = ## ## This proc doesn't raise an exception on error, beware. ## - ## .. code-block:: nim + ## ```nim ## block: openDefaultBrowser("https://nim-lang.org") + ## ``` doAssert url.len > 0, "URL must not be empty string" - openDefaultBrowserImpl(url) + openDefaultBrowserRaw(url) -proc openDefaultBrowser*() {.since: (1, 1).} = - ## Opens the user's default browser without any `url` (blank page). This does not block. - ## Implements IETF RFC-6694 Section 3, "about:blank" must be reserved for a blank page. +proc openDefaultBrowser*() {.since: (1, 1), deprecated: + "not implemented, please open with a specific url instead".} = + ## Intends to open the user's default browser without any `url` (blank page). + ## This does not block. + ## Intends to implement IETF RFC-6694 Section 3, + ## ("about:blank" is reserved for a blank page). ## - ## Under Windows, `ShellExecute` is used. Under Mac OS X the `open` - ## command is used. Under Unix, it is checked if `xdg-open` exists and - ## used if it does. Otherwise the environment variable `BROWSER` is - ## used to determine the default browser to use. + ## Beware that this intended behavior is **not** implemented and + ## considered not worthy to implement here. + ## + ## The following describes the behavior of current implementation: + ## + ## - Under Windows, this will only cause a pop-up dialog \ + ## asking the assocated application with `about` \ + ## (as Windows simply treats `about:` as a protocol like `http`). + ## - Under Mac OS X the `open "about:blank"` command is used. + ## - Under Unix, it is checked if `xdg-open` exists and used \ + ## if it does and open the application assocated with `text/html` mime \ + ## (not `x-scheme-handler/http`, so maybe html-viewer \ + ## other than your default browser is opened). \ + ## Otherwise the environment variable `BROWSER` is used \ + ## to determine the default browser to use. ## ## This proc doesn't raise an exception on error, beware. ## + ## ```nim + ## block: openDefaultBrowser() + ## ``` + ## ## **See also:** ## ## * https://tools.ietf.org/html/rfc6694#section-3 - ## - ## .. code-block:: nim - ## block: openDefaultBrowser() - openDefaultBrowserImpl("http:about:blank") # See IETF RFC-6694 Section 3. + openDefaultBrowserRaw("about:blank") # See IETF RFC-6694 Section 3. diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index fd5089dbb4..034f224ac7 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -9,29 +9,32 @@ ## This module implements helper procs for CGI applications. Example: ## -## .. code-block:: Nim +## ```Nim +## import std/[strtabs, cgi] ## -## import std/[strtabs, cgi] -## -## # Fill the values when debugging: -## when debug: -## setTestData("name", "Klaus", "password", "123456") -## # read the data into `myData` -## var myData = readData() -## # check that the data's variable names are "name" or "password" -## validateData(myData, "name", "password") -## # start generating content: -## writeContentType() -## # generate content: -## write(stdout, "\n") -## write(stdout, "Test\n") -## writeLine(stdout, "your name: " & myData["name"]) -## writeLine(stdout, "your password: " & myData["password"]) -## writeLine(stdout, "") +## # Fill the values when debugging: +## when debug: +## setTestData("name", "Klaus", "password", "123456") +## # read the data into `myData` +## var myData = readData() +## # check that the data's variable names are "name" or "password" +## validateData(myData, "name", "password") +## # start generating content: +## writeContentType() +## # generate content: +## write(stdout, "\n") +## write(stdout, "Test\n") +## writeLine(stdout, "your name: " & myData["name"]) +## writeLine(stdout, "your password: " & myData["password"]) +## writeLine(stdout, "") +## ``` import std/[strutils, os, strtabs, cookies, uri] export uri.encodeUrl, uri.decodeUrl +when defined(nimPreviewSlimSystem): + import std/syncio + proc addXmlChar(dest: var string, c: char) {.inline.} = case c @@ -249,9 +252,9 @@ proc setTestData*(keysvalues: varargs[string]) = ## Fills the appropriate environment variables to test your CGI application. ## This can only simulate the 'GET' request method. `keysvalues` should ## provide embedded (name, value)-pairs. Example: - ## - ## .. code-block:: Nim - ## setTestData("name", "Hanz", "password", "12345") + ## ```Nim + ## setTestData("name", "Hanz", "password", "12345") + ## ``` putEnv("REQUEST_METHOD", "GET") var i = 0 var query = "" @@ -266,9 +269,9 @@ proc setTestData*(keysvalues: varargs[string]) = proc writeContentType*() = ## Calls this before starting to send your HTML data to `stdout`. This ## implements this part of the CGI protocol: - ## - ## .. code-block:: Nim - ## write(stdout, "Content-type: text/html\n\n") + ## ```Nim + ## write(stdout, "Content-type: text/html\n\n") + ## ``` write(stdout, "Content-type: text/html\n\n") proc resetForStacktrace() = diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index e129849954..24257dacb9 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -36,6 +36,9 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + type NodeObj[T] {.acyclic.} = object byte: int ## byte index of the difference @@ -197,7 +200,7 @@ proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = discard exclImpl(c, key) result = c.count == oldCount -proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = +proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: sink T): bool = ## Returns true if `c` contains the given `key`. If the key does not exist, ## `c[key] = val` is performed. ## @@ -270,7 +273,7 @@ proc incl*(c: var CritBitTree[void], key: string) = discard rawInsert(c, key) -proc incl*[T](c: var CritBitTree[T], key: string, val: T) = +proc incl*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Inserts `key` with value `val` into `c`. ## ## **See also:** @@ -284,7 +287,7 @@ proc incl*[T](c: var CritBitTree[T], key: string, val: T) = var n = rawInsert(c, key) n.val = val -proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) = +proc `[]=`*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Alias for `incl <#incl,CritBitTree[T],string,T>`_. ## ## **See also:** @@ -300,7 +303,7 @@ template get[T](c: CritBitTree[T], key: string): T = n.val -func `[]`*[T](c: CritBitTree[T], key: string): T {.inline.} = +func `[]`*[T](c: CritBitTree[T], key: string): lent T {.inline.} = ## Retrieves the value at `c[key]`. If `key` is not in `t`, the ## `KeyError` exception is raised. One can check with `hasKey` whether ## the key exists. @@ -342,7 +345,7 @@ iterator keys*[T](c: CritBitTree[T]): string = for x in leaves(c.root): yield x.key -iterator values*[T](c: CritBitTree[T]): T = +iterator values*[T](c: CritBitTree[T]): lent T = ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. ## @@ -415,7 +418,7 @@ iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string = let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.key -iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): T = +iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): lent T = ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. ## @@ -518,7 +521,7 @@ func commonPrefixLen*[T](c: CritBitTree[T]): int {.inline, since((1, 3)).} = else: c.root.byte else: 0 -proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = +proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `pairs`. runnableExamples: doAssert {"a": "0", "b": "1", "c": "2"}.toCritBitTree is CritBitTree[string] @@ -526,7 +529,7 @@ proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: ( for item in pairs: result.incl item[0], item[1] -proc toCritBitTree*(items: openArray[string]): CritBitTree[void] {.since: (1, 3).} = +proc toCritBitTree*(items: sink openArray[string]): CritBitTree[void] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `items`. runnableExamples: doAssert ["a", "b", "c"].toCritBitTree is CritBitTree[void] diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index c8bbb1cc67..d2b0099f23 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -10,8 +10,9 @@ ## An implementation of a `deque`:idx: (double-ended queue). ## The underlying implementation uses a `seq`. ## -## Note that none of the procs that get an individual value from the deque should be used -## on an empty deque. +## .. note:: None of the procs that get an individual value from the deque should be used +## on an empty deque. +## ## If compiled with the `boundChecks` option, those procs will raise an `IndexDefect` ## on such access. This should not be relied upon, as `-d:danger` or `--checks:off` will ## disable those checks and then the procs may return garbage or crash the program. @@ -46,11 +47,10 @@ runnableExamples: ## See also ## ======== ## * `lists module `_ for singly and doubly linked lists and rings -## * `channels module `_ for inter-thread communication import std/private/since -import math +import std/[assertions, hashes, math] type Deque*[T] = object @@ -59,20 +59,28 @@ type ## To initialize an empty deque, ## use the `initDeque proc <#initDeque,int>`_. data: seq[T] - head, tail, count, mask: int + + # `head` and `tail` are masked only when accessing an element of `data` + # so that `tail - head == data.len` when the deque is full. + # They are uint so that incrementing/decrementing them doesn't cause + # over/underflow. You can get a number of items with `tail - head` + # even if `tail` or `head` is wraps around and `tail < head`, because + # `tail - head == (uint.high + 1 + tail) - head` when `tail < head`. + head, tail: uint const defaultInitialSize* = 4 template initImpl(result: typed, initialSize: int) = let correctSize = nextPowerOfTwo(initialSize) - result.mask = correctSize - 1 newSeq(result.data, correctSize) template checkIfInitialized(deq: typed) = - when compiles(defaultInitialSize): - if deq.mask == 0: - initImpl(deq, defaultInitialSize) + if deq.data.len == 0: + initImpl(deq, defaultInitialSize) + +func mask[T](deq: Deque[T]): uint {.inline.} = + uint(deq.data.len) - 1 proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## Creates a new empty deque. @@ -86,41 +94,27 @@ proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## * `toDeque proc <#toDeque,openArray[T]>`_ result.initImpl(initialSize) -proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = - ## Creates a new deque that contains the elements of `x` (in the same order). - ## - ## **See also:** - ## * `initDeque proc <#initDeque,int>`_ - runnableExamples: - let a = toDeque([7, 8, 9]) - assert len(a) == 3 - assert $a == "[7, 8, 9]" - - result.initImpl(x.len) - for item in items(x): - result.addLast(item) - -proc len*[T](deq: Deque[T]): int {.inline.} = +func len*[T](deq: Deque[T]): int {.inline.} = ## Returns the number of elements of `deq`. - result = deq.count + int(deq.tail - deq.head) template emptyCheck(deq) = # Bounds check for the regular deque access. when compileOption("boundChecks"): - if unlikely(deq.count < 1): + if unlikely(deq.len < 1): raise newException(IndexDefect, "Empty deque.") template xBoundsCheck(deq, i) = # Bounds check for the array like accesses. when compileOption("boundChecks"): # `-d:danger` or `--checks:off` should disable this. - if unlikely(i >= deq.count): # x < deq.low is taken care by the Natural parameter + if unlikely(i >= deq.len): # x < deq.low is taken care by the Natural parameter raise newException(IndexDefect, - "Out of bounds: " & $i & " > " & $(deq.count - 1)) + "Out of bounds: " & $i & " > " & $(deq.len - 1)) if unlikely(i < 0): # when used with BackwardsIndex raise newException(IndexDefect, "Out of bounds: " & $i & " < 0") -proc `[]`*[T](deq: Deque[T], i: Natural): T {.inline.} = +proc `[]`*[T](deq: Deque[T], i: Natural): lent T {.inline.} = ## Accesses the `i`-th element of `deq`. runnableExamples: let a = [10, 20, 30, 40, 50].toDeque @@ -129,7 +123,7 @@ proc `[]`*[T](deq: Deque[T], i: Natural): T {.inline.} = doAssertRaises(IndexDefect, echo a[8]) xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = ## Accesses the `i`-th element of `deq` and returns a mutable @@ -140,9 +134,9 @@ proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = assert a[0] == 11 xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] -proc `[]=`*[T](deq: var Deque[T], i: Natural, val: T) {.inline.} = +proc `[]=`*[T](deq: var Deque[T], i: Natural, val: sink T) {.inline.} = ## Sets the `i`-th element of `deq` to `val`. runnableExamples: var a = [10, 20, 30, 40, 50].toDeque @@ -152,9 +146,9 @@ proc `[]=`*[T](deq: var Deque[T], i: Natural, val: T) {.inline.} = checkIfInitialized(deq) xBoundsCheck(deq, i) - deq.data[(deq.head + i) and deq.mask] = val + deq.data[(deq.head + i.uint) and deq.mask] = val -proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): T {.inline.} = +proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): lent T {.inline.} = ## Accesses the backwards indexed `i`-th element. ## ## `deq[^1]` is the last element. @@ -180,7 +174,7 @@ proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = xBoundsCheck(deq, deq.len - int(i)) return deq[deq.len - int(i)] -proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = +proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: sink T) {.inline.} = ## Sets the backwards indexed `i`-th element of `deq` to `x`. ## ## `deq[^1]` is the last element. @@ -194,27 +188,25 @@ proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = xBoundsCheck(deq, deq.len - int(i)) deq[deq.len - int(i)] = x -iterator items*[T](deq: Deque[T]): T = +iterator items*[T](deq: Deque[T]): lent T = ## Yields every element of `deq`. ## ## **See also:** - ## * `mitems iterator <#mitems,Deque[T]>`_ + ## * `mitems iterator <#mitems.i,Deque[T]>`_ runnableExamples: from std/sequtils import toSeq let a = [10, 20, 30, 40, 50].toDeque assert toSeq(a.items) == @[10, 20, 30, 40, 50] - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator mitems*[T](deq: var Deque[T]): var T = ## Yields every element of `deq`, which can be modified. ## ## **See also:** - ## * `items iterator <#items,Deque[T]>`_ + ## * `items iterator <#items.i,Deque[T]>`_ runnableExamples: var a = [10, 20, 30, 40, 50].toDeque assert $a == "[10, 20, 30, 40, 50]" @@ -222,10 +214,8 @@ iterator mitems*[T](deq: var Deque[T]): var T = x = 5 * x - 1 assert $a == "[49, 99, 149, 199, 249]" - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = ## Yields every `(position, value)`-pair of `deq`. @@ -235,10 +225,8 @@ iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = let a = [10, 20, 30].toDeque assert toSeq(a.pairs) == @[(0, 10), (1, 20), (2, 30)] - var i = deq.head - for c in 0 ..< deq.count: - yield (c, deq.data[i]) - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield (c, deq.data[(deq.head + c.uint) and deq.mask]) proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = ## Returns true if `item` is in `deq` or false if not found. @@ -257,24 +245,24 @@ proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = proc expandIfNeeded[T](deq: var Deque[T]) = checkIfInitialized(deq) - var cap = deq.mask + 1 - if unlikely(deq.count >= cap): + let cap = deq.data.len + assert deq.len <= cap + if unlikely(deq.len == cap): var n = newSeq[T](cap * 2) var i = 0 for x in mitems(deq): - when nimVM: n[i] = x # workaround for VM bug + when nimvm: n[i] = x # workaround for VM bug else: n[i] = move(x) inc i deq.data = move(n) - deq.mask = cap * 2 - 1 - deq.tail = deq.count + deq.tail = cap.uint deq.head = 0 -proc addFirst*[T](deq: var Deque[T], item: T) = +proc addFirst*[T](deq: var Deque[T], item: sink T) = ## Adds an `item` to the beginning of `deq`. ## ## **See also:** - ## * `addLast proc <#addLast,Deque[T],T>`_ + ## * `addLast proc <#addLast,Deque[T],sinkT>`_ runnableExamples: var a = initDeque[int]() for i in 1 .. 5: @@ -282,15 +270,14 @@ proc addFirst*[T](deq: var Deque[T], item: T) = assert $a == "[50, 40, 30, 20, 10]" expandIfNeeded(deq) - inc deq.count - deq.head = (deq.head - 1) and deq.mask - deq.data[deq.head] = item + dec deq.head + deq.data[deq.head and deq.mask] = item -proc addLast*[T](deq: var Deque[T], item: T) = +proc addLast*[T](deq: var Deque[T], item: sink T) = ## Adds an `item` to the end of `deq`. ## ## **See also:** - ## * `addFirst proc <#addFirst,Deque[T],T>`_ + ## * `addFirst proc <#addFirst,Deque[T],sinkT>`_ runnableExamples: var a = initDeque[int]() for i in 1 .. 5: @@ -298,11 +285,24 @@ proc addLast*[T](deq: var Deque[T], item: T) = assert $a == "[10, 20, 30, 40, 50]" expandIfNeeded(deq) - inc deq.count - deq.data[deq.tail] = item - deq.tail = (deq.tail + 1) and deq.mask + deq.data[deq.tail and deq.mask] = item + inc deq.tail -proc peekFirst*[T](deq: Deque[T]): T {.inline.} = +proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = + ## Creates a new deque that contains the elements of `x` (in the same order). + ## + ## **See also:** + ## * `initDeque proc <#initDeque,int>`_ + runnableExamples: + let a = toDeque([7, 8, 9]) + assert len(a) == 3 + assert $a == "[7, 8, 9]" + + result.initImpl(x.len) + for item in items(x): + result.addLast(item) + +proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the first element of `deq`, but does not remove it from the deque. ## ## **See also:** @@ -315,9 +315,9 @@ proc peekFirst*[T](deq: Deque[T]): T {.inline.} = assert len(a) == 5 emptyCheck(deq) - result = deq.data[deq.head] + result = deq.data[deq.head and deq.mask] -proc peekLast*[T](deq: Deque[T]): T {.inline.} = +proc peekLast*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the last element of `deq`, but does not remove it from the deque. ## ## **See also:** @@ -345,7 +345,7 @@ proc peekFirst*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = assert $a == "[99, 20, 30, 40, 50]" emptyCheck(deq) - result = deq.data[deq.head] + result = deq.data[deq.head and deq.mask] proc peekLast*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = ## Returns a mutable reference to the last element of `deq`, @@ -378,10 +378,8 @@ proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = assert $a == "[20, 30, 40, 50]" emptyCheck(deq) - dec deq.count - result = deq.data[deq.head] - destroy(deq.data[deq.head]) - deq.head = (deq.head + 1) and deq.mask + result = move deq.data[deq.head and deq.mask] + inc deq.head proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = ## Removes and returns the last element of the `deq`. @@ -396,10 +394,8 @@ proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = assert $a == "[10, 20, 30, 40]" emptyCheck(deq) - dec deq.count - deq.tail = (deq.tail - 1) and deq.mask - result = deq.data[deq.tail] - destroy(deq.data[deq.tail]) + dec deq.tail + result = move deq.data[deq.tail and deq.mask] proc clear*[T](deq: var Deque[T]) {.inline.} = ## Resets the deque so that it is empty. @@ -413,7 +409,6 @@ proc clear*[T](deq: var Deque[T]) {.inline.} = assert len(a) == 0 for el in mitems(deq): destroy(el) - deq.count = 0 deq.tail = deq.head proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = @@ -433,19 +428,17 @@ proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = a.shrink(fromFirst = 2, fromLast = 1) assert $a == "[30, 40]" - if fromFirst + fromLast > deq.count: + if fromFirst + fromLast > deq.len: clear(deq) return for i in 0 ..< fromFirst: - destroy(deq.data[deq.head]) - deq.head = (deq.head + 1) and deq.mask + destroy(deq.data[deq.head and deq.mask]) + inc deq.head for i in 0 ..< fromLast: - destroy(deq.data[deq.tail]) - deq.tail = (deq.tail - 1) and deq.mask - - dec deq.count, fromFirst + fromLast + destroy(deq.data[(deq.tail - 1) and deq.mask]) + dec deq.tail proc `$`*[T](deq: Deque[T]): string = ## Turns a deque into its string representation. @@ -458,3 +451,30 @@ proc `$`*[T](deq: Deque[T]): string = if result.len > 1: result.add(", ") result.addQuoted(x) result.add("]") + +func `==`*[T](deq1, deq2: Deque[T]): bool = + ## The `==` operator for Deque. + ## Returns `true` if both deques contains the same values in the same order. + runnableExamples: + var a, b = initDeque[int]() + a.addFirst(2) + a.addFirst(1) + b.addLast(1) + b.addLast(2) + doAssert a == b + + if deq1.len != deq2.len: + return false + + for i in 0 ..< deq1.len: + if deq1.data[(deq1.head + i.uint) and deq1.mask] != deq2.data[(deq2.head + i.uint) and deq2.mask]: + return false + + true + +func hash*[T](deq: Deque[T]): Hash = + ## Hashing of Deque. + var h: Hash = 0 + for x in deq: + h = h !& hash(x) + !$h diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index 0304461766..17785c8c70 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -10,6 +10,11 @@ # An `include` file which contains common code for # hash sets and tables. +when defined(nimPreviewSlimSystem): + import std/assertions + +import std / outparams + const growthFactor = 2 @@ -33,16 +38,6 @@ proc slotsNeeded(count: Natural): int {.inline.} = # Make sure to synchronize with `mustRehash` above result = nextPowerOfTwo(count * 3 div 2 + 4) -proc rightSize*(count: Natural): int {.inline, deprecated: "Deprecated since 1.4.0".} = - ## **Deprecated since Nim v1.4.0**, it is not needed anymore - ## because picking the correct size is done internally. - ## - ## Return the value of `initialSize` to support `count` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - result = count - template rawGetKnownHCImpl() {.dirty.} = if t.dataLen == 0: return -1 @@ -63,7 +58,10 @@ proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} = template genHashImpl(key, hc: typed) = hc = hash(key) if hc == 0: # This almost never taken branch should be very predictable. - hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + when sizeof(int) < 4: + hc = 31415 # Value doesn't matter; Any non-zero favorite is fine <= 16-bit. + else: + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. template genHash(key: typed): Hash = var res: Hash @@ -74,5 +72,5 @@ template rawGetImpl() {.dirty.} = genHashImpl(key, hc) rawGetKnownHCImpl() -proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetImpl() diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim index 89e532951a..96f9b44300 100644 --- a/lib/pure/collections/heapqueue.nim +++ b/lib/pure/collections/heapqueue.nim @@ -47,6 +47,9 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + type HeapQueue*[T] = object ## A heap queue, commonly known as a priority queue. data: seq[T] @@ -59,7 +62,7 @@ proc initHeapQueue*[T](): HeapQueue[T] = ## ## **See also:** ## * `toHeapQueue proc <#toHeapQueue,openArray[T]>`_ - discard + result = default(HeapQueue[T]) proc len*[T](heap: HeapQueue[T]): int {.inline.} = ## Returns the number of elements of `heap`. @@ -73,6 +76,13 @@ proc `[]`*[T](heap: HeapQueue[T], i: Natural): lent T {.inline.} = ## Accesses the i-th element of `heap`. heap.data[i] +iterator items*[T](heap: HeapQueue[T]): lent T {.inline, since: (2, 1, 1).} = + ## Iterates over each item of `heap`. + let L = len(heap) + for i in 0 .. high(heap.data): + yield heap.data[i] + assert(len(heap) == L, "the length of the HeapQueue changed while iterating over it") + proc heapCmp[T](x, y: T): bool {.inline.} = x < y proc siftup[T](heap: var HeapQueue[T], startpos, p: int) = @@ -178,6 +188,11 @@ proc find*[T](heap: HeapQueue[T], x: T): int {.since: (1, 3).} = for i in 0 ..< heap.len: if heap[i] == x: return i +proc contains*[T](heap: HeapQueue[T], x: T): bool {.since: (2, 1, 1).} = + ## Returns true if `x` is in `heap` or false if not found. This is a shortcut + ## for `find(heap, x) >= 0`. + result = find(heap, x) >= 0 + proc del*[T](heap: var HeapQueue[T], index: Natural) = ## Removes the element at `index` from `heap`, maintaining the heap invariant. runnableExamples: diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 05e5addcc1..765a23e973 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -8,7 +8,7 @@ # ## Specialization of the generic `packedsets module `_ -## for ordinal sparse sets. +## (see its documentation for more examples) for ordinal sparse sets. import std/private/since import std/packedsets diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index c1fcb07433..6b88747ef3 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -19,15 +19,15 @@ ## ## ## Lists runnableExamples: - var l = initDoublyLinkedList[int]() + var list = initDoublyLinkedList[int]() let a = newDoublyLinkedNode[int](3) b = newDoublyLinkedNode[int](7) c = newDoublyLinkedNode[int](9) - l.add(a) - l.add(b) - l.prepend(c) + list.add(a) + list.add(b) + list.prepend(c) assert a.next == b assert a.prev == c @@ -38,15 +38,15 @@ runnableExamples: ## ## Rings runnableExamples: - var l = initSinglyLinkedRing[int]() + var ring = initSinglyLinkedRing[int]() let a = newSinglyLinkedNode[int](3) b = newSinglyLinkedNode[int](7) c = newSinglyLinkedNode[int](9) - l.add(a) - l.add(b) - l.prepend(c) + ring.add(a) + ring.add(b) + ring.prepend(c) assert c.next == a assert a.next == b @@ -56,19 +56,18 @@ runnableExamples: ## # See also ## * `deques module `_ for double-ended queues -## * `sharedlist module `_ for shared singly-linked lists import std/private/since -when not defined(nimHasCursor): - {.pragma: cursor.} +when defined(nimPreviewSlimSystem): + import std/assertions type DoublyLinkedNodeObj*[T] = object ## A node of a doubly linked list. ## ## It consists of a `value` field, and pointers to `next` and `prev`. - next*: (DoublyLinkedNode[T]) + next*: DoublyLinkedNode[T] prev* {.cursor.}: DoublyLinkedNode[T] value*: T DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T] @@ -77,23 +76,23 @@ type ## A node of a singly linked list. ## ## It consists of a `value` field, and a pointer to `next`. - next*: (SinglyLinkedNode[T]) + next*: SinglyLinkedNode[T] value*: T SinglyLinkedNode*[T] = ref SinglyLinkedNodeObj[T] SinglyLinkedList*[T] = object ## A singly linked list. - head*: (SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] DoublyLinkedList*[T] = object ## A doubly linked list. - head*: (DoublyLinkedNode[T]) + head*: DoublyLinkedNode[T] tail* {.cursor.}: DoublyLinkedNode[T] SinglyLinkedRing*[T] = object ## A singly linked ring. - head*: (SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] DoublyLinkedRing*[T] = object @@ -148,7 +147,7 @@ proc initDoublyLinkedRing*[T](): DoublyLinkedRing[T] = discard -proc newDoublyLinkedNode*[T](value: T): (DoublyLinkedNode[T]) = +proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] = ## Creates a new doubly linked node with the given `value`. runnableExamples: let n = newDoublyLinkedNode[int](5) @@ -157,7 +156,7 @@ proc newDoublyLinkedNode*[T](value: T): (DoublyLinkedNode[T]) = new(result) result.value = value -proc newSinglyLinkedNode*[T](value: T): (SinglyLinkedNode[T]) = +proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] = ## Creates a new singly linked node with the given `value`. runnableExamples: let n = newSinglyLinkedNode[int](5) @@ -166,44 +165,22 @@ proc newSinglyLinkedNode*[T](value: T): (SinglyLinkedNode[T]) = new(result) result.value = value -func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = - ## Creates a new `SinglyLinkedList` from the members of `elems`. - runnableExamples: - from std/sequtils import toSeq - let a = [1, 2, 3, 4, 5].toSinglyLinkedList - assert a.toSeq == [1, 2, 3, 4, 5] - - result = initSinglyLinkedList[T]() - for elem in elems.items: - result.add(elem) - -func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = - ## Creates a new `DoublyLinkedList` from the members of `elems`. - runnableExamples: - from std/sequtils import toSeq - let a = [1, 2, 3, 4, 5].toDoublyLinkedList - assert a.toSeq == [1, 2, 3, 4, 5] - - result = initDoublyLinkedList[T]() - for elem in elems.items: - result.add(elem) - template itemsListImpl() {.dirty.} = - var it = list.head + var it {.cursor.} = L.head while it != nil: yield it.value it = it.next template itemsRingImpl() {.dirty.} = - var it = ring.head + var it {.cursor.} = L.head if it != nil: while true: yield it.value it = it.next - if it == ring.head: break + if it == L.head: break -iterator items*[T](list: SomeLinkedList[T]): T = - ## Yields every value of `list`. +iterator items*[T](L: SomeLinkedList[T]): T = + ## Yields every value of `L`. ## ## **See also:** ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ @@ -218,8 +195,8 @@ iterator items*[T](list: SomeLinkedList[T]): T = itemsListImpl() -iterator items*[T](ring: SomeLinkedRing[T]): T = - ## Yields every value of `ring`. +iterator items*[T](L: SomeLinkedRing[T]): T = + ## Yields every value of `L`. ## ## **See also:** ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ @@ -234,8 +211,8 @@ iterator items*[T](ring: SomeLinkedRing[T]): T = itemsRingImpl() -iterator mitems*[T](list: var SomeLinkedList[T]): var T = - ## Yields every value of `list` so that you can modify it. +iterator mitems*[T](L: var SomeLinkedList[T]): var T = + ## Yields every value of `L` so that you can modify it. ## ## **See also:** ## * `items iterator <#items.i,SomeLinkedList[T]>`_ @@ -251,8 +228,8 @@ iterator mitems*[T](list: var SomeLinkedList[T]): var T = itemsListImpl() -iterator mitems*[T](ring: var SomeLinkedRing[T]): var T = - ## Yields every value of `ring` so that you can modify it. +iterator mitems*[T](L: var SomeLinkedRing[T]): var T = + ## Yields every value of `L` so that you can modify it. ## ## **See also:** ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ @@ -268,7 +245,7 @@ iterator mitems*[T](ring: var SomeLinkedRing[T]): var T = itemsRingImpl() -iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] = +iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. ## @@ -287,13 +264,13 @@ iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] = x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = list.head + var it {.cursor.} = L.head while it != nil: let nxt = it.next yield it it = nxt -iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] = +iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. ## @@ -312,27 +289,27 @@ iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] = x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = ring.head + var it {.cursor.} = L.head if it != nil: while true: let nxt = it.next yield it it = nxt - if it == ring.head: break + if it == L.head: break -proc `$`*[T](l: SomeLinkedCollection[T]): string = +proc `$`*[T](L: SomeLinkedCollection[T]): string = ## Turns a list into its string representation for logging and printing. runnableExamples: let a = [1, 2, 3, 4].toSinglyLinkedList assert $a == "[1, 2, 3, 4]" result = "[" - for x in nodes(l): + for x in nodes(L): if result.len > 1: result.add(", ") result.addQuoted(x.value) result.add("]") -proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = +proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = ## Searches in the list for a value. Returns `nil` if the value does not ## exist. ## @@ -343,10 +320,10 @@ proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = assert a.find(9).value == 9 assert a.find(1) == nil - for x in nodes(l): + for x in nodes(L): if x.value == value: return x -proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} = +proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} = ## Searches in the list for a value. Returns `false` if the value does not ## exist, `true` otherwise. This allows the usage of the `in` and `notin` ## operators. @@ -360,7 +337,7 @@ proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} = assert(not a.contains(1)) assert 2 notin a - result = find(l, value) != nil + result = find(L, value) != nil proc prepend*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = ## Prepends a shallow copy of `b` to the beginning of `a`. @@ -407,12 +384,10 @@ proc prependMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = assert s == [0, 1, 0, 1, 0, 1] b.addMoved(a) - when defined(js): # XXX: swap broken in js; bug #16771 - (b, a) = (a, b) - else: swap a, b + swap a, b -proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = - ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value @@ -426,14 +401,14 @@ proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = assert a.contains(9) n.next = nil - if list.tail != nil: - assert(list.tail.next == nil) - list.tail.next = n - list.tail = n - if list.head == nil: list.head = n + if L.tail != nil: + assert(L.tail.next == nil) + L.tail.next = n + L.tail = n + if L.head == nil: L.head = n -proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = - ## Appends (adds to the end) a value to `list`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value @@ -446,11 +421,11 @@ proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = a.add(8) assert a.contains(9) - add(list, newSinglyLinkedNode(value)) + add(L, newSinglyLinkedNode(value)) -proc prepend*[T](list: var SinglyLinkedList[T], +proc prepend*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = - ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ @@ -463,12 +438,12 @@ proc prepend*[T](list: var SinglyLinkedList[T], a.prepend(n) assert a.contains(9) - n.next = list.head - list.head = n - if list.tail == nil: list.tail = n + n.next = L.head + L.head = n + if L.tail == nil: L.tail = n -proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = - ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1). +proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ @@ -482,7 +457,7 @@ proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = a.prepend(8) assert a.contains(9) - prepend(list, newSinglyLinkedNode(value)) + prepend(L, newSinglyLinkedNode(value)) func copy*[T](a: SinglyLinkedList[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = ## Creates a shallow copy of `a`. @@ -531,17 +506,18 @@ proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} = ci assert s == [0, 1, 0, 1, 0, 1] - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil -proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value @@ -557,15 +533,15 @@ proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = assert a.contains(9) n.next = nil - n.prev = list.tail - if list.tail != nil: - assert(list.tail.next == nil) - list.tail.next = n - list.tail = n - if list.head == nil: list.head = n + n.prev = L.tail + if L.tail != nil: + assert(L.tail.next == nil) + L.tail.next = n + L.tail = n + if L.head == nil: L.head = n -proc add*[T](list: var DoublyLinkedList[T], value: T) = - ## Appends (adds to the end) a value to `list`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedList[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ @@ -581,10 +557,10 @@ proc add*[T](list: var DoublyLinkedList[T], value: T) = a.add(8) assert a.contains(9) - add(list, newDoublyLinkedNode(value)) + add(L, newDoublyLinkedNode(value)) -proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## Prepends (adds to the beginning) a node `n` to `list`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ @@ -600,15 +576,15 @@ proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = assert a.contains(9) n.prev = nil - n.next = list.head - if list.head != nil: - assert(list.head.prev == nil) - list.head.prev = n - list.head = n - if list.tail == nil: list.tail = n + n.next = L.head + if L.head != nil: + assert(L.head.prev == nil) + L.head.prev = n + L.head = n + if L.tail == nil: L.tail = n -proc prepend*[T](list: var DoublyLinkedList[T], value: T) = - ## Prepends (adds to the beginning) a value to `list`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedList[T], value: T) = + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ @@ -624,7 +600,7 @@ proc prepend*[T](list: var DoublyLinkedList[T], value: T) = a.prepend(8) assert a.contains(9) - prepend(list, newDoublyLinkedNode(value)) + prepend(L, newDoublyLinkedNode(value)) func copy*[T](a: DoublyLinkedList[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = ## Creates a shallow copy of `a`. @@ -675,12 +651,12 @@ proc addMoved*[T](a, b: var DoublyLinkedList[T]) {.since: (1, 5, 1).} = assert s == [0, 1, 0, 1, 0, 1] if b.head != nil: - b.head.prev = a.tail - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if a.head == nil: + a.head = b.head + else: + b.head.prev = a.tail + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil @@ -705,9 +681,9 @@ proc add*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = var tmp = b.copy a.addMoved(tmp) -proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} = - ## Removes a node `n` from `list`. - ## Returns `true` if `n` was found in `list`. +proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} = + ## Removes a node `n` from `L`. + ## Returns `true` if `n` was found in `L`. ## Efficiency: O(n); the list is traversed until `n` is found. ## Attempting to remove an element not contained in the list is a no-op. ## When the list is cyclic, the cycle is preserved after removal. @@ -728,22 +704,24 @@ proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.d ai assert s == [2, 2, 2, 2] - if n == list.head: - list.head = n.next - if list.tail.next == n: - list.tail.next = list.head # restore cycle + if n == L.head: + L.head = n.next + if L.tail.next == n: + L.tail.next = L.head # restore cycle else: - var prev = list.head + var prev {.cursor.} = L.head while prev.next != n and prev.next != nil: prev = prev.next if prev.next == nil: return false prev.next = n.next + if L.tail == n: + L.tail = prev # update tail if we removed the last node true -proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## Removes a node `n` from `list`. Efficiency: O(1). - ## This function assumes, for the sake of efficiency, that `n` is contained in `list`, +proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = + ## Removes a node `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, ## otherwise the effects are undefined. ## When the list is cyclic, the cycle is preserved after removal. runnableExamples: @@ -763,15 +741,15 @@ proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = ai assert s == [2, 2, 2, 2] - if n == list.tail: list.tail = n.prev - if n == list.head: list.head = n.next + if n == L.tail: L.tail = n.prev + if n == L.head: L.head = n.next if n.next != nil: n.next.prev = n.prev if n.prev != nil: n.prev.next = n.next -proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value @@ -784,17 +762,17 @@ proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = a.add(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - assert(ring.tail != nil) - ring.tail.next = n + if L.head != nil: + n.next = L.head + assert(L.tail != nil) + L.tail.next = n else: n.next = n - ring.head = n - ring.tail = n + L.head = n + L.tail = n -proc add*[T](ring: var SinglyLinkedRing[T], value: T) = - ## Appends (adds to the end) a value to `ring`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedRing[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ @@ -808,10 +786,10 @@ proc add*[T](ring: var SinglyLinkedRing[T], value: T) = a.add(8) assert a.contains(9) - add(ring, newSinglyLinkedNode(value)) + add(L, newSinglyLinkedNode(value)) -proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1). +proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ @@ -824,17 +802,17 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = a.prepend(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - assert(ring.tail != nil) - ring.tail.next = n + if L.head != nil: + n.next = L.head + assert(L.tail != nil) + L.tail.next = n else: n.next = n - ring.tail = n - ring.head = n + L.tail = n + L.head = n -proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) = - ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1). +proc prepend*[T](L: var SinglyLinkedRing[T], value: T) = + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ @@ -848,12 +826,12 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) = a.prepend(8) assert a.contains(9) - prepend(ring, newSinglyLinkedNode(value)) + prepend(L, newSinglyLinkedNode(value)) -proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value @@ -868,18 +846,18 @@ proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = a.add(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - n.prev = ring.head.prev - ring.head.prev.next = n - ring.head.prev = n + if L.head != nil: + n.next = L.head + n.prev = L.head.prev + L.head.prev.next = n + L.head.prev = n else: n.prev = n n.next = n - ring.head = n + L.head = n -proc add*[T](ring: var DoublyLinkedRing[T], value: T) = - ## Appends (adds to the end) a value to `ring`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedRing[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ @@ -895,10 +873,10 @@ proc add*[T](ring: var DoublyLinkedRing[T], value: T) = a.add(8) assert a.contains(9) - add(ring, newDoublyLinkedNode(value)) + add(L, newDoublyLinkedNode(value)) -proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ @@ -913,18 +891,18 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = a.prepend(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - n.prev = ring.head.prev - ring.head.prev.next = n - ring.head.prev = n + if L.head != nil: + n.next = L.head + n.prev = L.head.prev + L.head.prev.next = n + L.head.prev = n else: n.prev = n n.next = n - ring.head = n + L.head = n -proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) = - ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedRing[T], value: T) = + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ @@ -940,11 +918,11 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) = a.prepend(8) assert a.contains(9) - prepend(ring, newDoublyLinkedNode(value)) + prepend(L, newDoublyLinkedNode(value)) -proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## Removes `n` from `ring`. Efficiency: O(1). - ## This function assumes, for the sake of efficiency, that `n` is contained in `ring`, +proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = + ## Removes `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, ## otherwise the effects are undefined. runnableExamples: var a = initDoublyLinkedRing[int]() @@ -956,13 +934,13 @@ proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = n.next.prev = n.prev n.prev.next = n.next - if n == ring.head: - let p = ring.head.prev - if p == ring.head: + if n == L.head: + let p = L.head.prev + if p == L.head: # only one element left: - ring.head = nil + L.head = nil else: - ring.head = p + L.head = p proc append*[T](a: var (SinglyLinkedList[T] | SinglyLinkedRing[T]), b: SinglyLinkedList[T] | SinglyLinkedNode[T] | T) = @@ -991,3 +969,47 @@ proc appendMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = ## * `addMoved proc <#addMoved,SinglyLinkedList[T],SinglyLinkedList[T]>`_ ## * `addMoved proc <#addMoved,DoublyLinkedList[T],DoublyLinkedList[T]>`_ a.addMoved(b) + +func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `SinglyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toSinglyLinkedRing*[T](elems: openArray[T]): SinglyLinkedRing[T] = + ## Creates a new `SinglyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedRing[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `DoublyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedRing*[T](elems: openArray[T]): DoublyLinkedRing[T] = + ## Creates a new `DoublyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedRing[T]() + for elem in elems.items: + result.add(elem) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index e937b83944..3c0d8dc0eb 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -82,7 +82,17 @@ runnableExamples: import std/private/since -import macros +import std/macros +from std/typetraits import supportsCopyMem + +when defined(nimPreviewSlimSystem): + import std/assertions + + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped = @@ -131,6 +141,22 @@ func concat*[T](seqs: varargs[seq[T]]): seq[T] = result[i] = itm inc(i) +func addUnique*[T](s: var seq[T], x: sink T) = + ## Adds `x` to the container `s` if it is not already present. + ## Uses `==` to check if the item is already present. + runnableExamples: + var a = @[1, 2, 3] + a.addUnique(4) + a.addUnique(4) + assert a == @[1, 2, 3, 4] + + for i in 0..high(s): + if s[i] == x: return + when declared(ensureMove): + s.add ensureMove(x) + else: + s.add x + func count*[T](s: openArray[T], x: T): int = ## Returns the number of occurrences of the item `x` in the container `s`. ## @@ -164,7 +190,7 @@ func cycle*[T](s: openArray[T], n: Natural): seq[T] = result[o] = e inc o -func repeat*[T](x: T, n: Natural): seq[T] = +proc repeat*[T](x: T, n: Natural): seq[T] = ## Returns a new sequence with the item `x` repeated `n` times. ## `n` must be a non-negative number (zero or more). ## @@ -239,9 +265,18 @@ func maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = for i in 1..high(s): if s[i] > s[result]: result = i +func minmax*[T](x: openArray[T]): (T, T) = + ## The minimum and maximum values of `x`. `T` needs to have a `<` operator. + var l = x[0] + var h = x[0] + for i in 1..high(x): + if x[i] < l: l = x[i] + if h < x[i]: h = x[i] + result = (l, h) + template zipImpl(s1, s2, retType: untyped): untyped = - func zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = + proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = ## Returns a new sequence with a combination of the two input containers. ## ## The input containers can be of different types. @@ -284,7 +319,7 @@ when (NimMajor, NimMinor) <= (1, 0): else: zipImpl(s1, s2, seq[(S, T)]) -func unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = +proc unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = ## Returns a tuple of two sequences split out from a sequence of 2-field tuples. runnableExamples: let @@ -293,8 +328,7 @@ func unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = unzipped2 = @['a', 'b', 'c'] assert zipped.unzip() == (unzipped1, unzipped2) assert zip(unzipped1, unzipped2).unzip() == (unzipped1, unzipped2) - result[0] = newSeq[S](s.len) - result[1] = newSeq[T](s.len) + result = (newSeq[S](s.len), newSeq[T](s.len)) for i in 0..= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + template defaultImpl = + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + while i < newLen: + when defined(gcDestructors): + s[i] = move(s[j]) + else: + s[i].shallowCopy(s[j]) + inc(i) + inc(j) + setLen(s, newLen) + when nimvm: defaultImpl() + else: + when defined(js): + let n = slice.b - slice.a + 1 + let first = slice.a + {.emit: "`s`.splice(`first`, `n`);".} + else: + defaultImpl() + +func delete*[T](s: var seq[T]; first, last: Natural) {.deprecated: "use `delete(s, first..last)`".} = ## Deletes the items of a sequence `s` at positions `first..last` ## (including both ends of the range). ## This modifies `s` itself, it does not return a copy. - ## - runnableExamples: + runnableExamples("--warning:deprecated:off"): let outcome = @[1, 1, 1, 1, 1, 1, 1, 1] var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1] dest.delete(3, 8) @@ -646,7 +719,7 @@ since (1, 1): if pred: result += 1 result -proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if every item fulfills the ## predicate. ## @@ -688,7 +761,7 @@ template allIt*(s, pred: untyped): bool = break result -proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if at least one item ## fulfills the predicate. ## @@ -743,7 +816,7 @@ template toSeq1(s: not iterator): untyped = i += 1 result else: - var result: seq[OutType] = @[] + var result: seq[OutType]# = @[] for it in s: result.add(it) result @@ -760,7 +833,7 @@ template toSeq2(iter: iterator): untyped = result else: type OutType = typeof(iter2()) - var result: seq[OutType] = @[] + var result: seq[OutType]# = @[] when compiles(iter2()): evalOnceAs(iter4, iter, false) let iter3 = iter4() @@ -866,7 +939,7 @@ template foldl*(sequence, operation, first): untyped = ## ## The `operation` parameter should be an expression which uses the variables ## `a` and `b` for each step of the fold. The `first` parameter is the - ## start value (the first `a`) and therefor defines the type of the result. + ## start value (the first `a`) and therefore defines the type of the result. ## ## **See also:** ## * `foldr template<#foldr.t,untyped,untyped>`_ @@ -972,7 +1045,7 @@ template mapIt*(s: typed, op: untyped): untyped = i += 1 result else: - var result: seq[OutType] = @[] + var result: seq[OutType]# = @[] # use `items` to avoid https://github.com/nim-lang/Nim/issues/12639 for it {.inject.} in items(s): result.add(op) @@ -1014,27 +1087,31 @@ template applyIt*(varSeq, op: untyped) = template newSeqWith*(len: int, init: untyped): untyped = - ## Creates a new sequence of length `len`, calling `init` to initialize - ## each value of the sequence. - ## - ## Useful for creating "2D" sequences - sequences containing other sequences - ## or to populate fields of the created sequence. + ## Creates a new `seq` of length `len`, calling `init` to initialize + ## each value of the seq. ## + ## Useful for creating "2D" seqs - seqs containing other seqs + ## or to populate fields of the created seq. runnableExamples: - ## Creates a sequence containing 5 bool sequences, each of length of 3. + ## Creates a seq containing 5 bool seqs, each of length of 3. var seq2D = newSeqWith(5, newSeq[bool](3)) assert seq2D.len == 5 assert seq2D[0].len == 3 assert seq2D[4][2] == false - ## Creates a sequence of 20 random numbers from 1 to 10 - import random - var seqRand = newSeqWith(20, rand(10)) - - var result = newSeq[typeof(init)](len) - for i in 0 ..< len: + ## Creates a seq with random numbers + import std/random + var seqRand = newSeqWith(20, rand(1.0)) + assert seqRand[0] != seqRand[1] + type T = typeof(init) + let newLen = len + when supportsCopyMem(T) and declared(newSeqUninit): + var result = newSeqUninit[T](newLen) + else: # TODO: use `newSeqUnsafe` when that's available + var result = newSeq[T](newLen) + for i in 0 ..< newLen: result[i] = init - result + move(result) # refs bug #7295 func mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; filter = nnkLiterals): NimNode = diff --git a/lib/pure/collections/setimpl.nim b/lib/pure/collections/setimpl.nim index 7ebd227604..360a075d67 100644 --- a/lib/pure/collections/setimpl.nim +++ b/lib/pure/collections/setimpl.nim @@ -62,6 +62,7 @@ template containsOrInclImpl() {.dirty.} = if index >= 0: result = true else: + result = false if mustRehash(s): enlarge(s) index = rawGetKnownHC(s, key, hc) @@ -86,7 +87,9 @@ proc exclImpl[A](s: var HashSet[A], key: A): bool {.inline.} = var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. s.data[i].hcode = 0 # mark current EMPTY - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): i = (i + 1) and msk # increment mod table size if isEmpty(s.data[i].hcode): # end of collision cluster; So all done diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 3f91d9030a..af13135aab 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -25,7 +25,9 @@ ## `difference <#difference,HashSet[A],HashSet[A]>`_, and ## `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_ ## -## .. code-block:: +## **Examples:** +## +## ```Nim ## echo toHashSet([9, 5, 1]) # {9, 1, 5} ## echo toOrderedSet([9, 5, 1]) # {9, 5, 1} ## @@ -37,7 +39,7 @@ ## echo s1 - s2 # {1, 9} ## echo s1 * s2 # {5} ## echo s1 -+- s2 # {9, 1, 3, 7} -## +## ``` ## ## Note: The data types declared here have *value semantics*: This means ## that `=` performs a copy of the set. @@ -48,7 +50,10 @@ import - hashes, math + std/[hashes, math] + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} {.pragma: myShallow.} # For "integer-like A" that are too big for intsets/bit-vectors to be practical, @@ -61,7 +66,7 @@ type HashSet*[A] {.myShallow.} = object ## \ ## A generic hash set. ## - ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet,int>`_ + ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int @@ -125,7 +130,7 @@ proc initHashSet*[A](initialSize = defaultInitialSize): HashSet[A] = var a = initHashSet[int]() a.incl(3) assert len(a) == 1 - + result = default(HashSet[A]) result.init(initialSize) proc `[]`*[A](s: var HashSet[A], key: A): var A = @@ -246,7 +251,7 @@ iterator items*[A](s: HashSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template `_. ## - ## .. code-block:: + ## ```Nim ## type ## pair = tuple[a, b: int] ## var @@ -259,6 +264,7 @@ iterator items*[A](s: HashSet[A]): A = ## assert a.len == 2 ## echo b ## # --> {(a: 1, b: 3), (a: 0, b: 4)} + ## ``` let length = s.len for h in 0 .. high(s.data): if isFilled(s.data[h].hcode): @@ -344,7 +350,7 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = proc pop*[A](s: var HashSet[A]): A = ## Removes and returns an arbitrary element from the set `s`. ## - ## Raises KeyError if the set `s` is empty. + ## Raises `KeyError` if the set `s` is empty. ## ## See also: ## * `clear proc <#clear,HashSet[A]>`_ @@ -376,7 +382,9 @@ proc clear*[A](s: var HashSet[A]) = s.counter = 0 for i in 0 ..< s.data.len: s.data[i].hcode = 0 - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc union*[A](s1, s2: HashSet[A]): HashSet[A] = @@ -422,7 +430,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = assert c == toHashSet(["b"]) result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2)) - + # iterate over the elements of the smaller set if s1.data.len < s2.data.len: for item in s1: @@ -430,7 +438,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = else: for item in s2: if item in s1: incl(result, item) - + proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. @@ -556,7 +564,7 @@ proc `==`*[A](s, t: HashSet[A]): bool = s.counter == t.counter and s <= t -proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = +proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] {.effectsOf: op.} = ## Returns a new set after applying `op` proc on each of the elements of ##`data` set. ## @@ -583,12 +591,12 @@ proc `$`*[A](s: HashSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toHashSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toHashSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() @@ -603,12 +611,10 @@ proc isValid*[A](s: HashSet[A]): bool {.deprecated: ## Returns `true` if the set has been initialized (with `initHashSet proc ## <#initHashSet>`_ or `init proc <#init,HashSet[A]>`_). ## - ## **Examples:** - ## - ## .. code-block :: - ## proc savePreferences(options: HashSet[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! + runnableExamples: + proc savePreferences(options: HashSet[string]) = + assert options.isValid, "Pass an initialized set!" + # Do stuff here, may crash in release builds! result = s.data.len > 0 @@ -812,7 +818,9 @@ proc clear*[A](s: var OrderedSet[A]) = for i in 0 ..< s.data.len: s.data[i].hcode = 0 s.data[i].next = 0 - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc len*[A](s: OrderedSet[A]): int {.inline.} = ## Returns the number of elements in `s`. @@ -873,12 +881,12 @@ proc `$`*[A](s: OrderedSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toOrderedSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() @@ -889,7 +897,7 @@ iterator items*[A](s: OrderedSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template `_. ## - ## .. code-block:: + ## ```Nim ## var a = initOrderedSet[int]() ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: ## a.incl(value) @@ -901,6 +909,7 @@ iterator items*[A](s: OrderedSet[A]): A = ## # --> Got 5 ## # --> Got 8 ## # --> Got 4 + ## ``` let length = s.len forAllOrderedPairs: yield s.data[h].key diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index c72477675b..ec8f1cd866 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -11,10 +11,12 @@ ## ## Unstable API. +{.deprecated.} + {.push stackTrace: off.} import - locks + std/locks const ElemsPerNode = 100 @@ -35,8 +37,10 @@ template withLock(t, x: untyped) = release(t.lock) proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = - ## iterates over the list. If 'action' returns true, the + ## Iterates over the list. If `action` returns true, the ## current item is removed from the list. + ## + ## .. warning:: It may not preserve the element order after some modifications. withLock(x): var n = x.head while n != nil: @@ -47,8 +51,9 @@ proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = if action(n.d[i]): acquire(x.lock) let t = x.tail + dec t.dataLen # TODO considering t.dataLen == 0, + # probably the module should be refactored using doubly linked lists n.d[i] = t.d[t.dataLen] - dec t.dataLen else: acquire(x.lock) inc i diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 4ac3befb63..b474ecd315 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -14,8 +14,10 @@ ## ## Unstable API. +{.deprecated.} + import - hashes, math, locks + std/[hashes, math, locks] type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] @@ -60,17 +62,27 @@ template withLock(t, x: untyped) = template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + table.withValue("a", value): + assert value[] == "x" + + table.withValue("b", value): + value[] = "modified" + + table.withValue("b", value): + assert value[] == "modified" + + table.withValue("nonexistent", value): + assert false # not called acquire(t.lock) try: var hc: Hash @@ -84,20 +96,33 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body1, body2: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## do: - ## # block is executed when `key` not in `t` - ## raise newException(KeyError, "Key not found") - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + + table.withValue("a", value): + value[] = "m" + + var flag = false + table.withValue("d", value): + discard value + doAssert false + do: # if "d" notin table + flag = true + + if flag: + table["d"] = "n" + + assert table.mget("a") == "m" + assert table.mget("d") == "n" + acquire(t.lock) try: var hc: Hash @@ -112,7 +137,7 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, release(t.lock) proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = - ## retrieves the value at `t[key]`. The value can be modified. + ## Retrieves the value at `t[key]`. The value can be modified. ## If `key` is not in `t`, the `KeyError` exception is raised. withLock t: var hc: Hash @@ -126,7 +151,7 @@ proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = raise newException(KeyError, "key not found") proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = - ## retrieves value at `t[key]` or puts `val` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. **Note**: This is inherently ## unsafe in the context of multi-threading since it returns a pointer ## to `B`. @@ -134,7 +159,7 @@ proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool = - ## returns true if `key` is in the table, otherwise inserts `value`. + ## Returns true if `key` is in the table, otherwise inserts `value`. withLock t: hasKeyOrPutImpl(enlarge) @@ -166,8 +191,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## ## Example usage: ## - ## .. code-block:: nim - ## + ## ```nim ## # If value exists, decrement it. ## # If it becomes zero or less, delete the key ## t.withKey(1'i64) do (k: int64, v: var int, pairExists: var bool): @@ -175,6 +199,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## dec v ## if v <= 0: ## pairExists = false + ## ``` withLock t: var hc: Hash var index = rawGet(t, key, hc) @@ -191,28 +216,28 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, st_maybeRehashPutImpl(enlarge) proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. + ## Puts a (key, value)-pair into `t`. withLock t: putImpl(enlarge) proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if `t[key]` already exists. + ## Puts a new (key, value)-pair into `t` even if `t[key]` already exists. ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) proc del*[A, B](t: var SharedTable[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## Deletes `key` from hash table `t`. withLock t: delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) proc len*[A, B](t: var SharedTable[A, B]): int = - ## number of elements in `t` + ## Number of elements in `t`. withLock t: result = t.counter proc init*[A, B](t: var SharedTable[A, B], initialSize = 32) = - ## creates a new hash table that is empty. + ## Creates a new hash table that is empty. ## ## This proc must be called before any other usage of `t`. let initialSize = slotsNeeded(initialSize) diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index 27c3391775..3542741fac 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -11,6 +11,9 @@ include hashcommon +const + defaultInitialSize* = 32 + template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add genHashImpl(key, hc) var h: Hash = hc and maxHash(t) @@ -23,7 +26,7 @@ template rawInsertImpl() {.dirty.} = data[h].val = val data[h].hcode = hc -proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetDeepImpl() proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], @@ -31,9 +34,8 @@ proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], rawInsertImpl() template checkIfInitialized() = - when compiles(defaultInitialSize): - if t.dataLen == 0: - initImpl(t, defaultInitialSize) + if t.dataLen == 0: + initImpl(t, defaultInitialSize) template addImpl(enlarge) {.dirty.} = checkIfInitialized() @@ -43,7 +45,7 @@ template addImpl(enlarge) {.dirty.} = rawInsert(t, t.data, key, val, hc, j) inc(t.counter) -template maybeRehashPutImpl(enlarge) {.dirty.} = +template maybeRehashPutImpl(enlarge, val) {.dirty.} = checkIfInitialized() if mustRehash(t): enlarge(t) @@ -54,28 +56,41 @@ template maybeRehashPutImpl(enlarge) {.dirty.} = template putImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index >= 0: t.data[index].val = val - else: maybeRehashPutImpl(enlarge) + else: maybeRehashPutImpl(enlarge, val) template mgetOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: # not present: insert (flipping index) - maybeRehashPutImpl(enlarge) + when declared(val): + maybeRehashPutImpl(enlarge, val) + else: + maybeRehashPutImpl(enlarge, default(B)) # either way return modifiable val result = t.data[index].val +# template mgetOrPutDefaultImpl(enlarge) {.dirty.} = +# checkIfInitialized() +# var hc: Hash = default(Hash) +# var index = rawGet(t, key, hc) +# if index < 0: +# # not present: insert (flipping index) +# maybeRehashPutImpl(enlarge, default(B)) +# # either way return modifiable val +# result = t.data[index].val + template hasKeyOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: result = false - maybeRehashPutImpl(enlarge) + maybeRehashPutImpl(enlarge, val) else: result = true # delImplIdx is KnuthV3 Algo6.4R adapted to i=i+1 (from i=i-1) which has come to @@ -117,8 +132,10 @@ template delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) = var j = i # The correctness of this depends on (h+1) in nextTry var r = j # though may be adaptable to other simple sequences. makeEmpty(i) # mark current EMPTY - t.data[i].key = default(typeof(t.data[i].key)) - t.data[i].val = default(typeof(t.data[i].val)) + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} while true: i = (i + 1) and msk # increment mod table size if cellEmpty(i): # end of collision cluster; So all done @@ -149,8 +166,10 @@ template clearImpl() {.dirty.} = for i in 0 ..< t.dataLen: when compiles(t.data[i].hcode): # CountTable records don't contain a hcode t.data[i].hcode = 0 - t.data[i].key = default(typeof(t.data[i].key)) - t.data[i].val = default(typeof(t.data[i].val)) + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} t.counter = 0 template ctAnd(a, b): bool = @@ -208,3 +227,5 @@ template equalsImpl(s, t: typed) = if not t.hasKey(key): return false if t.getOrDefault(key) != val: return false return true + else: + return false diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index f776423491..d414caeed4 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -136,12 +136,11 @@ runnableExamples: ## a more complex object as a key you will be greeted by a strange compiler ## error: ## -## Error: type mismatch: got (Person) -## but expected one of: -## hashes.hash(x: openArray[A]): Hash -## hashes.hash(x: int): Hash -## hashes.hash(x: float): Hash -## … +## Error: type mismatch: got (Person) +## but expected one of: +## hashes.hash(x: openArray[A]): Hash +## hashes.hash(x: int): Hash +## hashes.hash(x: float): Hash ## ## What is happening here is that the types used for table keys require to have ## a `hash()` proc which will convert them to a `Hash `_ @@ -189,7 +188,6 @@ runnableExamples: ## ## * `json module`_ for table-like structure which allows ## heterogeneous members -## * `sharedtables module`_ for shared hash table support ## * `strtabs module`_ for efficient hash tables ## mapping from strings to strings ## * `hashes module`_ for helper functions for hashing @@ -198,6 +196,10 @@ runnableExamples: import std/private/since import std/[hashes, math, algorithm] + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] KeyValuePairSeq[A, B] = seq[KeyValuePair[A, B]] @@ -215,8 +217,6 @@ type ## For creating a new empty TableRef, use `newTable proc ## <#newTable>`_. -const - defaultInitialSize* = 32 # ------------------------------ helpers --------------------------------- @@ -227,6 +227,12 @@ template dataLen(t): untyped = len(t.data) include tableimpl +proc raiseKeyError[T](key: T) {.noinline, noreturn.} = + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") + template get(t, key): untyped = ## retrieves the value at `t[key]`. The value can be modified. ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -235,10 +241,7 @@ template get(t, key): untyped = var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + raiseKeyError(key) proc enlarge[A, B](t: var Table[A, B]) = var n: KeyValuePairSeq[A, B] @@ -275,6 +278,7 @@ proc initTable*[A, B](initialSize = defaultInitialSize): Table[A, B] = let a = initTable[int, string]() b = initTable[char, seq[int]]() + result = default(Table[A, B]) initImpl(result, initialSize) proc `[]=`*[A, B](t: var Table[A, B], key: A, val: sink B) = @@ -309,7 +313,7 @@ proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = result = initTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: Table[A, B], key: A): B = +proc `[]`*[A, B](t: Table[A, B], key: A): lent B = ## Retrieves the value at `t[key]`. ## ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -321,7 +325,7 @@ proc `[]`*[A, B](t: Table[A, B], key: A): B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in ## the table @@ -342,7 +346,7 @@ proc `[]`*[A, B](t: var Table[A, B], key: A): var B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in ## the table @@ -412,7 +416,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = @@ -430,7 +434,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = @@ -439,7 +443,7 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = ## ## ## Note that while the value returned is of type `var B`, - ## it is easy to accidentally create an copy of the value at `t[key]`. + ## it is easy to accidentally create a copy of the value at `t[key]`. ## Remember that seqs and strings are value types, and therefore ## cannot be copied into a separate variable for modification. ## See the example below. @@ -471,6 +475,18 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var Table[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + + mgetOrPutImpl(enlarge) + proc len*[A, B](t: Table[A, B]): int = ## Returns the number of keys in `t`. runnableExamples: @@ -485,7 +501,7 @@ proc add*[A, B](t: var Table[A, B], key: A, val: sink B) {.deprecated: ## ## **This can introduce duplicate keys into the table!** ## - ## Use `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## Use `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table without introducing duplicates. addImpl(enlarge) @@ -496,6 +512,9 @@ template tabCellHash(i) = t.data[i].hcode proc del*[A, B](t: var Table[A, B], key: A) = ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## ## See also: ## * `pop proc<#pop,Table[A,B],A,B>`_ ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table @@ -514,6 +533,9 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool = ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## ## See also: ## * `del proc<#del,Table[A,B],A>`_ ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table @@ -594,17 +616,17 @@ template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - t.withValue(2, value) do: + t.withValue(2, value): value.name = "No" value.uid = 521 - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + assert t[1].name == "Nim" + assert t[1].uid == 1314 mixin rawGet var hc: Hash @@ -629,16 +651,22 @@ template withValue*[A, B](t: var Table[A, B], key: A, let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - # do: - # # block is executed when `key` not in `t` - # raise newException(KeyError, "Key not found") - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + t.withValue(521, value): + doAssert false + do: + # block is executed when `key` not in `t` + t[1314] = User(name: "exist", uid: 521) + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + assert t[1314].name == "exist" + assert t[1314].uid == 521 + mixin rawGet var hc: Hash var index = rawGet(t, key, hc) @@ -660,7 +688,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -674,6 +702,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -702,7 +731,7 @@ iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: Table[A, B]): A = +iterator keys*[A, B](t: Table[A, B]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -723,7 +752,7 @@ iterator keys*[A, B](t: Table[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: Table[A, B]): B = +iterator values*[A, B](t: Table[A, B]): lent B = ## Iterates over any value in the table `t`. ## ## See also: @@ -795,7 +824,7 @@ iterator allValues*[A, B](t: Table[A, B]; key: A): B {.deprecated: # ------------------------------------------------------------------- -proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = +proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = ## Creates a new ref hash table that is empty. ## ## See also: @@ -808,9 +837,10 @@ proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = b = newTable[char, seq[int]]() new(result) - result[] = initTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initTable[A, B](initialSize) -proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = +proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = ## Creates a new ref hash table that contains the given `pairs`. ## ## `pairs` is a container consisting of `(key, value)` tuples. @@ -824,14 +854,16 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = assert b == {'a': 5, 'b': 9}.newTable new(result) - result[] = toTable[A, B](pairs) + {.noSideEffect.}: + result[] = toTable[A, B](pairs) -proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = +proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] result = newTable[C, B]() - for item in collection: - result[index(item)] = item + {.noSideEffect.}: + for item in collection: + result[index(item)] = item proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = ## Retrieves the value at `t[key]`. @@ -901,7 +933,7 @@ proc contains*[A, B](t: TableRef[A, B], key: A): bool = return hasKey[A, B](t, key) -proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = +proc hasKeyOrPut*[A, B](t: TableRef[A, B], key: A, val: B): bool = ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: @@ -994,6 +1026,18 @@ proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = doAssert t[25] == @[25, 35] t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: TableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + + t[].mgetOrPut(key) + proc len*[A, B](t: TableRef[A, B]): int = ## Returns the number of keys in `t`. runnableExamples: @@ -1015,7 +1059,8 @@ proc add*[A, B](t: TableRef[A, B], key: A, val: sink B) {.deprecated: proc del*[A, B](t: TableRef[A, B], key: A) = ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## - ## **If duplicate keys were added, this may need to be called multiple times.** + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. ## ## See also: ## * `pop proc<#pop,TableRef[A,B],A,B>`_ @@ -1035,7 +1080,8 @@ proc pop*[A, B](t: TableRef[A, B], key: A, val: var B): bool = ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## - ## **If duplicate keys were added, this may need to be called multiple times.** + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. ## ## See also: ## * `del proc<#del,TableRef[A,B],A>`_ @@ -1104,7 +1150,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1118,6 +1164,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -1146,7 +1193,7 @@ iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: TableRef[A, B]): A = +iterator keys*[A, B](t: TableRef[A, B]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -1167,7 +1214,7 @@ iterator keys*[A, B](t: TableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: TableRef[A, B]): B = +iterator values*[A, B](t: TableRef[A, B]): lent B = ## Iterates over any value in the table `t`. ## ## See also: @@ -1300,6 +1347,7 @@ proc initOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTable[A, let a = initOrderedTable[int, string]() b = initOrderedTable[char, seq[int]]() + result = default(OrderedTable[A, B]) initImpl(result, initialSize) proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: sink B) = @@ -1335,7 +1383,7 @@ proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = result = initOrderedTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = +proc `[]`*[A, B](t: OrderedTable[A, B], key: A): lent B = ## Retrieves the value at `t[key]`. ## ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -1391,7 +1439,7 @@ proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = doAssert a.hasKey('a') == true doAssert a.hasKey('z') == false - var hc: Hash + var hc: Hash = default(Hash) result = rawGet(t, key, hc) >= 0 proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = @@ -1440,7 +1488,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = @@ -1458,7 +1506,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = @@ -1481,6 +1529,18 @@ proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + mgetOrPutImpl(enlarge) + proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = ## Returns the number of keys in `t`. runnableExamples: @@ -1579,7 +1639,7 @@ proc clear*[A, B](t: var OrderedTable[A, B]) = t.last = -1 proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list @@ -1680,7 +1740,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1694,6 +1754,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -1722,7 +1783,7 @@ iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTable[A, B]): A = +iterator keys*[A, B](t: OrderedTable[A, B]): lent A = ## Iterates over any key in the table `t` in insertion order. ## ## See also: @@ -1743,7 +1804,7 @@ iterator keys*[A, B](t: OrderedTable[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTable[A, B]): B = +iterator values*[A, B](t: OrderedTable[A, B]): lent B = ## Iterates over any value in the table `t` in insertion order. ## ## See also: @@ -1790,7 +1851,7 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = # --------------------------- OrderedTableRef ------------------------------- # --------------------------------------------------------------------------- -proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that is empty. ## ## See also: @@ -1803,9 +1864,10 @@ proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableR a = newOrderedTable[int, string]() b = newOrderedTable[char, seq[int]]() new(result) - result[] = initOrderedTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initOrderedTable[A, B](initialSize) -proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that contains the given `pairs`. ## ## `pairs` is a container consisting of `(key, value)` tuples. @@ -1820,7 +1882,8 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] assert b == {'a': 5, 'b': 9}.newOrderedTable result = newOrderedTable[A, B](pairs.len) - for key, val in items(pairs): result[key] = val + {.noSideEffect.}: + for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = @@ -1890,7 +1953,7 @@ proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = return hasKey[A, B](t, key) -proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = +proc hasKeyOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): bool = ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: @@ -1967,6 +2030,18 @@ proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = result = t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + t[].mgetOrPut(key) + proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = ## Returns the number of keys in `t`. runnableExamples: @@ -2036,7 +2111,7 @@ proc clear*[A, B](t: OrderedTableRef[A, B]) = clear(t[]) proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list @@ -2088,7 +2163,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -2102,6 +2177,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -2130,7 +2206,7 @@ iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTableRef[A, B]): A = +iterator keys*[A, B](t: OrderedTableRef[A, B]): lent A = ## Iterates over any key in the table `t` in insertion order. ## ## See also: @@ -2151,7 +2227,7 @@ iterator keys*[A, B](t: OrderedTableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTableRef[A, B]): B = +iterator values*[A, B](t: OrderedTableRef[A, B]): lent B = ## Iterates over any value in the table `t` in insertion order. ## ## See also: @@ -2262,6 +2338,7 @@ proc initCountTable*[A](initialSize = defaultInitialSize): CountTable[A] = ## * `toCountTable proc<#toCountTable,openArray[A]>`_ ## * `newCountTable proc<#newCountTable>`_ for creating a ## `CountTableRef` + result = default(CountTable[A]) initImpl(result, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = @@ -2371,7 +2448,7 @@ proc contains*[A](t: CountTable[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2501,7 +2578,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = toCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2518,6 +2595,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2543,7 +2621,7 @@ iterator mpairs*[A](t: var CountTable[A]): (A, var int) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A](t: CountTable[A]): A = +iterator keys*[A](t: CountTable[A]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -2610,7 +2688,7 @@ iterator mvalues*[A](t: var CountTable[A]): var int = proc inc*[A](t: CountTableRef[A], key: A, val = 1) -proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = +proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## Creates a new ref count table that is empty. ## ## See also: @@ -2619,13 +2697,15 @@ proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## * `initCountTable proc<#initCountTable>`_ for creating a ## `CountTable` new(result) - result[] = initCountTable[A](initialSize) + {.noSideEffect.}: + result[] = initCountTable[A](initialSize) -proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = +proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = ## Creates a new ref count table with every member of a container `keys` ## having a count of how many times it occurs in that container. result = newCountTable[A](keys.len) - for key in items(keys): result.inc(key) + {.noSideEffect.}: + for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTableRef[A], key: A): int = ## Retrieves the value at `t[key]` if `key` is in `t`. @@ -2649,7 +2729,8 @@ proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a ## value of a key assert val > 0 - t[][key] = val + {.noSideEffect.}: + t[][key] = val proc inc*[A](t: CountTableRef[A], key: A, val = 1) = ## Increments `t[key]` by `val` (default: 1). @@ -2658,7 +2739,8 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) = a.inc('a') a.inc('b', 10) doAssert a == newCountTable("aaabbbbbbbbbbb") - t[].inc(key, val) + {.noSideEffect.}: + t[].inc(key, val) proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) @@ -2691,7 +2773,7 @@ proc contains*[A](t: CountTableRef[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2777,7 +2859,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = newCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2794,6 +2876,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2872,3 +2955,18 @@ iterator mvalues*[A](t: CountTableRef[A]): var int = if t.data[h].val != 0: yield t.data[h].val assert(len(t) == L, "the length of the table changed while iterating over it") + +proc hash*[K,V](s: Table[K,V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result + +proc hash*[K,V](s: OrderedTable[K,V]): Hash = + for p in pairs(s): + result = result !& hash(p) + result = !$result + +proc hash*[V](s: CountTable[V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 4a3f47e925..d3e6dc0631 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -6,24 +6,26 @@ # distribution, for details about the copyright. # -## This module implements color handling for Nim. +## This module implements color handling for Nim, +## namely color mixing and parsing the CSS color names. -import strutils -from algorithm import binarySearch +import std/strutils +from std/algorithm import binarySearch type Color* = distinct int ## A color stored as RGB, e.g. `0xff00cc`. -proc `==` *(a, b: Color): bool {.borrow.} +proc `==`*(a, b: Color): bool {.borrow.} ## Compares two colors. ## - ## .. code-block:: + ## ```Nim ## var ## a = Color(0xff_00_ff) ## b = colFuchsia ## c = Color(0x00_ff_cc) ## assert a == b - ## assert not a == c + ## assert not (a == c) + ## ``` template extract(a: Color, r, g, b: untyped) = var r = a.int shr 16 and 0xff @@ -170,6 +172,7 @@ const colDarkGoldenRod* = Color(0xB8860B) colDarkGray* = Color(0xA9A9A9) colDarkGreen* = Color(0x006400) + colDarkGrey* = Color(0xA9A9A9) colDarkKhaki* = Color(0xBDB76B) colDarkMagenta* = Color(0x8B008B) colDarkOliveGreen* = Color(0x556B2F) @@ -180,11 +183,13 @@ const colDarkSeaGreen* = Color(0x8FBC8F) colDarkSlateBlue* = Color(0x483D8B) colDarkSlateGray* = Color(0x2F4F4F) + colDarkSlateGrey* = Color(0x2F4F4F) colDarkTurquoise* = Color(0x00CED1) colDarkViolet* = Color(0x9400D3) colDeepPink* = Color(0xFF1493) colDeepSkyBlue* = Color(0x00BFFF) colDimGray* = Color(0x696969) + colDimGrey* = Color(0x696969) colDodgerBlue* = Color(0x1E90FF) colFireBrick* = Color(0xB22222) colFloralWhite* = Color(0xFFFAF0) @@ -197,6 +202,7 @@ const colGray* = Color(0x808080) colGreen* = Color(0x008000) colGreenYellow* = Color(0xADFF2F) + colGrey* = Color(0x808080) colHoneyDew* = Color(0xF0FFF0) colHotPink* = Color(0xFF69B4) colIndianRed* = Color(0xCD5C5C) @@ -211,13 +217,15 @@ const colLightCoral* = Color(0xF08080) colLightCyan* = Color(0xE0FFFF) colLightGoldenRodYellow* = Color(0xFAFAD2) - colLightGrey* = Color(0xD3D3D3) + colLightGray* = Color(0xD3D3D3) colLightGreen* = Color(0x90EE90) + colLightGrey* = Color(0xD3D3D3) colLightPink* = Color(0xFFB6C1) colLightSalmon* = Color(0xFFA07A) colLightSeaGreen* = Color(0x20B2AA) colLightSkyBlue* = Color(0x87CEFA) colLightSlateGray* = Color(0x778899) + colLightSlateGrey* = Color(0x778899) colLightSteelBlue* = Color(0xB0C4DE) colLightYellow* = Color(0xFFFFE0) colLime* = Color(0x00FF00) @@ -228,7 +236,7 @@ const colMediumAquaMarine* = Color(0x66CDAA) colMediumBlue* = Color(0x0000CD) colMediumOrchid* = Color(0xBA55D3) - colMediumPurple* = Color(0x9370D8) + colMediumPurple* = Color(0x9370DB) colMediumSeaGreen* = Color(0x3CB371) colMediumSlateBlue* = Color(0x7B68EE) colMediumSpringGreen* = Color(0x00FA9A) @@ -249,7 +257,7 @@ const colPaleGoldenRod* = Color(0xEEE8AA) colPaleGreen* = Color(0x98FB98) colPaleTurquoise* = Color(0xAFEEEE) - colPaleVioletRed* = Color(0xD87093) + colPaleVioletRed* = Color(0xDB7093) colPapayaWhip* = Color(0xFFEFD5) colPeachPuff* = Color(0xFFDAB9) colPeru* = Color(0xCD853F) @@ -257,6 +265,7 @@ const colPlum* = Color(0xDDA0DD) colPowderBlue* = Color(0xB0E0E6) colPurple* = Color(0x800080) + colRebeccaPurple* = Color(0x663399) colRed* = Color(0xFF0000) colRosyBrown* = Color(0xBC8F8F) colRoyalBlue* = Color(0x4169E1) @@ -270,6 +279,7 @@ const colSkyBlue* = Color(0x87CEEB) colSlateBlue* = Color(0x6A5ACD) colSlateGray* = Color(0x708090) + colSlateGrey* = Color(0x708090) colSnow* = Color(0xFFFAFA) colSpringGreen* = Color(0x00FF7F) colSteelBlue* = Color(0x4682B4) @@ -285,147 +295,155 @@ const colYellow* = Color(0xFFFF00) colYellowGreen* = Color(0x9ACD32) - colorNames = [ - ("aliceblue", colAliceBlue), - ("antiquewhite", colAntiqueWhite), - ("aqua", colAqua), - ("aquamarine", colAquamarine), - ("azure", colAzure), - ("beige", colBeige), - ("bisque", colBisque), - ("black", colBlack), - ("blanchedalmond", colBlanchedAlmond), - ("blue", colBlue), - ("blueviolet", colBlueViolet), - ("brown", colBrown), - ("burlywood", colBurlyWood), - ("cadetblue", colCadetBlue), - ("chartreuse", colChartreuse), - ("chocolate", colChocolate), - ("coral", colCoral), - ("cornflowerblue", colCornflowerBlue), - ("cornsilk", colCornsilk), - ("crimson", colCrimson), - ("cyan", colCyan), - ("darkblue", colDarkBlue), - ("darkcyan", colDarkCyan), - ("darkgoldenrod", colDarkGoldenRod), - ("darkgray", colDarkGray), - ("darkgreen", colDarkGreen), - ("darkkhaki", colDarkKhaki), - ("darkmagenta", colDarkMagenta), - ("darkolivegreen", colDarkOliveGreen), - ("darkorange", colDarkorange), - ("darkorchid", colDarkOrchid), - ("darkred", colDarkRed), - ("darksalmon", colDarkSalmon), - ("darkseagreen", colDarkSeaGreen), - ("darkslateblue", colDarkSlateBlue), - ("darkslategray", colDarkSlateGray), - ("darkturquoise", colDarkTurquoise), - ("darkviolet", colDarkViolet), - ("deeppink", colDeepPink), - ("deepskyblue", colDeepSkyBlue), - ("dimgray", colDimGray), - ("dodgerblue", colDodgerBlue), - ("firebrick", colFireBrick), - ("floralwhite", colFloralWhite), - ("forestgreen", colForestGreen), - ("fuchsia", colFuchsia), - ("gainsboro", colGainsboro), - ("ghostwhite", colGhostWhite), - ("gold", colGold), - ("goldenrod", colGoldenRod), - ("gray", colGray), - ("green", colGreen), - ("greenyellow", colGreenYellow), - ("honeydew", colHoneyDew), - ("hotpink", colHotPink), - ("indianred", colIndianRed), - ("indigo", colIndigo), - ("ivory", colIvory), - ("khaki", colKhaki), - ("lavender", colLavender), - ("lavenderblush", colLavenderBlush), - ("lawngreen", colLawnGreen), - ("lemonchiffon", colLemonChiffon), - ("lightblue", colLightBlue), - ("lightcoral", colLightCoral), - ("lightcyan", colLightCyan), - ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgrey", colLightGrey), - ("lightgreen", colLightGreen), - ("lightpink", colLightPink), - ("lightsalmon", colLightSalmon), - ("lightseagreen", colLightSeaGreen), - ("lightskyblue", colLightSkyBlue), - ("lightslategray", colLightSlateGray), - ("lightsteelblue", colLightSteelBlue), - ("lightyellow", colLightYellow), - ("lime", colLime), - ("limegreen", colLimeGreen), - ("linen", colLinen), - ("magenta", colMagenta), - ("maroon", colMaroon), - ("mediumaquamarine", colMediumAquaMarine), - ("mediumblue", colMediumBlue), - ("mediumorchid", colMediumOrchid), - ("mediumpurple", colMediumPurple), - ("mediumseagreen", colMediumSeaGreen), - ("mediumslateblue", colMediumSlateBlue), - ("mediumspringgreen", colMediumSpringGreen), - ("mediumturquoise", colMediumTurquoise), - ("mediumvioletred", colMediumVioletRed), - ("midnightblue", colMidnightBlue), - ("mintcream", colMintCream), - ("mistyrose", colMistyRose), - ("moccasin", colMoccasin), - ("navajowhite", colNavajoWhite), - ("navy", colNavy), - ("oldlace", colOldLace), - ("olive", colOlive), - ("olivedrab", colOliveDrab), - ("orange", colOrange), - ("orangered", colOrangeRed), - ("orchid", colOrchid), - ("palegoldenrod", colPaleGoldenRod), - ("palegreen", colPaleGreen), - ("paleturquoise", colPaleTurquoise), - ("palevioletred", colPaleVioletRed), - ("papayawhip", colPapayaWhip), - ("peachpuff", colPeachPuff), - ("peru", colPeru), - ("pink", colPink), - ("plum", colPlum), - ("powderblue", colPowderBlue), - ("purple", colPurple), - ("red", colRed), - ("rosybrown", colRosyBrown), - ("royalblue", colRoyalBlue), - ("saddlebrown", colSaddleBrown), - ("salmon", colSalmon), - ("sandybrown", colSandyBrown), - ("seagreen", colSeaGreen), - ("seashell", colSeaShell), - ("sienna", colSienna), - ("silver", colSilver), - ("skyblue", colSkyBlue), - ("slateblue", colSlateBlue), - ("slategray", colSlateGray), - ("snow", colSnow), - ("springgreen", colSpringGreen), - ("steelblue", colSteelBlue), - ("tan", colTan), - ("teal", colTeal), - ("thistle", colThistle), - ("tomato", colTomato), - ("turquoise", colTurquoise), - ("violet", colViolet), - ("wheat", colWheat), - ("white", colWhite), - ("whitesmoke", colWhiteSmoke), - ("yellow", colYellow), - ("yellowgreen", colYellowGreen)] + colorNames = { + "aliceblue": colAliceBlue, + "antiquewhite": colAntiqueWhite, + "aqua": colAqua, + "aquamarine": colAquamarine, + "azure": colAzure, + "beige": colBeige, + "bisque": colBisque, + "black": colBlack, + "blanchedalmond": colBlanchedAlmond, + "blue": colBlue, + "blueviolet": colBlueViolet, + "brown": colBrown, + "burlywood": colBurlyWood, + "cadetblue": colCadetBlue, + "chartreuse": colChartreuse, + "chocolate": colChocolate, + "coral": colCoral, + "cornflowerblue": colCornflowerBlue, + "cornsilk": colCornsilk, + "crimson": colCrimson, + "cyan": colCyan, + "darkblue": colDarkBlue, + "darkcyan": colDarkCyan, + "darkgoldenrod": colDarkGoldenRod, + "darkgray": colDarkGray, + "darkgreen": colDarkGreen, + "darkgrey": colDarkGrey, + "darkkhaki": colDarkKhaki, + "darkmagenta": colDarkMagenta, + "darkolivegreen": colDarkOliveGreen, + "darkorange": colDarkorange, + "darkorchid": colDarkOrchid, + "darkred": colDarkRed, + "darksalmon": colDarkSalmon, + "darkseagreen": colDarkSeaGreen, + "darkslateblue": colDarkSlateBlue, + "darkslategray": colDarkSlateGray, + "darkslategrey": colDarkSlateGrey, + "darkturquoise": colDarkTurquoise, + "darkviolet": colDarkViolet, + "deeppink": colDeepPink, + "deepskyblue": colDeepSkyBlue, + "dimgray": colDimGray, + "dimgrey": colDimGrey, + "dodgerblue": colDodgerBlue, + "firebrick": colFireBrick, + "floralwhite": colFloralWhite, + "forestgreen": colForestGreen, + "fuchsia": colFuchsia, + "gainsboro": colGainsboro, + "ghostwhite": colGhostWhite, + "gold": colGold, + "goldenrod": colGoldenRod, + "gray": colGray, + "green": colGreen, + "greenyellow": colGreenYellow, + "grey": colGrey, + "honeydew": colHoneyDew, + "hotpink": colHotPink, + "indianred": colIndianRed, + "indigo": colIndigo, + "ivory": colIvory, + "khaki": colKhaki, + "lavender": colLavender, + "lavenderblush": colLavenderBlush, + "lawngreen": colLawnGreen, + "lemonchiffon": colLemonChiffon, + "lightblue": colLightBlue, + "lightcoral": colLightCoral, + "lightcyan": colLightCyan, + "lightgoldenrodyellow": colLightGoldenRodYellow, + "lightgray": colLightGray, + "lightgreen": colLightGreen, + "lightgrey": colLightGrey, + "lightpink": colLightPink, + "lightsalmon": colLightSalmon, + "lightseagreen": colLightSeaGreen, + "lightskyblue": colLightSkyBlue, + "lightslategray": colLightSlateGray, + "lightslategrey": colLightSlateGrey, + "lightsteelblue": colLightSteelBlue, + "lightyellow": colLightYellow, + "lime": colLime, + "limegreen": colLimeGreen, + "linen": colLinen, + "magenta": colMagenta, + "maroon": colMaroon, + "mediumaquamarine": colMediumAquaMarine, + "mediumblue": colMediumBlue, + "mediumorchid": colMediumOrchid, + "mediumpurple": colMediumPurple, + "mediumseagreen": colMediumSeaGreen, + "mediumslateblue": colMediumSlateBlue, + "mediumspringgreen": colMediumSpringGreen, + "mediumturquoise": colMediumTurquoise, + "mediumvioletred": colMediumVioletRed, + "midnightblue": colMidnightBlue, + "mintcream": colMintCream, + "mistyrose": colMistyRose, + "moccasin": colMoccasin, + "navajowhite": colNavajoWhite, + "navy": colNavy, + "oldlace": colOldLace, + "olive": colOlive, + "olivedrab": colOliveDrab, + "orange": colOrange, + "orangered": colOrangeRed, + "orchid": colOrchid, + "palegoldenrod": colPaleGoldenRod, + "palegreen": colPaleGreen, + "paleturquoise": colPaleTurquoise, + "palevioletred": colPaleVioletRed, + "papayawhip": colPapayaWhip, + "peachpuff": colPeachPuff, + "peru": colPeru, + "pink": colPink, + "plum": colPlum, + "powderblue": colPowderBlue, + "purple": colPurple, + "rebeccapurple": colRebeccaPurple, + "red": colRed, + "rosybrown": colRosyBrown, + "royalblue": colRoyalBlue, + "saddlebrown": colSaddleBrown, + "salmon": colSalmon, + "sandybrown": colSandyBrown, + "seagreen": colSeaGreen, + "seashell": colSeaShell, + "sienna": colSienna, + "silver": colSilver, + "skyblue": colSkyBlue, + "slateblue": colSlateBlue, + "slategray": colSlateGray, + "slategrey": colSlateGrey, + "snow": colSnow, + "springgreen": colSpringGreen, + "steelblue": colSteelBlue, + "tan": colTan, + "teal": colTeal, + "thistle": colThistle, + "tomato": colTomato, + "turquoise": colTurquoise, + "violet": colViolet, + "wheat": colWheat, + "white": colWhite, + "whitesmoke": colWhiteSmoke, + "yellow": colYellow, + "yellowgreen": colYellowGreen} proc `$`*(c: Color): string = ## Converts a color into its textual representation. diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index b9371c1e17..b48811eaed 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -15,9 +15,6 @@ runnableExamples: from std/math import almostEqual, sqrt - func almostEqual(a, b: Complex): bool = - almostEqual(a.re, b.re) and almostEqual(a.im, b.im) - let z1 = complex(1.0, 2.0) z2 = complex(3.0, -4.0) @@ -36,7 +33,7 @@ runnableExamples: {.push checks: off, line_dir: off, stack_trace: off, debugger: off.} # the user does not want to trace a part of the standard library! -import std/math +import std/[math, strformat] type Complex*[T: SomeFloat] = object @@ -82,6 +79,13 @@ func abs2*[T](z: Complex[T]): T = ## This is more efficient than `abs(z) ^ 2`. result = z.re * z.re + z.im * z.im +func sgn*[T](z: Complex[T]): Complex[T] = + ## Returns the phase of `z` as a unit complex number, + ## or 0 if `z` is 0. + let a = abs(z) + if a != 0: + result = z / a + func conjugate*[T](z: Complex[T]): Complex[T] = ## Returns the complex conjugate of `z` (`complex(z.re, -z.im)`). result.re = z.re @@ -156,18 +160,7 @@ func `/`*[T](x: T; y: Complex[T]): Complex[T] = func `/`*[T](x, y: Complex[T]): Complex[T] = ## Divides two complex numbers. - var r, den: T - if abs(y.re) < abs(y.im): - r = y.re / y.im - den = y.im + r * y.re - result.re = (x.re * r + x.im) / den - result.im = (x.im * r - x.re) / den - else: - r = y.im / y.re - den = y.re + r * y.im - result.re = (x.re + r * x.im) / den - result.im = (x.im - r * x.re) / den - + x * conjugate(y) / abs2(y) func `+=`*[T](x: var Complex[T]; y: Complex[T]) = ## Adds `y` to `x`. @@ -253,10 +246,31 @@ func pow*[T](x, y: Complex[T]): Complex[T] = else: result.re = 0.0 result.im = 0.0 - elif y.re == 1.0 and y.im == 0.0: - result = x - elif y.re == -1.0 and y.im == 0.0: - result = T(1.0) / x + elif y.im == 0.0: + if y.re == 1.0: + result = x + elif y.re == -1.0: + result = T(1.0) / x + elif y.re == 2.0: + result = x * x + elif y.re == 0.5: + result = sqrt(x) + elif x.im == 0.0: + # Revert to real pow when both base and exponent are real + result.re = pow(x.re, y.re) + result.im = 0.0 + else: + # Special case when the exponent is real + let + rho = abs(x) + theta = arctan2(x.im, x.re) + s = pow(rho, y.re) + r = y.re * theta + result.re = s * cos(r) + result.im = s * sin(r) + elif x.im == 0.0 and x.re == E: + # Special case Euler's formula + result = exp(y) else: let rho = abs(x) @@ -395,6 +409,24 @@ func rect*[T](r, phi: T): Complex[T] = ## * `polar func<#polar,Complex[T]>`_ for the inverse operation complex(r * cos(phi), r * sin(phi)) +func almostEqual*[T: SomeFloat](x, y: Complex[T]; unitsInLastPlace: Natural = 4): bool = + ## Checks if two complex values are almost equal, using the + ## [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon). + ## + ## Two complex values are considered almost equal if their real and imaginary + ## components are almost equal. + ## + ## `unitsInLastPlace` is the max number of + ## [units in the last place](https://en.wikipedia.org/wiki/Unit_in_the_last_place) + ## difference tolerated when comparing two numbers. The larger the value, the + ## more error is allowed. A `0` value means that two numbers must be exactly the + ## same to be considered equal. + ## + ## The machine epsilon has to be scaled to the magnitude of the values used + ## and multiplied by the desired precision in ULPs unless the difference is + ## subnormal. + almostEqual(x.re, y.re, unitsInLastPlace = unitsInLastPlace) and + almostEqual(x.im, y.im, unitsInLastPlace = unitsInLastPlace) func `$`*(z: Complex): string = ## Returns `z`'s string representation as `"(re, im)"`. @@ -403,4 +435,39 @@ func `$`*(z: Complex): string = result = "(" & $z.re & ", " & $z.im & ")" +proc formatValueAsTuple(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (real, imaginary) tuple. + result.add "(" + formatValue(result, value.re, specifier) + result.add ", " + formatValue(result, value.im, specifier) + result.add ")" + +proc formatValueAsComplexNumber(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (RE+IMj) number + ## By default, the real and imaginary parts are formatted using the general ('g') format + let specifier = if specifier.contains({'e', 'E', 'f', 'F', 'g', 'G'}): + specifier.replace("j") + else: + specifier.replace('j', 'g') + result.add "(" + formatValue(result, value.re, specifier) + if value.im >= 0 and not specifier.contains({'+', '-'}): + result.add "+" + formatValue(result, value.im, specifier) + result.add "j)" + +proc formatValue*(result: var string; value: Complex; specifier: string) = + ## Standard format implementation for `Complex`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + ## For complex numbers, we add a specific 'j' specifier, which formats + ## the value as (A+Bj) like in mathematics. + if specifier.len == 0: + result.add $value + elif 'j' in specifier: + formatValueAsComplexNumber(result, value, specifier) + else: + formatValueAsTuple(result, value, specifier) + {.pop.} diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index bdf1e8cc21..c7b881bc55 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -211,8 +211,8 @@ else: # MSVC intrinsics proc interlockedExchange(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8".} - proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange".} - proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange".} proc interlockedExchange(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64".} proc interlockedCompareExchange(location: pointer; desired, expected: int8): int8 {.importc: "_InterlockedCompareExchange8".} @@ -286,26 +286,23 @@ else: moSequentiallyConsistent type - # Atomic* {.importcpp: "_Atomic('0)".} [T] = object + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object - AtomicInt8 {.importc: "_Atomic NI8", size: 1.} = object - AtomicInt16 {.importc: "_Atomic NI16", size: 2.} = object - AtomicInt32 {.importc: "_Atomic NI32", size: 4.} = object - AtomicInt64 {.importc: "_Atomic NI64", size: 8.} = object - - template atomicType*(T: typedesc[Trivial]): untyped = - # Maps the size of a trivial type to it's internal atomic type - when sizeof(T) == 1: AtomicInt8 - elif sizeof(T) == 2: AtomicInt16 - elif sizeof(T) == 4: AtomicInt32 - elif sizeof(T) == 8: AtomicInt64 + AtomicInt8 {.importc: "_Atomic NI8".} = int8 + AtomicInt16 {.importc: "_Atomic NI16".} = int16 + AtomicInt32 {.importc: "_Atomic NI32".} = int32 + AtomicInt64 {.importc: "_Atomic NI64".} = int64 type AtomicFlag* {.importc: "atomic_flag", size: 1.} = object Atomic*[T] = object when T is Trivial: - value: T.atomicType + # Maps the size of a trivial type to it's internal atomic type + when sizeof(T) == 1: value: AtomicInt8 + elif sizeof(T) == 2: value: AtomicInt16 + elif sizeof(T) == 4: value: AtomicInt32 + elif sizeof(T) == 8: value: AtomicInt64 else: nonAtomicValue: T guard: AtomicFlag @@ -364,11 +361,11 @@ else: cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = - while location.guard.testAndSet(moAcquire): discard + while testAndSet(location.guard, moAcquire): discard try: body finally: - location.guard.clear(moRelease) + clear(location.guard, moRelease) proc load*[T: not Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = withLock(location, order): diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index ee43b8e11f..0b2c915b24 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -16,7 +16,10 @@ runnableExamples: include "system/inclrtl" when defined(posix) and not (defined(macosx) or defined(bsd)): - import posix + import std/posix + +when defined(windows): + import std/private/win_getsysteminfo when defined(freebsd) or defined(macosx): {.emit: "#include ".} @@ -37,7 +40,7 @@ when defined(macosx) or defined(bsd): importc: "sysctl", nodecl.} when defined(genode): - include genode/env + import genode/env proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. importcpp: "@->cpu().affinity_space().total()".} @@ -54,25 +57,10 @@ proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. when defined(windows): - type - SYSTEM_INFO {.final, pure.} = object - u1: int32 - dwPageSize: int32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr int32 - dwNumberOfProcessors: int32 - dwProcessorType: int32 - dwAllocationGranularity: int32 - wProcessorLevel: int16 - wProcessorRevision: int16 - - proc GetSystemInfo(lpSystemInfo: var SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".} - var - si: SYSTEM_INFO - GetSystemInfo(si) - result = si.dwNumberOfProcessors + si: SystemInfo + getSystemInfo(addr si) + result = int(si.dwNumberOfProcessors) elif defined(macosx) or defined(bsd): var mib: array[0..3, cint] diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 841d58d86b..bfbf167212 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -13,11 +13,14 @@ ## Unstable API. when defined(windows): - import winlean, os, strutils, math + import std/[winlean, os, strutils, math] proc `-`(a, b: FILETIME): int64 = a.rdFileTime - b.rdFileTime elif defined(linux): - from cpuinfo import countProcessors + from std/cpuinfo import countProcessors + +when defined(nimPreviewSlimSystem): + import std/syncio type ThreadPoolAdvice* = enum @@ -84,7 +87,7 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = inc s.calls when not defined(testing) and isMainModule and not defined(nimdoc): - import random + import std/random proc busyLoop() = while true: diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 905e668ce9..06ed2fe542 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -7,21 +7,25 @@ # distribution, for details about the copyright. # +{.deprecated: "use the nimble packages `malebolgia`, `taskpools` or `weave` instead".} + ## Implements Nim's `parallel & spawn statements `_. ## ## Unstable API. ## ## See also ## ======== -## * `threads module `_ for basic thread support -## * `channels module `_ for message passing support +## * `threads module `_ for basic thread support ## * `locks module `_ for locks and condition variables ## * `asyncdispatch module `_ for asynchronous IO when not compileOption("threads"): {.error: "Threadpool requires --threads:on option.".} -import cpuinfo, cpuload, locks, os +import std/[cpuinfo, cpuload, locks, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, typedthreads, sysatomics] {.push stackTrace:off.} @@ -52,17 +56,14 @@ proc signal(cv: var Semaphore) = release(cv.L) signal(cv.c) -const CacheLineSize = 32 # true for most archs +const CacheLineSize = 64 # true for most archs type Barrier {.compilerproc.} = object entered: int cv: Semaphore # Semaphore takes 3 words at least - when sizeof(int) < 8: - cacheAlign: array[CacheLineSize-4*sizeof(int), byte] - left: int - cacheAlign2: array[CacheLineSize-sizeof(int), byte] - interest: bool # whether the master is interested in the "all done" event + left {.align(CacheLineSize).}: int + interest {.align(CacheLineSize).} : bool # whether the master is interested in the "all done" event proc barrierEnter(b: ptr Barrier) {.compilerproc, inline.} = # due to the signaling between threads, it is ensured we are the only @@ -103,7 +104,7 @@ type idx: int FlowVarBase* = ref FlowVarBaseObj ## Untyped base class for `FlowVar[T] <#FlowVar>`_. - FlowVarBaseObj = object of RootObj + FlowVarBaseObj {.acyclic.} = object of RootObj ready, usesSemaphore, awaited: bool cv: Semaphore # for 'blockUntilAny' support ai: ptr AwaitInfo @@ -112,7 +113,7 @@ type # be RootRef here otherwise the wrong GC keeps track of it! owner: pointer # ptr Worker - FlowVarObj[T] = object of FlowVarBaseObj + FlowVarObj[T] {.acyclic.} = object of FlowVarBaseObj blob: T FlowVar*[T] {.compilerproc.} = ref FlowVarObj[T] ## A data flow variable. @@ -451,19 +452,21 @@ proc preferSpawn*(): bool = ## <#spawnX.t>`_ instead. result = gSomeReady.counter > 0 -proc spawn*(call: sink typed): void {.magic: "Spawn".} +proc spawn*(call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task, so that the `call` is never executed on ## the calling thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn3` internally" -proc pinnedSpawn*(id: ThreadId; call: sink typed): void {.magic: "Spawn".} +proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task on the worker thread with `id`, so that ## the `call` is **always** executed on the thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn4` internally" template spawnX*(call) = ## Spawns a new task if a CPU core is ready, otherwise executes the diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 8d9cc0c958..f628aaf6b7 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -11,6 +11,9 @@ import std/[strtabs, times, options] +when defined(nimPreviewSlimSystem): + import std/assertions + type SameSite* {.pure.} = enum ## The SameSite cookie attribute. @@ -25,7 +28,7 @@ proc parseCookies*(s: string): StringTableRef = ## "Set-Cookie" header set by servers. runnableExamples: import std/strtabs - let cookieJar = parseCookies("a=1; foo=bar") + let cookieJar = parseCookies("a=1; foo=bar") assert cookieJar["a"] == "1" assert cookieJar["foo"] == "bar" @@ -46,10 +49,12 @@ proc parseCookies*(s: string): StringTableRef = proc setCookie*(key, value: string, domain = "", path = "", expires = "", noName = false, - secure = false, httpOnly = false, + secure = false, httpOnly = false, maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of ## `Set-Cookie: key=value; Domain=...; ...` + ## + ## .. tip:: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value @@ -73,4 +78,4 @@ proc setCookie*(key, value: string, expires: DateTime|Time, ## `Set-Cookie: key=value; Domain=...; ...` result = setCookie(key, value, domain, path, format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), - noname, secure, httpOnly, maxAge, sameSite) + noName, secure, httpOnly, maxAge, sameSite) diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index 21675928d6..24836e3164 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -8,11 +8,11 @@ # ## Nim coroutines implementation, supports several context switching methods: -## -------- ------------ +## ======== ============ ## ucontext available on unix and alike (default) ## setjmp available on unix and alike (x86/64 only) ## fibers available and required on windows. -## -------- ------------ +## ======== ============ ## ## -d:nimCoroutines Required to build this module. ## -d:nimCoroutinesUcontext Use ucontext backend. @@ -21,18 +21,22 @@ ## ## Unstable API. +import system/coro_detection + when not nimCoroutines and not defined(nimdoc): when defined(noNimCoroutines): {.error: "Coroutines can not be used with -d:noNimCoroutines".} else: {.error: "Coroutines require -d:nimCoroutines".} -import os -import lists +import std/[os, lists] include system/timers +when defined(nimPreviewSlimSystem): + import std/assertions + const defaultStackSize = 512 * 1024 -const useOrcArc = defined(gcArc) or defined(gcOrc) +const useOrcArc = defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc) when useOrcArc: proc nimGC_setStackBottom*(theStackBottom: pointer) = discard @@ -63,7 +67,7 @@ else: const coroBackend = CORO_BACKEND_UCONTEXT when coroBackend == CORO_BACKEND_FIBERS: - import windows/winlean + import std/winlean type Context = pointer @@ -219,7 +223,7 @@ proc switchTo(current, to: CoroutinePtr) = elif to.state == CORO_CREATED: # Coroutine is started. coroExecWithStack(runCurrentTask, to.stack.bottom) - #doAssert false + #raiseAssert "unreachable" else: {.error: "Invalid coroutine backend set.".} # Execution was just resumed. Restore frame information and set active stack. @@ -261,7 +265,7 @@ proc runCurrentTask() = current.state = CORO_FINISHED nimGC_setStackBottom(ctx.ncbottom) suspend(0) # Exit coroutine without returning from coroExecWithStack() - doAssert false + raiseAssert "unreachable" proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discardable.} = ## Schedule coroutine for execution. It does not run immediately. @@ -273,13 +277,11 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine))) coro.execContext = CreateFiberEx(stacksize, stacksize, FIBER_FLAG_FLOAT_SWITCH, - (proc(p: pointer): void {.stdcall.} = runCurrentTask()), - nil) - coro.stack.size = stacksize + (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) - coro.stack.top = cast[pointer](cast[ByteAddress](coro) + sizeof(Coroutine)) - coro.stack.bottom = cast[pointer](cast[ByteAddress](coro.stack.top) + stacksize) + coro.stack.top = cast[pointer](cast[int](coro) + sizeof(Coroutine)) + coro.stack.bottom = cast[pointer](cast[int](coro.stack.top) + stacksize) when coroBackend == CORO_BACKEND_UCONTEXT: discard getcontext(coro.execContext) coro.execContext.uc_stack.ss_sp = coro.stack.top @@ -294,9 +296,9 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar return coro.reference proc run*() = - initialize() ## Starts main coroutine scheduler loop which exits when all coroutines exit. ## Calling this proc starts execution of first coroutine. + initialize() ctx.current = ctx.coroutines.head var minDelay: float = 0 while ctx.current != nil: diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim index 0eae00fbaa..c907e54d8f 100644 --- a/lib/pure/cstrutils.nim +++ b/lib/pure/cstrutils.nim @@ -75,7 +75,7 @@ func cmpIgnoreStyle*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreStyle".} = ## for that. Returns: ## * 0 if `a == b` ## * < 0 if `a < b` - ## * > 0 if `a > b` + ## * \> 0 if `a > b` runnableExamples: assert cmpIgnoreStyle(cstring"hello", cstring"H_e_L_Lo") == 0 @@ -101,7 +101,7 @@ func cmpIgnoreCase*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## * 0 if `a == b` ## * < 0 if `a < b` - ## * > 0 if `a > b` + ## * \> 0 if `a > b` runnableExamples: assert cmpIgnoreCase(cstring"hello", cstring"HeLLo") == 0 assert cmpIgnoreCase(cstring"echo", cstring"hello") < 0 diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim deleted file mode 100644 index 852c8e1c39..0000000000 --- a/lib/pure/db_common.nim +++ /dev/null @@ -1,100 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Common datatypes and definitions for all `db_*.nim` ( -## `db_mysql `_, `db_postgres `_, -## and `db_sqlite `_) modules. - -type - DbError* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string - - - DbEffect* = object of IOEffect ## effect that denotes a database operation - ReadDbEffect* = object of DbEffect ## effect that denotes a read operation - WriteDbEffect* = object of DbEffect ## effect that denotes a write operation - - DbTypeKind* = enum ## a superset of datatypes that might be supported. - dbUnknown, ## unknown datatype - dbSerial, ## datatype used for primary auto-increment keys - dbNull, ## datatype used for the NULL value - dbBit, ## bit datatype - dbBool, ## boolean datatype - dbBlob, ## blob datatype - dbFixedChar, ## string of fixed length - dbVarchar, ## string datatype - dbJson, ## JSON datatype - dbXml, ## XML datatype - dbInt, ## some integer type - dbUInt, ## some unsigned integer type - dbDecimal, ## decimal numbers (fixed-point number) - dbFloat, ## some floating point type - dbDate, ## a year-month-day description - dbTime, ## HH:MM:SS information - dbDatetime, ## year-month-day and HH:MM:SS information, - ## plus optional time or timezone information - dbTimestamp, ## Timestamp values are stored as the number of seconds - ## since the epoch ('1970-01-01 00:00:00' UTC). - dbTimeInterval, ## an interval [a,b] of times - dbEnum, ## some enum - dbSet, ## set of enum values - dbArray, ## an array of values - dbComposite, ## composite type (record, struct, etc) - dbUrl, ## a URL - dbUuid, ## a UUID - dbInet, ## an IP address - dbMacAddress, ## a MAC address - dbGeometry, ## some geometric type - dbPoint, ## Point on a plane (x,y) - dbLine, ## Infinite line ((x1,y1),(x2,y2)) - dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) - dbBox, ## Rectangular box ((x1,y1),(x2,y2)) - dbPath, ## Closed or open path (similar to polygon) ((x1,y1),...) - dbPolygon, ## Polygon (similar to closed path) ((x1,y1),...) - dbCircle, ## Circle <(x,y),r> (center point and radius) - dbUser1, ## user definable datatype 1 (for unknown extensions) - dbUser2, ## user definable datatype 2 (for unknown extensions) - dbUser3, ## user definable datatype 3 (for unknown extensions) - dbUser4, ## user definable datatype 4 (for unknown extensions) - dbUser5 ## user definable datatype 5 (for unknown extensions) - - DbType* = object ## describes a database type - kind*: DbTypeKind ## the kind of the described type - notNull*: bool ## does the type contain NULL? - name*: string ## the name of the type - size*: Natural ## the size of the datatype; 0 if of variable size - maxReprLen*: Natural ## maximal length required for the representation - precision*, scale*: Natural ## precision and scale of the number - min*, max*: BiggestInt ## the minimum and maximum of allowed values - validValues*: seq[string] ## valid values of an enum or a set - - DbColumn* = object ## information about a database column - name*: string ## name of the column - tableName*: string ## name of the table the column belongs to (optional) - typ*: DbType ## type of the column - primaryKey*: bool ## is this a primary key? - foreignKey*: bool ## is this a foreign key? - DbColumns* = seq[DbColumn] - -template sql*(query: string): SqlQuery = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## `sql"update user set counter = counter + 1"` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - SqlQuery(query) - -proc dbError*(msg: string) {.noreturn, noinline.} = - ## raises an DbError exception with message `msg`. - var e: ref DbError - new(e) - e.msg = msg - raise e diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 2b119b92cc..9e71d4ce09 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -9,21 +9,20 @@ ## This module implements the basics for Linux distribution ("distro") ## detection and the OS's native package manager. Its primary purpose is to -## produce output for Nimble packages, like:: +## produce output for Nimble packages, like: ## -## To complete the installation, run: +## To complete the installation, run: ## -## sudo apt-get install libblas-dev -## sudo apt-get install libvoodoo +## sudo apt-get install libblas-dev +## sudo apt-get install libvoodoo ## ## The above output could be the result of a code snippet like: ## -## .. code-block:: nim -## +## ```nim ## if detectOs(Ubuntu): ## foreignDep "lbiblas-dev" ## foreignDep "libvoodoo" -## +## ``` ## ## See `packaging `_ for hints on distributing Nim using OS packages. @@ -31,7 +30,7 @@ from std/strutils import contains, toLowerAscii when not defined(nimscript): from std/osproc import execProcess - from std/os import existsEnv + from std/envvars import existsEnv type Distribution* {.pure.} = enum ## the list of known distributions @@ -52,6 +51,7 @@ type CentOS Deepin ArchLinux + Artix Antergos PCLinuxOS Mageia @@ -108,7 +108,7 @@ type Clonezilla SteamOS Absolute - NixOS ## NixOS or a Nix build environment + NixOS ## NixOS or a Nix build environment AUSTRUMI Arya Porteus @@ -123,9 +123,11 @@ type ExTiX Rockstor GoboLinux + Void BSD FreeBSD + NetBSD OpenBSD DragonFlyBSD @@ -133,7 +135,9 @@ type const - LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux} + LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, + Distribution.ArchLinux, Distribution.Artix, Distribution.Antergos, + Distribution.BlackArch, Distribution.ArchBang} # we cache the result of the 'cmdRelease' # execution for faster platform detections. @@ -152,7 +156,8 @@ template hostnamectl(): untyped = cmdRelease("hostnamectl", hostnamectlRes) proc detectOsWithAllCmd(d: Distribution): bool = let dd = toLowerAscii($d) result = dd in toLowerAscii(osReleaseID()) or dd in toLowerAscii(release()) or - dd in toLowerAscii(uname()) or ("operating system: " & dd) in toLowerAscii(hostnamectl()) + dd in toLowerAscii(uname()) or ("operating system: " & dd) in + toLowerAscii(hostnamectl()) proc detectOsImpl(d: Distribution): bool = case d @@ -164,7 +169,7 @@ proc detectOsImpl(d: Distribution): bool = else: when defined(bsd): case d - of Distribution.FreeBSD, Distribution.OpenBSD: + of Distribution.FreeBSD, Distribution.NetBSD, Distribution.OpenBSD: result = $d in uname() else: result = false @@ -172,14 +177,16 @@ proc detectOsImpl(d: Distribution): bool = case d of Distribution.Gentoo: result = ("-" & $d & " ") in uname() - of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, Distribution.Fedora, - Distribution.OpenMandriva, Distribution.CentOS, Distribution.Alpine, - Distribution.Mageia, Distribution.Zorin: + of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, + Distribution.Fedora, Distribution.OpenMandriva, Distribution.CentOS, + Distribution.Alpine, Distribution.Mageia, Distribution.Zorin, Distribution.Void: result = toLowerAscii($d) in osReleaseID() of Distribution.RedHat: result = "rhel" in osReleaseID() of Distribution.ArchLinux: result = "arch" in osReleaseID() + of Distribution.Artix: + result = "artix" in osReleaseID() of Distribution.NixOS: # Check if this is a Nix build or NixOS environment result = existsEnv("NIX_BUILD_TOP") or existsEnv("__NIXOS_SET_ENVIRONMENT_DONE") @@ -204,7 +211,7 @@ template detectOs*(d: untyped): bool = detectOsImpl(Distribution.d) when not defined(nimble): - var foreignDeps: seq[string] = @[] + var foreignDeps*: seq[string] = @[] ## Registered foreign deps. proc foreignCmd*(cmd: string; requiresSudo = false) = ## Registers a foreign command to the internal list of commands @@ -245,12 +252,14 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("nix-env -i " & p, false) elif detectOs(Solaris) or detectOs(FreeBSD): result = ("pkg install " & p, true) - elif detectOs(OpenBSD): + elif detectOs(NetBSD) or detectOs(OpenBSD): result = ("pkg_add " & p, true) elif detectOs(PCLinuxOS): result = ("rpm -ivh " & p, true) - elif detectOs(ArchLinux) or detectOs(Manjaro): + elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): result = ("pacman -S " & p, true) + elif detectOs(Void): + result = ("xbps-install " & p, true) else: result = (" install " & p, true) elif defined(haiku): diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index 3e1d84729d..a162fe37fa 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -17,75 +17,64 @@ ## Loading a simple C function ## --------------------------- ## -## The following example demonstrates loading a function called 'greet' +## The following example demonstrates loading a function called `greet` ## from a library that is determined at runtime based upon a language choice. -## If the library fails to load or the function 'greet' is not found, +## If the library fails to load or the function `greet` is not found, ## it quits with a failure error code. ## -## .. code-block::nim -## -## import std/dynlib -## -## type -## greetFunction = proc(): cstring {.gcsafe, stdcall.} -## -## let lang = stdin.readLine() -## -## let lib = case lang -## of "french": -## loadLib("french.dll") -## else: -## loadLib("english.dll") -## -## if lib == nil: -## echo "Error loading library" -## quit(QuitFailure) -## -## let greet = cast[greetFunction](lib.symAddr("greet")) -## -## if greet == nil: -## echo "Error loading 'greet' function from library" -## quit(QuitFailure) -## -## let greeting = greet() -## -## echo greeting -## -## unloadLib(lib) -## +runnableExamples: + type + GreetFunction = proc (): cstring {.gcsafe, stdcall.} -import strutils + proc loadGreet(lang: string) = + let lib = + case lang + of "french": + loadLib("french.dll") + else: + loadLib("english.dll") + assert lib != nil, "Error loading library" + + let greet = cast[GreetFunction](lib.symAddr("greet")) + assert greet != nil, "Error loading 'greet' function from library" + + echo greet() + + unloadLib(lib) + + +import std/strutils type - LibHandle* = pointer ## a handle to a dynamically loaded library + LibHandle* = pointer ## A handle to a dynamically loaded library. proc loadLib*(path: string, globalSymbols = false): LibHandle {.gcsafe.} - ## loads a library from `path`. Returns nil if the library could not + ## Loads a library from `path`. Returns nil if the library could not ## be loaded. proc loadLib*(): LibHandle {.gcsafe.} - ## gets the handle from the current executable. Returns nil if the + ## Gets the handle from the current executable. Returns nil if the ## library could not be loaded. proc unloadLib*(lib: LibHandle) {.gcsafe.} - ## unloads the library `lib` + ## Unloads the library `lib`. proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} = - ## raises an `EInvalidLibrary` exception. + ## Raises a `LibraryError` exception. raise newException(LibraryError, "could not find symbol: " & $name) proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.} - ## retrieves the address of a procedure/variable from `lib`. Returns nil + ## Retrieves the address of a procedure/variable from `lib`. Returns nil ## if the symbol could not be found. proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer = - ## retrieves the address of a procedure/variable from `lib`. Raises - ## `EInvalidLibrary` if the symbol could not be found. + ## Retrieves the address of a procedure/variable from `lib`. Raises + ## `LibraryError` if the symbol could not be found. result = symAddr(lib, name) if result == nil: raiseInvalidLibrary(name) proc libCandidates*(s: string, dest: var seq[string]) = - ## given a library name pattern `s` write possible library names to `dest`. + ## Given a library name pattern `s`, write possible library names to `dest`. var le = strutils.find(s, '(') var ri = strutils.find(s, ')', le+1) if le >= 0 and ri > le: @@ -97,8 +86,9 @@ proc libCandidates*(s: string, dest: var seq[string]) = add(dest, s) proc loadLibPattern*(pattern: string, globalSymbols = false): LibHandle = - ## loads a library with name matching `pattern`, similar to what `dynlib` + ## Loads a library with name matching `pattern`, similar to what the `dynlib` ## pragma does. Returns nil if the library could not be loaded. + ## ## .. warning:: this proc uses the GC and so cannot be used to load the GC. var candidates = newSeq[string]() libCandidates(pattern, candidates) @@ -115,7 +105,7 @@ when defined(posix) and not defined(nintendoswitch): # as an emulation layer on top of native functions. # ========================================================================= # - import posix + import std/posix proc loadLib(path: string, globalSymbols = false): LibHandle = let flags = @@ -150,6 +140,32 @@ elif defined(nintendoswitch): proc symAddr(lib: LibHandle, name: cstring): pointer = raise newException(OSError, "symAddr not implemented on Nintendo Switch!") +elif defined(genode): + # + # ========================================================================= + # Not implemented for Genode without POSIX. Raise an error if called. + # ========================================================================= + # + + template raiseErr(prc: string) = + raise newException(OSError, prc & " not implemented, compile with POSIX support") + + proc dlclose(lib: LibHandle) = + raiseErr(OSError, "dlclose") + proc dlopen(path: cstring, mode: int): LibHandle = + raiseErr(OSError, "dlopen") + proc dlsym(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "dlsym") + proc loadLib(path: string, global_symbols = false): LibHandle = + raiseErr(OSError, "loadLib") + proc loadLib(): LibHandle = + raiseErr(OSError, "loadLib") + proc unloadLib(lib: LibHandle) = + raiseErr(OSError, "unloadLib") + proc symAddr(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "symAddr") + + elif defined(windows) or defined(dos): # # ======================================================================= diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 1d8512018e..bbadca6552 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -7,15 +7,46 @@ # distribution, for details about the copyright. # -## Converts between different character encodings. On UNIX, this uses +## Routines for converting between different character encodings. On UNIX, this uses ## the `iconv`:idx: library, on Windows the Windows API. +## +## The following example shows how to change character encodings. +runnableExamples: + when defined(windows): + let + orig = "öäüß" + # convert `orig` from "UTF-8" to "CP1252" + cp1252 = convert(orig, "CP1252", "UTF-8") + # convert `cp1252` from "CP1252" to "ibm850" + ibm850 = convert(cp1252, "ibm850", "CP1252") + current = getCurrentEncoding() + assert orig == "\195\182\195\164\195\188\195\159" + assert ibm850 == "\148\132\129\225" + assert convert(ibm850, current, "ibm850") == orig -import os +## The example below uses a reuseable `EncodingConverter` object which is +## created by `open` with `destEncoding` and `srcEncoding` specified. You can use +## `convert` on this object multiple times. +runnableExamples: + when defined(windows): + var fromGB2312 = open("utf-8", "gb2312") + let first = "\203\173\197\194\163\191\210\187" & + "\203\242\209\204\211\234\200\206\198\189\201\250" + assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生" + + let second = "\211\208\176\215\205\183\200\231" & + "\208\194\163\172\199\227\184\199\200\231\185\202" + assert fromGB2312.convert(second) == "有白头如新，倾盖如故" + + +import std/os +when defined(nimPreviewSlimSystem): + import std/assertions when not defined(windows): type ConverterObj = object - EncodingConverter* = ptr ConverterObj ## can convert between two character sets + EncodingConverter* = ptr ConverterObj ## Can convert between two character sets. else: type @@ -24,11 +55,11 @@ else: dest, src: CodePage type - EncodingError* = object of ValueError ## exception that is raised - ## for encoding errors + EncodingError* = object of ValueError ## Exception that is raised + ## for encoding errors. when defined(windows): - import parseutils, strutils + import std/[parseutils, strutils] proc eqEncodingNames(a, b: string): bool = var i = 0 var j = 0 @@ -72,6 +103,7 @@ when defined(windows): (875, "cp875"), # IBM EBCDIC Greek Modern (932, "shift_jis"), # ANSI/OEM Japanese; Japanese (Shift-JIS) (936, "gb2312"), # ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) + (936, "gbk"), # Alias for GB2312 encoding (949, "ks_c_5601-1987"), # ANSI/OEM Korean (Unified Hangul Code) (950, "big5"), # ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) (1026, "IBM1026"), # IBM EBCDIC Turkish (Latin 5) @@ -281,8 +313,10 @@ else: var errno {.importc, header: "".}: cint - when defined(freebsd) or defined(netbsd): + when defined(bsd): {.pragma: importIconv, cdecl, header: "".} + when defined(openbsd): + {.passL: "-liconv".} else: {.pragma: importIconv, cdecl, dynlib: iconvDll.} @@ -295,7 +329,7 @@ else: importc: "iconv", importIconv.} proc getCurrentEncoding*(uiApp = false): string = - ## retrieves the current encoding. On Unix, always "UTF-8" is returned. + ## Retrieves the current encoding. On Unix, "UTF-8" is always returned. ## The `uiApp` parameter is Windows specific. If true, the UI's code-page ## is returned, if false, the Console's code-page is returned. when defined(windows): @@ -304,11 +338,11 @@ proc getCurrentEncoding*(uiApp = false): string = result = "UTF-8" proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = - ## opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `IOError` if it cannot fulfill the request. + ## Opens a converter that can convert from `srcEncoding` to `destEncoding`. + ## Raises `EncodingError` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) - if result == nil: + if result == cast[EncodingConverter](-1): raise newException(EncodingError, "cannot create encoding converter from " & srcEncoding & " to " & destEncoding) @@ -323,7 +357,7 @@ proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = "cannot find encoding " & srcEncoding) proc close*(c: EncodingConverter) = - ## frees the resources the converter `c` holds. + ## Frees the resources the converter `c` holds. when not defined(windows): iconvClose(c) @@ -418,12 +452,13 @@ when defined(windows): else: convertFromWideString(codePageTo, wideString) proc convert*(c: EncodingConverter, s: string): string = - ## converts `s` to `destEncoding` that was given to the converter `c`. It - ## assumed that `s` is in `srcEncoding`. - ## utf-16BE, utf-32 conversions not supported on windows result = convertWin(c.src, c.dest, s) else: proc convert*(c: EncodingConverter, s: string): string = + ## Converts `s` to `destEncoding` that was given to the converter `c`. It + ## assumes that `s` is in `srcEncoding`. + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. result = newString(s.len) var inLen = csize_t len(s) var outLen = csize_t len(result) @@ -464,84 +499,13 @@ else: proc convert*(s: string, destEncoding = "UTF-8", srcEncoding = "CP1252"): string = - ## converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. + ## Converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. ## This opens a converter, uses it and closes it again and is thus more ## convenient but also likely less efficient than re-using a converter. - ## utf-16BE, utf-32 conversions not supported on windows + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. var c = open(destEncoding, srcEncoding) try: result = convert(c, s) finally: close(c) - -when not defined(testing) and isMainModule: - let - orig = "öäüß" - cp1252 = convert(orig, "CP1252", "UTF-8") - ibm850 = convert(cp1252, "ibm850", "CP1252") - current = getCurrentEncoding() - echo "Original string from source code: ", orig - echo "Forced ibm850 encoding: ", ibm850 - echo "Current encoding: ", current - echo "From ibm850 to current: ", convert(ibm850, current, "ibm850") - -when not defined(testing) and isMainModule and defined(windows): - block should_throw_on_unsupported_conversions: - let original = "some string" - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "unicodeFFFE") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32BE") - - doAssertRaises(EncodingError): - discard convert(original, "unicodeFFFE", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32BE", "utf-8") - - block should_convert_from_utf16_to_utf8: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "utf-8", "utf-16") - doAssert(result == "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82") - - block should_convert_from_utf16_to_win1251: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "windows-1251", "utf-16") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_win1251_to_koi8r: - let original = "\xf2\xe5\xf1\xf2" # win1251 test string "тест" - let result = convert(original, "koi8-r", "windows-1251") - doAssert(result == "\xd4\xc5\xd3\xd4") - - block should_convert_from_koi8r_to_win1251: - let original = "\xd4\xc5\xd3\xd4" # koi8r test string "тест" - let result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_win1251: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "windows-1251", "utf-8") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_utf16: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "utf-16", "utf-8") - doAssert(result == "\x42\x04\x35\x04\x41\x04\x42\x04") - - block should_handle_empty_string_for_any_conversion: - let original = "" - var result = convert(original, "utf-16", "utf-8") - doAssert(result == "") - result = convert(original, "utf-8", "utf-16") - doAssert(result == "") - result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "") diff --git a/lib/pure/endians.nim b/lib/pure/endians.nim index 29fde1c805..4c1d45ae5e 100644 --- a/lib/pure/endians.nim +++ b/lib/pure/endians.nim @@ -10,6 +10,11 @@ ## This module contains helpers that deal with different byte orders ## (`endian`:idx:). ## +## Endianness is the order of bytes of a value in memory. Big-endian means that +## the most significant byte is stored at the smallest memory address, +## while little endian means that the least-significant byte is stored +## at the smallest address. See also https://en.wikipedia.org/wiki/Endianness. +## ## Unstable API. when defined(gcc) or defined(llvm_gcc) or defined(clang): @@ -47,7 +52,7 @@ else: when useBuiltinSwap: template swapOpImpl(T: typedesc, op: untyped) = - ## We have to use `copyMem` here instead of a simple deference because they + ## We have to use `copyMem` here instead of a simple dereference because they ## may point to a unaligned address. A sufficiently smart compiler _should_ ## be able to elide them when they're not necessary. var tmp: T @@ -56,18 +61,40 @@ when useBuiltinSwap: copyMem(outp, addr tmp, sizeof(T)) proc swapEndian64*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 8 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4, 5, 6, 7, 8] + var b: array[8, uint8] + swapEndian64(addr b, addr a) + assert b == [8'u8, 7, 6, 5, 4, 3, 2, 1] + swapOpImpl(uint64, builtin_bswap64) proc swapEndian32*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 4 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4] + var b: array[4, uint8] + swapEndian32(addr b, addr a) + assert b == [4'u8, 3, 2, 1] + swapOpImpl(uint32, builtin_bswap32) proc swapEndian16*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 2 bytes. + runnableExamples: + var a = [1'u8, 2] + var b: array[2, uint8] + swapEndian16(addr b, addr a) + assert b == [2'u8, 1] + swapOpImpl(uint16, builtin_bswap16) else: proc swapEndian64*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 8 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[7] @@ -80,8 +107,6 @@ else: o[7] = i[0] proc swapEndian32*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 4 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[3] @@ -90,8 +115,6 @@ else: o[3] = i[0] proc swapEndian16*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 2 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[1] @@ -106,8 +129,20 @@ when system.cpuEndian == bigEndian: proc bigEndian16*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 2) else: proc littleEndian64*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 8) + ## Copies `inp` to `outp`, storing it in 64-bit little-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc littleEndian32*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 4) + ## Copies `inp` to `outp`, storing it in 32-bit little-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc littleEndian16*(outp, inp: pointer){.inline.} = copyMem(outp, inp, 2) + ## Copies `inp` to `outp`, storing it in 16-bit little-endian order. + ## Both buffers are supposed to contain at least 2 bytes. proc bigEndian64*(outp, inp: pointer) {.inline.} = swapEndian64(outp, inp) + ## Copies `inp` to `outp`, storing it in 64-bit big-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc bigEndian32*(outp, inp: pointer) {.inline.} = swapEndian32(outp, inp) + ## Copies `inp` to `outp`, storing it in 32-bit big-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc bigEndian16*(outp, inp: pointer) {.inline.} = swapEndian16(outp, inp) + ## Copies `inp` to `outp`, storing it in 16-bit big-endian order. + ## Both buffers are supposed to contain at least 2 bytes. diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index ddb6a1a0c3..1d96fd6be6 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -12,7 +12,7 @@ ## The types, vars and procs are bindings for the C standard library ## [](https://en.cppreference.com/w/c/numeric/fenv) header. -when defined(posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} var diff --git a/lib/pure/future.nim b/lib/pure/future.nim deleted file mode 100644 index 40dba78469..0000000000 --- a/lib/pure/future.nim +++ /dev/null @@ -1,4 +0,0 @@ - -{.deprecated: "Use the new 'sugar' module instead".} - -include sugar diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index c526c976fe..5182995116 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -21,7 +21,7 @@ runnableExamples: foo: int bar: string - iterator items(x: Something): int = + iterator items(x: Something): Hash = yield hash(x.foo) yield hash(x.bar) @@ -51,18 +51,26 @@ runnableExamples: h = h !& hash(x.bar) result = !$h -## **Note:** If the type has a `==` operator, the following must hold: -## If two values compare equal, their hashes must also be equal. +## .. important:: Use `-d:nimPreviewHashRef` to +## enable hashing `ref`s. It is expected that this behavior +## becomes the new default in upcoming versions. +## +## .. note:: If the type has a `==` operator, the following must hold: +## If two values compare equal, their hashes must also be equal. ## ## See also ## ======== ## * `md5 module `_ for the MD5 checksum algorithm ## * `base64 module `_ for a Base64 encoder and decoder -## * `std/sha1 module `_ for a SHA-1 encoder and decoder +## * `sha1 module `_ for the SHA-1 checksum algorithm ## * `tables module `_ for hash tables import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + type Hash* = int ## A hash value. Hash tables using these values should ## always have a size of a power of two so they can use the `and` @@ -182,7 +190,7 @@ proc hashData*(data: pointer, size: int): Hash = var h: Hash = 0 when defined(js): var p: cstring - asm """`p` = `Data`;""" + {.emit: """`p` = `Data`""".} else: var p = cast[cstring](data) var i = 0 @@ -193,32 +201,6 @@ proc hashData*(data: pointer, size: int): Hash = dec(s) result = !$h -when defined(js): - var objectID = 0 - -proc hash*(x: pointer): Hash {.inline.} = - ## Efficient hashing of pointers. - when defined(js): - asm """ - if (typeof `x` == "object") { - if ("_NimID" in `x`) - `result` = `x`["_NimID"]; - else { - `result` = ++`objectID`; - `x`["_NimID"] = `result`; - } - } - """ - else: - result = cast[Hash](cast[uint](x) shr 3) # skip the alignment - -proc hash*[T: proc](x: T): Hash {.inline.} = - ## Efficient hashing of proc vars. Closures are supported too. - when T is "closure": - result = hash(rawProc(x)) !& hash(rawEnv(x)) - else: - result = hash(pointer(x)) - proc hashIdentity*[T: Ordinal|enum](x: T): Hash {.inline, since: (1, 3).} = ## The identity hash, i.e. `hashIdentity(x) = x`. cast[Hash](ord(x)) @@ -232,6 +214,61 @@ else: ## Efficient hashing of integers. hashWangYi1(uint64(ord(x))) +when defined(js): + var objectID = 0 + proc getObjectId(x: pointer): int = + {.emit: """ + if (typeof `x` == "object") { + if ("_NimID" in `x`) + `result` = `x`["_NimID"]; + else { + `result` = ++`objectID`; + `x`["_NimID"] = `result`; + } + } + """.} + +proc hash*(x: pointer): Hash {.inline.} = + ## Efficient `hash` overload. + when defined(js): + let y = getObjectId(x) + else: + let y = cast[int](x) + hash(y) # consistent with code expecting scrambled hashes depending on `nimIntHash1`. + +proc hash*[T](x: ptr[T]): Hash {.inline.} = + ## Efficient `hash` overload. + runnableExamples: + var a: array[10, uint8] + assert a[0].addr.hash != a[1].addr.hash + assert cast[pointer](a[0].addr).hash == a[0].addr.hash + hash(cast[pointer](x)) + +when defined(nimPreviewHashRef) or defined(nimdoc): + proc hash*[T](x: ref[T]): Hash {.inline.} = + ## Efficient `hash` overload. + ## + ## .. important:: Use `-d:nimPreviewHashRef` to + ## enable hashing `ref`s. It is expected that this behavior + ## becomes the new default in upcoming versions. + runnableExamples("-d:nimPreviewHashRef"): + type A = ref object + x: int + let a = A(x: 3) + let ha = a.hash + assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + a.x = 4 + assert ha == a.hash # the hash only depends on the address + runnableExamples("-d:nimPreviewHashRef"): + # you can overload `hash` if you want to customize semantics + type A[T] = ref object + x, y: T + proc hash(a: A): Hash = hash(a.x) + assert A[int](x: 3, y: 4).hash == A[int](x: 3, y: 5).hash + # xxx pending bug #17733, merge as `proc hash*(pointer | ref | ptr): Hash` + # or `proc hash*[T: ref | ptr](x: T): Hash` + hash(cast[pointer](x)) + proc hash*(x: float): Hash {.inline.} = ## Efficient hashing of floats. let y = x + 0.0 # for denormalization @@ -282,16 +319,24 @@ proc murmurHash(x: openArray[byte]): Hash = h1: uint32 i = 0 + + template impl = + var j = stepSize + while j > 0: + dec j + k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + # body while i < n * stepSize: var k1: uint32 - when defined(js) or defined(sparc) or defined(sparc64): - var j = stepSize - while j > 0: - dec j - k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + + when nimvm: + impl() else: - k1 = cast[ptr uint32](unsafeAddr x[i])[] + when declared(copyMem): + copyMem(addr k1, addr x[i], 4) + else: + impl() inc i, stepSize k1 = imul(k1, c1) @@ -323,16 +368,16 @@ proc murmurHash(x: openArray[byte]): Hash = return cast[Hash](h1) proc hashVmImpl(x: cstring, sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImpl(x: string, sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplChar(x: openArray[char], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplByte(x: openArray[byte], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hash*(x: string): Hash = ## Efficient hashing of strings. @@ -343,16 +388,10 @@ proc hash*(x: string): Hash = runnableExamples: doAssert hash("abracadabra") != hash("AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - for c in x: - result = result !& ord(c) - result = !$result + when nimvm: + result = hashVmImpl(x, 0, high(x)) else: - when nimvm: - result = hashVmImpl(x, 0, high(x)) - else: - result = murmurHash(toOpenArrayByte(x, 0, high(x))) + result = murmurHash(toOpenArrayByte(x, 0, high(x))) proc hash*(x: cstring): Hash = ## Efficient hashing of null-terminated strings. @@ -361,22 +400,14 @@ proc hash*(x: cstring): Hash = doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA") doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - var i = 0 - while x[i] != '\0': - result = result !& ord(x[i]) - inc i - result = !$result + when nimvm: + hashVmImpl(x, 0, high(x)) else: - when nimvm: - hashVmImpl(x, 0, high(x)) + when not defined(js): + murmurHash(toOpenArrayByte(x, 0, x.high)) else: - when not defined(js) and defined(nimToOpenArrayCString): - murmurHash(toOpenArrayByte(x, 0, x.high)) - else: - let xx = $x - murmurHash(toOpenArrayByte(xx, 0, high(xx))) + let xx = $x + murmurHash(toOpenArrayByte(xx, 0, high(xx))) proc hash*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting @@ -387,13 +418,7 @@ proc hash*(sBuf: string, sPos, ePos: int): Hash = var a = "abracadabra" doAssert hash(a, 0, 3) == hash(a, 7, 10) - when not defined(nimToOpenArrayCString): - result = 0 - for i in sPos..ePos: - result = result !& ord(sBuf[i]) - result = !$result - else: - murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) + murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) proc hashIgnoreStyle*(x: string): Hash = ## Efficient hashing of strings; style is ignored. @@ -484,11 +509,10 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h - -proc hash*[T: tuple | object](x: T): Hash = - ## Efficient hashing of tuples and objects. - ## There must be a `hash` proc defined for each of the field types. +proc hash*[T: tuple | object | proc | iterator {.closure.}](x: T): Hash = + ## Efficient `hash` overload. runnableExamples: + # for `tuple|object`, `hash` must be defined for each component of `x`. type Obj = object x: int y: string @@ -499,9 +523,31 @@ proc hash*[T: tuple | object](x: T): Hash = # you can define custom hashes for objects (even if they're generic): proc hash(a: Obj2): Hash = hash((a.x)) assert hash(Obj2[float](x: 520, y: "Nim")) == hash(Obj2[float](x: 520, y: "Nim2")) - for f in fields(x): - result = result !& hash(f) - result = !$result + runnableExamples: + # proc + proc fn1() = discard + const fn1b = fn1 + assert hash(fn1b) == hash(fn1) + + # closure + proc outer = + var a = 0 + proc fn2() = a.inc + assert fn2 is "closure" + let fn2b = fn2 + assert hash(fn2b) == hash(fn2) + assert hash(fn2) != hash(fn1) + outer() + + when T is "closure": + result = hash((rawProc(x), rawEnv(x))) + elif T is (proc): + result = hash(cast[pointer](x)) + else: + result = 0 + for f in fields(x): + result = result !& hash(f) + result = !$result proc hash*[A](x: openArray[A]): Hash = ## Efficient hashing of arrays and sequences. @@ -514,6 +560,7 @@ proc hash*[A](x: openArray[A]): Hash = else: result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: + result = 0 for a in x: result = result !& hash(a) result = !$result @@ -546,6 +593,7 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = proc hash*[A](x: set[A]): Hash = ## Efficient hashing of sets. ## There must be a `hash` proc defined for the element type `A`. + result = 0 for it in items(x): result = result !& hash(it) result = !$result diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index 04b1c5202d..fafa72463b 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -8,7 +8,7 @@ # ## Do yourself a favor and import the module -## as `from htmlgen import nil` and then fully qualify the macros. +## as `from std/htmlgen import nil` and then fully qualify the macros. ## ## *Note*: The Karax project (`nimble install karax`) has a better ## way to achieve the same, see https://github.com/pragmagic/karax/blob/master/tests/nativehtmlgen.nim @@ -30,17 +30,18 @@ ## Examples ## ======== ## -## .. code-block:: Nim +## ```Nim ## var nim = "Nim" ## echo h1(a(href="https://nim-lang.org", nim)) +## ``` ## -## Writes the string:: +## Writes the string: ## -##

    Nim

    +##

    Nim

    ## import - macros, strutils + std/[macros, strutils] const coreAttr* = " accesskey class contenteditable dir hidden id lang " & @@ -56,7 +57,7 @@ const ariaAttr* = " role " ## HTML DOM Aria Attributes commonAttr* = coreAttr & eventAttr & ariaAttr ## HTML DOM Common Attributes -proc getIdent(e: NimNode): string {.compileTime.} = +proc getIdent(e: NimNode): string = case e.kind of nnkIdent: result = e.strVal.normalize @@ -75,7 +76,7 @@ proc delete[T](s: var seq[T], attr: T): bool = result = true proc xmlCheckedTag*(argsList: NimNode, tag: string, optAttr = "", reqAttr = "", - isLeaf = false): NimNode {.compileTime.} = + isLeaf = false): NimNode = ## use this procedure to define a new XML tag # copy the attributes; when iterating over them these lists @@ -322,7 +323,7 @@ macro html*(e: varargs[untyped]): untyped = macro hr*(): untyped = ## Generates the HTML `hr` element. - result = xmlCheckedTag(newNimNode(nnkArglist), "hr", commonAttr, "", true) + result = xmlCheckedTag(newNimNode(nnkArgList), "hr", commonAttr, "", true) macro i*(e: varargs[untyped]): untyped = ## Generates the HTML `i` element. diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index c572146a70..62919546f2 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -12,10 +12,9 @@ ## ## It can be used to parse a wild HTML document and output it as valid XHTML ## document (well, if you are lucky): -## -## .. code-block:: Nim -## +## ```Nim ## echo loadHtml("mydirty.html") +## ``` ## ## Every tag in the resulting tree is in lower case. ## @@ -29,9 +28,7 @@ ## and write back the modified version. In this case we look for hyperlinks ## ending with the extension `.rst` and convert them to `.html`. ## -## .. code-block:: Nim -## :test: -## +## ```Nim test ## import std/htmlparser ## import std/xmltree # To use '$' for XmlNode ## import std/strtabs # To access XmlAttributes @@ -48,8 +45,14 @@ ## a.attrs["href"] = dir / filename & ".html" ## ## writeFile("output.html", $html) +## ``` -import strutils, streams, parsexml, xmltree, unicode, strtabs +{.deprecated: "use `nimble install htmlparser` and import `pkg/htmlparser` instead".} + +import std/[strutils, streams, parsexml, xmltree, unicode, strtabs] + +when defined(nimPreviewSlimSystem): + import std/syncio type HtmlTag* = enum ## list of all supported HTML tags; order will always be @@ -215,7 +218,7 @@ const tagMenu, tagNoframes} SingleTags* = {tagArea, tagBase, tagBasefont, tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, - tagLink, tagMeta, tagParam, tagWbr} + tagLink, tagMeta, tagParam, tagWbr, tagSource} proc allLower(s: string): bool = for c in s: @@ -391,10 +394,10 @@ proc entityToRune*(entity: string): Rune = case entity[1] of '0'..'9': try: runeValue = parseInt(entity[1..^1]) - except: discard + except ValueError: discard of 'x', 'X': # not case sensitive here try: runeValue = parseHexInt(entity[2..^1]) - except: discard + except ValueError: discard else: discard # other entities are not defined with prefix `#` if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values return Rune(runeValue) @@ -2061,7 +2064,7 @@ proc loadHtml*(path: string): XmlNode = result = loadHtml(path, errors) when not defined(testing) and isMainModule: - import os + import std/os var errors: seq[string] = @[] var x = loadHtml(paramStr(1), errors) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 14bcfd2fb8..08ea99627c 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -10,28 +10,38 @@ ## This module implements a simple HTTP client that can be used to retrieve ## webpages and other data. ## +## .. warning:: Validate untrusted inputs: URI parsers and getters are not detecting malicious URIs. +## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve ## `http://google.com`: ## -## .. code-block:: Nim +## ```Nim ## import std/httpclient ## var client = newHttpClient() -## echo client.getContent("http://google.com") +## try: +## echo client.getContent("http://google.com") +## finally: +## client.close() +## ``` ## ## The same action can also be performed asynchronously, simply use the ## `AsyncHttpClient`: ## -## .. code-block:: Nim +## ```Nim ## import std/[asyncdispatch, httpclient] ## ## proc asyncProc(): Future[string] {.async.} = ## var client = newAsyncHttpClient() -## return await client.getContent("http://example.com") +## try: +## return await client.getContent("http://google.com") +## finally: +## client.close() ## ## echo waitFor asyncProc() +## ``` ## ## The functionality implemented by `HttpClient` and `AsyncHttpClient` ## is the same, so you can use whichever one suits you best in the examples @@ -40,6 +50,10 @@ ## **Note:** You need to run asynchronous examples in an async proc ## otherwise you will get an `Undeclared identifier: 'await'` error. ## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. +## ## Using HTTP POST ## =============== ## @@ -47,33 +61,39 @@ ## uses `multipart/form-data` as the `Content-Type` to send the HTML to be ## validated to the server. ## -## .. code-block:: Nim +## ```Nim ## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" ## data["uploaded_file"] = ("test.html", "text/html", ## "

    test

    ") -## -## echo client.postContent("http://validator.w3.org/check", multipart=data) +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` ## ## To stream files from disk when performing the request, use `addFiles`. ## ## **Note:** This will allocate a new `Mimetypes` database every time you call ## it, you can pass your own via the `mimeDb` parameter to avoid this. ## -## .. code-block:: Nim +## ```Nim ## let mimes = newMimetypes() ## var client = newHttpClient() ## var data = newMultipartData() ## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes) -## -## echo client.postContent("http://validator.w3.org/check", multipart=data) +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` ## ## You can also make post requests with custom headers. ## This example sets `Content-Type` to `application/json` ## and uses a json object for the body ## -## .. code-block:: Nim +## ```Nim ## import std/[httpclient, json] ## ## let client = newHttpClient() @@ -81,8 +101,12 @@ ## let body = %*{ ## "data": "some text" ## } -## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) -## echo response.status +## try: +## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) +## echo response.status +## finally: +## client.close() +## ``` ## ## Progress reporting ## ================== @@ -91,31 +115,36 @@ ## This callback will be executed every second with information about the ## progress of the HTTP request. ## -## .. code-block:: Nim -## import std/[asyncdispatch, httpclient] +## ```Nim +## import std/[asyncdispatch, httpclient] ## -## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = -## echo("Downloaded ", progress, " of ", total) -## echo("Current rate: ", speed div 1000, "kb/s") +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") ## -## proc asyncProc() {.async.} = -## var client = newAsyncHttpClient() -## client.onProgressChanged = onProgressChanged -## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## try: +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## finally: +## client.close() ## -## waitFor asyncProc() +## waitFor asyncProc() +## ``` ## ## If you would like to remove the callback simply set it to `nil`. ## -## .. code-block:: Nim +## ```Nim ## client.onProgressChanged = nil +## ``` ## ## .. warning:: The `total` reported by httpclient may be 0 in some cases. ## ## ## SSL/TLS support ## =============== -## This requires the OpenSSL library, fortunately it's widely used and installed +## This requires the OpenSSL library. Fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give ## any of the functions a url with the `https` schema, for example: ## `https://github.com/`. @@ -123,12 +152,26 @@ ## You will also have to compile with `ssl` defined like so: ## `nim c -d:ssl ...`. ## -## Certificate validation is NOT performed by default. -## This will change in the future. +## Certificate validation is performed by default. ## ## A set of directories and files from the `ssl_certs `_ ## module are scanned to locate CA certificates. ## +## Example of setting SSL verification parameters in a new client: +## +## ```Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## ``` +## +## There are three options for verify mode: +## +## * ``CVerifyNone``: certificates are not verified; +## * ``CVerifyPeer``: certificates are verified; +## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional +## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to +## locate certificates +## ## See `newContext `_ to tweak or disable certificate validation. ## ## Timeouts @@ -148,10 +191,11 @@ ## ## Here is how to set a timeout when creating an `HttpClient` instance: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(timeout = 42) +## let client = newHttpClient(timeout = 42) +## ``` ## ## Proxy ## ===== @@ -162,28 +206,39 @@ ## ## Some examples on how to configure a Proxy for `HttpClient`: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## let myProxy = newProxy("http://myproxy.network") -## let client = newHttpClient(proxy = myProxy) +## let myProxy = newProxy("http://myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Use proxies with basic authentication: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Get Proxy URL from environment variables: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## var url = "" -## try: -## if existsEnv("http_proxy"): -## url = getEnv("http_proxy") -## elif existsEnv("https_proxy"): -## url = getEnv("https_proxy") -## except ValueError: -## echo "Unable to parse proxy from environment variables." +## var url = "" +## try: +## if existsEnv("http_proxy"): +## url = getEnv("http_proxy") +## elif existsEnv("https_proxy"): +## url = getEnv("https_proxy") +## except ValueError: +## echo "Unable to parse proxy from environment variables." ## -## let myProxy = newProxy(url = url) -## let client = newHttpClient(proxy = myProxy) +## let myProxy = newProxy(url = url) +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Redirects ## ========= @@ -194,10 +249,11 @@ ## ## Here you can see an example about how to set the `maxRedirects` of `HttpClient`: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(maxRedirects = 0) +## let client = newHttpClient(maxRedirects = 0) +## ``` ## import std/private/since @@ -208,6 +264,9 @@ import std/[ asyncnet, asyncdispatch, asyncfile, nativesockets, ] +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + export httpcore except parseHeader # TODO: The `except` doesn't work type @@ -245,9 +304,9 @@ proc contentLength*(response: Response | AsyncResponse): int = ## This is effectively the value of the "Content-Length" header. ## ## A `ValueError` exception will be raised if the value is not an integer. - var contentLengthHeader = response.headers.getOrDefault("Content-Length") + ## If the Content-Length header is not set in the response, ContentLength is set to the value -1. + var contentLengthHeader = response.headers.getOrDefault("Content-Length", HttpHeaderValues(@["-1"])) result = contentLengthHeader.parseInt() - doAssert(result >= 0 and result <= high(int32)) proc lastModified*(response: Response | AsyncResponse): DateTime = ## Retrieves the specified response's last modified time. @@ -300,7 +359,7 @@ type ## and `postContent` proc, ## when the server returns an error -const defUserAgent* = "Nim httpclient/" & NimVersion +const defUserAgent* = "Nim-httpclient/" & NimVersion proc httpError(msg: string) = var e: ref ProtocolError @@ -383,8 +442,9 @@ proc add*(p: MultipartData, xs: MultipartEntries): MultipartData ## Add a list of multipart entries to the multipart data `p`. All values are ## added without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data.add({"action": "login", "format": "json"}) + ## ``` for name, content in xs.items: p.add(name, content) result = p @@ -393,8 +453,9 @@ proc newMultipartData*(xs: MultipartEntries): MultipartData = ## Create a new multipart data object and fill it with the entries `xs` ## directly. ## - ## .. code-block:: Nim + ## ```Nim ## var data = newMultipartData({"action": "login", "format": "json"}) + ## ``` result = MultipartData() for entry in xs: result.add(entry.name, entry.content) @@ -409,8 +470,9 @@ proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]], ## Raises an `IOError` if the file cannot be opened or reading fails. To ## manually specify file content, filename and MIME type, use `[]=` instead. ## - ## .. code-block:: Nim + ## ```Nim ## data.addFiles({"uploaded_file": "public/test.html"}) + ## ``` for name, file in xs.items: var contentType: string let (_, fName, ext) = splitFile(file) @@ -424,8 +486,9 @@ proc `[]=`*(p: MultipartData, name, content: string) {.inline.} = ## Add a multipart entry to the multipart data `p`. The value is added ## without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data["username"] = "NimUser" + ## ``` p.add(name, content) proc `[]=`*(p: MultipartData, name: string, @@ -433,9 +496,10 @@ proc `[]=`*(p: MultipartData, name: string, ## Add a file to the multipart data `p`, specifying filename, contentType ## and content manually. ## - ## .. code-block:: Nim + ## ```Nim ## data["uploaded_file"] = ("test.html", "text/html", ## "

    test

    ") + ## ``` p.add(name, file.content, file.name, file.contentType, useStream = false) proc getBoundary(p: MultipartData): string = @@ -510,7 +574,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade # Proxy auth header. if not proxy.isNil and proxy.auth != "": let auth = base64.encode(proxy.auth) - add(result, "Proxy-Authorization: basic " & auth & httpNewLine) + add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) for key, val in headers: add(result, key & ": " & val & httpNewLine) @@ -574,15 +638,11 @@ proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5, ## ## `headers` specifies the HTTP Headers. runnableExamples: - import std/[asyncdispatch, httpclient, strutils] + import std/strutils - proc asyncProc(): Future[string] {.async.} = - var client = newAsyncHttpClient() - return await client.getContent("http://example.com") - - let exampleHtml = waitFor asyncProc() + let exampleHtml = newHttpClient().getContent("http://example.com") assert "Example Domain" in exampleHtml - assert not ("Pizza" in exampleHtml) + assert "Pizza" notin exampleHtml new result result.headers = headers @@ -616,6 +676,17 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, ## connections. ## ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/[asyncdispatch, strutils] + + proc asyncProc(): Future[string] {.async.} = + let client = newAsyncHttpClient() + result = await client.getContent("http://example.com") + + let exampleHtml = waitFor asyncProc() + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + new result result.headers = headers result.userAgent = userAgent @@ -635,15 +706,15 @@ proc close*(client: HttpClient | AsyncHttpClient) = client.connected = false proc getSocket*(client: HttpClient): Socket {.inline.} = - ## Get network socket, useful if you want to find out more details about the connection + ## Get network socket, useful if you want to find out more details about the connection. ## - ## this example shows info about local and remote endpoints + ## This example shows info about local and remote endpoints: ## - ## .. code-block:: Nim + ## ```Nim ## if client.connected: ## echo client.getSocket.getLocalAddr ## echo client.getSocket.getPeerAddr - ## + ## ``` return client.socket proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} = @@ -653,7 +724,7 @@ proc reportProgress(client: HttpClient | AsyncHttpClient, progress: BiggestInt) {.multisync.} = client.contentProgress += progress client.oneSecondProgress += progress - if (getMonoTime() - client.lastProgressReport).inSeconds > 1: + if (getMonoTime() - client.lastProgressReport).inSeconds >= 1: if not client.onProgressChanged.isNil: await client.onProgressChanged(client.contentTotal, client.contentProgress, @@ -751,7 +822,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, httpError("Got disconnected while trying to read body.") if recvLen != length: httpError("Received length doesn't match expected length. Wanted " & - $length & " got " & $recvLen) + $length & " got: " & $recvLen) else: # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO @@ -785,6 +856,7 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, var parsedStatus = false var linei = 0 var fullyRead = false + var lastHeaderName = "" var line = "" result.headers = newHttpHeaders() while true: @@ -819,16 +891,29 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, parsedStatus = true else: # Parse headers - var name = "" - var le = parseUntil(line, name, ':', linei) - if le <= 0: httpError("invalid headers") - inc(linei, le) - if line[linei] != ':': httpError("invalid headers") - inc(linei) # Skip : - - result.headers.add(name, line[linei .. ^1].strip()) - if result.headers.len > headerLimit: - httpError("too many headers") + # There's at least one char because empty lines are handled above (with client.close) + if line[0] in {' ', '\t'}: + # Check if it's a multiline header value, if so, append to the header we're currently parsing + # This works because a line with a header must start with the header name without any leading space + # See https://datatracker.ietf.org/doc/html/rfc7230, section 3.2 and 3.2.4 + # Multiline headers are deprecated in the spec, but it's better to parse them than crash + if lastHeaderName == "": + # Some extra unparsable lines in the HTTP output - we ignore them + discard + else: + result.headers.table[result.headers.toCaseInsensitive(lastHeaderName)][^1].add "\n" & line + else: + var name = "" + var le = parseUntil(line, name, ':', linei) + if le <= 0: httpError("Invalid headers - received empty header name") + if line.len == le: httpError("Invalid headers - no colon after header name") + inc(linei, le) # Skip the parsed header name + inc(linei) # Skip : + # If we want to be HTTP spec compliant later, error on linei == line.len (for empty header value) + lastHeaderName = name # Remember the header name for the possible multi-line header + result.headers.add(name, line[linei .. ^1].strip()) + if result.headers.len > headerLimit: + httpError("too many headers") if not fullyRead: httpError("Connection was closed before full request has been made") @@ -849,6 +934,9 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, client.parseBodyFut.addCallback do(): if client.parseBodyFut.failed: client.bodyStream.fail(client.parseBodyFut.error) + else: + when client is AsyncHttpClient: + result.bodyStream.complete() proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = @@ -955,19 +1043,22 @@ proc format(client: HttpClient | AsyncHttpClient, if entry.isFile: length += entry.fileSize + httpNewLine.len - result.add "--" & bound & "--" + result.add "--" & bound & "--" & httpNewLine for s in result: length += s.len client.headers["Content-Length"] = $length proc override(fallback, override: HttpHeaders): HttpHeaders = # Right-biased map union for `HttpHeaders` - if override.isNil: - return fallback result = newHttpHeaders() # Copy by value result.table[] = fallback.table[] + + if override.isNil: + # Return the copy of fallback so it does not get modified + return result + for k, vs in override.table: result[k] = vs @@ -987,12 +1078,16 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, await newConnection(client, url) - let newHeaders = client.headers.override(headers) + var newHeaders: HttpHeaders var data: seq[string] if multipart != nil and multipart.content.len > 0: + # `format` modifies `client.headers`, see + # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 data = await client.format(multipart) + newHeaders = client.headers.override(headers) else: + newHeaders = client.headers.override(headers) # Only change headers if they have not been specified already if not newHeaders.hasKey("Content-Length"): if body.len != 0: @@ -1137,7 +1232,7 @@ proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync ## A `HttpRequestError` will be raised if the server responds with a ## client error (status code 4xx) or a server error (status code 5xx). if resp.code.is4xx or resp.code.is5xx: - raise newException(HttpRequestError, resp.status) + raise newException(HttpRequestError, resp.status.move) else: return await resp.bodyStream.readAll() diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index fdd5926f72..45365c1851 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -12,7 +12,7 @@ ## ## Unstable API. import std/private/since -import tables, strutils, parseutils +import std/[tables, strutils, parseutils] type HttpHeaders* = ref object @@ -126,23 +126,20 @@ func toTitleCase(s: string): string = result[i] = if upper: toUpperAscii(s[i]) else: toLowerAscii(s[i]) upper = s[i] == '-' -func toCaseInsensitive(headers: HttpHeaders, s: string): string {.inline.} = +func toCaseInsensitive*(headers: HttpHeaders, s: string): string {.inline.} = + ## For internal usage only. Do not use. return if headers.isTitleCase: toTitleCase(s) else: toLowerAscii(s) func newHttpHeaders*(titleCase=false): HttpHeaders = ## Returns a new `HttpHeaders` object. if `titleCase` is set to true, ## headers are passed to the server in title case (e.g. "Content-Length") - new result - result.table = newTable[string, seq[string]]() - result.isTitleCase = titleCase + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) func newHttpHeaders*(keyValuePairs: openArray[tuple[key: string, val: string]], titleCase=false): HttpHeaders = ## Returns a new `HttpHeaders` object from an array. if `titleCase` is set to true, ## headers are passed to the server in title case (e.g. "Content-Length") - new result - result.table = newTable[string, seq[string]]() - result.isTitleCase = titleCase + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) for pair in keyValuePairs: let key = result.toCaseInsensitive(pair.key) @@ -237,10 +234,9 @@ func parseList(line: string, list: var seq[string], start: int): int = while start+i < line.len and line[start + i] notin {'\c', '\l'}: i += line.skipWhitespace(start + i) i += line.parseUntil(current, {'\c', '\l', ','}, start + i) - list.add(current) + list.add(move current) # implicit current.setLen(0) if start+i < line.len and line[start + i] == ',': i.inc # Skip , - current.setLen(0) func parseHeader*(line: string): tuple[key: string, value: seq[string]] = ## Parses a single raw header HTTP line into key value pairs. @@ -347,35 +343,25 @@ func `$`*(code: HttpCode): string = func `==`*(a, b: HttpCode): bool {.borrow.} -proc `==`*(rawCode: string, code: HttpCode): bool - {.deprecated: "Deprecated since v1.2; use rawCode == $code instead".} = - ## Compare the string form of the status code with a HttpCode - ## - ## **Note**: According to HTTP/1.1 specification, the reason phrase is - ## optional and should be ignored by the client, making this - ## proc only suitable for comparing the `HttpCode` against the - ## string form of itself. - return cmpIgnoreCase(rawCode, $code) == 0 - func is1xx*(code: HttpCode): bool {.inline, since: (1, 5).} = ## Determines whether `code` is a 1xx HTTP status code. runnableExamples: doAssert is1xx(HttpCode(103)) - code.int in {100 .. 199} + code.int in 100 .. 199 func is2xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 2xx HTTP status code. - code.int in {200 .. 299} + code.int in 200 .. 299 func is3xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 3xx HTTP status code. - code.int in {300 .. 399} + code.int in 300 .. 399 func is4xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 4xx HTTP status code. - code.int in {400 .. 499} + code.int in 400 .. 499 func is5xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 5xx HTTP status code. - code.int in {500 .. 599} + code.int in 500 .. 599 diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim deleted file mode 100644 index d0c92d5661..0000000000 --- a/lib/pure/includes/osenv.nim +++ /dev/null @@ -1,267 +0,0 @@ -# Include file that implements 'getEnv' and friends. Do not import it! - -when not declared(os) and not declared(ospaths): - {.error: "This is an include file for os.nim!".} - -when defined(nodejs): - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - var ret = default.cstring - let key2 = key.cstring - {.emit: "const value = process.env[`key2`];".} - {.emit: "if (value !== undefined) { `ret` = value };".} - result = $ret - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - var key2 = key.cstring - var ret: bool - {.emit: "`ret` = `key2` in process.env;".} - result = ret - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - var val2 = val.cstring - {.emit: "process.env[`key2`] = `val2`;".} - - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - {.emit: "delete process.env[`key2`];".} - - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - var num: int - var keys: RootObj - {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} - for i in 0..".} - proc c_putenv(env: cstring): cint {. - importc: "putenv", header: "".} - proc c_unsetenv(env: cstring): cint {. - importc: "unsetenv", header: "".} - - # Environment handling cannot be put into RTL, because the `envPairs` - # iterator depends on `environment`. - - var - envComputed {.threadvar.}: bool - environment {.threadvar.}: seq[string] - - when defined(nimV2): - proc unpairedEnvAllocs*(): int = - result = environment.len - if result > 0: inc result - - when defined(windows) and not defined(nimscript): - # because we support Windows GUI applications, things get really - # messy here... - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "".} - else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importc: "wcschr", header: "".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {. - importc: "strchr", header: "".} - - proc getEnvVarsC() = - if not envComputed: - environment = @[] - when useWinUnicode: - var - env = getEnvironmentStringsW() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[WideCString](cast[ByteAddress](eend)+2) - if eend[1].int == 0: break - discard freeEnvironmentStringsW(env) - else: - var - env = getEnvironmentStringsA() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[cstring](cast[ByteAddress](eend)+1) - if eend[1] == '\0': break - discard freeEnvironmentStringsA(env) - envComputed = true - - else: - const - useNSGetEnviron = (defined(macosx) and not defined(ios) and not defined(emscripten)) or defined(nimscript) - - when useNSGetEnviron: - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # , can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {. - importc: "_NSGetEnviron", header: "".} - elif defined(haiku): - var gEnv {.importc: "environ", header: "".}: cstringArray - else: - var gEnv {.importc: "environ".}: cstringArray - - proc getEnvVarsC() = - # retrieves the variables of char** env of C's main proc - if not envComputed: - environment = @[] - when useNSGetEnviron: - var gEnv = NSGetEnviron()[] - var i = 0 - while gEnv[i] != nil: - add environment, $gEnv[i] - inc(i) - envComputed = true - - proc findEnvVar(key: string): int = - getEnvVarsC() - var temp = key & '=' - for i in 0..high(environment): - when defined(windows): - if skipIgnoreCase(environment[i], temp) == len(temp): return i - else: - if startsWith(environment[i], temp): return i - return -1 - - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - ## Returns the value of the `environment variable`:idx: named `key`. - ## - ## If the variable does not exist, `""` is returned. To distinguish - ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc <#existsEnv,string>`_. - ## - ## See also: - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert getEnv("unknownEnv") == "" - assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - - when nimvm: - discard "built into the compiler" - else: - var i = findEnvVar(key) - if i >= 0: - return substr(environment[i], find(environment[i], '=')+1) - else: - var env = c_getenv(key) - if env == nil: return default - result = $env - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - ## Checks whether the environment variable named `key` exists. - ## Returns true if it exists, false otherwise. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert not existsEnv("unknownEnv") - - when nimvm: - discard "built into the compiler" - else: - if c_getenv(key) != nil: return true - else: return findEnvVar(key) >= 0 - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `OSError` is raised. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - - # Note: by storing the string in the environment sequence, - # we guarantee that we don't free the memory before the program - # ends (this is needed for POSIX compliance). It is also needed so that - # the process itself may access its modified environment variables! - when nimvm: - discard "built into the compiler" - else: - var indx = findEnvVar(key) - if indx >= 0: - environment[indx] = key & '=' & val - else: - add environment, (key & '=' & val) - indx = high(environment) - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - var v = newWideCString(val) - if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError()) - else: - if c_putenv(environment[indx]) != 0'i32: - raiseOSError(osLastError()) - - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - ## Deletes the `environment variable`:idx: named `key`. - ## If an error occurs, `OSError` is raised. - ## - ## See also:ven - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - when nimvm: - discard "built into the compiler" - else: - var indx = findEnvVar(key) - if indx < 0: return # Do nothing if the env var is not already set - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - if setEnvironmentVariableW(k, nil) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, nil) == 0'i32: raiseOSError(osLastError()) - else: - if c_unsetenv(key) != 0'i32: - raiseOSError(osLastError()) - environment.delete(indx) - - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. - ## - ## In the first component of the tuple is the name of the current variable stored, - ## in the second its value. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - getEnvVarsC() - for i in 0..high(environment): - var p = find(environment[i], '=') - yield (substr(environment[i], 0, p-1), - substr(environment[i], p+1)) diff --git a/lib/pure/includes/unicode_ranges.nim b/lib/pure/includes/unicode_ranges.nim index 74c8c6399f..04ccfb7472 100644 --- a/lib/pure/includes/unicode_ranges.nim +++ b/lib/pure/includes/unicode_ranges.nim @@ -2,1988 +2,1979 @@ const toLowerRanges = [ - 0x00041, 0x0005A, 532, - 0x000C0, 0x000D6, 532, - 0x000D8, 0x000DE, 532, - 0x00189, 0x0018A, 705, - 0x001B1, 0x001B2, 717, - 0x00388, 0x0038A, 537, - 0x0038E, 0x0038F, 563, - 0x00391, 0x003A1, 532, - 0x003A3, 0x003AB, 532, - 0x003FD, 0x003FF, 370, - 0x00400, 0x0040F, 580, - 0x00410, 0x0042F, 532, - 0x00531, 0x00556, 548, - 0x010A0, 0x010C5, 7764, - 0x013A0, 0x013EF, 39364, - 0x013F0, 0x013F5, 508, - 0x01C90, 0x01CBA, -2508, - 0x01CBD, 0x01CBF, -2508, - 0x01F08, 0x01F0F, 492, - 0x01F18, 0x01F1D, 492, - 0x01F28, 0x01F2F, 492, - 0x01F38, 0x01F3F, 492, - 0x01F48, 0x01F4D, 492, - 0x01F68, 0x01F6F, 492, - 0x01F88, 0x01F8F, 492, - 0x01F98, 0x01F9F, 492, - 0x01FA8, 0x01FAF, 492, - 0x01FB8, 0x01FB9, 492, - 0x01FBA, 0x01FBB, 426, - 0x01FC8, 0x01FCB, 414, - 0x01FD8, 0x01FD9, 492, - 0x01FDA, 0x01FDB, 400, - 0x01FE8, 0x01FE9, 492, - 0x01FEA, 0x01FEB, 388, - 0x01FF8, 0x01FF9, 372, - 0x01FFA, 0x01FFB, 374, - 0x02C00, 0x02C2E, 548, - 0x02C7E, 0x02C7F, -10315, - 0x0FF21, 0x0FF3A, 532, - 0x10400, 0x10427, 540, - 0x104B0, 0x104D3, 540, - 0x10C80, 0x10CB2, 564, - 0x118A0, 0x118BF, 532, - 0x16E40, 0x16E5F, 532, - 0x1E900, 0x1E921, 534, + 0x00041'i32, 0x0005A'i32, 532, + 0x000C0'i32, 0x000D6'i32, 532, + 0x000D8'i32, 0x000DE'i32, 532, + 0x00189'i32, 0x0018A'i32, 705, + 0x001B1'i32, 0x001B2'i32, 717, + 0x00388'i32, 0x0038A'i32, 537, + 0x0038E'i32, 0x0038F'i32, 563, + 0x00391'i32, 0x003A1'i32, 532, + 0x003A3'i32, 0x003AB'i32, 532, + 0x003FD'i32, 0x003FF'i32, 370, + 0x00400'i32, 0x0040F'i32, 580, + 0x00410'i32, 0x0042F'i32, 532, + 0x00531'i32, 0x00556'i32, 548, + 0x010A0'i32, 0x010C5'i32, 7764, + 0x013A0'i32, 0x013EF'i32, 39364, + 0x013F0'i32, 0x013F5'i32, 508, + 0x01C90'i32, 0x01CBA'i32, -2508, + 0x01CBD'i32, 0x01CBF'i32, -2508, + 0x01F08'i32, 0x01F0F'i32, 492, + 0x01F18'i32, 0x01F1D'i32, 492, + 0x01F28'i32, 0x01F2F'i32, 492, + 0x01F38'i32, 0x01F3F'i32, 492, + 0x01F48'i32, 0x01F4D'i32, 492, + 0x01F68'i32, 0x01F6F'i32, 492, + 0x01F88'i32, 0x01F8F'i32, 492, + 0x01F98'i32, 0x01F9F'i32, 492, + 0x01FA8'i32, 0x01FAF'i32, 492, + 0x01FB8'i32, 0x01FB9'i32, 492, + 0x01FBA'i32, 0x01FBB'i32, 426, + 0x01FC8'i32, 0x01FCB'i32, 414, + 0x01FD8'i32, 0x01FD9'i32, 492, + 0x01FDA'i32, 0x01FDB'i32, 400, + 0x01FE8'i32, 0x01FE9'i32, 492, + 0x01FEA'i32, 0x01FEB'i32, 388, + 0x01FF8'i32, 0x01FF9'i32, 372, + 0x01FFA'i32, 0x01FFB'i32, 374, + 0x02C00'i32, 0x02C2E'i32, 548, + 0x02C7E'i32, 0x02C7F'i32, -10315, + 0x0FF21'i32, 0x0FF3A'i32, 532, + 0x10400'i32, 0x10427'i32, 540, + 0x104B0'i32, 0x104D3'i32, 540, + 0x10C80'i32, 0x10CB2'i32, 564, + 0x118A0'i32, 0x118BF'i32, 532, + 0x16E40'i32, 0x16E5F'i32, 532, + 0x1E900'i32, 0x1E921'i32, 534, ] toLowerSinglets = [ - 0x00100, 501, - 0x00102, 501, - 0x00104, 501, - 0x00106, 501, - 0x00108, 501, - 0x0010A, 501, - 0x0010C, 501, - 0x0010E, 501, - 0x00110, 501, - 0x00112, 501, - 0x00114, 501, - 0x00116, 501, - 0x00118, 501, - 0x0011A, 501, - 0x0011C, 501, - 0x0011E, 501, - 0x00120, 501, - 0x00122, 501, - 0x00124, 501, - 0x00126, 501, - 0x00128, 501, - 0x0012A, 501, - 0x0012C, 501, - 0x0012E, 501, - 0x00130, 301, - 0x00132, 501, - 0x00134, 501, - 0x00136, 501, - 0x00139, 501, - 0x0013B, 501, - 0x0013D, 501, - 0x0013F, 501, - 0x00141, 501, - 0x00143, 501, - 0x00145, 501, - 0x00147, 501, - 0x0014A, 501, - 0x0014C, 501, - 0x0014E, 501, - 0x00150, 501, - 0x00152, 501, - 0x00154, 501, - 0x00156, 501, - 0x00158, 501, - 0x0015A, 501, - 0x0015C, 501, - 0x0015E, 501, - 0x00160, 501, - 0x00162, 501, - 0x00164, 501, - 0x00166, 501, - 0x00168, 501, - 0x0016A, 501, - 0x0016C, 501, - 0x0016E, 501, - 0x00170, 501, - 0x00172, 501, - 0x00174, 501, - 0x00176, 501, - 0x00178, 379, - 0x00179, 501, - 0x0017B, 501, - 0x0017D, 501, - 0x00181, 710, - 0x00182, 501, - 0x00184, 501, - 0x00186, 706, - 0x00187, 501, - 0x0018B, 501, - 0x0018E, 579, - 0x0018F, 702, - 0x00190, 703, - 0x00191, 501, - 0x00193, 705, - 0x00194, 707, - 0x00196, 711, - 0x00197, 709, - 0x00198, 501, - 0x0019C, 711, - 0x0019D, 713, - 0x0019F, 714, - 0x001A0, 501, - 0x001A2, 501, - 0x001A4, 501, - 0x001A6, 718, - 0x001A7, 501, - 0x001A9, 718, - 0x001AC, 501, - 0x001AE, 718, - 0x001AF, 501, - 0x001B3, 501, - 0x001B5, 501, - 0x001B7, 719, - 0x001B8, 501, - 0x001BC, 501, - 0x001C4, 502, - 0x001C5, 501, - 0x001C7, 502, - 0x001C8, 501, - 0x001CA, 502, - 0x001CB, 501, - 0x001CD, 501, - 0x001CF, 501, - 0x001D1, 501, - 0x001D3, 501, - 0x001D5, 501, - 0x001D7, 501, - 0x001D9, 501, - 0x001DB, 501, - 0x001DE, 501, - 0x001E0, 501, - 0x001E2, 501, - 0x001E4, 501, - 0x001E6, 501, - 0x001E8, 501, - 0x001EA, 501, - 0x001EC, 501, - 0x001EE, 501, - 0x001F1, 502, - 0x001F2, 501, - 0x001F4, 501, - 0x001F6, 403, - 0x001F7, 444, - 0x001F8, 501, - 0x001FA, 501, - 0x001FC, 501, - 0x001FE, 501, - 0x00200, 501, - 0x00202, 501, - 0x00204, 501, - 0x00206, 501, - 0x00208, 501, - 0x0020A, 501, - 0x0020C, 501, - 0x0020E, 501, - 0x00210, 501, - 0x00212, 501, - 0x00214, 501, - 0x00216, 501, - 0x00218, 501, - 0x0021A, 501, - 0x0021C, 501, - 0x0021E, 501, - 0x00220, 370, - 0x00222, 501, - 0x00224, 501, - 0x00226, 501, - 0x00228, 501, - 0x0022A, 501, - 0x0022C, 501, - 0x0022E, 501, - 0x00230, 501, - 0x00232, 501, - 0x0023A, 11295, - 0x0023B, 501, - 0x0023D, 337, - 0x0023E, 11292, - 0x00241, 501, - 0x00243, 305, - 0x00244, 569, - 0x00245, 571, - 0x00246, 501, - 0x00248, 501, - 0x0024A, 501, - 0x0024C, 501, - 0x0024E, 501, - 0x00370, 501, - 0x00372, 501, - 0x00376, 501, - 0x0037F, 616, - 0x00386, 538, - 0x0038C, 564, - 0x003CF, 508, - 0x003D8, 501, - 0x003DA, 501, - 0x003DC, 501, - 0x003DE, 501, - 0x003E0, 501, - 0x003E2, 501, - 0x003E4, 501, - 0x003E6, 501, - 0x003E8, 501, - 0x003EA, 501, - 0x003EC, 501, - 0x003EE, 501, - 0x003F4, 440, - 0x003F7, 501, - 0x003F9, 493, - 0x003FA, 501, - 0x00460, 501, - 0x00462, 501, - 0x00464, 501, - 0x00466, 501, - 0x00468, 501, - 0x0046A, 501, - 0x0046C, 501, - 0x0046E, 501, - 0x00470, 501, - 0x00472, 501, - 0x00474, 501, - 0x00476, 501, - 0x00478, 501, - 0x0047A, 501, - 0x0047C, 501, - 0x0047E, 501, - 0x00480, 501, - 0x0048A, 501, - 0x0048C, 501, - 0x0048E, 501, - 0x00490, 501, - 0x00492, 501, - 0x00494, 501, - 0x00496, 501, - 0x00498, 501, - 0x0049A, 501, - 0x0049C, 501, - 0x0049E, 501, - 0x004A0, 501, - 0x004A2, 501, - 0x004A4, 501, - 0x004A6, 501, - 0x004A8, 501, - 0x004AA, 501, - 0x004AC, 501, - 0x004AE, 501, - 0x004B0, 501, - 0x004B2, 501, - 0x004B4, 501, - 0x004B6, 501, - 0x004B8, 501, - 0x004BA, 501, - 0x004BC, 501, - 0x004BE, 501, - 0x004C0, 515, - 0x004C1, 501, - 0x004C3, 501, - 0x004C5, 501, - 0x004C7, 501, - 0x004C9, 501, - 0x004CB, 501, - 0x004CD, 501, - 0x004D0, 501, - 0x004D2, 501, - 0x004D4, 501, - 0x004D6, 501, - 0x004D8, 501, - 0x004DA, 501, - 0x004DC, 501, - 0x004DE, 501, - 0x004E0, 501, - 0x004E2, 501, - 0x004E4, 501, - 0x004E6, 501, - 0x004E8, 501, - 0x004EA, 501, - 0x004EC, 501, - 0x004EE, 501, - 0x004F0, 501, - 0x004F2, 501, - 0x004F4, 501, - 0x004F6, 501, - 0x004F8, 501, - 0x004FA, 501, - 0x004FC, 501, - 0x004FE, 501, - 0x00500, 501, - 0x00502, 501, - 0x00504, 501, - 0x00506, 501, - 0x00508, 501, - 0x0050A, 501, - 0x0050C, 501, - 0x0050E, 501, - 0x00510, 501, - 0x00512, 501, - 0x00514, 501, - 0x00516, 501, - 0x00518, 501, - 0x0051A, 501, - 0x0051C, 501, - 0x0051E, 501, - 0x00520, 501, - 0x00522, 501, - 0x00524, 501, - 0x00526, 501, - 0x00528, 501, - 0x0052A, 501, - 0x0052C, 501, - 0x0052E, 501, - 0x010C7, 7764, - 0x010CD, 7764, - 0x01E00, 501, - 0x01E02, 501, - 0x01E04, 501, - 0x01E06, 501, - 0x01E08, 501, - 0x01E0A, 501, - 0x01E0C, 501, - 0x01E0E, 501, - 0x01E10, 501, - 0x01E12, 501, - 0x01E14, 501, - 0x01E16, 501, - 0x01E18, 501, - 0x01E1A, 501, - 0x01E1C, 501, - 0x01E1E, 501, - 0x01E20, 501, - 0x01E22, 501, - 0x01E24, 501, - 0x01E26, 501, - 0x01E28, 501, - 0x01E2A, 501, - 0x01E2C, 501, - 0x01E2E, 501, - 0x01E30, 501, - 0x01E32, 501, - 0x01E34, 501, - 0x01E36, 501, - 0x01E38, 501, - 0x01E3A, 501, - 0x01E3C, 501, - 0x01E3E, 501, - 0x01E40, 501, - 0x01E42, 501, - 0x01E44, 501, - 0x01E46, 501, - 0x01E48, 501, - 0x01E4A, 501, - 0x01E4C, 501, - 0x01E4E, 501, - 0x01E50, 501, - 0x01E52, 501, - 0x01E54, 501, - 0x01E56, 501, - 0x01E58, 501, - 0x01E5A, 501, - 0x01E5C, 501, - 0x01E5E, 501, - 0x01E60, 501, - 0x01E62, 501, - 0x01E64, 501, - 0x01E66, 501, - 0x01E68, 501, - 0x01E6A, 501, - 0x01E6C, 501, - 0x01E6E, 501, - 0x01E70, 501, - 0x01E72, 501, - 0x01E74, 501, - 0x01E76, 501, - 0x01E78, 501, - 0x01E7A, 501, - 0x01E7C, 501, - 0x01E7E, 501, - 0x01E80, 501, - 0x01E82, 501, - 0x01E84, 501, - 0x01E86, 501, - 0x01E88, 501, - 0x01E8A, 501, - 0x01E8C, 501, - 0x01E8E, 501, - 0x01E90, 501, - 0x01E92, 501, - 0x01E94, 501, - 0x01E9E, -7115, - 0x01EA0, 501, - 0x01EA2, 501, - 0x01EA4, 501, - 0x01EA6, 501, - 0x01EA8, 501, - 0x01EAA, 501, - 0x01EAC, 501, - 0x01EAE, 501, - 0x01EB0, 501, - 0x01EB2, 501, - 0x01EB4, 501, - 0x01EB6, 501, - 0x01EB8, 501, - 0x01EBA, 501, - 0x01EBC, 501, - 0x01EBE, 501, - 0x01EC0, 501, - 0x01EC2, 501, - 0x01EC4, 501, - 0x01EC6, 501, - 0x01EC8, 501, - 0x01ECA, 501, - 0x01ECC, 501, - 0x01ECE, 501, - 0x01ED0, 501, - 0x01ED2, 501, - 0x01ED4, 501, - 0x01ED6, 501, - 0x01ED8, 501, - 0x01EDA, 501, - 0x01EDC, 501, - 0x01EDE, 501, - 0x01EE0, 501, - 0x01EE2, 501, - 0x01EE4, 501, - 0x01EE6, 501, - 0x01EE8, 501, - 0x01EEA, 501, - 0x01EEC, 501, - 0x01EEE, 501, - 0x01EF0, 501, - 0x01EF2, 501, - 0x01EF4, 501, - 0x01EF6, 501, - 0x01EF8, 501, - 0x01EFA, 501, - 0x01EFC, 501, - 0x01EFE, 501, - 0x01F59, 492, - 0x01F5B, 492, - 0x01F5D, 492, - 0x01F5F, 492, - 0x01FBC, 491, - 0x01FCC, 491, - 0x01FEC, 493, - 0x01FFC, 491, - 0x02126, -7017, - 0x0212A, -7883, - 0x0212B, -7762, - 0x02132, 528, - 0x02183, 501, - 0x02C60, 501, - 0x02C62, -10243, - 0x02C63, -3314, - 0x02C64, -10227, - 0x02C67, 501, - 0x02C69, 501, - 0x02C6B, 501, - 0x02C6D, -10280, - 0x02C6E, -10249, - 0x02C6F, -10283, - 0x02C70, -10282, - 0x02C72, 501, - 0x02C75, 501, - 0x02C80, 501, - 0x02C82, 501, - 0x02C84, 501, - 0x02C86, 501, - 0x02C88, 501, - 0x02C8A, 501, - 0x02C8C, 501, - 0x02C8E, 501, - 0x02C90, 501, - 0x02C92, 501, - 0x02C94, 501, - 0x02C96, 501, - 0x02C98, 501, - 0x02C9A, 501, - 0x02C9C, 501, - 0x02C9E, 501, - 0x02CA0, 501, - 0x02CA2, 501, - 0x02CA4, 501, - 0x02CA6, 501, - 0x02CA8, 501, - 0x02CAA, 501, - 0x02CAC, 501, - 0x02CAE, 501, - 0x02CB0, 501, - 0x02CB2, 501, - 0x02CB4, 501, - 0x02CB6, 501, - 0x02CB8, 501, - 0x02CBA, 501, - 0x02CBC, 501, - 0x02CBE, 501, - 0x02CC0, 501, - 0x02CC2, 501, - 0x02CC4, 501, - 0x02CC6, 501, - 0x02CC8, 501, - 0x02CCA, 501, - 0x02CCC, 501, - 0x02CCE, 501, - 0x02CD0, 501, - 0x02CD2, 501, - 0x02CD4, 501, - 0x02CD6, 501, - 0x02CD8, 501, - 0x02CDA, 501, - 0x02CDC, 501, - 0x02CDE, 501, - 0x02CE0, 501, - 0x02CE2, 501, - 0x02CEB, 501, - 0x02CED, 501, - 0x02CF2, 501, - 0x0A640, 501, - 0x0A642, 501, - 0x0A644, 501, - 0x0A646, 501, - 0x0A648, 501, - 0x0A64A, 501, - 0x0A64C, 501, - 0x0A64E, 501, - 0x0A650, 501, - 0x0A652, 501, - 0x0A654, 501, - 0x0A656, 501, - 0x0A658, 501, - 0x0A65A, 501, - 0x0A65C, 501, - 0x0A65E, 501, - 0x0A660, 501, - 0x0A662, 501, - 0x0A664, 501, - 0x0A666, 501, - 0x0A668, 501, - 0x0A66A, 501, - 0x0A66C, 501, - 0x0A680, 501, - 0x0A682, 501, - 0x0A684, 501, - 0x0A686, 501, - 0x0A688, 501, - 0x0A68A, 501, - 0x0A68C, 501, - 0x0A68E, 501, - 0x0A690, 501, - 0x0A692, 501, - 0x0A694, 501, - 0x0A696, 501, - 0x0A698, 501, - 0x0A69A, 501, - 0x0A722, 501, - 0x0A724, 501, - 0x0A726, 501, - 0x0A728, 501, - 0x0A72A, 501, - 0x0A72C, 501, - 0x0A72E, 501, - 0x0A732, 501, - 0x0A734, 501, - 0x0A736, 501, - 0x0A738, 501, - 0x0A73A, 501, - 0x0A73C, 501, - 0x0A73E, 501, - 0x0A740, 501, - 0x0A742, 501, - 0x0A744, 501, - 0x0A746, 501, - 0x0A748, 501, - 0x0A74A, 501, - 0x0A74C, 501, - 0x0A74E, 501, - 0x0A750, 501, - 0x0A752, 501, - 0x0A754, 501, - 0x0A756, 501, - 0x0A758, 501, - 0x0A75A, 501, - 0x0A75C, 501, - 0x0A75E, 501, - 0x0A760, 501, - 0x0A762, 501, - 0x0A764, 501, - 0x0A766, 501, - 0x0A768, 501, - 0x0A76A, 501, - 0x0A76C, 501, - 0x0A76E, 501, - 0x0A779, 501, - 0x0A77B, 501, - 0x0A77D, -34832, - 0x0A77E, 501, - 0x0A780, 501, - 0x0A782, 501, - 0x0A784, 501, - 0x0A786, 501, - 0x0A78B, 501, - 0x0A78D, -41780, - 0x0A790, 501, - 0x0A792, 501, - 0x0A796, 501, - 0x0A798, 501, - 0x0A79A, 501, - 0x0A79C, 501, - 0x0A79E, 501, - 0x0A7A0, 501, - 0x0A7A2, 501, - 0x0A7A4, 501, - 0x0A7A6, 501, - 0x0A7A8, 501, - 0x0A7AA, -41808, - 0x0A7AB, -41819, - 0x0A7AC, -41815, - 0x0A7AD, -41805, - 0x0A7AE, -41808, - 0x0A7B0, -41758, - 0x0A7B1, -41782, - 0x0A7B2, -41761, - 0x0A7B3, 1428, - 0x0A7B4, 501, - 0x0A7B6, 501, - 0x0A7B8, 501, - 0x0A7BA, 501, - 0x0A7BC, 501, - 0x0A7BE, 501, - 0x0A7C2, 501, - 0x0A7C4, 452, - 0x0A7C5, -41807, - 0x0A7C6, -34884, + 0x00100'i32, 501, + 0x00102'i32, 501, + 0x00104'i32, 501, + 0x00106'i32, 501, + 0x00108'i32, 501, + 0x0010A'i32, 501, + 0x0010C'i32, 501, + 0x0010E'i32, 501, + 0x00110'i32, 501, + 0x00112'i32, 501, + 0x00114'i32, 501, + 0x00116'i32, 501, + 0x00118'i32, 501, + 0x0011A'i32, 501, + 0x0011C'i32, 501, + 0x0011E'i32, 501, + 0x00120'i32, 501, + 0x00122'i32, 501, + 0x00124'i32, 501, + 0x00126'i32, 501, + 0x00128'i32, 501, + 0x0012A'i32, 501, + 0x0012C'i32, 501, + 0x0012E'i32, 501, + 0x00130'i32, 301, + 0x00132'i32, 501, + 0x00134'i32, 501, + 0x00136'i32, 501, + 0x00139'i32, 501, + 0x0013B'i32, 501, + 0x0013D'i32, 501, + 0x0013F'i32, 501, + 0x00141'i32, 501, + 0x00143'i32, 501, + 0x00145'i32, 501, + 0x00147'i32, 501, + 0x0014A'i32, 501, + 0x0014C'i32, 501, + 0x0014E'i32, 501, + 0x00150'i32, 501, + 0x00152'i32, 501, + 0x00154'i32, 501, + 0x00156'i32, 501, + 0x00158'i32, 501, + 0x0015A'i32, 501, + 0x0015C'i32, 501, + 0x0015E'i32, 501, + 0x00160'i32, 501, + 0x00162'i32, 501, + 0x00164'i32, 501, + 0x00166'i32, 501, + 0x00168'i32, 501, + 0x0016A'i32, 501, + 0x0016C'i32, 501, + 0x0016E'i32, 501, + 0x00170'i32, 501, + 0x00172'i32, 501, + 0x00174'i32, 501, + 0x00176'i32, 501, + 0x00178'i32, 379, + 0x00179'i32, 501, + 0x0017B'i32, 501, + 0x0017D'i32, 501, + 0x00181'i32, 710, + 0x00182'i32, 501, + 0x00184'i32, 501, + 0x00186'i32, 706, + 0x00187'i32, 501, + 0x0018B'i32, 501, + 0x0018E'i32, 579, + 0x0018F'i32, 702, + 0x00190'i32, 703, + 0x00191'i32, 501, + 0x00193'i32, 705, + 0x00194'i32, 707, + 0x00196'i32, 711, + 0x00197'i32, 709, + 0x00198'i32, 501, + 0x0019C'i32, 711, + 0x0019D'i32, 713, + 0x0019F'i32, 714, + 0x001A0'i32, 501, + 0x001A2'i32, 501, + 0x001A4'i32, 501, + 0x001A6'i32, 718, + 0x001A7'i32, 501, + 0x001A9'i32, 718, + 0x001AC'i32, 501, + 0x001AE'i32, 718, + 0x001AF'i32, 501, + 0x001B3'i32, 501, + 0x001B5'i32, 501, + 0x001B7'i32, 719, + 0x001B8'i32, 501, + 0x001BC'i32, 501, + 0x001C4'i32, 502, + 0x001C5'i32, 501, + 0x001C7'i32, 502, + 0x001C8'i32, 501, + 0x001CA'i32, 502, + 0x001CB'i32, 501, + 0x001CD'i32, 501, + 0x001CF'i32, 501, + 0x001D1'i32, 501, + 0x001D3'i32, 501, + 0x001D5'i32, 501, + 0x001D7'i32, 501, + 0x001D9'i32, 501, + 0x001DB'i32, 501, + 0x001DE'i32, 501, + 0x001E0'i32, 501, + 0x001E2'i32, 501, + 0x001E4'i32, 501, + 0x001E6'i32, 501, + 0x001E8'i32, 501, + 0x001EA'i32, 501, + 0x001EC'i32, 501, + 0x001EE'i32, 501, + 0x001F1'i32, 502, + 0x001F2'i32, 501, + 0x001F4'i32, 501, + 0x001F6'i32, 403, + 0x001F7'i32, 444, + 0x001F8'i32, 501, + 0x001FA'i32, 501, + 0x001FC'i32, 501, + 0x001FE'i32, 501, + 0x00200'i32, 501, + 0x00202'i32, 501, + 0x00204'i32, 501, + 0x00206'i32, 501, + 0x00208'i32, 501, + 0x0020A'i32, 501, + 0x0020C'i32, 501, + 0x0020E'i32, 501, + 0x00210'i32, 501, + 0x00212'i32, 501, + 0x00214'i32, 501, + 0x00216'i32, 501, + 0x00218'i32, 501, + 0x0021A'i32, 501, + 0x0021C'i32, 501, + 0x0021E'i32, 501, + 0x00220'i32, 370, + 0x00222'i32, 501, + 0x00224'i32, 501, + 0x00226'i32, 501, + 0x00228'i32, 501, + 0x0022A'i32, 501, + 0x0022C'i32, 501, + 0x0022E'i32, 501, + 0x00230'i32, 501, + 0x00232'i32, 501, + 0x0023A'i32, 11295, + 0x0023B'i32, 501, + 0x0023D'i32, 337, + 0x0023E'i32, 11292, + 0x00241'i32, 501, + 0x00243'i32, 305, + 0x00244'i32, 569, + 0x00245'i32, 571, + 0x00246'i32, 501, + 0x00248'i32, 501, + 0x0024A'i32, 501, + 0x0024C'i32, 501, + 0x0024E'i32, 501, + 0x00370'i32, 501, + 0x00372'i32, 501, + 0x00376'i32, 501, + 0x0037F'i32, 616, + 0x00386'i32, 538, + 0x0038C'i32, 564, + 0x003CF'i32, 508, + 0x003D8'i32, 501, + 0x003DA'i32, 501, + 0x003DC'i32, 501, + 0x003DE'i32, 501, + 0x003E0'i32, 501, + 0x003E2'i32, 501, + 0x003E4'i32, 501, + 0x003E6'i32, 501, + 0x003E8'i32, 501, + 0x003EA'i32, 501, + 0x003EC'i32, 501, + 0x003EE'i32, 501, + 0x003F4'i32, 440, + 0x003F7'i32, 501, + 0x003F9'i32, 493, + 0x003FA'i32, 501, + 0x00460'i32, 501, + 0x00462'i32, 501, + 0x00464'i32, 501, + 0x00466'i32, 501, + 0x00468'i32, 501, + 0x0046A'i32, 501, + 0x0046C'i32, 501, + 0x0046E'i32, 501, + 0x00470'i32, 501, + 0x00472'i32, 501, + 0x00474'i32, 501, + 0x00476'i32, 501, + 0x00478'i32, 501, + 0x0047A'i32, 501, + 0x0047C'i32, 501, + 0x0047E'i32, 501, + 0x00480'i32, 501, + 0x0048A'i32, 501, + 0x0048C'i32, 501, + 0x0048E'i32, 501, + 0x00490'i32, 501, + 0x00492'i32, 501, + 0x00494'i32, 501, + 0x00496'i32, 501, + 0x00498'i32, 501, + 0x0049A'i32, 501, + 0x0049C'i32, 501, + 0x0049E'i32, 501, + 0x004A0'i32, 501, + 0x004A2'i32, 501, + 0x004A4'i32, 501, + 0x004A6'i32, 501, + 0x004A8'i32, 501, + 0x004AA'i32, 501, + 0x004AC'i32, 501, + 0x004AE'i32, 501, + 0x004B0'i32, 501, + 0x004B2'i32, 501, + 0x004B4'i32, 501, + 0x004B6'i32, 501, + 0x004B8'i32, 501, + 0x004BA'i32, 501, + 0x004BC'i32, 501, + 0x004BE'i32, 501, + 0x004C0'i32, 515, + 0x004C1'i32, 501, + 0x004C3'i32, 501, + 0x004C5'i32, 501, + 0x004C7'i32, 501, + 0x004C9'i32, 501, + 0x004CB'i32, 501, + 0x004CD'i32, 501, + 0x004D0'i32, 501, + 0x004D2'i32, 501, + 0x004D4'i32, 501, + 0x004D6'i32, 501, + 0x004D8'i32, 501, + 0x004DA'i32, 501, + 0x004DC'i32, 501, + 0x004DE'i32, 501, + 0x004E0'i32, 501, + 0x004E2'i32, 501, + 0x004E4'i32, 501, + 0x004E6'i32, 501, + 0x004E8'i32, 501, + 0x004EA'i32, 501, + 0x004EC'i32, 501, + 0x004EE'i32, 501, + 0x004F0'i32, 501, + 0x004F2'i32, 501, + 0x004F4'i32, 501, + 0x004F6'i32, 501, + 0x004F8'i32, 501, + 0x004FA'i32, 501, + 0x004FC'i32, 501, + 0x004FE'i32, 501, + 0x00500'i32, 501, + 0x00502'i32, 501, + 0x00504'i32, 501, + 0x00506'i32, 501, + 0x00508'i32, 501, + 0x0050A'i32, 501, + 0x0050C'i32, 501, + 0x0050E'i32, 501, + 0x00510'i32, 501, + 0x00512'i32, 501, + 0x00514'i32, 501, + 0x00516'i32, 501, + 0x00518'i32, 501, + 0x0051A'i32, 501, + 0x0051C'i32, 501, + 0x0051E'i32, 501, + 0x00520'i32, 501, + 0x00522'i32, 501, + 0x00524'i32, 501, + 0x00526'i32, 501, + 0x00528'i32, 501, + 0x0052A'i32, 501, + 0x0052C'i32, 501, + 0x0052E'i32, 501, + 0x010C7'i32, 7764, + 0x010CD'i32, 7764, + 0x01E00'i32, 501, + 0x01E02'i32, 501, + 0x01E04'i32, 501, + 0x01E06'i32, 501, + 0x01E08'i32, 501, + 0x01E0A'i32, 501, + 0x01E0C'i32, 501, + 0x01E0E'i32, 501, + 0x01E10'i32, 501, + 0x01E12'i32, 501, + 0x01E14'i32, 501, + 0x01E16'i32, 501, + 0x01E18'i32, 501, + 0x01E1A'i32, 501, + 0x01E1C'i32, 501, + 0x01E1E'i32, 501, + 0x01E20'i32, 501, + 0x01E22'i32, 501, + 0x01E24'i32, 501, + 0x01E26'i32, 501, + 0x01E28'i32, 501, + 0x01E2A'i32, 501, + 0x01E2C'i32, 501, + 0x01E2E'i32, 501, + 0x01E30'i32, 501, + 0x01E32'i32, 501, + 0x01E34'i32, 501, + 0x01E36'i32, 501, + 0x01E38'i32, 501, + 0x01E3A'i32, 501, + 0x01E3C'i32, 501, + 0x01E3E'i32, 501, + 0x01E40'i32, 501, + 0x01E42'i32, 501, + 0x01E44'i32, 501, + 0x01E46'i32, 501, + 0x01E48'i32, 501, + 0x01E4A'i32, 501, + 0x01E4C'i32, 501, + 0x01E4E'i32, 501, + 0x01E50'i32, 501, + 0x01E52'i32, 501, + 0x01E54'i32, 501, + 0x01E56'i32, 501, + 0x01E58'i32, 501, + 0x01E5A'i32, 501, + 0x01E5C'i32, 501, + 0x01E5E'i32, 501, + 0x01E60'i32, 501, + 0x01E62'i32, 501, + 0x01E64'i32, 501, + 0x01E66'i32, 501, + 0x01E68'i32, 501, + 0x01E6A'i32, 501, + 0x01E6C'i32, 501, + 0x01E6E'i32, 501, + 0x01E70'i32, 501, + 0x01E72'i32, 501, + 0x01E74'i32, 501, + 0x01E76'i32, 501, + 0x01E78'i32, 501, + 0x01E7A'i32, 501, + 0x01E7C'i32, 501, + 0x01E7E'i32, 501, + 0x01E80'i32, 501, + 0x01E82'i32, 501, + 0x01E84'i32, 501, + 0x01E86'i32, 501, + 0x01E88'i32, 501, + 0x01E8A'i32, 501, + 0x01E8C'i32, 501, + 0x01E8E'i32, 501, + 0x01E90'i32, 501, + 0x01E92'i32, 501, + 0x01E94'i32, 501, + 0x01E9E'i32, -7115, + 0x01EA0'i32, 501, + 0x01EA2'i32, 501, + 0x01EA4'i32, 501, + 0x01EA6'i32, 501, + 0x01EA8'i32, 501, + 0x01EAA'i32, 501, + 0x01EAC'i32, 501, + 0x01EAE'i32, 501, + 0x01EB0'i32, 501, + 0x01EB2'i32, 501, + 0x01EB4'i32, 501, + 0x01EB6'i32, 501, + 0x01EB8'i32, 501, + 0x01EBA'i32, 501, + 0x01EBC'i32, 501, + 0x01EBE'i32, 501, + 0x01EC0'i32, 501, + 0x01EC2'i32, 501, + 0x01EC4'i32, 501, + 0x01EC6'i32, 501, + 0x01EC8'i32, 501, + 0x01ECA'i32, 501, + 0x01ECC'i32, 501, + 0x01ECE'i32, 501, + 0x01ED0'i32, 501, + 0x01ED2'i32, 501, + 0x01ED4'i32, 501, + 0x01ED6'i32, 501, + 0x01ED8'i32, 501, + 0x01EDA'i32, 501, + 0x01EDC'i32, 501, + 0x01EDE'i32, 501, + 0x01EE0'i32, 501, + 0x01EE2'i32, 501, + 0x01EE4'i32, 501, + 0x01EE6'i32, 501, + 0x01EE8'i32, 501, + 0x01EEA'i32, 501, + 0x01EEC'i32, 501, + 0x01EEE'i32, 501, + 0x01EF0'i32, 501, + 0x01EF2'i32, 501, + 0x01EF4'i32, 501, + 0x01EF6'i32, 501, + 0x01EF8'i32, 501, + 0x01EFA'i32, 501, + 0x01EFC'i32, 501, + 0x01EFE'i32, 501, + 0x01F59'i32, 492, + 0x01F5B'i32, 492, + 0x01F5D'i32, 492, + 0x01F5F'i32, 492, + 0x01FBC'i32, 491, + 0x01FCC'i32, 491, + 0x01FEC'i32, 493, + 0x01FFC'i32, 491, + 0x02126'i32, -7017, + 0x0212A'i32, -7883, + 0x0212B'i32, -7762, + 0x02132'i32, 528, + 0x02183'i32, 501, + 0x02C60'i32, 501, + 0x02C62'i32, -10243, + 0x02C63'i32, -3314, + 0x02C64'i32, -10227, + 0x02C67'i32, 501, + 0x02C69'i32, 501, + 0x02C6B'i32, 501, + 0x02C6D'i32, -10280, + 0x02C6E'i32, -10249, + 0x02C6F'i32, -10283, + 0x02C70'i32, -10282, + 0x02C72'i32, 501, + 0x02C75'i32, 501, + 0x02C80'i32, 501, + 0x02C82'i32, 501, + 0x02C84'i32, 501, + 0x02C86'i32, 501, + 0x02C88'i32, 501, + 0x02C8A'i32, 501, + 0x02C8C'i32, 501, + 0x02C8E'i32, 501, + 0x02C90'i32, 501, + 0x02C92'i32, 501, + 0x02C94'i32, 501, + 0x02C96'i32, 501, + 0x02C98'i32, 501, + 0x02C9A'i32, 501, + 0x02C9C'i32, 501, + 0x02C9E'i32, 501, + 0x02CA0'i32, 501, + 0x02CA2'i32, 501, + 0x02CA4'i32, 501, + 0x02CA6'i32, 501, + 0x02CA8'i32, 501, + 0x02CAA'i32, 501, + 0x02CAC'i32, 501, + 0x02CAE'i32, 501, + 0x02CB0'i32, 501, + 0x02CB2'i32, 501, + 0x02CB4'i32, 501, + 0x02CB6'i32, 501, + 0x02CB8'i32, 501, + 0x02CBA'i32, 501, + 0x02CBC'i32, 501, + 0x02CBE'i32, 501, + 0x02CC0'i32, 501, + 0x02CC2'i32, 501, + 0x02CC4'i32, 501, + 0x02CC6'i32, 501, + 0x02CC8'i32, 501, + 0x02CCA'i32, 501, + 0x02CCC'i32, 501, + 0x02CCE'i32, 501, + 0x02CD0'i32, 501, + 0x02CD2'i32, 501, + 0x02CD4'i32, 501, + 0x02CD6'i32, 501, + 0x02CD8'i32, 501, + 0x02CDA'i32, 501, + 0x02CDC'i32, 501, + 0x02CDE'i32, 501, + 0x02CE0'i32, 501, + 0x02CE2'i32, 501, + 0x02CEB'i32, 501, + 0x02CED'i32, 501, + 0x02CF2'i32, 501, + 0x0A640'i32, 501, + 0x0A642'i32, 501, + 0x0A644'i32, 501, + 0x0A646'i32, 501, + 0x0A648'i32, 501, + 0x0A64A'i32, 501, + 0x0A64C'i32, 501, + 0x0A64E'i32, 501, + 0x0A650'i32, 501, + 0x0A652'i32, 501, + 0x0A654'i32, 501, + 0x0A656'i32, 501, + 0x0A658'i32, 501, + 0x0A65A'i32, 501, + 0x0A65C'i32, 501, + 0x0A65E'i32, 501, + 0x0A660'i32, 501, + 0x0A662'i32, 501, + 0x0A664'i32, 501, + 0x0A666'i32, 501, + 0x0A668'i32, 501, + 0x0A66A'i32, 501, + 0x0A66C'i32, 501, + 0x0A680'i32, 501, + 0x0A682'i32, 501, + 0x0A684'i32, 501, + 0x0A686'i32, 501, + 0x0A688'i32, 501, + 0x0A68A'i32, 501, + 0x0A68C'i32, 501, + 0x0A68E'i32, 501, + 0x0A690'i32, 501, + 0x0A692'i32, 501, + 0x0A694'i32, 501, + 0x0A696'i32, 501, + 0x0A698'i32, 501, + 0x0A69A'i32, 501, + 0x0A722'i32, 501, + 0x0A724'i32, 501, + 0x0A726'i32, 501, + 0x0A728'i32, 501, + 0x0A72A'i32, 501, + 0x0A72C'i32, 501, + 0x0A72E'i32, 501, + 0x0A732'i32, 501, + 0x0A734'i32, 501, + 0x0A736'i32, 501, + 0x0A738'i32, 501, + 0x0A73A'i32, 501, + 0x0A73C'i32, 501, + 0x0A73E'i32, 501, + 0x0A740'i32, 501, + 0x0A742'i32, 501, + 0x0A744'i32, 501, + 0x0A746'i32, 501, + 0x0A748'i32, 501, + 0x0A74A'i32, 501, + 0x0A74C'i32, 501, + 0x0A74E'i32, 501, + 0x0A750'i32, 501, + 0x0A752'i32, 501, + 0x0A754'i32, 501, + 0x0A756'i32, 501, + 0x0A758'i32, 501, + 0x0A75A'i32, 501, + 0x0A75C'i32, 501, + 0x0A75E'i32, 501, + 0x0A760'i32, 501, + 0x0A762'i32, 501, + 0x0A764'i32, 501, + 0x0A766'i32, 501, + 0x0A768'i32, 501, + 0x0A76A'i32, 501, + 0x0A76C'i32, 501, + 0x0A76E'i32, 501, + 0x0A779'i32, 501, + 0x0A77B'i32, 501, + 0x0A77D'i32, -34832, + 0x0A77E'i32, 501, + 0x0A780'i32, 501, + 0x0A782'i32, 501, + 0x0A784'i32, 501, + 0x0A786'i32, 501, + 0x0A78B'i32, 501, + 0x0A78D'i32, -41780, + 0x0A790'i32, 501, + 0x0A792'i32, 501, + 0x0A796'i32, 501, + 0x0A798'i32, 501, + 0x0A79A'i32, 501, + 0x0A79C'i32, 501, + 0x0A79E'i32, 501, + 0x0A7A0'i32, 501, + 0x0A7A2'i32, 501, + 0x0A7A4'i32, 501, + 0x0A7A6'i32, 501, + 0x0A7A8'i32, 501, + 0x0A7AA'i32, -41808, + 0x0A7AB'i32, -41819, + 0x0A7AC'i32, -41815, + 0x0A7AD'i32, -41805, + 0x0A7AE'i32, -41808, + 0x0A7B0'i32, -41758, + 0x0A7B1'i32, -41782, + 0x0A7B2'i32, -41761, + 0x0A7B3'i32, 1428, + 0x0A7B4'i32, 501, + 0x0A7B6'i32, 501, + 0x0A7B8'i32, 501, + 0x0A7BA'i32, 501, + 0x0A7BC'i32, 501, + 0x0A7BE'i32, 501, + 0x0A7C2'i32, 501, + 0x0A7C4'i32, 452, + 0x0A7C5'i32, -41807, + 0x0A7C6'i32, -34884, ] toUpperRanges = [ - 0x00061, 0x0007A, 468, - 0x000E0, 0x000F6, 468, - 0x000F8, 0x000FE, 468, - 0x0023F, 0x00240, 11315, - 0x00256, 0x00257, 295, - 0x0028A, 0x0028B, 283, - 0x0037B, 0x0037D, 630, - 0x003AD, 0x003AF, 463, - 0x003B1, 0x003C1, 468, - 0x003C3, 0x003CB, 468, - 0x003CD, 0x003CE, 437, - 0x00430, 0x0044F, 468, - 0x00450, 0x0045F, 420, - 0x00561, 0x00586, 452, - 0x010D0, 0x010FA, 3508, - 0x010FD, 0x010FF, 3508, - 0x013F8, 0x013FD, 492, - 0x01C83, 0x01C84, -5742, - 0x01F00, 0x01F07, 508, - 0x01F10, 0x01F15, 508, - 0x01F20, 0x01F27, 508, - 0x01F30, 0x01F37, 508, - 0x01F40, 0x01F45, 508, - 0x01F60, 0x01F67, 508, - 0x01F70, 0x01F71, 574, - 0x01F72, 0x01F75, 586, - 0x01F76, 0x01F77, 600, - 0x01F78, 0x01F79, 628, - 0x01F7A, 0x01F7B, 612, - 0x01F7C, 0x01F7D, 626, - 0x01F80, 0x01F87, 508, - 0x01F90, 0x01F97, 508, - 0x01FA0, 0x01FA7, 508, - 0x01FB0, 0x01FB1, 508, - 0x01FD0, 0x01FD1, 508, - 0x01FE0, 0x01FE1, 508, - 0x02C30, 0x02C5E, 452, - 0x02D00, 0x02D25, -6764, - 0x0AB70, 0x0ABBF, -38364, - 0x0FF41, 0x0FF5A, 468, - 0x10428, 0x1044F, 460, - 0x104D8, 0x104FB, 460, - 0x10CC0, 0x10CF2, 436, - 0x118C0, 0x118DF, 468, - 0x16E60, 0x16E7F, 468, - 0x1E922, 0x1E943, 466, + 0x00061'i32, 0x0007A'i32, 468, + 0x000E0'i32, 0x000F6'i32, 468, + 0x000F8'i32, 0x000FE'i32, 468, + 0x0023F'i32, 0x00240'i32, 11315, + 0x00256'i32, 0x00257'i32, 295, + 0x0028A'i32, 0x0028B'i32, 283, + 0x0037B'i32, 0x0037D'i32, 630, + 0x003AD'i32, 0x003AF'i32, 463, + 0x003B1'i32, 0x003C1'i32, 468, + 0x003C3'i32, 0x003CB'i32, 468, + 0x003CD'i32, 0x003CE'i32, 437, + 0x00430'i32, 0x0044F'i32, 468, + 0x00450'i32, 0x0045F'i32, 420, + 0x00561'i32, 0x00586'i32, 452, + 0x010D0'i32, 0x010FA'i32, 3508, + 0x010FD'i32, 0x010FF'i32, 3508, + 0x013F8'i32, 0x013FD'i32, 492, + 0x01C83'i32, 0x01C84'i32, -5742, + 0x01F00'i32, 0x01F07'i32, 508, + 0x01F10'i32, 0x01F15'i32, 508, + 0x01F20'i32, 0x01F27'i32, 508, + 0x01F30'i32, 0x01F37'i32, 508, + 0x01F40'i32, 0x01F45'i32, 508, + 0x01F60'i32, 0x01F67'i32, 508, + 0x01F70'i32, 0x01F71'i32, 574, + 0x01F72'i32, 0x01F75'i32, 586, + 0x01F76'i32, 0x01F77'i32, 600, + 0x01F78'i32, 0x01F79'i32, 628, + 0x01F7A'i32, 0x01F7B'i32, 612, + 0x01F7C'i32, 0x01F7D'i32, 626, + 0x01F80'i32, 0x01F87'i32, 508, + 0x01F90'i32, 0x01F97'i32, 508, + 0x01FA0'i32, 0x01FA7'i32, 508, + 0x01FB0'i32, 0x01FB1'i32, 508, + 0x01FD0'i32, 0x01FD1'i32, 508, + 0x01FE0'i32, 0x01FE1'i32, 508, + 0x02C30'i32, 0x02C5E'i32, 452, + 0x02D00'i32, 0x02D25'i32, -6764, + 0x0AB70'i32, 0x0ABBF'i32, -38364, + 0x0FF41'i32, 0x0FF5A'i32, 468, + 0x10428'i32, 0x1044F'i32, 460, + 0x104D8'i32, 0x104FB'i32, 460, + 0x10CC0'i32, 0x10CF2'i32, 436, + 0x118C0'i32, 0x118DF'i32, 468, + 0x16E60'i32, 0x16E7F'i32, 468, + 0x1E922'i32, 0x1E943'i32, 466, ] toUpperSinglets = [ - 0x000B5, 1243, - 0x000FF, 621, - 0x00101, 499, - 0x00103, 499, - 0x00105, 499, - 0x00107, 499, - 0x00109, 499, - 0x0010B, 499, - 0x0010D, 499, - 0x0010F, 499, - 0x00111, 499, - 0x00113, 499, - 0x00115, 499, - 0x00117, 499, - 0x00119, 499, - 0x0011B, 499, - 0x0011D, 499, - 0x0011F, 499, - 0x00121, 499, - 0x00123, 499, - 0x00125, 499, - 0x00127, 499, - 0x00129, 499, - 0x0012B, 499, - 0x0012D, 499, - 0x0012F, 499, - 0x00131, 268, - 0x00133, 499, - 0x00135, 499, - 0x00137, 499, - 0x0013A, 499, - 0x0013C, 499, - 0x0013E, 499, - 0x00140, 499, - 0x00142, 499, - 0x00144, 499, - 0x00146, 499, - 0x00148, 499, - 0x0014B, 499, - 0x0014D, 499, - 0x0014F, 499, - 0x00151, 499, - 0x00153, 499, - 0x00155, 499, - 0x00157, 499, - 0x00159, 499, - 0x0015B, 499, - 0x0015D, 499, - 0x0015F, 499, - 0x00161, 499, - 0x00163, 499, - 0x00165, 499, - 0x00167, 499, - 0x00169, 499, - 0x0016B, 499, - 0x0016D, 499, - 0x0016F, 499, - 0x00171, 499, - 0x00173, 499, - 0x00175, 499, - 0x00177, 499, - 0x0017A, 499, - 0x0017C, 499, - 0x0017E, 499, - 0x0017F, 200, - 0x00180, 695, - 0x00183, 499, - 0x00185, 499, - 0x00188, 499, - 0x0018C, 499, - 0x00192, 499, - 0x00195, 597, - 0x00199, 499, - 0x0019A, 663, - 0x0019E, 630, - 0x001A1, 499, - 0x001A3, 499, - 0x001A5, 499, - 0x001A8, 499, - 0x001AD, 499, - 0x001B0, 499, - 0x001B4, 499, - 0x001B6, 499, - 0x001B9, 499, - 0x001BD, 499, - 0x001BF, 556, - 0x001C5, 499, - 0x001C6, 498, - 0x001C8, 499, - 0x001C9, 498, - 0x001CB, 499, - 0x001CC, 498, - 0x001CE, 499, - 0x001D0, 499, - 0x001D2, 499, - 0x001D4, 499, - 0x001D6, 499, - 0x001D8, 499, - 0x001DA, 499, - 0x001DC, 499, - 0x001DD, 421, - 0x001DF, 499, - 0x001E1, 499, - 0x001E3, 499, - 0x001E5, 499, - 0x001E7, 499, - 0x001E9, 499, - 0x001EB, 499, - 0x001ED, 499, - 0x001EF, 499, - 0x001F2, 499, - 0x001F3, 498, - 0x001F5, 499, - 0x001F9, 499, - 0x001FB, 499, - 0x001FD, 499, - 0x001FF, 499, - 0x00201, 499, - 0x00203, 499, - 0x00205, 499, - 0x00207, 499, - 0x00209, 499, - 0x0020B, 499, - 0x0020D, 499, - 0x0020F, 499, - 0x00211, 499, - 0x00213, 499, - 0x00215, 499, - 0x00217, 499, - 0x00219, 499, - 0x0021B, 499, - 0x0021D, 499, - 0x0021F, 499, - 0x00223, 499, - 0x00225, 499, - 0x00227, 499, - 0x00229, 499, - 0x0022B, 499, - 0x0022D, 499, - 0x0022F, 499, - 0x00231, 499, - 0x00233, 499, - 0x0023C, 499, - 0x00242, 499, - 0x00247, 499, - 0x00249, 499, - 0x0024B, 499, - 0x0024D, 499, - 0x0024F, 499, - 0x00250, 11283, - 0x00251, 11280, - 0x00252, 11282, - 0x00253, 290, - 0x00254, 294, - 0x00259, 298, - 0x0025B, 297, - 0x0025C, 42819, - 0x00260, 295, - 0x00261, 42815, - 0x00263, 293, - 0x00265, 42780, - 0x00266, 42808, - 0x00268, 291, - 0x00269, 289, - 0x0026A, 42808, - 0x0026B, 11243, - 0x0026C, 42805, - 0x0026F, 289, - 0x00271, 11249, - 0x00272, 287, - 0x00275, 286, - 0x0027D, 11227, - 0x00280, 282, - 0x00282, 42807, - 0x00283, 282, - 0x00287, 42782, - 0x00288, 282, - 0x00289, 431, - 0x0028C, 429, - 0x00292, 281, - 0x0029D, 42761, - 0x0029E, 42758, - 0x00371, 499, - 0x00373, 499, - 0x00377, 499, - 0x003AC, 462, - 0x003C2, 469, - 0x003CC, 436, - 0x003D0, 438, - 0x003D1, 443, - 0x003D5, 453, - 0x003D6, 446, - 0x003D7, 492, - 0x003D9, 499, - 0x003DB, 499, - 0x003DD, 499, - 0x003DF, 499, - 0x003E1, 499, - 0x003E3, 499, - 0x003E5, 499, - 0x003E7, 499, - 0x003E9, 499, - 0x003EB, 499, - 0x003ED, 499, - 0x003EF, 499, - 0x003F0, 414, - 0x003F1, 420, - 0x003F2, 507, - 0x003F3, 384, - 0x003F5, 404, - 0x003F8, 499, - 0x003FB, 499, - 0x00461, 499, - 0x00463, 499, - 0x00465, 499, - 0x00467, 499, - 0x00469, 499, - 0x0046B, 499, - 0x0046D, 499, - 0x0046F, 499, - 0x00471, 499, - 0x00473, 499, - 0x00475, 499, - 0x00477, 499, - 0x00479, 499, - 0x0047B, 499, - 0x0047D, 499, - 0x0047F, 499, - 0x00481, 499, - 0x0048B, 499, - 0x0048D, 499, - 0x0048F, 499, - 0x00491, 499, - 0x00493, 499, - 0x00495, 499, - 0x00497, 499, - 0x00499, 499, - 0x0049B, 499, - 0x0049D, 499, - 0x0049F, 499, - 0x004A1, 499, - 0x004A3, 499, - 0x004A5, 499, - 0x004A7, 499, - 0x004A9, 499, - 0x004AB, 499, - 0x004AD, 499, - 0x004AF, 499, - 0x004B1, 499, - 0x004B3, 499, - 0x004B5, 499, - 0x004B7, 499, - 0x004B9, 499, - 0x004BB, 499, - 0x004BD, 499, - 0x004BF, 499, - 0x004C2, 499, - 0x004C4, 499, - 0x004C6, 499, - 0x004C8, 499, - 0x004CA, 499, - 0x004CC, 499, - 0x004CE, 499, - 0x004CF, 485, - 0x004D1, 499, - 0x004D3, 499, - 0x004D5, 499, - 0x004D7, 499, - 0x004D9, 499, - 0x004DB, 499, - 0x004DD, 499, - 0x004DF, 499, - 0x004E1, 499, - 0x004E3, 499, - 0x004E5, 499, - 0x004E7, 499, - 0x004E9, 499, - 0x004EB, 499, - 0x004ED, 499, - 0x004EF, 499, - 0x004F1, 499, - 0x004F3, 499, - 0x004F5, 499, - 0x004F7, 499, - 0x004F9, 499, - 0x004FB, 499, - 0x004FD, 499, - 0x004FF, 499, - 0x00501, 499, - 0x00503, 499, - 0x00505, 499, - 0x00507, 499, - 0x00509, 499, - 0x0050B, 499, - 0x0050D, 499, - 0x0050F, 499, - 0x00511, 499, - 0x00513, 499, - 0x00515, 499, - 0x00517, 499, - 0x00519, 499, - 0x0051B, 499, - 0x0051D, 499, - 0x0051F, 499, - 0x00521, 499, - 0x00523, 499, - 0x00525, 499, - 0x00527, 499, - 0x00529, 499, - 0x0052B, 499, - 0x0052D, 499, - 0x0052F, 499, - 0x01C80, -5754, - 0x01C81, -5753, - 0x01C82, -5744, - 0x01C85, -5743, - 0x01C86, -5736, - 0x01C87, -5681, - 0x01C88, 35766, - 0x01D79, 35832, - 0x01D7D, 4314, - 0x01D8E, 35884, - 0x01E01, 499, - 0x01E03, 499, - 0x01E05, 499, - 0x01E07, 499, - 0x01E09, 499, - 0x01E0B, 499, - 0x01E0D, 499, - 0x01E0F, 499, - 0x01E11, 499, - 0x01E13, 499, - 0x01E15, 499, - 0x01E17, 499, - 0x01E19, 499, - 0x01E1B, 499, - 0x01E1D, 499, - 0x01E1F, 499, - 0x01E21, 499, - 0x01E23, 499, - 0x01E25, 499, - 0x01E27, 499, - 0x01E29, 499, - 0x01E2B, 499, - 0x01E2D, 499, - 0x01E2F, 499, - 0x01E31, 499, - 0x01E33, 499, - 0x01E35, 499, - 0x01E37, 499, - 0x01E39, 499, - 0x01E3B, 499, - 0x01E3D, 499, - 0x01E3F, 499, - 0x01E41, 499, - 0x01E43, 499, - 0x01E45, 499, - 0x01E47, 499, - 0x01E49, 499, - 0x01E4B, 499, - 0x01E4D, 499, - 0x01E4F, 499, - 0x01E51, 499, - 0x01E53, 499, - 0x01E55, 499, - 0x01E57, 499, - 0x01E59, 499, - 0x01E5B, 499, - 0x01E5D, 499, - 0x01E5F, 499, - 0x01E61, 499, - 0x01E63, 499, - 0x01E65, 499, - 0x01E67, 499, - 0x01E69, 499, - 0x01E6B, 499, - 0x01E6D, 499, - 0x01E6F, 499, - 0x01E71, 499, - 0x01E73, 499, - 0x01E75, 499, - 0x01E77, 499, - 0x01E79, 499, - 0x01E7B, 499, - 0x01E7D, 499, - 0x01E7F, 499, - 0x01E81, 499, - 0x01E83, 499, - 0x01E85, 499, - 0x01E87, 499, - 0x01E89, 499, - 0x01E8B, 499, - 0x01E8D, 499, - 0x01E8F, 499, - 0x01E91, 499, - 0x01E93, 499, - 0x01E95, 499, - 0x01E9B, 441, - 0x01EA1, 499, - 0x01EA3, 499, - 0x01EA5, 499, - 0x01EA7, 499, - 0x01EA9, 499, - 0x01EAB, 499, - 0x01EAD, 499, - 0x01EAF, 499, - 0x01EB1, 499, - 0x01EB3, 499, - 0x01EB5, 499, - 0x01EB7, 499, - 0x01EB9, 499, - 0x01EBB, 499, - 0x01EBD, 499, - 0x01EBF, 499, - 0x01EC1, 499, - 0x01EC3, 499, - 0x01EC5, 499, - 0x01EC7, 499, - 0x01EC9, 499, - 0x01ECB, 499, - 0x01ECD, 499, - 0x01ECF, 499, - 0x01ED1, 499, - 0x01ED3, 499, - 0x01ED5, 499, - 0x01ED7, 499, - 0x01ED9, 499, - 0x01EDB, 499, - 0x01EDD, 499, - 0x01EDF, 499, - 0x01EE1, 499, - 0x01EE3, 499, - 0x01EE5, 499, - 0x01EE7, 499, - 0x01EE9, 499, - 0x01EEB, 499, - 0x01EED, 499, - 0x01EEF, 499, - 0x01EF1, 499, - 0x01EF3, 499, - 0x01EF5, 499, - 0x01EF7, 499, - 0x01EF9, 499, - 0x01EFB, 499, - 0x01EFD, 499, - 0x01EFF, 499, - 0x01F51, 508, - 0x01F53, 508, - 0x01F55, 508, - 0x01F57, 508, - 0x01FB3, 509, - 0x01FBE, -6705, - 0x01FC3, 509, - 0x01FE5, 507, - 0x01FF3, 509, - 0x0214E, 472, - 0x02184, 499, - 0x02C61, 499, - 0x02C65, -10295, - 0x02C66, -10292, - 0x02C68, 499, - 0x02C6A, 499, - 0x02C6C, 499, - 0x02C73, 499, - 0x02C76, 499, - 0x02C81, 499, - 0x02C83, 499, - 0x02C85, 499, - 0x02C87, 499, - 0x02C89, 499, - 0x02C8B, 499, - 0x02C8D, 499, - 0x02C8F, 499, - 0x02C91, 499, - 0x02C93, 499, - 0x02C95, 499, - 0x02C97, 499, - 0x02C99, 499, - 0x02C9B, 499, - 0x02C9D, 499, - 0x02C9F, 499, - 0x02CA1, 499, - 0x02CA3, 499, - 0x02CA5, 499, - 0x02CA7, 499, - 0x02CA9, 499, - 0x02CAB, 499, - 0x02CAD, 499, - 0x02CAF, 499, - 0x02CB1, 499, - 0x02CB3, 499, - 0x02CB5, 499, - 0x02CB7, 499, - 0x02CB9, 499, - 0x02CBB, 499, - 0x02CBD, 499, - 0x02CBF, 499, - 0x02CC1, 499, - 0x02CC3, 499, - 0x02CC5, 499, - 0x02CC7, 499, - 0x02CC9, 499, - 0x02CCB, 499, - 0x02CCD, 499, - 0x02CCF, 499, - 0x02CD1, 499, - 0x02CD3, 499, - 0x02CD5, 499, - 0x02CD7, 499, - 0x02CD9, 499, - 0x02CDB, 499, - 0x02CDD, 499, - 0x02CDF, 499, - 0x02CE1, 499, - 0x02CE3, 499, - 0x02CEC, 499, - 0x02CEE, 499, - 0x02CF3, 499, - 0x02D27, -6764, - 0x02D2D, -6764, - 0x0A641, 499, - 0x0A643, 499, - 0x0A645, 499, - 0x0A647, 499, - 0x0A649, 499, - 0x0A64B, 499, - 0x0A64D, 499, - 0x0A64F, 499, - 0x0A651, 499, - 0x0A653, 499, - 0x0A655, 499, - 0x0A657, 499, - 0x0A659, 499, - 0x0A65B, 499, - 0x0A65D, 499, - 0x0A65F, 499, - 0x0A661, 499, - 0x0A663, 499, - 0x0A665, 499, - 0x0A667, 499, - 0x0A669, 499, - 0x0A66B, 499, - 0x0A66D, 499, - 0x0A681, 499, - 0x0A683, 499, - 0x0A685, 499, - 0x0A687, 499, - 0x0A689, 499, - 0x0A68B, 499, - 0x0A68D, 499, - 0x0A68F, 499, - 0x0A691, 499, - 0x0A693, 499, - 0x0A695, 499, - 0x0A697, 499, - 0x0A699, 499, - 0x0A69B, 499, - 0x0A723, 499, - 0x0A725, 499, - 0x0A727, 499, - 0x0A729, 499, - 0x0A72B, 499, - 0x0A72D, 499, - 0x0A72F, 499, - 0x0A733, 499, - 0x0A735, 499, - 0x0A737, 499, - 0x0A739, 499, - 0x0A73B, 499, - 0x0A73D, 499, - 0x0A73F, 499, - 0x0A741, 499, - 0x0A743, 499, - 0x0A745, 499, - 0x0A747, 499, - 0x0A749, 499, - 0x0A74B, 499, - 0x0A74D, 499, - 0x0A74F, 499, - 0x0A751, 499, - 0x0A753, 499, - 0x0A755, 499, - 0x0A757, 499, - 0x0A759, 499, - 0x0A75B, 499, - 0x0A75D, 499, - 0x0A75F, 499, - 0x0A761, 499, - 0x0A763, 499, - 0x0A765, 499, - 0x0A767, 499, - 0x0A769, 499, - 0x0A76B, 499, - 0x0A76D, 499, - 0x0A76F, 499, - 0x0A77A, 499, - 0x0A77C, 499, - 0x0A77F, 499, - 0x0A781, 499, - 0x0A783, 499, - 0x0A785, 499, - 0x0A787, 499, - 0x0A78C, 499, - 0x0A791, 499, - 0x0A793, 499, - 0x0A794, 548, - 0x0A797, 499, - 0x0A799, 499, - 0x0A79B, 499, - 0x0A79D, 499, - 0x0A79F, 499, - 0x0A7A1, 499, - 0x0A7A3, 499, - 0x0A7A5, 499, - 0x0A7A7, 499, - 0x0A7A9, 499, - 0x0A7B5, 499, - 0x0A7B7, 499, - 0x0A7B9, 499, - 0x0A7BB, 499, - 0x0A7BD, 499, - 0x0A7BF, 499, - 0x0A7C3, 499, - 0x0AB53, -428, + 0x000B5'i32, 1243, + 0x000FF'i32, 621, + 0x00101'i32, 499, + 0x00103'i32, 499, + 0x00105'i32, 499, + 0x00107'i32, 499, + 0x00109'i32, 499, + 0x0010B'i32, 499, + 0x0010D'i32, 499, + 0x0010F'i32, 499, + 0x00111'i32, 499, + 0x00113'i32, 499, + 0x00115'i32, 499, + 0x00117'i32, 499, + 0x00119'i32, 499, + 0x0011B'i32, 499, + 0x0011D'i32, 499, + 0x0011F'i32, 499, + 0x00121'i32, 499, + 0x00123'i32, 499, + 0x00125'i32, 499, + 0x00127'i32, 499, + 0x00129'i32, 499, + 0x0012B'i32, 499, + 0x0012D'i32, 499, + 0x0012F'i32, 499, + 0x00131'i32, 268, + 0x00133'i32, 499, + 0x00135'i32, 499, + 0x00137'i32, 499, + 0x0013A'i32, 499, + 0x0013C'i32, 499, + 0x0013E'i32, 499, + 0x00140'i32, 499, + 0x00142'i32, 499, + 0x00144'i32, 499, + 0x00146'i32, 499, + 0x00148'i32, 499, + 0x0014B'i32, 499, + 0x0014D'i32, 499, + 0x0014F'i32, 499, + 0x00151'i32, 499, + 0x00153'i32, 499, + 0x00155'i32, 499, + 0x00157'i32, 499, + 0x00159'i32, 499, + 0x0015B'i32, 499, + 0x0015D'i32, 499, + 0x0015F'i32, 499, + 0x00161'i32, 499, + 0x00163'i32, 499, + 0x00165'i32, 499, + 0x00167'i32, 499, + 0x00169'i32, 499, + 0x0016B'i32, 499, + 0x0016D'i32, 499, + 0x0016F'i32, 499, + 0x00171'i32, 499, + 0x00173'i32, 499, + 0x00175'i32, 499, + 0x00177'i32, 499, + 0x0017A'i32, 499, + 0x0017C'i32, 499, + 0x0017E'i32, 499, + 0x0017F'i32, 200, + 0x00180'i32, 695, + 0x00183'i32, 499, + 0x00185'i32, 499, + 0x00188'i32, 499, + 0x0018C'i32, 499, + 0x00192'i32, 499, + 0x00195'i32, 597, + 0x00199'i32, 499, + 0x0019A'i32, 663, + 0x0019E'i32, 630, + 0x001A1'i32, 499, + 0x001A3'i32, 499, + 0x001A5'i32, 499, + 0x001A8'i32, 499, + 0x001AD'i32, 499, + 0x001B0'i32, 499, + 0x001B4'i32, 499, + 0x001B6'i32, 499, + 0x001B9'i32, 499, + 0x001BD'i32, 499, + 0x001BF'i32, 556, + 0x001C5'i32, 499, + 0x001C6'i32, 498, + 0x001C8'i32, 499, + 0x001C9'i32, 498, + 0x001CB'i32, 499, + 0x001CC'i32, 498, + 0x001CE'i32, 499, + 0x001D0'i32, 499, + 0x001D2'i32, 499, + 0x001D4'i32, 499, + 0x001D6'i32, 499, + 0x001D8'i32, 499, + 0x001DA'i32, 499, + 0x001DC'i32, 499, + 0x001DD'i32, 421, + 0x001DF'i32, 499, + 0x001E1'i32, 499, + 0x001E3'i32, 499, + 0x001E5'i32, 499, + 0x001E7'i32, 499, + 0x001E9'i32, 499, + 0x001EB'i32, 499, + 0x001ED'i32, 499, + 0x001EF'i32, 499, + 0x001F2'i32, 499, + 0x001F3'i32, 498, + 0x001F5'i32, 499, + 0x001F9'i32, 499, + 0x001FB'i32, 499, + 0x001FD'i32, 499, + 0x001FF'i32, 499, + 0x00201'i32, 499, + 0x00203'i32, 499, + 0x00205'i32, 499, + 0x00207'i32, 499, + 0x00209'i32, 499, + 0x0020B'i32, 499, + 0x0020D'i32, 499, + 0x0020F'i32, 499, + 0x00211'i32, 499, + 0x00213'i32, 499, + 0x00215'i32, 499, + 0x00217'i32, 499, + 0x00219'i32, 499, + 0x0021B'i32, 499, + 0x0021D'i32, 499, + 0x0021F'i32, 499, + 0x00223'i32, 499, + 0x00225'i32, 499, + 0x00227'i32, 499, + 0x00229'i32, 499, + 0x0022B'i32, 499, + 0x0022D'i32, 499, + 0x0022F'i32, 499, + 0x00231'i32, 499, + 0x00233'i32, 499, + 0x0023C'i32, 499, + 0x00242'i32, 499, + 0x00247'i32, 499, + 0x00249'i32, 499, + 0x0024B'i32, 499, + 0x0024D'i32, 499, + 0x0024F'i32, 499, + 0x00250'i32, 11283, + 0x00251'i32, 11280, + 0x00252'i32, 11282, + 0x00253'i32, 290, + 0x00254'i32, 294, + 0x00259'i32, 298, + 0x0025B'i32, 297, + 0x0025C'i32, 42819, + 0x00260'i32, 295, + 0x00261'i32, 42815, + 0x00263'i32, 293, + 0x00265'i32, 42780, + 0x00266'i32, 42808, + 0x00268'i32, 291, + 0x00269'i32, 289, + 0x0026A'i32, 42808, + 0x0026B'i32, 11243, + 0x0026C'i32, 42805, + 0x0026F'i32, 289, + 0x00271'i32, 11249, + 0x00272'i32, 287, + 0x00275'i32, 286, + 0x0027D'i32, 11227, + 0x00280'i32, 282, + 0x00282'i32, 42807, + 0x00283'i32, 282, + 0x00287'i32, 42782, + 0x00288'i32, 282, + 0x00289'i32, 431, + 0x0028C'i32, 429, + 0x00292'i32, 281, + 0x0029D'i32, 42761, + 0x0029E'i32, 42758, + 0x00371'i32, 499, + 0x00373'i32, 499, + 0x00377'i32, 499, + 0x003AC'i32, 462, + 0x003C2'i32, 469, + 0x003CC'i32, 436, + 0x003D0'i32, 438, + 0x003D1'i32, 443, + 0x003D5'i32, 453, + 0x003D6'i32, 446, + 0x003D7'i32, 492, + 0x003D9'i32, 499, + 0x003DB'i32, 499, + 0x003DD'i32, 499, + 0x003DF'i32, 499, + 0x003E1'i32, 499, + 0x003E3'i32, 499, + 0x003E5'i32, 499, + 0x003E7'i32, 499, + 0x003E9'i32, 499, + 0x003EB'i32, 499, + 0x003ED'i32, 499, + 0x003EF'i32, 499, + 0x003F0'i32, 414, + 0x003F1'i32, 420, + 0x003F2'i32, 507, + 0x003F3'i32, 384, + 0x003F5'i32, 404, + 0x003F8'i32, 499, + 0x003FB'i32, 499, + 0x00461'i32, 499, + 0x00463'i32, 499, + 0x00465'i32, 499, + 0x00467'i32, 499, + 0x00469'i32, 499, + 0x0046B'i32, 499, + 0x0046D'i32, 499, + 0x0046F'i32, 499, + 0x00471'i32, 499, + 0x00473'i32, 499, + 0x00475'i32, 499, + 0x00477'i32, 499, + 0x00479'i32, 499, + 0x0047B'i32, 499, + 0x0047D'i32, 499, + 0x0047F'i32, 499, + 0x00481'i32, 499, + 0x0048B'i32, 499, + 0x0048D'i32, 499, + 0x0048F'i32, 499, + 0x00491'i32, 499, + 0x00493'i32, 499, + 0x00495'i32, 499, + 0x00497'i32, 499, + 0x00499'i32, 499, + 0x0049B'i32, 499, + 0x0049D'i32, 499, + 0x0049F'i32, 499, + 0x004A1'i32, 499, + 0x004A3'i32, 499, + 0x004A5'i32, 499, + 0x004A7'i32, 499, + 0x004A9'i32, 499, + 0x004AB'i32, 499, + 0x004AD'i32, 499, + 0x004AF'i32, 499, + 0x004B1'i32, 499, + 0x004B3'i32, 499, + 0x004B5'i32, 499, + 0x004B7'i32, 499, + 0x004B9'i32, 499, + 0x004BB'i32, 499, + 0x004BD'i32, 499, + 0x004BF'i32, 499, + 0x004C2'i32, 499, + 0x004C4'i32, 499, + 0x004C6'i32, 499, + 0x004C8'i32, 499, + 0x004CA'i32, 499, + 0x004CC'i32, 499, + 0x004CE'i32, 499, + 0x004CF'i32, 485, + 0x004D1'i32, 499, + 0x004D3'i32, 499, + 0x004D5'i32, 499, + 0x004D7'i32, 499, + 0x004D9'i32, 499, + 0x004DB'i32, 499, + 0x004DD'i32, 499, + 0x004DF'i32, 499, + 0x004E1'i32, 499, + 0x004E3'i32, 499, + 0x004E5'i32, 499, + 0x004E7'i32, 499, + 0x004E9'i32, 499, + 0x004EB'i32, 499, + 0x004ED'i32, 499, + 0x004EF'i32, 499, + 0x004F1'i32, 499, + 0x004F3'i32, 499, + 0x004F5'i32, 499, + 0x004F7'i32, 499, + 0x004F9'i32, 499, + 0x004FB'i32, 499, + 0x004FD'i32, 499, + 0x004FF'i32, 499, + 0x00501'i32, 499, + 0x00503'i32, 499, + 0x00505'i32, 499, + 0x00507'i32, 499, + 0x00509'i32, 499, + 0x0050B'i32, 499, + 0x0050D'i32, 499, + 0x0050F'i32, 499, + 0x00511'i32, 499, + 0x00513'i32, 499, + 0x00515'i32, 499, + 0x00517'i32, 499, + 0x00519'i32, 499, + 0x0051B'i32, 499, + 0x0051D'i32, 499, + 0x0051F'i32, 499, + 0x00521'i32, 499, + 0x00523'i32, 499, + 0x00525'i32, 499, + 0x00527'i32, 499, + 0x00529'i32, 499, + 0x0052B'i32, 499, + 0x0052D'i32, 499, + 0x0052F'i32, 499, + 0x01C80'i32, -5754, + 0x01C81'i32, -5753, + 0x01C82'i32, -5744, + 0x01C85'i32, -5743, + 0x01C86'i32, -5736, + 0x01C87'i32, -5681, + 0x01C88'i32, 35766, + 0x01D79'i32, 35832, + 0x01D7D'i32, 4314, + 0x01D8E'i32, 35884, + 0x01E01'i32, 499, + 0x01E03'i32, 499, + 0x01E05'i32, 499, + 0x01E07'i32, 499, + 0x01E09'i32, 499, + 0x01E0B'i32, 499, + 0x01E0D'i32, 499, + 0x01E0F'i32, 499, + 0x01E11'i32, 499, + 0x01E13'i32, 499, + 0x01E15'i32, 499, + 0x01E17'i32, 499, + 0x01E19'i32, 499, + 0x01E1B'i32, 499, + 0x01E1D'i32, 499, + 0x01E1F'i32, 499, + 0x01E21'i32, 499, + 0x01E23'i32, 499, + 0x01E25'i32, 499, + 0x01E27'i32, 499, + 0x01E29'i32, 499, + 0x01E2B'i32, 499, + 0x01E2D'i32, 499, + 0x01E2F'i32, 499, + 0x01E31'i32, 499, + 0x01E33'i32, 499, + 0x01E35'i32, 499, + 0x01E37'i32, 499, + 0x01E39'i32, 499, + 0x01E3B'i32, 499, + 0x01E3D'i32, 499, + 0x01E3F'i32, 499, + 0x01E41'i32, 499, + 0x01E43'i32, 499, + 0x01E45'i32, 499, + 0x01E47'i32, 499, + 0x01E49'i32, 499, + 0x01E4B'i32, 499, + 0x01E4D'i32, 499, + 0x01E4F'i32, 499, + 0x01E51'i32, 499, + 0x01E53'i32, 499, + 0x01E55'i32, 499, + 0x01E57'i32, 499, + 0x01E59'i32, 499, + 0x01E5B'i32, 499, + 0x01E5D'i32, 499, + 0x01E5F'i32, 499, + 0x01E61'i32, 499, + 0x01E63'i32, 499, + 0x01E65'i32, 499, + 0x01E67'i32, 499, + 0x01E69'i32, 499, + 0x01E6B'i32, 499, + 0x01E6D'i32, 499, + 0x01E6F'i32, 499, + 0x01E71'i32, 499, + 0x01E73'i32, 499, + 0x01E75'i32, 499, + 0x01E77'i32, 499, + 0x01E79'i32, 499, + 0x01E7B'i32, 499, + 0x01E7D'i32, 499, + 0x01E7F'i32, 499, + 0x01E81'i32, 499, + 0x01E83'i32, 499, + 0x01E85'i32, 499, + 0x01E87'i32, 499, + 0x01E89'i32, 499, + 0x01E8B'i32, 499, + 0x01E8D'i32, 499, + 0x01E8F'i32, 499, + 0x01E91'i32, 499, + 0x01E93'i32, 499, + 0x01E95'i32, 499, + 0x01E9B'i32, 441, + 0x01EA1'i32, 499, + 0x01EA3'i32, 499, + 0x01EA5'i32, 499, + 0x01EA7'i32, 499, + 0x01EA9'i32, 499, + 0x01EAB'i32, 499, + 0x01EAD'i32, 499, + 0x01EAF'i32, 499, + 0x01EB1'i32, 499, + 0x01EB3'i32, 499, + 0x01EB5'i32, 499, + 0x01EB7'i32, 499, + 0x01EB9'i32, 499, + 0x01EBB'i32, 499, + 0x01EBD'i32, 499, + 0x01EBF'i32, 499, + 0x01EC1'i32, 499, + 0x01EC3'i32, 499, + 0x01EC5'i32, 499, + 0x01EC7'i32, 499, + 0x01EC9'i32, 499, + 0x01ECB'i32, 499, + 0x01ECD'i32, 499, + 0x01ECF'i32, 499, + 0x01ED1'i32, 499, + 0x01ED3'i32, 499, + 0x01ED5'i32, 499, + 0x01ED7'i32, 499, + 0x01ED9'i32, 499, + 0x01EDB'i32, 499, + 0x01EDD'i32, 499, + 0x01EDF'i32, 499, + 0x01EE1'i32, 499, + 0x01EE3'i32, 499, + 0x01EE5'i32, 499, + 0x01EE7'i32, 499, + 0x01EE9'i32, 499, + 0x01EEB'i32, 499, + 0x01EED'i32, 499, + 0x01EEF'i32, 499, + 0x01EF1'i32, 499, + 0x01EF3'i32, 499, + 0x01EF5'i32, 499, + 0x01EF7'i32, 499, + 0x01EF9'i32, 499, + 0x01EFB'i32, 499, + 0x01EFD'i32, 499, + 0x01EFF'i32, 499, + 0x01F51'i32, 508, + 0x01F53'i32, 508, + 0x01F55'i32, 508, + 0x01F57'i32, 508, + 0x01FB3'i32, 509, + 0x01FBE'i32, -6705, + 0x01FC3'i32, 509, + 0x01FE5'i32, 507, + 0x01FF3'i32, 509, + 0x0214E'i32, 472, + 0x02184'i32, 499, + 0x02C61'i32, 499, + 0x02C65'i32, -10295, + 0x02C66'i32, -10292, + 0x02C68'i32, 499, + 0x02C6A'i32, 499, + 0x02C6C'i32, 499, + 0x02C73'i32, 499, + 0x02C76'i32, 499, + 0x02C81'i32, 499, + 0x02C83'i32, 499, + 0x02C85'i32, 499, + 0x02C87'i32, 499, + 0x02C89'i32, 499, + 0x02C8B'i32, 499, + 0x02C8D'i32, 499, + 0x02C8F'i32, 499, + 0x02C91'i32, 499, + 0x02C93'i32, 499, + 0x02C95'i32, 499, + 0x02C97'i32, 499, + 0x02C99'i32, 499, + 0x02C9B'i32, 499, + 0x02C9D'i32, 499, + 0x02C9F'i32, 499, + 0x02CA1'i32, 499, + 0x02CA3'i32, 499, + 0x02CA5'i32, 499, + 0x02CA7'i32, 499, + 0x02CA9'i32, 499, + 0x02CAB'i32, 499, + 0x02CAD'i32, 499, + 0x02CAF'i32, 499, + 0x02CB1'i32, 499, + 0x02CB3'i32, 499, + 0x02CB5'i32, 499, + 0x02CB7'i32, 499, + 0x02CB9'i32, 499, + 0x02CBB'i32, 499, + 0x02CBD'i32, 499, + 0x02CBF'i32, 499, + 0x02CC1'i32, 499, + 0x02CC3'i32, 499, + 0x02CC5'i32, 499, + 0x02CC7'i32, 499, + 0x02CC9'i32, 499, + 0x02CCB'i32, 499, + 0x02CCD'i32, 499, + 0x02CCF'i32, 499, + 0x02CD1'i32, 499, + 0x02CD3'i32, 499, + 0x02CD5'i32, 499, + 0x02CD7'i32, 499, + 0x02CD9'i32, 499, + 0x02CDB'i32, 499, + 0x02CDD'i32, 499, + 0x02CDF'i32, 499, + 0x02CE1'i32, 499, + 0x02CE3'i32, 499, + 0x02CEC'i32, 499, + 0x02CEE'i32, 499, + 0x02CF3'i32, 499, + 0x02D27'i32, -6764, + 0x02D2D'i32, -6764, + 0x0A641'i32, 499, + 0x0A643'i32, 499, + 0x0A645'i32, 499, + 0x0A647'i32, 499, + 0x0A649'i32, 499, + 0x0A64B'i32, 499, + 0x0A64D'i32, 499, + 0x0A64F'i32, 499, + 0x0A651'i32, 499, + 0x0A653'i32, 499, + 0x0A655'i32, 499, + 0x0A657'i32, 499, + 0x0A659'i32, 499, + 0x0A65B'i32, 499, + 0x0A65D'i32, 499, + 0x0A65F'i32, 499, + 0x0A661'i32, 499, + 0x0A663'i32, 499, + 0x0A665'i32, 499, + 0x0A667'i32, 499, + 0x0A669'i32, 499, + 0x0A66B'i32, 499, + 0x0A66D'i32, 499, + 0x0A681'i32, 499, + 0x0A683'i32, 499, + 0x0A685'i32, 499, + 0x0A687'i32, 499, + 0x0A689'i32, 499, + 0x0A68B'i32, 499, + 0x0A68D'i32, 499, + 0x0A68F'i32, 499, + 0x0A691'i32, 499, + 0x0A693'i32, 499, + 0x0A695'i32, 499, + 0x0A697'i32, 499, + 0x0A699'i32, 499, + 0x0A69B'i32, 499, + 0x0A723'i32, 499, + 0x0A725'i32, 499, + 0x0A727'i32, 499, + 0x0A729'i32, 499, + 0x0A72B'i32, 499, + 0x0A72D'i32, 499, + 0x0A72F'i32, 499, + 0x0A733'i32, 499, + 0x0A735'i32, 499, + 0x0A737'i32, 499, + 0x0A739'i32, 499, + 0x0A73B'i32, 499, + 0x0A73D'i32, 499, + 0x0A73F'i32, 499, + 0x0A741'i32, 499, + 0x0A743'i32, 499, + 0x0A745'i32, 499, + 0x0A747'i32, 499, + 0x0A749'i32, 499, + 0x0A74B'i32, 499, + 0x0A74D'i32, 499, + 0x0A74F'i32, 499, + 0x0A751'i32, 499, + 0x0A753'i32, 499, + 0x0A755'i32, 499, + 0x0A757'i32, 499, + 0x0A759'i32, 499, + 0x0A75B'i32, 499, + 0x0A75D'i32, 499, + 0x0A75F'i32, 499, + 0x0A761'i32, 499, + 0x0A763'i32, 499, + 0x0A765'i32, 499, + 0x0A767'i32, 499, + 0x0A769'i32, 499, + 0x0A76B'i32, 499, + 0x0A76D'i32, 499, + 0x0A76F'i32, 499, + 0x0A77A'i32, 499, + 0x0A77C'i32, 499, + 0x0A77F'i32, 499, + 0x0A781'i32, 499, + 0x0A783'i32, 499, + 0x0A785'i32, 499, + 0x0A787'i32, 499, + 0x0A78C'i32, 499, + 0x0A791'i32, 499, + 0x0A793'i32, 499, + 0x0A794'i32, 548, + 0x0A797'i32, 499, + 0x0A799'i32, 499, + 0x0A79B'i32, 499, + 0x0A79D'i32, 499, + 0x0A79F'i32, 499, + 0x0A7A1'i32, 499, + 0x0A7A3'i32, 499, + 0x0A7A5'i32, 499, + 0x0A7A7'i32, 499, + 0x0A7A9'i32, 499, + 0x0A7B5'i32, 499, + 0x0A7B7'i32, 499, + 0x0A7B9'i32, 499, + 0x0A7BB'i32, 499, + 0x0A7BD'i32, 499, + 0x0A7BF'i32, 499, + 0x0A7C3'i32, 499, + 0x0AB53'i32, -428, ] toTitleSinglets = [ - 0x001C4, 501, - 0x001C6, 499, - 0x001C7, 501, - 0x001C9, 499, - 0x001CA, 501, - 0x001CC, 499, - 0x001F1, 501, - 0x001F3, 499, + 0x001C4'i32, 501, + 0x001C6'i32, 499, + 0x001C7'i32, 501, + 0x001C9'i32, 499, + 0x001CA'i32, 501, + 0x001CC'i32, 499, + 0x001F1'i32, 501, + 0x001F3'i32, 499, ] alphaRanges = [ - 0x00041, 0x0005A, - 0x00061, 0x0007A, - 0x000C0, 0x000D6, - 0x000D8, 0x000F6, - 0x000F8, 0x002C1, - 0x002C6, 0x002D1, - 0x002E0, 0x002E4, - 0x00370, 0x00374, - 0x00376, 0x00377, - 0x0037A, 0x0037D, - 0x00388, 0x0038A, - 0x0038E, 0x003A1, - 0x003A3, 0x003F5, - 0x003F7, 0x00481, - 0x0048A, 0x0052F, - 0x00531, 0x00556, - 0x00560, 0x00588, - 0x005D0, 0x005EA, - 0x005EF, 0x005F2, - 0x00620, 0x0064A, - 0x0066E, 0x0066F, - 0x00671, 0x006D3, - 0x006E5, 0x006E6, - 0x006EE, 0x006EF, - 0x006FA, 0x006FC, - 0x00712, 0x0072F, - 0x0074D, 0x007A5, - 0x007CA, 0x007EA, - 0x007F4, 0x007F5, - 0x00800, 0x00815, - 0x00840, 0x00858, - 0x00860, 0x0086A, - 0x008A0, 0x008B4, - 0x008B6, 0x008BD, - 0x00904, 0x00939, - 0x00958, 0x00961, - 0x00971, 0x00980, - 0x00985, 0x0098C, - 0x0098F, 0x00990, - 0x00993, 0x009A8, - 0x009AA, 0x009B0, - 0x009B6, 0x009B9, - 0x009DC, 0x009DD, - 0x009DF, 0x009E1, - 0x009F0, 0x009F1, - 0x00A05, 0x00A0A, - 0x00A0F, 0x00A10, - 0x00A13, 0x00A28, - 0x00A2A, 0x00A30, - 0x00A32, 0x00A33, - 0x00A35, 0x00A36, - 0x00A38, 0x00A39, - 0x00A59, 0x00A5C, - 0x00A72, 0x00A74, - 0x00A85, 0x00A8D, - 0x00A8F, 0x00A91, - 0x00A93, 0x00AA8, - 0x00AAA, 0x00AB0, - 0x00AB2, 0x00AB3, - 0x00AB5, 0x00AB9, - 0x00AE0, 0x00AE1, - 0x00B05, 0x00B0C, - 0x00B0F, 0x00B10, - 0x00B13, 0x00B28, - 0x00B2A, 0x00B30, - 0x00B32, 0x00B33, - 0x00B35, 0x00B39, - 0x00B5C, 0x00B5D, - 0x00B5F, 0x00B61, - 0x00B85, 0x00B8A, - 0x00B8E, 0x00B90, - 0x00B92, 0x00B95, - 0x00B99, 0x00B9A, - 0x00B9E, 0x00B9F, - 0x00BA3, 0x00BA4, - 0x00BA8, 0x00BAA, - 0x00BAE, 0x00BB9, - 0x00C05, 0x00C0C, - 0x00C0E, 0x00C10, - 0x00C12, 0x00C28, - 0x00C2A, 0x00C39, - 0x00C58, 0x00C5A, - 0x00C60, 0x00C61, - 0x00C85, 0x00C8C, - 0x00C8E, 0x00C90, - 0x00C92, 0x00CA8, - 0x00CAA, 0x00CB3, - 0x00CB5, 0x00CB9, - 0x00CE0, 0x00CE1, - 0x00CF1, 0x00CF2, - 0x00D05, 0x00D0C, - 0x00D0E, 0x00D10, - 0x00D12, 0x00D3A, - 0x00D54, 0x00D56, - 0x00D5F, 0x00D61, - 0x00D7A, 0x00D7F, - 0x00D85, 0x00D96, - 0x00D9A, 0x00DB1, - 0x00DB3, 0x00DBB, - 0x00DC0, 0x00DC6, - 0x00E01, 0x00E30, - 0x00E32, 0x00E33, - 0x00E40, 0x00E46, - 0x00E81, 0x00E82, - 0x00E86, 0x00E8A, - 0x00E8C, 0x00EA3, - 0x00EA7, 0x00EB0, - 0x00EB2, 0x00EB3, - 0x00EC0, 0x00EC4, - 0x00EDC, 0x00EDF, - 0x00F40, 0x00F47, - 0x00F49, 0x00F6C, - 0x00F88, 0x00F8C, - 0x01000, 0x0102A, - 0x01050, 0x01055, - 0x0105A, 0x0105D, - 0x01065, 0x01066, - 0x0106E, 0x01070, - 0x01075, 0x01081, - 0x010A0, 0x010C5, - 0x010D0, 0x010FA, - 0x010FC, 0x01248, - 0x0124A, 0x0124D, - 0x01250, 0x01256, - 0x0125A, 0x0125D, - 0x01260, 0x01288, - 0x0128A, 0x0128D, - 0x01290, 0x012B0, - 0x012B2, 0x012B5, - 0x012B8, 0x012BE, - 0x012C2, 0x012C5, - 0x012C8, 0x012D6, - 0x012D8, 0x01310, - 0x01312, 0x01315, - 0x01318, 0x0135A, - 0x01380, 0x0138F, - 0x013A0, 0x013F5, - 0x013F8, 0x013FD, - 0x01401, 0x0166C, - 0x0166F, 0x0167F, - 0x01681, 0x0169A, - 0x016A0, 0x016EA, - 0x016F1, 0x016F8, - 0x01700, 0x0170C, - 0x0170E, 0x01711, - 0x01720, 0x01731, - 0x01740, 0x01751, - 0x01760, 0x0176C, - 0x0176E, 0x01770, - 0x01780, 0x017B3, - 0x01820, 0x01878, - 0x01880, 0x01884, - 0x01887, 0x018A8, - 0x018B0, 0x018F5, - 0x01900, 0x0191E, - 0x01950, 0x0196D, - 0x01970, 0x01974, - 0x01980, 0x019AB, - 0x019B0, 0x019C9, - 0x01A00, 0x01A16, - 0x01A20, 0x01A54, - 0x01B05, 0x01B33, - 0x01B45, 0x01B4B, - 0x01B83, 0x01BA0, - 0x01BAE, 0x01BAF, - 0x01BBA, 0x01BE5, - 0x01C00, 0x01C23, - 0x01C4D, 0x01C4F, - 0x01C5A, 0x01C7D, - 0x01C80, 0x01C88, - 0x01C90, 0x01CBA, - 0x01CBD, 0x01CBF, - 0x01CE9, 0x01CEC, - 0x01CEE, 0x01CF3, - 0x01CF5, 0x01CF6, - 0x01D00, 0x01DBF, - 0x01E00, 0x01F15, - 0x01F18, 0x01F1D, - 0x01F20, 0x01F45, - 0x01F48, 0x01F4D, - 0x01F50, 0x01F57, - 0x01F5F, 0x01F7D, - 0x01F80, 0x01FB4, - 0x01FB6, 0x01FBC, - 0x01FC2, 0x01FC4, - 0x01FC6, 0x01FCC, - 0x01FD0, 0x01FD3, - 0x01FD6, 0x01FDB, - 0x01FE0, 0x01FEC, - 0x01FF2, 0x01FF4, - 0x01FF6, 0x01FFC, - 0x02090, 0x0209C, - 0x0210A, 0x02113, - 0x02119, 0x0211D, - 0x0212A, 0x0212D, - 0x0212F, 0x02139, - 0x0213C, 0x0213F, - 0x02145, 0x02149, - 0x02183, 0x02184, - 0x02C00, 0x02C2E, - 0x02C30, 0x02C5E, - 0x02C60, 0x02CE4, - 0x02CEB, 0x02CEE, - 0x02CF2, 0x02CF3, - 0x02D00, 0x02D25, - 0x02D30, 0x02D67, - 0x02D80, 0x02D96, - 0x02DA0, 0x02DA6, - 0x02DA8, 0x02DAE, - 0x02DB0, 0x02DB6, - 0x02DB8, 0x02DBE, - 0x02DC0, 0x02DC6, - 0x02DC8, 0x02DCE, - 0x02DD0, 0x02DD6, - 0x02DD8, 0x02DDE, - 0x03005, 0x03006, - 0x03031, 0x03035, - 0x0303B, 0x0303C, - 0x03041, 0x03096, - 0x0309D, 0x0309F, - 0x030A1, 0x030FA, - 0x030FC, 0x030FF, - 0x03105, 0x0312F, - 0x03131, 0x0318E, - 0x031A0, 0x031BA, - 0x031F0, 0x031FF, - 0x0A000, 0x0A48C, - 0x0A4D0, 0x0A4FD, - 0x0A500, 0x0A60C, - 0x0A610, 0x0A61F, - 0x0A62A, 0x0A62B, - 0x0A640, 0x0A66E, - 0x0A67F, 0x0A69D, - 0x0A6A0, 0x0A6E5, - 0x0A717, 0x0A71F, - 0x0A722, 0x0A788, - 0x0A78B, 0x0A7BF, - 0x0A7C2, 0x0A7C6, - 0x0A7F7, 0x0A801, - 0x0A803, 0x0A805, - 0x0A807, 0x0A80A, - 0x0A80C, 0x0A822, - 0x0A840, 0x0A873, - 0x0A882, 0x0A8B3, - 0x0A8F2, 0x0A8F7, - 0x0A8FD, 0x0A8FE, - 0x0A90A, 0x0A925, - 0x0A930, 0x0A946, - 0x0A960, 0x0A97C, - 0x0A984, 0x0A9B2, - 0x0A9E0, 0x0A9E4, - 0x0A9E6, 0x0A9EF, - 0x0A9FA, 0x0A9FE, - 0x0AA00, 0x0AA28, - 0x0AA40, 0x0AA42, - 0x0AA44, 0x0AA4B, - 0x0AA60, 0x0AA76, - 0x0AA7E, 0x0AAAF, - 0x0AAB5, 0x0AAB6, - 0x0AAB9, 0x0AABD, - 0x0AADB, 0x0AADD, - 0x0AAE0, 0x0AAEA, - 0x0AAF2, 0x0AAF4, - 0x0AB01, 0x0AB06, - 0x0AB09, 0x0AB0E, - 0x0AB11, 0x0AB16, - 0x0AB20, 0x0AB26, - 0x0AB28, 0x0AB2E, - 0x0AB30, 0x0AB5A, - 0x0AB5C, 0x0AB67, - 0x0AB70, 0x0ABE2, - 0x0D7B0, 0x0D7C6, - 0x0D7CB, 0x0D7FB, - 0x0F900, 0x0FA6D, - 0x0FA70, 0x0FAD9, - 0x0FB00, 0x0FB06, - 0x0FB13, 0x0FB17, - 0x0FB1F, 0x0FB28, - 0x0FB2A, 0x0FB36, - 0x0FB38, 0x0FB3C, - 0x0FB40, 0x0FB41, - 0x0FB43, 0x0FB44, - 0x0FB46, 0x0FBB1, - 0x0FBD3, 0x0FD3D, - 0x0FD50, 0x0FD8F, - 0x0FD92, 0x0FDC7, - 0x0FDF0, 0x0FDFB, - 0x0FE70, 0x0FE74, - 0x0FE76, 0x0FEFC, - 0x0FF21, 0x0FF3A, - 0x0FF41, 0x0FF5A, - 0x0FF66, 0x0FFBE, - 0x0FFC2, 0x0FFC7, - 0x0FFCA, 0x0FFCF, - 0x0FFD2, 0x0FFD7, - 0x0FFDA, 0x0FFDC, - 0x10000, 0x1000B, - 0x1000D, 0x10026, - 0x10028, 0x1003A, - 0x1003C, 0x1003D, - 0x1003F, 0x1004D, - 0x10050, 0x1005D, - 0x10080, 0x100FA, - 0x10280, 0x1029C, - 0x102A0, 0x102D0, - 0x10300, 0x1031F, - 0x1032D, 0x10340, - 0x10342, 0x10349, - 0x10350, 0x10375, - 0x10380, 0x1039D, - 0x103A0, 0x103C3, - 0x103C8, 0x103CF, - 0x10400, 0x1049D, - 0x104B0, 0x104D3, - 0x104D8, 0x104FB, - 0x10500, 0x10527, - 0x10530, 0x10563, - 0x10600, 0x10736, - 0x10740, 0x10755, - 0x10760, 0x10767, - 0x10800, 0x10805, - 0x1080A, 0x10835, - 0x10837, 0x10838, - 0x1083F, 0x10855, - 0x10860, 0x10876, - 0x10880, 0x1089E, - 0x108E0, 0x108F2, - 0x108F4, 0x108F5, - 0x10900, 0x10915, - 0x10920, 0x10939, - 0x10980, 0x109B7, - 0x109BE, 0x109BF, - 0x10A10, 0x10A13, - 0x10A15, 0x10A17, - 0x10A19, 0x10A35, - 0x10A60, 0x10A7C, - 0x10A80, 0x10A9C, - 0x10AC0, 0x10AC7, - 0x10AC9, 0x10AE4, - 0x10B00, 0x10B35, - 0x10B40, 0x10B55, - 0x10B60, 0x10B72, - 0x10B80, 0x10B91, - 0x10C00, 0x10C48, - 0x10C80, 0x10CB2, - 0x10CC0, 0x10CF2, - 0x10D00, 0x10D23, - 0x10F00, 0x10F1C, - 0x10F30, 0x10F45, - 0x10FE0, 0x10FF6, - 0x11003, 0x11037, - 0x11083, 0x110AF, - 0x110D0, 0x110E8, - 0x11103, 0x11126, - 0x11150, 0x11172, - 0x11183, 0x111B2, - 0x111C1, 0x111C4, - 0x11200, 0x11211, - 0x11213, 0x1122B, - 0x11280, 0x11286, - 0x1128A, 0x1128D, - 0x1128F, 0x1129D, - 0x1129F, 0x112A8, - 0x112B0, 0x112DE, - 0x11305, 0x1130C, - 0x1130F, 0x11310, - 0x11313, 0x11328, - 0x1132A, 0x11330, - 0x11332, 0x11333, - 0x11335, 0x11339, - 0x1135D, 0x11361, - 0x11400, 0x11434, - 0x11447, 0x1144A, - 0x11480, 0x114AF, - 0x114C4, 0x114C5, - 0x11580, 0x115AE, - 0x115D8, 0x115DB, - 0x11600, 0x1162F, - 0x11680, 0x116AA, - 0x11700, 0x1171A, - 0x11800, 0x1182B, - 0x118A0, 0x118DF, - 0x119A0, 0x119A7, - 0x119AA, 0x119D0, - 0x11A0B, 0x11A32, - 0x11A5C, 0x11A89, - 0x11AC0, 0x11AF8, - 0x11C00, 0x11C08, - 0x11C0A, 0x11C2E, - 0x11C72, 0x11C8F, - 0x11D00, 0x11D06, - 0x11D08, 0x11D09, - 0x11D0B, 0x11D30, - 0x11D60, 0x11D65, - 0x11D67, 0x11D68, - 0x11D6A, 0x11D89, - 0x11EE0, 0x11EF2, - 0x12000, 0x12399, - 0x12480, 0x12543, - 0x13000, 0x1342E, - 0x14400, 0x14646, - 0x16800, 0x16A38, - 0x16A40, 0x16A5E, - 0x16AD0, 0x16AED, - 0x16B00, 0x16B2F, - 0x16B40, 0x16B43, - 0x16B63, 0x16B77, - 0x16B7D, 0x16B8F, - 0x16E40, 0x16E7F, - 0x16F00, 0x16F4A, - 0x16F93, 0x16F9F, - 0x16FE0, 0x16FE1, - 0x18800, 0x18AF2, - 0x1B000, 0x1B11E, - 0x1B150, 0x1B152, - 0x1B164, 0x1B167, - 0x1B170, 0x1B2FB, - 0x1BC00, 0x1BC6A, - 0x1BC70, 0x1BC7C, - 0x1BC80, 0x1BC88, - 0x1BC90, 0x1BC99, - 0x1D400, 0x1D454, - 0x1D456, 0x1D49C, - 0x1D49E, 0x1D49F, - 0x1D4A5, 0x1D4A6, - 0x1D4A9, 0x1D4AC, - 0x1D4AE, 0x1D4B9, - 0x1D4BD, 0x1D4C3, - 0x1D4C5, 0x1D505, - 0x1D507, 0x1D50A, - 0x1D50D, 0x1D514, - 0x1D516, 0x1D51C, - 0x1D51E, 0x1D539, - 0x1D53B, 0x1D53E, - 0x1D540, 0x1D544, - 0x1D54A, 0x1D550, - 0x1D552, 0x1D6A5, - 0x1D6A8, 0x1D6C0, - 0x1D6C2, 0x1D6DA, - 0x1D6DC, 0x1D6FA, - 0x1D6FC, 0x1D714, - 0x1D716, 0x1D734, - 0x1D736, 0x1D74E, - 0x1D750, 0x1D76E, - 0x1D770, 0x1D788, - 0x1D78A, 0x1D7A8, - 0x1D7AA, 0x1D7C2, - 0x1D7C4, 0x1D7CB, - 0x1E100, 0x1E12C, - 0x1E137, 0x1E13D, - 0x1E2C0, 0x1E2EB, - 0x1E800, 0x1E8C4, - 0x1E900, 0x1E943, - 0x1EE00, 0x1EE03, - 0x1EE05, 0x1EE1F, - 0x1EE21, 0x1EE22, - 0x1EE29, 0x1EE32, - 0x1EE34, 0x1EE37, - 0x1EE4D, 0x1EE4F, - 0x1EE51, 0x1EE52, - 0x1EE61, 0x1EE62, - 0x1EE67, 0x1EE6A, - 0x1EE6C, 0x1EE72, - 0x1EE74, 0x1EE77, - 0x1EE79, 0x1EE7C, - 0x1EE80, 0x1EE89, - 0x1EE8B, 0x1EE9B, - 0x1EEA1, 0x1EEA3, - 0x1EEA5, 0x1EEA9, - 0x1EEAB, 0x1EEBB, - 0x2F800, 0x2FA1D, + 0x00041'i32, 0x0005A'i32, + 0x00061'i32, 0x0007A'i32, + 0x000C0'i32, 0x000D6'i32, + 0x000D8'i32, 0x000F6'i32, + 0x000F8'i32, 0x002C1'i32, + 0x002C6'i32, 0x002D1'i32, + 0x002E0'i32, 0x002E4'i32, + 0x00370'i32, 0x00374'i32, + 0x00376'i32, 0x00377'i32, + 0x0037A'i32, 0x0037D'i32, + 0x00388'i32, 0x0038A'i32, + 0x0038E'i32, 0x003A1'i32, + 0x003A3'i32, 0x003F5'i32, + 0x003F7'i32, 0x00481'i32, + 0x0048A'i32, 0x0052F'i32, + 0x00531'i32, 0x00556'i32, + 0x00560'i32, 0x00588'i32, + 0x005D0'i32, 0x005EA'i32, + 0x005EF'i32, 0x005F2'i32, + 0x00620'i32, 0x0064A'i32, + 0x0066E'i32, 0x0066F'i32, + 0x00671'i32, 0x006D3'i32, + 0x006E5'i32, 0x006E6'i32, + 0x006EE'i32, 0x006EF'i32, + 0x006FA'i32, 0x006FC'i32, + 0x00712'i32, 0x0072F'i32, + 0x0074D'i32, 0x007A5'i32, + 0x007CA'i32, 0x007EA'i32, + 0x007F4'i32, 0x007F5'i32, + 0x00800'i32, 0x00815'i32, + 0x00840'i32, 0x00858'i32, + 0x00860'i32, 0x0086A'i32, + 0x008A0'i32, 0x008B4'i32, + 0x008B6'i32, 0x008BD'i32, + 0x00904'i32, 0x00939'i32, + 0x00958'i32, 0x00961'i32, + 0x00971'i32, 0x00980'i32, + 0x00985'i32, 0x0098C'i32, + 0x0098F'i32, 0x00990'i32, + 0x00993'i32, 0x009A8'i32, + 0x009AA'i32, 0x009B0'i32, + 0x009B6'i32, 0x009B9'i32, + 0x009DC'i32, 0x009DD'i32, + 0x009DF'i32, 0x009E1'i32, + 0x009F0'i32, 0x009F1'i32, + 0x00A05'i32, 0x00A0A'i32, + 0x00A0F'i32, 0x00A10'i32, + 0x00A13'i32, 0x00A28'i32, + 0x00A2A'i32, 0x00A30'i32, + 0x00A32'i32, 0x00A33'i32, + 0x00A35'i32, 0x00A36'i32, + 0x00A38'i32, 0x00A39'i32, + 0x00A59'i32, 0x00A5C'i32, + 0x00A72'i32, 0x00A74'i32, + 0x00A85'i32, 0x00A8D'i32, + 0x00A8F'i32, 0x00A91'i32, + 0x00A93'i32, 0x00AA8'i32, + 0x00AAA'i32, 0x00AB0'i32, + 0x00AB2'i32, 0x00AB3'i32, + 0x00AB5'i32, 0x00AB9'i32, + 0x00AE0'i32, 0x00AE1'i32, + 0x00B05'i32, 0x00B0C'i32, + 0x00B0F'i32, 0x00B10'i32, + 0x00B13'i32, 0x00B28'i32, + 0x00B2A'i32, 0x00B30'i32, + 0x00B32'i32, 0x00B33'i32, + 0x00B35'i32, 0x00B39'i32, + 0x00B5C'i32, 0x00B5D'i32, + 0x00B5F'i32, 0x00B61'i32, + 0x00B85'i32, 0x00B8A'i32, + 0x00B8E'i32, 0x00B90'i32, + 0x00B92'i32, 0x00B95'i32, + 0x00B99'i32, 0x00B9A'i32, + 0x00B9E'i32, 0x00B9F'i32, + 0x00BA3'i32, 0x00BA4'i32, + 0x00BA8'i32, 0x00BAA'i32, + 0x00BAE'i32, 0x00BB9'i32, + 0x00C05'i32, 0x00C0C'i32, + 0x00C0E'i32, 0x00C10'i32, + 0x00C12'i32, 0x00C28'i32, + 0x00C2A'i32, 0x00C39'i32, + 0x00C58'i32, 0x00C5A'i32, + 0x00C60'i32, 0x00C61'i32, + 0x00C85'i32, 0x00C8C'i32, + 0x00C8E'i32, 0x00C90'i32, + 0x00C92'i32, 0x00CA8'i32, + 0x00CAA'i32, 0x00CB3'i32, + 0x00CB5'i32, 0x00CB9'i32, + 0x00CE0'i32, 0x00CE1'i32, + 0x00CF1'i32, 0x00CF2'i32, + 0x00D05'i32, 0x00D0C'i32, + 0x00D0E'i32, 0x00D10'i32, + 0x00D12'i32, 0x00D3A'i32, + 0x00D54'i32, 0x00D56'i32, + 0x00D5F'i32, 0x00D61'i32, + 0x00D7A'i32, 0x00D7F'i32, + 0x00D85'i32, 0x00D96'i32, + 0x00D9A'i32, 0x00DB1'i32, + 0x00DB3'i32, 0x00DBB'i32, + 0x00DC0'i32, 0x00DC6'i32, + 0x00E01'i32, 0x00E30'i32, + 0x00E32'i32, 0x00E33'i32, + 0x00E40'i32, 0x00E46'i32, + 0x00E81'i32, 0x00E82'i32, + 0x00E86'i32, 0x00E8A'i32, + 0x00E8C'i32, 0x00EA3'i32, + 0x00EA7'i32, 0x00EB0'i32, + 0x00EB2'i32, 0x00EB3'i32, + 0x00EC0'i32, 0x00EC4'i32, + 0x00EDC'i32, 0x00EDF'i32, + 0x00F40'i32, 0x00F47'i32, + 0x00F49'i32, 0x00F6C'i32, + 0x00F88'i32, 0x00F8C'i32, + 0x01000'i32, 0x0102A'i32, + 0x01050'i32, 0x01055'i32, + 0x0105A'i32, 0x0105D'i32, + 0x01065'i32, 0x01066'i32, + 0x0106E'i32, 0x01070'i32, + 0x01075'i32, 0x01081'i32, + 0x010A0'i32, 0x010C5'i32, + 0x010D0'i32, 0x010FA'i32, + 0x010FC'i32, 0x01248'i32, + 0x0124A'i32, 0x0124D'i32, + 0x01250'i32, 0x01256'i32, + 0x0125A'i32, 0x0125D'i32, + 0x01260'i32, 0x01288'i32, + 0x0128A'i32, 0x0128D'i32, + 0x01290'i32, 0x012B0'i32, + 0x012B2'i32, 0x012B5'i32, + 0x012B8'i32, 0x012BE'i32, + 0x012C2'i32, 0x012C5'i32, + 0x012C8'i32, 0x012D6'i32, + 0x012D8'i32, 0x01310'i32, + 0x01312'i32, 0x01315'i32, + 0x01318'i32, 0x0135A'i32, + 0x01380'i32, 0x0138F'i32, + 0x013A0'i32, 0x013F5'i32, + 0x013F8'i32, 0x013FD'i32, + 0x01401'i32, 0x0166C'i32, + 0x0166F'i32, 0x0167F'i32, + 0x01681'i32, 0x0169A'i32, + 0x016A0'i32, 0x016EA'i32, + 0x016F1'i32, 0x016F8'i32, + 0x01700'i32, 0x0170C'i32, + 0x0170E'i32, 0x01711'i32, + 0x01720'i32, 0x01731'i32, + 0x01740'i32, 0x01751'i32, + 0x01760'i32, 0x0176C'i32, + 0x0176E'i32, 0x01770'i32, + 0x01780'i32, 0x017B3'i32, + 0x01820'i32, 0x01878'i32, + 0x01880'i32, 0x01884'i32, + 0x01887'i32, 0x018A8'i32, + 0x018B0'i32, 0x018F5'i32, + 0x01900'i32, 0x0191E'i32, + 0x01950'i32, 0x0196D'i32, + 0x01970'i32, 0x01974'i32, + 0x01980'i32, 0x019AB'i32, + 0x019B0'i32, 0x019C9'i32, + 0x01A00'i32, 0x01A16'i32, + 0x01A20'i32, 0x01A54'i32, + 0x01B05'i32, 0x01B33'i32, + 0x01B45'i32, 0x01B4B'i32, + 0x01B83'i32, 0x01BA0'i32, + 0x01BAE'i32, 0x01BAF'i32, + 0x01BBA'i32, 0x01BE5'i32, + 0x01C00'i32, 0x01C23'i32, + 0x01C4D'i32, 0x01C4F'i32, + 0x01C5A'i32, 0x01C7D'i32, + 0x01C80'i32, 0x01C88'i32, + 0x01C90'i32, 0x01CBA'i32, + 0x01CBD'i32, 0x01CBF'i32, + 0x01CE9'i32, 0x01CEC'i32, + 0x01CEE'i32, 0x01CF3'i32, + 0x01CF5'i32, 0x01CF6'i32, + 0x01D00'i32, 0x01DBF'i32, + 0x01E00'i32, 0x01F15'i32, + 0x01F18'i32, 0x01F1D'i32, + 0x01F20'i32, 0x01F45'i32, + 0x01F48'i32, 0x01F4D'i32, + 0x01F50'i32, 0x01F57'i32, + 0x01F5F'i32, 0x01F7D'i32, + 0x01F80'i32, 0x01FB4'i32, + 0x01FB6'i32, 0x01FBC'i32, + 0x01FC2'i32, 0x01FC4'i32, + 0x01FC6'i32, 0x01FCC'i32, + 0x01FD0'i32, 0x01FD3'i32, + 0x01FD6'i32, 0x01FDB'i32, + 0x01FE0'i32, 0x01FEC'i32, + 0x01FF2'i32, 0x01FF4'i32, + 0x01FF6'i32, 0x01FFC'i32, + 0x02090'i32, 0x0209C'i32, + 0x0210A'i32, 0x02113'i32, + 0x02119'i32, 0x0211D'i32, + 0x0212A'i32, 0x0212D'i32, + 0x0212F'i32, 0x02139'i32, + 0x0213C'i32, 0x0213F'i32, + 0x02145'i32, 0x02149'i32, + 0x02183'i32, 0x02184'i32, + 0x02C00'i32, 0x02C2E'i32, + 0x02C30'i32, 0x02C5E'i32, + 0x02C60'i32, 0x02CE4'i32, + 0x02CEB'i32, 0x02CEE'i32, + 0x02CF2'i32, 0x02CF3'i32, + 0x02D00'i32, 0x02D25'i32, + 0x02D30'i32, 0x02D67'i32, + 0x02D80'i32, 0x02D96'i32, + 0x02DA0'i32, 0x02DA6'i32, + 0x02DA8'i32, 0x02DAE'i32, + 0x02DB0'i32, 0x02DB6'i32, + 0x02DB8'i32, 0x02DBE'i32, + 0x02DC0'i32, 0x02DC6'i32, + 0x02DC8'i32, 0x02DCE'i32, + 0x02DD0'i32, 0x02DD6'i32, + 0x02DD8'i32, 0x02DDE'i32, + 0x03005'i32, 0x03006'i32, + 0x03031'i32, 0x03035'i32, + 0x0303B'i32, 0x0303C'i32, + 0x03041'i32, 0x03096'i32, + 0x0309D'i32, 0x0309F'i32, + 0x030A1'i32, 0x030FA'i32, + 0x030FC'i32, 0x030FF'i32, + 0x03105'i32, 0x0312F'i32, + 0x03131'i32, 0x0318E'i32, + 0x031A0'i32, 0x031BA'i32, + 0x031F0'i32, 0x031FF'i32, + 0x03400'i32, 0x04DB5'i32, + 0x04E00'i32, 0x09FEF'i32, + 0x0A000'i32, 0x0A48C'i32, + 0x0A4D0'i32, 0x0A4FD'i32, + 0x0A500'i32, 0x0A60C'i32, + 0x0A610'i32, 0x0A61F'i32, + 0x0A62A'i32, 0x0A62B'i32, + 0x0A640'i32, 0x0A66E'i32, + 0x0A67F'i32, 0x0A69D'i32, + 0x0A6A0'i32, 0x0A6E5'i32, + 0x0A717'i32, 0x0A71F'i32, + 0x0A722'i32, 0x0A788'i32, + 0x0A78B'i32, 0x0A7BF'i32, + 0x0A7C2'i32, 0x0A7C6'i32, + 0x0A7F7'i32, 0x0A801'i32, + 0x0A803'i32, 0x0A805'i32, + 0x0A807'i32, 0x0A80A'i32, + 0x0A80C'i32, 0x0A822'i32, + 0x0A840'i32, 0x0A873'i32, + 0x0A882'i32, 0x0A8B3'i32, + 0x0A8F2'i32, 0x0A8F7'i32, + 0x0A8FD'i32, 0x0A8FE'i32, + 0x0A90A'i32, 0x0A925'i32, + 0x0A930'i32, 0x0A946'i32, + 0x0A960'i32, 0x0A97C'i32, + 0x0A984'i32, 0x0A9B2'i32, + 0x0A9E0'i32, 0x0A9E4'i32, + 0x0A9E6'i32, 0x0A9EF'i32, + 0x0A9FA'i32, 0x0A9FE'i32, + 0x0AA00'i32, 0x0AA28'i32, + 0x0AA40'i32, 0x0AA42'i32, + 0x0AA44'i32, 0x0AA4B'i32, + 0x0AA60'i32, 0x0AA76'i32, + 0x0AA7E'i32, 0x0AAAF'i32, + 0x0AAB5'i32, 0x0AAB6'i32, + 0x0AAB9'i32, 0x0AABD'i32, + 0x0AADB'i32, 0x0AADD'i32, + 0x0AAE0'i32, 0x0AAEA'i32, + 0x0AAF2'i32, 0x0AAF4'i32, + 0x0AB01'i32, 0x0AB06'i32, + 0x0AB09'i32, 0x0AB0E'i32, + 0x0AB11'i32, 0x0AB16'i32, + 0x0AB20'i32, 0x0AB26'i32, + 0x0AB28'i32, 0x0AB2E'i32, + 0x0AB30'i32, 0x0AB5A'i32, + 0x0AB5C'i32, 0x0AB67'i32, + 0x0AB70'i32, 0x0ABE2'i32, + 0x0AC00'i32, 0x0D7A3'i32, + 0x0D7B0'i32, 0x0D7C6'i32, + 0x0D7CB'i32, 0x0D7FB'i32, + 0x0F900'i32, 0x0FA6D'i32, + 0x0FA70'i32, 0x0FAD9'i32, + 0x0FB00'i32, 0x0FB06'i32, + 0x0FB13'i32, 0x0FB17'i32, + 0x0FB1F'i32, 0x0FB28'i32, + 0x0FB2A'i32, 0x0FB36'i32, + 0x0FB38'i32, 0x0FB3C'i32, + 0x0FB40'i32, 0x0FB41'i32, + 0x0FB43'i32, 0x0FB44'i32, + 0x0FB46'i32, 0x0FBB1'i32, + 0x0FBD3'i32, 0x0FD3D'i32, + 0x0FD50'i32, 0x0FD8F'i32, + 0x0FD92'i32, 0x0FDC7'i32, + 0x0FDF0'i32, 0x0FDFB'i32, + 0x0FE70'i32, 0x0FE74'i32, + 0x0FE76'i32, 0x0FEFC'i32, + 0x0FF21'i32, 0x0FF3A'i32, + 0x0FF41'i32, 0x0FF5A'i32, + 0x0FF66'i32, 0x0FFBE'i32, + 0x0FFC2'i32, 0x0FFC7'i32, + 0x0FFCA'i32, 0x0FFCF'i32, + 0x0FFD2'i32, 0x0FFD7'i32, + 0x0FFDA'i32, 0x0FFDC'i32, + 0x10000'i32, 0x1000B'i32, + 0x1000D'i32, 0x10026'i32, + 0x10028'i32, 0x1003A'i32, + 0x1003C'i32, 0x1003D'i32, + 0x1003F'i32, 0x1004D'i32, + 0x10050'i32, 0x1005D'i32, + 0x10080'i32, 0x100FA'i32, + 0x10280'i32, 0x1029C'i32, + 0x102A0'i32, 0x102D0'i32, + 0x10300'i32, 0x1031F'i32, + 0x1032D'i32, 0x10340'i32, + 0x10342'i32, 0x10349'i32, + 0x10350'i32, 0x10375'i32, + 0x10380'i32, 0x1039D'i32, + 0x103A0'i32, 0x103C3'i32, + 0x103C8'i32, 0x103CF'i32, + 0x10400'i32, 0x1049D'i32, + 0x104B0'i32, 0x104D3'i32, + 0x104D8'i32, 0x104FB'i32, + 0x10500'i32, 0x10527'i32, + 0x10530'i32, 0x10563'i32, + 0x10600'i32, 0x10736'i32, + 0x10740'i32, 0x10755'i32, + 0x10760'i32, 0x10767'i32, + 0x10800'i32, 0x10805'i32, + 0x1080A'i32, 0x10835'i32, + 0x10837'i32, 0x10838'i32, + 0x1083F'i32, 0x10855'i32, + 0x10860'i32, 0x10876'i32, + 0x10880'i32, 0x1089E'i32, + 0x108E0'i32, 0x108F2'i32, + 0x108F4'i32, 0x108F5'i32, + 0x10900'i32, 0x10915'i32, + 0x10920'i32, 0x10939'i32, + 0x10980'i32, 0x109B7'i32, + 0x109BE'i32, 0x109BF'i32, + 0x10A10'i32, 0x10A13'i32, + 0x10A15'i32, 0x10A17'i32, + 0x10A19'i32, 0x10A35'i32, + 0x10A60'i32, 0x10A7C'i32, + 0x10A80'i32, 0x10A9C'i32, + 0x10AC0'i32, 0x10AC7'i32, + 0x10AC9'i32, 0x10AE4'i32, + 0x10B00'i32, 0x10B35'i32, + 0x10B40'i32, 0x10B55'i32, + 0x10B60'i32, 0x10B72'i32, + 0x10B80'i32, 0x10B91'i32, + 0x10C00'i32, 0x10C48'i32, + 0x10C80'i32, 0x10CB2'i32, + 0x10CC0'i32, 0x10CF2'i32, + 0x10D00'i32, 0x10D23'i32, + 0x10F00'i32, 0x10F1C'i32, + 0x10F30'i32, 0x10F45'i32, + 0x10FE0'i32, 0x10FF6'i32, + 0x11003'i32, 0x11037'i32, + 0x11083'i32, 0x110AF'i32, + 0x110D0'i32, 0x110E8'i32, + 0x11103'i32, 0x11126'i32, + 0x11150'i32, 0x11172'i32, + 0x11183'i32, 0x111B2'i32, + 0x111C1'i32, 0x111C4'i32, + 0x11200'i32, 0x11211'i32, + 0x11213'i32, 0x1122B'i32, + 0x11280'i32, 0x11286'i32, + 0x1128A'i32, 0x1128D'i32, + 0x1128F'i32, 0x1129D'i32, + 0x1129F'i32, 0x112A8'i32, + 0x112B0'i32, 0x112DE'i32, + 0x11305'i32, 0x1130C'i32, + 0x1130F'i32, 0x11310'i32, + 0x11313'i32, 0x11328'i32, + 0x1132A'i32, 0x11330'i32, + 0x11332'i32, 0x11333'i32, + 0x11335'i32, 0x11339'i32, + 0x1135D'i32, 0x11361'i32, + 0x11400'i32, 0x11434'i32, + 0x11447'i32, 0x1144A'i32, + 0x11480'i32, 0x114AF'i32, + 0x114C4'i32, 0x114C5'i32, + 0x11580'i32, 0x115AE'i32, + 0x115D8'i32, 0x115DB'i32, + 0x11600'i32, 0x1162F'i32, + 0x11680'i32, 0x116AA'i32, + 0x11700'i32, 0x1171A'i32, + 0x11800'i32, 0x1182B'i32, + 0x118A0'i32, 0x118DF'i32, + 0x119A0'i32, 0x119A7'i32, + 0x119AA'i32, 0x119D0'i32, + 0x11A0B'i32, 0x11A32'i32, + 0x11A5C'i32, 0x11A89'i32, + 0x11AC0'i32, 0x11AF8'i32, + 0x11C00'i32, 0x11C08'i32, + 0x11C0A'i32, 0x11C2E'i32, + 0x11C72'i32, 0x11C8F'i32, + 0x11D00'i32, 0x11D06'i32, + 0x11D08'i32, 0x11D09'i32, + 0x11D0B'i32, 0x11D30'i32, + 0x11D60'i32, 0x11D65'i32, + 0x11D67'i32, 0x11D68'i32, + 0x11D6A'i32, 0x11D89'i32, + 0x11EE0'i32, 0x11EF2'i32, + 0x12000'i32, 0x12399'i32, + 0x12480'i32, 0x12543'i32, + 0x13000'i32, 0x1342E'i32, + 0x14400'i32, 0x14646'i32, + 0x16800'i32, 0x16A38'i32, + 0x16A40'i32, 0x16A5E'i32, + 0x16AD0'i32, 0x16AED'i32, + 0x16B00'i32, 0x16B2F'i32, + 0x16B40'i32, 0x16B43'i32, + 0x16B63'i32, 0x16B77'i32, + 0x16B7D'i32, 0x16B8F'i32, + 0x16E40'i32, 0x16E7F'i32, + 0x16F00'i32, 0x16F4A'i32, + 0x16F93'i32, 0x16F9F'i32, + 0x16FE0'i32, 0x16FE1'i32, + 0x17000'i32, 0x187F7'i32, + 0x18800'i32, 0x18AF2'i32, + 0x1B000'i32, 0x1B11E'i32, + 0x1B150'i32, 0x1B152'i32, + 0x1B164'i32, 0x1B167'i32, + 0x1B170'i32, 0x1B2FB'i32, + 0x1BC00'i32, 0x1BC6A'i32, + 0x1BC70'i32, 0x1BC7C'i32, + 0x1BC80'i32, 0x1BC88'i32, + 0x1BC90'i32, 0x1BC99'i32, + 0x1D400'i32, 0x1D454'i32, + 0x1D456'i32, 0x1D49C'i32, + 0x1D49E'i32, 0x1D49F'i32, + 0x1D4A5'i32, 0x1D4A6'i32, + 0x1D4A9'i32, 0x1D4AC'i32, + 0x1D4AE'i32, 0x1D4B9'i32, + 0x1D4BD'i32, 0x1D4C3'i32, + 0x1D4C5'i32, 0x1D505'i32, + 0x1D507'i32, 0x1D50A'i32, + 0x1D50D'i32, 0x1D514'i32, + 0x1D516'i32, 0x1D51C'i32, + 0x1D51E'i32, 0x1D539'i32, + 0x1D53B'i32, 0x1D53E'i32, + 0x1D540'i32, 0x1D544'i32, + 0x1D54A'i32, 0x1D550'i32, + 0x1D552'i32, 0x1D6A5'i32, + 0x1D6A8'i32, 0x1D6C0'i32, + 0x1D6C2'i32, 0x1D6DA'i32, + 0x1D6DC'i32, 0x1D6FA'i32, + 0x1D6FC'i32, 0x1D714'i32, + 0x1D716'i32, 0x1D734'i32, + 0x1D736'i32, 0x1D74E'i32, + 0x1D750'i32, 0x1D76E'i32, + 0x1D770'i32, 0x1D788'i32, + 0x1D78A'i32, 0x1D7A8'i32, + 0x1D7AA'i32, 0x1D7C2'i32, + 0x1D7C4'i32, 0x1D7CB'i32, + 0x1E100'i32, 0x1E12C'i32, + 0x1E137'i32, 0x1E13D'i32, + 0x1E2C0'i32, 0x1E2EB'i32, + 0x1E800'i32, 0x1E8C4'i32, + 0x1E900'i32, 0x1E943'i32, + 0x1EE00'i32, 0x1EE03'i32, + 0x1EE05'i32, 0x1EE1F'i32, + 0x1EE21'i32, 0x1EE22'i32, + 0x1EE29'i32, 0x1EE32'i32, + 0x1EE34'i32, 0x1EE37'i32, + 0x1EE4D'i32, 0x1EE4F'i32, + 0x1EE51'i32, 0x1EE52'i32, + 0x1EE61'i32, 0x1EE62'i32, + 0x1EE67'i32, 0x1EE6A'i32, + 0x1EE6C'i32, 0x1EE72'i32, + 0x1EE74'i32, 0x1EE77'i32, + 0x1EE79'i32, 0x1EE7C'i32, + 0x1EE80'i32, 0x1EE89'i32, + 0x1EE8B'i32, 0x1EE9B'i32, + 0x1EEA1'i32, 0x1EEA3'i32, + 0x1EEA5'i32, 0x1EEA9'i32, + 0x1EEAB'i32, 0x1EEBB'i32, + 0x20000'i32, 0x2A6D6'i32, + 0x2A700'i32, 0x2B734'i32, + 0x2B740'i32, 0x2B81D'i32, + 0x2B820'i32, 0x2CEA1'i32, + 0x2CEB0'i32, 0x2EBE0'i32, + 0x2F800'i32, 0x2FA1D'i32, ] alphaSinglets = [ - 0x000AA, - 0x000B5, - 0x000BA, - 0x002EC, - 0x002EE, - 0x0037F, - 0x00386, - 0x0038C, - 0x00559, - 0x006D5, - 0x006FF, - 0x00710, - 0x007B1, - 0x007FA, - 0x0081A, - 0x00824, - 0x00828, - 0x0093D, - 0x00950, - 0x009B2, - 0x009BD, - 0x009CE, - 0x009FC, - 0x00A5E, - 0x00ABD, - 0x00AD0, - 0x00AF9, - 0x00B3D, - 0x00B71, - 0x00B83, - 0x00B9C, - 0x00BD0, - 0x00C3D, - 0x00C80, - 0x00CBD, - 0x00CDE, - 0x00D3D, - 0x00D4E, - 0x00DBD, - 0x00E84, - 0x00EA5, - 0x00EBD, - 0x00EC6, - 0x00F00, - 0x0103F, - 0x01061, - 0x0108E, - 0x010C7, - 0x010CD, - 0x01258, - 0x012C0, - 0x017D7, - 0x017DC, - 0x018AA, - 0x01AA7, - 0x01CFA, - 0x01F59, - 0x01F5B, - 0x01F5D, - 0x01FBE, - 0x02071, - 0x0207F, - 0x02102, - 0x02107, - 0x02115, - 0x02124, - 0x02126, - 0x02128, - 0x0214E, - 0x02D27, - 0x02D2D, - 0x02D6F, - 0x02E2F, - 0x03400, - 0x04DB5, - 0x04E00, - 0x09FEF, - 0x0A8FB, - 0x0A9CF, - 0x0AA7A, - 0x0AAB1, - 0x0AAC0, - 0x0AAC2, - 0x0AC00, - 0x0D7A3, - 0x0FB1D, - 0x0FB3E, - 0x10808, - 0x1083C, - 0x10A00, - 0x10F27, - 0x11144, - 0x11176, - 0x111DA, - 0x111DC, - 0x11288, - 0x1133D, - 0x11350, - 0x1145F, - 0x114C7, - 0x11644, - 0x116B8, - 0x118FF, - 0x119E1, - 0x119E3, - 0x11A00, - 0x11A3A, - 0x11A50, - 0x11A9D, - 0x11C40, - 0x11D46, - 0x11D98, - 0x16F50, - 0x16FE3, - 0x17000, - 0x187F7, - 0x1D4A2, - 0x1D4BB, - 0x1D546, - 0x1E14E, - 0x1E94B, - 0x1EE24, - 0x1EE27, - 0x1EE39, - 0x1EE3B, - 0x1EE42, - 0x1EE47, - 0x1EE49, - 0x1EE4B, - 0x1EE54, - 0x1EE57, - 0x1EE59, - 0x1EE5B, - 0x1EE5D, - 0x1EE5F, - 0x1EE64, - 0x1EE7E, - 0x20000, - 0x2A6D6, - 0x2A700, - 0x2B734, - 0x2B740, - 0x2B81D, - 0x2B820, - 0x2CEA1, - 0x2CEB0, - 0x2EBE0, + 0x000AA'i32, + 0x000B5'i32, + 0x000BA'i32, + 0x002EC'i32, + 0x002EE'i32, + 0x0037F'i32, + 0x00386'i32, + 0x0038C'i32, + 0x00559'i32, + 0x006D5'i32, + 0x006FF'i32, + 0x00710'i32, + 0x007B1'i32, + 0x007FA'i32, + 0x0081A'i32, + 0x00824'i32, + 0x00828'i32, + 0x0093D'i32, + 0x00950'i32, + 0x009B2'i32, + 0x009BD'i32, + 0x009CE'i32, + 0x009FC'i32, + 0x00A5E'i32, + 0x00ABD'i32, + 0x00AD0'i32, + 0x00AF9'i32, + 0x00B3D'i32, + 0x00B71'i32, + 0x00B83'i32, + 0x00B9C'i32, + 0x00BD0'i32, + 0x00C3D'i32, + 0x00C80'i32, + 0x00CBD'i32, + 0x00CDE'i32, + 0x00D3D'i32, + 0x00D4E'i32, + 0x00DBD'i32, + 0x00E84'i32, + 0x00EA5'i32, + 0x00EBD'i32, + 0x00EC6'i32, + 0x00F00'i32, + 0x0103F'i32, + 0x01061'i32, + 0x0108E'i32, + 0x010C7'i32, + 0x010CD'i32, + 0x01258'i32, + 0x012C0'i32, + 0x017D7'i32, + 0x017DC'i32, + 0x018AA'i32, + 0x01AA7'i32, + 0x01CFA'i32, + 0x01F59'i32, + 0x01F5B'i32, + 0x01F5D'i32, + 0x01FBE'i32, + 0x02071'i32, + 0x0207F'i32, + 0x02102'i32, + 0x02107'i32, + 0x02115'i32, + 0x02124'i32, + 0x02126'i32, + 0x02128'i32, + 0x0214E'i32, + 0x02D27'i32, + 0x02D2D'i32, + 0x02D6F'i32, + 0x02E2F'i32, + 0x0A8FB'i32, + 0x0A9CF'i32, + 0x0AA7A'i32, + 0x0AAB1'i32, + 0x0AAC0'i32, + 0x0AAC2'i32, + 0x0FB1D'i32, + 0x0FB3E'i32, + 0x10808'i32, + 0x1083C'i32, + 0x10A00'i32, + 0x10F27'i32, + 0x11144'i32, + 0x11176'i32, + 0x111DA'i32, + 0x111DC'i32, + 0x11288'i32, + 0x1133D'i32, + 0x11350'i32, + 0x1145F'i32, + 0x114C7'i32, + 0x11644'i32, + 0x116B8'i32, + 0x118FF'i32, + 0x119E1'i32, + 0x119E3'i32, + 0x11A00'i32, + 0x11A3A'i32, + 0x11A50'i32, + 0x11A9D'i32, + 0x11C40'i32, + 0x11D46'i32, + 0x11D98'i32, + 0x16F50'i32, + 0x16FE3'i32, + 0x1D4A2'i32, + 0x1D4BB'i32, + 0x1D546'i32, + 0x1E14E'i32, + 0x1E94B'i32, + 0x1EE24'i32, + 0x1EE27'i32, + 0x1EE39'i32, + 0x1EE3B'i32, + 0x1EE42'i32, + 0x1EE47'i32, + 0x1EE49'i32, + 0x1EE4B'i32, + 0x1EE54'i32, + 0x1EE57'i32, + 0x1EE59'i32, + 0x1EE5B'i32, + 0x1EE5D'i32, + 0x1EE5F'i32, + 0x1EE64'i32, + 0x1EE7E'i32, ] spaceRanges = [ - 0x00009, 0x0000D, - 0x00020, 0x00020, - 0x00085, 0x00085, - 0x000A0, 0x000A0, - 0x01680, 0x01680, - 0x02000, 0x0200A, - 0x02028, 0x02029, - 0x0202F, 0x0202F, - 0x0205F, 0x0205F, - 0x03000, 0x03000, + 0x00009'i32, 0x0000D'i32, + 0x00020'i32, 0x00020'i32, + 0x00085'i32, 0x00085'i32, + 0x000A0'i32, 0x000A0'i32, + 0x01680'i32, 0x01680'i32, + 0x02000'i32, 0x0200A'i32, + 0x02028'i32, 0x02029'i32, + 0x0202F'i32, 0x0202F'i32, + 0x0205F'i32, 0x0205F'i32, + 0x03000'i32, 0x03000'i32, ] unicodeSpaces = [ diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index b62b4c2db6..10658b78ef 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -9,7 +9,7 @@ # This module implements Linux epoll(). -import posix, times, epoll +import std/[posix, times, epoll] # Maximum number of events that can be returned const MAX_EPOLL_EVENTS = 64 @@ -72,14 +72,16 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = + proc initialNumFD(): int {.inline.} = + when defined(nuttx): + result = NEPOLL_MAX + else: + result = 1024 # Retrieve the maximum fd count (for current OS) via getrlimit() - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseOSError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand - const numFD = 1024 + let numFD = initialNumFD() var epollFD = epoll_create1(O_CLOEXEC) if epollFD < 0: @@ -136,7 +138,7 @@ template checkFd(s, f) = var numFD = s.numFD while numFD <= f: numFD *= 2 when hasThreadSupport: - s.fds = reallocSharedArray(s.fds, numFD) + s.fds = reallocSharedArray(s.fds, s.numFD, numFD) else: s.fds.setLen(numFD) for i in s.numFD ..< numFD: @@ -529,4 +531,4 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2 proc getFd*[T](s: Selector[T]): int = - return s.epollFd.int + return s.epollFD.int diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 68be969c7f..e28218a97f 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -9,7 +9,7 @@ # This module implements BSD kqueue(). -import posix, times, kqueue, nativesockets +import std/[posix, times, kqueue, nativesockets] const # Maximum number of events that can be returned. diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 6b10e19ae0..7c53471563 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -9,11 +9,17 @@ # This module implements Posix poll(). -import posix, times +import std/[posix, times] # Maximum number of events that can be returned const MAX_POLL_EVENTS = 64 +const hasEventFds = defined(zephyr) or defined(nimPollHasEventFds) + +when hasEventFds: + proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "".} + when hasThreadSupport: type SelectorImpl[T] = object @@ -53,10 +59,7 @@ else: body proc newSelector*[T](): Selector[T] = - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseIOSelectorsError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() when hasThreadSupport: result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) @@ -187,14 +190,22 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = s.pollRemove(fdi.cint) proc newSelectEvent*(): SelectEvent = - var fds: array[2, cint] - if posix.pipe(fds) != 0: - raiseIOSelectorsError(osLastError()) - setNonBlocking(fds[0]) - setNonBlocking(fds[1]) - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rfd = fds[0] - result.wfd = fds[1] + when not hasEventFds: + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + else: + let fdci = eventfd(0, posix.O_NONBLOCK) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fdci + result.wfd = fdci proc trigger*(ev: SelectEvent) = var data: uint64 = 1 @@ -203,7 +214,10 @@ proc trigger*(ev: SelectEvent) = proc close*(ev: SelectEvent) = let res1 = posix.close(ev.rfd) - let res2 = posix.close(ev.wfd) + let res2 = + when hasEventFds: 0 + else: posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) if res1 != 0 or res2 != 0: raiseIOSelectorsError(osLastError()) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 6f216ac859..11bc62b78d 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -9,10 +9,10 @@ # This module implements Posix and Windows select(). -import times, nativesockets +import std/[times, nativesockets] when defined(windows): - import winlean + import std/winlean when defined(gcc): {.passl: "-lws2_32".} elif defined(vcc): @@ -26,27 +26,27 @@ else: #include #include """ type - Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object + FdSet {.importc: "fd_set", header: platformHeaders, pure, final.} = object var FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint -proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_SET(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} -proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_CLR(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} -proc IOFD_ZERO(fdset: ptr Fdset) +proc IOFD_ZERO(fdset: ptr FdSet) {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} when defined(windows): - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.stdcall, importc: "select", header: platformHeaders.} else: - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.cdecl, importc: "select", header: platformHeaders.} @@ -313,7 +313,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode) or defined(freertos): + when defined(genode) or defined(freertos) or defined(zephyr) or defined(nuttx): tv.tv_sec = Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 @@ -398,18 +398,6 @@ proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = if s.fds[i].ident == fdi: return true -when hasThreadSupport: - template withSelectLock[T](s: Selector[T], body: untyped) = - acquire(s.lock) - {.locks: [s.lock].}: - try: - body - finally: - release(s.lock) -else: - template withSelectLock[T](s: Selector[T], body: untyped) = - body - proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = s.withSelectLock(): let fdi = int(fd) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 13cd3fb854..485b8918c6 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -41,13 +41,14 @@ ## For a `JsonNode` who's kind is `JObject`, you can access its fields using ## the `[]` operator. The following example shows how to do this: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode.kind == JObject ## doAssert jsonNode["key"].kind == JFloat +## ``` ## ## Reading values ## -------------- @@ -62,12 +63,13 @@ ## ## To retrieve the value of `"key"` you can do the following: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode["key"].getFloat() == 3.14 +## ``` ## ## **Important:** The `[]` operator will raise an exception when the ## specified field does not exist. @@ -79,7 +81,7 @@ ## when the field is not found. The `get`-family of procedures will return a ## type's default value when called on `nil`. ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("{}") @@ -88,6 +90,7 @@ ## doAssert jsonNode{"nope"}.getFloat() == 0 ## doAssert jsonNode{"nope"}.getStr() == "" ## doAssert jsonNode{"nope"}.getBool() == false +## ``` ## ## Using default values ## -------------------- @@ -95,7 +98,7 @@ ## The `get`-family helpers also accept an additional parameter which allow ## you to fallback to a default value should the key's values be `null`: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14, "key2": null}""") @@ -103,6 +106,7 @@ ## doAssert jsonNode["key"].getFloat(6.28) == 3.14 ## doAssert jsonNode["key2"].getFloat(3.14) == 3.14 ## doAssert jsonNode{"nope"}.getFloat(3.14) == 3.14 # note the {} +## ``` ## ## Unmarshalling ## ------------- @@ -113,7 +117,7 @@ ## Note: Use `Option `_ for keys sometimes missing in json ## responses, and backticks around keys with a reserved keyword as name. ## -## .. code-block:: Nim +## ```Nim ## import std/json ## import std/options ## @@ -127,6 +131,7 @@ ## let user = to(userJson, User) ## if user.`type`.isSome(): ## assert user.`type`.get() != "robot" +## ``` ## ## Creating JSON ## ============= @@ -134,7 +139,7 @@ ## This module can also be used to comfortably create JSON using the `%*` ## operator: ## -## .. code-block:: nim +## ```nim ## import std/json ## ## var hisName = "John" @@ -148,6 +153,7 @@ ## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} ## j2["details"] = %* {"age":35, "pi":3.1415} ## echo j2 +## ``` ## ## See also: std/jsonutils for hookable json serialization/deserialization ## of arbitrary types. @@ -159,12 +165,14 @@ runnableExamples: a1, a2, a0, a3, a4: int doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" -import - std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] +import std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] import std/options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 import std/private/since +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, formatfloat] + export tables.`$` @@ -203,6 +211,8 @@ type of JArray: elems*: seq[JsonNode] +const DepthLimit = 1000 + proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. result = JsonNode(kind: JString, str: s) @@ -214,10 +224,6 @@ proc newJRawNumber(s: string): JsonNode = ## to the string representation without the quotes. result = JsonNode(kind: JString, str: s, isUnquoted: true) -proc newJStringMove(s: string): JsonNode = - result = JsonNode(kind: JString) - shallowCopy(result.str, s) - proc newJInt*(n: BiggestInt): JsonNode = ## Creates a new `JInt JsonNode`. result = JsonNode(kind: JInt, num: n) @@ -334,7 +340,16 @@ proc `%`*(n: BiggestInt): JsonNode = proc `%`*(n: float): JsonNode = ## Generic constructor for JSON data. Creates a new `JFloat JsonNode`. - result = JsonNode(kind: JFloat, fnum: n) + runnableExamples: + assert $(%[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2]) == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert (%NaN).kind == JString + assert (%0.0).kind == JFloat + # for those special cases, we could also have used `newJRawNumber` but then + # it would've been inconsisten with the case of `parseJson` vs `%` for representing them. + if n != n: newJString("nan") + elif n == Inf: newJString("inf") + elif n == -Inf: newJString("-inf") + else: JsonNode(kind: JFloat, fnum: n) proc `%`*(b: bool): JsonNode = ## Generic constructor for JSON data. Creates a new `JBool JsonNode`. @@ -398,7 +413,7 @@ proc `%`*(o: enum): JsonNode = ## string. Creates a new `JString JsonNode`. result = %($o) -proc toJsonImpl(x: NimNode): NimNode {.compileTime.} = +proc toJsonImpl(x: NimNode): NimNode = case x.kind of nnkBracket: # array if x.len == 0: return newCall(bindSym"newJArray") @@ -429,7 +444,7 @@ macro `%*`*(x: untyped): untyped = ## `%` for every element. result = toJsonImpl(x) -proc `==`*(a, b: JsonNode): bool = +proc `==`*(a, b: JsonNode): bool {.noSideEffect, raises: [].} = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -449,19 +464,25 @@ proc `==`*(a, b: JsonNode): bool = of JNull: result = true of JArray: - result = a.elems == b.elems + {.cast(raises: []).}: # bug #19303 + result = a.elems == b.elems of JObject: # we cannot use OrderedTable's equality here as # the order does not matter for equality here. if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - if b.fields[key] != val: return false + {.cast(raises: []).}: + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: + if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} -proc hash*(n: JsonNode): Hash = +proc hash*(n: JsonNode): Hash {.noSideEffect.} = ## Compute the hash for a JSON node case n.kind of JArray: @@ -523,6 +544,24 @@ proc `[]`*(node: JsonNode, index: BackwardsIndex): JsonNode {.inline, since: (1, `[]`(node, node.len - int(index)) +proc `[]`*[U, V](a: JsonNode, x: HSlice[U, V]): JsonNode = + ## Slice operation for JArray. + ## + ## Returns the inclusive range `[a[x.a], a[x.b]]`: + runnableExamples: + import std/json + let arr = %[0,1,2,3,4,5] + doAssert arr[2..4] == %[2,3,4] + doAssert arr[2..^2] == %[2,3,4] + doAssert arr[^4..^2] == %[2,3,4] + + assert(a.kind == JArray) + result = newJArray() + let xa = (when x.a is BackwardsIndex: a.len - int(x.a) else: int(x.a)) + let L = (when x.b is BackwardsIndex: a.len - int(x.b) else: int(x.b)) - xa + 1 + for i in 0.. DepthLimit: + raiseParseErr(p, "}") result = newJObject() discard getTok(p) while p.tok != tkCurlyRi: @@ -866,16 +908,18 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = var key = p.a discard getTok(p) eat(p, tkColon) - var val = parseJson(p, rawIntegers, rawFloats) + var val = parseJson(p, rawIntegers, rawFloats, depth+1) result[key] = val if p.tok != tkComma: break discard getTok(p) eat(p, tkCurlyRi) of tkBracketLe: + if depth > DepthLimit: + raiseParseErr(p, "]") result = newJArray() discard getTok(p) while p.tok != tkBracketRi: - result.add(parseJson(p, rawIntegers, rawFloats)) + result.add(parseJson(p, rawIntegers, rawFloats, depth+1)) if p.tok != tkComma: break discard getTok(p) eat(p, tkBracketRi) @@ -922,21 +966,22 @@ proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats when defined(js): from std/math import `mod` - from std/jsffi import JSObject, `[]`, to + from std/jsffi import JsObject, `[]`, to from std/private/jsutils import getProtoName, isInteger, isSafeInteger - proc parseNativeJson(x: cstring): JSObject {.importjs: "JSON.parse(#)".} + proc parseNativeJson(x: cstring): JsObject {.importjs: "JSON.parse(#)".} - proc getVarType(x: JSObject): JsonNodeKind = + proc getVarType(x: JsObject, isRawNumber: var bool): JsonNodeKind = result = JNull case $getProtoName(x) # TODO: Implicit returns fail here. of "[object Array]": return JArray of "[object Object]": return JObject of "[object Number]": - if isInteger(x): + if isInteger(x) and 1.0 / cast[float](x) != -Inf: # preserve -0.0 as float if isSafeInteger(x): return JInt else: + isRawNumber = true return JString else: return JFloat @@ -945,35 +990,41 @@ when defined(js): of "[object String]": return JString else: assert false - proc len(x: JSObject): int = - assert x.getVarType == JArray - asm """ + proc len(x: JsObject): int = + {.emit: """ `result` = `x`.length; - """ + """.} - proc convertObject(x: JSObject): JsonNode = - case getVarType(x) + proc convertObject(x: JsObject): JsonNode = + var isRawNumber = false + case getVarType(x, isRawNumber) of JArray: result = newJArray() for i in 0 ..< x.len: result.add(x[i].convertObject()) of JObject: result = newJObject() - asm """for (var property in `x`) { + {.emit: """for (var property in `x`) { if (`x`.hasOwnProperty(property)) { - """ + """.} + var nimProperty: cstring - var nimValue: JSObject - asm "`nimProperty` = property; `nimValue` = `x`[property];" + var nimValue: JsObject + {.emit: "`nimProperty` = property; `nimValue` = `x`[property];".} result[$nimProperty] = nimValue.convertObject() - asm "}}" + {.emit: "}}".} of JInt: result = newJInt(x.to(int)) of JFloat: result = newJFloat(x.to(float)) of JString: # Dunno what to do with isUnquoted here - result = newJString($x.to(cstring)) + if isRawNumber: + var value: cstring + {.emit: "`value` = `x`.toString();".} + result = newJRawNumber($value) + else: + result = newJString($x.to(cstring)) of JBool: result = newJBool(x.to(bool)) of JNull: @@ -1017,297 +1068,306 @@ template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ] raise newException(JsonKindError, msg) -when defined(nimFixedForwardGeneric): +macro isRefSkipDistinct*(arg: typed): untyped = + ## internal only, do not use + var impl = getTypeImpl(arg) + if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): + impl = getTypeImpl(impl[1]) + while impl.kind == nnkDistinctTy: + impl = getTypeImpl(impl[0]) + result = newLit(impl.kind == nnkRefTy) - macro isRefSkipDistinct*(arg: typed): untyped = - ## internal only, do not use - var impl = getTypeImpl(arg) - if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): - impl = getTypeImpl(impl[1]) - while impl.kind == nnkDistinctTy: - impl = getTypeImpl(impl[0]) - result = newLit(impl.kind == nnkRefTy) +# The following forward declarations don't work in older versions of Nim - # The following forward declarations don't work in older versions of Nim +# forward declare all initFromJson - # forward declare all initFromJson +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) +# initFromJson definitions - # initFromJson definitions +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) + # since strings don't have a nil state anymore, this mapping of + # JNull to the default string is questionable. `none(string)` and + # `some("")` have the same potentional json value `JNull`. + if jsonNode.kind == JNull: + dst = "" + else: + dst = jsonNode.str - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) - # since strings don't have a nil state anymore, this mapping of - # JNull to the default string is questionable. `none(string)` and - # `some("")` have the same potentional json value `JNull`. - if jsonNode.kind == JNull: - dst = "" +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JBool}, jsonPath) + dst = jsonNode.bval + +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode == nil: + raise newException(KeyError, "key not found: " & jsonPath) + dst = jsonNode.copy + +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = + when T is uint|uint64 or int.sizeof == 4: + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) + case jsonNode.kind + of JString: + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) else: - dst = jsonNode.str + dst = T(jsonNode.num) + else: + verifyJsonKind(jsonNode, {JInt}, jsonPath) + dst = cast[T](jsonNode.num) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JBool}, jsonPath) - dst = jsonNode.bval - - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode == nil: - raise newException(KeyError, "key not found: " & jsonPath) - dst = jsonNode.copy - - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = - when T is uint|uint64: - case jsonNode.kind - of JString: - dst = T(parseBiggestUInt(jsonNode.str)) - else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = T(jsonNode.num) - else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = cast[T](jsonNode.num) - - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JInt, JFloat, JString}, jsonPath) + if jsonNode.kind == JString: + case jsonNode.str + of "nan": + let b = NaN + dst = T(b) + # dst = NaN # would fail some tests because range conversions would cause CT error + # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. + of "inf": + let b = Inf + dst = T(b) + of "-inf": + let b = -Inf + dst = T(b) + else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) + else: if jsonNode.kind == JFloat: dst = T(jsonNode.fnum) else: dst = T(jsonNode.num) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString}, jsonPath) - dst = parseEnum[T](jsonNode.getStr) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString}, jsonPath) + dst = parseEnum[T](jsonNode.getStr) - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - dst.setLen jsonNode.len - let orignalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i], jsonNode[i], jsonPath) - jsonPath.setLen orignalJsonPathLen +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + dst.setLen jsonNode.len + let orignalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i], jsonNode[i], jsonPath) + jsonPath.setLen orignalJsonPathLen - proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - let originalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays - jsonPath.setLen originalJsonPathLen +proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays + jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initTable[string, T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen +proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initTable[string, T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initOrderedTable[string,T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen +proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initOrderedTable[string,T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) - if jsonNode.kind == JNull: - dst = nil +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) + if jsonNode.kind == JNull: + dst = nil + else: + dst = new(T) + initFromJson(dst[], jsonNode, jsonPath) + +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode != nil and jsonNode.kind != JNull: + when T is ref: + dst = some(new(T)) else: - dst = new(T) - initFromJson(dst[], jsonNode, jsonPath) + dst = some(default(T)) + initFromJson(dst.get, jsonNode, jsonPath) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode != nil and jsonNode.kind != JNull: - when T is ref: - dst = some(new(T)) - else: - dst = some(default(T)) - initFromJson(dst.get, jsonNode, jsonPath) +macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = + let typInst = getTypeInst(dst) + let typImpl = getTypeImpl(dst) + let baseTyp = typImpl[0] - macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = - let typInst = getTypeInst(dst) - let typImpl = getTypeImpl(dst) - let baseTyp = typImpl[0] + result = quote do: + initFromJson(`baseTyp`(`dst`), `jsonNode`, `jsonPath`) - result = quote do: +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignDistinctImpl(dst, jsonNode, jsonPath) + +proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = + if typeExpr.kind == nnkTupleConstr: + error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) + +proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = + case typeNode.kind + of nnkEmpty: + discard + of nnkRecList, nnkTupleTy: + for it in typeNode: + foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkIdentDefs: + typeNode.expectLen 3 + let fieldSym = typeNode[0] + let fieldNameLit = newLit(fieldSym.strVal) + let fieldPathLit = newLit("." & fieldSym.strVal) + let fieldType = typeNode[1] + + # Detecting incompatiple tuple types in `assignObjectImpl` only + # would be much cleaner, but the ast for tuple types does not + # contain usable type information. + detectIncompatibleType(fieldType, fieldSym) + + dst.add quote do: + jsonPath.add `fieldPathLit` when nimvm: - # workaround #12282 - var tmp: `baseTyp` - initFromJson( tmp, `jsonNode`, `jsonPath`) - `dst` = `typInst`(tmp) - else: - initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignDistinctImpl(dst, jsonNode, jsonPath) - - proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode): void = - if typeExpr.kind == nnkTupleConstr: - error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - - proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode): void {.compileTime.} = - case typeNode.kind - of nnkEmpty: - discard - of nnkRecList, nnkTupleTy: - for it in typeNode: - foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkIdentDefs: - typeNode.expectLen 3 - let fieldSym = typeNode[0] - let fieldNameLit = newLit(fieldSym.strVal) - let fieldPathLit = newLit("." & fieldSym.strVal) - let fieldType = typeNode[1] - - # Detecting incompatiple tuple types in `assignObjectImpl` only - # would be much cleaner, but the ast for tuple types does not - # contain usable type information. - detectIncompatibleType(fieldType, fieldSym) - - dst.add quote do: - jsonPath.add `fieldPathLit` - when nimvm: - when isRefSkipDistinct(`tmpSym`.`fieldSym`): - # workaround #12489 - var tmp: `fieldType` - initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - `tmpSym`.`fieldSym` = tmp - else: - initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + when isRefSkipDistinct(`tmpSym`.`fieldSym`): + # workaround #12489 + var tmp: `fieldType` + initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + `tmpSym`.`fieldSym` = tmp else: initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - jsonPath.setLen `originalJsonPathLen` + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + jsonPath.setLen `originalJsonPathLen` - of nnkRecCase: - let kindSym = typeNode[0][0] - let kindNameLit = newLit(kindSym.strVal) - let kindPathLit = newLit("." & kindSym.strVal) - let kindType = typeNode[0][1] - let kindOffsetLit = newLit(uint(getOffset(kindSym))) - dst.add quote do: - var kindTmp: `kindType` - jsonPath.add `kindPathLit` - initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - when defined js: + of nnkRecCase: + let kindSym = typeNode[0][0] + let kindNameLit = newLit(kindSym.strVal) + let kindPathLit = newLit("." & kindSym.strVal) + let kindType = typeNode[0][1] + let kindOffsetLit = newLit(uint(getOffset(kindSym))) + dst.add quote do: + var kindTmp: `kindType` + jsonPath.add `kindPathLit` + initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + when defined js: + `tmpSym`.`kindSym` = kindTmp + else: + when nimvm: `tmpSym`.`kindSym` = kindTmp else: - when nimvm: - `tmpSym`.`kindSym` = kindTmp - else: - # fuck it, assign kind field anyway - ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp - dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) - for i in 1 ..< typeNode.len: - foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # fuck it, assign kind field anyway + ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp + dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) + for i in 1 ..< typeNode.len: + foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - of nnkOfBranch, nnkElse: - let ofBranch = newNimNode(typeNode.kind) - for i in 0 ..< typeNode.len-1: - ofBranch.add copyNimTree(typeNode[i]) - let dstInner = newNimNode(nnkStmtListExpr) - foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - # resOuter now contains the inner stmtList - ofBranch.add dstInner - dst[^1].expectKind nnkCaseStmt - dst[^1].add ofBranch + of nnkOfBranch, nnkElse: + let ofBranch = newNimNode(typeNode.kind) + for i in 0 ..< typeNode.len-1: + ofBranch.add copyNimTree(typeNode[i]) + let dstInner = newNimNode(nnkStmtListExpr) + foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # resOuter now contains the inner stmtList + ofBranch.add dstInner + dst[^1].expectKind nnkCaseStmt + dst[^1].add ofBranch - of nnkObjectTy: - typeNode[0].expectKind nnkEmpty - typeNode[1].expectKind {nnkEmpty, nnkOfInherit} - if typeNode[1].kind == nnkOfInherit: - let base = typeNode[1][0] - var impl = getTypeImpl(base) - while impl.kind in {nnkRefTy, nnkPtrTy}: - impl = getTypeImpl(impl[0]) - foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - let body = typeNode[2] - foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + of nnkObjectTy: + typeNode[0].expectKind nnkEmpty + typeNode[1].expectKind {nnkEmpty, nnkOfInherit} + if typeNode[1].kind == nnkOfInherit: + let base = typeNode[1][0] + var impl = getTypeImpl(base) + while impl.kind in {nnkRefTy, nnkPtrTy}: + impl = getTypeImpl(impl[0]) + foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + let body = typeNode[2] + foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - else: - error("unhandled kind: " & $typeNode.kind, typeNode) + else: + error("unhandled kind: " & $typeNode.kind, typeNode) - macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - let typeSym = getTypeInst(dst) - let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") - result = newStmtList() - result.add quote do: - let `originalJsonPathLen` = len(`jsonPath`) - if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: - # both, `dst` and `typeSym` don't have good lineinfo. But nothing - # else is available here. - detectIncompatibleType(typeSym, dst) - foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) - else: - foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) +macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + let typeSym = getTypeInst(dst) + let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") + result = newStmtList() + result.add quote do: + let `originalJsonPathLen` = len(`jsonPath`) + if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: + # both, `dst` and `typeSym` don't have good lineinfo. But nothing + # else is available here. + detectIncompatibleType(typeSym, dst) + foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) + else: + foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignObjectImpl(dst, jsonNode, jsonPath) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignObjectImpl(dst, jsonNode, jsonPath) - proc to*[T](node: JsonNode, t: typedesc[T]): T = - ## `Unmarshals`:idx: the specified node into the object type specified. - ## - ## Known limitations: - ## - ## * Heterogeneous arrays are not supported. - ## * Sets in object variants are not supported. - ## * Not nil annotations are not supported. - ## - runnableExamples: - let jsonNode = parseJson(""" - { - "person": { - "name": "Nimmer", - "age": 21 - }, - "list": [1, 2, 3, 4] - } - """) +proc to*[T](node: JsonNode, t: typedesc[T]): T = + ## `Unmarshals`:idx: the specified node into the object type specified. + ## + ## Known limitations: + ## + ## * Heterogeneous arrays are not supported. + ## * Sets in object variants are not supported. + ## * Not nil annotations are not supported. + ## + runnableExamples: + let jsonNode = parseJson(""" + { + "person": { + "name": "Nimmer", + "age": 21 + }, + "list": [1, 2, 3, 4] + } + """) - type - Person = object - name: string - age: int + type + Person = object + name: string + age: int - Data = object - person: Person - list: seq[int] + Data = object + person: Person + list: seq[int] - var data = to(jsonNode, Data) - doAssert data.person.name == "Nimmer" - doAssert data.person.age == 21 - doAssert data.list == @[1, 2, 3, 4] + var data = to(jsonNode, Data) + doAssert data.person.name == "Nimmer" + doAssert data.person.age == 21 + doAssert data.list == @[1, 2, 3, 4] - var jsonPath = "" - initFromJson(result, node, jsonPath) + var jsonPath = "" + result = default(T) + initFromJson(result, node, jsonPath) when false: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: JsonParser diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 166d8286b6..1b6b2b3a2c 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2009 Andreas Rumpf # # See the file "copying.txt", included in this @@ -12,7 +12,10 @@ ## needs refilling. import - strutils, streams + std/[strutils, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions const EndOfFile* = '\0' ## end of file marker diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index b2ace79ab7..e3f0240a29 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -17,10 +17,11 @@ ## ## To get started, first create a logger: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var logger = newConsoleLogger() +## ``` ## ## The logger that was created above logs to the console, but this module ## also provides loggers that log to files, such as the @@ -30,9 +31,10 @@ ## Once a logger has been created, call its `log proc ## <#log.e,ConsoleLogger,Level,varargs[string,]>`_ to log a message: ## -## .. code-block:: +## ```Nim ## logger.log(lvlInfo, "a log message") ## # Output: INFO a log message +## ``` ## ## The ``INFO`` within the output is the result of a format string being ## prepended to the message, and it will differ depending on the message's @@ -47,9 +49,8 @@ ## ## .. warning:: ## For loggers that log to a console or to files, only error and fatal -## messages will cause their output buffers to be flushed immediately. -## Use the `flushFile proc `_ to flush the buffer -## manually if needed. +## messages will cause their output buffers to be flushed immediately by default. +## set ``flushThreshold`` when creating the logger to change this. ## ## Handlers ## -------- @@ -59,7 +60,7 @@ ## used with the `addHandler proc<#addHandler,Logger>`_, which is demonstrated ## in the following example: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var consoleLog = newConsoleLogger() @@ -69,17 +70,19 @@ ## addHandler(consoleLog) ## addHandler(fileLog) ## addHandler(rollingLog) +## ``` ## ## After doing this, use either the `log template ## <#log.t,Level,varargs[string,]>`_ or one of the level-specific templates, ## such as the `error template<#error.t,varargs[string,]>`_, to log messages ## to all registered handlers at once. ## -## .. code-block:: +## ```Nim ## # This example uses the loggers created above ## log(lvlError, "an error occurred") ## error("an error occurred") # Equivalent to the above line ## info("something normal happened") # Will not be written to errors.log +## ``` ## ## Note that a message's level is still checked against each handler's ## ``levelThreshold`` and the global log filter. @@ -117,12 +120,13 @@ ## ## The following example illustrates how to use format strings: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var logger = newConsoleLogger(fmtStr="[$time] - $levelname: ") ## logger.log(lvlInfo, "this is a message") ## # Output: [19:50:13] - INFO: this is a message +## ``` ## ## Notes when using multiple threads ## --------------------------------- @@ -142,9 +146,12 @@ ## * `strscans module`_ for ``scanf`` and ``scanp`` macros, which ## offer easier substring extraction than regular expressions -import strutils, times +import std/[strutils, times] when not defined(js): - import os + import std/os + +when defined(nimPreviewSlimSystem): + import std/syncio type Level* = enum ## \ @@ -197,6 +204,15 @@ const ## If a different format string is preferred, refer to the ## `documentation about format strings<#basic-usage-format-strings>`_ ## for more information, including a list of available variables. + defaultFlushThreshold = when NimMajor >= 2: + when defined(nimV1LogFlushBehavior): lvlError else: lvlAll + else: + when defined(nimFlushAllLogs): lvlAll else: lvlError + ## The threshold above which log messages to file-like loggers + ## are automatically flushed. + ## + ## By default, only error and fatal messages are logged, + ## but defining ``-d:nimFlushAllLogs`` will make all levels be flushed type Logger* = ref object of RootObj @@ -225,6 +241,8 @@ type ## * `FileLogger<#FileLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ useStderr*: bool ## If true, writes to stderr; otherwise, writes to stdout + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately when not defined(js): type @@ -240,6 +258,8 @@ when not defined(js): ## * `ConsoleLogger<#ConsoleLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ file*: File ## The wrapped file + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately RollingFileLogger* = ref object of FileLogger ## A logger that writes log messages to a file while performing log @@ -345,8 +365,8 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## `setLogFilter proc<#setLogFilter,Level>`_. ## ## **Note:** Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## ## See also: ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ @@ -357,14 +377,15 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var consoleLog = newConsoleLogger() ## consoleLog.log(lvlInfo, "this is a message") ## consoleLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: let ln = substituteLog(logger.fmtStr, level, args) when defined(js): - let cln: cstring = ln + let cln = ln.cstring case level of lvlDebug: {.emit: "console.debug(`cln`);".} of lvlInfo: {.emit: "console.info(`cln`);".} @@ -377,12 +398,12 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = if logger.useStderr: handle = stderr writeLine(handle, ln) - if level in {lvlError, lvlFatal}: flushFile(handle) + if level >= logger.flushThreshold: flushFile(handle) except IOError: discard proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, - useStderr = false): ConsoleLogger = + useStderr = false, flushThreshold = defaultFlushThreshold): ConsoleLogger = ## Creates a new `ConsoleLogger<#ConsoleLogger>`_. ## ## By default, log messages are written to ``stdout``. If ``useStderr`` is @@ -399,13 +420,15 @@ proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newConsoleLogger() ## var formatLog = newConsoleLogger(fmtStr=verboseFmtStr) ## var errorLog = newConsoleLogger(levelThreshold=lvlError, useStderr=true) + ## ``` new result result.fmtStr = fmtStr result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.useStderr = useStderr when not defined(js): @@ -421,8 +444,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -434,13 +457,14 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var fileLog = newFileLogger("messages.log") ## fileLog.log(lvlInfo, "this is a message") ## fileLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) proc defaultFilename*(): string = ## Returns the filename that is used by default when naming log files. @@ -451,7 +475,8 @@ when not defined(js): proc newFileLogger*(file: File, levelThreshold = lvlAll, - fmtStr = defaultFmtStr): FileLogger = + fmtStr = defaultFmtStr, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that uses the given file handle. ## ## **Note:** This proc is not available for the JavaScript backend. @@ -464,7 +489,7 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var messages = open("messages.log", fmWrite) ## var formatted = open("formatted.log", fmWrite) ## var errors = open("errors.log", fmWrite) @@ -472,16 +497,19 @@ when not defined(js): ## var normalLog = newFileLogger(messages) ## var formatLog = newFileLogger(formatted, fmtStr=verboseFmtStr) ## var errorLog = newFileLogger(errors, levelThreshold=lvlError) + ## ``` new(result) result.file = file result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.fmtStr = fmtStr proc newFileLogger*(filename = defaultFilename(), mode: FileMode = fmAppend, levelThreshold = lvlAll, fmtStr = defaultFmtStr, - bufSize: int = -1): FileLogger = + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that logs to a file with the ## given filename. ## @@ -500,12 +528,13 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newFileLogger("messages.log") ## var formatLog = newFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var errorLog = newFileLogger("errors.log", levelThreshold=lvlError) + ## ``` let file = open(filename, mode, bufSize = bufSize) - newFileLogger(file, levelThreshold, fmtStr) + newFileLogger(file, levelThreshold, fmtStr, flushThreshold) # ------ @@ -537,7 +566,8 @@ when not defined(js): levelThreshold = lvlAll, fmtStr = defaultFmtStr, maxLines: Positive = 1000, - bufSize: int = -1): RollingFileLogger = + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): RollingFileLogger = ## Creates a new `RollingFileLogger<#RollingFileLogger>`_. ## ## Once the current log file being written to contains ``maxLines`` lines, @@ -559,11 +589,12 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newRollingFileLogger("messages.log") ## var formatLog = newRollingFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var shortLog = newRollingFileLogger("short.log", maxLines=200) ## var errorLog = newRollingFileLogger("errors.log", levelThreshold=lvlError) + ## ``` new(result) result.levelThreshold = levelThreshold result.fmtStr = fmtStr @@ -573,6 +604,7 @@ when not defined(js): result.curLine = 0 result.baseName = filename result.baseMode = mode + result.flushThreshold = flushThreshold result.logFiles = countFiles(filename) @@ -599,8 +631,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -612,10 +644,11 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var rollingLog = newRollingFileLogger("messages.log") ## rollingLog.log(lvlInfo, "this is a message") ## rollingLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: if logger.curLine >= logger.maxLines: logger.file.close() @@ -626,7 +659,7 @@ when not defined(js): bufSize = logger.bufSize) writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) logger.curLine.inc # -------- @@ -645,11 +678,12 @@ template log*(level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## log(lvlInfo, "This is an example.") + ## ``` ## ## See also: ## * `debug template<#debug.t,varargs[string,]>`_ @@ -674,11 +708,12 @@ template debug*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## debug("myProc called with arguments: foo, 5") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -695,11 +730,12 @@ template info*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## info("Application started successfully.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -716,11 +752,12 @@ template notice*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## notice("An important operation has completed.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -736,11 +773,12 @@ template warn*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## warn("The previous operation took too long to process.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -758,11 +796,12 @@ template error*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## error("An exception occurred while processing the form.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -779,11 +818,12 @@ template fatal*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## fatal("Can't open database -- exiting.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 936b8fe94d..f9b3d3e4ca 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -56,6 +56,9 @@ Please contribute a new implementation.""".} import std/[streams, typeinfo, json, intsets, tables, unicode] +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + proc ptrToInt(x: pointer): int {.inline.} = result = cast[int](x) # don't skip alignment @@ -163,7 +166,10 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akSequence: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + invokeNewSeq(a, 0) + else: + setPointer(a, nil) next(p) of jsonArrayStart: next(p) @@ -204,7 +210,8 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = setPointer(a, nil) next(p) of jsonInt: - setPointer(a, t.getOrDefault(p.getInt)) + var raw = t.getOrDefault(p.getInt) + setPointer(a, addr raw) next(p) of jsonArrayStart: next(p) @@ -230,7 +237,10 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akString: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + setString(a, "") + else: + setPointer(a, nil) next(p) of jsonString: setString(a, p.str) @@ -282,7 +292,7 @@ proc load*[T](s: Stream, data: var T) = var tab = initTable[BiggestInt, pointer]() loadAny(s, toAny(data), tab) -proc store*[T](s: Stream, data: T) = +proc store*[T](s: Stream, data: sink T) = ## Stores `data` into the stream `s`. Raises `IOError` in case of an error. runnableExamples: import std/streams @@ -295,10 +305,16 @@ proc store*[T](s: Stream, data: T) = var stored = initIntSet() var d: T - shallowCopy(d, data) + when defined(gcArc) or defined(gcOrc)or defined(gcAtomicArc): + d = data + else: + shallowCopy(d, data) storeAny(s, toAny(d), stored) -proc `$$`*[T](x: T): string = +proc loadVM[T](typ: typedesc[T], x: T): string = + discard "the implementation is in the compiler/vmops" + +proc `$$`*[T](x: sink T): string = ## Returns a string representation of `x` (serialization, marshalling). ## ## **Note:** to serialize `x` to JSON use `%x` from the `json` module @@ -313,12 +329,21 @@ proc `$$`*[T](x: T): string = let y = $$x assert y == """{"id": 1, "bar": "baz"}""" - var stored = initIntSet() - var d: T - shallowCopy(d, x) - var s = newStringStream() - storeAny(s, toAny(d), stored) - result = s.data + when nimvm: + result = loadVM(T, x) + else: + var stored = initIntSet() + var d: T + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + d = x + else: + shallowCopy(d, x) + var s = newStringStream() + storeAny(s, toAny(d), stored) + result = s.data + +proc toVM[T](typ: typedesc[T], data: string): T = + discard "the implementation is in the compiler/vmops" proc to*[T](data: string): T = ## Reads data and transforms it to a type `T` (deserialization, unmarshalling). @@ -335,5 +360,8 @@ proc to*[T](data: string): T = assert z.id == 1 assert z.bar == "baz" - var tab = initTable[BiggestInt, pointer]() - loadAny(newStringStream(data), toAny(result), tab) + when nimvm: + result = toVM(T, data) + else: + var tab = initTable[BiggestInt, pointer]() + loadAny(newStringStream(data), toAny(result), tab) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 74a98e655b..73a60e9744 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -47,7 +47,6 @@ runnableExamples: ## * `fenv module `_ for handling of floating-point rounding ## and exceptions (overflow, zero-divide, etc.) ## * `random module `_ for a fast and tiny random number generator -## * `mersenne module `_ for the Mersenne Twister random number generator ## * `stats module `_ for statistical analysis ## * `strformat module `_ for formatting floats for printing ## * `system module `_ for some very basic and trivial math operators @@ -60,6 +59,10 @@ import std/private/since import std/[bitops, fenv] +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(c) or defined(cpp): proc c_isnan(x: float): bool {.importc: "isnan", header: "".} # a generic like `x: SomeFloat` might work too if this is implemented via a C macro. @@ -69,16 +72,46 @@ when defined(c) or defined(cpp): proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "".} - func c_frexp*(x: cfloat, exponent: var cint): cfloat {. - importc: "frexpf", header: "", deprecated: "Use `frexp` instead".} - func c_frexp*(x: cdouble, exponent: var cint): cdouble {. - importc: "frexp", header: "", deprecated: "Use `frexp` instead".} - # don't export `c_frexp` in the future and remove `c_frexp2`. func c_frexp2(x: cfloat, exponent: var cint): cfloat {. importc: "frexpf", header: "".} func c_frexp2(x: cdouble, exponent: var cint): cdouble {. importc: "frexp", header: "".} + + type + div_t {.importc, header: "".} = object + quot: cint + rem: cint + ldiv_t {.importc, header: "".} = object + quot: clong + rem: clong + lldiv_t {.importc, header: "".} = object + quot: clonglong + rem: clonglong + + when cint isnot clong: + func divmod_c(x, y: cint): div_t {.importc: "div", header: "".} + when clong isnot clonglong: + func divmod_c(x, y: clonglong): lldiv_t {.importc: "lldiv", header: "".} + func divmod_c(x, y: clong): ldiv_t {.importc: "ldiv", header: "".} + func divmod*[T: SomeInteger](x, y: T): (T, T) {.inline.} = + ## Specialized instructions for computing both division and modulus. + ## Return structure is: (quotient, remainder) + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + when T is cint | clong | clonglong: + when compileOption("overflowChecks"): + if y == 0: + raise new(DivByZeroDefect) + elif (x == T.low and y == -1.T): + raise new(OverflowDefect) + let res = divmod_c(x, y) + result[0] = res.quot + result[1] = res.rem + else: + result[0] = x div y + result[1] = x mod y func binom*(n, k: int): int = ## Computes the [binomial coefficient](https://en.wikipedia.org/wiki/Binomial_coefficient). @@ -122,7 +155,7 @@ func fac*(n: int): int = {.push checks: off, line_dir: off, stack_trace: off.} -when defined(posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} const @@ -167,7 +200,7 @@ func isNaN*(x: SomeFloat): bool {.inline, since: (1,5,1).} = template fn: untyped = result = x != x when nimvm: fn() else: - when defined(js): fn() + when defined(js) or defined(nimscript): fn() else: result = c_isnan(x) when defined(js): @@ -186,13 +219,13 @@ when defined(js): let a = newFloat64Array(buffer) let b = newUint32Array(buffer) a[0] = x - asm """ + {.emit: """ function updateBit(num, bitPos, bitVal) { return (num & ~(1 << bitPos)) | (bitVal << bitPos); } `b`[1] = updateBit(`b`[1], 31, `sgn`); `result` = `a`[0] - """ + """.} proc signbit*(x: SomeFloat): bool {.inline, since: (1, 5, 1).} = ## Returns true if `x` is negative, false otherwise. @@ -237,7 +270,6 @@ func classify*(x: float): FloatClass = ## Classifies a floating point value. ## ## Returns `x`'s class as specified by the `FloatClass enum<#FloatClass>`_. - ## Doesn't work with `--passc:-ffast-math`. runnableExamples: doAssert classify(0.3) == fcNormal doAssert classify(0.0) == fcZero @@ -246,6 +278,7 @@ func classify*(x: float): FloatClass = doAssert classify(5.0e-324) == fcSubnormal # JavaScript and most C compilers have no classify: + if isNan(x): return fcNan if x == 0.0: if 1.0 / x == Inf: return fcZero @@ -254,7 +287,6 @@ func classify*(x: float): FloatClass = if x * 0.5 == x: if x > 0.0: return fcInf else: return fcNegInf - if x != x: return fcNan if abs(x) < MinFloatNormal: return fcSubnormal return fcNormal @@ -328,68 +360,8 @@ func nextPowerOfTwo*(x: int): int = result = result or (result shr 1) result += 1 + ord(x <= 0) -func sum*[T](x: openArray[T]): T = - ## Computes the sum of the elements in `x`. - ## - ## If `x` is empty, 0 is returned. - ## - ## **See also:** - ## * `prod func <#prod,openArray[T]>`_ - ## * `cumsum func <#cumsum,openArray[T]>`_ - ## * `cumsummed func <#cumsummed,openArray[T]>`_ - runnableExamples: - doAssert sum([1, 2, 3, 4]) == 10 - doAssert sum([-4, 3, 5]) == 4 - for i in items(x): result = result + i -func prod*[T](x: openArray[T]): T = - ## Computes the product of the elements in `x`. - ## - ## If `x` is empty, 1 is returned. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `fac func <#fac,int>`_ - runnableExamples: - doAssert prod([1, 2, 3, 4]) == 24 - doAssert prod([-4, 3, 5]) == -60 - - result = T(1) - for i in items(x): result = result * i - -func cumsummed*[T](x: openArray[T]): seq[T] = - ## Returns the cumulative (aka prefix) summation of `x`. - ## - ## If `x` is empty, `@[]` is returned. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version - runnableExamples: - doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] - - let xLen = x.len - if xLen == 0: - return @[] - result.setLen(xLen) - result[0] = x[0] - for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] - -func cumsum*[T](x: var openArray[T]) = - ## Transforms `x` in-place (must be declared as `var`) into its - ## cumulative (aka prefix) summation. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which - ## returns a cumsummed sequence - runnableExamples: - var a = [1, 2, 3, 4] - cumsum(a) - doAssert a == @[1, 3, 6, 10] - - for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] when not defined(js): # C func sqrt*(x: float32): float32 {.importc: "sqrtf", header: "".} @@ -855,6 +827,14 @@ else: # JS doAssert -6.5 mod 2.5 == -1.5 doAssert 6.5 mod -2.5 == 1.5 doAssert -6.5 mod -2.5 == -1.5 + + func divmod*[T:SomeInteger](num, denom: T): (T, T) = + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + result[0] = num div denom + result[1] = num mod denom + func round*[T: float32|float64](x: T, places: int): T = ## Decimal rounding on a binary floating point number. @@ -941,6 +921,58 @@ func euclMod*[T: SomeNumber](x, y: T): T {.since: (1, 5, 1).} = if result < 0: result += abs(y) +func ceilDiv*[T: SomeInteger](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Ceil division is conceptually defined as `ceil(x / y)`. + ## + ## Assumes `x >= 0` and `y > 0` (and `x + y - 1 <= high(T)` if T is SomeUnsignedInt). + ## + ## This is different from the `system.div `_ + ## operator, which works like `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `ceilDiv` rounds up. + ## + ## This function has the above input limitation, because that allows the + ## compiler to generate faster code and it is rarely used with + ## negative values or unsigned integers close to `high(T)/2`. + ## If you need a `ceilDiv` that works with any input, see: + ## https://github.com/demotomohiro/divmath. + ## + ## **See also:** + ## * `system.div proc `_ for integer division + ## * `floorDiv func <#floorDiv,T,T>`_ for integer division which rounds down. + runnableExamples: + assert ceilDiv(12, 3) == 4 + assert ceilDiv(13, 3) == 5 + + when sizeof(T) == 8: + type UT = uint64 + elif sizeof(T) == 4: + type UT = uint32 + elif sizeof(T) == 2: + type UT = uint16 + elif sizeof(T) == 1: + type UT = uint8 + else: + {.fatal: "Unsupported int type".} + + assert x >= 0 and y > 0 + when T is SomeUnsignedInt: + assert x + y - 1 >= x + + # If the divisor is const, the backend C/C++ compiler generates code without a `div` + # instruction, as it is slow on most CPUs. + # If the divisor is a power of 2 and a const unsigned integer type, the + # compiler generates faster code. + # If the divisor is const and a signed integer, generated code becomes slower + # than the code with unsigned integers, because division with signed integers + # need to works for both positive and negative value without `idiv`/`sdiv`. + # That is why this code convert parameters to unsigned. + # This post contains a comparison of the performance of signed/unsigned integers: + # https://github.com/nim-lang/Nim/pull/18596#issuecomment-894420984. + # If signed integer arguments were not converted to unsigned integers, + # `ceilDiv` wouldn't work for any positive signed integer value, because + # `x + (y - 1)` can overflow. + ((x.UT + (y.UT - 1.UT)) div y.UT).T + func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = ## Splits `x` into a normalized fraction `frac` and an integral power of 2 `exp`, ## such that `abs(frac) in 0.5..<1` and `x == frac * 2 ^ exp`, except for special @@ -949,18 +981,26 @@ func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = doAssert frexp(8.0) == (0.5, 4) doAssert frexp(-8.0) == (-0.5, 4) doAssert frexp(0.0) == (0.0, 0) + # special cases: - when not defined(windows): - doAssert frexp(-0.0) == (-0.0, 0) # signbit preserved for +-0 + when sizeof(int) == 8: + doAssert frexp(-0.0).frac.signbit # signbit preserved for +-0 doAssert frexp(Inf).frac == Inf # +- Inf preserved doAssert frexp(NaN).frac.isNaN + when not defined(js): var exp: cint result.frac = c_frexp2(x, exp) result.exp = exp else: if x == 0.0: - result = (0.0, 0) + # reuse signbit implementation + let uintBuffer = toBitsImpl(x) + if (uintBuffer[1] shr 31) != 0: + # x is -0.0 + result = (-0.0, 0) + else: + result = (0.0, 0) elif x < 0.0: result = frexp(-x) result.frac = -result.frac @@ -980,6 +1020,7 @@ func frexp*[T: float32|float64](x: T, exponent: var int): T {.inline.} = var x: int doAssert frexp(5.0, x) == 0.625 doAssert x == 3 + (result, exponent) = frexp(x) @@ -988,7 +1029,7 @@ when not defined(js): # taken from Go-lang Math.Log2 const ln2 = 0.693147180559945309417232121458176568075500134360255254120680009 template log2Impl[T](x: T): T = - var exp: int32 + var exp: int var frac = frexp(x, exp) # Make sure exact powers of two give an exact answer. # Don't depend on Log(0.5)*(1/Ln2)+exp being exactly exp-1. @@ -1074,6 +1115,69 @@ func sgn*[T: SomeNumber](x: T): int {.inline.} = {.pop.} {.pop.} +func sum*[T](x: openArray[T]): T = + ## Computes the sum of the elements in `x`. + ## + ## If `x` is empty, 0 is returned. + ## + ## **See also:** + ## * `prod func <#prod,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ + runnableExamples: + doAssert sum([1, 2, 3, 4]) == 10 + doAssert sum([-4, 3, 5]) == 4 + + for i in items(x): result = result + i + +func prod*[T](x: openArray[T]): T = + ## Computes the product of the elements in `x`. + ## + ## If `x` is empty, 1 is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `fac func <#fac,int>`_ + runnableExamples: + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([-4, 3, 5]) == -60 + + result = T(1) + for i in items(x): result = result * i + +func cumsummed*[T](x: openArray[T]): seq[T] = + ## Returns the cumulative (aka prefix) summation of `x`. + ## + ## If `x` is empty, `@[]` is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version + runnableExamples: + doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] + + let xLen = x.len + if xLen == 0: + return @[] + result.setLen(xLen) + result[0] = x[0] + for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] + +func cumsum*[T](x: var openArray[T]) = + ## Transforms `x` in-place (must be declared as `var`) into its + ## cumulative (aka prefix) summation. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which + ## returns a cumsummed sequence + runnableExamples: + var a = [1, 2, 3, 4] + cumsum(a) + doAssert a == @[1, 3, 6, 10] + + for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] + func `^`*[T: SomeNumber](x: T, y: Natural): T = ## Computes `x` to the power of `y`. ## diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 11c3245484..c65a9c2daf 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -9,15 +9,17 @@ ## Module for computing [MD5 checksums](https://en.wikipedia.org/wiki/MD5). ## -## **Note:** The procs in this module can be used at compile time. +## This module also works at compile time and in JavaScript. ## ## See also ## ======== -## * `base64 module`_ implements a Base64 encoder and decoder -## * `std/sha1 module `_ for a SHA-1 encoder and decoder +## * `base64 module`_ for a Base64 encoder and decoder +## * `sha1 module `_ for the SHA-1 checksum algorithm ## * `hashes module`_ for efficient computations of hash values ## for diverse Nim types +{.deprecated: "use command `nimble install checksums` and import `checksums/md5` instead".} + when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -34,15 +36,16 @@ type buffer: MD5Buffer const - padding: cstring = "\x80\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0" + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] proc F(x, y, z: uint32): uint32 {.inline.} = result = (x and y) or ((not x) and z) @@ -79,7 +82,7 @@ proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = rot(a, s) a = a + b -proc encode(dest: var MD5Block, src: cstring) = +proc encode(dest: var MD5Block, src: openArray[uint8]) = var j = 0 for i in 0..high(dest): dest[i] = uint32(ord(src[j])) or @@ -97,10 +100,48 @@ proc decode(dest: var openArray[uint8], src: openArray[uint32]) = dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) inc(i, 4) -proc transform(buffer: pointer, state: var MD5State) = +template slice(s: string, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + var s2 = newSeq[uint8](s.len) + for i in 0 ..< s2.len: + s2[i] = uint8(s[i]) + s2 + else: + s.toOpenArrayByte(a, b) + +template slice(s: cstring, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + slice($s, a, b) + else: + when defined(js): + # toOpenArrayByte for cstring is not implemented in JS + slice($s, a, b) + else: + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + when nimvm: + s[a .. b] + else: + s.toOpenArray(a, b) + +const useMem = declared(copyMem) + +template memOrNot(withMem, withoutMem): untyped = + when nimvm: + withoutMem + else: + when useMem: + withMem + else: + withoutMem + +proc transform(buffer: openArray[uint8], state: var MD5State) = var myBlock: MD5Block - encode(myBlock, cast[cstring](buffer)) + encode(myBlock, buffer) var a = state[0] var b = state[1] var c = state[2] @@ -175,10 +216,18 @@ proc transform(buffer: pointer, state: var MD5State) = state[3] = state[3] + d proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], tags: [], gcsafe.} proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) + proc toMD5*(s: string): MD5Digest = ## Computes the `MD5Digest` value for a string `s`. @@ -192,7 +241,7 @@ proc toMD5*(s: string): MD5Digest = var c: MD5Context md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, result) proc `$`*(d: MD5Digest): string = @@ -215,7 +264,7 @@ proc getMD5*(s: string): string = c: MD5Context d: MD5Digest md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, d) result = $d @@ -226,6 +275,12 @@ proc `==`*(D1, D2: MD5Digest): bool = return true +proc clearBuffer(c: var MD5Context) {.inline.} = + memOrNot: + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + do: + reset(c.buffer) + proc md5Init*(c: var MD5Context) = ## Initializes an `MD5Context`. ## @@ -237,29 +292,39 @@ proc md5Init*(c: var MD5Context) = c.state[3] = 0x10325476'u32 c.count[0] = 0'u32 c.count[1] = 0'u32 - zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + clearBuffer(c) -proc md5Update*(c: var MD5Context, input: cstring, len: int) = - ## Updates the `MD5Context` with the `input` data of length `len`. +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + memOrNot: + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + do: + # cannot use system.`[]=` for arrays and openarrays as + # it can raise RangeDefect which gets tracked + for i in 0..`_, there's no need to call this ## function explicitly. - var input = input var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(len) shl 3) - if c.count[0] < (uint32(len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(len) shr 29) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) var PartLen = 64 - Index - if len >= PartLen: - copyMem(addr(c.buffer[Index]), input, PartLen) - transform(addr(c.buffer), c.state) + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) var i = PartLen - while i + 63 < len: - transform(addr(input[i]), c.state) + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) inc(i, 64) - copyMem(addr(c.buffer[0]), addr(input[i]), len-i) - else: - copyMem(addr(c.buffer[Index]), addr(input[0]), len) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) proc md5Final*(c: var MD5Context, digest: var MD5Digest) = ## Finishes the `MD5Context` and stores the result in `digest`. @@ -273,11 +338,11 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) = var Index = int((c.count[0] shr 3) and 0x3F) if Index < 56: PadLen = 56 - Index else: PadLen = 120 - Index - md5Update(c, padding, PadLen) - md5Update(c, cast[cstring](addr(Bits)), 8) + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) decode(digest, c.state) - zeroMem(addr(c), sizeof(MD5Context)) + clearBuffer(c) when defined(nimHasStyleChecks): - {.pop.} #{.push styleChecks: off.} + {.pop.} #{.push styleChecks: off.} \ No newline at end of file diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 407a358faa..df5b8c46ff 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -16,18 +16,50 @@ ## other "line-like", variable length, delimited records). when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs elif defined(posix): - import posix + import std/posix else: {.error: "the memfiles module is not supported on your operating system!".} -import os, streams +import std/streams +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + proc newEIO(msg: string): ref IOError = new(result) result.msg = msg +proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1. + ## Space is only allocated if that is cheaper than writing to the file. This + ## routine returns the last OSErrorCode found rather than raising to support + ## old rollback/clean-up code style. [ Should maybe move to std/osfiles. ] + if newFileSize == -1: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable, but allows SEGV + elif e != 0: + result = osLastError() + type MemFile* = object ## represents a memory mapped file mem*: pointer ## a pointer to the memory mapped file. The pointer @@ -118,7 +150,7 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## var ## mm, mm_full, mm_half: MemFile ## @@ -130,6 +162,7 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## # Read the first 512 bytes ## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512) + ## ``` # The file can be resized only when write mode is used: if mode == fmAppend: @@ -167,25 +200,13 @@ proc open*(filename: string, mode: FileMode = fmRead, else: FILE_ATTRIBUTE_NORMAL or flags, 0) - when useWinUnicode: - result.fHandle = callCreateFile(createFileW, newWideCString(filename)) - else: - result.fHandle = callCreateFile(createFileA, filename) + result.fHandle = callCreateFile(createFileW, newWideCString(filename)) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -219,7 +240,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.wasOpened = true if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: - if closeHandle(result.fHandle) == 0: + if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE else: @@ -242,9 +263,8 @@ proc open*(filename: string, mode: FileMode = fmRead, # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") if mappedSize != -1: result.size = mappedSize @@ -299,34 +319,58 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -when defined(posix) or defined(nimdoc): - proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## resize and re-map the file underlying an `allowRemap MemFile`. - ## **Note**: this assumes the entire file is mapped read-write at offset zero. - ## Also, the value of `.mem` will probably change. - ## **Note**: This is not (yet) available on Windows. - when defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if ftruncate(f.handle, newFileSize) == -1: - raiseOSError(osLastError()) - when defined(linux): #Maybe NetBSD, too? - #On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize; flags: cint): - pointer {.importc: "mremap", header: "".} - let newAddr = mremap(f.mem, csize(f.size), csize(newFileSize), cint(1)) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the @@ -377,7 +421,7 @@ proc `$`*(ms: MemSlice): string {.inline.} = copyMem(addr(result[0]), ms.data, ms.size) iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline.} = - ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`. + ## Iterates over \[optional `eat`] `delim`-delimited slices in MemFile `mfile`. ## ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l) ## style on on a line-by-line basis. I.e., not every line needs the same ending. @@ -400,15 +444,15 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ## functions, not str* functions). ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## var count = 0 ## for slice in memSlices(memfiles.open("foo")): ## if slice.size > 0 and cast[cstring](slice.data)[0] != '#': ## inc(count) ## echo count + ## ``` - proc c_memchr(cstr: pointer, c: char, n: csize): pointer {. + proc c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. importc: "memchr", header: "".} proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q) var ms: MemSlice @@ -416,7 +460,7 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ms.data = mfile.mem var remaining = mfile.size while remaining > 0: - ending = c_memchr(ms.data, delim, remaining) + ending = c_memchr(ms.data, delim, csize_t(remaining)) if ending == nil: # unterminated final slice ms.size = remaining # Weird case..check eat? yield ms @@ -431,16 +475,16 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline iterator lines*(mfile: MemFile, buf: var string, delim = '\l', eat = '\r'): string {.inline.} = ## Replace contents of passed buffer with each new line, like - ## `readLine(File) `_. + ## `readLine(File) `_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## var buffer: string = "" ## for line in lines(memfiles.open("foo"), buffer): ## echo line + ## ``` for ms in memSlices(mfile, delim, eat): setLen(buf, ms.size) @@ -450,15 +494,15 @@ iterator lines*(mfile: MemFile, buf: var string, delim = '\l', iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): string {.inline.} = ## Return each line in a file as a Nim string, like - ## `lines(File) `_. + ## `lines(File) `_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## for line in lines(memfiles.open("foo")): ## echo line + ## ``` var buf = newStringOfCap(80) for line in lines(mfile, buf, delim, eat): @@ -488,8 +532,8 @@ proc mmsSetPosition(s: Stream, pos: int) = proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem) - let p = cast[ByteAddress](MemMapFileStream(s).pos) + let startAddress = cast[int](MemMapFileStream(s).mf.mem) + let p = cast[int](MemMapFileStream(s).pos) let l = min(bufLen, MemMapFileStream(s).mf.size - p) moveMem(buffer, cast[pointer](startAddress + p), l) result = l @@ -504,8 +548,8 @@ proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = let size = MemMapFileStream(s).mf.size if MemMapFileStream(s).pos + bufLen > size: raise newEIO("cannot write to stream") - let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) + - cast[ByteAddress](MemMapFileStream(s).pos) + let p = cast[int](MemMapFileStream(s).mf.mem) + + cast[int](MemMapFileStream(s).pos) moveMem(cast[pointer](p), buffer, bufLen) inc(MemMapFileStream(s).pos, bufLen) diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index c10739fecb..ff639e8e5d 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -26,1856 +26,716 @@ runnableExamples: doAssert m.getMimetype("fakext") == "text/fakelang" doAssert m.getMimetype("FaKeXT") == "text/fakelang" -import strtabs -from strutils import startsWith, toLowerAscii, strip +import std/tables +from std/strutils import startsWith, toLowerAscii, strip + +when defined(nimPreviewSlimSystem): + import std/assertions + type MimeDB* = object - mimes: StringTableRef + mimes: OrderedTable[string, string] const mimes* = { - "123": "application/vnd.lotus-1-2-3", - "1km": "application/vnd.1000minds.decision-model+xml", - "323": "text/h323", - "3dm": "text/vnd.in3d.3dml", - "3dmf": "x-world/x-3dmf", - "3dml": "text/vnd.in3d.3dml", - "3ds": "image/x-3ds", - "3g2": "video/3gpp2", - "3gp": "video/3gpp", - "3gpp": "audio/3gpp", - "3gpp2": "video/3gpp2", - "3mf": "application/vnd.ms-3mfdocument", - "669": "audio/x-mod", - "726": "audio/32kadpcm", - "7z": "application/x-7z-compressed", - "a": "text/plain", - "a2l": "application/a2l", - "aa3": "audio/atrac3", - "aab": "application/x-authorware-bin", - "aac": "audio/x-aac", - "aal": "audio/atrac-advanced-lossless", - "aam": "application/x-authorware-map", - "aas": "application/x-authorware-seg", - "abc": "text/vnd.abc", - "abw": "application/x-abiword", - "ac": "application/pkix-attr-cert", - "ac3": "audio/ac3", - "acc": "application/vnd.americandynamics.acc", - "ace": "application/x-ace-compressed", - "acn": "audio/asc", - "acu": "application/vnd.acucobol", - "acutc": "application/vnd.acucorp", - "acx": "application/internet-property-stream", - "adp": "audio/adpcm", - "aep": "application/vnd.audiograph", - "afl": "video/animaflex", - "afm": "application/x-font-type1", - "afp": "application/vnd.ibm.modcap", - "ahead": "application/vnd.ahead.space", - "ai": "application/postscript", - "aif": "audio/x-aiff", - "aifc": "audio/x-aiff", - "aiff": "audio/x-aiff", - "aim": "application/x-aim", - "aip": "text/x-audiosoft-intra", - "air": "application/vnd.adobe.air-application-installer-package+zip", - "ait": "application/vnd.dvb.ait", - "alc": "chemical/x-alchemy", - "ami": "application/vnd.amiga.ami", - "aml": "application/aml", - "amr": "audio/amr", - "ani": "application/x-navi-animation", - "anx": "application/x-annodex", - "aos": "application/x-nokia-9000-communicator-add-on-software", - "apinotes": "text/apinotes", - "apk": "application/vnd.android.package-archive", - "apkg": "application/vnd.anki", - "apng": "image/apng", - "appcache": "text/cache-manifest", - "appimage": "application/appimage", - "application": "application/x-ms-application", - "apr": "application/vnd.lotus-approach", - "aps": "application/mime", - "apxml": "application/auth-policy+xml", - "arc": "application/x-freearc", - "arj": "application/x-arj", - "art": "message/rfc822", - "asar": "binary/asar", - "asc": "text/plain", - "ascii": "text/vnd.ascii-art", - "asf": "application/vnd.ms-asf", - "asice": "application/vnd.etsi.asic-e+zip", - "asics": "application/vnd.etsi.asic-s+zip", - "asm": "text/x-asm", - "asn": "chemical/x-ncbi-asn1-spec", - "aso": "application/vnd.accpac.simply.aso", - "asp": "text/asp", - "asr": "video/x-ms-asf", - "asx": "video/x-ms-asf", - "at3": "audio/atrac3", - "atc": "application/vnd.acucorp", - "atf": "application/atf", - "atfx": "application/atfx", + "ez": "application/andrew-inset", + "aw": "application/applixware", "atom": "application/atom+xml", "atomcat": "application/atomcat+xml", - "atomdeleted": "application/atomdeleted+xml", - "atomsrv": "application/atomserv+xml", "atomsvc": "application/atomsvc+xml", - "atx": "application/vnd.antix.game-component", - "atxml": "application/atxml", - "au": "audio/basic", - "auc": "application/tamp-apex-update-confirm", - "avi": "video/x-msvideo", - "avs": "video/avs-video", - "aw": "application/applixware", - "awb": "audio/amr-wb", - "axa": "audio/x-annodex", - "axs": "application/olescript", - "axv": "video/x-annodex", - "azf": "application/vnd.airzip.filesecure.azf", - "azs": "application/vnd.airzip.filesecure.azs", - "azv": "image/vnd.airzip.accelerator.azv", - "azw": "application/vnd.amazon.ebook", - "azw3": "application/vnd.amazon.mobi8-ebook", - "b": "chemical/x-molconn-Z", - "bak": "application/x-trash", - "bar": "application/vnd.qualcomm.brew-app-res", - "bas": "text/plain", - "bash": "text/shell", - "bat": "application/x-msdos-program", - "bcpio": "application/x-bcpio", - "bdf": "application/x-font-bdf", - "bdm": "application/vnd.syncml.dm+wbxml", - "bdoc": "application/bdoc", - "bed": "application/vnd.realvnc.bed", - "bh2": "application/vnd.fujitsu.oasysprs", - "bib": "text/x-bibtex", - "bik": "video/vnd.radgamettools.bink", - "bin": "application/octet-stream", - "bk2": "video/vnd.radgamettools.bink", - "bkm": "application/vnd.nervana", - "blb": "application/x-blorb", - "blend": "binary/blender", - "blorb": "application/x-blorb", - "bm": "image/bmp", - "bmed": "multipart/vnd.bint.med-plus", - "bmi": "application/vnd.bmi", - "bmml": "application/vnd.balsamiq.bmml+xml", - "bmp": "image/bmp", - "bmpr": "application/vnd.balsamiq.bmpr", - "boo": "application/book", - "book": "application/book", - "box": "application/vnd.previewsystems.box", - "boz": "application/x-bzip2", - "bpd": "application/vnd.hbci", - "bpk": "application/octet-stream", - "brf": "text/plain", - "bsd": "chemical/x-crossfire", - "bsh": "application/x-bsh", - "bsp": "model/vnd.valve.source.compiled-map", - "btf": "image/prs.btif", - "btif": "image/prs.btif", - "bz": "application/x-bzip", - "bz2": "application/x-bzip2", - "c": "text/x-csrc", - "c++": "text/x-c++src", - "c11amc": "application/vnd.cluetrust.cartomobile-config", - "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", - "c3d": "chemical/x-chem3d", - "c3ex": "application/cccex", - "c4d": "application/vnd.clonk.c4group", - "c4f": "application/vnd.clonk.c4group", - "c4g": "application/vnd.clonk.c4group", - "c4p": "application/vnd.clonk.c4group", - "c4u": "application/vnd.clonk.c4group", - "cab": "application/vnd.ms-cab-compressed", - "cac": "chemical/x-cache", - "cache": "application/x-cache", - "caf": "audio/x-caf", - "cap": "application/vnd.tcpdump.pcap", - "car": "application/vnd.curl.car", - "cascii": "chemical/x-cactvs-binary", - "cat": "application/vnd.ms-pki.seccat", - "cb7": "application/x-cbr", - "cba": "application/x-cbr", - "cbin": "chemical/x-cactvs-binary", - "cbor": "application/cbor", - "cbr": "application/x-cbr", - "cbt": "application/x-cbr", - "cbz": "application/vnd.comicbook+zip", - "cc": "text/plain", - "ccad": "application/clariscad", - "ccc": "text/vnd.net2phone.commcenter.command", - "ccmp": "application/ccmp+xml", - "cco": "application/x-cocoa", - "cct": "application/x-director", "ccxml": "application/ccxml+xml", - "cda": "application/x-cdf", - "cdbcmsg": "application/vnd.contact.cmsg", - "cdf": "application/x-netcdf", - "cdfx": "application/cdfx+xml", - "cdkey": "application/vnd.mediastation.cdkey", "cdmia": "application/cdmi-capability", "cdmic": "application/cdmi-container", "cdmid": "application/cdmi-domain", "cdmio": "application/cdmi-object", "cdmiq": "application/cdmi-queue", - "cdr": "image/x-coreldraw", - "cdt": "image/x-coreldrawtemplate", - "cdx": "chemical/x-cdx", - "cdxml": "application/vnd.chemdraw+xml", - "cdy": "application/vnd.cinderella", - "cea": "application/cea", - "cef": "chemical/x-cxf", - "cellml": "application/cellml+xml", - "cer": "application/pkix-cert", - "cfg": "text/cfg", - "cfs": "application/x-cfs-compressed", - "cgm": "image/cgm", - "cha": "application/x-chat", - "chat": "application/x-chat", - "chm": "application/vnd.ms-htmlhelp", - "chrt": "application/vnd.kde.kchart", - "cif": "chemical/x-cif", - "cii": "application/vnd.anser-web-certificate-issue-initiation", - "cil": "application/vnd.ms-artgalry", - "cl": "application/simple-filter+xml", - "cla": "application/vnd.claymore", + "cu": "application/cu-seeme", + "davmount": "application/davmount+xml", + "dbk": "application/docbook+xml", + "dssc": "application/dssc+der", + "xdssc": "application/dssc+xml", + "ecma": "application/ecmascript", + "emma": "application/emma+xml", + "epub": "application/epub+zip", + "exi": "application/exi", + "pfr": "application/font-tdpfr", + "gml": "application/gml+xml", + "gpx": "application/gpx+xml", + "gxf": "application/gxf", + "stk": "application/hyperstudio", + "ink": "application/inkml+xml", + "inkml": "application/inkml+xml", + "ipfix": "application/ipfix", + "jar": "application/java-archive", + "ser": "application/java-serialized-object", "class": "application/java-vm", + "json": "application/json", + "jsonml": "application/jsonml+json", + "lostxml": "application/lost+xml", + "hqx": "application/mac-binhex40", + "cpt": "application/mac-compactpro", + "mads": "application/mads+xml", + "mrc": "application/marc", + "mrcx": "application/marcxml+xml", + "ma": "application/mathematica", + "nb": "application/mathematica", + "mb": "application/mathematica", + "mathml": "application/mathml+xml", + "mbox": "application/mbox", + "mscml": "application/mediaservercontrol+xml", + "metalink": "application/metalink+xml", + "meta4": "application/metalink4+xml", + "mets": "application/mets+xml", + "mods": "application/mods+xml", + "m21": "application/mp21", + "mp21": "application/mp21", + "mp4s": "application/mp4", + "doc": "application/msword", + "dot": "application/msword", + "mxf": "application/mxf", + "bin": "application/octet-stream", + "dms": "application/octet-stream", + "lrf": "application/octet-stream", + "mar": "application/octet-stream", + "so": "application/octet-stream", + "dist": "application/octet-stream", + "distz": "application/octet-stream", + "pkg": "application/octet-stream", + "bpk": "application/octet-stream", + "dump": "application/octet-stream", + "elc": "application/octet-stream", + "deploy": "application/octet-stream", + "oda": "application/oda", + "opf": "application/oebps-package+xml", + "ogx": "application/ogg", + "omdoc": "application/omdoc+xml", + "onetoc": "application/onenote", + "onetoc2": "application/onenote", + "onetmp": "application/onenote", + "onepkg": "application/onenote", + "oxps": "application/oxps", + "xer": "application/patch-ops-error+xml", + "pdf": "application/pdf", + "pgp": "application/pgp-encrypted", + "asc": "application/pgp-signature", + "sig": "application/pgp-signature", + "prf": "application/pics-rules", + "p10": "application/pkcs10", + "p7m": "application/pkcs7-mime", + "p7c": "application/pkcs7-mime", + "p7s": "application/pkcs7-signature", + "p8": "application/pkcs8", + "ac": "application/pkix-attr-cert", + "cer": "application/pkix-cert", + "crl": "application/pkix-crl", + "pkipath": "application/pkix-pkipath", + "pki": "application/pkixcmp", + "pls": "application/pls+xml", + "ai": "application/postscript", + "eps": "application/postscript", + "ps": "application/postscript", + "cww": "application/prs.cww", + "pskcxml": "application/pskc+xml", + "rdf": "application/rdf+xml", + "rif": "application/reginfo+xml", + "rnc": "application/relax-ng-compact-syntax", + "rl": "application/resource-lists+xml", + "rld": "application/resource-lists-diff+xml", + "rs": "application/rls-services+xml", + "gbr": "application/rpki-ghostbusters", + "mft": "application/rpki-manifest", + "roa": "application/rpki-roa", + "rsd": "application/rsd+xml", + "rss": "application/rss+xml", + "rtf": "application/rtf", + "sbml": "application/sbml+xml", + "scq": "application/scvp-cv-request", + "scs": "application/scvp-cv-response", + "spq": "application/scvp-vp-request", + "spp": "application/scvp-vp-response", + "sdp": "application/sdp", + "setpay": "application/set-payment-initiation", + "setreg": "application/set-registration-initiation", + "shf": "application/shf+xml", + "smi": "application/smil+xml", + "smil": "application/smil+xml", + "rq": "application/sparql-query", + "srx": "application/sparql-results+xml", + "gram": "application/srgs", + "grxml": "application/srgs+xml", + "sru": "application/sru+xml", + "ssdl": "application/ssdl+xml", + "ssml": "application/ssml+xml", + "tei": "application/tei+xml", + "teicorpus": "application/tei+xml", + "tfi": "application/thraud+xml", + "tsd": "application/timestamped-data", + "plb": "application/vnd.3gpp.pic-bw-large", + "psb": "application/vnd.3gpp.pic-bw-small", + "pvb": "application/vnd.3gpp.pic-bw-var", + "tcap": "application/vnd.3gpp2.tcap", + "pwn": "application/vnd.3m.post-it-notes", + "aso": "application/vnd.accpac.simply.aso", + "imp": "application/vnd.accpac.simply.imp", + "acu": "application/vnd.acucobol", + "atc": "application/vnd.acucorp", + "acutc": "application/vnd.acucorp", + "air": "application/vnd.adobe.air-application-installer-package+zip", + "fcdt": "application/vnd.adobe.formscentral.fcdt", + "fxp": "application/vnd.adobe.fxp", + "fxpl": "application/vnd.adobe.fxp", + "xdp": "application/vnd.adobe.xdp+xml", + "xfdf": "application/vnd.adobe.xfdf", + "ahead": "application/vnd.ahead.space", + "azf": "application/vnd.airzip.filesecure.azf", + "azs": "application/vnd.airzip.filesecure.azs", + "azw": "application/vnd.amazon.ebook", + "acc": "application/vnd.americandynamics.acc", + "ami": "application/vnd.amiga.ami", + "apk": "application/vnd.android.package-archive", + "cii": "application/vnd.anser-web-certificate-issue-initiation", + "fti": "application/vnd.anser-web-funds-transfer-initiation", + "atx": "application/vnd.antix.game-component", + "mpkg": "application/vnd.apple.installer+xml", + "m3u8": "application/vnd.apple.mpegurl", + "swi": "application/vnd.aristanetworks.swi", + "iota": "application/vnd.astraea-software.iota", + "aep": "application/vnd.audiograph", + "mpm": "application/vnd.blueice.multipass", + "bmi": "application/vnd.bmi", + "rep": "application/vnd.businessobjects", + "cdxml": "application/vnd.chemdraw+xml", + "mmd": "application/vnd.chipnuts.karaoke-mmd", + "cdy": "application/vnd.cinderella", + "cla": "application/vnd.claymore", + "rp9": "application/vnd.cloanto.rp9", + "c4g": "application/vnd.clonk.c4group", + "c4d": "application/vnd.clonk.c4group", + "c4f": "application/vnd.clonk.c4group", + "c4p": "application/vnd.clonk.c4group", + "c4u": "application/vnd.clonk.c4group", + "c11amc": "application/vnd.cluetrust.cartomobile-config", + "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", + "csp": "application/vnd.commonspace", + "cdbcmsg": "application/vnd.contact.cmsg", + "cmc": "application/vnd.cosmocaller", + "clkx": "application/vnd.crick.clicker", "clkk": "application/vnd.crick.clicker.keyboard", "clkp": "application/vnd.crick.clicker.palette", "clkt": "application/vnd.crick.clicker.template", "clkw": "application/vnd.crick.clicker.wordbank", - "clkx": "application/vnd.crick.clicker", - "clp": "application/x-msclip", - "cls": "text/x-tex", - "clue": "application/clue_info+xml", - "cmake": "text/cmake", - "cmc": "application/vnd.cosmocaller", - "cmdf": "chemical/x-cmdf", - "cml": "chemical/x-cml", - "cmp": "application/vnd.yellowriver-custom-menu", - "cmsc": "application/cms", - "cmx": "image/x-cmx", - "cnd": "text/jcr-cnd", - "cnf": "text/cnf", - "cod": "application/vnd.rim.cod", - "coffee": "application/vnd.coffeescript", - "com": "application/x-msdos-program", - "conf": "text/plain", - "copyright": "text/vnd.debian.copyright", - "cpa": "chemical/x-compass", - "cpio": "application/x-cpio", - "cpkg": "application/vnd.xmpie.cpkg", - "cpl": "application/cpl+xml", - "cpp": "text/x-c++src", - "cpt": "application/mac-compactpro", - "cr2": "image/x-canon-cr2", - "crd": "application/x-mscardfile", - "crl": "application/pkix-crl", - "crt": "application/x-x509-ca-cert", - "crtr": "application/vnd.multiad.creator", - "crw": "image/x-canon-crw", - "crx": "application/x-chrome-extension", - "cryptonote": "application/vnd.rig.cryptonote", - "cs": "text/c#", - "csf": "chemical/x-cache-csf", - "csh": "application/x-csh", - "csl": "application/vnd.citationstyles.style+xml", - "csm": "chemical/x-csml", - "csml": "chemical/x-csml", - "cson": "text/cson", - "csp": "application/vnd.commonspace", - "csrattrs": "application/csrattrs", - "css": "text/css", - "cst": "application/vnd.commonspace", - "csv": "text/csv", - "csvs": "text/csv-schema", - "ctab": "chemical/x-cactvs-binary", - "ctx": "chemical/x-ctx", - "cu": "application/cu-seeme", - "cub": "chemical/x-gaussian-cube", - "cuc": "application/tamp-community-update-confirm", - "curl": "text/vnd.curl", - "cw": "application/prs.cww", - "cww": "application/prs.cww", - "cxf": "chemical/x-cxf", - "cxt": "application/x-director", - "cxx": "text/plain", - "d": "text/x-dsrc", - "dae": "model/vnd.collada+xml", - "daf": "application/vnd.mobius.daf", + "wbs": "application/vnd.criticaltools.wbs+xml", + "pml": "application/vnd.ctc-posml", + "ppd": "application/vnd.cups-ppd", + "car": "application/vnd.curl.car", + "pcurl": "application/vnd.curl.pcurl", "dart": "application/vnd.dart", - "dat": "application/x-ns-proxy-autoconfig", - "dataless": "application/vnd.fdsn.seed", - "davmount": "application/davmount+xml", - "dbk": "application/docbook+xml", - "dcd": "application/dcd", - "dcf": "application/vnd.oma.drm.content", - "dcm": "application/dicom", - "dcr": "application/x-director", - "dcurl": "text/vnd.curl.dcurl", - "dd": "application/vnd.oma.dd+xml", - "dd2": "application/vnd.oma.dd2+xml", - "ddd": "application/vnd.fujixerox.ddd", - "ddf": "application/vnd.syncml.dmddf+xml", - "deb": "application/vnd.debian.binary-package", - "deepv": "application/x-deepv", - "def": "text/plain", - "deploy": "application/octet-stream", - "der": "application/x-x509-ca-cert", - "dfac": "application/vnd.dreamfactory", - "dgc": "application/x-dgc-compressed", - "dib": "image/bmp", - "dic": "text/x-c", - "dif": "video/x-dv", - "diff": "text/x-diff", - "dii": "application/dii", - "dim": "application/vnd.fastcopy-disk-image", - "dir": "application/x-director", - "dis": "application/vnd.mobius.dis", - "disposition-notification": "message/disposition-notification", - "dist": "application/vnd.apple.installer+xml", - "distz": "application/vnd.apple.installer+xml", - "dit": "application/dit", - "djv": "image/vnd.djvu", - "djvu": "image/vnd.djvu", - "dl": "video/dl", - "dll": "application/x-msdos-program", - "dls": "audio/dls", - "dm": "application/vnd.oma.drm.message", - "dmg": "application/x-apple-diskimage", - "dmp": "application/vnd.tcpdump.pcap", - "dms": "text/vnd.dmclientscript", + "rdz": "application/vnd.data-vision.rdz", + "uvf": "application/vnd.dece.data", + "uvvf": "application/vnd.dece.data", + "uvd": "application/vnd.dece.data", + "uvvd": "application/vnd.dece.data", + "uvt": "application/vnd.dece.ttml+xml", + "uvvt": "application/vnd.dece.ttml+xml", + "uvx": "application/vnd.dece.unspecified", + "uvvx": "application/vnd.dece.unspecified", + "uvz": "application/vnd.dece.zip", + "uvvz": "application/vnd.dece.zip", + "fe_launch": "application/vnd.denovo.fcselayout-link", "dna": "application/vnd.dna", - "doc": "application/msword", - "docjson": "application/vnd.document+json", - "docm": "application/vnd.ms-word.document.macroenabled.12", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "dor": "model/vnd.gdl", - "dot": "text/vnd.graphviz", - "dotm": "application/vnd.ms-word.template.macroenabled.12", - "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", - "dp": "application/vnd.osgi.dp", + "mlp": "application/vnd.dolby.mlp", "dpg": "application/vnd.dpgraph", - "dpgraph": "application/vnd.dpgraph", - "dpkg": "application/vnd.xmpie.dpkg", - "dr": "application/vnd.oma.drm.rights+xml", - "dra": "audio/vnd.dra", - "drc": "application/vnd.oma.drm.rights+wbxml", - "drle": "image/dicom-rle", - "drw": "application/drafting", - "dsc": "text/prs.lines.tag", - "dsm": "application/vnd.desmume.movie", - "dssc": "application/dssc+der", - "dtb": "application/x-dtbook+xml", - "dtd": "application/xml-dtd", - "dts": "audio/vnd.dts", - "dtshd": "audio/vnd.dts.hd", - "dump": "application/octet-stream", - "dv": "video/x-dv", - "dvb": "video/vnd.dvb.file", - "dvc": "application/dvcs", - "dvi": "application/x-dvi", - "dwf": "model/vnd.dwf", - "dwg": "image/vnd.dwg", - "dx": "chemical/x-jcamp-dx", - "dxf": "image/vnd.dxf", - "dxp": "application/vnd.spotfire.dxp", - "dxr": "application/x-director", - "dzr": "application/vnd.dzr", - "ear": "binary/zip", - "ecelp4800": "audio/vnd.nuera.ecelp4800", - "ecelp7470": "audio/vnd.nuera.ecelp7470", - "ecelp9600": "audio/vnd.nuera.ecelp9600", - "ecig": "application/vnd.evolv.ecig.settings", - "ecigprofile": "application/vnd.evolv.ecig.profile", - "ecigtheme": "application/vnd.evolv.ecig.theme", - "ecma": "application/ecmascript", - "edm": "application/vnd.novadigm.edm", - "edx": "application/vnd.novadigm.edx", - "efi": "application/efi", - "efif": "application/vnd.picsel", - "ei6": "application/vnd.pg.osasli", - "ejs": "text/ejs", - "el": "text/plain", - "elc": "application/x-bytecode.elisp", - "emb": "chemical/x-embl-dl-nucleotide", - "embl": "chemical/x-embl-dl-nucleotide", - "emf": "image/emf", - "eml": "message/rfc822", - "emm": "application/vnd.ibm.electronic-media", - "emma": "application/emma+xml", - "emotionml": "application/emotionml+xml", - "emz": "application/x-msmetafile", - "ent": "text/xml-external-parsed-entity", - "entity": "application/vnd.nervana", - "env": "application/x-envoy", - "enw": "audio/evrcnw", - "eol": "audio/vnd.digital-winds", - "eot": "application/vnd.ms-fontobject", - "ep": "application/vnd.bluetooth.ep.oob", - "eps": "application/postscript", - "eps2": "application/postscript", - "eps3": "application/postscript", - "epsf": "application/postscript", - "epsi": "application/postscript", - "epub": "application/epub+zip", - "erb": "text/erb", - "erf": "image/x-epson-erf", - "es": "application/ecmascript", - "es3": "application/vnd.eszigno3+xml", - "esa": "application/vnd.osgi.subsystem", - "escn": "text/godot", + "dfac": "application/vnd.dreamfactory", + "kpxx": "application/vnd.ds-keypoint", + "ait": "application/vnd.dvb.ait", + "svc": "application/vnd.dvb.service", + "geo": "application/vnd.dynageo", + "mag": "application/vnd.ecowin.chart", + "nml": "application/vnd.enliven", "esf": "application/vnd.epson.esf", - "espass": "application/vnd.espass-espass+zip", + "msf": "application/vnd.epson.msf", + "qam": "application/vnd.epson.quickanime", + "slt": "application/vnd.epson.salt", + "ssf": "application/vnd.epson.ssf", + "es3": "application/vnd.eszigno3+xml", "et3": "application/vnd.eszigno3+xml", - "etx": "text/x-setext", - "eva": "application/x-eva", - "evb": "audio/evrcb", - "evc": "audio/evrc", - "evw": "audio/evrcwb", - "evy": "application/x-envoy", - "exe": "application/x-msdos-program", - "exi": "application/exi", - "exr": "image/aces", - "ext": "application/vnd.novadigm.ext", - "eyaml": "text/yaml", - "ez": "application/andrew-inset", "ez2": "application/vnd.ezpix-album", "ez3": "application/vnd.ezpix-package", - "f": "text/x-fortran", - "f4v": "video/x-f4v", - "f77": "text/x-fortran", - "f90": "text/plain", - "fb": "application/x-maker", - "fbdoc": "application/x-maker", - "fbs": "image/vnd.fastbidsheet", - "fbx": "model/filmbox", - "fcdt": "application/vnd.adobe.formscentral.fcdt", - "fch": "chemical/x-gaussian-checkpoint", - "fchk": "chemical/x-gaussian-checkpoint", - "fcs": "application/vnd.isac.fcs", "fdf": "application/vnd.fdf", - "fdt": "application/fdt+xml", - "fe_launch": "application/vnd.denovo.fcselayout-link", - "feature": "text/gherkin", - "fg5": "application/vnd.fujitsu.oasysgp", - "fgd": "application/x-director", - "fh": "image/x-freehand", - "fh4": "image/x-freehand", - "fh5": "image/x-freehand", - "fh7": "image/x-freehand", - "fhc": "image/x-freehand", - "fif": "image/fif", - "fig": "application/x-xfig", - "finf": "application/fastinfoset", - "fish": "text/fish", - "fit": "image/fits", - "fits": "image/fits", - "fla": "application/vnd.dtg.local.flash", - "flac": "audio/x-flac", - "fli": "video/x-fli", - "flo": "application/vnd.micrografx.flo", - "flr": "x-world/x-vrml", - "flv": "video/x-flv", - "flw": "application/vnd.kde.kivio", - "flx": "text/vnd.fmi.flexstor", - "fly": "text/vnd.fly", - "fm": "application/vnd.framemaker", - "fmf": "video/x-atomic3d-feature", - "fnc": "application/vnd.frogans.fnc", - "fo": "application/vnd.software602.filler.form+xml", - "for": "text/x-fortran", - "fpx": "image/vnd.fpx", - "frame": "application/vnd.framemaker", - "frl": "application/freeloader", - "frm": "application/vnd.ufdl", - "fsc": "application/vnd.fsc.weblaunch", - "fst": "image/vnd.fst", - "ftc": "application/vnd.fluxtime.clip", - "fti": "application/vnd.anser-web-funds-transfer-initiation", - "fts": "image/fits", - "funk": "audio/make", - "fvt": "video/vnd.fvt", - "fxm": "video/x-javafx", - "fxp": "application/vnd.adobe.fxp", - "fxpl": "application/vnd.adobe.fxp", - "fzs": "application/vnd.fuzzysheet", - "g": "text/plain", - "g2w": "application/vnd.geoplan", - "g3": "image/g3fax", - "g3w": "application/vnd.geospace", - "gac": "application/vnd.groove-account", - "gal": "chemical/x-gaussian-log", - "gam": "application/x-tads", - "gamin": "chemical/x-gamess-input", - "gau": "chemical/x-gaussian-input", - "gbr": "application/rpki-ghostbusters", - "gca": "application/x-gca-compressed", - "gcd": "text/x-pcs-gcd", - "gcf": "application/x-graphing-calculator", - "gcg": "chemical/x-gcg8-sequence", - "gdl": "model/vnd.gdl", - "gdoc": "application/vnd.google-apps.document", - "gemspec": "text/ruby", - "gen": "chemical/x-genbank", - "geo": "application/vnd.dynageo", - "geojson": "application/geo+json", - "gex": "application/vnd.geometry-explorer", - "gf": "application/x-tex-gf", - "ggb": "application/vnd.geogebra.file", - "ggt": "application/vnd.geogebra.tool", - "ghf": "application/vnd.groove-help", - "gif": "image/gif", - "gim": "application/vnd.groove-identity-message", - "gjc": "chemical/x-gaussian-input", - "gjf": "chemical/x-gaussian-input", - "gl": "video/gl", - "glb": "model/gltf-binary", - "gltf": "model/gltf+json", - "gml": "application/gml+xml", - "gmx": "application/vnd.gmx", - "gnumeric": "application/x-gnumeric", - "go": "text/go", - "gotmpl": "text/gotmpl", + "mseed": "application/vnd.fdsn.mseed", + "seed": "application/vnd.fdsn.seed", + "dataless": "application/vnd.fdsn.seed", "gph": "application/vnd.flographit", - "gpt": "chemical/x-mopac-graph", - "gpx": "application/gpx+xml", - "gqf": "application/vnd.grafeq", - "gqs": "application/vnd.grafeq", - "gradle": "text/groovy", - "gram": "application/srgs", - "gramps": "application/x-gramps-xml", + "ftc": "application/vnd.fluxtime.clip", + "fm": "application/vnd.framemaker", + "frame": "application/vnd.framemaker", + "maker": "application/vnd.framemaker", + "book": "application/vnd.framemaker", + "fnc": "application/vnd.frogans.fnc", + "ltf": "application/vnd.frogans.ltf", + "fsc": "application/vnd.fsc.weblaunch", + "oas": "application/vnd.fujitsu.oasys", + "oa2": "application/vnd.fujitsu.oasys2", + "oa3": "application/vnd.fujitsu.oasys3", + "fg5": "application/vnd.fujitsu.oasysgp", + "bh2": "application/vnd.fujitsu.oasysprs", + "ddd": "application/vnd.fujixerox.ddd", + "xdw": "application/vnd.fujixerox.docuworks", + "xbd": "application/vnd.fujixerox.docuworks.binder", + "fzs": "application/vnd.fuzzysheet", + "txd": "application/vnd.genomatix.tuxedo", + "ggb": "application/vnd.geogebra.file", + "ggs": "application/vnd.geogebra.slides", + "ggt": "application/vnd.geogebra.tool", + "gex": "application/vnd.geometry-explorer", "gre": "application/vnd.geometry-explorer", - "groovy": "text/groovy", - "grv": "application/vnd.groove-injector", - "grxml": "application/srgs+xml", - "gsd": "audio/x-gsm", - "gsf": "application/x-font-ghostscript", - "gsheet": "application/vnd.google-apps.spreadsheet", - "gslides": "application/vnd.google-apps.presentation", - "gsm": "model/vnd.gdl", - "gsp": "application/x-gsp", - "gss": "application/x-gss", - "gtar": "application/x-gtar", - "gtm": "application/vnd.groove-tool-message", - "gtw": "model/vnd.gtw", - "gv": "text/vnd.graphviz", - "gxf": "application/gxf", "gxt": "application/vnd.geonext", - "gyb": "text/gyb", - "gyp": "text/gyp", - "gypi": "text/gyp", - "gz": "application/gzip", - "h": "text/x-chdr", - "h++": "text/x-c++hdr", - "h261": "video/h261", - "h263": "video/h263", - "h264": "video/h264", - "hal": "application/vnd.hal+xml", - "hbc": "application/vnd.hbci", - "hbci": "application/vnd.hbci", - "hbs": "text/x-handlebars-template", - "hdd": "application/x-virtualbox-hdd", - "hdf": "application/x-hdf", - "hdr": "image/vnd.radiance", - "hdt": "application/vnd.hdt", - "heic": "image/heic", - "heics": "image/heic-sequence", - "heif": "image/heif", - "heifs": "image/heif-sequence", - "help": "application/x-helpfile", - "hgl": "application/vnd.hp-hpgl", - "hh": "text/plain", - "hin": "chemical/x-hin", - "hjson": "application/hjson", - "hlb": "text/x-script", - "hlp": "application/winhlp", - "hpg": "application/vnd.hp-hpgl", - "hpgl": "application/vnd.hp-hpgl", - "hpi": "application/vnd.hp-hpid", - "hpid": "application/vnd.hp-hpid", - "hpp": "text/x-c++hdr", - "hps": "application/vnd.hp-hps", - "hpub": "application/prs.hpub+zip", - "hqx": "application/mac-binhex40", - "hs": "text/x-haskell", - "hta": "application/hta", - "htc": "text/x-component", - "htke": "application/vnd.kenameaapp", - "html": "text/html", - "htt": "text/webviewhtml", - "hvd": "application/vnd.yamaha.hv-dic", - "hvp": "application/vnd.yamaha.hv-voice", - "hvs": "application/vnd.yamaha.hv-script", - "hx": "text/haxe", - "hxml": "text/haxe", - "hxx": "text/plain", - "i2g": "application/vnd.intergeo", - "ic0": "application/vnd.commerce-battelle", - "ic1": "application/vnd.commerce-battelle", - "ic2": "application/vnd.commerce-battelle", - "ic3": "application/vnd.commerce-battelle", - "ic4": "application/vnd.commerce-battelle", - "ic5": "application/vnd.commerce-battelle", - "ic6": "application/vnd.commerce-battelle", - "ic7": "application/vnd.commerce-battelle", - "ic8": "application/vnd.commerce-battelle", - "ica": "application/vnd.commerce-battelle", - "icc": "application/vnd.iccprofile", - "icd": "application/vnd.commerce-battelle", - "ice": "x-conference/x-cooltalk", - "icf": "application/vnd.commerce-battelle", - "icm": "application/vnd.iccprofile", - "icns": "binary/icns", - "ico": "image/x-icon", - "ics": "text/calendar", - "icz": "text/calendar", - "idc": "text/plain", - "idl": "text/idl", - "ief": "image/ief", - "iefs": "image/ief", - "ifb": "text/calendar", - "ifm": "application/vnd.shana.informed.formdata", - "iges": "model/iges", - "igl": "application/vnd.igloader", - "igm": "application/vnd.insors.igm", - "ign": "application/vnd.coreos.ignition+json", - "ignition": "application/vnd.coreos.ignition+json", - "igs": "model/iges", - "igx": "application/vnd.micrografx.igx", - "iif": "application/vnd.shana.informed.interchange", - "iii": "application/x-iphone", - "ima": "application/x-ima", - "imap": "application/x-httpd-imap", - "imf": "application/vnd.imagemeter.folder+zip", - "img": "application/octet-stream", - "imgcal": "application/vnd.3lightssoftware.imagescal", - "imi": "application/vnd.imagemeter.image+zip", - "imp": "application/vnd.accpac.simply.imp", - "ims": "application/vnd.ms-ims", - "imscc": "application/vnd.ims.imsccv1p1", - "in": "text/plain", - "inc": "text/inc", - "inf": "application/inf", - "info": "application/x-info", - "ini": "text/ini", - "ink": "application/inkml+xml", - "inkml": "application/inkml+xml", - "inp": "chemical/x-gamess-input", - "ins": "application/x-internet-signup", - "install": "application/x-install-instructions", - "iota": "application/vnd.astraea-software.iota", - "ip": "application/x-ip2", - "ipfix": "application/ipfix", - "ipk": "application/vnd.shana.informed.package", - "irm": "application/vnd.ibm.rights-management", - "irp": "application/vnd.irepository.package+xml", - "ism": "model/vnd.gdl", - "iso": "application/x-iso9660-image", - "isp": "application/x-internet-signup", - "ist": "chemical/x-isostar", - "istr": "chemical/x-isostar", - "isu": "video/x-isvideo", - "it": "audio/it", - "itp": "application/vnd.shana.informed.formtemplate", - "its": "application/its+xml", - "iv": "application/x-inventor", - "ivp": "application/vnd.immervision-ivp", - "ivr": "i-world/i-vrml", - "ivu": "application/vnd.immervision-ivu", - "ivy": "application/x-livescreen", - "j2": "text/jinja", - "jad": "text/vnd.sun.j2me.app-descriptor", - "jade": "text/jade", - "jam": "application/vnd.jam", - "jar": "application/x-java-archive", - "jardiff": "application/x-java-archive-diff", - "java": "text/x-java-source", - "jcm": "application/x-java-commerce", - "jdx": "chemical/x-jcamp-dx", - "jenkinsfile": "text/groovy", - "jfif": "image/jpeg", - "jinja": "text/jinja", - "jinja2": "text/jinja", - "jisp": "application/vnd.jisp", - "jls": "image/jls", - "jlt": "application/vnd.hp-jlyt", - "jl": "text/julia", - "jmz": "application/x-jmol", - "jng": "image/x-jng", - "jnlp": "application/x-java-jnlp-file", - "joda": "application/vnd.joost.joda-archive", - "jp2": "image/jp2", - "jpe": "image/jpeg", - "jpeg": "image/jpeg", - "jpf": "image/jpx", - "jpg": "image/jpeg", - "jpg2": "image/jp2", - "jpgm": "image/jpm", - "jpgv": "video/jpeg", - "jpm": "image/jpm", - "jps": "image/x-jps", - "jpx": "image/jpx", - "jrd": "application/jrd+json", - "js": "application/javascript", - "json": "application/json", - "json-patch": "application/json-patch+json", - "json5": "application/json5", - "jsonld": "application/ld+json", - "jsonml": "application/jsonml+json", - "jsx": "text/jsx", - "jtd": "text/vnd.esmertec.theme-descriptor", - "jut": "image/jutvision", - "kar": "audio/midi", - "karbon": "application/vnd.kde.karbon", - "kcm": "application/vnd.nervana", - "key": "application/pgp-keys", - "keynote": "application/vnd.apple.keynote", - "kfo": "application/vnd.kde.kformula", - "kia": "application/vnd.kidspiration", - "kil": "application/x-killustrator", - "kin": "chemical/x-kinemage", + "g2w": "application/vnd.geoplan", + "g3w": "application/vnd.geospace", + "gmx": "application/vnd.gmx", "kml": "application/vnd.google-earth.kml+xml", "kmz": "application/vnd.google-earth.kmz", - "kne": "application/vnd.kinar", - "knp": "application/vnd.kinar", - "kom": "application/vnd.hbci", + "gqf": "application/vnd.grafeq", + "gqs": "application/vnd.grafeq", + "gac": "application/vnd.groove-account", + "ghf": "application/vnd.groove-help", + "gim": "application/vnd.groove-identity-message", + "grv": "application/vnd.groove-injector", + "gtm": "application/vnd.groove-tool-message", + "tpl": "application/vnd.groove-tool-template", + "vcg": "application/vnd.groove-vcard", + "hal": "application/vnd.hal+xml", + "zmm": "application/vnd.handheld-entertainment+xml", + "hbci": "application/vnd.hbci", + "les": "application/vnd.hhe.lesson-player", + "hpgl": "application/vnd.hp-hpgl", + "hpid": "application/vnd.hp-hpid", + "hps": "application/vnd.hp-hps", + "jlt": "application/vnd.hp-jlyt", + "pcl": "application/vnd.hp-pcl", + "pclxl": "application/vnd.hp-pclxl", + "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "mpy": "application/vnd.ibm.minipay", + "afp": "application/vnd.ibm.modcap", + "listafp": "application/vnd.ibm.modcap", + "list3820": "application/vnd.ibm.modcap", + "irm": "application/vnd.ibm.rights-management", + "sc": "application/vnd.ibm.secure-container", + "icc": "application/vnd.iccprofile", + "icm": "application/vnd.iccprofile", + "igl": "application/vnd.igloader", + "ivp": "application/vnd.immervision-ivp", + "ivu": "application/vnd.immervision-ivu", + "igm": "application/vnd.insors.igm", + "xpw": "application/vnd.intercon.formnet", + "xpx": "application/vnd.intercon.formnet", + "i2g": "application/vnd.intergeo", + "qbo": "application/vnd.intu.qbo", + "qfx": "application/vnd.intu.qfx", + "rcprofile": "application/vnd.ipunplugged.rcprofile", + "irp": "application/vnd.irepository.package+xml", + "xpr": "application/vnd.is-xpr", + "fcs": "application/vnd.isac.fcs", + "jam": "application/vnd.jam", + "rms": "application/vnd.jcp.javame.midlet-rms", + "jisp": "application/vnd.jisp", + "joda": "application/vnd.joost.joda-archive", + "ktz": "application/vnd.kahootz", + "ktr": "application/vnd.kahootz", + "karbon": "application/vnd.kde.karbon", + "chrt": "application/vnd.kde.kchart", + "kfo": "application/vnd.kde.kformula", + "flw": "application/vnd.kde.kivio", "kon": "application/vnd.kde.kontour", - "koz": "audio/vnd.audikoz", "kpr": "application/vnd.kde.kpresenter", "kpt": "application/vnd.kde.kpresenter", - "kpxx": "application/vnd.ds-keypoint", - "ksh": "application/x-ksh", "ksp": "application/vnd.kde.kspread", - "kt": "text/kotlin", - "ktr": "application/vnd.kahootz", - "ktx": "image/ktx", - "ktz": "application/vnd.kahootz", "kwd": "application/vnd.kde.kword", "kwt": "application/vnd.kde.kword", - "l16": "audio/l16", - "la": "audio/nspaudio", - "lam": "audio/x-liveaudio", - "lasjson": "application/vnd.las.las+json", + "htke": "application/vnd.kenameaapp", + "kia": "application/vnd.kidspiration", + "kne": "application/vnd.kinar", + "knp": "application/vnd.kinar", + "skp": "application/vnd.koan", + "skd": "application/vnd.koan", + "skt": "application/vnd.koan", + "skm": "application/vnd.koan", + "sse": "application/vnd.kodak-descriptor", "lasxml": "application/vnd.las.las+xml", - "latex": "application/x-latex", - "lbc": "audio/ilbc", "lbd": "application/vnd.llamagraphics.life-balance.desktop", "lbe": "application/vnd.llamagraphics.life-balance.exchange+xml", - "le": "application/vnd.bluetooth.le.oob", - "les": "application/vnd.hhe.lesson-player", - "less": "text/less", - "lgr": "application/lgr+xml", - "lha": "application/octet-stream", - "lhs": "text/x-literate-haskell", - "lhx": "application/octet-stream", - "lin": "application/bbolin", - "link66": "application/vnd.route66.link66+xml", - "list": "text/plain", - "list3820": "application/vnd.ibm.modcap", - "listafp": "application/vnd.ibm.modcap", - "lmp": "model/vnd.gdl", - "lnk": "application/x-ms-shortcut", - "log": "text/plain", - "lostsyncxml": "application/lostsync+xml", - "lostxml": "application/lost+xml", - "lrf": "application/octet-stream", - "lrm": "application/vnd.ms-lrm", - "lsf": "video/x-la-asf", - "lsp": "text/x-script.lisp", - "lst": "text/plain", - "lsx": "video/x-la-asf", - "ltf": "application/vnd.frogans.ltf", - "ltx": "application/x-latex", - "lua": "text/x-lua", - "luac": "application/x-lua-bytecode", - "lvp": "audio/vnd.lucent.voice", + "123": "application/vnd.lotus-1-2-3", + "apr": "application/vnd.lotus-approach", + "pre": "application/vnd.lotus-freelance", + "nsf": "application/vnd.lotus-notes", + "org": "application/vnd.lotus-organizer", + "scm": "application/vnd.lotus-screencam", "lwp": "application/vnd.lotus-wordpro", - "lxf": "application/lxf", - "lyx": "application/x-lyx", - "lzh": "application/octet-stream", - "lzx": "application/x-lzx", - "m": "application/vnd.wolfram.mathematica.package", - "m13": "application/x-msmediaview", - "m14": "application/x-msmediaview", - "m15": "audio/x-mod", - "m1v": "video/mpeg", - "m21": "application/mp21", - "m2a": "audio/mpeg", - "m2v": "video/mpeg", - "m3a": "audio/mpeg", - "m3g": "application/m3g", - "m3u": "audio/x-mpegurl", - "m3u8": "application/vnd.apple.mpegurl", - "m4a": "audio/x-m4a", - "m4s": "video/iso.segment", - "m4u": "video/vnd.mpegurl", - "m4v": "video/x-m4v", - "ma": "application/mathematica", - "mads": "application/mads+xml", - "mag": "application/vnd.ecowin.chart", - "mail": "message/rfc822", - "maker": "application/vnd.framemaker", - "man": "application/x-troff-man", - "manifest": "text/cache-manifest", - "map": "application/x-navimap", - "mar": "text/plain", - "markdown": "text/markdown", - "mathml": "application/mathml+xml", - "mb": "application/mathematica", - "mbd": "application/mbedlet", - "mbk": "application/vnd.mobius.mbk", - "mbox": "application/mbox", - "mc$": "application/x-magic-cap-package-1.0", - "mc1": "application/vnd.medcalcdata", + "portpkg": "application/vnd.macports.portpkg", "mcd": "application/vnd.mcd", - "mcf": "image/vasa", - "mcif": "chemical/x-mmcif", - "mcm": "chemical/x-macmolecule", - "mcp": "application/netmc", - "mcurl": "text/vnd.curl.mcurl", - "md": "text/markdown", - "mdb": "application/x-msaccess", - "mdc": "application/vnd.marlin.drm.mdcf", - "mdi": "image/vnd.ms-modi", - "me": "application/x-troff-me", - "med": "audio/x-mod", - "mesh": "model/mesh", - "meta4": "application/metalink4+xml", - "metalink": "application/metalink+xml", - "mets": "application/mets+xml", - "mf4": "application/mf4", + "mc1": "application/vnd.medcalcdata", + "cdkey": "application/vnd.mediastation.cdkey", + "mwf": "application/vnd.mfer", "mfm": "application/vnd.mfmp", - "mft": "application/rpki-manifest", - "mgp": "application/vnd.osgeo.mapguide.package", - "mgz": "application/vnd.proteus.magazine", - "mht": "message/rfc822", - "mhtml": "message/rfc822", - "mib": "text/mib", - "mid": "audio/midi", - "midi": "audio/midi", - "mie": "application/x-mie", - "mif": "application/x-mif", - "mime": "message/rfc822", - "miz": "text/mizar", - "mj2": "video/mj2", - "mjf": "audio/x-vnd.audioexplosion.mjuicemediafile", - "mjp2": "video/mj2", - "mjpg": "video/x-motion-jpeg", - "mjs": "application/javascript", - "mk": "text/makefile", - "mk3d": "video/x-matroska-3d", - "mka": "audio/x-matroska", - "mkd": "text/x-markdown", - "mks": "video/x-matroska", - "mkv": "video/x-matroska", - "mlp": "application/vnd.dolby.mlp", - "mm": "application/x-freemind", - "mmd": "application/vnd.chipnuts.karaoke-mmd", - "mmdb": "application/vnd.maxmind.maxmind-db", - "mme": "application/base64", - "mmf": "application/vnd.smaf", - "mml": "text/mathml", - "mmod": "chemical/x-macromodel-input", - "mmr": "image/vnd.fujixerox.edmics-mmr", - "mms": "application/vnd.wap.mms-message", - "mng": "video/x-mng", - "mny": "application/x-msmoney", - "mobi": "application/x-mobipocket-ebook", - "moc": "text/x-moc", - "mod": "audio/x-mod", - "model-inter": "application/vnd.vd-study", - "mods": "application/mods+xml", - "modulemap": "text/modulemap", - "mol": "chemical/x-mdl-molfile", - "mol2": "chemical/x-mol2", - "moml": "model/vnd.moml+xml", - "moo": "chemical/x-mopac-out", - "moov": "video/quicktime", - "mop": "chemical/x-mopac-input", - "mopcrt": "chemical/x-mopac-input", - "mov": "video/quicktime", - "movie": "video/x-sgi-movie", - "mp1": "audio/mpeg", - "mp2": "audio/mpeg", - "mp21": "application/mp21", - "mp2a": "audio/mpeg", - "mp3": "audio/mp3", - "mp4": "video/mp4", - "mp4a": "audio/mp4", - "mp4s": "application/mp4", - "mp4v": "video/mp4", - "mpa": "video/mpeg", - "mpc": "application/vnd.mophun.certificate", - "mpd": "application/dash+xml", - "mpdd": "application/dashdelta", - "mpe": "video/mpeg", - "mpeg": "video/mpeg", - "mpega": "audio/mpeg", - "mpf": "text/vnd.ms-mediapackage", - "mpg": "video/mpeg", - "mpg4": "video/mp4", - "mpga": "audio/mpeg", - "mpkg": "application/vnd.apple.installer+xml", - "mpm": "application/vnd.blueice.multipass", + "flo": "application/vnd.micrografx.flo", + "igx": "application/vnd.micrografx.igx", + "mif": "application/vnd.mif", + "daf": "application/vnd.mobius.daf", + "dis": "application/vnd.mobius.dis", + "mbk": "application/vnd.mobius.mbk", + "mqy": "application/vnd.mobius.mqy", + "msl": "application/vnd.mobius.msl", + "plc": "application/vnd.mobius.plc", + "txf": "application/vnd.mobius.txf", "mpn": "application/vnd.mophun.application", + "mpc": "application/vnd.mophun.certificate", + "xul": "application/vnd.mozilla.xul+xml", + "cil": "application/vnd.ms-artgalry", + "cab": "application/vnd.ms-cab-compressed", + "xls": "application/vnd.ms-excel", + "xlm": "application/vnd.ms-excel", + "xla": "application/vnd.ms-excel", + "xlc": "application/vnd.ms-excel", + "xlt": "application/vnd.ms-excel", + "xlw": "application/vnd.ms-excel", + "xlam": "application/vnd.ms-excel.addin.macroenabled.12", + "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", + "xltm": "application/vnd.ms-excel.template.macroenabled.12", + "eot": "application/vnd.ms-fontobject", + "chm": "application/vnd.ms-htmlhelp", + "ims": "application/vnd.ms-ims", + "lrm": "application/vnd.ms-lrm", + "thmx": "application/vnd.ms-officetheme", + "cat": "application/vnd.ms-pki.seccat", + "stl": "application/vnd.ms-pki.stl", + "ppt": "application/vnd.ms-powerpoint", + "pps": "application/vnd.ms-powerpoint", + "pot": "application/vnd.ms-powerpoint", + "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", + "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", + "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", + "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", + "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", "mpp": "application/vnd.ms-project", "mpt": "application/vnd.ms-project", - "mpv": "application/x-project", - "mpv2": "video/mpeg", - "mpx": "application/x-project", - "mpy": "application/vnd.ibm.minipay", - "mqy": "application/vnd.mobius.mqy", - "mrc": "application/marc", - "mrcx": "application/marcxml+xml", - "ms": "application/x-troff-ms", - "msa": "application/vnd.msa-disk-image", - "mscml": "application/mediaservercontrol+xml", - "msd": "application/vnd.fdsn.mseed", - "mseed": "application/vnd.fdsn.mseed", + "docm": "application/vnd.ms-word.document.macroenabled.12", + "dotm": "application/vnd.ms-word.template.macroenabled.12", + "wps": "application/vnd.ms-works", + "wks": "application/vnd.ms-works", + "wcm": "application/vnd.ms-works", + "wdb": "application/vnd.ms-works", + "wpl": "application/vnd.ms-wpl", + "xps": "application/vnd.ms-xpsdocument", "mseq": "application/vnd.mseq", - "msf": "application/vnd.epson.msf", - "msg": "application/vnd.ms-outlook", - "msh": "model/mesh", - "msi": "application/x-msi", - "msl": "application/vnd.mobius.msl", - "msm": "model/vnd.gdl", - "msty": "application/vnd.muvee.style", - "mtm": "audio/x-mod", - "mts": "model/vnd.mts", - "multitrack": "audio/vnd.presonus.multitrack", "mus": "application/vnd.musician", - "musd": "application/mmt-usd+xml", - "musicxml": "application/vnd.recordare.musicxml+xml", - "mv": "video/x-sgi-movie", - "mvb": "application/x-msmediaview", - "mvt": "application/vnd.mapbox-vector-tile", - "mwc": "application/vnd.dpgraph", - "mwf": "application/vnd.mfer", - "mxf": "application/mxf", - "mxi": "application/vnd.vd-study", - "mxl": "application/vnd.recordare.musicxml", - "mxmf": "audio/mobile-xmf", - "mxml": "application/xv+xml", - "mxs": "application/vnd.triscape.mxs", - "mxu": "video/vnd.mpegurl", - "my": "audio/make", - "mzz": "application/x-vnd.audioexplosion.mzz", - "n-gage": "application/vnd.nokia.n-gage.symbian.install", - "n3": "text/n3", - "nap": "image/naplps", - "naplps": "image/naplps", - "nb": "application/mathematica", - "nbp": "application/vnd.wolfram.player", - "nc": "application/x-netcdf", - "ncm": "application/vnd.nokia.configuration-message", - "ncx": "application/x-dtbncx+xml", - "ndc": "application/vnd.osa.netdeploy", - "ndjson": "application/json", - "ndl": "application/vnd.lotus-notes", - "nds": "application/vnd.nintendo.nitro.rom", - "nef": "image/x-nikon-nef", - "nfo": "text/x-nfo", - "ngdat": "application/vnd.nokia.n-gage.data", - "ngdoc": "text/ngdoc", - "nif": "image/x-niff", - "niff": "image/x-niff", + "msty": "application/vnd.muvee.style", + "taglet": "application/vnd.mynfc", + "nlu": "application/vnd.neurolanguage.nlu", "nim": "text/nim", "nimble": "text/nimble", "nimf": "text/nim", "nims": "text/nim", + "ntf": "application/vnd.nitf", "nitf": "application/vnd.nitf", - "nix": "application/x-mix-transfer", - "nlu": "application/vnd.neurolanguage.nlu", - "nml": "application/vnd.enliven", "nnd": "application/vnd.noblenet-directory", "nns": "application/vnd.noblenet-sealer", "nnw": "application/vnd.noblenet-web", - "notebook": "application/vnd.smart.notebook", - "npx": "image/vnd.net-fpx", - "nq": "application/n-quads", - "ns2": "application/vnd.lotus-notes", - "ns3": "application/vnd.lotus-notes", - "ns4": "application/vnd.lotus-notes", - "nsc": "application/x-conference", - "nsf": "application/vnd.lotus-notes", - "nsg": "application/vnd.lotus-notes", - "nsh": "application/vnd.lotus-notes", - "nt": "application/n-triples", - "ntf": "application/vnd.lotus-notes", - "numbers": "application/vnd.apple.numbers", - "nvd": "application/x-navidoc", - "nwc": "application/x-nwc", - "nws": "message/rfc822", - "nzb": "application/x-nzb", - "o": "application/x-object", - "o4a": "application/vnd.oma.drm.dcf", - "o4v": "application/vnd.oma.drm.dcf", - "oa2": "application/vnd.fujitsu.oasys2", - "oa3": "application/vnd.fujitsu.oasys3", - "oas": "application/vnd.fujitsu.oasys", - "obd": "application/x-msbinder", - "obg": "application/vnd.openblox.game-binary", - "obgx": "application/vnd.openblox.game+xml", - "obj": "application/x-tgif", - "oda": "application/oda", - "odb": "application/vnd.oasis.opendocument.database", + "ngdat": "application/vnd.nokia.n-gage.data", + "n-gage": "application/vnd.nokia.n-gage.symbian.install", + "rpst": "application/vnd.nokia.radio-preset", + "rpss": "application/vnd.nokia.radio-presets", + "edm": "application/vnd.novadigm.edm", + "edx": "application/vnd.novadigm.edx", + "ext": "application/vnd.novadigm.ext", "odc": "application/vnd.oasis.opendocument.chart", - "odd": "application/tei+xml", + "otc": "application/vnd.oasis.opendocument.chart-template", + "odb": "application/vnd.oasis.opendocument.database", "odf": "application/vnd.oasis.opendocument.formula", "odft": "application/vnd.oasis.opendocument.formula-template", "odg": "application/vnd.oasis.opendocument.graphics", - "odi": "application/vnd.oasis.opendocument.image", - "odm": "application/vnd.oasis.opendocument.text-master", - "odp": "application/vnd.oasis.opendocument.presentation", - "ods": "application/vnd.oasis.opendocument.spreadsheet", - "odt": "application/vnd.oasis.opendocument.text", - "odx": "application/odx", - "oeb": "application/vnd.openeye.oeb", - "oga": "audio/ogg", - "ogex": "model/vnd.opengex", - "ogg": "audio/ogg", - "ogv": "video/ogg", - "ogx": "application/ogg", - "old": "application/x-trash", - "omc": "application/x-omc", - "omcd": "application/x-omcdatamaker", - "omcr": "application/x-omcregerator", - "omdoc": "application/omdoc+xml", - "omg": "audio/atrac3", - "onepkg": "application/onenote", - "onetmp": "application/onenote", - "onetoc": "application/onenote", - "onetoc2": "application/onenote", - "opf": "application/oebps-package+xml", - "opml": "text/x-opml", - "oprc": "application/vnd.palm", - "opus": "audio/ogg", - "or2": "application/vnd.lotus-organizer", - "or3": "application/vnd.lotus-organizer", - "orf": "image/x-olympus-orf", - "org": "text/x-org", - "orq": "application/ocsp-request", - "ors": "application/ocsp-response", - "osf": "application/vnd.yamaha.openscoreformat", - "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", - "osm": "application/vnd.openstreetmap.data+xml", - "otc": "application/vnd.oasis.opendocument.chart-template", - "otf": "font/otf", "otg": "application/vnd.oasis.opendocument.graphics-template", - "oth": "application/vnd.oasis.opendocument.text-web", + "odi": "application/vnd.oasis.opendocument.image", "oti": "application/vnd.oasis.opendocument.image-template", + "odp": "application/vnd.oasis.opendocument.presentation", "otp": "application/vnd.oasis.opendocument.presentation-template", + "ods": "application/vnd.oasis.opendocument.spreadsheet", "ots": "application/vnd.oasis.opendocument.spreadsheet-template", + "odt": "application/vnd.oasis.opendocument.text", + "odm": "application/vnd.oasis.opendocument.text-master", "ott": "application/vnd.oasis.opendocument.text-template", - "ova": "application/x-virtualbox-ova", - "ovf": "application/x-virtualbox-ovf", - "owx": "application/owl+xml", - "oxlicg": "application/vnd.oxli.countgraph", - "oxps": "application/oxps", + "oth": "application/vnd.oasis.opendocument.text-web", + "xo": "application/vnd.olpc-sugar", + "dd2": "application/vnd.oma.dd2+xml", "oxt": "application/vnd.openofficeorg.extension", - "oza": "application/x-oz-application", - "p": "text/x-pascal", - "p10": "application/pkcs10", - "p12": "application/pkcs12", - "p2p": "application/vnd.wfa.p2p", - "p7a": "application/x-pkcs7-signature", - "p7b": "application/x-pkcs7-certificates", - "p7c": "application/pkcs7-mime", - "p7m": "application/pkcs7-mime", - "p7r": "application/x-pkcs7-certreqresp", - "p7s": "application/pkcs7-signature", - "p8": "application/pkcs8", - "pac": "application/x-ns-proxy-autoconfig", - "pack": "application/x-java-pack200", - "package": "application/vnd.autopackage", - "pages": "application/vnd.apple.pages", - "par": "text/plain-bas", - "part": "application/pro_eng", - "pas": "text/pascal", - "pat": "image/x-coreldrawpattern", - "patch": "text/x-diff", - "paw": "application/vnd.pawaafile", - "pbd": "application/vnd.powerbuilder6", - "pbm": "image/x-portable-bitmap", - "pcap": "application/vnd.tcpdump.pcap", - "pcf": "application/x-font-pcf", - "pcl": "application/vnd.hp-pcl", - "pclxl": "application/vnd.hp-pclxl", - "pct": "image/x-pict", - "pcurl": "application/vnd.curl.pcurl", - "pcx": "image/x-pcx", - "pdb": "application/vnd.palm", - "pde": "text/x-processing", - "pdf": "application/pdf", - "pdx": "application/pdx", - "pem": "text/pem", - "pfa": "application/x-font-type1", - "pfb": "application/x-font-type1", - "pfm": "application/x-font-type1", - "pfr": "application/font-tdpfr", - "pfunk": "audio/make", - "pfx": "application/pkcs12", - "pgb": "image/vnd.globalgraphics.pgb", - "pgm": "image/x-portable-graymap", - "pgn": "application/x-chess-pgn", - "pgp": "application/pgp-encrypted", - "php": "application/x-httpd-php", - "php3": "application/x-httpd-php3", - "php3p": "application/x-httpd-php3-preprocessed", - "php4": "application/x-httpd-php4", - "php5": "application/x-httpd-php5", - "phps": "application/x-httpd-php-source", - "pht": "application/x-httpd-php", - "phtml": "application/x-httpd-php", - "pic": "image/pict", - "pict": "image/pict", - "pil": "application/vnd.piaccess.application-license", - "pk": "application/x-tex-pk", - "pkd": "application/vnd.hbci", - "pkg": "application/vnd.apple.installer+xml", - "pki": "application/pkixcmp", - "pkipath": "application/pkix-pkipath", - "pko": "application/ynd.ms-pkipko", - "pkpass": "application/vnd.apple.pkpass", - "pl": "application/x-perl", - "plantuml": "text/plantuml", - "plb": "application/vnd.3gpp.pic-bw-large", - "plc": "application/vnd.mobius.plc", - "plf": "application/vnd.pocketlearn", - "plj": "audio/vnd.everad.plj", - "plp": "application/vnd.panoply", - "pls": "application/pls+xml", - "plx": "application/x-pixclscript", - "ply": "model/stanford", - "pm": "text/plain", - "pm4": "application/x-pagemaker", - "pm5": "application/x-pagemaker", - "pma": "application/x-perfmon", - "pmc": "application/x-perfmon", - "pml": "application/vnd.ctc-posml", - "pmr": "application/x-perfmon", - "pmw": "application/x-perfmon", - "png": "image/png", - "pnm": "image/x-portable-anymap", - "po": "text/pofile", - "pod": "text/x-pod", - "portpkg": "application/vnd.macports.portpkg", - "pot": "application/vnd.ms-powerpoint", - "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", - "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", - "pov": "model/x-pov", - "pp": "text/puppet", - "ppa": "application/vnd.ms-powerpoint", - "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", - "ppd": "application/vnd.cups-ppd", - "ppkg": "application/vnd.xmpie.ppkg", - "ppm": "image/x-portable-pixmap", - "pps": "application/vnd.ms-powerpoint", - "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", - "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", - "ppt": "application/vnd.ms-powerpoint", - "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "ppz": "application/mspowerpoint", + "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "mgp": "application/vnd.osgeo.mapguide.package", + "dp": "application/vnd.osgi.dp", + "esa": "application/vnd.osgi.subsystem", + "pdb": "application/vnd.palm", "pqa": "application/vnd.palm", - "prc": "application/vnd.palm", - "pre": "application/vnd.lotus-freelance", - "preminet": "application/vnd.preminet", - "prf": "application/pics-rules", - "proto": "text/proto", - "provn": "text/provenance-notation", - "provx": "application/provenance+xml", - "prt": "application/pro_eng", - "prz": "application/vnd.lotus-freelance", - "ps": "application/postscript", - "psb": "application/vnd.3gpp.pic-bw-small", - "psd": "image/vnd.adobe.photoshop", - "pseg3820": "application/vnd.ibm.modcap", - "psf": "application/x-font-linux-psf", - "psid": "audio/prs.sid", - "pskcxml": "application/pskc+xml", - "pti": "image/prs.pti", - "ptid": "application/vnd.pvi.ptid1", - "pub": "application/x-mspublisher", - "purs": "text/purescript", - "pvb": "application/vnd.3gpp.pic-bw-var", - "pvu": "paleovu/x-pv", - "pwn": "application/vnd.3m.post-it-notes", - "pwz": "application/vnd.ms-powerpoint", - "pxd": "text/cython", - "pxi": "text/cython", - "py": "text/x-script.phyton", - "pya": "audio/vnd.ms-playready.media.pya", - "pyc": "application/x-python-code", - "pyi": "text/pyi", - "pyo": "application/x-python-code", - "pyv": "video/vnd.ms-playready.media.pyv", - "pyx": "text/cython", - "qam": "application/vnd.epson.quickanime", - "qbo": "application/vnd.intu.qbo", - "qca": "application/vnd.ericsson.quickcall", - "qcall": "application/vnd.ericsson.quickcall", - "qcp": "audio/qcelp", - "qd3": "x-world/x-3dmf", - "qd3d": "x-world/x-3dmf", - "qfx": "application/vnd.intu.qfx", - "qgs": "application/x-qgis", - "qif": "image/x-quicktime", + "oprc": "application/vnd.palm", + "paw": "application/vnd.pawaafile", + "str": "application/vnd.pg.format", + "ei6": "application/vnd.pg.osasli", + "efif": "application/vnd.picsel", + "wg": "application/vnd.pmi.widget", + "plf": "application/vnd.pocketlearn", + "pbd": "application/vnd.powerbuilder6", + "box": "application/vnd.previewsystems.box", + "mgz": "application/vnd.proteus.magazine", "qps": "application/vnd.publishare-delta-tree", - "qt": "video/quicktime", - "qtc": "video/x-qtc", - "qti": "image/x-quicktime", - "qtif": "image/x-quicktime", - "qtl": "application/x-quicktimeplayer", - "quiz": "application/vnd.quobject-quoxdocument", - "quox": "application/vnd.quobject-quoxdocument", - "qvd": "application/vnd.theqvd", + "ptid": "application/vnd.pvi.ptid1", + "qxd": "application/vnd.quark.quarkxpress", + "qxt": "application/vnd.quark.quarkxpress", "qwd": "application/vnd.quark.quarkxpress", "qwt": "application/vnd.quark.quarkxpress", - "qxb": "application/vnd.quark.quarkxpress", - "qxd": "application/vnd.quark.quarkxpress", "qxl": "application/vnd.quark.quarkxpress", - "qxt": "application/vnd.quark.quarkxpress", - "r": "text/r", - "ra": "audio/x-realaudio", - "ram": "audio/x-pn-realaudio", - "raml": "application/raml+yaml", - "rapd": "application/route-apd+xml", - "rar": "application/x-rar-compressed", - "ras": "image/x-cmu-raster", - "rast": "image/cmu-raster", - "rb": "application/x-ruby", - "rcprofile": "application/vnd.ipunplugged.rcprofile", - "rct": "application/prs.nprend", - "rd": "chemical/x-mdl-rdfile", - "rda": "text/r", - "rdata": "text/r", - "rds": "text/r", - "rdf": "application/rdf+xml", - "rdf-crypt": "application/prs.rdf-xml-crypt", - "rdz": "application/vnd.data-vision.rdz", - "relo": "application/p2p-overlay+xml", - "rep": "application/vnd.businessobjects", - "request": "application/vnd.nervana", - "res": "application/x-dtbresource+xml", - "rexx": "text/x-script.rexx", - "rf": "image/vnd.rn-realflash", - "rfcxml": "application/rfc+xml", - "rgb": "image/x-rgb", - "rgbe": "image/vnd.radiance", - "rhtml": "application/x-httpd-eruby", - "rif": "application/reginfo+xml", - "rip": "audio/vnd.rip", - "ris": "application/x-research-info-systems", - "rl": "application/resource-lists+xml", - "rlc": "image/vnd.fujixerox.edmics-rlc", - "rld": "application/resource-lists-diff+xml", - "rlib": "text/rust", + "qxb": "application/vnd.quark.quarkxpress", + "bed": "application/vnd.realvnc.bed", + "mxl": "application/vnd.recordare.musicxml", + "musicxml": "application/vnd.recordare.musicxml+xml", + "cryptonote": "application/vnd.rig.cryptonote", + "cod": "application/vnd.rim.cod", "rm": "application/vnd.rn-realmedia", - "rmi": "audio/mid", - "rmm": "audio/x-pn-realaudio", - "rmp": "audio/x-pn-realaudio-plugin", - "rms": "application/vnd.jcp.javame.midlet-rms", "rmvb": "application/vnd.rn-realmedia-vbr", - "rnc": "application/relax-ng-compact-syntax", - "rnd": "application/prs.nprend", - "rng": "text/xml", - "rnx": "application/vnd.rn-realplayer", - "roa": "application/rpki-roa", - "roff": "text/troff", - "ros": "chemical/x-rosdal", - "rp": "image/vnd.rn-realpix", - "rp9": "application/vnd.cloanto.rp9", - "rpm": "application/x-redhat-package-manager", - "rpss": "application/vnd.nokia.radio-presets", - "rpst": "application/vnd.nokia.radio-preset", - "rq": "application/sparql-query", - "rs": "application/rls-services+xml", - "rsd": "application/rsd+xml", - "rsheet": "application/urc-ressheet+xml", - "rsm": "model/vnd.gdl", - "rss": "application/rss+xml", - "rst": "text/prs.fallenstein.rst", - "rt": "text/richtext", - "rtf": "text/rtf", - "rtx": "text/richtext", - "run": "application/x-makeself", - "rusd": "application/route-usd+xml", - "rv": "video/vnd.rn-realvideo", - "rxn": "chemical/x-mdl-rxnfile", - "s": "text/x-asm", - "s11": "video/vnd.sealed.mpeg1", - "s14": "video/vnd.sealed.mpeg4", - "s1a": "application/vnd.sealedmedia.softseal.pdf", - "s1e": "application/vnd.sealed.xls", - "s1g": "image/vnd.sealedmedia.softseal.gif", - "s1h": "application/vnd.sealedmedia.softseal.html", - "s1j": "image/vnd.sealedmedia.softseal.jpg", - "s1m": "audio/vnd.sealedmedia.softseal.mpeg", - "s1n": "image/vnd.sealed.png", - "s1p": "application/vnd.sealed.ppt", - "s1q": "video/vnd.sealedmedia.softseal.mov", - "s1w": "application/vnd.sealed.doc", - "s3df": "application/vnd.sealed.3df", - "s3m": "audio/s3m", - "sac": "application/tamp-sequence-adjust-confirm", - "saf": "application/vnd.yamaha.smaf-audio", - "sam": "application/vnd.lotus-wordpro", - "sandboxed": "text/html-sandboxed", - "sass": "text/x-sass", - "saveme": "application/octet-stream", - "sbk": "application/x-tbook", - "sbml": "application/sbml+xml", - "sc": "application/vnd.ibm.secure-container", - "scala": "text/x-scala", - "scd": "application/x-msschedule", - "sce": "application/vnd.etsi.asic-e+zip", - "scim": "application/scim+json", - "scld": "application/vnd.doremir.scorecloud-binary-document", - "scm": "application/vnd.lotus-screencam", - "scq": "application/scvp-cv-request", - "scr": "application/x-silverlight", - "scs": "application/scvp-cv-response", - "scsf": "application/vnd.sealed.csf", - "scss": "text/x-scss", - "sct": "text/scriptlet", - "scurl": "text/vnd.curl.scurl", - "sd": "chemical/x-mdl-sdfile", - "sd2": "audio/x-sd2", - "sda": "application/vnd.stardivision.draw", - "sdc": "application/vnd.stardivision.calc", - "sdd": "application/vnd.stardivision.impress", - "sdf": "application/vnd.kinar", - "sdkd": "application/vnd.solent.sdkm+xml", - "sdkm": "application/vnd.solent.sdkm+xml", - "sdml": "text/plain", - "sdo": "application/vnd.sealed.doc", - "sdoc": "application/vnd.sealed.doc", - "sdp": "application/sdp", - "sdr": "application/sounder", - "sdw": "application/vnd.stardivision.writer", - "sea": "application/x-sea", + "link66": "application/vnd.route66.link66+xml", + "st": "application/vnd.sailingtracker.track", "see": "application/vnd.seemail", - "seed": "application/vnd.fdsn.seed", - "sem": "application/vnd.sealed.eml", "sema": "application/vnd.sema", "semd": "application/vnd.semd", "semf": "application/vnd.semf", - "seml": "application/vnd.sealed.eml", - "ser": "application/java-serialized-object", - "set": "application/set", - "setpay": "application/set-payment-initiation", - "setreg": "application/set-registration-initiation", - "sfc": "application/vnd.nintendo.snes.rom", - "sfd": "application/vnd.font-fontforge-sfd", - "sfd-hdstx": "application/vnd.hydrostatix.sof-data", - "sfs": "application/vnd.spotfire.sfs", - "sfv": "text/x-sfv", - "sgf": "application/x-go-sgf", - "sgi": "image/sgi", - "sgif": "image/vnd.sealedmedia.softseal.gif", - "sgl": "application/vnd.stardivision.writer-global", - "sgm": "text/sgml", - "sgml": "text/sgml", - "sh": "application/x-sh", - "shar": "application/x-shar", - "shex": "text/shex", - "shf": "application/shf+xml", - "shp": "application/x-qgis", - "shx": "application/x-qgis", - "si": "text/vnd.wap.si", - "sic": "application/vnd.wap.sic", - "sid": "image/x-mrsid-image", - "sieve": "application/sieve", - "sig": "application/pgp-signature", - "sik": "application/x-trash", - "sil": "audio/silk", - "silo": "model/mesh", - "sis": "application/vnd.symbian.install", - "sisx": "x-epoc/x-sisx-app", - "sit": "application/x-stuffit", - "sitx": "application/x-stuffitx", - "siv": "application/sieve", - "sjp": "image/vnd.sealedmedia.softseal.jpg", - "sjpg": "image/vnd.sealedmedia.softseal.jpg", - "skd": "application/vnd.koan", - "skm": "application/vnd.koan", - "skp": "application/vnd.koan", - "skt": "application/vnd.koan", - "sl": "text/vnd.wap.sl", - "sla": "application/vnd.scribus", - "slaz": "application/vnd.scribus", - "slc": "application/vnd.wap.slc", - "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", - "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", - "sls": "application/route-s-tsid+xml", - "slt": "application/vnd.epson.salt", - "sm": "application/vnd.stepmania.stepchart", - "smc": "application/vnd.nintendo.snes.rom", - "smf": "application/vnd.stardivision.math", - "smh": "application/vnd.sealed.mht", - "smht": "application/vnd.sealed.mht", - "smi": "application/smil+xml", - "smil": "application/smil+xml", - "smk": "video/vnd.radgamettools.smacker", - "sml": "application/smil+xml", - "smo": "video/vnd.sealedmedia.softseal.mov", - "smov": "video/vnd.sealedmedia.softseal.mov", - "smp": "audio/vnd.sealedmedia.softseal.mpeg", - "smp3": "audio/vnd.sealedmedia.softseal.mpeg", - "smpg": "video/vnd.sealed.mpeg1", - "sms": "application/vnd.3gpp2.sms", - "smv": "video/x-smv", - "smzip": "application/vnd.stepmania.package", - "snd": "audio/basic", - "snf": "application/x-font-snf", - "so": "application/octet-stream", - "soa": "text/dns", - "soc": "application/sgml-open-catalog", - "sol": "application/solids", - "spc": "text/x-speech", - "spd": "application/vnd.sealedmedia.softseal.pdf", - "spdf": "application/vnd.sealedmedia.softseal.pdf", - "spec": "text/spec", - "spf": "application/vnd.yamaha.smaf-phrase", - "spl": "application/x-futuresplash", - "spn": "image/vnd.sealed.png", - "spng": "image/vnd.sealed.png", - "spo": "text/vnd.in3d.spot", - "spot": "text/vnd.in3d.spot", - "spp": "application/scvp-vp-response", - "sppt": "application/vnd.sealed.ppt", - "spq": "application/scvp-vp-request", - "spr": "application/x-sprite", - "sprite": "application/x-sprite", - "spx": "audio/ogg", - "sql": "application/x-sql", - "sr": "application/vnd.sigrok.session", - "src": "application/x-wais-source", - "srt": "application/x-subrip", - "sru": "application/sru+xml", - "srx": "application/sparql-results+xml", - "ssdl": "application/ssdl+xml", - "sse": "application/vnd.kodak-descriptor", - "ssf": "application/vnd.epson.ssf", - "ssi": "text/x-server-parsed-html", - "ssm": "application/streamingmedia", - "ssml": "application/ssml+xml", - "sst": "application/vnd.ms-pki.certstore", - "ssw": "video/vnd.sealed.swf", - "sswf": "video/vnd.sealed.swf", - "st": "application/vnd.sailingtracker.track", - "stc": "application/vnd.sun.xml.calc.template", - "std": "application/vnd.sun.xml.draw.template", - "step": "application/step", - "stf": "application/vnd.wt.stf", - "sti": "application/vnd.sun.xml.impress.template", - "stif": "application/vnd.sealed.tiff", - "stk": "application/hyperstudio", - "stl": "application/vnd.ms-pki.stl", - "stm": "audio/x-stm", - "stml": "application/vnd.sealedmedia.softseal.html", - "stp": "application/step", - "str": "application/vnd.pg.format", - "study-inter": "application/vnd.vd-study", - "stw": "application/vnd.sun.xml.writer.template", - "sty": "text/x-tex", - "styl": "text/stylus", - "sub": "text/vnd.dvb.subtitle", - "sus": "application/vnd.sus-calendar", - "susp": "application/vnd.sus-calendar", - "sv4cpio": "application/x-sv4cpio", - "sv4crc": "application/x-sv4crc", - "svc": "application/vnd.dvb.service", - "svd": "application/vnd.svd", - "svf": "image/x-dwg", - "svg": "image/svg+xml", - "svgz": "image/svg+xml", - "sw": "chemical/x-swissprot", - "swa": "application/x-director", - "swf": "application/x-shockwave-flash", - "swfl": "application/x-shockwave-flash", - "swi": "application/vnd.aristanetworks.swi", - "swift": "text/swift", - "swiftdeps": "text/swiftdeps", - "sxc": "application/vnd.sun.xml.calc", - "sxd": "application/vnd.sun.xml.draw", - "sxg": "application/vnd.sun.xml.writer.global", - "sxi": "application/vnd.sun.xml.impress", - "sxl": "application/vnd.sealed.xls", - "sxls": "application/vnd.sealed.xls", - "sxm": "application/vnd.sun.xml.math", - "sxw": "application/vnd.sun.xml.writer", - "t": "text/troff", - "t3": "application/x-t3vm-image", - "t38": "image/t38", - "tac": "text/twisted", - "tag": "text/prs.lines.tag", - "taglet": "application/vnd.mynfc", - "talk": "text/x-speech", - "tam": "application/vnd.onepager", - "tamp": "application/vnd.onepagertamp", - "tamx": "application/vnd.onepagertamx", - "tao": "application/vnd.tao.intent-module-archive", - "tap": "image/vnd.tencent.tap", - "tar": "application/x-tar", - "tat": "application/vnd.onepagertat", - "tatp": "application/vnd.onepagertatp", - "tatx": "application/vnd.onepagertatx", - "tau": "application/tamp-apex-update", - "taz": "application/x-gtar", - "tbk": "application/toolbook", - "tcap": "application/vnd.3gpp2.tcap", - "tcl": "application/x-tcl", - "tcsh": "text/x-script.tcsh", - "tcu": "application/tamp-community-update", - "td": "application/urc-targetdesc+xml", - "teacher": "application/vnd.smart.teacher", - "tei": "application/tei+xml", - "teicorpus": "application/tei+xml", - "ter": "application/tamp-error", - "tex": "application/x-tex", - "texi": "application/x-texinfo", - "texinfo": "application/x-texinfo", - "text": "text/plain", - "tf": "text/terraform", - "tfi": "application/thraud+xml", - "tfm": "application/x-tex-tfm", - "tfx": "image/tiff-fx", - "tga": "image/x-tga", - "tgf": "chemical/x-mdl-tgf", - "tgz": "application/gzip", - "thmx": "application/vnd.ms-officetheme", - "thrift": "text/thrift", - "tif": "image/tiff", - "tiff": "image/tiff", - "tk": "text/x-tcl", - "tlclient": "application/vnd.cendio.thinlinc.clientconf", - "tm": "text/texmacs", - "tmo": "application/vnd.tmobile-livetv", - "tnef": "application/vnd.ms-tnef", - "tnf": "application/vnd.ms-tnef", - "toml": "text/toml", - "torrent": "application/x-bittorrent", - "tpl": "application/vnd.groove-tool-template", - "tpt": "application/vnd.trid.tpt", - "tr": "text/troff", - "tra": "application/vnd.trueapp", - "tree": "application/vnd.rainstor.data", - "trig": "application/trig", - "trm": "application/x-msterminal", - "ts": "video/mp2t", - "tsa": "application/tamp-sequence-adjust", - "tscn": "text/godot", - "tsd": "application/timestamped-data", - "tsi": "audio/tsp-audio", - "tsp": "audio/tsplayer", - "tsq": "application/timestamp-query", - "tsr": "application/timestamp-reply", - "tst": "application/vnd.etsi.timestamp-token", - "tsv": "text/tab-separated-values", - "tsx": "text/tsx", - "ttc": "font/collection", - "ttf": "font/ttf", - "ttl": "text/turtle", - "ttml": "application/ttml+xml", - "tuc": "application/tamp-update-confirm", - "tur": "application/tamp-update", - "turbot": "image/florian", + "ifm": "application/vnd.shana.informed.formdata", + "itp": "application/vnd.shana.informed.formtemplate", + "iif": "application/vnd.shana.informed.interchange", + "ipk": "application/vnd.shana.informed.package", "twd": "application/vnd.simtech-mindmapper", "twds": "application/vnd.simtech-mindmapper", - "txd": "application/vnd.genomatix.tuxedo", - "txf": "application/vnd.mobius.txf", - "txt": "text/plain", - "u32": "application/x-authorware-bin", - "u8dsn": "message/global-delivery-status", - "u8hdr": "message/global-headers", - "u8mdn": "message/global-disposition-notification", - "u8msg": "message/global", - "udeb": "application/vnd.debian.binary-package", + "mmf": "application/vnd.smaf", + "teacher": "application/vnd.smart.teacher", + "sdkm": "application/vnd.solent.sdkm+xml", + "sdkd": "application/vnd.solent.sdkm+xml", + "dxp": "application/vnd.spotfire.dxp", + "sfs": "application/vnd.spotfire.sfs", + "sdc": "application/vnd.stardivision.calc", + "sda": "application/vnd.stardivision.draw", + "sdd": "application/vnd.stardivision.impress", + "smf": "application/vnd.stardivision.math", + "sdw": "application/vnd.stardivision.writer", + "vor": "application/vnd.stardivision.writer", + "sgl": "application/vnd.stardivision.writer-global", + "smzip": "application/vnd.stepmania.package", + "sm": "application/vnd.stepmania.stepchart", + "sxc": "application/vnd.sun.xml.calc", + "stc": "application/vnd.sun.xml.calc.template", + "sxd": "application/vnd.sun.xml.draw", + "std": "application/vnd.sun.xml.draw.template", + "sxi": "application/vnd.sun.xml.impress", + "sti": "application/vnd.sun.xml.impress.template", + "sxm": "application/vnd.sun.xml.math", + "sxw": "application/vnd.sun.xml.writer", + "sxg": "application/vnd.sun.xml.writer.global", + "stw": "application/vnd.sun.xml.writer.template", + "sus": "application/vnd.sus-calendar", + "susp": "application/vnd.sus-calendar", + "svd": "application/vnd.svd", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "xsm": "application/vnd.syncml+xml", + "bdm": "application/vnd.syncml.dm+wbxml", + "xdm": "application/vnd.syncml.dm+xml", + "tao": "application/vnd.tao.intent-module-archive", + "pcap": "application/vnd.tcpdump.pcap", + "cap": "application/vnd.tcpdump.pcap", + "dmp": "application/vnd.tcpdump.pcap", + "tmo": "application/vnd.tmobile-livetv", + "tpt": "application/vnd.trid.tpt", + "mxs": "application/vnd.triscape.mxs", + "tra": "application/vnd.trueapp", "ufd": "application/vnd.ufdl", "ufdl": "application/vnd.ufdl", - "uil": "text/x-uil", - "uis": "application/urc-uisocketdesc+xml", - "uls": "text/iuls", - "ult": "audio/x-mod", - "ulx": "application/x-glulx", - "umj": "application/vnd.umajin", - "uni": "audio/x-mod", - "unis": "text/uri-list", - "unityweb": "application/vnd.unity", - "unv": "application/i-deas", - "uo": "application/vnd.uoml+xml", - "uoml": "application/vnd.uoml+xml", - "upa": "application/vnd.hbci", - "uri": "text/uri-list", - "uric": "text/vnd.si.uricatalogue", - "urim": "application/vnd.uri-map", - "urimap": "application/vnd.uri-map", - "uris": "text/uri-list", - "urls": "text/uri-list", - "ustar": "application/x-ustar", "utz": "application/vnd.uiq.theme", - "uu": "text/x-uuencode", - "uue": "text/x-uuencode", - "uva": "audio/vnd.dece.audio", - "uvd": "application/vnd.dece.data", - "uvf": "application/vnd.dece.data", - "uvg": "image/vnd.dece.graphic", - "uvh": "video/vnd.dece.hd", - "uvi": "image/vnd.dece.graphic", - "uvm": "video/vnd.dece.mobile", - "uvp": "video/vnd.dece.pd", - "uvs": "video/vnd.dece.sd", - "uvt": "application/vnd.dece.ttml+xml", - "uvu": "video/vnd.dece.mp4", - "uvv": "video/vnd.dece.video", - "uvva": "audio/vnd.dece.audio", - "uvvd": "application/vnd.dece.data", - "uvvf": "application/vnd.dece.data", - "uvvg": "image/vnd.dece.graphic", - "uvvh": "video/vnd.dece.hd", - "uvvi": "image/vnd.dece.graphic", - "uvvm": "video/vnd.dece.mobile", - "uvvp": "video/vnd.dece.pd", - "uvvs": "video/vnd.dece.sd", - "uvvt": "application/vnd.dece.ttml+xml", - "uvvu": "video/vnd.dece.mp4", - "uvvv": "video/vnd.dece.video", - "uvvx": "application/vnd.dece.unspecified", - "uvvz": "application/vnd.dece.zip", - "uvx": "application/vnd.dece.unspecified", - "uvz": "application/vnd.dece.zip", - "val": "chemical/x-ncbi-asn1-binary", - "vbk": "audio/vnd.nortel.vbk", - "vbox": "application/x-virtualbox-vbox", - "vbox-extpack": "application/x-virtualbox-vbox-extpack", - "vcard": "text/vcard", - "vcd": "application/x-cdlink", - "vcf": "text/x-vcard", - "vcg": "application/vnd.groove-vcard", - "vcs": "text/x-vcalendar", + "umj": "application/vnd.umajin", + "unityweb": "application/vnd.unity", + "uoml": "application/vnd.uoml+xml", "vcx": "application/vnd.vcx", - "vda": "application/vda", - "vdi": "application/x-virtualbox-vdi", - "vdo": "video/vdo", - "vdx": "text/vdx", - "vew": "application/vnd.lotus-approach", - "vfr": "application/vnd.tml", - "vhd": "application/x-virtualbox-vhd", - "viaframe": "application/vnd.tml", - "vim": "text/vim", - "vis": "application/vnd.visionary", - "viv": "video/vnd.vivo", - "vivo": "video/vivo", - "vmd": "application/vocaltec-media-desc", - "vmdk": "application/x-virtualbox-vmdk", - "vmf": "application/vocaltec-media-file", - "vms": "chemical/x-vamas-iso14976", - "vmt": "application/vnd.valve.source.material", - "vob": "video/x-ms-vob", - "voc": "audio/voc", - "vor": "application/vnd.stardivision.writer", - "vos": "video/vosaic", - "vox": "audio/voxware", - "vpm": "multipart/voice-message", - "vqe": "audio/x-twinvq-plugin", - "vqf": "audio/x-twinvq", - "vql": "audio/x-twinvq-plugin", - "vrm": "x-world/x-vrml", - "vrml": "model/vrml", - "vrt": "x-world/x-vrt", - "vsc": "application/vnd.vidsoft.vidconference", "vsd": "application/vnd.visio", - "vsf": "application/vnd.vsf", - "vss": "application/vnd.visio", "vst": "application/vnd.visio", + "vss": "application/vnd.visio", "vsw": "application/vnd.visio", - "vtf": "image/vnd.valve.source.texture", - "vtt": "text/vtt", - "vtu": "model/vnd.vtu", - "vue": "text/vue", - "vwx": "application/vnd.vectorworks", - "vxml": "application/voicexml+xml", - "w3d": "application/x-director", - "w60": "application/wordperfect6.0", - "w61": "application/wordperfect6.1", - "w6w": "application/msword", - "wad": "application/x-doom", - "wadl": "application/vnd.sun.wadl+xml", - "war": "binary/zip", - "wasm": "application/wasm", - "wav": "audio/wave", - "wax": "audio/x-ms-wax", - "wb1": "application/x-qpro", - "wbmp": "image/vnd.wap.wbmp", - "wbs": "application/vnd.criticaltools.wbs+xml", + "vis": "application/vnd.visionary", + "vsf": "application/vnd.vsf", "wbxml": "application/vnd.wap.wbxml", - "wcm": "application/vnd.ms-works", - "wdb": "application/vnd.ms-works", - "wdp": "image/vnd.ms-photo", - "web": "application/vnd.xara", - "weba": "audio/webm", - "webapp": "application/x-web-app-manifest+json", - "webm": "video/webm", - "webmanifest": "application/manifest+json", - "webp": "image/webp", - "wg": "application/vnd.pmi.widget", - "wgt": "application/widget", - "whl": "binary/wheel", - "wif": "application/watcherinfo+xml", - "win": "model/vnd.gdl", - "wiz": "application/msword", - "wk": "application/x-123", - "wk1": "application/vnd.lotus-1-2-3", - "wk3": "application/vnd.lotus-1-2-3", - "wk4": "application/vnd.lotus-1-2-3", - "wks": "application/vnd.ms-works", - "wkt": "text/wkt", - "wlnk": "application/link-format", - "wm": "video/x-ms-wm", - "wma": "audio/x-ms-wma", - "wmc": "application/vnd.wmc", - "wmd": "application/x-ms-wmd", - "wmf": "image/wmf", - "wml": "text/vnd.wap.wml", "wmlc": "application/vnd.wap.wmlc", - "wmls": "text/vnd.wap.wmlscript", "wmlsc": "application/vnd.wap.wmlscriptc", - "wmv": "video/x-ms-wmv", - "wmx": "video/x-ms-wmx", - "wmz": "application/x-ms-wmz", - "woff": "font/woff", - "woff2": "font/woff2", - "word": "application/msword", - "wp": "application/wordperfect", - "wp5": "application/wordperfect", - "wp6": "application/wordperfect", - "wpd": "application/vnd.wordperfect", - "wpl": "application/vnd.ms-wpl", - "wps": "application/vnd.ms-works", - "wq1": "application/x-lotus", - "wqd": "application/vnd.wqd", - "wri": "application/x-mswrite", - "wrl": "model/vrml", - "wrz": "model/vrml", - "wsc": "message/vnd.wfa.wsc", - "wsdl": "application/wsdl+xml", - "wsgi": "text/wsgi", - "wspolicy": "application/wspolicy+xml", - "wsrc": "application/x-wais-source", "wtb": "application/vnd.webturbo", - "wtk": "application/x-wintalk", - "wv": "application/vnd.wv.csp+wbxml", - "wvx": "video/x-ms-wvx", - "wz": "application/x-wingz", - "x-png": "image/png", - "x32": "application/x-authorware-bin", - "x3d": "application/vnd.hzn-3d-crossword", - "x3db": "model/x3d+xml", - "x3dbz": "model/x3d+binary", - "x3dv": "model/x3d-vrml", - "x3dvz": "model/x3d-vrml", - "x3dz": "model/x3d+xml", - "x_b": "model/vnd.parasolid.transmit.binary", - "x_t": "model/vnd.parasolid.transmit.text", - "xaf": "x-world/x-vrml", - "xaml": "application/xaml+xml", - "xap": "application/x-silverlight-app", + "nbp": "application/vnd.wolfram.player", + "wpd": "application/vnd.wordperfect", + "wqd": "application/vnd.wqd", + "stf": "application/vnd.wt.stf", "xar": "application/vnd.xara", - "xav": "application/xcap-att+xml", - "xbap": "application/x-ms-xbap", - "xbd": "application/vnd.fujixerox.docuworks.binder", - "xbm": "image/x-xbitmap", - "xca": "application/xcap-caps+xml", - "xcf": "application/x-xcf", - "xcs": "application/calendar+xml", - "xct": "application/vnd.fujixerox.docuworks.container", - "xdd": "application/bacnet-xdd+zip", - "xdf": "application/xcap-diff+xml", - "xdm": "application/vnd.syncml.dm+xml", - "xdp": "application/vnd.adobe.xdp+xml", - "xdr": "video/x-amt-demorun", - "xdssc": "application/dssc+xml", - "xdw": "application/vnd.fujixerox.docuworks", - "xel": "application/xcap-el+xml", - "xenc": "application/xenc+xml", - "xer": "application/patch-ops-error+xml", - "xfd": "application/vnd.xfdl", - "xfdf": "application/vnd.adobe.xfdf", "xfdl": "application/vnd.xfdl", - "xgz": "xgl/drawing", - "xht": "application/xhtml+xml", - "xhtm": "application/xhtml+xml", - "xhtml": "application/xhtml+xml", - "xhvml": "application/xv+xml", - "xif": "image/vnd.xiff", - "xl": "application/excel", - "xla": "application/vnd.ms-excel", - "xlam": "application/vnd.ms-excel.addin.macroenabled.12", - "xlb": "application/vndms-excel", - "xlc": "application/vnd.ms-excel", + "hvd": "application/vnd.yamaha.hv-dic", + "hvs": "application/vnd.yamaha.hv-script", + "hvp": "application/vnd.yamaha.hv-voice", + "osf": "application/vnd.yamaha.openscoreformat", + "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", + "saf": "application/vnd.yamaha.smaf-audio", + "spf": "application/vnd.yamaha.smaf-phrase", + "cmp": "application/vnd.yellowriver-custom-menu", + "zir": "application/vnd.zul", + "zirz": "application/vnd.zul", + "zaz": "application/vnd.zzazz.deck+xml", + "vxml": "application/voicexml+xml", + "wasm": "application/wasm", + "wgt": "application/widget", + "hlp": "application/winhlp", + "wsdl": "application/wsdl+xml", + "wspolicy": "application/wspolicy+xml", + "7z": "application/x-7z-compressed", + "abw": "application/x-abiword", + "ace": "application/x-ace-compressed", + "dmg": "application/x-apple-diskimage", + "aab": "application/x-authorware-bin", + "x32": "application/x-authorware-bin", + "u32": "application/x-authorware-bin", + "vox": "application/x-authorware-bin", + "aam": "application/x-authorware-map", + "aas": "application/x-authorware-seg", + "bcpio": "application/x-bcpio", + "torrent": "application/x-bittorrent", + "blb": "application/x-blorb", + "blorb": "application/x-blorb", + "bz": "application/x-bzip", + "bz2": "application/x-bzip2", + "boz": "application/x-bzip2", + "cbr": "application/x-cbr", + "cba": "application/x-cbr", + "cbt": "application/x-cbr", + "cbz": "application/x-cbr", + "cb7": "application/x-cbr", + "vcd": "application/x-cdlink", + "cfs": "application/x-cfs-compressed", + "chat": "application/x-chat", + "pgn": "application/x-chess-pgn", + "nsc": "application/x-conference", + "cpio": "application/x-cpio", + "csh": "application/x-csh", + "deb": "application/x-debian-package", + "udeb": "application/x-debian-package", + "dgc": "application/x-dgc-compressed", + "dir": "application/x-director", + "dcr": "application/x-director", + "dxr": "application/x-director", + "cst": "application/x-director", + "cct": "application/x-director", + "cxt": "application/x-director", + "w3d": "application/x-director", + "fgd": "application/x-director", + "swa": "application/x-director", + "wad": "application/x-doom", + "ncx": "application/x-dtbncx+xml", + "dtb": "application/x-dtbook+xml", + "res": "application/x-dtbresource+xml", + "dvi": "application/x-dvi", + "evy": "application/x-envoy", + "eva": "application/x-eva", + "bdf": "application/x-font-bdf", + "gsf": "application/x-font-ghostscript", + "psf": "application/x-font-linux-psf", + "pcf": "application/x-font-pcf", + "snf": "application/x-font-snf", + "pfa": "application/x-font-type1", + "pfb": "application/x-font-type1", + "pfm": "application/x-font-type1", + "afm": "application/x-font-type1", + "arc": "application/x-freearc", + "spl": "application/x-futuresplash", + "gca": "application/x-gca-compressed", + "ulx": "application/x-glulx", + "gnumeric": "application/x-gnumeric", + "gramps": "application/x-gramps-xml", + "gtar": "application/x-gtar", + "hdf": "application/x-hdf", + "install": "application/x-install-instructions", + "iso": "application/x-iso9660-image", + "jnlp": "application/x-java-jnlp-file", + "latex": "application/x-latex", + "lzh": "application/x-lzh-compressed", + "lha": "application/x-lzh-compressed", + "mie": "application/x-mie", + "prc": "application/x-mobipocket-ebook", + "mobi": "application/x-mobipocket-ebook", + "application": "application/x-ms-application", + "lnk": "application/x-ms-shortcut", + "wmd": "application/x-ms-wmd", + "wmz": "application/x-ms-wmz", + "xbap": "application/x-ms-xbap", + "mdb": "application/x-msaccess", + "obd": "application/x-msbinder", + "crd": "application/x-mscardfile", + "clp": "application/x-msclip", + "exe": "application/x-msdownload", + "dll": "application/x-msdownload", + "com": "application/x-msdownload", + "bat": "application/x-msdownload", + "msi": "application/x-msdownload", + "mvb": "application/x-msmediaview", + "m13": "application/x-msmediaview", + "m14": "application/x-msmediaview", + "wmf": "application/x-msmetafile", + "wmz": "application/x-msmetafile", + "emf": "application/x-msmetafile", + "emz": "application/x-msmetafile", + "mny": "application/x-msmoney", + "pub": "application/x-mspublisher", + "scd": "application/x-msschedule", + "trm": "application/x-msterminal", + "wri": "application/x-mswrite", + "nc": "application/x-netcdf", + "cdf": "application/x-netcdf", + "nzb": "application/x-nzb", + "p12": "application/x-pkcs12", + "pfx": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "spc": "application/x-pkcs7-certificates", + "p7r": "application/x-pkcs7-certreqresp", + "rar": "application/x-rar-compressed", + "ris": "application/x-research-info-systems", + "sh": "application/x-sh", + "shar": "application/x-shar", + "swf": "application/x-shockwave-flash", + "xap": "application/x-silverlight-app", + "sql": "application/x-sql", + "sit": "application/x-stuffit", + "sitx": "application/x-stuffitx", + "srt": "application/x-subrip", + "sv4cpio": "application/x-sv4cpio", + "sv4crc": "application/x-sv4crc", + "t3": "application/x-t3vm-image", + "gam": "application/x-tads", + "tar": "application/x-tar", + "tcl": "application/x-tcl", + "tex": "application/x-tex", + "tfm": "application/x-tex-tfm", + "texinfo": "application/x-texinfo", + "texi": "application/x-texinfo", + "obj": "application/x-tgif", + "ustar": "application/x-ustar", + "src": "application/x-wais-source", + "der": "application/x-x509-ca-cert", + "crt": "application/x-x509-ca-cert", + "fig": "application/x-xfig", "xlf": "application/x-xliff+xml", - "xlim": "application/vnd.xmpie.xlim", - "xlm": "application/vnd.ms-excel", - "xls": "application/vnd.ms-excel", - "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", - "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xlt": "application/vnd.ms-excel", - "xltm": "application/vnd.ms-excel.template.macroenabled.12", - "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", - "xlw": "application/vnd.ms-excel", - "xm": "audio/xm", - "xml": "text/xml", - "xmls": "application/dskpp+xml", - "xmt_bin": "model/vnd.parasolid.transmit.binary", - "xmt_txt": "model/vnd.parasolid.transmit.text", - "xmz": "xgl/movie", - "xns": "application/xcap-ns+xml", - "xo": "application/vnd.olpc-sugar", - "xof": "x-world/x-vrml", - "xop": "application/xop+xml", - "xpdl": "application/xml", "xpi": "application/x-xpinstall", - "xpix": "application/x-vnd.ls-xpix", - "xpl": "application/xproc+xml", - "xpm": "image/x-xpixmap", - "xpr": "application/vnd.is-xpr", - "xps": "application/vnd.ms-xpsdocument", - "xpw": "application/vnd.intercon.formnet", - "xpx": "application/vnd.intercon.formnet", - "xq": "text/xquery", - "xql": "text/xquery", - "xqm": "text/xquery", - "xqu": "text/xquery", - "xquery": "text/xquery", - "xqy": "text/xquery", - "xsd": "text/xml", - "xsf": "application/prs.xsf+xml", - "xsl": "application/xslt+xml", - "xslt": "application/xslt+xml", - "xsm": "application/vnd.syncml+xml", - "xspf": "application/xspf+xml", - "xsr": "video/x-amt-showrun", - "xtel": "chemical/x-xtel", - "xul": "application/vnd.mozilla.xul+xml", - "xvm": "application/xv+xml", - "xvml": "application/xv+xml", - "xwd": "image/x-xwindowdump", - "xyz": "chemical/x-xyz", - "xyze": "image/vnd.radiance", "xz": "application/x-xz", - "yaml": "text/yaml", - "yang": "application/yang", - "yin": "application/yin+xml", - "yme": "application/vnd.yaoweme", - "yml": "text/yaml", - "ymp": "text/x-suse-ymp", "z1": "application/x-zmachine", "z2": "application/x-zmachine", "z3": "application/x-zmachine", @@ -1884,26 +744,301 @@ const mimes* = { "z6": "application/x-zmachine", "z7": "application/x-zmachine", "z8": "application/x-zmachine", - "zaz": "application/vnd.zzazz.deck+xml", - "zfc": "application/vnd.filmit.zfc", - "zfo": "application/vnd.software602.filler.form-xml-zip", - "zig": "text/zig", + "xaml": "application/xaml+xml", + "xdf": "application/xcap-diff+xml", + "xenc": "application/xenc+xml", + "xhtml": "application/xhtml+xml", + "xht": "application/xhtml+xml", + "xml": "application/xml", + "xsl": "application/xml", + "dtd": "application/xml-dtd", + "xop": "application/xop+xml", + "xpl": "application/xproc+xml", + "xslt": "application/xslt+xml", + "xspf": "application/xspf+xml", + "mxml": "application/xv+xml", + "xhvml": "application/xv+xml", + "xvml": "application/xv+xml", + "xvm": "application/xv+xml", + "yang": "application/yang", + "yin": "application/yin+xml", "zip": "application/zip", - "zir": "application/vnd.zul", - "zirz": "application/vnd.zul", - "zmm": "application/vnd.handheld-entertainment+xml", - "zmt": "chemical/x-mopac-input", - "zone": "text/dns", - "zoo": "application/octet-stream", - "zsh": "text/x-script.zsh", - "~": "application/x-trash" + "adp": "audio/adpcm", + "au": "audio/basic", + "snd": "audio/basic", + "mid": "audio/midi", + "midi": "audio/midi", + "kar": "audio/midi", + "rmi": "audio/midi", + "m4a": "audio/mp4", + "mp4a": "audio/mp4", + "mpga": "audio/mpeg", + "mp2": "audio/mpeg", + "mp2a": "audio/mpeg", + "mp3": "audio/mpeg", + "m2a": "audio/mpeg", + "m3a": "audio/mpeg", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "spx": "audio/ogg", + "opus": "audio/ogg", + "s3m": "audio/s3m", + "sil": "audio/silk", + "uva": "audio/vnd.dece.audio", + "uvva": "audio/vnd.dece.audio", + "eol": "audio/vnd.digital-winds", + "dra": "audio/vnd.dra", + "dts": "audio/vnd.dts", + "dtshd": "audio/vnd.dts.hd", + "lvp": "audio/vnd.lucent.voice", + "pya": "audio/vnd.ms-playready.media.pya", + "ecelp4800": "audio/vnd.nuera.ecelp4800", + "ecelp7470": "audio/vnd.nuera.ecelp7470", + "ecelp9600": "audio/vnd.nuera.ecelp9600", + "rip": "audio/vnd.rip", + "weba": "audio/webm", + "aac": "audio/x-aac", + "aif": "audio/x-aiff", + "aiff": "audio/x-aiff", + "aifc": "audio/x-aiff", + "caf": "audio/x-caf", + "flac": "audio/x-flac", + "mka": "audio/x-matroska", + "m3u": "audio/x-mpegurl", + "wax": "audio/x-ms-wax", + "wma": "audio/x-ms-wma", + "ram": "audio/x-pn-realaudio", + "ra": "audio/x-pn-realaudio", + "rmp": "audio/x-pn-realaudio-plugin", + "wav": "audio/x-wav", + "xm": "audio/xm", + "cdx": "chemical/x-cdx", + "cif": "chemical/x-cif", + "cmdf": "chemical/x-cmdf", + "cml": "chemical/x-cml", + "csml": "chemical/x-csml", + "xyz": "chemical/x-xyz", + "ttc": "font/collection", + "otf": "font/otf", + "ttf": "font/ttf", + "woff": "font/woff", + "woff2": "font/woff2", + "bmp": "image/bmp", + "cgm": "image/cgm", + "g3": "image/g3fax", + "gif": "image/gif", + "ief": "image/ief", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "jpe": "image/jpeg", + "ktx": "image/ktx", + "png": "image/png", + "btif": "image/prs.btif", + "sgi": "image/sgi", + "svg": "image/svg+xml", + "svgz": "image/svg+xml", + "tiff": "image/tiff", + "tif": "image/tiff", + "psd": "image/vnd.adobe.photoshop", + "uvi": "image/vnd.dece.graphic", + "uvvi": "image/vnd.dece.graphic", + "uvg": "image/vnd.dece.graphic", + "uvvg": "image/vnd.dece.graphic", + "djvu": "image/vnd.djvu", + "djv": "image/vnd.djvu", + "sub": "image/vnd.dvb.subtitle", + "dwg": "image/vnd.dwg", + "dxf": "image/vnd.dxf", + "fbs": "image/vnd.fastbidsheet", + "fpx": "image/vnd.fpx", + "fst": "image/vnd.fst", + "mmr": "image/vnd.fujixerox.edmics-mmr", + "rlc": "image/vnd.fujixerox.edmics-rlc", + "mdi": "image/vnd.ms-modi", + "wdp": "image/vnd.ms-photo", + "npx": "image/vnd.net-fpx", + "wbmp": "image/vnd.wap.wbmp", + "xif": "image/vnd.xiff", + "webp": "image/webp", + "3ds": "image/x-3ds", + "ras": "image/x-cmu-raster", + "cmx": "image/x-cmx", + "fh": "image/x-freehand", + "fhc": "image/x-freehand", + "fh4": "image/x-freehand", + "fh5": "image/x-freehand", + "fh7": "image/x-freehand", + "ico": "image/x-icon", + "sid": "image/x-mrsid-image", + "pcx": "image/x-pcx", + "pic": "image/x-pict", + "pct": "image/x-pict", + "pnm": "image/x-portable-anymap", + "pbm": "image/x-portable-bitmap", + "pgm": "image/x-portable-graymap", + "ppm": "image/x-portable-pixmap", + "rgb": "image/x-rgb", + "tga": "image/x-tga", + "xbm": "image/x-xbitmap", + "xpm": "image/x-xpixmap", + "xwd": "image/x-xwindowdump", + "eml": "message/rfc822", + "mime": "message/rfc822", + "igs": "model/iges", + "iges": "model/iges", + "msh": "model/mesh", + "mesh": "model/mesh", + "silo": "model/mesh", + "dae": "model/vnd.collada+xml", + "dwf": "model/vnd.dwf", + "gdl": "model/vnd.gdl", + "gtw": "model/vnd.gtw", + "mts": "model/vnd.mts", + "vtu": "model/vnd.vtu", + "wrl": "model/vrml", + "vrml": "model/vrml", + "x3db": "model/x3d+binary", + "x3dbz": "model/x3d+binary", + "x3dv": "model/x3d+vrml", + "x3dvz": "model/x3d+vrml", + "x3d": "model/x3d+xml", + "x3dz": "model/x3d+xml", + "appcache": "text/cache-manifest", + "ics": "text/calendar", + "ifb": "text/calendar", + "css": "text/css", + "csv": "text/csv", + "html": "text/html", + "htm": "text/html", + "js": "text/javascript", + "mjs": "text/javascript", + "n3": "text/n3", + "txt": "text/plain", + "text": "text/plain", + "conf": "text/plain", + "def": "text/plain", + "list": "text/plain", + "log": "text/plain", + "in": "text/plain", + "dsc": "text/prs.lines.tag", + "rtx": "text/richtext", + "sgml": "text/sgml", + "sgm": "text/sgml", + "tsv": "text/tab-separated-values", + "t": "text/troff", + "tr": "text/troff", + "roff": "text/troff", + "man": "text/troff", + "me": "text/troff", + "ms": "text/troff", + "ttl": "text/turtle", + "uri": "text/uri-list", + "uris": "text/uri-list", + "urls": "text/uri-list", + "vcard": "text/vcard", + "curl": "text/vnd.curl", + "dcurl": "text/vnd.curl.dcurl", + "mcurl": "text/vnd.curl.mcurl", + "scurl": "text/vnd.curl.scurl", + "sub": "text/vnd.dvb.subtitle", + "fly": "text/vnd.fly", + "flx": "text/vnd.fmi.flexstor", + "gv": "text/vnd.graphviz", + "3dml": "text/vnd.in3d.3dml", + "spot": "text/vnd.in3d.spot", + "jad": "text/vnd.sun.j2me.app-descriptor", + "wml": "text/vnd.wap.wml", + "wmls": "text/vnd.wap.wmlscript", + "s": "text/x-asm", + "asm": "text/x-asm", + "c": "text/x-c", + "cc": "text/x-c", + "cxx": "text/x-c", + "cpp": "text/x-c", + "h": "text/x-c", + "hh": "text/x-c", + "dic": "text/x-c", + "f": "text/x-fortran", + "for": "text/x-fortran", + "f77": "text/x-fortran", + "f90": "text/x-fortran", + "java": "text/x-java-source", + "nfo": "text/x-nfo", + "opml": "text/x-opml", + "p": "text/x-pascal", + "pas": "text/x-pascal", + "etx": "text/x-setext", + "sfv": "text/x-sfv", + "uu": "text/x-uuencode", + "vcs": "text/x-vcalendar", + "vcf": "text/x-vcard", + "3gp": "video/3gpp", + "3g2": "video/3gpp2", + "h261": "video/h261", + "h263": "video/h263", + "h264": "video/h264", + "jpgv": "video/jpeg", + "jpm": "video/jpm", + "jpgm": "video/jpm", + "mj2": "video/mj2", + "mjp2": "video/mj2", + "mp4": "video/mp4", + "mp4v": "video/mp4", + "mpg4": "video/mp4", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpe": "video/mpeg", + "m1v": "video/mpeg", + "m2v": "video/mpeg", + "ogv": "video/ogg", + "qt": "video/quicktime", + "mov": "video/quicktime", + "uvh": "video/vnd.dece.hd", + "uvvh": "video/vnd.dece.hd", + "uvm": "video/vnd.dece.mobile", + "uvvm": "video/vnd.dece.mobile", + "uvp": "video/vnd.dece.pd", + "uvvp": "video/vnd.dece.pd", + "uvs": "video/vnd.dece.sd", + "uvvs": "video/vnd.dece.sd", + "uvv": "video/vnd.dece.video", + "uvvv": "video/vnd.dece.video", + "dvb": "video/vnd.dvb.file", + "fvt": "video/vnd.fvt", + "mxu": "video/vnd.mpegurl", + "m4u": "video/vnd.mpegurl", + "pyv": "video/vnd.ms-playready.media.pyv", + "uvu": "video/vnd.uvvu.mp4", + "uvvu": "video/vnd.uvvu.mp4", + "viv": "video/vnd.vivo", + "webm": "video/webm", + "f4v": "video/x-f4v", + "fli": "video/x-fli", + "flv": "video/x-flv", + "m4v": "video/x-m4v", + "mkv": "video/x-matroska", + "mk3d": "video/x-matroska", + "mks": "video/x-matroska", + "mng": "video/x-mng", + "asf": "video/x-ms-asf", + "asx": "video/x-ms-asf", + "vob": "video/x-ms-vob", + "wm": "video/x-ms-wm", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wvx": "video/x-ms-wvx", + "avi": "video/x-msvideo", + "movie": "video/x-sgi-movie", + "smv": "video/x-smv", + "ice": "x-conference/x-cooltalk", } func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. - result.mimes = mimes.newStringTable() + {.cast(noSideEffect).}: + result.mimes = mimes.toOrderedTable() func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 605f623212..20ea9d77a1 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -12,31 +12,39 @@ # TODO: Clean up the exports a bit and everything else in general. -import os, options +import std/[os, options] import std/private/since +import std/strbasics +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] when hostOS == "solaris": {.passl: "-lsocket -lnsl".} const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when useWinVersion: - import winlean + import std/winlean export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, WSANOTINITIALISED, WSAENOTSOCK, WSAEINPROGRESS, WSAEINTR, WSAEDISCON, ERROR_NETNAME_DELETED else: - import posix + import std/posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET, EBADF export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, Sockaddr_in6, Sockaddr_storage, - inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto, + recv, `==`, connect, send, accept, recvfrom, sendto, freeAddrInfo +when not useNimNetLite: + export inet_ntoa + export SO_ERROR, SOL_SOCKET, @@ -59,7 +67,7 @@ type ## some procedures, such as getaddrinfo) AF_UNIX = 1, ## for local socket (using a file). Unsupported on Windows. AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = when defined(macosx): 30 else: 23 ## for network protocol IPv6. + AF_INET6 = when defined(macosx): 30 elif defined(windows): 23 else: 10 ## for network protocol IPv6. SockType* = enum ## second argument to `socket` proc SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets @@ -162,7 +170,7 @@ when not useWinVersion: else: proc toInt(domain: Domain): cint = - result = toU32(ord(domain)).cint + result = cast[cint](uint32(ord(domain))) proc toKnownDomain*(family: cint): Option[Domain] = ## Converts the platform-dependent `cint` to the Domain or none(), @@ -209,7 +217,7 @@ proc getProtoByName*(name: string): int {.since: (1, 3, 5).} = let protoent = posix.getprotobyname(name.cstring) if protoent == nil: - raise newException(OSError, "protocol not found") + raise newException(OSError, "protocol not found: " & name) result = protoent.p_proto.int @@ -295,9 +303,9 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, if domain == AF_INET6: hints.ai_flags = AI_V4MAPPED let socketPort = if sockType == SOCK_RAW: "" else: $port - var gaiResult = getaddrinfo(address, socketPort, addr(hints), result) + var gaiResult = getaddrinfo(address, socketPort.cstring, addr(hints), result) if gaiResult != 0'i32: - when useWinVersion or defined(freertos): + when useWinVersion or defined(freertos) or defined(nuttx): raiseOSError(osLastError()) else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) @@ -331,125 +339,6 @@ template htons*(x: uint16): untyped = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by `name` matches the s_name member - ## and the protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by `port` matches the s_port member and the - ## protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = - when defined(android4): - posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, - cint(posix.AF_INET)) - else: - posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).SockLen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when useWinVersion: - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostname*(): string {.tags: [ReadIOEffect].} = - ## Returns the local hostname (not the FQDN) - # https://tools.ietf.org/html/rfc1035#section-2.3.1 - # https://tools.ietf.org/html/rfc2181#section-11 - const size = 64 - result = newString(size) - when useWinVersion: - let success = winlean.gethostname(result, size) - else: - # Posix - let success = posix.gethostname(result, size) - if success != 0.cint: - raiseOSError(osLastError()) - let x = len(cstring(result)) - result.setLen(x) - proc getSockDomain*(socket: SocketHandle): Domain = ## Returns the socket's domain (AF_INET or AF_INET6). var name: Sockaddr_in6 @@ -463,161 +352,383 @@ proc getSockDomain*(socket: SocketHandle): Domain = else: raise newException(IOError, "Unknown socket family in getSockDomain") -proc getAddrString*(sockAddr: ptr SockAddr): string = - ## Returns the string representation of address within sockAddr - if sockAddr.sa_family.cint == nativeAfInet: - result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) - elif sockAddr.sa_family.cint == nativeAfInet6: - let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN - else: 46 # it's actually 46 in both cases - result = newString(addrLen) - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result = result.substr("::ffff:".len) +when not useNimNetLite: + proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the service name specified by `name` matches the s_name member + ## and the protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. + when useWinVersion: + var s = winlean.getservbyname(name, proto) else: - if winlean.inet_ntop(winlean.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - setLen(result, len(cstring(result))) - else: - when defined(posix) and not defined(nimdoc): - if sockAddr.sa_family.cint == nativeAfUnix: - return "unix" - raise newException(IOError, "Unknown socket family in getAddrString") + var s = posix.getservbyname(name, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto -proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) = - ## Stores in `strAddress` the string representation of the address inside - ## `sockAddr` - ## - ## **Note** - ## * `strAddress` must be initialized to 46 in length. - assert(46 == len(strAddress), - "`strAddress` was not initialized correctly. 46 != `len(strAddress)`") - if sockAddr.sa_family.cint == nativeAfInet: - let addr4 = addr cast[ptr Sockaddr_in](sockAddr).sin_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET, addr4, addr strAddress[0], - strAddress.len.int32) == nil: - raiseOSError(osLastError()) + proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the port specified by `port` matches the s_port member and the + ## protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. + when useWinVersion: + var s = winlean.getservbyport(uint16(port).cint, proto) else: - if winlean.inet_ntop(winlean.AF_INET, addr4, addr strAddress[0], - strAddress.len.int32) == nil: - raiseOSError(osLastError()) - elif sockAddr.sa_family.cint == nativeAfInet6: - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr strAddress[0], - strAddress.len.int32) == nil: - raiseOSError(osLastError()) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - strAddress = strAddress.substr("::ffff:".len) + var s = posix.getservbyport(uint16(port).cint, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto + + proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = + ## This function will lookup the hostname of an IP Address. + var + addrInfo = getAddrInfo(ip, Port(0), AF_UNSPEC) + myAddr: pointer + addrLen = 0 + family = 0 + + defer: freeAddrInfo(addrInfo) + + if addrInfo.ai_addr.sa_family.cint == nativeAfInet: + family = nativeAfInet + myAddr = addr cast[ptr Sockaddr_in](addrInfo.ai_addr).sin_addr + addrLen = 4 + elif addrInfo.ai_addr.sa_family.cint == nativeAfInet6: + family = nativeAfInet6 + myAddr = addr cast[ptr Sockaddr_in6](addrInfo.ai_addr).sin6_addr + addrLen = 16 else: - if winlean.inet_ntop(winlean.AF_INET6, addr6, addr strAddress[0], - strAddress.len.int32) == nil: + raise newException(IOError, "Unknown socket family in `getHostByAddr()`") + + when useWinVersion: + var s = winlean.gethostbyaddr(cast[ptr InAddr](myAddr), addrLen.cuint, + cint(family)) + if s == nil: raiseOSError(osLastError()) + else: + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](myAddr), addrLen.cint, + cint(family)) + else: + posix.gethostbyaddr(myAddr, addrLen.SockLen, + cint(family)) + if s == nil: + raiseOSError(osLastError(), $hstrerror(h_errno)) + + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) + when useWinVersion: + result.addrtype = Domain(s.h_addrtype) + else: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raiseOSError(osLastError(), "unknown h_addrtype") + if result.addrtype == AF_INET: + result.addrList = @[] + var i = 0 + while not isNil(s.h_addr_list[i]): + var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrList.add($inet_ntoa(inaddrPtr[])) + inc(i) + else: + let strAddrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 + var i = 0 + while not isNil(s.h_addr_list[i]): + var ipStr = newString(strAddrLen) + if inet_ntop(nativeAfInet6, cast[pointer](s.h_addr_list[i]), + cstring(ipStr), len(ipStr).int32) == nil: + raiseOSError(osLastError()) + when not useWinVersion: + if posix.IN6_IS_ADDR_V4MAPPED(cast[ptr In6Addr](s.h_addr_list[i])) != 0: + ipStr.setSlice("::ffff:".len..= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + + proc getSockName*(socket: SocketHandle): Port = + ## Returns the socket's associated port number. + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + #name.sin_port = htons(cint16(port)) + #name.sin_addr.s_addr = htonl(INADDR_ANY) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = Port(nativesockets.ntohs(name.sin_port)) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: raiseOSError(osLastError()) - else: - raise newException(IOError, "Unknown socket family in getAddrString") - setLen(strAddress, len(cstring(strAddress))) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") -when defined(posix) and not defined(nimdoc): - proc makeUnixAddr*(path: string): Sockaddr_un = - result.sun_family = AF_UNIX.TSa_Family - if path.len >= Sockaddr_un_path_length: - raise newException(ValueError, "socket path too long") - copyMem(addr result.sun_path, path.cstring, path.len + 1) + proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's peer address and port number. + ## + ## Similar to POSIX's `getpeername`:idx: + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + +when useNimNetLite: -proc getSockName*(socket: SocketHandle): Port = - ## Returns the socket's associated port number. - var name: Sockaddr_in when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(nativesockets.ntohs(name.sin_port)) + const + INET_ADDRSTRLEN = 16 + INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases -proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's local address and port number. - ## - ## Similar to POSIX's `getsockname`:idx:. - case domain - of AF_INET: - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) - else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: - raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc sockAddrToStr(sa: ptr SockAddr): string {.noinit.} = + let af_family = sa.sa_family + var nl, v4Slice: cint + var si_addr: ptr InAddr -proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's peer address and port number. - ## - ## Similar to POSIX's `getpeername`:idx: - case domain - of AF_INET: - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) + if af_family == AF_INET.TSa_Family: + nl = INET_ADDRSTRLEN + si_addr = cast[ptr Sockaddr_in](sa).sin_addr.addr() + elif af_family == AF_INET6.TSa_Family: + nl = INET6_ADDRSTRLEN + let si6_addr = cast[ptr Sockaddr_in6](sa).sin6_addr.addr() + si_addr = cast[ptr InAddr](si6_addr) # let's us reuse logic below + when defined(posix) and not defined(nimdoc) and not defined(zephyr): + if posix.IN6_IS_ADDR_V4MAPPED(si6_addr) != 0: + v4Slice = "::ffff:".len() else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) + when defined(posix) and not defined(nimdoc): + if af_family.cint == nativeAfUnix: + return "unix" + return "" + + result = newString(nl) + let namePtr = result.cstring() + if namePtr == inet_ntop(af_family.cint, si_addr, namePtr, nl): + result.setLen(len(namePtr)) + if v4Slice > 0: result.setSlice(v4Slice.int ..< nl.int) else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: + return "" + + proc sockAddrToStr(sa: var Sockaddr_in | var Sockaddr_in6): string = + result = sockAddrToStr(cast[ptr SockAddr](unsafeAddr(sa))) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + result = sockAddrToStr(sockAddr) + if result.len() == 0: raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: - raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) {.noinit.} = + strAddress = getAddrString(sockAddr) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + template sockGetNameOrRaiseError(socket: untyped, name: untyped) = + var namelen = sizeof(socket).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + case domain + of AF_INET: + var name = Sockaddr_in(sin_family: TSa_Family(posix.AF_INET)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name = Sockaddr_in6(sin6_family: TSa_Family(posix.AF_INET6)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin6_port))) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc getSockOptInt*(socket: SocketHandle, level, optname: int): int {. tags: [ReadIOEffect].} = @@ -731,14 +842,14 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke ## child processes. ## ## Returns (osInvalidSocket, "") if an error occurred. - var sockAddress: Sockaddr_in + var sockAddress: SockAddr var addrLen = sizeof(sockAddress).SockLen var sock = when (defined(linux) or defined(bsd)) and not defined(nimdoc): - accept4(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen), + accept4(fd, addr(sockAddress), addr(addrLen), if inheritable: 0 else: SOCK_CLOEXEC) else: - accept(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + accept(fd, addr(sockAddress), addr(addrLen)) when declared(setInheritable) and not (defined(linux) or defined(bsd)): if not setInheritable(sock, inheritable): close sock @@ -746,7 +857,11 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke if sock == osInvalidSocket: return (osInvalidSocket, "") else: - return (sock, $inet_ntoa(sockAddress.sin_addr)) + when useNimNetLite: + var name = sockAddrToStr(addr sockAddress) + return (sock, name) + else: + return (sock, $inet_ntoa(cast[Sockaddr_in](sockAddress).sin_addr)) when defined(windows): var wsa: WSAData diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 37e6e8f5a5..7f6a365579 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -44,64 +44,73 @@ ## After you create a socket with the `newSocket` procedure, you can easily ## connect it to a server running at a known hostname (or IP address) and port. ## To do so over TCP, use the example below. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.connect("google.com", Port(80)) -## -## For SSL, use the following example (and make sure to compile with `-d:ssl`): -## -## .. code-block:: Nim -## var socket = newSocket() -## var ctx = newContext() -## wrapSocket(ctx, socket) -## socket.connect("google.com", Port(443)) -## + +runnableExamples("-r:off"): + let socket = newSocket() + socket.connect("google.com", Port(80)) + +## For SSL, use the following example: + +runnableExamples("-r:off -d:ssl"): + let socket = newSocket() + let ctx = newContext() + wrapSocket(ctx, socket) + socket.connect("google.com", Port(443)) + ## UDP is a connectionless protocol, so UDP sockets don't have to explicitly ## call the `connect `_ procedure. They can ## simply start sending data immediately. -## -## .. code-block:: Nim -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.sendTo("192.168.0.1", Port(27960), "status\n") -## + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + socket.sendTo("192.168.0.1", Port(27960), "status\n") + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + let ip = parseIpAddress("192.168.0.1") + doAssert socket.sendTo(ip, Port(27960), "status\c\l") == 8 + ## Creating a server ## ----------------- ## ## After you create a socket with the `newSocket` procedure, you can create a ## TCP server by calling the `bindAddr` and `listen` procedures. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.bindAddr(Port(1234)) -## socket.listen() -## -## You can then begin accepting connections using the `accept` procedure. -## -## .. code-block:: Nim -## var client: Socket -## var address = "" -## while true: -## socket.acceptAddr(client, address) -## echo("Client connected from: ", address) + +runnableExamples("-r:off"): + let socket = newSocket() + socket.bindAddr(Port(1234)) + socket.listen() + + # You can then begin accepting connections using the `accept` procedure. + var client: Socket + var address = "" + while true: + socket.acceptAddr(client, address) + echo "Client connected from: ", address import std/private/since -import nativesockets, os, strutils, times, sets, options, std/monotimes -import ssl_config +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/nativesockets +import std/[os, strutils, times, sets, options, monotimes] +import std/ssl_config export nativesockets.Port, nativesockets.`$`, nativesockets.`==` export Domain, SockType, Protocol const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) const defineSsl = defined(ssl) or defined(nimdoc) when useWinVersion: - from winlean import WSAESHUTDOWN + from std/winlean import WSAESHUTDOWN when defineSsl: - import openssl + import std/openssl when not defined(nimDisableCertificateValidation): - from ssl_certs import scanSSLCertificates + from std/ssl_certs import scanSSLCertificates # Note: The enumerations are mapped to Window's constants. @@ -198,9 +207,33 @@ type when defined(nimHasStyleChecks): {.pop.} + +when defined(posix) and not defined(lwip): + from std/posix import TPollfd, POLLIN, POLLPRI, POLLOUT, POLLWRBAND, Tnfds + + template monitorPollEvent(x: var SocketHandle, y: cint, timeout: int): int = + var tpollfd: TPollfd + tpollfd.fd = cast[cint](x) + tpollfd.events = y + posix.poll(addr(tpollfd), Tnfds(1), timeout) + +proc timeoutRead(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectRead(fds, timeout) + else: + monitorPollEvent(fd, POLLIN or POLLPRI, timeout) + +proc timeoutWrite(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectWrite(fds, timeout) + else: + monitorPollEvent(fd, POLLOUT or POLLWRBAND, timeout) + proc socketError*(socket: Socket, err: int = -1, async = false, lastError = (-1).OSErrorCode, - flags: set[SocketFlag] = {}): void {.gcsafe.} + flags: set[SocketFlag] = {}) {.gcsafe.} proc isDisconnectionError*(flags: set[SocketFlag], lastError: OSErrorCode): bool = @@ -283,14 +316,20 @@ proc parseIPv4Address(addressStr: string): IpAddress = byteCount = 0 currentByte: uint16 = 0 separatorValid = false + leadingZero = false result = IpAddress(family: IpAddressFamily.IPv4) for i in 0 .. high(addressStr): if addressStr[i] in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers are not allowed") currentByte = currentByte * 10 + cast[uint16](ord(addressStr[i]) - ord('0')) - if currentByte > 255'u16: + if currentByte == 0'u16: + leadingZero = true + elif currentByte > 255'u16: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -302,6 +341,7 @@ proc parseIPv4Address(addressStr: string): IpAddress = currentByte = 0 byteCount.inc separatorValid = false + leadingZero = false else: raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -390,10 +430,16 @@ proc parseIPv6Address(addressStr: string): IpAddress = result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) groupCount.inc() else: # Must parse IPv4 address + var leadingZero = false for i, c in addressStr[v4StartPos..high(addressStr)]: if c in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers not allowed") currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: + if currentShort == 0'u32: + leadingZero = true + elif currentShort > 255'u32: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -404,6 +450,7 @@ proc parseIPv6Address(addressStr: string): IpAddress = currentShort = 0 byteCount.inc() separatorValid = false + leadingZero = false else: # Invalid character raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -433,7 +480,12 @@ proc parseIPv6Address(addressStr: string): IpAddress = proc parseIpAddress*(addressStr: string): IpAddress = ## Parses an IP address - ## Raises ValueError on error + ## + ## Raises ValueError on error. + ## + ## For IPv4 addresses, only the strict form as + ## defined in RFC 6943 is considered valid, see + ## https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1. if addressStr.len == 0: raise newException(ValueError, "IP Address string is empty") if addressStr.contains(':'): @@ -495,13 +547,20 @@ proc fromSockAddr*(sa: Sockaddr_storage | SockAddr | Sockaddr_in | Sockaddr_in6, fromSockAddrAux(cast[ptr Sockaddr_storage](unsafeAddr sa), sl, address, port) when defineSsl: - CRYPTO_malloc_init() - doAssert SslLibraryInit() == 1 - SSL_load_error_strings() - ERR_load_BIO_strings() - OpenSSL_add_all_algorithms() + # OpenSSL >= 1.1.0 does not need explicit init. + when not useOpenssl3: + CRYPTO_malloc_init() + doAssert SslLibraryInit() == 1 + SSL_load_error_strings() + ERR_load_BIO_strings() + OpenSSL_add_all_algorithms() - proc raiseSSLError*(s = "") = + proc sslHandle*(self: Socket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + + proc raiseSSLError*(s = "") {.raises: [SslError].}= ## Raises a new SSL error. if s != "": raise newException(SslError, s) @@ -563,12 +622,11 @@ when defineSsl: proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, certFile = "", keyFile = "", cipherList = CiphersIntermediate, - caDir = "", caFile = ""): SslContext = + caDir = "", caFile = "", ciphersuites = CiphersModern): SslContext = ## Creates an SSL context. ## - ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 - ## are available with the addition of `protSSLv23` which allows for - ## compatibility with all of them. + ## Protocol version is currently ignored by default and TLS is used. + ## With `-d:openssl10`, only SSLv23 and TLSv1 may be used. ## ## There are three options for verify mode: ## `CVerifyNone`: certificates are not verified; @@ -582,10 +640,10 @@ when defineSsl: ## ## CA certificates will be loaded, in the following order, from: ## - ## - caFile, caDir, parameters, if set - ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`, - ## the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used - ## - a set of files and directories from the `ssl_certs `_ file. + ## - caFile, caDir, parameters, if set + ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`, + ## the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used + ## - a set of files and directories from the `ssl_certs `_ file. ## ## The last two parameters specify the certificate file path and the key file ## path, a server socket will most likely not work without these. @@ -595,31 +653,39 @@ when defineSsl: ## or using ECDSA: ## - `openssl ecparam -out mykey.pem -name secp256k1 -genkey` ## - `openssl req -new -key mykey.pem -x509 -nodes -days 365 -out mycert.pem` - var newCTX: SslCtx - case protVersion - of protSSLv23: - newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. - of protSSLv2: - raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") - of protSSLv3: - raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") - of protTLSv1: - newCTX = SSL_CTX_new(TLSv1_method()) + var mtd: PSSL_METHOD + when defined(openssl10): + case protVersion + of protSSLv23: + mtd = SSLv23_method() + of protSSLv2: + raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") + of protSSLv3: + raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") + of protTLSv1: + mtd = TLSv1_method() + else: + mtd = TLS_method() + if mtd == nil: + raiseSSLError("Failed to create TLS context") + var newCTX = SSL_CTX_new(mtd) + if newCTX == nil: + raiseSSLError("Failed to create TLS context") if newCTX.SSL_CTX_set_cipher_list(cipherList) != 1: raiseSSLError() when not defined(openssl10) and not defined(libressl): let sslVersion = getOpenSSLVersion() - if sslVersion >= 0x010101000 and not sslVersion == 0x020000000: + if sslVersion >= 0x010101000 and sslVersion != 0x020000000: # In OpenSSL >= 1.1.1, TLSv1.3 cipher suites can only be configured via # this API. - if newCTX.SSL_CTX_set_ciphersuites(cipherList) != 1: + if newCTX.SSL_CTX_set_ciphersuites(ciphersuites) != 1: raiseSSLError() # Automatically the best ECDH curve for client exchange. Without this, ECDH # ciphers will be ignored by the server. # # From OpenSSL >= 1.1.0, this setting is set by default and can't be - # overriden. + # overridden. if newCTX.SSL_CTX_set_ecdh_auto(1) != 1: raiseSSLError() @@ -644,15 +710,20 @@ when defineSsl: if verifyMode != CVerifyNone: # Use the caDir and caFile parameters if set if caDir != "" or caFile != "": - if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != VerifySuccess: + if newCTX.SSL_CTX_load_verify_locations(if caFile == "": nil else: caFile.cstring, if caDir == "": nil else: caDir.cstring) != VerifySuccess: raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).") else: # Scan for certs in known locations. For CVerifyPeerUseEnvVars also scan # the SSL_CERT_FILE and SSL_CERT_DIR env vars var found = false - for fn in scanSSLCertificates(): - if newCTX.SSL_CTX_load_verify_locations(fn, nil) == VerifySuccess: + let useEnvVars = (if verifyMode == CVerifyPeerUseEnvVars: true else: false) + for fn in scanSSLCertificates(useEnvVars = useEnvVars): + if fn.extractFilename == "": + if newCTX.SSL_CTX_load_verify_locations(nil, cstring(fn.normalizePathEnd(false))) == VerifySuccess: + found = true + break + elif newCTX.SSL_CTX_load_verify_locations(cstring(fn), nil) == VerifySuccess: found = true break if not found: @@ -685,17 +756,16 @@ when defineSsl: return ctx.getExtraInternal().clientGetPskFunc proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; - max_identity_len: cuint; psk: ptr cuchar; + max_identity_len: cuint; psk: ptr uint8; max_psk_len: cuint): cuint {.cdecl.} = let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let hintString = if hint == nil: "" else: $hint let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) - if psk.len.cuint > max_psk_len: + if pskString.len.cuint > max_psk_len: return 0 if identityString.len.cuint >= max_identity_len: return 0 - - copyMem(identity, identityString.cstring, pskString.len + 1) # with the last zero byte + copyMem(identity, identityString.cstring, identityString.len + 1) # with the last zero byte copyMem(psk, pskString.cstring, pskString.len) return pskString.len.cuint @@ -712,11 +782,11 @@ when defineSsl: proc serverGetPskFunc*(ctx: SslContext): SslServerGetPskFunc = return ctx.getExtraInternal().serverGetPskFunc - proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; + proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.} = let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let pskString = (ctx.serverGetPskFunc)($identity) - if psk.len.cint > max_psk_len: + if pskString.len.cint > max_psk_len: return 0 copyMem(psk, pskString.cstring, pskString.len) @@ -759,23 +829,28 @@ when defineSsl: if SSL_set_fd(socket.sslHandle, socket.fd) != 1: raiseSSLError() - proc checkCertName(socket: Socket, hostname: string) = + proc checkCertName(socket: Socket, hostname: string) {.raises: [SslError], tags:[RootEffect].} = ## Check if the certificate Subject Alternative Name (SAN) or Subject CommonName (CN) matches hostname. ## Wildcards match only in the left-most label. ## When name starts with a dot it will be matched by a certificate valid for any subdomain when not defined(nimDisableCertificateValidation) and not defined(windows): assert socket.isSsl - let certificate = socket.sslHandle.SSL_get_peer_certificate() - if certificate.isNil: - raiseSSLError("No SSL certificate found.") + try: + let certificate = socket.sslHandle.SSL_get_peer_certificate() + if certificate.isNil: + raiseSSLError("No SSL certificate found.") - const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint - const size = 1024 - var peername: string = newString(size) - let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint, - X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, peername) - if match != 1: - raiseSSLError("SSL Certificate check failed.") + const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint + # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html + let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint, + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil) + # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html + X509_free(certificate) + if match != 1: + raiseSSLError("SSL Certificate check failed.") + + except LibraryError: + raiseSSLError("SSL import failed") proc wrapConnectedSocket*(ctx: SslContext, socket: Socket, handshake: SslHandshakeType, @@ -802,6 +877,7 @@ when defineSsl: let ret = SSL_connect(socket.sslHandle) socketError(socket, ret) when not defined(nimDisableCertificateValidation) and not defined(windows): + # FIXME: this should be skipped on CVerifyNone if hostname.len > 0 and not isIpAddress(hostname): socket.checkCertName(hostname) of handshakeAsServer: @@ -954,14 +1030,16 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) - raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) + var address2: string + address2.addQuoted address + raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) + freeAddrInfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, flags = {SocketFlag.SafeDisconn}, inheritable = defined(nimInheritHandles)) {. - tags: [ReadIOEffect], gcsafe, locks: 0.} = + tags: [ReadIOEffect], gcsafe.} = ## Blocks until a connection is being made from a client. When a connection ## is made sets `client` to the client socket and `address` to the address ## of the connecting client. @@ -1076,7 +1154,7 @@ proc accept*(server: Socket, client: var owned(Socket), acceptAddr(server, client, addrDummy, flags) when defined(posix) and not defined(lwip): - from posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, + from std/posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, sigprocmask, pthread_sigmask, SIGPIPE, SIG_BLOCK, SIG_UNBLOCK template blockSigpipe(body: untyped): untyped = @@ -1190,9 +1268,9 @@ proc close*(socket: Socket, flags = {SocketFlag.SafeDisconn}) = socket.fd = osInvalidSocket when defined(posix): - from posix import TCP_NODELAY + from std/posix import TCP_NODELAY else: - from winlean import TCP_NODELAY + from std/winlean import TCP_NODELAY proc toCInt*(opt: SOBool): cint = ## Converts a `SOBool` into its Socket Option cint representation. @@ -1219,25 +1297,24 @@ proc getLocalAddr*(socket: Socket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: Socket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: Socket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = ## Sets option `opt` to a boolean value specified by `value`. - ## - ## .. code-block:: Nim - ## var socket = newSocket() - ## socket.setSockOpt(OptReusePort, true) - ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) - ## + runnableExamples("-r:off"): + let socket = newSocket() + socket.setSockOpt(OptReusePort, true) + socket.setSockOpt(OptNoDelay, true, level = IPPROTO_TCP.cint) var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(posix) or defined(nimdoc): +when defined(nimdoc) or (defined(posix) and not useNimNetLite): proc connectUnix*(socket: Socket, path: string) = ## Connects to Unix socket on `path`. ## This only works on Unix-style systems: Mac OS X, BSD and Linux @@ -1256,7 +1333,7 @@ when defined(posix) or defined(nimdoc): (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: raiseOSError(osLastError()) -when defined(ssl): +when defineSsl: proc gotHandshake*(socket: Socket): bool = ## Determines whether a handshake has occurred between a client (`socket`) ## and the server that `socket` is connected to. @@ -1277,14 +1354,6 @@ proc hasDataBuffered*(s: Socket): bool = if s.isSsl and not result: result = s.sslHasPeekChar -proc select(readfd: Socket, timeout = 500): int = - ## Used for socket operation timeouts. - if readfd.hasDataBuffered: - return 1 - - var fds = @[readfd.fd] - result = selectRead(fds, timeout) - proc isClosed(socket: Socket): bool = socket.fd == osInvalidSocket @@ -1398,7 +1467,9 @@ proc waitFor(socket: Socket, waited: var Duration, timeout, size: int, return min(sslPending, size) var startTime = getMonoTime() - let selRet = select(socket, (timeout - waited.inMilliseconds).int) + let selRet = if socket.hasDataBuffered: 1 + else: + timeoutRead(socket.fd, (timeout - waited.inMilliseconds).int) if selRet < 0: raiseOSError(osLastError()) if selRet != 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") @@ -1426,7 +1497,7 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): int = ## Higher-level version of `recv`. ## - ## Reads **up to** `size` bytes from `socket` into `buf`. + ## Reads **up to** `size` bytes from `socket` into `data`. ## ## For buffered sockets this function will attempt to read all the requested ## data. It will read this data in `BufferSize` chunks. @@ -1443,8 +1514,6 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## A timeout may be specified in milliseconds, if enough data is not received ## within the time specified a TimeoutError exception will be raised. ## - ## **Note**: `data` must be initialised. - ## ## .. warning:: Only the `SafeDisconn` flag is currently supported. data.setLen(size) result = @@ -1463,7 +1532,7 @@ proc recv*(socket: Socket, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): string {.inline.} = ## Higher-level version of `recv` which returns a string. ## - ## Reads **up to** `size` bytes from `socket` into `buf`. + ## Reads **up to** `size` bytes from `socket` into the result. ## ## For buffered sockets this function will attempt to read all the requested ## data. It will read this data in `BufferSize` chunks. @@ -1534,6 +1603,7 @@ proc readLine*(socket: Socket, line: var string, timeout = -1, if flags.isDisconnectionError(lastError): setLen(line, 0) socket.socketError(n, lastError = lastError, flags = flags) + return var waited: Duration @@ -1583,11 +1653,12 @@ proc recvLine*(socket: Socket, timeout = -1, result = "" readLine(socket, result, timeout, flags, maxLength) -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. +proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: int, + address: var T, port: var Port, flags = 0'i32): int {. tags: [ReadIOEffect].} = ## Receives data from `socket`. This function should normally be used with - ## connection-less sockets (UDP sockets). + ## connection-less sockets (UDP sockets). The source address of the data + ## packet is stored in the `address` argument as either a string or an IpAddress. ## ## If an error occurs an OSError exception will be raised. Otherwise the return ## value will be the length of data received. @@ -1596,31 +1667,37 @@ proc recvFrom*(socket: Socket, data: var string, length: int, ## so when `socket` is buffered the non-buffered implementation will be ## used. Therefore if `socket` contains something in its buffer this ## function will make no effort to return it. - template adaptRecvFromToDomain(domain: Domain) = - var addrLen = sizeof(sockAddress).SockLen + template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = + var addrLen = SockLen(sizeof(sockAddress)) result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) if result != -1: data.setLen(result) - address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) - when domain == AF_INET6: - port = ntohs(sockAddress.sin6_port).Port + + when typeof(address) is string: + address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) + when domain == AF_INET6: + port = ntohs(sockAddress.sin6_port).Port + else: + port = ntohs(sockAddress.sin_port).Port else: - port = ntohs(sockAddress.sin_port).Port + data.setLen(result) + sockAddress.fromSockAddr(addrLen, address, port) else: raiseOSError(osLastError()) assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket") # TODO: Buffered sockets data.setLen(length) + case socket.domain of AF_INET6: var sockAddress: Sockaddr_in6 - adaptRecvFromToDomain(AF_INET6) + adaptRecvFromToDomain(sockAddress, AF_INET6) of AF_INET: var sockAddress: Sockaddr_in - adaptRecvFromToDomain(AF_INET) + adaptRecvFromToDomain(sockAddress, AF_INET) else: raise newException(ValueError, "Unknown socket address family") @@ -1659,15 +1736,36 @@ proc send*(socket: Socket, data: pointer, size: int): int {. result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) proc send*(socket: Socket, data: string, - flags = {SocketFlag.SafeDisconn}) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - let sent = send(socket, cstring(data), data.len) - if sent < 0: - let lastError = osLastError() - socketError(socket, lastError = lastError, flags = flags) + flags = {SocketFlag.SafeDisconn}, maxRetries = 100) {.tags: [WriteIOEffect].} = + ## Sends data to a socket. Will try to send all the data by handling interrupts + ## and incomplete writes up to `maxRetries`. + var written = 0 + var attempts = 0 + while data.len - written > 0: + let sent = send(socket, cstring(data), data.len) - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") + if sent < 0: + let lastError = osLastError() + let isBlockingErr = + when defined(nimdoc): + false + elif useWinVersion: + lastError.int32 == WSAEINTR or + lastError.int32 == WSAEWOULDBLOCK + else: + lastError.int32 == EINTR or + lastError.int32 == EWOULDBLOCK or + lastError.int32 == EAGAIN + + if not isBlockingErr: + let lastError = osLastError() + socketError(socket, lastError = lastError, flags = flags) + else: + attempts.inc() + if attempts > maxRetries: + raiseOSError(osLastError(), "Could not send all data.") + else: + written.inc(sent) template `&=`*(socket: Socket; data: typed) = ## an alias for 'send'. @@ -1683,7 +1781,8 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, tags: [WriteIOEffect].} = ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. + ## this function will try each IP of that hostname. This function + ## should normally be used with connection-less sockets (UDP sockets). ## ## If an error occurs an OSError exception will be raised. ## @@ -1707,7 +1806,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, it = it.ai_next let osError = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(osError) @@ -1718,11 +1817,37 @@ proc sendTo*(socket: Socket, address: string, port: Port, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. ## + ## Generally for use with connection-less (UDP) sockets. + ## ## If an error occurs an OSError exception will be raised. ## ## This is the high-level version of the above `sendTo` function. socket.sendTo(address, port, cstring(data), data.len, socket.domain) +proc sendTo*(socket: Socket, address: IpAddress, port: Port, + data: string, flags = 0'i32): int {. + discardable, tags: [WriteIOEffect].} = + ## This proc sends `data` to the specified `IpAddress` and returns + ## the number of bytes written. + ## + ## Generally for use with connection-less (UDP) sockets. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This is the high-level version of the above `sendTo` function. + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") + + var sa: Sockaddr_storage + var sl: SockLen + toSockAddr(address, port, sa, sl) + result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, + cast[ptr SockAddr](addr sa), sl) + + if result == -1'i32: + let osError = osLastError() + raiseOSError(osError) + proc isSsl*(socket: Socket): bool = ## Determines whether `socket` is a SSL socket. @@ -1734,6 +1859,16 @@ proc isSsl*(socket: Socket): bool = proc getFd*(socket: Socket): SocketHandle = return socket.fd ## Returns the socket's file descriptor +when defined(zephyr) or defined(nimNetSocketExtras): # Remove in future + proc getDomain*(socket: Socket): Domain = return socket.domain + ## Returns the socket's domain + + proc getType*(socket: Socket): SockType = return socket.sockType + ## Returns the socket's type + + proc getProtocol*(socket: Socket): Protocol = return socket.protocol + ## Returns the socket's protocol + when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -1785,14 +1920,18 @@ proc `==`*(lhs, rhs: IpAddress): bool = proc `$`*(address: IpAddress): string = ## Converts an IpAddress into the textual representation - result = "" case address.family of IpAddressFamily.IPv4: - for i in 0 .. 3: - if i != 0: - result.add('.') - result.add($address.address_v4[i]) + result = newStringOfCap(15) + result.addInt address.address_v4[0] + result.add '.' + result.addInt address.address_v4[1] + result.add '.' + result.addInt address.address_v4[2] + result.add '.' + result.addInt address.address_v4[3] of IpAddressFamily.IPv6: + result = newStringOfCap(39) var currentZeroStart = -1 currentZeroCount = 0 @@ -1886,7 +2025,7 @@ proc dial*(address: string, port: Port, # network system problem (e.g. not enough FDs), and not an unreachable # address. let err = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds() raiseOSError(err) fdPerDomain[ord(domain)] = lastFd @@ -1895,18 +2034,20 @@ proc dial*(address: string, port: Port, break lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds(ord(domain)) if success: - result = newSocket(lastFd, domain, sockType, protocol) + result = newSocket(lastFd, domain, sockType, protocol, buffered) elif lastError != 0.OSErrorCode: + lastFd.close() raiseOSError(lastError) else: + lastFd.close() raise newException(IOError, "Couldn't resolve address: " & address) proc connect*(socket: Socket, address: string, - port = Port(0)) {.tags: [ReadIOEffect].} = + port = Port(0)) {.tags: [ReadIOEffect, RootEffect].} = ## Connects socket to `address`:`port`. `Address` can be an IP address or a ## host name. If `address` is a host name, this function will try each IP ## of that host name. `htons` is already performed on `port` so you must @@ -1925,7 +2066,7 @@ proc connect*(socket: Socket, address: string, else: lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) when defineSsl: @@ -1977,11 +2118,11 @@ proc connectAsync(socket: Socket, name: string, port = Port(0), it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) proc connect*(socket: Socket, address: string, port = Port(0), - timeout: int) {.tags: [ReadIOEffect, WriteIOEffect].} = + timeout: int) {.tags: [ReadIOEffect, WriteIOEffect, RootEffect].} = ## Connects to server as specified by `address` on port specified by `port`. ## ## The `timeout` parameter specifies the time in milliseconds to allow for @@ -1989,8 +2130,7 @@ proc connect*(socket: Socket, address: string, port = Port(0), socket.fd.setBlocking(false) socket.connectAsync(address, port, socket.domain) - var s = @[socket.fd] - if selectWrite(s, timeout) != 1: + if timeoutWrite(socket.fd, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: let res = getSockOptInt(socket.fd, SOL_SOCKET, SO_ERROR) @@ -2021,10 +2161,8 @@ proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress = ## ## Supports IPv4 and v6. ## Raises OSError if external networking is not set up. - ## - ## .. code-block:: Nim - ## echo $getPrimaryIPAddr() # "192.168.1.2" - + runnableExamples("-r:off"): + echo getPrimaryIPAddr() # "192.168.1.2" let socket = if dest.family == IpAddressFamily.IPv4: newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 721ae35c3a..bf8367d1de 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -9,18 +9,21 @@ ## Profiling support for Nim. This is an embedded profiler that requires ## `--profiler:on`. You only need to import this module to get a profiling -## report at program exit. +## report at program exit. See `Embedded Stack Trace Profiler `_ +## for usage. when not defined(profiler) and not defined(memProfiler): {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} -when defined(nimHasUsed): - {.used.} +{.used.} # We don't want to profile the profiling code ... {.push profiler: off.} -import hashes, algorithm, strutils, tables, sets +import std/[hashes, algorithm, strutils, tables, sets] + +when defined(nimPreviewSlimSystem): + import std/[syncio, sysatomics] when not defined(memProfiler): include "system/timers" @@ -66,7 +69,7 @@ when not defined(memProfiler): else: interval = intervalInUs * 1000 - tickCountCorrection when withThreads: - import locks + import std/locks var profilingLock: Lock @@ -121,13 +124,13 @@ when defined(memProfiler): var gTicker {.threadvar.}: int - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if gTicker == 0: gTicker = SamplingInterval result = true dec gTicker - proc hook(st: StackTrace, size: int) {.nimcall, locks: 0.} = + proc hook(st: StackTrace, size: int) {.nimcall.} = when defined(ignoreAllocationSize): hookAux(st, 1) else: @@ -139,7 +142,7 @@ else: gTicker: int # we use an additional counter to # avoid calling 'getTicks' too frequently - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if interval == 0: result = true elif gTicker == 0: gTicker = 500 @@ -148,7 +151,7 @@ else: else: dec gTicker - proc hook(st: StackTrace) {.nimcall, locks: 0.} = + proc hook(st: StackTrace) {.nimcall.} = #echo "profiling! ", interval if interval == 0: hookAux(st, 1) diff --git a/lib/pure/nimtracker.nim b/lib/pure/nimtracker.nim deleted file mode 100644 index a66dfc2eab..0000000000 --- a/lib/pure/nimtracker.nim +++ /dev/null @@ -1,88 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Memory tracking support for Nim. - -when not defined(memTracker) and not isMainModule: - {.error: "Memory tracking support is turned off!".} - -{.push memtracker: off.} -# we import the low level wrapper and are careful not to use Nim's -# memory manager for anything here. -import sqlite3 - -var - dbHandle: PSqlite3 - insertStmt {.threadvar.}: Pstmt - -const insertQuery = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - -template sbind(x: int; value) = - when value is cstring: - let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT) - if ret != SQLITE_OK: - quit "could not bind value" - else: - let ret = insertStmt.bindInt64(x, value) - if ret != SQLITE_OK: - quit "could not bind value" - -when defined(memTracker): - proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [], gcsafe.} = - if insertStmt.isNil: - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) != SQLITE_OK: - quit "could not bind query to insertStmt " & $sqlite3.errmsg(dbHandle) - for i in 0..log.count-1: - var success = false - let e = log.data[i] - discard sqlite3.reset(insertStmt) - discard clearBindings(insertStmt) - sbind 1, e.op - sbind(2, cast[int](e.address)) - sbind 3, e.size - sbind 4, e.file - sbind 5, e.line - if step(insertStmt) == SQLITE_DONE: - success = true - if not success: - quit "could not write to database! " & $sqlite3.errmsg(dbHandle) - -proc execQuery(q: string) = - var s: Pstmt - if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK: - discard step(s) - if finalize(s) != SQLITE_OK: - quit "could not finalize " & $sqlite3.errmsg(dbHandle) - else: - quit "could not prepare statement " & $sqlite3.errmsg(dbHandle) - -proc setupDb() = - execQuery """create table if not exists tracking( - id integer primary key, - op varchar not null, - address integer not null, - size integer not null, - file varchar not null, - line integer not null - )""" - execQuery "delete from tracking" - -if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK: - setupDb() - const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) == SQLITE_OK: - when defined(memTracker): - setTrackLogger logEntries - else: - quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle) -else: - quit "could not setup sqlite " & $sqlite3.errmsg(dbHandle) -{.pop.} diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index fb70047b6d..4d6ceefd79 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -9,8 +9,7 @@ ## Nim OID support. An OID is a global ID that consists of a timestamp, ## a unique counter and a random value. This combination should suffice to -## produce a globally distributed unique ID. This implementation was extracted -## from the MongoDB interface and is thus binary compatible with a MongoDB OID. +## produce a globally distributed unique ID. ## ## This implementation calls `initRand()` for the first call of ## `genOid`. @@ -18,9 +17,12 @@ import std/[hashes, times, endians, random] from std/private/decode_helpers import handleHexChar +when defined(nimPreviewSlimSystem): + import std/sysatomics + type Oid* = object ## An OID. - time: int32 + time: int64 fuzz: int32 count: int32 @@ -42,44 +44,29 @@ proc hexbyte*(hex: char): int {.inline.} = proc parseOid*(str: cstring): Oid = ## Parses an OID. - var bytes = cast[cstring](addr(result.time)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(result.time)) + 4)) var i = 0 while i < 12: bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) inc(i) -template toStringImpl[T: string | cstring](result: var T, oid: Oid) = - ## Stringifies `oid`. +proc `$`*(oid: Oid): string = + ## Converts an OID to a string. const hex = "0123456789abcdef" - const N = 24 - when T is string: - result.setLen N + result.setLen 24 var o = oid - var bytes = cast[cstring](addr(o)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(o)) + 4)) var i = 0 while i < 12: let b = bytes[i].ord result[2 * i] = hex[(b and 0xF0) shr 4] result[2 * i + 1] = hex[b and 0xF] inc(i) - when T is cstring: - result[N] = '\0' - -proc oidToString*(oid: Oid, str: cstring) {.deprecated: "unsafe; use `$`".} = - ## Converts an oid to a string which must have space allocated for 25 elements. - # work around a compiler bug: - var str = str - toStringImpl(str, oid) - -proc `$`*(oid: Oid): string = - ## Converts an OID to a string. - toStringImpl(result, oid) - let - t = getTime().toUnix.int32 + t = getTime().toUnix var seed = initRand(t) @@ -89,10 +76,10 @@ let fuzz = cast[int32](seed.rand(high(int))) template genOid(result: var Oid, incr: var int, fuzz: int32) = - var time = getTime().toUnix.int32 + var time = getTime().toUnix var i = cast[int32](atomicInc(incr)) - bigEndian32(addr result.time, addr(time)) + bigEndian64(addr result.time, addr(time)) result.fuzz = fuzz bigEndian32(addr result.count, addr(i)) @@ -106,7 +93,7 @@ proc genOid*(): Oid = proc generatedTime*(oid: Oid): Time = ## Returns the generated timestamp of the OID. - var tmp: int32 + var tmp: int64 var dummy = oid.time - bigEndian32(addr(tmp), addr(dummy)) + bigEndian64(addr(tmp), addr(dummy)) result = fromUnix(tmp) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 63e3598f09..b34ff72c0d 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -53,7 +53,7 @@ Pattern matching supports pattern matching on `Option`s, with the `Some()` and `None()` patterns. -.. code-block:: nim + ```nim {.experimental: "caseStmtMacros".} import fusion/matching @@ -65,15 +65,24 @@ supports pattern matching on `Option`s, with the `Some()` and assert false assertMatch(some(some(none(int))), Some(Some(None()))) + ``` ]## # xxx pending https://github.com/timotheecour/Nim/issues/376 use `runnableExamples` and `whichModule` +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} import std/typetraits +when defined(nimPreviewSlimSystem): + import std/assertions + + when (NimMajor, NimMinor) >= (1, 1): type - SomePointer = ref | ptr | pointer | proc + SomePointer = ref | ptr | pointer | proc | iterator {.closure.} else: type SomePointer = ref | ptr | pointer @@ -81,7 +90,7 @@ else: type Option*[T] = object ## An optional type that may or may not contain a value of type `T`. - ## When `T` is a a pointer type (`ptr`, `pointer`, `ref` or `proc`), + ## When `T` is a a pointer type (`ptr`, `pointer`, `ref`, `proc` or `iterator {.closure.}`), ## `none(T)` is represented as `nil`. when T is SomePointer: val: T @@ -92,7 +101,7 @@ type UnpackDefect* = object of Defect UnpackError* {.deprecated: "See corresponding Defect".} = UnpackDefect -proc option*[T](val: T): Option[T] {.inline.} = +proc option*[T](val: sink T): Option[T] {.inline.} = ## Can be used to convert a pointer type (`ptr`, `pointer`, `ref` or `proc`) to an option type. ## It converts `nil` to `none(T)`. When `T` is no pointer type, this is equivalent to `some(val)`. ## @@ -108,11 +117,12 @@ proc option*[T](val: T): Option[T] {.inline.} = assert option[Foo](nil).isNone assert option(42).isSome - result.val = val - when T isnot SomePointer: - result.has = true + when T is SomePointer: + result = Option[T](val: val) + else: + result = Option[T](has: true, val: val) -proc some*[T](val: T): Option[T] {.inline.} = +proc some*[T](val: sink T): Option[T] {.inline.} = ## Returns an `Option` that has the value `val`. ## ## **See also:** @@ -127,10 +137,9 @@ proc some*[T](val: T): Option[T] {.inline.} = when T is SomePointer: assert not val.isNil - result.val = val + result = Option[T](val: val) else: - result.has = true - result.val = val + result = Option[T](has: true, val: val) proc none*(T: typedesc): Option[T] {.inline.} = ## Returns an `Option` for this type that has no value. @@ -143,7 +152,7 @@ proc none*(T: typedesc): Option[T] {.inline.} = assert none(int).isNone # the default is the none type - discard + result = Option[T]() proc none*[T]: Option[T] {.inline.} = ## Alias for `none(T) <#none,typedesc>`_. @@ -222,7 +231,7 @@ proc get*[T](self: var Option[T]): var T {.inline.} = raise newException(UnpackDefect, "Can't obtain a value from a `none`") return self.val -proc map*[T](self: Option[T], callback: proc (input: T)) {.inline.} = +proc map*[T](self: Option[T], callback: proc (input: T)) {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option`, if it has one. ## ## **See also:** @@ -241,7 +250,7 @@ proc map*[T](self: Option[T], callback: proc (input: T)) {.inline.} = if self.isSome: callback(self.val) -proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] {.inline.} = +proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option` and returns an ## `Option` containing the new value. ## @@ -278,7 +287,7 @@ proc flatten*[T](self: Option[Option[T]]): Option[T] {.inline.} = none(T) proc flatMap*[T, R](self: Option[T], - callback: proc (input: T): Option[R]): Option[R] {.inline.} = + callback: proc (input: T): Option[R]): Option[R] {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option` and returns the new value. ## ## If the `Option` has no value, `none(R)` will be returned. @@ -303,7 +312,7 @@ proc flatMap*[T, R](self: Option[T], map(self, callback).flatten() -proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] {.inline.} = +proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] {.inline, effectsOf: callback.} = ## Applies a `callback` to the value of the `Option`. ## ## If the `callback` returns `true`, the option is returned as `some`. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index d0b3aef1a9..6a7b4af1b2 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -8,51 +8,62 @@ # ## This module contains basic operating system facilities like -## retrieving environment variables, reading command line arguments, -## working with directories, running shell commands, etc. -## -## .. code-block:: -## import std/os -## -## let myFile = "/path/to/my/file.nim" -## -## let pathSplit = splitPath(myFile) -## assert pathSplit.head == "/path/to/my" -## assert pathSplit.tail == "file.nim" -## -## assert parentDir(myFile) == "/path/to/my" -## -## let fileSplit = splitFile(myFile) -## assert fileSplit.dir == "/path/to/my" -## assert fileSplit.name == "file" -## assert fileSplit.ext == ".nim" -## -## assert myFile.changeFileExt("c") == "/path/to/my/file.c" +## retrieving environment variables, working with directories, +## running shell commands, etc. + +## .. importdoc:: symlinks.nim, appdirs.nim, dirs.nim, ospaths2.nim + +runnableExamples: + let myFile = "/path/to/my/file.nim" + assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim") + when defined(posix): + assert parentDir(myFile) == "/path/to/my" + assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim") + assert myFile.changeFileExt("c") == "/path/to/my/file.c" -## -## ## **See also:** +## * `paths `_ and `files `_ modules for high-level file manipulation ## * `osproc module `_ for process communication beyond -## `execShellCmd proc <#execShellCmd,string>`_ -## * `parseopt module `_ for command-line parser beyond -## `parseCmdLine proc <#parseCmdLine,string>`_ +## `execShellCmd proc`_ ## * `uri module `_ ## * `distros module `_ ## * `dynlib module `_ ## * `streams module `_ +import std/private/ospaths2 +export ospaths2 + +import std/private/osfiles +export osfiles + +import std/private/osdirs +export osdirs + +import std/private/ossymlinks +export ossymlinks + +import std/private/osappdirs +export osappdirs + +import std/private/oscommon include system/inclrtl import std/private/since +import std/cmdline +export cmdline + import std/[strutils, pathnorm] +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + const weirdTarget = defined(nimscript) or defined(js) since (1, 1): const invalidFilenameChars* = {'/', '\\', ':', '*', '?', '"', '<', '>', '|', '^', '\0'} ## \ - ## Characters that may produce invalid filenames across Linux, Windows, Mac, etc. - ## You can check if your filename contains these char and strip them for safety. + ## Characters that may produce invalid filenames across Linux, Windows and Mac. + ## You can check if your filename contains any of these chars and strip them for safety. ## Mac bans ``':'``, Linux bans ``'/'``, Windows bans all others. invalidFilenames* = [ "CON", "PRN", "AUX", "NUL", @@ -87,943 +98,35 @@ elif defined(js): else: {.pragma: noNimJs.} -proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} -type - ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read - ## from an environment variable. - WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write - ## to an environment variable. +import std/oserrors +export oserrors +import std/envvars +export envvars - ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read - ## operation from the directory - ## structure. - WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write - ## operation to - ## the directory structure. +import std/private/osseps +export osseps - OSErrorCode* = distinct int32 ## Specifies an OS Error Code. -include "includes/osseps" - -proc absolutePathInternal(path: string): string {.gcsafe.} - -proc normalizePathEnd(path: var string, trailingSep = false) = - ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on - ## ``trailingSep``, and taking care of edge cases: it preservers whether - ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, - ## not `AltSep`. Trailing `/.` are compressed, see examples. - if path.len == 0: return - var i = path.len - while i >= 1: - if path[i-1] in {DirSep, AltSep}: dec(i) - elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) - else: break - if trailingSep: - # foo// => foo - path.setLen(i) - # foo => foo/ - path.add DirSep - elif i > 0: - # foo// => foo - path.setLen(i) - else: - # // => / (empty case was already taken care of) - path = $DirSep - -proc normalizePathEnd(path: string, trailingSep = false): string = - ## outplace overload - runnableExamples: - when defined(posix): - assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" - assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" - assert normalizePathEnd(".//./.", trailingSep = false) == "." - assert normalizePathEnd("", trailingSep = true) == "" # not / ! - assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! - result = path - result.normalizePathEnd(trailingSep) - -since((1, 1)): - export normalizePathEnd - -template endsWith(a: string, b: set[char]): bool = - a.len > 0 and a[^1] in b - -proc joinPathImpl(result: var string, state: var int, tail: string) = - let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) - normalizePathEnd(result, trailingSep=false) - addNormalizePath(tail, result, state, DirSep) - normalizePathEnd(result, trailingSep=trailingSep) - -proc joinPath*(head, tail: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Joins two directory names to one. - ## - ## returns normalized path concatenation of `head` and `tail`, preserving - ## whether or not `tail` has a trailing slash (or, if tail if empty, whether - ## head has one). - ## - ## See also: - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `uri.combine proc `_ - ## * `uri./ proc `_ - runnableExamples: - when defined(posix): - assert joinPath("usr", "lib") == "usr/lib" - assert joinPath("usr", "lib/") == "usr/lib/" - assert joinPath("usr", "") == "usr" - assert joinPath("usr/", "") == "usr/" - assert joinPath("", "") == "" - assert joinPath("", "lib") == "lib" - assert joinPath("", "/lib") == "/lib" - assert joinPath("usr/", "/lib") == "usr/lib" - assert joinPath("usr/lib", "../bin") == "usr/bin" - - result = newStringOfCap(head.len + tail.len) - var state = 0 - joinPathImpl(result, state, head) - joinPathImpl(result, state, tail) - when false: - if len(head) == 0: - result = tail - elif head[len(head)-1] in {DirSep, AltSep}: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & substr(tail, 1) - else: - result = head & tail - else: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & tail - else: - result = head & DirSep & tail - -proc joinPath*(parts: varargs[string]): string {.noSideEffect, - rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_, - ## but works with any number of directory parts. - ## - ## You need to pass at least one element or the proc - ## will assert in debug builds and crash on release builds. - ## - ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - runnableExamples: - when defined(posix): - assert joinPath("a") == "a" - assert joinPath("a", "b", "c") == "a/b/c" - assert joinPath("usr/lib", "../../var", "log") == "var/log" - - var estimatedLen = 0 - for p in parts: estimatedLen += p.len - result = newStringOfCap(estimatedLen) - var state = 0 - for i in 0..high(parts): - joinPathImpl(result, state, parts[i]) - -proc `/`*(head, tail: string): string {.noSideEffect, inline.} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_. - ## - ## See also: - ## * `/../ proc <#/../,string,string>`_ - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `uri.combine proc `_ - ## * `uri./ proc `_ - runnableExamples: - when defined(posix): - assert "usr" / "" == "usr" - assert "" / "lib" == "lib" - assert "" / "/lib" == "/lib" - assert "usr/" / "/lib/" == "usr/lib/" - assert "usr" / "lib" / "../bin" == "usr/bin" - - result = joinPath(head, tail) - -proc splitPath*(path: string): tuple[head, tail: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a directory into `(head, tail)` tuple, so that - ## ``head / tail == path`` (except for edge cases like "/usr"). - ## - ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `relativePath proc <#relativePath,string,string>`_ - runnableExamples: - assert splitPath("usr/local/bin") == ("usr/local", "bin") - assert splitPath("usr/local/bin/") == ("usr/local/bin", "") - assert splitPath("/bin/") == ("/bin", "") - when (NimMajor, NimMinor) <= (1, 0): - assert splitPath("/bin") == ("", "bin") - else: - assert splitPath("/bin") == ("/", "bin") - assert splitPath("bin") == ("", "bin") - assert splitPath("") == ("", "") - - var sepPos = -1 - for i in countdown(len(path)-1, 0): - if path[i] in {DirSep, AltSep}: - sepPos = i - break - if sepPos >= 0: - result.head = substr(path, 0, - when (NimMajor, NimMinor) <= (1, 0): - sepPos-1 - else: - if likely(sepPos >= 1): sepPos-1 else: 0 - ) - result.tail = substr(path, sepPos+1) - else: - result.head = "" - result.tail = path - -proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = - ## Checks whether a given `path` is absolute. - ## - ## On Windows, network paths are considered absolute too. - runnableExamples: - assert not "".isAbsolute - assert not ".".isAbsolute - when defined(posix): - assert "/".isAbsolute - assert not "a/".isAbsolute - assert "/a/".isAbsolute - - if len(path) == 0: return false - - when doslikeFileSystem: - var len = len(path) - result = (path[0] in {'/', '\\'}) or - (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') - elif defined(macos): - # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path - result = path[0] != ':' - elif defined(RISCOS): - result = path[0] == '$' - elif defined(posix) or defined(js): - # `or defined(js)` wouldn't be needed pending https://github.com/nim-lang/Nim/issues/13469 - # This works around the problem for posix, but windows is still broken with nim js -d:nodejs - result = path[0] == '/' - else: - doAssert false # if ever hits here, adapt as needed - -when FileSystemCaseSensitive: - template `!=?`(a, b: char): bool = a != b -else: - template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) - -when doslikeFileSystem: - proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: []} = - ## An absolute path from the root of the current drive (e.g. "\foo") - path.len > 0 and - (path[0] == AltSep or - (path[0] == DirSep and - (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) - - proc isUNCPrefix(path: string): bool {.noSideEffect, raises: []} = - path[0] == DirSep and path[1] == DirSep - - proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: []} = - ## Return true if path1 and path2 have a same root. - ## - ## Detail of windows path formats: - ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats - - assert(isAbsolute(path1)) - assert(isAbsolute(path2)) - - let - len1 = path1.len - len2 = path2.len - assert(len1 != 0 and len2 != 0) - - if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): - return true - elif len1 == 1 or len2 == 1: - return false - else: - if path1[1] == ':' and path2[1] == ':': - return path1[0].toLowerAscii() == path2[0].toLowerAscii() - else: - var - p1, p2: PathIter - pp1 = next(p1, path1) - pp2 = next(p2, path2) - if pp1[1] - pp1[0] == 1 and pp2[1] - pp2[0] == 1 and - isUNCPrefix(path1) and isUNCPrefix(path2): - #UNC - var h = 0 - while p1.hasNext(path1) and p2.hasNext(path2) and h < 2: - pp1 = next(p1, path1) - pp2 = next(p2, path2) - let diff = pp1[1] - pp1[0] - if diff != pp2[1] - pp2[0]: - return false - for i in 0..diff: - if path1[i + pp1[0]] !=? path2[i + pp2[0]]: - return false - inc h - return h == 2 - else: - return false - -proc relativePath*(path, base: string, sep = DirSep): string {. - rtl, extern: "nos$1".} = - ## Converts `path` to a path relative to `base`. - ## - ## The `sep` (default: `DirSep <#DirSep>`_) is used for the path normalizations, - ## this can be useful to ensure the relative path only contains `'/'` - ## so that it can be used for URL constructions. - ## - ## On windows, if a root of `path` and a root of `base` are different, - ## returns `path` as is because it is impossible to make a relative path. - ## That means an absolute path can be returned. - ## - ## See also: - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - runnableExamples: - assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" - assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" - assert relativePath("", "/users/moo", '/') == "" - assert relativePath("foo", ".", '/') == "foo" - assert relativePath("foo", "foo", '/') == "." - - if path.len == 0: return "" - var base = if base == ".": "" else: base - var path = path - path.normalizePathAux - base.normalizePathAux - let a1 = isAbsolute(path) - let a2 = isAbsolute(base) - if a1 and not a2: - base = absolutePathInternal(base) - elif a2 and not a1: - path = absolutePathInternal(path) - - when doslikeFileSystem: - if isAbsolute(path) and isAbsolute(base): - if not sameRoot(path, base): - return path - - var f = default PathIter - var b = default PathIter - var ff = (0, -1) - var bb = (0, -1) # (int, int) - result = newStringOfCap(path.len) - # skip the common prefix: - while f.hasNext(path) and b.hasNext(base): - ff = next(f, path) - bb = next(b, base) - let diff = ff[1] - ff[0] - if diff != bb[1] - bb[0]: break - var same = true - for i in 0..diff: - if path[i + ff[0]] !=? base[i + bb[0]]: - same = false - break - if not same: break - ff = (0, -1) - bb = (0, -1) - # for i in 0..diff: - # result.add base[i + bb[0]] - - # /foo/bar/xxx/ -- base - # /foo/bar/baz -- path path - # ../baz - # every directory that is in 'base', needs to add '..' - while true: - if bb[1] >= bb[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - result.add ".." - if not b.hasNext(base): break - bb = b.next(base) - - # add the rest of 'path': - while true: - if ff[1] >= ff[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - for i in 0..ff[1] - ff[0]: - result.add path[i + ff[0]] - if not f.hasNext(path): break - ff = f.next(path) - - when not defined(nimOldRelativePathBehavior): - if result.len == 0: result.add "." - -proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = - ## Returns true if `path` is relative to `base`. - runnableExamples: - doAssert isRelativeTo("./foo//bar", "foo") - doAssert isRelativeTo("foo/bar", ".") - doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") - doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") - let path = path.normalizePath - let base = base.normalizePath - let ret = relativePath(path, base) - result = path.len > 0 and not ret.startsWith ".." - -proc parentDirPos(path: string): int = - var q = 1 - if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 - for i in countdown(len(path)-q, 0): - if path[i] in {DirSep, AltSep}: return i - result = -1 - -proc parentDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the parent directory of `path`. - ## - ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end - ## in a dir separator, but also takes care of path normalizations. - ## The remainder can be obtained with `lastPathPart(path) proc - ## <#lastPathPart,string>`_. - ## - ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - ## * `parentDirs iterator <#parentDirs.i,string>`_ - runnableExamples: - assert parentDir("") == "" - when defined(posix): - assert parentDir("/usr/local/bin") == "/usr/local" - assert parentDir("foo/bar//") == "foo" - assert parentDir("//foo//bar//.") == "/foo" - assert parentDir("./foo") == "." - assert parentDir("/./foo//./") == "/" - assert parentDir("a//./") == "." - assert parentDir("a/b/c/..") == "a" - result = pathnorm.normalizePath(path) - var sepPos = parentDirPos(result) - if sepPos >= 0: - result = substr(result, 0, sepPos) - normalizePathEnd(result) - elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: - # `.` => `..` and .. => `../..`(etc) would be a sensible alternative - # `/` => `/` (as done with splitFile) would be a sensible alternative - result = "" - else: - result = "." - -proc tailDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the tail part of `path`. - ## - ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - runnableExamples: - assert tailDir("/bin") == "bin" - assert tailDir("bin") == "" - assert tailDir("bin/") == "" - assert tailDir("/usr/local/bin") == "usr/local/bin" - assert tailDir("//usr//local//bin//") == "usr//local//bin//" - assert tailDir("./usr/local/bin") == "usr/local/bin" - assert tailDir("usr/local/bin") == "local/bin" - - var i = 0 - while i < len(path): - if path[i] in {DirSep, AltSep}: - while i < len(path) and path[i] in {DirSep, AltSep}: inc i - return substr(path, i) - inc i - result = "" - -proc isRootDir*(path: string): bool {. - noSideEffect, rtl, extern: "nos$1".} = - ## Checks whether a given `path` is a root directory. - runnableExamples: - assert isRootDir("") - assert isRootDir(".") - assert isRootDir("/") - assert isRootDir("a") - assert not isRootDir("/a") - assert not isRootDir("a/b/c") - - result = parentDirPos(path) < 0 - -iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = - ## Walks over all parent directories of a given `path`. - ## - ## If `fromRoot` is true (default: false), the traversal will start from - ## the file system root directory. - ## If `inclusive` is true (default), the original argument will be included - ## in the traversal. - ## - ## Relative paths won't be expanded by this iterator. Instead, it will traverse - ## only the directories appearing in the relative path. - ## - ## See also: - ## * `parentDir proc <#parentDir,string>`_ - ## - ## **Examples:** - ## - ## .. code-block:: - ## let g = "a/b/c" - ## - ## for p in g.parentDirs: - ## echo p - ## # a/b/c - ## # a/b - ## # a - ## - ## for p in g.parentDirs(fromRoot=true): - ## echo p - ## # a/ - ## # a/b/ - ## # a/b/c - ## - ## for p in g.parentDirs(inclusive=false): - ## echo p - ## # a/b - ## # a - - if not fromRoot: - var current = path - if inclusive: yield path - while true: - if current.isRootDir: break - current = current.parentDir - yield current - else: - for i in countup(0, path.len - 2): # ignore the last / - # deal with non-normalized paths such as /foo//bar//baz - if path[i] in {DirSep, AltSep} and - (i == 0 or path[i-1] notin {DirSep, AltSep}): - yield path.substr(0, i) - - if inclusive: yield path - -proc `/../`*(head, tail: string): string {.noSideEffect.} = - ## The same as ``parentDir(head) / tail``, unless there is no parent - ## directory. Then ``head / tail`` is performed instead. - ## - ## See also: - ## * `/ proc <#/,string,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - runnableExamples: - when defined(posix): - assert "a/b/c" /../ "d/e" == "a/b/d/e" - assert "a" /../ "d/e" == "a/d/e" - - let sepPos = parentDirPos(head) - if sepPos >= 0: - result = substr(head, 0, sepPos-1) / tail - else: - result = head / tail - -proc normExt(ext: string): string = - if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here - else: result = ExtSep & ext - -proc searchExtPos*(path: string): int = - ## Returns index of the `'.'` char in `path` if it signifies the beginning - ## of extension. Returns -1 otherwise. - ## - ## See also: - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert searchExtPos("a/b/c") == -1 - assert searchExtPos("c.nim") == 1 - assert searchExtPos("a/b/c.nim") == 5 - assert searchExtPos("a.b.c.nim") == 5 - - # BUGFIX: do not search until 0! .DS_Store is no file extension! - result = -1 - for i in countdown(len(path)-1, 1): - if path[i] == ExtSep: - result = i - break - elif path[i] in {DirSep, AltSep}: - break # do not skip over path - -proc splitFile*(path: string): tuple[dir, name, ext: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a filename into `(dir, name, extension)` tuple. - ## - ## `dir` does not end in `DirSep <#DirSep>`_ unless it's `/`. - ## `extension` includes the leading dot. - ## - ## If `path` has no extension, `ext` is the empty string. - ## If `path` has no directory component, `dir` is the empty string. - ## If `path` has no filename component, `name` and `ext` are empty strings. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - var (dir, name, ext) = splitFile("usr/local/nimc.html") - assert dir == "usr/local" - assert name == "nimc" - assert ext == ".html" - (dir, name, ext) = splitFile("/usr/local/os") - assert dir == "/usr/local" - assert name == "os" - assert ext == "" - (dir, name, ext) = splitFile("/usr/local/") - assert dir == "/usr/local" - assert name == "" - assert ext == "" - (dir, name, ext) = splitFile("/tmp.txt") - assert dir == "/" - assert name == "tmp" - assert ext == ".txt" - - var namePos = 0 - var dotPos = 0 - for i in countdown(len(path) - 1, 0): - if path[i] in {DirSep, AltSep} or i == 0: - if path[i] in {DirSep, AltSep}: - result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) - namePos = i + 1 - if dotPos > i: - result.name = substr(path, namePos, dotPos - 1) - result.ext = substr(path, dotPos) - else: - result.name = substr(path, namePos) - break - elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and - path[i - 1] notin {DirSep, AltSep} and - path[i + 1] != ExtSep and dotPos == 0: - dotPos = i - -proc extractFilename*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Extracts the filename of a given `path`. - ## - ## This is the same as ``name & ext`` from `splitFile(path) proc - ## <#splitFile,string>`_. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert extractFilename("foo/bar/") == "" - assert extractFilename("foo/bar") == "bar" - assert extractFilename("foo/bar.baz") == "bar.baz" - - if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: - result = "" - else: - result = splitPath(path).tail - -proc lastPathPart*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Like `extractFilename proc <#extractFilename,string>`_, but ignores - ## trailing dir separator; aka: `baseName`:idx: in some other languages. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert lastPathPart("foo/bar/") == "bar" - assert lastPathPart("foo/bar") == "bar" - - let path = path.normalizePathEnd(trailingSep = false) - result = extractFilename(path) - -proc changeFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Changes the file extension to `ext`. - ## - ## If the `filename` has no extension, `ext` will be added. - ## If `ext` == "" then any extension is removed. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. (Although I know - ## of none such beast.) - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert changeFileExt("foo.bar", "baz") == "foo.baz" - assert changeFileExt("foo.bar", "") == "foo" - assert changeFileExt("foo", "baz") == "foo.baz" - - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = substr(filename, 0, extPos-1) & normExt(ext) - -proc addFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Adds the file extension `ext` to `filename`, unless - ## `filename` already has an extension. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. - ## (Although I know of none such beast.) - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - runnableExamples: - assert addFileExt("foo.bar", "baz") == "foo.bar" - assert addFileExt("foo.bar", "") == "foo.bar" - assert addFileExt("foo", "baz") == "foo.baz" - - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = filename - -proc cmpPaths*(pathA, pathB: string): int {. - noSideEffect, rtl, extern: "nos$1".} = - ## Compares two paths. - ## - ## On a case-sensitive filesystem this is done - ## case-sensitively otherwise case-insensitively. Returns: - ## - ## | 0 if pathA == pathB - ## | < 0 if pathA < pathB - ## | > 0 if pathA > pathB - runnableExamples: - when defined(macosx): - assert cmpPaths("foo", "Foo") == 0 - elif defined(posix): - assert cmpPaths("foo", "Foo") > 0 - - let a = normalizePath(pathA) - let b = normalizePath(pathB) - if FileSystemCaseSensitive: - result = cmp(a, b) - else: - when defined(nimscript): - result = cmpic(a, b) - elif defined(nimdoc): discard - else: - result = cmpIgnoreCase(a, b) - -proc unixToNativePath*(path: string, drive=""): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Converts an UNIX-like path to a native one. - ## - ## On an UNIX system this does nothing. Else it converts - ## `'/'`, `'.'`, `'..'` to the appropriate things. - ## - ## On systems with a concept of "drives", `drive` is used to determine - ## which drive label to use during absolute path conversion. - ## `drive` defaults to the drive of the current working directory, and is - ## ignored on systems that do not have a concept of "drives". - when defined(unix): - result = path - else: - if path.len == 0: return "" - - var start: int - if path[0] == '/': - # an absolute path - when doslikeFileSystem: - if drive != "": - result = drive & ":" & DirSep - else: - result = $DirSep - elif defined(macos): - result = "" # must not start with ':' - else: - result = $DirSep - start = 1 - elif path[0] == '.' and (path.len == 1 or path[1] == '/'): - # current directory - result = $CurDir - start = when doslikeFileSystem: 1 else: 2 - else: - result = "" - start = 0 - - var i = start - while i < len(path): # ../../../ --> :::: - if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': - # parent directory - when defined(macos): - if result[high(result)] == ':': - add result, ':' - else: - add result, ParDir - else: - add result, ParDir & DirSep - inc(i, 3) - elif path[i] == '/': - add result, DirSep - inc(i) - else: - add result, path[i] - inc(i) - -include "includes/oserr" -when not defined(nimscript): - include "includes/osenv" - -proc getHomeDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the home directory of the current user. - ## - ## This proc is wrapped by the `expandTilde proc <#expandTilde,string>`_ - ## for the convenience of processing paths coming from user configuration files. - ## - ## See also: - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - runnableExamples: - assert getHomeDir() == expandTilde("~") - # `getHomeDir()` doesn't end in `DirSep` even if `$HOME` (on posix) or - # `$USERPROFILE` (on windows) does, unless `-d:nimLegacyHomeDir` is specified. - from std/strutils import endsWith - assert not getHomeDir().endsWith DirSep - - when defined(windows): result = getEnv("USERPROFILE") - else: result = getEnv("HOME") - result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir)) - -proc getConfigDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the config directory of the current user for applications. - ## - ## On non-Windows OSs, this proc conforms to the XDG Base Directory - ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment - ## variable if it is set, otherwise it returns the default configuration - ## directory ("~/.config"). - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - runnableExamples: - from std/strutils import endsWith - # See `getHomeDir` for behavior regarding trailing DirSep. - assert not getConfigDir().endsWith DirSep - when defined(windows): - result = getEnv("APPDATA") - else: - result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") - result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir)) - -when defined(windows): - type DWORD = uint32 - - proc getTempPath( - nBufferLength: DWORD, lpBuffer: WideCString - ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = - ## Retrieves the path of the directory designated for temporary files. - -template getEnvImpl(result: var string, tempDirList: openArray[string]) = - for dir in tempDirList: - if existsEnv(dir): - result = getEnv(dir) - break - -template getTempDirImpl(result: var string) = - when defined(windows): - getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) - else: - getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) - -proc getTempDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the temporary directory of the current user for applications to - ## save temporary files in. - ## - ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). - ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. - ## On all platforms, `/tmp` will be returned if the procs fails. - ## - ## You can override this implementation - ## by adding `-d:tempDir=mytempname` to your compiler invocation. - ## - ## **Note:** This proc does not check whether the returned path exists. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - runnableExamples: - from std/strutils import endsWith - # See `getHomeDir` for behavior regarding trailing DirSep. - assert not getTempDir().endsWith(DirSep) - const tempDirDefault = "/tmp" - when defined(tempDir): - const tempDir {.strdefine.}: string = tempDirDefault - result = tempDir - else: - when nimvm: - getTempDirImpl(result) - else: - when defined(windows): - let size = getTempPath(0, nil) - # If the function fails, the return value is zero. - if size > 0: - let buffer = newWideCString(size.int) - if getTempPath(size, buffer) > 0: - result = $buffer - elif defined(android): result = "/data/local/tmp" - else: - getTempDirImpl(result) - if result.len == 0: - result = tempDirDefault - result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir)) proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing - ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## ``~`` with `getHomeDir()`_ (otherwise returns ``path`` unmodified). ## - ## Windows: this is still supported despite Windows platform not having this + ## Windows: this is still supported despite the Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. - ## - ## .. warning:: `~bob` and `~bob/` are not yet handled correctly. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ runnableExamples: assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar" assert expandTilde("/foo/bar") == "/foo/bar" - assert expandTilde("~") == getHomeDir() - from std/strutils import endsWith - assert not expandTilde("~").endsWith(DirSep) if len(path) == 0 or path[0] != '~': result = path @@ -1035,15 +138,12 @@ proc expandTilde*(path: string): string {. # TODO: handle `~bob` and `~bob/` which means home of bob result = path -# TODO: consider whether quoteShellPosix, quoteShellWindows, quoteShell, quoteShellCommand -# belong in `strutils` instead; they are not specific to paths proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote `s`, so it can be safely passed to Windows API. ## ## Based on Python's `subprocess.list2cmdline`. ## See `this link `_ ## for more details. - let needQuote = {' ', '\t'} in s or s.len == 0 result = "" var backslashBuff = "" @@ -1054,8 +154,8 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" if c == '\\': backslashBuff.add(c) elif c == '\"': - result.add(backslashBuff) - result.add(backslashBuff) + for i in 0.. 0: + result.add(backslashBuff) if needQuote: + result.add(backslashBuff) result.add("\"") + proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. - ## Based on Python's `pipes.quote`. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} if s.len == 0: - return "''" - - let safe = s.allCharsInSet(safeUnixChars) - - if safe: - return s + result = "''" + elif s.allCharsInSet(safeUnixChars): + result = s else: - return "'" & s.replace("'", "'\"'\"'") & "'" + result = "'" & s.replace("'", "'\"'\"'") & "'" when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to shell. ## - ## When on Windows, it calls `quoteShellWindows proc - ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc - ## <#quoteShellPosix,string>`_. + ## When on Windows, it calls `quoteShellWindows proc`_. + ## Otherwise, calls `quoteShellPosix proc`_. when defined(windows): - return quoteShellWindows(s) + result = quoteShellWindows(s) else: - return quoteShellPosix(s) + result = quoteShellPosix(s) proc quoteShellCommand*(args: openArray[string]): string = ## Concatenates and quotes shell arguments `args`. @@ -1113,106 +212,10 @@ when not weirdTarget: importc: "system", header: "".} when not defined(windows): - proc c_rename(oldname, newname: cstring): cint {. - importc: "rename", header: "".} - proc c_strlen(a: cstring): cint {. - importc: "strlen", header: "", noSideEffect.} proc c_free(p: pointer) {. importc: "free", header: "".} -when defined(windows) and not weirdTarget: - when useWinUnicode: - template wrapUnary(varname, winApiProc, arg: untyped) = - var varname = winApiProc(newWideCString(arg)) - - template wrapBinary(varname, winApiProc, arg, arg2: untyped) = - var varname = winApiProc(newWideCString(arg), arg2) - proc findFirstFile(a: string, b: var WIN32_FIND_DATA): Handle = - result = findFirstFileW(newWideCString(a), b) - template findNextFile(a, b: untyped): untyped = findNextFileW(a, b) - template getCommandLine(): untyped = getCommandLineW() - - template getFilename(f: untyped): untyped = - $cast[WideCString](addr(f.cFileName[0])) - else: - template findFirstFile(a, b: untyped): untyped = findFirstFileA(a, b) - template findNextFile(a, b: untyped): untyped = findNextFileA(a, b) - template getCommandLine(): untyped = getCommandLineA() - - template getFilename(f: untyped): untyped = $cstring(addr f.cFileName) - - proc skipFindData(f: WIN32_FIND_DATA): bool {.inline.} = - # Note - takes advantage of null delimiter in the cstring - const dot = ord('.') - result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or - f.cFileName[1].int == dot and f.cFileName[2].int == 0) - -proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noNimJs.} = - ## Returns true if `filename` exists and is a regular file or symlink. - ## - ## Directories, device files, named pipes and sockets return false. - ## - ## See also: - ## * `dirExists proc <#dirExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, filename) - else: - var a = getFileAttributesA(filename) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 - else: - var res: Stat - return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) - -proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], - noNimJs.} = - ## Returns true if the directory `dir` exists. If `dir` is a file, false - ## is returned. Follows symlinks. - ## - ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, dir) - else: - var a = getFileAttributesA(dir) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - else: - var res: Stat - return stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) - -proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], - noWeirdTarget.} = - ## Returns true if the symlink `link` exists. Will return true - ## regardless of whether the link points to a directory or file. - ## - ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `dirExists proc <#dirExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, link) - else: - var a = getFileAttributesA(link) - if a != -1'i32: - # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` - # may also be needed. - result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 - else: - var res: Stat - return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) - - -when not defined(windows): - const maxSymlinkLen = 1024 - const ExeExts* = ## Platform specific file extension for executables. ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``. @@ -1225,7 +228,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## in directories listed in the ``PATH`` environment variable. ## ## Returns `""` if the `exe` cannot be found. `exe` - ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. + ## is added the `ExeExts`_ file extensions if it has none. ## ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired @@ -1252,16 +255,16 @@ proc findExe*(exe: string, followSymlinks: bool = true; for ext in extensions: var x = addFileExt(x, ext) if fileExists(x): - when not defined(windows): + when not (defined(windows) or defined(nintendoswitch)): while followSymlinks: # doubles as if here if x.symlinkExists: var r = newString(maxSymlinkLen) - var len = readlink(x, r, maxSymlinkLen) + var len = readlink(x.cstring, r.cstring, maxSymlinkLen) if len < 0: raiseOSError(osLastError(), exe) if len > maxSymlinkLen: r = newString(len+1) - len = readlink(x, r, len) + len = readlink(x.cstring, r.cstring, len) setLen(r, len) if isAbsolute(r): x = r @@ -1280,9 +283,9 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", ## Returns the `file`'s last modification time. ## ## See also: - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1298,9 +301,9 @@ proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeir ## Returns the `file`'s last read or write access time. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1320,9 +323,9 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdT ## `here `_ for details. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1339,9 +342,9 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = ## modification time is later than `b`'s. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ when defined(posix): # If we don't have access to nanosecond resolution, use '>=' when not StatHasNanoseconds: @@ -1351,356 +354,6 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = else: result = getLastModificationTime(a) > getLastModificationTime(b) -when not defined(nimscript): - proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = - ## Returns the `current working directory`:idx: i.e. where the built - ## binary is run. - ## - ## So the path returned by this proc is determined at run time. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - ## * `currentSourcePath template `_ - ## * `getProjectPath proc `_ - when defined(nodejs): - var ret: cstring - {.emit: "`ret` = process.cwd();".} - return $ret - elif defined(js): - doAssert false, "use -d:nodejs to have `getCurrentDir` defined" - elif defined(windows): - var bufsize = MAX_PATH.int32 - when useWinUnicode: - var res = newWideCString("", bufsize) - while true: - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - result = newString(bufsize) - while true: - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break - else: - var bufsize = 1024 # should be enough - result = newString(bufsize) - while true: - if getcwd(result, bufsize) != nil: - setLen(result, c_strlen(result)) - break - else: - let err = osLastError() - if err.int32 == ERANGE: - bufsize = bufsize shl 1 - doAssert(bufsize >= 0) - result = newString(bufsize) - else: - raiseOSError(osLastError()) - -proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = - ## Sets the `current working directory`:idx:; `OSError` - ## is raised if `newDir` cannot been set. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - when defined(windows): - when useWinUnicode: - if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: - raiseOSError(osLastError(), newDir) - else: - if setCurrentDirectoryA(newDir) == 0'i32: raiseOSError(osLastError(), newDir) - else: - if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) - - -proc absolutePath*(path: string, root = getCurrentDir()): string = - ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; - ## default: current directory). - ## If `path` is absolute, return it, ignoring `root`. - ## - ## See also: - ## * `normalizedPath proc <#normalizedPath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ - runnableExamples: - assert absolutePath("a") == getCurrentDir() / "a" - - if isAbsolute(path): path - else: - if not root.isAbsolute: - raise newException(ValueError, "The specified root is not absolute: " & root) - joinPath(root, path) - -proc absolutePathInternal(path: string): string = - absolutePath(path, getCurrentDir()) - -proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = - ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. - runnableExamples: - import std/sugar - when defined(posix): - doAssert "foo".dup(normalizeExe) == "./foo" - doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" - doAssert "".dup(normalizeExe) == "" - when defined(posix): - if file.len > 0 and DirSep notin file and file != "." and file != "..": - file = "./" & file - -proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = - ## Normalize a path. - ## - ## Consecutive directory separators are collapsed, including an initial double slash. - ## - ## On relative paths, double dot (`..`) sequences are collapsed if possible. - ## On absolute paths they are always collapsed. - ## - ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. - ## Triple dot is not handled. - ## - ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizedPath proc <#normalizedPath,string>`_ for outplace version - ## * `normalizeExe proc <#normalizeExe,string>`_ - runnableExamples: - when defined(posix): - var a = "a///b//..//c///d" - a.normalizePath() - assert a == "a/c/d" - - path = pathnorm.normalizePath(path) - when false: - let isAbs = isAbsolute(path) - var stack: seq[string] = @[] - for p in split(path, {DirSep}): - case p - of "", ".": - continue - of "..": - if stack.len == 0: - if isAbs: - discard # collapse all double dots on absoluta paths - else: - stack.add(p) - elif stack[^1] == "..": - stack.add(p) - else: - discard stack.pop() - else: - stack.add(p) - - if isAbs: - path = DirSep & join(stack, $DirSep) - elif stack.len > 0: - path = join(stack, $DirSep) - else: - path = "." - -proc normalizePathAux(path: var string) = normalizePath(path) - -proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = - ## Returns a normalized path for the current OS. - ## - ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ for the in-place version - runnableExamples: - when defined(posix): - assert normalizedPath("a///b//..//c///d") == "a/c/d" - result = pathnorm.normalizePath(path) - -when defined(windows) and not weirdTarget: - proc openHandle(path: string, followSymlink=true, writeAccess=false): Handle = - var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL - if not followSymlink: - flags = flags or FILE_FLAG_OPEN_REPARSE_POINT - let access = if writeAccess: GENERIC_WRITE else: 0'i32 - - when useWinUnicode: - result = createFileW( - newWideCString(path), access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - else: - result = createFileA( - path, access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - -proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noWeirdTarget.} = - ## Returns true if both pathname arguments refer to the same physical - ## file or directory. - ## - ## Raises `OSError` if any of the files does not - ## exist or information about it can not be obtained. - ## - ## This proc will return true if given two alternative hard-linked or - ## sym-linked paths to the same file or directory. - ## - ## See also: - ## * `sameFileContent proc <#sameFileContent,string,string>`_ - when defined(windows): - var success = true - var f1 = openHandle(path1) - var f2 = openHandle(path2) - - var lastErr: OSErrorCode - if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: - var fi1, fi2: BY_HANDLE_FILE_INFORMATION - - if getFileInformationByHandle(f1, addr(fi1)) != 0 and - getFileInformationByHandle(f2, addr(fi2)) != 0: - result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and - fi1.nFileIndexHigh == fi2.nFileIndexHigh and - fi1.nFileIndexLow == fi2.nFileIndexLow - else: - lastErr = osLastError() - success = false - else: - lastErr = osLastError() - success = false - - discard closeHandle(f1) - discard closeHandle(f2) - - if not success: raiseOSError(lastErr, $(path1, path2)) - else: - var a, b: Stat - if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: - raiseOSError(osLastError(), $(path1, path2)) - else: - result = a.st_dev == b.st_dev and a.st_ino == b.st_ino - -type - FilePermission* = enum ## File access permission, modelled after UNIX. - ## - ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FileInfo object <#FileInfo>`_ - fpUserExec, ## execute access for the file owner - fpUserWrite, ## write access for the file owner - fpUserRead, ## read access for the file owner - fpGroupExec, ## execute access for the group - fpGroupWrite, ## write access for the group - fpGroupRead, ## read access for the group - fpOthersExec, ## execute access for others - fpOthersWrite, ## write access for others - fpOthersRead ## read access for others - -proc getFilePermissions*(filename: string): set[FilePermission] {. - rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = - ## Retrieves file permissions for `filename`. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is checked, every other - ## permission is available in any case. - ## - ## See also: - ## * `setFilePermissions proc <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FilePermission enum <#FilePermission>`_ - when defined(posix): - var a: Stat - if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) - result = {} - if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) - if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) - if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) - - if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) - if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) - if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) - - if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) - if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) - if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError(), filename) - if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: - result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, - fpOthersExec, fpOthersRead} - else: - result = {fpUserExec..fpOthersRead} - -proc setFilePermissions*(filename: string, permissions: set[FilePermission], - followSymlinks = true) - {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], - noWeirdTarget.} = - ## Sets the file permissions for `filename`. - ## - ## If `followSymlinks` set to true (default) and ``filename`` points to a - ## symlink, permissions are set to the file symlink points to. - ## `followSymlinks` set to false is a noop on Windows and some POSIX - ## systems (including Linux) on which `lchmod` is either unavailable or always - ## fails, given that symlinks permissions there are not observed. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is changed, depending on - ## ``fpUserWrite`` permission. - ## - ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `FilePermission enum <#FilePermission>`_ - when defined(posix): - var p = 0.Mode - if fpUserRead in permissions: p = p or S_IRUSR.Mode - if fpUserWrite in permissions: p = p or S_IWUSR.Mode - if fpUserExec in permissions: p = p or S_IXUSR.Mode - - if fpGroupRead in permissions: p = p or S_IRGRP.Mode - if fpGroupWrite in permissions: p = p or S_IWGRP.Mode - if fpGroupExec in permissions: p = p or S_IXGRP.Mode - - if fpOthersRead in permissions: p = p or S_IROTH.Mode - if fpOthersWrite in permissions: p = p or S_IWOTH.Mode - if fpOthersExec in permissions: p = p or S_IXOTH.Mode - - if not followSymlinks and filename.symlinkExists: - when declared(lchmod): - if lchmod(filename, cast[Mode](p)) != 0: - raiseOSError(osLastError(), $(filename, permissions)) - else: - if chmod(filename, cast[Mode](p)) != 0: - raiseOSError(osLastError(), $(filename, permissions)) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError(), filename) - if fpUserWrite in permissions: - res = res and not FILE_ATTRIBUTE_READONLY - else: - res = res or FILE_ATTRIBUTE_READONLY - when useWinUnicode: - wrapBinary(res2, setFileAttributesW, filename, res) - else: - var res2 = setFileAttributesA(filename, res) - if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) proc isAdmin*: bool {.noWeirdTarget.} = ## Returns whether the caller's process is a member of the Administrators local @@ -1720,314 +373,19 @@ proc isAdmin*: bool {.noWeirdTarget.} = addr administratorsGroup)): raiseOSError(osLastError(), "could not get SID for Administrators group") - defer: + try: + var b: WINBOOL + if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): + raiseOSError(osLastError(), "could not check access token membership") + + result = isSuccess(b) + finally: if freeSid(administratorsGroup) != nil: raiseOSError(osLastError(), "failed to free SID for Administrators group") - var b: WINBOOL - if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): - raiseOSError(osLastError(), "could not check access token membership") - - return isSuccess(b) else: - return geteuid() == 0 + result = geteuid() == 0 -proc createSymlink*(src, dest: string) {.noWeirdTarget.} = - ## Create a symbolic link at `dest` which points to the item specified - ## by `src`. On most operating systems, will fail if a link already exists. - ## - ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation - ## of symlinks to root users (administrators) or users with developper mode enabled. - ## - ## See also: - ## * `createHardlink proc <#createHardlink,string,string>`_ - ## * `expandSymlink proc <#expandSymlink,string>`_ - - when defined(windows): - const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 - # allows anyone with developer mode on to create a link - let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if symlink(src, dest) != 0: - raiseOSError(osLastError(), $(src, dest)) - -proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = - ## Returns a string representing the path to which the symbolic link points. - ## - ## On Windows this is a noop, `symlinkPath` is simply returned. - ## - ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ - when defined(windows): - result = symlinkPath - else: - result = newString(maxSymlinkLen) - var len = readlink(symlinkPath, result, maxSymlinkLen) - if len < 0: - raiseOSError(osLastError(), symlinkPath) - if len > maxSymlinkLen: - result = newString(len+1) - len = readlink(symlinkPath, result, len) - setLen(result, len) - -const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) - # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` - -when hasCCopyfile: - # `copyfile` API available since osx 10.5. - {.push nodecl, header: "".} - type - copyfile_state_t {.nodecl.} = pointer - copyfile_flags_t = cint - proc copyfile_state_alloc(): copyfile_state_t - proc copyfile_state_free(state: copyfile_state_t): cint - proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} - # replace with `let` pending bootstrap >= 1.4.0 - var - COPYFILE_DATA {.nodecl.}: copyfile_flags_t - COPYFILE_XATTR {.nodecl.}: copyfile_flags_t - {.pop.} - -type CopyFlag* = enum ## Copy options. - cfSymlinkAsIs, ## Copy symlinks as symlinks - cfSymlinkFollow, ## Copy the files symlinks point to - cfSymlinkIgnore ## Ignore symlinks - -const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} - -proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, - extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], - noWeirdTarget.} = - ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will - ## copy the source file's attributes into dest. - ## - ## On other platforms you need - ## to use `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ - ## procs - ## to copy them by hand (or use the convenience `copyFileWithPermissions - ## proc <#copyFileWithPermissions,string,string>`_), - ## otherwise `dest` will inherit the default permissions of a newly - ## created file for the user. - ## - ## If `dest` already exists, the file attributes - ## will be preserved and the content overwritten. - ## - ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless - ## `-d:nimLegacyCopyFile` is used. - ## - ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - - doAssert card(copyFlagSymlink * options) == 1, "There should be exactly " & - "one cfSymlink* in options" - let isSymlink = source.symlinkExists - if isSymlink and (cfSymlinkIgnore in options or defined(windows)): - return - when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if copyFileW(s, d, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if copyFileA(source, dest, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if isSymlink and cfSymlinkAsIs in options: - createSymlink(expandSymlink(source), dest) - else: - when hasCCopyfile: - let state = copyfile_state_alloc() - # xxx `COPYFILE_STAT` could be used for one-shot - # `copyFileWithPermissions`. - let status = c_copyfile(source.cstring, dest.cstring, state, - COPYFILE_DATA) - if status != 0: - let err = osLastError() - discard copyfile_state_free(state) - raiseOSError(err, $(source, dest)) - let status2 = copyfile_state_free(state) - if status2 != 0: raiseOSError(osLastError(), $(source, dest)) - else: - # generic version of copyFile which works for any platform: - const bufSize = 8000 # better for memory manager - var d, s: File - if not open(s, source):raiseOSError(osLastError(), source) - if not open(d, dest, fmWrite): - close(s) - raiseOSError(osLastError(), dest) - var buf = alloc(bufSize) - while true: - var bytesread = readBuffer(s, buf, bufSize) - if bytesread > 0: - var byteswritten = writeBuffer(d, buf, bytesread) - if bytesread != byteswritten: - dealloc(buf) - close(s) - close(d) - raiseOSError(osLastError(), dest) - if bytesread != bufSize: break - dealloc(buf) - close(s) - flushFile(d) - close(d) - -proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}) - {.noWeirdTarget, since: (1,3,7).} = - ## Copies a file `source` into directory `dir`, which must exist. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyDir,string,string>`_ - if dir.len == 0: # treating "" as "." is error prone - raise newException(ValueError, "dest is empty") - copyFile(source, dir / source.lastPathPart, options) - -when not declared(ENOENT) and not defined(windows): - when NoFakeVars: - when not defined(haiku): - const ENOENT = cint(2) # 2 on most systems including Solaris - else: - const ENOENT = cint(-2147459069) - else: - var ENOENT {.importc, header: "".}: cint - -when defined(windows) and not weirdTarget: - when useWinUnicode: - template deleteFile(file: untyped): untyped = deleteFileW(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesW(file, attrs) - else: - template deleteFile(file: untyped): untyped = deleteFileA(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesA(file, attrs) - -proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = - ## Removes the `file`. - ## - ## If this fails, returns `false`. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - result = true - when defined(windows): - when useWinUnicode: - let f = newWideCString(file) - else: - let f = file - if deleteFile(f) == 0: - result = false - let err = getLastError() - if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: - result = true - elif err == ERROR_ACCESS_DENIED and - setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and - deleteFile(f) != 0: - result = true - else: - if unlink(file) != 0'i32 and errno != ENOENT: - result = false - -proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = - ## Removes the `file`. - ## - ## If this fails, `OSError` is raised. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - if not tryRemoveFile(file): - raiseOSError(osLastError(), file) - -proc tryMoveFSObject(source, dest: string): bool {.noWeirdTarget.} = - ## Moves a file or directory from `source` to `dest`. - ## - ## Returns false in case of `EXDEV` error. - ## In case of other errors `OSError` is raised. - ## Returns true in case of success. - when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) - else: - if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) - else: - if c_rename(source, dest) != 0'i32: - let err = osLastError() - if err == EXDEV.OSErrorCode: - return false - else: - # see whether `strerror(errno)` is redundant with what raiseOSError already shows - raiseOSError(err, $(source, dest, strerror(errno))) - return true - -proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = - ## Moves a file from `source` to `dest`. - ## - ## Symlinks are not followed: if `source` is a symlink, it is itself moved, - ## not its target. - ## - ## If this fails, `OSError` is raised. - ## If `dest` already exists, it will be overwritten. - ## - ## Can be used to `rename files`:idx:. - ## - ## See also: - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyFile(source, dest, {cfSymlinkAsIs}) - try: - removeFile(source) - except: - discard tryRemoveFile(dest) - raise proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. @@ -2053,122 +411,11 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## `_. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## discard execShellCmd("ls -la") + ## ``` result = exitStatusLikeShell(c_system(command)) -# Templates for filtering directories and files -when defined(windows) and not weirdTarget: - template isDir(f: WIN32_FIND_DATA): bool = - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - template isFile(f: WIN32_FIND_DATA): bool = - not isDir(f) -else: - template isDir(f: string): bool {.dirty.} = - dirExists(f) - template isFile(f: string): bool {.dirty.} = - fileExists(f) - -template defaultWalkFilter(item): bool = - ## Walk filter used to return true on both - ## files and directories - true - -template walkCommon(pattern: string, filter) = - ## Common code for getting the files and directories with the - ## specified `pattern` - when defined(windows): - var - f: WIN32_FIND_DATA - res: int - res = findFirstFile(pattern, f) - if res != -1: - defer: findClose(res) - let dotPos = searchExtPos(pattern) - while true: - if not skipFindData(f) and filter(f): - # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check - # that the file extensions have the same length ... - let ff = getFilename(f) - let idx = ff.len - pattern.len + dotPos - if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or - (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): - yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: # here we use glob - var - f: Glob - res: int - f.gl_offs = 0 - f.gl_pathc = 0 - f.gl_pathv = nil - res = glob(pattern, 0, nil, addr(f)) - defer: globfree(addr(f)) - if res == 0: - for i in 0.. f.gl_pathc - 1: - assert(f.gl_pathv[i] != nil) - let path = $f.gl_pathv[i] - if filter(path): - yield path - -iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the files and directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - runnableExamples: - import std/sequtils - let paths = toSeq(walkPattern("lib/pure/*")) # works on windows too - assert "lib/pure/concurrency".unixToNativePath in paths - assert "lib/pure/os.nim".unixToNativePath in paths - walkCommon(pattern, defaultWalkFilter) - -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the files that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - runnableExamples: - import std/sequtils - assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on windows too - walkCommon(pattern, isFile) - -iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - runnableExamples: - import std/sequtils - let paths = toSeq(walkDirs("lib/pure/*")) # works on windows too - assert "lib/pure/concurrency".unixToNativePath in paths - walkCommon(pattern, isDir) - proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = ## Returns the full (`absolute`:idx:) path of an existing file `filename`. @@ -2176,32 +423,18 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var unused: WideCString = nil - var res = newWideCString("", bufsize) - while true: - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - var unused: cstring = nil - result = newString(bufsize) - while true: - var L = getFullPathNameA(filename, bufsize, result, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var unused: WideCString = nil + var res = newWideCString(bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError(), filename) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break # getFullPathName doesn't do case corrections, so we have to use this convoluted # way of retrieving the true filename for x in walkFiles(result): @@ -2219,379 +452,14 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", result = $r c_free(cast[pointer](r)) -type - PathComponent* = enum ## Enumeration specifying a path component. - ## - ## See also: - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - ## * `FileInfo object <#FileInfo>`_ - pcFile, ## path refers to a file - pcLinkToFile, ## path refers to a symbolic link to a file - pcDir, ## path refers to a directory - pcLinkToDir ## path refers to a symbolic link to a directory - proc getCurrentCompilerExe*(): string {.compileTime.} = discard - ## This is `getAppFilename() <#getAppFilename>`_ at compile time. + ## Returns the path of the currently running Nim compiler or nimble executable. ## ## Can be used to retrieve the currently executing ## Nim compiler from a Nim or nimscript program, or the nimble binary ## inside a nimble program (likewise with other binaries built from ## compiler API). -when defined(posix) and not weirdTarget: - proc getSymlinkFileKind(path: string): PathComponent = - # Helper function. - var s: Stat - assert(path != "") - if stat(path, s) == 0'i32 and S_ISDIR(s.st_mode): - result = pcLinkToDir - else: - result = pcLinkToFile - -proc staticWalkDir(dir: string; relative: bool): seq[ - tuple[kind: PathComponent, path: string]] = - discard - -iterator walkDir*(dir: string; relative = false, checkDir = false): - tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = - ## Walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item are returned. - ## - ## Walking is not recursive. If ``relative`` is true (default: false) - ## the resulting path is shortened to be relative to ``dir``. - ## Example: This directory structure:: - ## dirA / dirB / fileB1.txt - ## / dirC - ## / fileA1.txt - ## / fileA2.txt - ## - ## and this code: - ## - ## .. code-block:: Nim - ## for kind, path in walkDir("dirA"): - ## echo(path) - ## - ## produce this output (but not necessarily in this order!):: - ## dirA/dirB - ## dirA/dirC - ## dirA/fileA1.txt - ## dirA/fileA2.txt - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - - when nimvm: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - else: - when weirdTarget: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - elif defined(windows): - var f: WIN32_FIND_DATA - var h = findFirstFile(dir / "*", f) - if h == -1: - if checkDir: - raiseOSError(osLastError(), dir) - else: - defer: findClose(h) - while true: - var k = pcFile - if not skipFindData(f): - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - yield (k, xx) - if findNextFile(h, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: - var d = opendir(dir) - if d == nil: - if checkDir: - raiseOSError(osLastError(), dir) - else: - defer: discard closedir(d) - while true: - var x = readdir(d) - if x == nil: break - var y = $cstring(addr x.d_name) - if y != "." and y != "..": - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile - - template kSetGeneric() = # pure Posix component `k` resolution - if lstat(path, s) < 0'i32: continue # don't yield - elif S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - case x.d_type - of DT_DIR: k = pcDir - of DT_LNK: - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - of DT_UNKNOWN: - kSetGeneric() - else: # e.g. DT_REG etc - discard # leave it as pcFile - else: # assuming that field `d_type` is not present - kSetGeneric() - - yield (k, y) - -iterator walkDirRec*(dir: string, - yieldFilter = {pcFile}, followFilter = {pcDir}, - relative = false, checkDir = false): string {.tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file - ## or directory in `dir`. - ## - ## If ``relative`` is true (default: false) the resulting path is - ## shortened to be relative to ``dir``, otherwise the full path is returned. - ## - ## .. warning:: Modifying the directory structure while the iterator - ## is traversing may result in undefined behavior! - ## - ## Walking is recursive. `followFilter` controls the behaviour of the iterator: - ## - ## --------------------- --------------------------------------------- - ## yieldFilter meaning - ## --------------------- --------------------------------------------- - ## ``pcFile`` yield real files (default) - ## ``pcLinkToFile`` yield symbolic links to files - ## ``pcDir`` yield real directories - ## ``pcLinkToDir`` yield symbolic links to directories - ## --------------------- --------------------------------------------- - ## - ## --------------------- --------------------------------------------- - ## followFilter meaning - ## --------------------- --------------------------------------------- - ## ``pcDir`` follow real directories (default) - ## ``pcLinkToDir`` follow symbolic links to directories - ## --------------------- --------------------------------------------- - ## - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - - var stack = @[""] - var checkDir = checkDir - while stack.len > 0: - let d = stack.pop() - for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): - let rel = d / p - if k in {pcDir, pcLinkToDir} and k in followFilter: - stack.add rel - if k in yieldFilter: - yield if relative: rel else: dir / rel - checkDir = false - # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong - # permissions), it'll abort iteration and there would be no way to - # continue iteration. - # Future work can provide a way to customize this and do error reporting. - -proc rawRemoveDir(dir: string) {.noWeirdTarget.} = - when defined(windows): - when useWinUnicode: - wrapUnary(res, removeDirectoryW, dir) - else: - var res = removeDirectoryA(dir) - let lastError = osLastError() - if res == 0'i32 and lastError.int32 != 3'i32 and - lastError.int32 != 18'i32 and lastError.int32 != 2'i32: - raiseOSError(lastError, dir) - else: - if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) - -proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ - WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = - ## Removes the directory `dir` including all subdirectories and files - ## in `dir` (recursively). - ## - ## If this fails, `OSError` is raised. This does not fail if the directory never - ## existed in the first place, unless `checkDir` = true - ## - ## See also: - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - for kind, path in walkDir(dir, checkDir = checkDir): - case kind - of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) - of pcDir: removeDir(path, true) - # for subdirectories there is no benefit in `checkDir = false` - # (unless perhaps for edge case of concurrent processes also deleting - # the same files) - rawRemoveDir(dir) - -proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = - # Try to create one directory (not the whole path). - # returns `true` for success, `false` if the path has previously existed - # - # This is a thin wrapper over mkDir (or alternatives on other systems), - # so in case of a pre-existing path we don't check that it is a directory. - when defined(solaris): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno in {EEXIST, ENOSYS}: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(haiku): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST or errno == EROFS: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(posix): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST: - result = false - else: - #echo res - raiseOSError(osLastError(), dir) - else: - when useWinUnicode: - wrapUnary(res, createDirectoryW, dir) - else: - let res = createDirectoryA(dir) - - if res != 0'i32: - result = true - elif getLastError() == 183'i32: - result = false - else: - raiseOSError(osLastError(), dir) - -proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = - ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. - ## - ## Does not create parent directories (raises `OSError` if parent directories do not exist). - ## Returns `true` if the directory already exists, and `false` otherwise. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - result = not rawCreateDir(dir) - if result: - # path already exists - need to check that it is indeed a directory - if not dirExists(dir): - raise newException(IOError, "Failed to create '" & dir & "'") - -proc createDir*(dir: string) {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = - ## Creates the `directory`:idx: `dir`. - ## - ## The directory may contain several subdirectories that do not exist yet. - ## The full path is created. If this fails, `OSError` is raised. - ## - ## It does **not** fail if the directory already exists because for - ## most usages this does not indicate an error. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - var omitNext = false - when doslikeFileSystem: - omitNext = isAbsolute(dir) - for i in 1.. dir.len-1: - if dir[i] in {DirSep, AltSep}: - if omitNext: - omitNext = false - else: - discard existsOrCreateDir(substr(dir, 0, i-1)) - - # The loop does not create the dir itself if it doesn't end in separator - if dir.len > 0 and not omitNext and - dir[^1] notin {DirSep, AltSep}: - discard existsOrCreateDir(dir) - -proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = - ## Copies a directory from `source` to `dest`. - ## - ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks - ## are skipped. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will copy the attributes from - ## `source` into `dest`. - ## - ## On other platforms created files and directories will inherit the - ## default permissions of a newly created file/directory for the user. - ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## to preserve attributes recursively on these platforms. - ## - ## See also: - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - createDir(dest) - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - if kind == pcDir: - copyDir(path, dest / noSource) - else: - copyFile(path, dest / noSource, {cfSymlinkAsIs}) - -proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = - ## Moves a directory from `source` to `dest`. - ## - ## Symlinks are not followed: if `source` contains symlinks, they themself are - ## moved, not their target. - ## - ## If this fails, `OSError` is raised. - ## - ## See also: - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyDir(source, dest) - removeDir(source) - proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## Create a hard link at `dest` which points to the item specified ## by `src`. @@ -2600,403 +468,39 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## root users (administrators). ## ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ + ## * `createSymlink proc`_ when defined(windows): - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createHardLinkW(wDst, wSrc, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createHardLinkA(dest, src, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + raiseOSError(osLastError(), $(src, dest)) else: if link(src, dest) != 0: raiseOSError(osLastError(), $(src, dest)) -proc copyFileWithPermissions*(source, dest: string, - ignorePermissionErrors = true, - options = {cfSymlinkFollow}) {.noWeirdTarget.} = - ## Copies a file from `source` to `dest` preserving file permissions. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_, - ## `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions<#setFilePermissions,string,set[FilePermission]>`_ - ## procs on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyFile proc - ## <#copyFile,string,string>`_ since that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file itself has - ## been copied, which won't happen atomically and could lead to a race - ## condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - copyFile(source, dest, options) - when not defined(windows): - try: - setFilePermissions(dest, getFilePermissions(source), followSymlinks = - (cfSymlinkFollow in options)) - except: - if not ignorePermissionErrors: - raise - -proc copyDirWithPermissions*(source, dest: string, - ignorePermissionErrors = true) - {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], - benign, noWeirdTarget.} = - ## Copies a directory from `source` to `dest` preserving file permissions. - ## - ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks - ## are skipped. - ## - ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir - ## <#copyDir,string,string>`_ and `copyFileWithPermissions - ## <#copyFileWithPermissions,string,string>`_ procs - ## on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyDir proc - ## <#copyDir,string,string>`_ since that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file or directory - ## itself has been copied, which won't happen atomically and could lead to a - ## race condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - createDir(dest) - when not defined(windows): - try: - setFilePermissions(dest, getFilePermissions(source), followSymlinks = - false) - except: - if not ignorePermissionErrors: - raise - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - if kind == pcDir: - copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) - else: - copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) - proc inclFilePermissions*(filename: string, permissions: set[FilePermission]) {. rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)+permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)+permissions) proc exclFilePermissions*(filename: string, permissions: set[FilePermission]) {. rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)-permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)-permissions) -proc parseCmdLine*(c: string): seq[string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a `command line`:idx: into several components. - ## - ## **Note**: This proc is only occasionally useful, better use the - ## `parseopt module `_. - ## - ## On Windows, it uses the `following parsing rules - ## `_: - ## - ## * Arguments are delimited by white space, which is either a space or a tab. - ## * The caret character (^) is not recognized as an escape character or - ## delimiter. The character is handled completely by the command-line parser - ## in the operating system before being passed to the argv array in the - ## program. - ## * A string surrounded by double quotation marks ("string") is interpreted - ## as a single argument, regardless of white space contained within. A - ## quoted string can be embedded in an argument. - ## * A double quotation mark preceded by a backslash (\") is interpreted as a - ## literal double quotation mark character ("). - ## * Backslashes are interpreted literally, unless they immediately precede - ## a double quotation mark. - ## * If an even number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is interpreted as a string delimiter. - ## * If an odd number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is "escaped" by the remaining backslash, - ## causing a literal double quotation mark (") to be placed in argv. - ## - ## On Posix systems, it uses the following parsing rules: - ## Components are separated by whitespace unless the whitespace - ## occurs within ``"`` or ``'`` quotes. - ## - ## See also: - ## * `parseopt module `_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - - result = @[] - var i = 0 - var a = "" - while true: - setLen(a, 0) - # eat all delimiting whitespace - while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) - if i >= c.len: break - when defined(windows): - # parse a single argument according to the above rules: - var inQuote = false - while i < c.len: - case c[i] - of '\\': - var j = i - while j < c.len and c[j] == '\\': inc(j) - if j < c.len and c[j] == '"': - for k in 1..(j-i) div 2: a.add('\\') - if (j-i) mod 2 == 0: - i = j - else: - a.add('"') - i = j+1 - else: - a.add(c[i]) - inc(i) - of '"': - inc(i) - if not inQuote: inQuote = true - elif i < c.len and c[i] == '"': - a.add(c[i]) - inc(i) - else: - inQuote = false - break - of ' ', '\t': - if not inQuote: break - a.add(c[i]) - inc(i) - else: - a.add(c[i]) - inc(i) - else: - case c[i] - of '\'', '\"': - var delim = c[i] - inc(i) # skip ' or " - while i < c.len and c[i] != delim: - add a, c[i] - inc(i) - if i < c.len: inc(i) - else: - while i < c.len and c[i] > ' ': - add(a, c[i]) - inc(i) - add(result, a) - -when defined(nimdoc): - # Common forward declaration docstring block for parameter retrieval procs. - proc paramCount*(): int {.tags: [ReadIOEffect].} = - ## Returns the number of `command line arguments`:idx: given to the - ## application. - ## - ## Unlike `argc`:idx: in C, if your binary was called without parameters this - ## will return zero. - ## You can query each individual parameter with `paramStr proc <#paramStr,int>`_ - ## or retrieve all of them in one go with `commandLineParams proc - ## <#commandLineParams>`_. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() `_. - ## - ## See also: - ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramCount): - ## # Use paramCount() here - ## else: - ## # Do something else! - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - ## Returns the `i`-th `command line argument`:idx: given to the application. - ## - ## `i` should be in the range `1..paramCount()`, the `IndexDefect` - ## exception will be raised for invalid values. Instead of iterating - ## over `paramCount() <#paramCount>`_ with this proc you can - ## call the convenience `commandLineParams() <#commandLineParams>`_. - ## - ## Similarly to `argv`:idx: in C, - ## it is possible to call `paramStr(0)` but this will return OS specific - ## contents (usually the name of the invoked executable). You should avoid - ## this and call `getAppFilename() <#getAppFilename>`_ instead. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() `_. - ## - ## See also: - ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## * `getAppFilename proc <#getAppFilename>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramStr): - ## # Use paramStr() here - ## else: - ## # Do something else! - -elif defined(nimscript): discard -elif defined(nodejs): - type Argv = object of JSRoot - let argv {.importjs: "process.argv".} : Argv - proc len(argv: Argv): int {.importjs: "#.length".} - proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} - - proc paramCount*(): int {.tags: [ReadDirEffect].} = - result = argv.len - 2 - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - let i = i + 1 - if i < argv.len and i >= 0: - result = $argv[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) -elif defined(nintendoswitch): - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on Nintendo Switch") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on Nintendo Switch") - -elif defined(windows): - # Since we support GUI applications with Nim, we sometimes generate - # a WinMain entry proc. But a WinMain proc has no access to the parsed - # command line arguments. The way to get them differs. Thus we parse them - # ourselves. This has the additional benefit that the program's behaviour - # is always the same -- independent of the used C compiler. - var - ownArgv {.threadvar.}: seq[string] - ownParsedArgv {.threadvar.}: bool - - proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - result = ownArgv.len-1 - - proc paramStr*(i: int): string {.rtl, extern: "nos$1", - tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - if i < ownArgv.len and i >= 0: - result = ownArgv[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) - -elif defined(genode): - proc paramStr*(i: int): string = - raise newException(OSError, "paramStr is not implemented on Genode") - - proc paramCount*(): int = - raise newException(OSError, "paramCount is not implemented on Genode") -elif weirdTarget: - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on current platform") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on current platform") -elif not defined(createNimRtl) and - not(defined(posix) and appType == "lib"): - # On Posix, there is no portable way to get the command line from a DLL. - var - cmdCount {.importc: "cmdCount".}: cint - cmdLine {.importc: "cmdLine".}: cstringArray - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if i < cmdCount and i >= 0: - result = $cmdLine[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - result = cmdCount-1 - -when declared(paramCount) or defined(nimdoc): - proc commandLineParams*(): seq[string] = - ## Convenience proc which returns the command line parameters. - ## - ## This returns **only** the parameters. If you want to get the application - ## executable filename, call `getAppFilename() <#getAppFilename>`_. - ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() - ## `_. - ## - ## See also: - ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `getAppFilename proc <#getAppFilename>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(commandLineParams): - ## # Use commandLineParams() here - ## else: - ## # Do something else! - result = @[] - for i in 1..paramCount(): - result.add(paramStr(i)) -else: - proc commandLineParams*(): seq[string] {.error: - "commandLineParams() unsupported by dynamic libraries".} = - discard - when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netbsd)): proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint {.importc: "sysctl",header: """#include - #include """} + #include """.} const CTL_KERN = 1 KERN_PROC = 14 @@ -3036,10 +540,10 @@ when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netb when not weirdTarget and (defined(linux) or defined(solaris) or defined(bsd) or defined(aix)): proc getApplAux(procPath: string): string = result = newString(maxSymlinkLen) - var len = readlink(procPath, result, maxSymlinkLen) + var len = readlink(procPath, result.cstring, maxSymlinkLen) if len > maxSymlinkLen: result = newString(len+1) - len = readlink(procPath, result, len) + len = readlink(procPath, result.cstring, len) setLen(result, len) when not weirdTarget and defined(openbsd): @@ -3108,7 +612,7 @@ when defined(haiku): B_FIND_PATH_IMAGE_PATH = 1000 proc find_path(codePointer: pointer, baseDirectory: cint, subPath: cstring, - pathBuffer: cstring, bufferSize: csize): int32 + pathBuffer: cstring, bufferSize: csize_t): int32 {.importc, header: "".} proc getApplHaiku(): string = @@ -3120,13 +624,15 @@ when defined(haiku): else: result = "" -proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = +proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget, raises: [].} = ## Returns the filename of the application's executable. ## This proc will resolve symlinks. ## + ## Returns empty string when name is unavailable + ## ## See also: - ## * `getAppDir proc <#getAppDir>`_ - ## * `getCurrentCompilerExe proc <#getCurrentCompilerExe>`_ + ## * `getAppDir proc`_ + ## * `getCurrentCompilerExe proc`_ # Linux: /proc//exe # Solaris: @@ -3134,63 +640,54 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW # /proc//path/a.out (complete pathname) when defined(windows): var bufsize = int32(MAX_PATH) - when useWinUnicode: - var buf = newWideCString("", bufsize) - while true: - var L = getModuleFileNameW(0, buf, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - buf = newWideCString("", L) - bufsize = L - else: - result = buf$L - break - else: - result = newString(bufsize) - while true: - var L = getModuleFileNameA(0, result, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var buf = newWideCString(bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString(L) + bufsize = L + else: + result = buf$L + break elif defined(macosx): var size = cuint32(0) getExecPath1(nil, size) result = newString(int(size)) - if getExecPath2(result, size): + if getExecPath2(result.cstring, size): result = "" # error! if result.len > 0: - result = result.expandFilename + try: + result = result.expandFilename + except OSError: + result = "" else: when defined(linux) or defined(aix): result = getApplAux("/proc/self/exe") elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") - elif defined(genode) or defined(nintendoswitch): - raiseOSError(OSErrorCode(-1), "POSIX command line not supported") + elif defined(genode): + result = "" # Not supported elif defined(freebsd) or defined(dragonfly) or defined(netbsd): result = getApplFreebsd() elif defined(haiku): result = getApplHaiku() elif defined(openbsd): - result = getApplOpenBsd() + result = try: getApplOpenBsd() except OSError: "" + elif defined(nintendoswitch): + result = "" # little heuristic that may work on other POSIX-like systems: if result.len == 0: - result = getApplHeuristic() + result = try: getApplHeuristic() except OSError: "" proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = ## Returns the directory of the application's executable. ## ## See also: - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `getAppFilename proc`_ result = splitFile(getAppFilename()).dir proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} = @@ -3214,11 +711,10 @@ proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", result = rdFileSize(a) findClose(resA) else: - var f: File - if open(f, file): - result = getFileSize(f) - close(f) - else: raiseOSError(osLastError(), file) + var rawInfo: Stat + if stat(file, rawInfo) < 0'i32: + raiseOSError(osLastError(), file) + rawInfo.st_size when defined(windows) or weirdTarget: type @@ -3234,9 +730,9 @@ type ## Contains information associated with a file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ id*: tuple[device: DeviceId, file: FileId] ## Device and file id. kind*: PathComponent ## Kind of file object - directory, symlink, etc. size*: BiggestInt ## Size of file. @@ -3247,21 +743,30 @@ type creationTime*: times.Time ## Time file was created. Not supported on all systems! blockSize*: int ## Preferred I/O block size for this object. ## In some filesystems, this may vary from file to file. + isSpecial*: bool ## Is file special? (on Unix some "files" + ## can be special=non-regular like FIFOs, + ## devices); for directories `isSpecial` + ## is always `false`, for symlinks it is + ## the same as for the link's target. template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(windows): - template merge(a, b): untyped = a or (b shl 32) + template merge[T](a, b): untyped = + cast[T]( + (uint64(cast[uint32](a))) or + (uint64(cast[uint32](b)) shl 32) + ) formalInfo.id.device = rawInfo.dwVolumeSerialNumber - formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) - formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) + formalInfo.id.file = merge[FileId](rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) + formalInfo.size = merge[BiggestInt](rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) formalInfo.linkCount = rawInfo.nNumberOfLinks formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) - formalInfo.blockSize = 8192 # xxx use windows API instead of hardcoding + formalInfo.blockSize = 8192 # xxx use Windows API instead of hardcoding # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: @@ -3303,14 +808,14 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = checkAndIncludeMode(S_IWOTH, fpOthersWrite) checkAndIncludeMode(S_IXOTH, fpOthersExec) - formalInfo.kind = + (formalInfo.kind, formalInfo.isSpecial) = if S_ISDIR(rawInfo.st_mode): - pcDir + (pcDir, false) elif S_ISLNK(rawInfo.st_mode): assert(path != "") # symlinks can't occur for file handles getSymlinkFileKind(path) else: - pcFile + (pcFile, not S_ISREG(rawInfo.st_mode)) when defined(js): when not declared(FileHandle): @@ -3326,8 +831,8 @@ proc getFileInfo*(handle: FileHandle): FileInfo {.noWeirdTarget.} = ## is invalid, `OSError` is raised. ## ## See also: - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ # Done: ID, Kind, Size, Permissions, Link Count when defined(windows): @@ -3348,8 +853,8 @@ proc getFileInfo*(file: File): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ if file.isNil: raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) @@ -3358,20 +863,21 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget. ## Retrieves file information for the file object pointed to by `path`. ## ## Due to intrinsic differences between operating systems, the information - ## contained by the returned `FileInfo object <#FileInfo>`_ will be slightly + ## contained by the returned `FileInfo object`_ will be slightly ## different across platforms, and in some cases, incomplete or inaccurate. ## ## When `followSymlink` is true (default), symlinks are followed and the ## information retrieved is information related to the symlink's target. - ## Otherwise, information on the symlink itself is retrieved. + ## Otherwise, information on the symlink itself is retrieved (however, + ## field `isSpecial` is still determined from the target on Unix). ## ## If the information cannot be retrieved, such as when the path doesn't ## exist, or when permission restrictions prevent the program from retrieving ## file information, `OSError` is raised. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ when defined(windows): var handle = openHandle(path, followSymlink) @@ -3398,7 +904,7 @@ proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", ## binary content. ## ## See also: - ## * `sameFile proc <#sameFile,string,string>`_ + ## * `sameFile proc`_ var a, b: File if not open(a, path1): return false @@ -3445,10 +951,7 @@ proc isHidden*(path: string): bool {.noWeirdTarget.} = assert ".foo/".isHidden when defined(windows): - when useWinUnicode: - wrapUnary(attributes, getFileAttributesW, path) - else: - var attributes = getFileAttributesA(path) + wrapUnary(attributes, getFileAttributesW, path) if attributes != -1'i32: result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 else: @@ -3484,37 +987,43 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noWeirdTarget.} = discard h.closeHandle if res == 0'i32: raiseOSError(osLastError(), file) + func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = - ## Returns true if ``filename`` is valid for crossplatform use. + ## Returns `true` if `filename` is valid for crossplatform use. ## ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. - ## You can pass full paths as argument too, but func only checks filenames. - ## It uses ``invalidFilenameChars``, ``invalidFilenames`` and ``maxLen`` to verify the specified ``filename``. + ## It uses `invalidFilenameChars`, `invalidFilenames` and `maxLen` to verify the specified `filename`. ## - ## .. code-block:: nim - ## assert not isValidFilename(" foo") ## Leading white space - ## assert not isValidFilename("foo ") ## Trailing white space - ## assert not isValidFilename("foo.") ## Ends with Dot - ## assert not isValidFilename("con.txt") ## "CON" is invalid (Windows) - ## assert not isValidFilename("OwO:UwU") ## ":" is invalid (Mac) - ## assert not isValidFilename("aux.bat") ## "AUX" is invalid (Windows) + ## See also: ## - # https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception - # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file - # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx + ## * https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception + ## * https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + ## * https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx + ## + ## .. warning:: This only checks filenames, not whole paths + ## (because basically you can mount anything as a path on Linux). + runnableExamples: + assert not isValidFilename(" foo") # Leading white space + assert not isValidFilename("foo ") # Trailing white space + assert not isValidFilename("foo.") # Ends with dot + assert not isValidFilename("con.txt") # "CON" is invalid (Windows) + assert not isValidFilename("OwO:UwU") # ":" is invalid (Mac) + assert not isValidFilename("aux.bat") # "AUX" is invalid (Windows) + assert not isValidFilename("") # Empty string + assert not isValidFilename("foo/") # Filename is empty + result = true let f = filename.splitFile() - if unlikely(f.name.len + f.ext.len > maxLen or + if unlikely(f.name.len + f.ext.len > maxLen or f.name.len == 0 or f.name[0] == ' ' or f.name[^1] == ' ' or f.name[^1] == '.' or find(f.name, invalidFilenameChars) != -1): return false for invalid in invalidFilenames: if cmpIgnoreCase(f.name, invalid) == 0: return false + # deprecated declarations -when not defined(nimscript): - when not defined(js): # `noNimJs` doesn't work with templates, this should improve. - template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = - fileExists(args) - template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = - dirExists(args) - # {.deprecated: [existsFile: fileExists].} # pending bug #14819; this would avoid above mentioned issue +when not weirdTarget: + template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = + fileExists(args) + template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = + dirExists(args) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6aefb8d6c7..9284e823af 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -18,18 +18,24 @@ include "system/inclrtl" import - strutils, os, strtabs, streams, cpuinfo, streamwrapper, - std/private/since + std/[strutils, os, strtabs, streams, cpuinfo, streamwrapper, + private/since] export quoteShell, quoteShellWindows, quoteShellPosix when defined(windows): - import winlean + import std/winlean else: - import posix + import std/posix when defined(linux) and defined(useClone): - import linux + import std/linux + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + when defined(windows): + import std/widestrs + type ProcessOption* = enum ## Options that can be passed to `startProcess proc @@ -66,16 +72,11 @@ type Process* = ref ProcessObj ## Represents an operating system process. -const poDemon* {.deprecated.} = poDaemon ## Nim versions before 0.20 - ## used the wrong spelling ("demon"). - ## Now `ProcessOption` uses the correct spelling ("daemon"), - ## and this is needed just for backward compatibility. - proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): - string {.rtl, extern: "nosp$1", + string {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. @@ -90,12 +91,12 @@ proc execProcess*(command: string, workingDir: string = "", ## * `execCmd proc <#execCmd,string>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) - ## let outp_shell = execProcess("nim c -r mytestfile.nim") - ## # Note: outp may have an interleave of text from the nim compile - ## # and any output from mytestfile when it runs + ## ```Nim + ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) + ## let outp_shell = execProcess("nim c -r mytestfile.nim") + ## # Note: outp may have an interleave of text from the nim compile + ## # and any output from mytestfile when it runs + ## ``` proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadIOEffect, RootEffect].} @@ -112,14 +113,14 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let errC = execCmd("nim c -r mytestfile.nim") + ## ```Nim + ## let errC = execCmd("nim c -r mytestfile.nim") + ## ``` proc startProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): - owned(Process) {.rtl, extern: "nosp$1", + owned(Process) {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory @@ -151,7 +152,7 @@ proc startProcess*(command: string, workingDir: string = "", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ -proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [WriteIOEffect].} +proc close*(p: Process) {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [WriteIOEffect].} ## When the process has finished executing, cleanup related handles. ## ## .. warning:: If the process has not finished executing, this will forcibly @@ -200,7 +201,7 @@ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## * `terminate proc <#terminate,Process>`_ ## * `posix_utils.sendSignal(pid: Pid, signal: int) `_ -proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} +proc running*(p: Process): bool {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Returns true if the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = @@ -211,7 +212,7 @@ proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, - extern: "nosp$1", tags: [].} + extern: "nosp$1", raises: [OSError, ValueError], tags: [].} ## Waits for the process to finish and returns `p`'s error code. ## ## .. warning:: Be careful when using `waitForExit` for processes created without @@ -219,8 +220,12 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. + ## + ## .. warning:: When working with `timeout` parameters, remember that the value is + ## typically expressed in milliseconds, and ensure that the correct unit of time + ## is used to avoid unexpected behavior. -proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -236,7 +241,7 @@ proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## * `outputStream proc <#outputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ -proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} +proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [].} ## Returns ``p``'s output stream for reading from. ## ## You cannot perform peek/write/setOption operations to this stream. @@ -288,7 +293,7 @@ proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], ## * `errorStream proc <#errorStream,Process>`_ ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ -proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", +proc inputHandle*(p: Process): FileHandle {.rtl, raises: [], extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. ## @@ -301,7 +306,7 @@ proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s output file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -313,7 +318,7 @@ proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s error file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -324,18 +329,23 @@ proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle -proc countProcessors*(): int {.rtl, extern: "nosp$1".} = +proc countProcessors*(): int {.rtl, extern: "nosp$1", raises: [].} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. ## It is implemented just calling `cpuinfo.countProcessors`. result = cpuinfo.countProcessors() +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", - tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect].} = + raises: [ValueError, OSError, IOError], + tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], + effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. ## Creates `n` processes that execute in parallel. ## @@ -447,7 +457,7 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), raises: [OSError, IOError, ValueError], tags: [ReadIOEffect].} = ## Convenience iterator for working with `startProcess` to read data from a ## background process. ## @@ -455,8 +465,7 @@ iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = ## * `readLines proc <#readLines,Process>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} ## var ps: seq[Process] ## for prog in ["a", "b"]: # run 2 progs in parallel @@ -468,15 +477,17 @@ iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = ## i.inc ## if i > 100: break ## p.close + ## ``` var outp = p.outputStream var line = newStringOfCap(120) - while true: - if outp.readLine(line): - yield line - else: - if p.peekExitCode != -1: break + while outp.readLine(line): + if keepNewLines: + line.add("\n") + yield line + discard waitForExit(p) -proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = +proc readLines*(p: Process): (seq[string], int) {.since: (1, 3), + raises: [OSError, IOError, ValueError], tags: [ReadIOEffect].} = ## Convenience function for working with `startProcess` to read data from a ## background process. ## @@ -484,8 +495,7 @@ proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## * `lines iterator <#lines.i,Process>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} ## var ps: seq[Process] ## for prog in ["a", "b"]: # run 2 progs in parallel @@ -495,6 +505,7 @@ proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## if exCode != 0: ## for line in lines: echo line ## p.close + ## ``` for line in p.lines: result[0].add(line) result[1] = p.peekExitCode @@ -510,6 +521,7 @@ when not defined(useNimRtl): var outp = outputStream(p) result = "" var line = newStringOfCap(120) + # consider `p.lines(keepNewLines=true)` to circumvent `running` busy-wait while true: # FIXME: converts CR-LF to LF. if outp.readLine(line): @@ -562,12 +574,8 @@ when defined(windows) and not defined(useNimRtl): if a == 0: raiseOSError(osLastError()) proc newFileHandleStream(handle: Handle): owned FileHandleStream = - new(result) - result.handle = handle - result.closeImpl = hsClose - result.atEndImpl = hsAtEnd - result.readDataImpl = hsReadData - result.writeDataImpl = hsWriteData + result = FileHandleStream(handle: handle, closeImpl: hsClose, atEndImpl: hsAtEnd, + readDataImpl: hsReadData, writeDataImpl: hsWriteData) proc buildCommandLine(a: string, args: openArray[string]): string = result = quoteShell(a) @@ -710,22 +718,15 @@ when defined(windows) and not defined(useNimRtl): if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) - when useWinUnicode: - var tmp = newWideCString(cmdl) - var ee = - if e.str.isNil: newWideCString(cstring(nil)) - else: newWideCString(e.str, e.len) - var wwd = newWideCString(wd) - var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT - if poDaemon in options: flags = flags or CREATE_NO_WINDOW - success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, - ee, wwd, si, procInfo) - else: - var ee = - if e.str.isNil: cstring(nil) - else: cstring(e.str) - success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, ee, wd, si, procInfo) + var tmp = newWideCString(cmdl) + var ee = + if e.str.isNil: newWideCString(cstring(nil)) + else: newWideCString(e.str, e.len) + var wwd = newWideCString(wd) + var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT + if poDaemon in options: flags = flags or CREATE_NO_WINDOW + success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, + ee, wwd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -741,7 +742,7 @@ when defined(windows) and not defined(useNimRtl): const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, - "Requested command not found: '$1'. OS error:" % command) + "Requested command not found: '" & command & "'. OS error:") else: raiseOSError(lastError, command) result.fProcessHandle = procInfo.hProcess @@ -866,13 +867,9 @@ when defined(windows) and not defined(useNimRtl): si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) - when useWinUnicode: - var c = newWideCString(command) - var res = winlean.createProcessW(nil, c, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) - else: - var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) + var c = newWideCString(command) + var res = winlean.createProcessW(nil, c, nil, nil, 0, + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: @@ -949,13 +946,13 @@ elif not defined(useNimRtl): not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} {.push stacktrace: off, profiler: off.} proc startProcessAfterFork(data: ptr StartProcessData) {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} {.pop.} proc startProcess(command: string, workingDir: string = "", @@ -1056,13 +1053,15 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - if poDaemon in data.options: - chck posix_spawnattr_setpgroup(attr, 0'i32) + when not defined(nuttx): + if poDaemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) var flags = POSIX_SPAWN_USEVFORK or POSIX_SPAWN_SETSIGMASK - if poDaemon in data.options: - flags = flags or POSIX_SPAWN_SETPGROUP + when not defined(nuttx): + if poDaemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP chck posix_spawnattr_setflags(attr, flags) if not (poParentStreams in data.options): @@ -1082,9 +1081,9 @@ elif not defined(useNimRtl): var pid: Pid if (poUsePath in data.options): - res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + res = posix_spawnp(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) else: - res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + res = posix_spawn(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) @@ -1125,8 +1124,7 @@ elif not defined(useNimRtl): let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): raiseOSError(osLastError(), - "Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + "Could not find command: '" & $data.sysCommand & "'. OS error: " & $strerror(error)) return pid @@ -1170,14 +1168,14 @@ elif not defined(useNimRtl): when defined(uClibc) or defined(linux) or defined(haiku): # uClibc environment (OpenWrt included) doesn't have the full execvpe let exe = findExe(data.sysCommand) - discard execve(exe, data.sysArgs, data.sysEnv) + discard execve(exe.cstring, data.sysArgs, data.sysEnv) else: # MacOSX doesn't have execvpe, so we need workaround. # On MacOSX we can arrive here only from fork, so this is safe: environ = data.sysEnv - discard execvp(data.sysCommand, data.sysArgs) + discard execvp(data.sysCommand.cstring, data.sysArgs) else: - discard execve(data.sysCommand, data.sysArgs, data.sysEnv) + discard execve(data.sysCommand.cstring, data.sysArgs, data.sysEnv) startProcessFail(data) {.pop.} @@ -1233,7 +1231,7 @@ elif not defined(useNimRtl): when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or defined(dragonfly): - import kqueue + import std/kqueue proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: @@ -1350,12 +1348,12 @@ elif not defined(useNimRtl): p.exitStatus = status break else: - doAssert false, "unreachable!" + raiseAssert "unreachable!" result = exitStatusLikeShell(p.exitStatus) else: - import times + import std/times const hasThreadSupport = compileOption("threads") and not defined(nimscript) @@ -1519,7 +1517,7 @@ elif not defined(useNimRtl): header: "".} proc execCmd(command: string): int = - when defined(linux): + when defined(posix): let tmp = csystem(command) result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: @@ -1572,7 +1570,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { poStdErrToStdOut, poUsePath}, env: StringTableRef = nil, workingDir = "", input = ""): tuple[ output: string, - exitCode: int] {.tags: + exitCode: int] {.raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = ## A convenience proc that runs the `command`, and returns its `output` and ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. @@ -1589,16 +1587,16 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## var result = execCmdEx("nim r --hints:off -", options = {}, input = "echo 3*4") ## import std/[strutils, strtabs] ## stripLineEnd(result[0]) ## portable way to remove trailing newline, if any ## doAssert result == ("12", 0) - ## doAssert execCmdEx("ls --nonexistant").exitCode != 0 + ## doAssert execCmdEx("ls --nonexistent").exitCode != 0 ## when defined(posix): ## assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) ## assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) + ## ``` when (NimMajor, NimMinor, NimPatch) < (1, 3, 5): doAssert input.len == 0 @@ -1618,6 +1616,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { inputStream(p).write(input) close inputStream(p) + # consider `p.lines(keepNewLines=true)` to avoid exit code test result = ("", -1) var line = newStringOfCap(120) while true: diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 23bb09ac4b..8a43daf54f 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -19,99 +19,97 @@ ## :literal: ## ## Here is an example of how to use the configuration file parser: -## -## .. code-block:: nim -## -## import std/[os, parsecfg, strutils, streams] -## -## var f = newFileStream(paramStr(1), fmRead) -## if f != nil: -## var p: CfgParser -## open(p, f, paramStr(1)) -## while true: -## var e = next(p) -## case e.kind -## of cfgEof: break -## of cfgSectionStart: ## a `[section]` has been parsed -## echo("new section: " & e.section) -## of cfgKeyValuePair: -## echo("key-value-pair: " & e.key & ": " & e.value) -## of cfgOption: -## echo("command: " & e.key & ": " & e.value) -## of cfgError: -## echo(e.msg) -## close(p) -## else: -## echo("cannot open: " & paramStr(1)) -## -## -## Examples -## ======== -## +runnableExamples("-r:off"): + import std/[strutils, streams] + + let configFile = "example.ini" + var f = newFileStream(configFile, fmRead) + assert f != nil, "cannot open " & configFile + var p: CfgParser + open(p, f, configFile) + while true: + var e = next(p) + case e.kind + of cfgEof: break + of cfgSectionStart: ## a `[section]` has been parsed + echo "new section: " & e.section + of cfgKeyValuePair: + echo "key-value-pair: " & e.key & ": " & e.value + of cfgOption: + echo "command: " & e.key & ": " & e.value + of cfgError: + echo e.msg + close(p) + +##[ ## Configuration file example -## -------------------------- -## -## .. code-block:: nim -## +]## + +## ```none ## charset = "utf-8" ## [Package] ## name = "hello" ## --threads:on ## [Author] -## name = "lihf8515" -## qq = "10214028" -## email = "lihaifeng@wxm.com" -## +## name = "nim-lang" +## website = "nim-lang.org" +## ``` + +##[ ## Creating a configuration file -## ----------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict=newConfig() -## dict.setSectionKey("","charset","utf-8") -## dict.setSectionKey("Package","name","hello") -## dict.setSectionKey("Package","--threads","on") -## dict.setSectionKey("Author","name","lihf8515") -## dict.setSectionKey("Author","qq","10214028") -## dict.setSectionKey("Author","email","lihaifeng@wxm.com") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples: + var dict = newConfig() + dict.setSectionKey("","charset", "utf-8") + dict.setSectionKey("Package", "name", "hello") + dict.setSectionKey("Package", "--threads", "on") + dict.setSectionKey("Author", "name", "nim-lang") + dict.setSectionKey("Author", "website", "nim-lang.org") + assert $dict == """ +charset=utf-8 +[Package] +name=hello +--threads:on +[Author] +name=nim-lang +website=nim-lang.org +""" + +##[ ## Reading a configuration file -## ---------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## var charset = dict.getSectionValue("","charset") -## var threads = dict.getSectionValue("Package","--threads") -## var pname = dict.getSectionValue("Package","name") -## var name = dict.getSectionValue("Author","name") -## var qq = dict.getSectionValue("Author","qq") -## var email = dict.getSectionValue("Author","email") -## echo pname & "\n" & name & "\n" & qq & "\n" & email -## +]## + +runnableExamples("-r:off"): + let dict = loadConfig("config.ini") + let charset = dict.getSectionValue("","charset") + let threads = dict.getSectionValue("Package","--threads") + let pname = dict.getSectionValue("Package","name") + let name = dict.getSectionValue("Author","name") + let website = dict.getSectionValue("Author","website") + echo pname & "\n" & name & "\n" & website + +##[ ## Modifying a configuration file -## ------------------------------ -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## dict.setSectionKey("Author","name","lhf") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.setSectionKey("Author", "name", "nim-lang") + dict.writeConfig("config.ini") + +##[ ## Deleting a section key in a configuration file -## ---------------------------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## dict.delSectionKey("Author","email") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.delSectionKey("Author", "website") + dict.writeConfig("config.ini") + +##[ ## Supported INI File structure -## ---------------------------- -## The examples below are supported: -## +]## # taken from https://docs.python.org/3/library/configparser.html#supported-ini-file-structure runnableExamples: @@ -150,33 +148,38 @@ runnableExamples: ) let section1 = "Simple Values" - doAssert dict.getSectionValue(section1, "key") == "value" - doAssert dict.getSectionValue(section1, "spaces in keys") == "allowed" - doAssert dict.getSectionValue(section1, "spaces in values") == "allowed as well" - doAssert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" - doAssert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" + assert dict.getSectionValue(section1, "key") == "value" + assert dict.getSectionValue(section1, "spaces in keys") == "allowed" + assert dict.getSectionValue(section1, "spaces in values") == "allowed as well" + assert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" + assert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" let section2 = "All Values Are Strings" - doAssert dict.getSectionValue(section2, "values like this") == "19990429" - doAssert dict.getSectionValue(section2, "or this") == "3.14159265359" - doAssert dict.getSectionValue(section2, "are they treated as numbers") == "no" - doAssert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" - doAssert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" + assert dict.getSectionValue(section2, "values like this") == "19990429" + assert dict.getSectionValue(section2, "or this") == "3.14159265359" + assert dict.getSectionValue(section2, "are they treated as numbers") == "no" + assert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" + assert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" let section3 = "Seletion A" - doAssert dict.getSectionValue(section3, + assert dict.getSectionValue(section3, "space around section name will be ignored", "not an empty value") == "" let section4 = "Sections Can Be Indented" - doAssert dict.getSectionValue(section4, "can_values_be_as_well") == "True" - doAssert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" - doAssert dict.getSectionValue(section4, "purpose") == "formatting for readability" + assert dict.getSectionValue(section4, "can_values_be_as_well") == "True" + assert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" + assert dict.getSectionValue(section4, "purpose") == "formatting for readability" -import strutils, lexbase, streams, tables +import std/[strutils, lexbase, streams, tables] import std/private/decode_helpers +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/syncio include "system/inclrtl" + type CfgEventKind* = enum ## enumeration of all events that may occur when parsing cfgEof, ## end of file reached @@ -449,7 +452,7 @@ proc getKeyValPair(c: var CfgParser, kind: CfgEventKind): CfgEvent = if c.tok.kind == tkSymbol: case kind of cfgOption, cfgKeyValuePair: - result = CfgEvent(kind: kind, key: c.tok.literal, value: "") + result = CfgEvent(kind: kind, key: c.tok.literal.move, value: "") else: discard rawGetTok(c, c.tok) if c.tok.kind in {tkEquals, tkColon}: @@ -478,7 +481,7 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = of tkBracketLe: rawGetTok(c, c.tok) if c.tok.kind == tkSymbol: - result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal) + result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal.move) else: result = CfgEvent(kind: cfgError, msg: errorStr(c, "symbol expected, but found: " & c.tok.literal)) @@ -495,17 +498,17 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = # ---------------- Configuration file related operations ---------------- type - Config* = OrderedTableRef[string, OrderedTableRef[string, string]] + Config* = OrderedTableRef[string, OrderedTableRef[string, string]] proc newConfig*(): Config = ## Creates a new configuration table. ## Useful when wanting to create a configuration file. - result = newOrderedTable[string, OrderedTableRef[string, string]]() + result = newOrderedTable[string, OrderedTableRef[string, string]]() -proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = +proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = ## Loads the specified configuration from stream into a new Config instance. ## `filename` parameter is only used for nicer error messages. - var dict = newOrderedTable[string, OrderedTableRef[string, string]]() + var dict = newOrderedTable[string, OrderedTableRef[string, string]]() var curSection = "" ## Current section, ## the default value of the current section is "", ## which means that the current section is a common @@ -535,7 +538,7 @@ proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = close(p) result = dict -proc loadConfig*(filename: string): Config = +proc loadConfig*(filename: string): Config = ## Loads the specified configuration file into a new Config instance. let file = open(filename, fmRead) let fileStream = newFileStream(file) @@ -563,7 +566,7 @@ proc replace(s: string): string = proc writeConfig*(dict: Config, stream: Stream) = ## Writes the contents of the table to the specified stream. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. for section, sectionData in dict.pairs(): if section != "": ## Not general section if not allCharsInSet(section, SymChars): ## Non system character @@ -603,7 +606,7 @@ proc writeConfig*(dict: Config, stream: Stream) = proc `$`*(dict: Config): string = ## Writes the contents of the table to string. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. let stream = newStringStream() defer: stream.close() dict.writeConfig(stream) @@ -612,7 +615,7 @@ proc `$`*(dict: Config): string = proc writeConfig*(dict: Config, filename: string) = ## Writes the contents of the table to the specified configuration file. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. let file = open(filename, fmWrite) defer: file.close() let fileStream = newFileStream(file) @@ -649,3 +652,8 @@ proc delSectionKey*(dict: var Config, section, key: string) = dict.del(section) else: dict[section].del(key) + +iterator sections*(dict: Config): lent string {.since: (1, 5).} = + ## Iterates through the sections in the `dict`. + for section in dict.keys: + yield section diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 1e34f3649f..c7bf0c9c11 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -13,7 +13,7 @@ ## Basic usage ## =========== ## -## .. code-block:: nim +## ```nim ## import std/parsecsv ## from std/os import paramStr ## from std/streams import newFileStream @@ -29,11 +29,12 @@ ## for val in items(x.row): ## echo "##", val, "##" ## close(x) +## ``` ## ## For CSV files with a header row, the header can be read and then used as a ## reference for item access with `rowEntry <#rowEntry,CsvParser,string>`_: ## -## .. code-block:: nim +## ```nim ## import std/parsecsv ## ## # Prepare a file @@ -52,6 +53,7 @@ ## for col in items(p.headers): ## echo "##", col, ":", p.rowEntry(col), "##" ## p.close() +## ``` ## ## See also ## ======== @@ -65,8 +67,10 @@ ## * `parsesql module `_ for a SQL parser ## * `other parsers `_ for other parsers -import - lexbase, streams +import std/[lexbase, streams] + +when defined(nimPreviewSlimSystem): + import std/syncio type CsvRow* = seq[string] ## A row in a CSV file. @@ -97,10 +101,10 @@ proc raiseEInvalidCsv(filename: string, line, col: int, e.msg = filename & "(" & $line & ", " & $col & ") Error: " & msg raise e -proc error(my: CsvParser, pos: int, msg: string) = - raiseEInvalidCsv(my.filename, my.lineNumber, getColNumber(my, pos), msg) +proc error(self: CsvParser, pos: int, msg: string) = + raiseEInvalidCsv(self.filename, self.lineNumber, getColNumber(self, pos), msg) -proc open*(my: var CsvParser, input: Stream, filename: string, +proc open*(self: var CsvParser, input: Stream, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = ## Initializes the parser with an input stream. `Filename` is only used @@ -127,16 +131,14 @@ proc open*(my: var CsvParser, input: Stream, filename: string, parser.close() strm.close() - lexbase.open(my, input) - my.filename = filename - my.sep = separator - my.quote = quote - my.esc = escape - my.skipWhite = skipInitialSpace - my.row = @[] - my.currRow = 0 + lexbase.open(self, input) + self.filename = filename + self.sep = separator + self.quote = quote + self.esc = escape + self.skipWhite = skipInitialSpace -proc open*(my: var CsvParser, filename: string, +proc open*(self: var CsvParser, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = ## Similar to the `other open proc<#open,CsvParser,Stream,string,char,char,char>`_, @@ -150,54 +152,54 @@ proc open*(my: var CsvParser, filename: string, removeFile("tmp.csv") var s = newFileStream(filename, fmRead) - if s == nil: my.error(0, "cannot open: " & filename) - open(my, s, filename, separator, + if s == nil: self.error(0, "cannot open: " & filename) + open(self, s, filename, separator, quote, escape, skipInitialSpace) -proc parseField(my: var CsvParser, a: var string) = - var pos = my.bufpos - if my.skipWhite: - while my.buf[pos] in {' ', '\t'}: inc(pos) +proc parseField(self: var CsvParser, a: var string) = + var pos = self.bufpos + if self.skipWhite: + while self.buf[pos] in {' ', '\t'}: inc(pos) setLen(a, 0) # reuse memory - if my.buf[pos] == my.quote and my.quote != '\0': + if self.buf[pos] == self.quote and self.quote != '\0': inc(pos) while true: - let c = my.buf[pos] + let c = self.buf[pos] if c == '\0': - my.bufpos = pos # can continue after exception? - error(my, pos, my.quote & " expected") + self.bufpos = pos # can continue after exception? + error(self, pos, self.quote & " expected") break - elif c == my.quote: - if my.esc == '\0' and my.buf[pos+1] == my.quote: - add(a, my.quote) + elif c == self.quote: + if self.esc == '\0' and self.buf[pos + 1] == self.quote: + add(a, self.quote) inc(pos, 2) else: inc(pos) break - elif c == my.esc: - add(a, my.buf[pos+1]) + elif c == self.esc: + add(a, self.buf[pos + 1]) inc(pos, 2) else: case c of '\c': - pos = handleCR(my, pos) + pos = handleCR(self, pos) add(a, "\n") of '\l': - pos = handleLF(my, pos) + pos = handleLF(self, pos) add(a, "\n") else: add(a, c) inc(pos) else: while true: - let c = my.buf[pos] - if c == my.sep: break + let c = self.buf[pos] + if c == self.sep: break if c in {'\c', '\l', '\0'}: break add(a, c) inc(pos) - my.bufpos = pos + self.bufpos = pos -proc processedRows*(my: var CsvParser): int = +proc processedRows*(self: var CsvParser): int {.inline.} = ## Returns number of the processed rows. ## ## But even if `readRow <#readRow,CsvParser,int>`_ arrived at EOF then @@ -220,9 +222,9 @@ proc processedRows*(my: var CsvParser): int = parser.close() strm.close() - return my.currRow + self.currRow -proc readRow*(my: var CsvParser, columns = 0): bool = +proc readRow*(self: var CsvParser, columns = 0): bool = ## Reads the next row; if `columns` > 0, it expects the row to have ## exactly this many columns. Returns false if the end of the file ## has been encountered else true. @@ -251,47 +253,47 @@ proc readRow*(my: var CsvParser, columns = 0): bool = strm.close() var col = 0 # current column - let oldpos = my.bufpos + let oldpos = self.bufpos # skip initial empty lines #8365 while true: - case my.buf[my.bufpos] - of '\c': my.bufpos = handleCR(my, my.bufpos) - of '\l': my.bufpos = handleLF(my, my.bufpos) + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) else: break - while my.buf[my.bufpos] != '\0': - let oldlen = my.row.len - if oldlen < col+1: - setLen(my.row, col+1) - my.row[col] = "" - parseField(my, my.row[col]) + while self.buf[self.bufpos] != '\0': + let oldlen = self.row.len + if oldlen < col + 1: + setLen(self.row, col + 1) + self.row[col] = "" + parseField(self, self.row[col]) inc(col) - if my.buf[my.bufpos] == my.sep: - inc(my.bufpos) + if self.buf[self.bufpos] == self.sep: + inc(self.bufpos) else: - case my.buf[my.bufpos] + case self.buf[self.bufpos] of '\c', '\l': # skip empty lines: while true: - case my.buf[my.bufpos] - of '\c': my.bufpos = handleCR(my, my.bufpos) - of '\l': my.bufpos = handleLF(my, my.bufpos) + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) else: break of '\0': discard - else: error(my, my.bufpos, my.sep & " expected") + else: error(self, self.bufpos, self.sep & " expected") break - setLen(my.row, col) + setLen(self.row, col) result = col > 0 if result and col != columns and columns > 0: - error(my, oldpos+1, $columns & " columns expected, but found " & + error(self, oldpos + 1, $columns & " columns expected, but found " & $col & " columns") - inc(my.currRow) + inc(self.currRow) -proc close*(my: var CsvParser) {.inline.} = - ## Closes the parser `my` and its associated input stream. - lexbase.close(my) +proc close*(self: var CsvParser) {.inline.} = + ## Closes the parser `self` and its associated input stream. + lexbase.close(self) -proc readHeaderRow*(my: var CsvParser) = +proc readHeaderRow*(self: var CsvParser) = ## Reads the first row and creates a look-up table for column numbers ## See also: ## * `rowEntry proc <#rowEntry,CsvParser,string>`_ @@ -313,16 +315,16 @@ proc readHeaderRow*(my: var CsvParser) = parser.close() strm.close() - let present = my.readRow() + let present = self.readRow() if present: - my.headers = my.row + self.headers = self.row -proc rowEntry*(my: var CsvParser, entry: string): var string = +proc rowEntry*(self: var CsvParser, entry: string): var string = ## Accesses a specified `entry` from the current row. ## ## Assumes that `readHeaderRow <#readHeaderRow,CsvParser>`_ has already been ## called. - ## + ## ## If specified `entry` does not exist, raises KeyError. runnableExamples: import std/streams @@ -340,14 +342,14 @@ proc rowEntry*(my: var CsvParser, entry: string): var string = parser.close() strm.close() - let index = my.headers.find(entry) + let index = self.headers.find(entry) if index >= 0: - result = my.row[index] + result = self.row[index] else: raise newException(KeyError, "Entry `" & entry & "` doesn't exist") when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: CsvParser diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim index 196d8c360b..9292a85964 100644 --- a/lib/pure/parsejson.nim +++ b/lib/pure/parsejson.nim @@ -11,9 +11,12 @@ ## and exported by the `json` standard library ## module, but can also be used in its own right. -import strutils, lexbase, streams, unicode +import std/[strutils, lexbase, streams, unicode] import std/private/decode_helpers +when defined(nimPreviewSlimSystem): + import std/assertions + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing @@ -218,7 +221,7 @@ proc parseString(my: var JsonParser): TokKind = add(my.a, 'u') inc(pos, 2) var pos2 = pos - var r = parseEscapedUTF16(my.buf, pos) + var r = parseEscapedUTF16(cstring(my.buf), pos) if r < 0: my.err = errInvalidToken break @@ -228,7 +231,7 @@ proc parseString(my: var JsonParser): TokKind = my.err = errInvalidToken break inc(pos, 2) - var s = parseEscapedUTF16(my.buf, pos) + var s = parseEscapedUTF16(cstring(my.buf), pos) if (s and 0xfc00) == 0xdc00 and s > 0: r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) else: diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index b2d024a39e..24c903b584 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -16,7 +16,7 @@ ## ## The following syntax is supported when arguments for the `shortNoVal` and ## `longNoVal` parameters, which are -## `described later<#shortnoval-and-longnoval>`_, are not provided: +## `described later<#nimshortnoval-and-nimlongnoval>`_, are not provided: ## ## 1. Short options: `-abcd`, `-e:5`, `-e=5` ## 2. Long options: `--foo:bar`, `--foo=bar`, `--foo` @@ -48,7 +48,7 @@ ## ## Here is an example: ## -## .. code-block:: +## ```Nim ## import std/parseopt ## ## var p = initOptParser("-ab -e:5 --foo --bar=20 file.txt") @@ -71,10 +71,34 @@ ## # Option: foo ## # Option and value: bar, 20 ## # Argument: file.txt +## ``` ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. ## +## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. +## Assign the default value to a variable before parsing. +## Then set the variable to the new value while parsing. +## +## Here is an example: +## +## ```Nim +## import std/parseopt +## +## var varName: string = "defaultValue" +## +## for kind, key, val in getopt(): +## case kind +## of cmdArgument: +## discard +## of cmdLongOption, cmdShortOption: +## case key: +## of "varName": # --varName: in the console when executing +## varName = val # do input sanitization in production systems +## of cmdEnd: +## discard +## ``` +## ## `shortNoVal` and `longNoVal` ## ============================ ## @@ -98,7 +122,7 @@ ## `shortNoVal` and `longNoVal`, which is the default, and providing ## arguments for those two parameters: ## -## .. code-block:: +## ```Nim ## import std/parseopt ## ## proc printToken(kind: CmdLineKind, key: string, val: string) = @@ -132,6 +156,7 @@ ## # Output: ## # Option and value: j, 4 ## # Option and value: first, bar +## ``` ## ## See also ## ======== @@ -151,7 +176,7 @@ include "system/inclrtl" -import os +import std/os type CmdLineKind* = enum ## The detected command line token. @@ -164,7 +189,7 @@ type ## ## To initialize it, use the ## `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_. - pos*: int + pos: int inShortState: bool allowWhitespaceAfterColon: bool shortNoVal: set[char] @@ -192,51 +217,6 @@ proc parseWord(s: string, i: int, w: var string, add(w, s[result]) inc(result) -proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser = - ## Initializes the command line parser. - ## - ## If `cmdline == ""`, the real command line as provided by the - ## `os` module is retrieved instead if it is available. If the - ## command line is not available, a `ValueError` will be raised. - ## - ## `shortNoVal` and `longNoVal` are used to specify which options - ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on - ## how this affects parsing. - ## - ## See also: - ## * `getopt iterator<#getopt.i,OptParser>`_ - runnableExamples: - var p = initOptParser() - p = initOptParser("--left --debug:3 -l -r:2") - p = initOptParser("--left --debug:3 -l -r:2", - shortNoVal = {'l'}, longNoVal = @["left"]) - - result.pos = 0 - result.idx = 0 - result.inShortState = false - result.shortNoVal = shortNoVal - result.longNoVal = longNoVal - result.allowWhitespaceAfterColon = allowWhitespaceAfterColon - if cmdline != "": - result.cmds = parseCmdLine(cmdline) - else: - when declared(paramCount): - result.cmds = newSeq[string](paramCount()) - for i in countup(1, paramCount()): - result.cmds[i-1] = paramStr(i) - else: - # we cannot provide this for NimRtl creation on Posix, because we can't - # access the command line arguments then! - doAssert false, "empty command line given but" & - " real command line is not accessible" - - result.kind = cmdEnd - result.key = "" - result.val = "" - proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = @@ -275,12 +255,38 @@ proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, else: # we cannot provide this for NimRtl creation on Posix, because we can't # access the command line arguments then! - doAssert false, "empty command line given but" & + raiseAssert "empty command line given but" & " real command line is not accessible" result.kind = cmdEnd result.key = "" result.val = "" +proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline == ""`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## This does not provide a way of passing default values to arguments. + ## + ## See also: + ## * `getopt iterator<#getopt.i,OptParser>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", + shortNoVal = {'l'}, longNoVal = @["left"]) + + initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) + proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos p.kind = cmdShortOption @@ -373,7 +379,7 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.key = p.cmds[p.idx] + p.key = p.cmds[p.idx] inc p.idx p.pos = 0 @@ -385,15 +391,14 @@ when declared(quoteShellCommand): ## * `remainingArgs proc<#remainingArgs,OptParser>`_ ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") ## while true: ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.cmdLineRest == "foo.txt bar.txt" + ## ``` result = p.cmds[p.idx .. ^1].quoteShellCommand proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = @@ -403,15 +408,14 @@ proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = ## * `cmdLineRest proc<#cmdLineRest,OptParser>`_ ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") ## while true: ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] + ## ``` result = @[] for i in p.idx..`_. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## # these are placeholders, of course ## proc writeHelp() = discard ## proc writeVersion() = discard @@ -447,6 +452,7 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## if filename == "": ## # no filename has been given, so we show the help ## writeHelp() + ## ``` p.pos = 0 p.idx = 0 while true: @@ -465,18 +471,18 @@ iterator getopt*(cmdline: seq[string] = @[], ## ## `shortNoVal` and `longNoVal` are used to specify which options ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,seq[string],set[char],seq[string]>`_ ## ## **Examples:** ## - ## .. code-block:: - ## + ## ```Nim ## # these are placeholders, of course ## proc writeHelp() = discard ## proc writeVersion() = discard @@ -496,6 +502,7 @@ iterator getopt*(cmdline: seq[string] = @[], ## if filename == "": ## # no filename has been written, so we show the help ## writeHelp() + ## ``` var p = initOptParser(cmdline, shortNoVal = shortNoVal, longNoVal = longNoVal) while true: diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index eb5d7c2cce..a7c938d018 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -12,9 +12,12 @@ ## ## Unstable API. -import strutils, lexbase +import std/[strutils, lexbase] import std/private/decode_helpers +when defined(nimPreviewSlimSystem): + import std/assertions + # ------------------- scanner ------------------------------------------------- type @@ -57,7 +60,7 @@ const reservedKeywords = @[ # statements - "select", "from", "where", "group", "limit", "having", + "select", "from", "where", "group", "limit", "offset", "having", # functions "count", ] @@ -506,6 +509,7 @@ type nkFromItemPair, nkGroup, nkLimit, + nkOffset, nkHaving, nkOrder, nkJoin, @@ -982,7 +986,7 @@ proc parseInsert(p: var SqlParser): SqlNode = parseParIdentList(p, n) result.add n else: - result.add(nil) + result.add(newNode(nkNone)) if isKeyw(p, "default"): getTok(p) eat(p, "values") @@ -1017,7 +1021,7 @@ proc parseUpdate(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseDelete(p: var SqlParser): SqlNode = getTok(p) @@ -1029,7 +1033,7 @@ proc parseDelete(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseSelect(p: var SqlParser): SqlNode = getTok(p) @@ -1123,6 +1127,11 @@ proc parseSelect(p: var SqlParser): SqlNode = var l = newNode(nkLimit) l.add(parseExpr(p)) result.add(l) + if isKeyw(p, "offset"): + getTok(p) + var o = newNode(nkOffset) + o.add(parseExpr(p)) + result.add(o) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1385,6 +1394,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkLimit: s.addKeyw("limit") s.addMulti(n) + of nkOffset: + s.addKeyw("offset") + s.addMulti(n) of nkHaving: s.addKeyw("having") s.addMulti(n) @@ -1451,7 +1463,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("enum") rs(n, s) -proc renderSQL*(n: SqlNode, upperCase = false): string = +proc renderSql*(n: SqlNode, upperCase = false): string = ## Converts an SQL abstract syntax tree to its string representation. var s: SqlWriter s.buffer = "" @@ -1460,8 +1472,8 @@ proc renderSQL*(n: SqlNode, upperCase = false): string = return s.buffer proc `$`*(n: SqlNode): string = - ## an alias for `renderSQL`. - renderSQL(n) + ## an alias for `renderSql`. + renderSql(n) proc treeReprAux(s: SqlNode, level: int, result: var string) = result.add('\n') @@ -1479,7 +1491,7 @@ proc treeRepr*(s: SqlNode): string = result = newStringOfCap(128) treeReprAux(s, 0, result) -import streams +import std/streams proc open(L: var SqlLexer, input: Stream, filename: string) = lexbase.open(L, input) @@ -1493,7 +1505,7 @@ proc open(p: var SqlParser, input: Stream, filename: string) = p.tok.literal = "" getTok(p) -proc parseSQL*(input: Stream, filename: string): SqlNode = +proc parseSql*(input: Stream, filename: string): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. @@ -1504,8 +1516,8 @@ proc parseSQL*(input: Stream, filename: string): SqlNode = finally: close(p) -proc parseSQL*(input: string, filename = ""): SqlNode = +proc parseSql*(input: string, filename = ""): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. - parseSQL(newStringStream(input), "") + parseSql(newStringStream(input), "") diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 64949428fa..ce642fcf73 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -12,29 +12,28 @@ ## ## To unpack raw bytes look at the `streams `_ module. ## -## .. code-block:: nim -## :test: +## ```nim test +## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] +## var outp: seq[string] ## -## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] -## var outp: seq[string] +## for log in logs: +## var res: string +## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 +## outp.add(res & " - " & captureBetween(log, ' ', '_')) +## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ``` ## -## for log in logs: -## var res: string -## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 -## outp.add(res & " - " & captureBetween(log, ' ', '_')) -## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ```nim test +## from std/strutils import Digits, parseInt ## -## .. code-block:: nim -## :test: -## from std/strutils import Digits, parseInt -## -## let -## input1 = "2019 school start" -## input2 = "3 years back" -## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 -## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 -## examYear = parseInt(startYear) + parseInt(yearsBack) -## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## let +## input1 = "2019 school start" +## input2 = "3 years back" +## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 +## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 +## examYear = parseInt(startYear) + parseInt(yearsBack) +## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## ``` ## ## **See also:** ## * `strutils module`_ for combined and identical parsing proc's @@ -50,6 +49,8 @@ include "system/inclrtl" +template toOa(s: string): openArray[char] = openArray[char](s) + const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} @@ -59,8 +60,7 @@ const proc toLower(c: char): char {.inline.} = result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c -proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseBin*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a binary number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -89,7 +89,7 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 doAssert num64 == 336784608873 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -104,10 +104,9 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseOct*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses an octal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -136,7 +135,7 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseOct("2346475523464755", num64) == 16 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -151,10 +150,9 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseHex*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a hexadecimal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -184,7 +182,7 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseHex("4E69ED4E69ED", num64) == 12 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -206,9 +204,9 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseIdent*(s: string, ident: var string, start = 0): int = +proc parseIdent*(s: openArray[char], ident: var string): int = ## Parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. ## If error, the value of `ident` is not changed. @@ -220,14 +218,14 @@ proc parseIdent*(s: string, ident: var string, start = 0): int = doAssert res == "ello" doAssert parseIdent("Hello World", res, 6) == 5 doAssert res == "World" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - ident = substr(s, start, i-1) - result = i-start + ident = substr(s.toOpenArray(0, i-1)) + result = i -proc parseIdent*(s: string, start = 0): string = +proc parseIdent*(s: openArray[char]): string = ## Parses an identifier and returns it or an empty string in ## case of an error. runnableExamples: @@ -236,13 +234,13 @@ proc parseIdent*(s: string, start = 0): string = doAssert parseIdent("Hello World", 5) == "" doAssert parseIdent("Hello World", 6) == "World" result = "" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - result = substr(s, start, i-1) + result = substr(s.toOpenArray(0, i - 1)) -proc parseChar*(s: string, c: var char, start = 0): int = +proc parseChar*(s: openArray[char], c: var char): int = ## Parses a single character, stores it in `c` and returns 1. ## In case of error (if start >= s.len) it returns 0 ## and the value of `c` is unchanged. @@ -252,11 +250,11 @@ proc parseChar*(s: string, c: var char, start = 0): int = doAssert c == '\0' doAssert "nim".parseChar(c, 0) == 1 doAssert c == 'n' - if start < s.len: - c = s[start] + if s.len > 0: + c = s[0] result = 1 -proc skipWhitespace*(s: string, start = 0): int {.inline.} = +proc skipWhitespace*(s: openArray[char]): int {.inline.} = ## Skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. runnableExamples: @@ -265,9 +263,9 @@ proc skipWhitespace*(s: string, start = 0): int {.inline.} = doAssert skipWhitespace("Hello World", 5) == 1 doAssert skipWhitespace("Hello World", 5) == 2 result = 0 - while start+result < s.len and s[start+result] in Whitespace: inc(result) + while result < s.len and s[result] in Whitespace: inc(result) -proc skip*(s, token: string, start = 0): int {.inline.} = +proc skip*(s, token: openArray[char]): int {.inline.} = ## Skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. runnableExamples: @@ -277,22 +275,22 @@ proc skip*(s, token: string, start = 0): int {.inline.} = doAssert skip("CAPlow", "CAP", 0) == 3 doAssert skip("CAPlow", "cap", 0) == 0 result = 0 - while start+result < s.len and result < token.len and - s[result+start] == token[result]: + while result < s.len and result < token.len and + s[result] == token[result]: inc(result) if result != token.len: result = 0 -proc skipIgnoreCase*(s, token: string, start = 0): int = +proc skipIgnoreCase*(s, token: openArray[char]): int = ## Same as `skip` but case is ignored for token matching. runnableExamples: doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 result = 0 - while start+result < s.len and result < token.len and - toLower(s[result+start]) == toLower(token[result]): inc(result) + while result < s.len and result < token.len and + toLower(s[result]) == toLower(token[result]): inc(result) if result != token.len: result = 0 -proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: set[char]): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -301,9 +299,9 @@ proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = doAssert skipUntil("Hello World", {'W'}, 0) == 6 doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 result = 0 - while start+result < s.len and s[result+start] notin until: inc(result) + while result < s.len and s[result] notin until: inc(result) -proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: char): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -313,24 +311,23 @@ proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = doAssert skipUntil("Hello World", 'W', 0) == 6 doAssert skipUntil("Hello World", 'w', 0) == 11 result = 0 - while start+result < s.len and s[result+start] != until: inc(result) + while result < s.len and s[result] != until: inc(result) -proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = - ## Skips all characters while one char from the set `token` is found. +proc skipWhile*(s: openArray[char], toSkip: set[char]): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. ## Returns number of characters skipped. runnableExamples: doAssert skipWhile("Hello World", {'H', 'e'}) == 2 doAssert skipWhile("Hello World", {'e'}) == 0 doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 result = 0 - while start+result < s.len and s[result+start] in toSkip: inc(result) + while result < s.len and s[result] in toSkip: inc(result) -proc fastSubstr(s: string; token: var string; start, length: int) = +proc fastSubstr(s: openArray[char]; token: var string; length: int) = token.setLen length - for i in 0 ..< length: token[i] = s[i+start] + for i in 0 ..< length: token[i] = s[i] -proc parseUntil*(s: string, token: var string, until: set[char], - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters notin `until`. @@ -342,14 +339,13 @@ proc parseUntil*(s: string, token: var string, until: set[char], doAssert myToken == "Hello " doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 doAssert myToken == "lo " - var i = start + var i = 0 while i < s.len and s[i] notin until: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: char, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: char): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that is not the `until` character. @@ -361,14 +357,13 @@ proc parseUntil*(s: string, token: var string, until: char, doAssert myToken == "Hell" doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 doAssert myToken == "ll" - var i = start + var i = 0 while i < s.len and s[i] != until: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: string, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: string): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that comes before the `until` token. @@ -382,7 +377,7 @@ proc parseUntil*(s: string, token: var string, until: string, if until.len == 0: token.setLen(0) return 0 - var i = start + var i = 0 while i < s.len: if until.len > 0 and s[i] == until[0]: var u = 1 @@ -390,12 +385,11 @@ proc parseUntil*(s: string, token: var string, until: string, inc u if u >= until.len: break inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseWhile*(s: string, token: var string, validChars: set[char], - start = 0): int {.inline.} = +proc parseWhile*(s: openArray[char], token: var string, validChars: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. @@ -405,22 +399,22 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], doAssert myToken.len() == 0 doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 doAssert myToken == "Wor" - var i = start + var i = 0 while i < s.len and s[i] in validChars: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = +proc captureBetween*(s: openArray[char], first: char, second = '\0'): string = ## Finds the first occurrence of ``first``, then returns everything from there ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). runnableExamples: doAssert captureBetween("Hello World", 'e') == "llo World" doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" - doAssert captureBetween("Hello World", 'l', start = 6) == "d" - var i = skipUntil(s, first, start)+1+start + doAssert captureBetween("Hello World".toOpenArray(6, "Hello World".high), 'l') == "d" + var i = skipUntil(s, first) + 1 result = "" - discard s.parseUntil(result, if second == '\0': first else: second, i) + discard parseUntil(s.toOpenArray(i, s.high), result, if second == '\0': first else: second) proc integerOutOfRangeError() {.noinline.} = raise newException(ValueError, "Parsed integer outside of valid range") @@ -429,10 +423,10 @@ proc integerOutOfRangeError() {.noinline.} = when defined(js): {.push overflowChecks: off.} -proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = +proc rawParseInt(s: openArray[char], b: var BiggestInt): int = var sign: BiggestInt = -1 - i = start + i = 0 if i < s.len: if s[i] == '+': inc(i) elif s[i] == '-': @@ -452,30 +446,30 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = integerOutOfRangeError() else: b = b * sign - result = i - start + result = i when defined(js): {.pop.} # overflowChecks: off -proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. +proc parseBiggestInt*(s: openArray[char], number: var BiggestInt): int {. rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: var res: BiggestInt - doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert parseBiggestInt("9223372036854775807", res) == 19 doAssert res == 9223372036854775807 var res = BiggestInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseInt(s, res, start) + result = rawParseInt(s, res) if result != 0: number = res -proc parseInt*(s: string, number: var int, start = 0): int {. +proc parseInt*(s: openArray[char], number: var int): int {. rtl, extern: "npuParseInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -485,14 +479,14 @@ proc parseInt*(s: string, number: var int, start = 0): int {. doAssert parseInt("2019", res, 2) == 2 doAssert res == 19 var res = BiggestInt(0) - result = parseBiggestInt(s, res, start) + result = parseBiggestInt(s, res) when sizeof(int) <= 4: if res < low(int) or res > high(int): integerOutOfRangeError() if result != 0: number = int(res) -proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. +proc parseSaturatedNatural*(s: openArray[char], b: var int): int {. raises: [].} = ## Parses a natural number into ``b``. This cannot raise an overflow ## error. ``high(int)`` is returned for an overflow. @@ -502,7 +496,7 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. var res = 0 discard parseSaturatedNatural("848", res) doAssert res == 848 - var i = start + var i = 0 if i < s.len and s[i] == '+': inc(i) if i < s.len and s[i] in {'0'..'9'}: b = 0 @@ -514,13 +508,13 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. b = high(int) inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored - result = i - start + result = i -proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = +proc rawParseUInt(s: openArray[char], b: var BiggestUInt): int = var res = 0.BiggestUInt prev = 0.BiggestUInt - i = start + i = 0 if i < s.len - 1 and s[i] == '-' and s[i + 1] in {'0'..'9'}: integerOutOfRangeError() if i < s.len and s[i] == '+': inc(i) # Allow @@ -534,11 +528,11 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res - result = i - start + result = i -proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. +proc parseBiggestUInt*(s: openArray[char], number: var BiggestUInt): int {. rtl, extern: "npuParseBiggestUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -550,13 +544,13 @@ proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. var res = BiggestUInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseUInt(s, res, start) + result = rawParseUInt(s, res) if result != 0: number = res -proc parseUInt*(s: string, number: var uint, start = 0): int {. +proc parseUInt*(s: openArray[char], number: var uint): int {. rtl, extern: "npuParseUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -566,22 +560,22 @@ proc parseUInt*(s: string, number: var uint, start = 0): int {. doAssert parseUInt("3450", res, 2) == 2 doAssert res == 50 var res = BiggestUInt(0) - result = parseBiggestUInt(s, res, start) + result = parseBiggestUInt(s, res) when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: if res > 0xFFFF_FFFF'u64: integerOutOfRangeError() if result != 0: number = uint(res) -proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. +proc parseBiggestFloat*(s: openArray[char], number: var BiggestFloat): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if a parsing error ## occurred. -proc parseFloat*(s: string, number: var float, start = 0): int {. +proc parseFloat*(s: openArray[char], number: var float): int {. rtl, extern: "npuParseFloat", noSideEffect.} = - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if there occurred a parsing ## error. runnableExamples: @@ -593,10 +587,75 @@ proc parseFloat*(s: string, number: var float, start = 0): int {. doAssert parseFloat("32.57", res, 3) == 2 doAssert res == 57.00 var bf = BiggestFloat(0.0) - result = parseBiggestFloat(s, bf, start) + result = parseBiggestFloat(s, bf) if result != 0: number = bf +func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c + +func parseSize*(s: openArray[char], size: var int64, alwaysBin=false): int = + ## Parse a size qualified by binary or metric units into `size`. This format + ## is often called "human readable". Result is the number of processed chars + ## or 0 on parse errors and size is rounded to the nearest integer. Trailing + ## garbage like "/s" in "1k/s" is allowed and detected by `result < s.len`. + ## + ## To simplify use, following non-rare wild conventions, and since fractional + ## data like milli-bytes is so rare, unit matching is case-insensitive but for + ## the 'i' distinguishing binary-metric from metric (which cannot be 'I'). + ## + ## An optional trailing 'B|b' is ignored but processed. I.e., you must still + ## know if units are bytes | bits or infer this fact via the case of s[^1] (if + ## users can even be relied upon to use 'B' for byte and 'b' for bit or have + ## that be s[^1]). + ## + ## If `alwaysBin==true` then scales are always binary-metric, but e.g. "KiB" + ## is still accepted for clarity. If the value would exceed the range of + ## `int64`, `size` saturates to `int64.high`. Supported metric prefix chars + ## include k, m, g, t, p, e, z, y (but z & y saturate unless the number is a + ## small fraction). + ## + ## **See also:** + ## * https://en.wikipedia.org/wiki/Binary_prefix + ## * `formatSize module`_ for formatting + runnableExamples: + var res: int64 # caller must still know if 'b' refers to bytes|bits + doAssert parseSize("10.5 MB", res) == 7 + doAssert res == 10_500_000 # decimal metric Mega prefix + doAssert parseSize("64 mib", res) == 6 + doAssert res == 67108864 # 64 shl 20 + doAssert parseSize("1G/h", res, true) == 2 # '/' stops parse + doAssert res == 1073741824 # 1 shl 30, forced binary metric + const prefix = "b" & "kmgtpezy" # byte|bit & lowCase metric-ish prefixes + const scaleM = [1.0, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24] # 10^(3*idx) + const scaleB = [1.0, 1024, 1048576, 1073741824, 1099511627776.0, # 2^(10*idx) + 1125899906842624.0, 1152921504606846976.0, # ldexp? + 1.180591620717411303424e21, 1.208925819614629174706176e24] + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + if result > 0: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = if alwaysBin: scaleB[si] else: scaleM[si] + if result < s.len and s[result] == 'i': + scale = scaleB[si] # Switch from default to binary-metric + inc result + if result < s.len and s[result].toLowerAscii == 'b': + inc result # Skip optional '[bB]' + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = if sizeF > 9223372036854774784.0: int64.high else: sizeF.int64 +# Above constant=2^63-1024 avoids C UB; github.com/nim-lang/Nim/issues/20102 or +# stackoverflow.com/questions/20923556/math-pow2-63-1-math-pow2-63-512-is-true + type InterpolatedKind* = enum ## Describes for `interpolatedFragments` ## which part of the interpolated string is @@ -606,7 +665,7 @@ type ikVar, ## ``var`` part of the interpolated string ikExpr ## ``expr`` part of the interpolated string -iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, +iterator interpolatedFragments*(s: openArray[char]): tuple[kind: InterpolatedKind, value: string] = ## Tokenizes the string `s` into substrings for interpolation purposes. ## @@ -641,7 +700,7 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, else: discard inc j raise newException(ValueError, - "Expected closing '}': " & substr(s, i, s.high)) + "Expected closing '}': " & substr(s.toOpenArray(i, s.high))) inc i, 2 # skip ${ kind = ikExpr elif j+1 < s.len and s[j+1] in IdentStartChars: @@ -655,15 +714,370 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, kind = ikDollar else: raise newException(ValueError, - "Unable to parse a variable name at " & substr(s, i, s.high)) + "Unable to parse a variable name at " & substr(s.toOpenArray(i, s.high))) else: while j < s.len and s[j] != '$': inc j kind = ikStr if j > i: # do not copy the trailing } for ikExpr: - yield (kind, substr(s, i, j-1-ord(kind == ikExpr))) + yield (kind, substr(s.toOpenArray(i, j-1-ord(kind == ikExpr)))) else: break i = j {.pop.} + + +proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a binary number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-bin character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseBin("0100_1110_0110_1001_1110_1101", num) == 29 + doAssert num == 5138925 + doAssert parseBin("3", num) == 0 + var num8: int8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8) == 32 + doAssert num8 == 0b1110_1101'i8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8, 3, 9) == 9 + doAssert num8 == 0b0100_1110'i8 + var num8u: uint8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8u) == 32 + doAssert num8u == 237 + var num64: int64 + doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 + doAssert num64 == 336784608873 + parseBin(s.toOpenArray(start, s.high), number, maxLen) + +proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses an octal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-oct character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseOct("0o23464755", num) == 10 + doAssert num == 5138925 + doAssert parseOct("8", num) == 0 + var num8: int8 + doAssert parseOct("0o_1464_755", num8) == 11 + doAssert num8 == -19 + doAssert parseOct("0o_1464_755", num8, 3, 3) == 3 + doAssert num8 == 102 + var num8u: uint8 + doAssert parseOct("1464755", num8u) == 7 + doAssert num8u == 237 + var num64: int64 + doAssert parseOct("2346475523464755", num64) == 16 + doAssert num64 == 86216859871725 + parseOct(s.toOpenArray(start, s.high), number, maxLen) + +proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a hexadecimal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-hex character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseHex("4E_69_ED", num) == 8 + doAssert num == 5138925 + doAssert parseHex("X", num) == 0 + doAssert parseHex("#ABC", num) == 4 + var num8: int8 + doAssert parseHex("0x_4E_69_ED", num8) == 11 + doAssert num8 == 0xED'i8 + doAssert parseHex("0x_4E_69_ED", num8, 3, 2) == 2 + doAssert num8 == 0x4E'i8 + var num8u: uint8 + doAssert parseHex("0x_4E_69_ED", num8u) == 11 + doAssert num8u == 237 + var num64: int64 + doAssert parseHex("4E69ED4E69ED", num64) == 12 + doAssert num64 == 86216859871725 + parseHex(s.toOpenArray(start, s.high), number, maxLen) + +proc parseIdent*(s: string, ident: var string, start = 0): int = + ## Parses an identifier and stores it in ``ident``. Returns + ## the number of the parsed characters or 0 in case of an error. + ## If error, the value of `ident` is not changed. + runnableExamples: + var res: string + doAssert parseIdent("Hello World", res, 0) == 5 + doAssert res == "Hello" + doAssert parseIdent("Hello World", res, 1) == 4 + doAssert res == "ello" + doAssert parseIdent("Hello World", res, 6) == 5 + doAssert res == "World" + parseIdent(s.toOpenArray(start, s.high), ident) + +proc parseIdent*(s: string, start = 0): string = + ## Parses an identifier and returns it or an empty string in + ## case of an error. + runnableExamples: + doAssert parseIdent("Hello World", 0) == "Hello" + doAssert parseIdent("Hello World", 1) == "ello" + doAssert parseIdent("Hello World", 5) == "" + doAssert parseIdent("Hello World", 6) == "World" + parseIdent(s.toOpenArray(start, s.high)) + +proc parseChar*(s: string, c: var char, start = 0): int = + ## Parses a single character, stores it in `c` and returns 1. + ## In case of error (if start >= s.len) it returns 0 + ## and the value of `c` is unchanged. + runnableExamples: + var c: char + doAssert "nim".parseChar(c, 3) == 0 + doAssert c == '\0' + doAssert "nim".parseChar(c, 0) == 1 + doAssert c == 'n' + parseChar(s.toOpenArray(start, s.high), c) + +proc skipWhitespace*(s: string, start = 0): int {.inline.} = + ## Skips the whitespace starting at ``s[start]``. Returns the number of + ## skipped characters. + runnableExamples: + doAssert skipWhitespace("Hello World", 0) == 0 + doAssert skipWhitespace(" Hello World", 0) == 1 + doAssert skipWhitespace("Hello World", 5) == 1 + doAssert skipWhitespace("Hello World", 5) == 2 + skipWhitespace(s.toOpenArray(start, s.high)) + +proc skip*(s, token: string, start = 0): int {.inline.} = + ## Skips the `token` starting at ``s[start]``. Returns the length of `token` + ## or 0 if there was no `token` at ``s[start]``. + runnableExamples: + doAssert skip("2019-01-22", "2019", 0) == 4 + doAssert skip("2019-01-22", "19", 0) == 0 + doAssert skip("2019-01-22", "19", 2) == 2 + doAssert skip("CAPlow", "CAP", 0) == 3 + doAssert skip("CAPlow", "cap", 0) == 0 + skip(s.toOpenArray(start, s.high), token) + +proc skipIgnoreCase*(s, token: string, start = 0): int = + ## Same as `skip` but case is ignored for token matching. + runnableExamples: + doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 + doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 + skipIgnoreCase(s.toOpenArray(start, s.high), token) + +proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = + ## Skips all characters until one char from the set `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 + doAssert skipUntil("Hello World", {'W'}, 0) == 6 + doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = + ## Skips all characters until the char `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", 'o', 0) == 4 + doAssert skipUntil("Hello World", 'o', 4) == 0 + doAssert skipUntil("Hello World", 'W', 0) == 6 + doAssert skipUntil("Hello World", 'w', 0) == 11 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipWhile("Hello World", {'H', 'e'}) == 2 + doAssert skipWhile("Hello World", {'e'}) == 0 + doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 + skipWhile(s.toOpenArray(start, s.high), toSkip) + +proc parseUntil*(s: string, token: var string, until: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters notin `until`. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, {'W', 'o', 'r'}) == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, {'W', 'r'}) == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 + doAssert myToken == "lo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: char, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that is not the `until` character. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, 'W') == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, 'o') == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 + doAssert myToken == "ll" + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: string, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, "Wor") == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 + doAssert myToken == "llo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseWhile*(s: string, token: var string, validChars: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters in `validChars`. + runnableExamples: + var myToken: string + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 0) == 0 + doAssert myToken.len() == 0 + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 + doAssert myToken == "Wor" + parseWhile(s.toOpenArray(start, s.high), token, validChars) + +proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = + ## Finds the first occurrence of ``first``, then returns everything from there + ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). + runnableExamples: + doAssert captureBetween("Hello World", 'e') == "llo World" + doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" + doAssert captureBetween("Hello World", 'l', start = 6) == "d" + captureBetween(s.toOpenArray(start, s.high), first, second) + +proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestInt + doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert res == 9223372036854775807 + parseBiggestInt(s.toOpenArray(start, s.high), number) + +proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: int + doAssert parseInt("2019", res, 0) == 4 + doAssert res == 2019 + doAssert parseInt("2019", res, 2) == 2 + doAssert res == 19 + parseInt(s.toOpenArray(start, s.high), number) + + +proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. + raises: [].} = + ## Parses a natural number into ``b``. This cannot raise an overflow + ## error. ``high(int)`` is returned for an overflow. + ## The number of processed character is returned. + ## This is usually what you really want to use instead of `parseInt`:idx:. + runnableExamples: + var res = 0 + discard parseSaturatedNatural("848", res) + doAssert res == 848 + parseSaturatedNatural(s.toOpenArray(start, s.high), b) + + +proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestUInt + doAssert parseBiggestUInt("12", res, 0) == 2 + doAssert res == 12 + doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 + doAssert res == 1111111111111111111'u64 + parseBiggestUInt(s.toOpenArray(start, s.high), number) + +proc parseUInt*(s: string, number: var uint, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: uint + doAssert parseUInt("3450", res) == 4 + doAssert res == 3450 + doAssert parseUInt("3450", res, 2) == 2 + doAssert res == 50 + parseUInt(s.toOpenArray(start, s.high), number) + +proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if a parsing error + ## occurred. + parseFloat(s.toOpenArray(start, s.high), number) + +proc parseFloat*(s: string, number: var float, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there occurred a parsing + ## error. + runnableExamples: + var res: float + doAssert parseFloat("32", res, 0) == 2 + doAssert res == 32.0 + doAssert parseFloat("32.57", res, 0) == 5 + doAssert res == 32.57 + doAssert parseFloat("32.57", res, 3) == 2 + doAssert res == 57.00 + parseFloat(s.toOpenArray(start, s.high), number) + +iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, + value: string] = + ## Tokenizes the string `s` into substrings for interpolation purposes. + ## + runnableExamples: + var outp: seq[tuple[kind: InterpolatedKind, value: string]] + for k, v in interpolatedFragments(" $this is ${an example} $$"): + outp.add (k, v) + doAssert outp == @[(ikStr, " "), + (ikVar, "this"), + (ikStr, " is "), + (ikExpr, "an example"), + (ikStr, " "), + (ikDollar, "$")] + for x in s.toOa.interpolatedFragments: + yield x + diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 79a9cc7307..c760799a20 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -36,43 +36,43 @@ The file ``examples/htmltitle.nim`` demonstrates how to use the XML parser to accomplish a simple task: To determine the title of an HTML document. -.. code-block:: nim + ```nim + # Example program to show the parsexml module + # This program reads an HTML file and writes its title to stdout. + # Errors and whitespace are ignored. - # Example program to show the parsexml module - # This program reads an HTML file and writes its title to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + if paramCount() < 1: + quit("Usage: htmltitle filename[.html]") - if paramCount() < 1: - quit("Usage: htmltitle filename[.html]") + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + while true: + x.next() + case x.kind + of xmlElementStart: + if cmpIgnoreCase(x.elementName, "title") == 0: + var title = "" + x.next() # skip "" + while x.kind == xmlCharData: + title.add(x.charData) + x.next() + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: + echo("Title: " & title) + quit(0) # Success! + else: + echo(x.errorMsgExpected("/title")) - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - while true: - x.next() - case x.kind - of xmlElementStart: - if cmpIgnoreCase(x.elementName, "title") == 0: - var title = "" - x.next() # skip "<title>" - while x.kind == xmlCharData: - title.add(x.charData) - x.next() - if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: - echo("Title: " & title) - quit(0) # Success! - else: - echo(x.errorMsgExpected("/title")) + of xmlEof: break # end of file reached + else: discard # ignore other events - of xmlEof: break # end of file reached - else: discard # ignore other events - - x.close() - quit("Could not determine title!") + x.close() + quit("Could not determine title!") + ``` ]## @@ -85,69 +85,72 @@ The file ``examples/htmlrefs.nim`` demonstrates how to use the XML parser to accomplish another simple task: To determine all the links an HTML document contains. -.. code-block:: nim + ```nim + # Example program to show the new parsexml module + # This program reads an HTML file and writes all its used links to stdout. + # Errors and whitespace are ignored. - # Example program to show the new parsexml module - # This program reads an HTML file and writes all its used links to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + proc `=?=` (a, b: string): bool = + # little trick: define our own comparator that ignores case + return cmpIgnoreCase(a, b) == 0 - proc `=?=` (a, b: string): bool = - # little trick: define our own comparator that ignores case - return cmpIgnoreCase(a, b) == 0 + if paramCount() < 1: + quit("Usage: htmlrefs filename[.html]") - if paramCount() < 1: - quit("Usage: htmlrefs filename[.html]") - - var links = 0 # count the number of links - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - next(x) # get first event - block mainLoop: - while true: - case x.kind - of xmlElementOpen: - # the <a href = "xyz"> tag we are interested in always has an attribute, - # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` - if x.elementName =?= "a": - x.next() - if x.kind == xmlAttribute: - if x.attrKey =?= "href": - var link = x.attrValue - inc(links) - # skip until we have an ``xmlElementClose`` event - while true: - x.next() - case x.kind - of xmlEof: break mainLoop - of xmlElementClose: break - else: discard - x.next() # skip ``xmlElementClose`` - # now we have the description for the ``a`` element - var desc = "" - while x.kind == xmlCharData: - desc.add(x.charData) - x.next() - echo(desc & ": " & link) - else: - x.next() - of xmlEof: break # end of file reached - of xmlError: - echo(errorMsg(x)) + var links = 0 # count the number of links + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + next(x) # get first event + block mainLoop: + while true: + case x.kind + of xmlElementOpen: + # the <a href = "xyz"> tag we are interested in always has an attribute, + # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` + if x.elementName =?= "a": x.next() - else: x.next() # skip other events + if x.kind == xmlAttribute: + if x.attrKey =?= "href": + var link = x.attrValue + inc(links) + # skip until we have an ``xmlElementClose`` event + while true: + x.next() + case x.kind + of xmlEof: break mainLoop + of xmlElementClose: break + else: discard + x.next() # skip ``xmlElementClose`` + # now we have the description for the ``a`` element + var desc = "" + while x.kind == xmlCharData: + desc.add(x.charData) + x.next() + echo(desc & ": " & link) + else: + x.next() + of xmlEof: break # end of file reached + of xmlError: + echo(errorMsg(x)) + x.next() + else: x.next() # skip other events - echo($links & " link(s) found!") - x.close() + echo($links & " link(s) found!") + x.close() + ``` ]## import - strutils, lexbase, streams, unicode + std/[strutils, lexbase, streams, unicode] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] # the parser treats ``<br />`` as ``<br></br>`` @@ -789,7 +792,7 @@ proc next*(my: var XmlParser) = my.state = stateNormal when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: XmlParser diff --git a/lib/pure/pathnorm.nim b/lib/pure/pathnorm.nim index 10a2a0b679..4cdc023037 100644 --- a/lib/pure/pathnorm.nim +++ b/lib/pure/pathnorm.nim @@ -14,7 +14,7 @@ # Yes, this uses import here, not include so that # we don't end up exporting these symbols from pathnorm and os: -import includes/osseps +import std/private/osseps type PathIter* = object @@ -29,10 +29,6 @@ proc next*(it: var PathIter; x: string): (int, int) = if not it.notFirst and x[it.i] in {DirSep, AltSep}: # absolute path: inc it.i - when doslikeFileSystem: # UNC paths have leading `\\` - if hasNext(it, x) and x[it.i] == DirSep and - it.i+1 < x.len and x[it.i+1] != DirSep: - inc it.i else: while it.i < x.len and x[it.i] notin {DirSep, AltSep}: inc it.i if it.i > it.prev: @@ -56,10 +52,23 @@ proc isDotDot(x: string; bounds: (int, int)): bool = proc isSlash(x: string; bounds: (int, int)): bool = bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep} +when doslikeFileSystem: + import std/private/ntpath + proc addNormalizePath*(x: string; result: var string; state: var int; dirSep = DirSep) = ## Low level proc. Undocumented. + when doslikeFileSystem: # Add Windows drive at start without normalization + var x = x + if result == "": + let (drive, file) = splitDrive(x) + x = file + result.add drive + for c in result.mitems: + if c in {DirSep, AltSep}: + c = dirSep + # state: 0th bit set if isAbsolute path. Other bits count # the number of path components. var it: PathIter diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 06a77af577..ca2e800212 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -16,15 +16,17 @@ ## include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] const useUnicode = true ## change this to deactivate proper UTF-8 support -import strutils, macros +import std/[strutils, macros] import std/private/decode_helpers when useUnicode: - import unicode + import std/unicode export unicode.`==` const @@ -83,30 +85,30 @@ type of pkChar, pkGreedyRepChar: ch: char of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal - of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] + of pkBackRef..pkBackRefIgnoreStyle: index: range[-MaxSubpatterns..MaxSubpatterns-1] else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj -proc kind*(p: Peg): PegKind = p.kind +func kind*(p: Peg): PegKind = p.kind ## Returns the *PegKind* of a given *Peg* object. -proc term*(p: Peg): string = p.term +func term*(p: Peg): string = p.term ## Returns the *string* representation of a given *Peg* variant object ## where present. -proc ch*(p: Peg): char = p.ch +func ch*(p: Peg): char = p.ch ## Returns the *char* representation of a given *Peg* variant object ## where present. -proc charChoice*(p: Peg): ref set[char] = p.charChoice +func charChoice*(p: Peg): ref set[char] = p.charChoice ## Returns the *charChoice* field of a given *Peg* variant object ## where present. -proc nt*(p: Peg): NonTerminal = p.nt +func nt*(p: Peg): NonTerminal = p.nt ## Returns the *NonTerminal* object of a given *Peg* variant object ## where present. -proc index*(p: Peg): range[0..MaxSubpatterns] = p.index +func index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index ## Returns the back-reference index of a captured sub-pattern in the ## *Captures* object for a given *Peg* variant object where present. @@ -120,59 +122,60 @@ iterator pairs*(p: Peg): (int, Peg) {.inline.} = for i in 0 ..< p.sons.len: yield (i, p.sons[i]) -proc name*(nt: NonTerminal): string = nt.name +func name*(nt: NonTerminal): string = nt.name ## Gets the name of the symbol represented by the parent *Peg* object variant ## of a given *NonTerminal*. -proc line*(nt: NonTerminal): int = nt.line +func line*(nt: NonTerminal): int = nt.line ## Gets the line number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc col*(nt: NonTerminal): int = nt.col +func col*(nt: NonTerminal): int = nt.col ## Gets the column number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags +func flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags ## Gets the *NonTerminalFlag*-typed flags field of the parent *Peg* variant ## object of a given *NonTerminal*. -proc rule*(nt: NonTerminal): Peg = nt.rule +func rule*(nt: NonTerminal): Peg = nt.rule ## Gets the *Peg* object representing the rule definition of the parent *Peg* ## object variant of a given *NonTerminal*. -proc term*(t: string): Peg {.noSideEffect, rtl, extern: "npegs$1Str".} = +func term*(t: string): Peg {.rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string if t.len != 1: result = Peg(kind: pkTerminal, term: t) else: result = Peg(kind: pkChar, ch: t[0]) -proc termIgnoreCase*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreCase*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore case for matching result = Peg(kind: pkTerminalIgnoreCase, term: t) -proc termIgnoreStyle*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreStyle*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore style for matching result = Peg(kind: pkTerminalIgnoreStyle, term: t) -proc term*(t: char): Peg {.noSideEffect, rtl, extern: "npegs$1Char".} = +func term*(t: char): Peg {.rtl, extern: "npegs$1Char".} = ## constructs a PEG from a terminal char assert t != '\0' result = Peg(kind: pkChar, ch: t) -proc charSet*(s: set[char]): Peg {.noSideEffect, rtl, extern: "npegs$1".} = +func charSet*(s: set[char]): Peg {.rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s result = Peg(kind: pkCharChoice) - new(result.charChoice) - result.charChoice[] = s + {.cast(noSideEffect).}: + new(result.charChoice) + result.charChoice[] = s -proc len(a: Peg): int {.inline.} = return a.sons.len -proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) +func len(a: Peg): int {.inline.} = return a.sons.len +func add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) -proc addChoice(dest: var Peg, elem: Peg) = +func addChoice(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkCharChoice: # caution! Do not introduce false aliasing here! @@ -195,12 +198,12 @@ template multipleOp(k: PegKind, localOpt: untyped) = if result.len == 1: result = result.sons[0] -proc `/`*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegsOrderedChoice".} = +func `/`*(a: varargs[Peg]): Peg {. + rtl, extern: "npegsOrderedChoice".} = ## constructs an ordered choice with the PEGs in `a` multipleOp(pkOrderedChoice, addChoice) -proc addSequence(dest: var Peg, elem: Peg) = +func addSequence(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkTerminal: # caution! Do not introduce false aliasing here! @@ -212,12 +215,12 @@ proc addSequence(dest: var Peg, elem: Peg) = else: add(dest, elem) else: add(dest, elem) -proc sequence*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func sequence*(a: varargs[Peg]): Peg {. + rtl, extern: "npegs$1".} = ## constructs a sequence with all the PEGs from `a` multipleOp(pkSequence, addSequence) -proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = +func `?`*(a: Peg): Peg {.rtl, extern: "npegsOptional".} = ## constructs an optional for the PEG `a` if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, pkGreedyRepSet}: @@ -227,7 +230,7 @@ proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = else: result = Peg(kind: pkOption, sons: @[a]) -proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = +func `*`*(a: Peg): Peg {.rtl, extern: "npegsGreedyRep".} = ## constructs a "greedy repetition" for the PEG `a` case a.kind of pkGreedyRep, pkGreedyRepChar, pkGreedyRepSet, pkGreedyAny, pkOption: @@ -242,96 +245,99 @@ proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = else: result = Peg(kind: pkGreedyRep, sons: @[a]) -proc `!*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsSearch".} = +func `!*`*(a: Peg): Peg {.rtl, extern: "npegsSearch".} = ## constructs a "search" for the PEG `a` result = Peg(kind: pkSearch, sons: @[a]) -proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, +func `!*\`*(a: Peg): Peg {.rtl, extern: "npgegsCapturedSearch".} = ## constructs a "captured search" for the PEG `a` result = Peg(kind: pkCapturedSearch, sons: @[a]) -proc `+`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyPosRep".} = +func `+`*(a: Peg): Peg {.rtl, extern: "npegsGreedyPosRep".} = ## constructs a "greedy positive repetition" with the PEG `a` return sequence(a, *a) -proc `&`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsAndPredicate".} = +func `&`*(a: Peg): Peg {.rtl, extern: "npegsAndPredicate".} = ## constructs an "and predicate" with the PEG `a` result = Peg(kind: pkAndPredicate, sons: @[a]) -proc `!`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsNotPredicate".} = +func `!`*(a: Peg): Peg {.rtl, extern: "npegsNotPredicate".} = ## constructs a "not predicate" with the PEG `a` result = Peg(kind: pkNotPredicate, sons: @[a]) -proc any*: Peg {.inline.} = +func any*: Peg {.inline.} = ## constructs the PEG `any character`:idx: (``.``) result = Peg(kind: pkAny) -proc anyRune*: Peg {.inline.} = +func anyRune*: Peg {.inline.} = ## constructs the PEG `any rune`:idx: (``_``) result = Peg(kind: pkAnyRune) -proc newLine*: Peg {.inline.} = +func newLine*: Peg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) result = Peg(kind: pkNewLine) -proc unicodeLetter*: Peg {.inline.} = +func unicodeLetter*: Peg {.inline.} = ## constructs the PEG ``\letter`` which matches any Unicode letter. result = Peg(kind: pkLetter) -proc unicodeLower*: Peg {.inline.} = +func unicodeLower*: Peg {.inline.} = ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. result = Peg(kind: pkLower) -proc unicodeUpper*: Peg {.inline.} = +func unicodeUpper*: Peg {.inline.} = ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. result = Peg(kind: pkUpper) -proc unicodeTitle*: Peg {.inline.} = +func unicodeTitle*: Peg {.inline.} = ## constructs the PEG ``\title`` which matches any Unicode title letter. result = Peg(kind: pkTitle) -proc unicodeWhitespace*: Peg {.inline.} = +func unicodeWhitespace*: Peg {.inline.} = ## constructs the PEG ``\white`` which matches any Unicode ## whitespace character. result = Peg(kind: pkWhitespace) -proc startAnchor*: Peg {.inline.} = +func startAnchor*: Peg {.inline.} = ## constructs the PEG ``^`` which matches the start of the input. result = Peg(kind: pkStartAnchor) -proc endAnchor*: Peg {.inline.} = +func endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = +func capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. - result = Peg(kind: pkBackRef, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. + result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores case for matching. - result = Peg(kind: pkBackRefIgnoreCase, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores case for matching. + result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores style for matching. - result = Peg(kind: pkBackRefIgnoreStyle, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores style for matching. + result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) -proc spaceCost(n: Peg): int = +func spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, - pkAny..pkWhitespace, pkGreedyAny: + pkAny..pkWhitespace, pkGreedyAny, pkBackRef..pkBackRefIgnoreStyle: result = 1 of pkNonTerminal: # we cannot inline a rule with a non-terminal @@ -341,8 +347,8 @@ proc spaceCost(n: Peg): int = inc(result, spaceCost(n.sons[i])) if result >= InlineThreshold: break -proc nonterminal*(n: NonTerminal): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func nonterminal*(n: NonTerminal): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG that consists of the nonterminal symbol assert n != nil if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: @@ -351,8 +357,8 @@ proc nonterminal*(n: NonTerminal): Peg {. else: result = Peg(kind: pkNonTerminal, nt: n) -proc newNonTerminal*(name: string, line, column: int): NonTerminal {. - noSideEffect, rtl, extern: "npegs$1".} = +func newNonTerminal*(name: string, line, column: int): NonTerminal {. + rtl, extern: "npegs$1".} = ## constructs a nonterminal symbol result = NonTerminal(name: name, line: line, col: column) @@ -387,7 +393,7 @@ template natural*: Peg = # ------------------------- debugging ----------------------------------------- -proc esc(c: char, reserved = {'\0'..'\255'}): string = +func esc(c: char, reserved = {'\0'..'\255'}): string = case c of '\b': result = "\\b" of '\t': result = "\\t" @@ -403,14 +409,14 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string = elif c in reserved: result = '\\' & c else: result = $c -proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" +func singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" -proc singleQuoteEsc(str: string): string = +func singleQuoteEsc(str: string): string = result = "'" for c in items(str): add result, esc(c, {'\''}) add result, '\'' -proc charSetEscAux(cc: set[char]): string = +func charSetEscAux(cc: set[char]): string = const reserved = {'^', '-', ']'} result = "" var c1 = 0 @@ -427,13 +433,13 @@ proc charSetEscAux(cc: set[char]): string = c1 = c2 inc(c1) -proc charSetEsc(cc: set[char]): string = +func charSetEsc(cc: set[char]): string = if card(cc) >= 128+64: result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' else: result = '[' & charSetEscAux(cc) & ']' -proc toStrAux(r: Peg, res: var string) = +func toStrAux(r: Peg, res: var string) = case r.kind of pkEmpty: add(res, "()") of pkAny: add(res, '.') @@ -519,7 +525,7 @@ proc toStrAux(r: Peg, res: var string) = of pkStartAnchor: add(res, '^') -proc `$` *(r: Peg): string {.noSideEffect, rtl, extern: "npegsToString".} = +func `$` *(r: Peg): string {.rtl, extern: "npegsToString".} = ## converts a PEG to its string representation result = "" toStrAux(r, result) @@ -532,7 +538,7 @@ type ml: int origStart: int -proc bounds*(c: Captures, +func bounds*(c: Captures, i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = ## returns the bounds ``[first..last]`` of the `i`'th capture. result = c.matches[i] @@ -545,24 +551,26 @@ when not useUnicode: inc(i) template runeLenAt(s, i): untyped = 1 - proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} - proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} - proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} - proc isTitle(a: char): bool {.inline.} = return false - proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} + func isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} + func isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} + func isLower(a: char): bool {.inline.} = return a in {'a'..'z'} + func isTitle(a: char): bool {.inline.} = return false + func isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} template matchOrParse(mopProc: untyped) = # Used to make the main matcher proc *rawMatch* as well as event parser # procs. For the former, *enter* and *leave* event handler code generators # are provided which just return *discard*. - proc mopProc(s: string, p: Peg, start: int, c: var Captures): int = + proc mopProc(s: string, p: Peg, start: int, c: var Captures): int {.gcsafe, raises: [].} = proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int = # Parse handler code must run in an *of* clause of its own for each # *PegKind*, so we encapsulate the identical clause body for # *pkBackRef..pkBackRefIgnoreStyle* here. - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] + var index = p.index + if index < 0: index.inc(c.ml) + if index < 0 or index >= c.ml: return -1 + var (a, b) = c.matches[index] var n: Peg case p.kind of pkBackRef: @@ -822,15 +830,22 @@ template matchOrParse(mopProc: untyped) = leave(pkNotPredicate, s, p, start, result) of pkCapture: enter(pkCapture, s, p, start) - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = mopProc(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture + if p.sons.len == 0 or p.sons[0].kind == pkEmpty: + # empty capture removes last match + dec(c.ml) + c.matches[c.ml] = (0, 0) + result = 0 # match of length 0 else: - c.ml = idx + var idx = c.ml # reserve a slot for the subpattern + result = mopProc(s, p.sons[0], start, c) + if result >= 0: + if idx < MaxSubpatterns: + if idx != c.ml: + for i in countdown(c.ml, idx): + c.matches[i+1] = c.matches[i] + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture + inc(c.ml) leave(pkCapture, s, p, start, result) of pkBackRef: enter(pkBackRef, s, p, start) @@ -851,8 +866,8 @@ template matchOrParse(mopProc: untyped) = leave(pkStartAnchor, s, p, start, result) of pkRule, pkList: assert false -proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int - {.noSideEffect, rtl, extern: "npegs$1".} = +func rawMatch*(s: string, p: Peg, start: int, c: var Captures): int + {.rtl, extern: "npegs$1".} = ## low-level matching proc that implements the PEG interpreter. Use this ## for maximum efficiency (every other PEG operation ends up calling this ## proc). @@ -864,13 +879,17 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int template leave(pk, s, p, start, length) = discard matchOrParse(matchIt) - result = matchIt(s, p, start, c) + {.cast(noSideEffect).}: + # This cast is allowed because the `matchOrParse` template is used for + # both matching and parsing, but side effects are only possible when it's + # used by `eventParser`. + result = matchIt(s, p, start, c) macro mkHandlerTplts(handlers: untyped): untyped = # Transforms the handler spec in *handlers* into handler templates. # The AST structure of *handlers[0]*: # - # .. code-block:: + # ``` # StmtList # Call # Ident "pkNonTerminal" @@ -891,7 +910,8 @@ macro mkHandlerTplts(handlers: untyped): untyped = # StmtList # <handler code block> # ... - proc mkEnter(hdName, body: NimNode): NimNode = + # ``` + func mkEnter(hdName, body: NimNode): NimNode = template helper(hdName, body) {.dirty.} = template hdName(s, p, start) = let s {.inject.} = s @@ -940,60 +960,61 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## match, else the length of the total match. The following example code ## evaluates an arithmetic expression defined by a simple PEG: ## - ## .. code-block:: nim - ## import std/[strutils, pegs] + ## ```nim + ## import std/[strutils, pegs] ## - ## let - ## pegAst = """ - ## Expr <- Sum - ## Sum <- Product (('+' / '-')Product)* - ## Product <- Value (('*' / '/')Value)* - ## Value <- [0-9]+ / '(' Expr ')' - ## """.peg - ## txt = "(5+3)/2-7*22" + ## let + ## pegAst = """ + ## Expr <- Sum + ## Sum <- Product (('+' / '-')Product)* + ## Product <- Value (('*' / '/')Value)* + ## Value <- [0-9]+ / '(' Expr ')' + ## """.peg + ## txt = "(5+3)/2-7*22" ## - ## var - ## pStack: seq[string] = @[] - ## valStack: seq[float] = @[] - ## opStack = "" - ## let - ## parseArithExpr = pegAst.eventParser: - ## pkNonTerminal: - ## enter: - ## pStack.add p.nt.name - ## leave: - ## pStack.setLen pStack.high - ## if length > 0: - ## let matchStr = s.substr(start, start+length-1) - ## case p.nt.name - ## of "Value": - ## try: - ## valStack.add matchStr.parseFloat - ## echo valStack - ## except ValueError: - ## discard - ## of "Sum", "Product": - ## try: - ## let val = matchStr.parseFloat - ## except ValueError: - ## if valStack.len > 1 and opStack.len > 0: - ## valStack[^2] = case opStack[^1] - ## of '+': valStack[^2] + valStack[^1] - ## of '-': valStack[^2] - valStack[^1] - ## of '*': valStack[^2] * valStack[^1] - ## else: valStack[^2] / valStack[^1] - ## valStack.setLen valStack.high - ## echo valStack - ## opStack.setLen opStack.high - ## echo opStack - ## pkChar: - ## leave: - ## if length == 1 and "Value" != pStack[^1]: - ## let matchChar = s[start] - ## opStack.add matchChar - ## echo opStack + ## var + ## pStack: seq[string] = @[] + ## valStack: seq[float] = @[] + ## opStack = "" + ## let + ## parseArithExpr = pegAst.eventParser: + ## pkNonTerminal: + ## enter: + ## pStack.add p.nt.name + ## leave: + ## pStack.setLen pStack.high + ## if length > 0: + ## let matchStr = s.substr(start, start+length-1) + ## case p.nt.name + ## of "Value": + ## try: + ## valStack.add matchStr.parseFloat + ## echo valStack + ## except ValueError: + ## discard + ## of "Sum", "Product": + ## try: + ## let val = matchStr.parseFloat + ## except ValueError: + ## if valStack.len > 1 and opStack.len > 0: + ## valStack[^2] = case opStack[^1] + ## of '+': valStack[^2] + valStack[^1] + ## of '-': valStack[^2] - valStack[^1] + ## of '*': valStack[^2] * valStack[^1] + ## else: valStack[^2] / valStack[^1] + ## valStack.setLen valStack.high + ## echo valStack + ## opStack.setLen opStack.high + ## echo opStack + ## pkChar: + ## leave: + ## if length == 1 and "Value" != pStack[^1]: + ## let matchChar = s[start] + ## opStack.add matchChar + ## echo opStack ## - ## let pLen = parseArithExpr(txt) + ## let pLen = parseArithExpr(txt) + ## ``` ## ## The *handlers* parameter consists of code blocks for *PegKinds*, ## which define the grammar elements of interest. Each block can contain @@ -1008,7 +1029,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## Symbols declared in an *enter* handler can be made visible in the ## corresponding *leave* handler by annotating them with an *inject* pragma. proc rawParse(s: string, p: Peg, start: int, c: var Captures): int - {.genSym.} = + {.gensym.} = # binding from *macros* bind strVal @@ -1023,7 +1044,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # by *mkHandlerTplts*. template mkDoEnter(hdPostf, s, pegNode, start) = when declared(`enter hdPostf`): - `enter hdPostf`(s, pegNode, start): + `enter hdPostf`(s, pegNode, start) else: discard let hdPostf = ident(substr(strVal(pegKind), 2)) @@ -1034,7 +1055,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # a grammar element of kind *pegKind*. template mkDoLeave(hdPostf, s, pegNode, start, length) = when declared(`leave hdPostf`): - `leave hdPostf`(s, pegNode, start, length): + `leave hdPostf`(s, pegNode, start, length) else: discard let hdPostf = ident(substr(strVal(pegKind), 2)) @@ -1043,7 +1064,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = matchOrParse(parseIt) parseIt(s, p, start, c) - proc parser(s: string): int {.genSym.} = + proc parser(s: string): int {.gensym.} = # the proc to be returned var ms: array[MaxSubpatterns, (int, int)] @@ -1060,8 +1081,8 @@ template fillMatches(s, caps, c) = else: caps[k] = "" -proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func matchLen*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1071,8 +1092,8 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], result = rawMatch(s, pattern, start, c) if result >= 0: fillMatches(s, matches, c) -proc matchLen*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func matchLen*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1081,22 +1102,22 @@ proc matchLen*(s: string, pattern: Peg, c.origStart = start result = rawMatch(s, pattern, start, c) -proc match*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func match*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and ## the captured substrings in the array ``matches``. If it does not ## match, nothing is written into ``matches`` and ``false`` is ## returned. result = matchLen(s, pattern, matches, start) != -1 -proc match*(s: string, pattern: Peg, - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1".} = +func match*(s: string, pattern: Peg, + start = 0): bool {.rtl, extern: "npegs$1".} = ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. result = matchLen(s, pattern, start) != -1 -proc find*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func find*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## returns the starting position of ``pattern`` in ``s`` and the captured ## substrings in the array ``matches``. If it does not match, nothing ## is written into ``matches`` and -1 is returned. @@ -1110,9 +1131,9 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string], return -1 # could also use the pattern here: (!P .)* P -proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], +func findBounds*(s: string, pattern: Peg, matches: var openArray[string], start = 0): tuple[first, last: int] {. - noSideEffect, rtl, extern: "npegs$1Capture".} = + rtl, extern: "npegs$1Capture".} = ## returns the starting position and end position of ``pattern`` in ``s`` ## and the captured ## substrings in the array ``matches``. If it does not match, nothing @@ -1127,8 +1148,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], return (i, i+L-1) return (-1, 0) -proc find*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func find*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not ## match, -1 is returned. var c: Captures @@ -1151,10 +1172,10 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = yield substr(s, i, i+L-1) inc(i, L) -proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func findAll*(s: string, pattern: Peg, start = 0): seq[string] {. + rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. - ## If it does not match, @[] is returned. + ## If it does not match, `@[]` is returned. result = @[] for it in findAll(s, pattern, start): result.add it @@ -1162,8 +1183,7 @@ template `=~`*(s: string, pattern: Peg): bool = ## This calls ``match`` with an implicit declared ``matches`` array that ## can be used in the scope of the ``=~`` call: ## - ## .. code-block:: nim - ## + ## ```nim ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": ## # matches a key=value pair: ## echo("Key: ", matches[0]) @@ -1175,50 +1195,51 @@ template `=~`*(s: string, pattern: Peg): bool = ## echo("comment: ", matches[0]) ## else: ## echo("syntax error") - ## + ## ``` bind MaxSubpatterns when not declaredInScope(matches): - var matches {.inject.}: array[0..MaxSubpatterns-1, string] + var matches {.inject.} = default(array[0..MaxSubpatterns-1, string]) match(s, pattern, matches) # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func contains*(s: string, pattern: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## same as ``find(s, pattern, start) >= 0`` return find(s, pattern, start) >= 0 -proc contains*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func contains*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## same as ``find(s, pattern, matches, start) >= 0`` return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func startsWith*(s: string, prefix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix, start) >= 0 -proc endsWith*(s: string, suffix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func endsWith*(s: string, suffix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` ends with the pattern `suffix` var c: Captures c.origStart = start for i in start .. s.len-1: if rawMatch(s, suffix, i, c) == s.len - i: return true -proc replacef*(s: string, sub: Peg, by: string): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replacef*(s: string, sub: Peg, by: string): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## - ## .. code-block:: nim + ## ```nim ## "var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1<-keykey; val2<-key2key2" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1235,8 +1256,8 @@ proc replacef*(s: string, sub: Peg, by: string): string {. inc(i, x) add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, by = ""): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replace*(s: string, sub: Peg, by = ""): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed ## in `by`. result = "" @@ -1252,9 +1273,9 @@ proc replace*(s: string, sub: Peg, by = ""): string {. inc(i, x) add(result, substr(s, i)) -proc parallelReplace*(s: string, subs: varargs[ +func parallelReplace*(s: string, subs: varargs[ tuple[pattern: Peg, repl: string]]): string {. - noSideEffect, rtl, extern: "npegs$1".} = + rtl, extern: "npegs$1".} = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -1276,16 +1297,18 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, cb: proc( +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +func replace*(s: string, sub: Peg, cb: proc( match: int, cnt: int, caps: openArray[string]): string): string {. - rtl, extern: "npegs$1cb".} = + rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. ## The callback proc receives the index of the current match (starting with 0), ## the count of captures and an open array with the captures of each match. Examples: ## - ## .. code-block:: nim - ## - ## proc handleMatches*(m: int, n: int, c: openArray[string]): string = + ## ```nim + ## func handleMatches*(m: int, n: int, c: openArray[string]): string = ## result = "" ## if m > 0: ## result.add ", " @@ -1296,12 +1319,13 @@ proc replace*(s: string, sub: Peg, cb: proc( ## ## let s = "Var1=key1;var2=Key2; VAR3" ## echo s.replace(peg"{\ident}('='{\ident})* ';'* \s*", handleMatches) + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1: 'key1', var2: 'Key2', var3: ''" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1339,18 +1363,19 @@ iterator split*(s: string, sep: Peg): string = ## Substrings are separated by the PEG `sep`. ## Examples: ## - ## .. code-block:: nim + ## ```nim ## for word in split("00232this02939is39an22example111", peg"\d+"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "this" ## "is" ## "an" ## "example" - ## + ## ``` var c: Captures var first = 0 @@ -1368,8 +1393,8 @@ iterator split*(s: string, sep: Peg): string = if first < last: yield substr(s, first, last-1) -proc split*(s: string, sep: Peg): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func split*(s: string, sep: Peg): seq[string] {. + rtl, extern: "npegs$1".} = ## Splits the string `s` into substrings. result = @[] for it in split(s, sep): result.add it @@ -1395,6 +1420,7 @@ type tkCurlyLe, ## '{' tkCurlyRi, ## '}' tkCurlyAt, ## '{@}' + tkEmptyCurl, ## '{}' tkArrow, ## '<-' tkBar, ## '/' tkStar, ## '*' @@ -1427,25 +1453,25 @@ type const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", ".", "_", "identifier", "string literal", - "character set", "(", ")", "{", "}", "{@}", + "character set", "(", ")", "{", "}", "{@}", "{}", "<-", "/", "*", "+", "&", "!", "?", "@", "built-in", "escaped", "$", "$", "^" ] -proc handleCR(L: var PegLexer, pos: int): int = +func handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result -proc handleLF(L: var PegLexer, pos: int): int = +func handleLF(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\L') inc(L.lineNumber) result = pos+1 L.lineStart = result -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = +func init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.buf = input L.bufpos = 0 L.lineNumber = line @@ -1453,18 +1479,18 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.lineStart = 0 L.filename = filename -proc getColumn(L: PegLexer): int {.inline.} = +func getColumn(L: PegLexer): int {.inline.} = result = abs(L.bufpos - L.lineStart) + L.colOffset -proc getLine(L: PegLexer): int {.inline.} = +func getLine(L: PegLexer): int {.inline.} = result = L.lineNumber -proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = +func errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = var line = if line < 0: getLine(L) else: line var col = if col < 0: getColumn(L) else: col result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] -proc getEscapedChar(c: var PegLexer, tok: var Token) = +func getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) if c.bufpos >= len(c.buf): tok.kind = tkInvalid @@ -1524,7 +1550,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc skip(c: var PegLexer) = +func skip(c: var PegLexer) = var pos = c.bufpos while pos < c.buf.len: case c.buf[pos] @@ -1541,7 +1567,7 @@ proc skip(c: var PegLexer) = break # EndOfFile also leaves the loop c.bufpos = pos -proc getString(c: var PegLexer, tok: var Token) = +func getString(c: var PegLexer, tok: var Token) = tok.kind = tkStringLit var pos = c.bufpos + 1 var quote = c.buf[pos-1] @@ -1562,19 +1588,27 @@ proc getString(c: var PegLexer, tok: var Token) = inc(pos) c.bufpos = pos -proc getDollar(c: var PegLexer, tok: var Token) = +func getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 + var neg = false + if pos < c.buf.len and c.buf[pos] == '^': + neg = true + inc(pos) if pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 while pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(c.buf[pos]) - ord('0') inc(pos) + if neg: + tok.index = -tok.index else: + if neg: + dec(pos) tok.kind = tkDollar c.bufpos = pos -proc getCharSet(c: var PegLexer, tok: var Token) = +func getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkCharSet tok.charset = {} var pos = c.bufpos + 1 @@ -1631,7 +1665,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset -proc getSymbol(c: var PegLexer, tok: var Token) = +func getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos while pos < c.buf.len: add(tok.literal, c.buf[pos]) @@ -1640,7 +1674,7 @@ proc getSymbol(c: var PegLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getBuiltin(c: var PegLexer, tok: var Token) = +func getBuiltin(c: var PegLexer, tok: var Token) = if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) @@ -1649,7 +1683,7 @@ proc getBuiltin(c: var PegLexer, tok: var Token) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -proc getTok(c: var PegLexer, tok: var Token) = +func getTok(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) @@ -1670,6 +1704,10 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") + elif c.buf[c.bufpos] == '}' and c.bufpos < c.buf.len: + tok.kind = tkEmptyCurl + inc(c.bufpos) + add(tok.literal, "{}") else: tok.kind = tkCurlyLe add(tok.literal, '{') @@ -1705,7 +1743,7 @@ proc getTok(c: var PegLexer, tok: var Token) = return if c.buf[c.bufpos] in {'\'', '"'} or c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and - c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos+1] in {'^', '0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1767,7 +1805,7 @@ proc getTok(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc arrowIsNextTok(c: PegLexer): bool = +func arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) @@ -1788,23 +1826,21 @@ type identIsVerbatim: bool skip: Peg -proc pegError(p: PegParser, msg: string, line = -1, col = -1) = - var e: ref EInvalidPeg - new(e) - e.msg = errorStr(p, msg, line, col) +func pegError(p: PegParser, msg: string, line = -1, col = -1) = + var e = (ref EInvalidPeg)(msg: errorStr(p, msg, line, col)) raise e -proc getTok(p: var PegParser) = +func getTok(p: var PegParser) = getTok(p, p.tok) if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") -proc eat(p: var PegParser, kind: TokKind) = +func eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) else: pegError(p, tokKindToStr[kind] & " expected") -proc parseExpr(p: var PegParser): Peg {.gcsafe.} +func parseExpr(p: var PegParser): Peg {.gcsafe.} -proc getNonTerminal(p: var PegParser, name: string): NonTerminal = +func getNonTerminal(p: var PegParser, name: string): NonTerminal = for i in 0..high(p.nonterms): result = p.nonterms[i] if cmpIgnoreStyle(result.name, name) == 0: return @@ -1812,19 +1848,22 @@ proc getNonTerminal(p: var PegParser, name: string): NonTerminal = result = newNonTerminal(name, getLine(p), getColumn(p)) add(p.nonterms, result) -proc modifiedTerm(s: string, m: Modifier): Peg = +func modifiedTerm(s: string, m: Modifier): Peg = case m of modNone, modVerbatim: result = term(s) of modIgnoreCase: result = termIgnoreCase(s) of modIgnoreStyle: result = termIgnoreStyle(s) -proc modifiedBackref(s: int, m: Modifier): Peg = +func modifiedBackref(s: int, m: Modifier): Peg = + var + reverse = s < 0 + index = if reverse: -s else: s case m - of modNone, modVerbatim: result = backref(s) - of modIgnoreCase: result = backrefIgnoreCase(s) - of modIgnoreStyle: result = backrefIgnoreStyle(s) + of modNone, modVerbatim: result = backref(index, reverse) + of modIgnoreCase: result = backrefIgnoreCase(index, reverse) + of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) -proc builtin(p: var PegParser): Peg = +func builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous case p.tok.literal of "n": result = newLine() @@ -1844,11 +1883,11 @@ proc builtin(p: var PegParser): Peg = of "white": result = unicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) -proc token(terminal: Peg, p: PegParser): Peg = +func token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) -proc primary(p: var PegParser): Peg = +func primary(p: var PegParser): Peg = case p.tok.kind of tkAmp: getTok(p) @@ -1872,7 +1911,8 @@ proc primary(p: var PegParser): Peg = getTok(p) elif not arrowIsNextTok(p): var nt = getNonTerminal(p, p.tok.literal) - incl(nt.flags, ntUsed) + {.cast(noSideEffect).}: + incl(nt.flags, ntUsed) result = nonterminal(nt).token(p) getTok(p) else: @@ -1896,6 +1936,9 @@ proc primary(p: var PegParser): Peg = result = capture(parseExpr(p)).token(p) eat(p, tkCurlyRi) inc(p.captures) + of tkEmptyCurl: + result = capture() + getTok(p) of tkAny: result = any().token(p) getTok(p) @@ -1915,11 +1958,11 @@ proc primary(p: var PegParser): Peg = result = startAnchor() getTok(p) of tkBackref: + if abs(p.tok.index) > p.captures or p.tok.index == 0: + pegError(p, "invalid back reference index: " & $p.tok.index) var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: - pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) else: pegError(p, "expression expected, but found: " & p.tok.literal) @@ -1937,13 +1980,13 @@ proc primary(p: var PegParser): Peg = getTok(p) else: break -proc seqExpr(p: var PegParser): Peg = +func seqExpr(p: var PegParser): Peg = result = primary(p) while true: case p.tok.kind of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, - tkHat, tkCurlyAt: + tkHat, tkCurlyAt, tkEmptyCurl: result = sequence(result, primary(p)) of tkIdentifier: if not arrowIsNextTok(p): @@ -1951,27 +1994,29 @@ proc seqExpr(p: var PegParser): Peg = else: break else: break -proc parseExpr(p: var PegParser): Peg = +func parseExpr(p: var PegParser): Peg = result = seqExpr(p) while p.tok.kind == tkBar: getTok(p) result = result / seqExpr(p) -proc parseRule(p: var PegParser): NonTerminal = +func parseRule(p: var PegParser): NonTerminal = if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = getNonTerminal(p, p.tok.literal) if ntDeclared in result.flags: pegError(p, "attempt to redefine: " & result.name) - result.line = getLine(p) - result.col = getColumn(p) + {.cast(noSideEffect).}: + result.line = getLine(p) + result.col = getColumn(p) getTok(p) eat(p, tkArrow) - result.rule = parseExpr(p) - incl(result.flags, ntDeclared) # NOW inlining may be attempted + {.cast(noSideEffect).}: + result.rule = parseExpr(p) + incl(result.flags, ntDeclared) # NOW inlining may be attempted else: pegError(p, "rule expected, but found: " & p.tok.literal) -proc rawParse(p: var PegParser): Peg = +func rawParse(p: var PegParser): Peg = ## parses a rule or a PEG expression while p.tok.kind == tkBuiltin: case p.tok.literal @@ -2001,7 +2046,7 @@ proc rawParse(p: var PegParser): Peg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = +func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are ## used for error messages, but they only provide start offsets. `parsePeg` ## keeps track of line and column numbers within `pattern`. @@ -2016,14 +2061,14 @@ proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = getTok(p) result = rawParse(p) -proc peg*(pattern: string): Peg = +func peg*(pattern: string): Peg = ## constructs a Peg object from the `pattern`. The short name has been - ## chosen to encourage its use as a raw string modifier:: + ## chosen to encourage its use as a raw string modifier: ## - ## peg"{\ident} \s* '=' \s* {.*}" + ## peg"{\ident} \s* '=' \s* {.*}" result = parsePeg(pattern, "pattern") -proc escapePeg*(s: string): string = +func escapePeg*(s: string): string = ## escapes `s` so that it is matched verbatim when used as a peg. result = "" var inQuote = false @@ -2041,147 +2086,3 @@ proc escapePeg*(s: string): string = inQuote = true result.add(c) if inQuote: result.add('\'') - -when isMainModule: - proc pegsTest() = - assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" - assert match("(a b c)", peg"'(' @ ')'") - assert match("W_HI_Le", peg"\y 'while'") - assert(not match("W_HI_L", peg"\y 'while'")) - assert(not match("W_HI_Le", peg"\y v'while'")) - assert match("W_HI_Le", peg"y'while'") - - assert($ +digits == $peg"\d+") - assert "0158787".match(peg"\d+") - assert "ABC 0232".match(peg"\w+\s+\d+") - assert "ABC".match(peg"\d+ / \w+") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", peg"\d+"): - accum.add(word) - assert(accum == @["this", "is", "an", "example"]) - - assert matchLen("key", ident) == 3 - - var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) - assert matchLen("key1= cal9", pattern) == 11 - - var ws = newNonTerminal("ws", 1, 1) - ws.rule = *whitespace - - var expr = newNonTerminal("expr", 1, 1) - expr.rule = sequence(capture(ident), *sequence( - nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - - var c: Captures - var s = "a+b + c +d+e+f" - assert rawMatch(s, expr.rule, 0, c) == len(s) - var a = "" - for i in 0..c.ml-1: - a.add(substr(s, c.matches[i][0], c.matches[i][1])) - assert a == "abcdef" - #echo expr.rule - - #const filename = "lib/devel/peg/grammar.txt" - #var grammar = parsePeg(newFileStream(filename, fmRead), filename) - #echo "a <- [abc]*?".match(grammar) - assert find("_____abc_______", term("abc"), 2) == 5 - assert match("_______ana", peg"A <- 'ana' / . A") - assert match("abcs%%%", peg"A <- ..A / .A / '%'") - - var matches: array[0..MaxSubpatterns-1, string] - if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": - assert matches[0] == "abc" - else: - assert false - - var g2 = peg"""S <- A B / C D - A <- 'a'+ - B <- 'b'+ - C <- 'c'+ - D <- 'd'+ - """ - assert($g2 == "((A B) / (C D))") - assert match("cccccdddddd", g2) - assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") - - if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": - assert matches[0] == "a" - else: - assert false - - if match("abcdefg", peg"c {d} ef {g}", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - accum = @[] - for x in findAll("abcdef", peg".", 3): - accum.add(x) - assert(accum == @["d", "e", "f"]) - - for x in findAll("abcdef", peg"^{.}", 3): - assert x == "d" - - if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": - assert matches[0] == "f" - assert matches[1] == "a, b" - else: - assert false - - assert match("eine übersicht und außerdem", peg"(\letter \white*)+") - # ß is not a lower cased letter?! - assert match("eine übersicht und auerdem", peg"(\lower \white*)+") - assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") - assert(not match("456678", peg"(\letter)+")) - - assert("var1 = key; var2 = key2".replacef( - peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey;var2<-key2key2") - - assert match("prefix/start", peg"^start$", 7) - - if "foo" =~ peg"{'a'}?.*": - assert matches[0].len == 0 - else: assert false - - if "foo" =~ peg"{''}.*": - assert matches[0] == "" - else: assert false - - if "foo" =~ peg"{'foo'}": - assert matches[0] == "foo" - else: assert false - - let empty_test = peg"^\d*" - let str = "XYZ" - - assert(str.find(empty_test) == 0) - assert(str.match(empty_test)) - - proc handleMatches(m: int, n: int, c: openArray[string]): string = - result = "" - - if m > 0: - result.add ", " - - result.add case n: - of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" - of 1: toLowerAscii(c[0]) & ": ''" - else: "" - - assert("Var1=key1;var2=Key2; VAR3". - replace(peg"{\ident}('='{\ident})* ';'* \s*", - handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''") - - - doAssert "test1".match(peg"""{@}$""") - doAssert "test2".match(peg"""{(!$ .)*} $""") - pegsTest() - static: - pegsTest() diff --git a/lib/pure/prelude.nim b/lib/pure/prelude.nim index f1728b5f72..9428f29ebc 100644 --- a/lib/pure/prelude.nim +++ b/lib/pure/prelude.nim @@ -14,7 +14,7 @@ when defined(nimdoc) and isMainModule: runnableExamples: include std/prelude # same as: - # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt] + # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] let x = 1 assert "foo $# $#" % [$x, "bar"] == "foo 1 bar" assert toSeq(1..3) == @[1, 2, 3] @@ -25,4 +25,4 @@ when defined(nimdoc) and isMainModule: # xxx `nim doc -b:js -d:nodejs --doccmd:-d:nodejs lib/pure/prelude.nim` fails for some reason # specific to `nim doc`, but the code otherwise works with nodejs. -import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt] +import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim deleted file mode 100644 index dfeeea35e4..0000000000 --- a/lib/pure/punycode.nim +++ /dev/null @@ -1,208 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements a representation of Unicode with the limited -## ASCII character subset. - -import strutils -import unicode - -# issue #3045 - -const - Base = 36 - TMin = 1 - TMax = 26 - Skew = 38 - Damp = 700 - InitialBias = 72 - InitialN = 128 - Delimiter = '-' - -type - PunyError* = object of ValueError - -proc decodeDigit(x: char): int {.raises: [PunyError].} = - if '0' <= x and x <= '9': - result = ord(x) - (ord('0') - 26) - elif 'A' <= x and x <= 'Z': - result = ord(x) - ord('A') - elif 'a' <= x and x <= 'z': - result = ord(x) - ord('a') - else: - raise newException(PunyError, "Bad input") - -proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = - if 0 <= digit and digit < 26: - result = Rune(digit + ord('a')) - elif 26 <= digit and digit < 36: - result = Rune(digit + (ord('0') - 26)) - else: - raise newException(PunyError, "internal error in punycode encoding") - -proc isBasic(c: char): bool = ord(c) < 0x80 -proc isBasic(r: Rune): bool = int(r) < 0x80 - -proc adapt(delta, numPoints: int, first: bool): int = - var d = if first: delta div Damp else: delta div 2 - d += d div numPoints - var k = 0 - while d > ((Base-TMin)*TMax) div 2: - d = d div (Base - TMin) - k += Base - result = k + (Base - TMin + 1) * d div (d + Skew) - -proc encode*(prefix, s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. - ## Prepend `prefix` to the result - result = prefix - var (d, n, bias) = (0, InitialN, InitialBias) - var (b, remaining) = (0, 0) - for r in s.runes: - if r.isBasic: - # basic Ascii character - inc b - result.add($r) - else: - # special character - inc remaining - - var h = b - if b > 0: - result.add(Delimiter) # we have some Ascii chars - while remaining != 0: - var m: int = high(int32) - for r in s.runes: - if m > int(r) and int(r) >= n: - m = int(r) - d += (m - n) * (h + 1) - if d < 0: - raise newException(PunyError, "invalid label " & s) - n = m - for r in s.runes: - if int(r) < n: - inc d - if d < 0: - raise newException(PunyError, "invalid label " & s) - continue - if int(r) > n: - continue - var q = d - var k = Base - while true: - var t = k - bias - if t < TMin: - t = TMin - elif t > TMax: - t = TMax - if q < t: - break - result.add($encodeDigit(t + (q - t) mod (Base - t))) - q = (q - t) div (Base - t) - k += Base - result.add($encodeDigit(q)) - bias = adapt(d, h + 1, h == b) - d = 0 - inc h - dec remaining - inc d - inc n - -proc encode*(s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. Prefix is empty. - result = encode("", s) - -proc decode*(encoded: string): string {.raises: [PunyError].} = - ## Decode a Punycode-encoded string - var - n = InitialN - i = 0 - bias = InitialBias - var d = rfind(encoded, Delimiter) - result = "" - - if d > 0: - # found Delimiter - for j in 0..<d: - var c = encoded[j] # char - if not c.isBasic: - raise newException(PunyError, "Encoded contains a non-basic char") - result.add(c) # add the character - inc d - else: - d = 0 # set to first index - - while (d < len(encoded)): - var oldi = i - var w = 1 - var k = Base - while true: - if d == len(encoded): - raise newException(PunyError, "Bad input: " & encoded) - var c = encoded[d]; inc d - var digit = int(decodeDigit(c)) - if digit > (high(int32) - i) div w: - raise newException(PunyError, "Too large a value: " & $digit) - i += digit * w - var t: int - if k <= bias: - t = TMin - elif k >= bias + TMax: - t = TMax - else: - t = k - bias - if digit < t: - break - w *= Base - t - k += Base - bias = adapt(i - oldi, runeLen(result) + 1, oldi == 0) - - if i div (runeLen(result) + 1) > high(int32) - n: - raise newException(PunyError, "Value too large") - - n += i div (runeLen(result) + 1) - i = i mod (runeLen(result) + 1) - insert(result, $Rune(n), i) - inc i - - -runnableExamples: - static: - block: - doAssert encode("") == "" - doAssert encode("a") == "a-" - doAssert encode("A") == "A-" - doAssert encode("3") == "3-" - doAssert encode("-") == "--" - doAssert encode("--") == "---" - doAssert encode("abc") == "abc-" - doAssert encode("London") == "London-" - doAssert encode("Lloyd-Atkinson") == "Lloyd-Atkinson-" - doAssert encode("This has spaces") == "This has spaces-" - doAssert encode("ü") == "tda" - doAssert encode("München") == "Mnchen-3ya" - doAssert encode("Mnchen-3ya") == "Mnchen-3ya-" - doAssert encode("München-Ost") == "Mnchen-Ost-9db" - doAssert encode("Bahnhof München-Ost") == "Bahnhof Mnchen-Ost-u6b" - block: - doAssert decode("") == "" - doAssert decode("a-") == "a" - doAssert decode("A-") == "A" - doAssert decode("3-") == "3" - doAssert decode("--") == "-" - doAssert decode("---") == "--" - doAssert decode("abc-") == "abc" - doAssert decode("London-") == "London" - doAssert decode("Lloyd-Atkinson-") == "Lloyd-Atkinson" - doAssert decode("This has spaces-") == "This has spaces" - doAssert decode("tda") == "ü" - doAssert decode("Mnchen-3ya") == "München" - doAssert decode("Mnchen-3ya-") == "Mnchen-3ya" - doAssert decode("Mnchen-Ost-9db") == "München-Ost" - doAssert decode("Bahnhof Mnchen-Ost-u6b") == "Bahnhof München-Ost" diff --git a/lib/pure/random.nim b/lib/pure/random.nim index f91d92731d..7499443762 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -66,14 +66,16 @@ runnableExamples: ## See also ## ======== ## * `std/sysrand module <sysrand.html>`_ for a cryptographically secure pseudorandom number generator -## * `mersenne module <mersenne.html>`_ for the Mersenne Twister random number generator ## * `math module <math.html>`_ for basic math routines ## * `stats module <stats.html>`_ for statistical analysis ## * `list of cryptographic and hashing modules <lib.html#pure-libraries-hashing>`_ ## in the standard library import std/[algorithm, math] -import std/private/since +import std/private/[since, jsutils] + +when defined(nimPreviewSlimSystem): + import std/[assertions] include system/inclrtl {.push debugger: off.} @@ -108,10 +110,19 @@ when defined(js): a0: 0x69B4C98Cu32, a1: 0xFED1DD30u32) # global for backwards compatibility else: - # racy for multi-threading but good enough for now: - var state = Rand( + const DefaultRandSeed = Rand( a0: 0x69B4C98CB8530805u64, - a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility + a1: 0xFED1DD3004688D67CAu64) + + # racy for multi-threading but good enough for now: + var state = DefaultRandSeed # global for backwards compatibility + +func isValid(r: Rand): bool {.inline.} = + ## Check whether state of `r` is valid. + ## + ## In `xoroshiro128+`, if all bits of `a0` and `a1` are zero, + ## they are always zero after calling `next(r: var Rand)`. + not (r.a0 == 0 and r.a1 == 0) since (1, 5): template randState*(): untyped = @@ -133,11 +144,10 @@ proc next*(r: var Rand): uint64 = ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type ## * `skipRandomNumbers proc<#skipRandomNumbers,Rand>`_ - runnableExamples: + runnableExamples("-r:off"): var r = initRand(2019) - doAssert r.next() == 138_744_656_611_299'u64 - doAssert r.next() == 979_810_537_855_049_344'u64 - doAssert r.next() == 3_628_232_584_225_300_704'u64 + assert r.next() == 13223559681708962501'u64 # implementation defined + assert r.next() == 7229677234260823147'u64 # ditto let s0 = r.a0 var s1 = r.a1 @@ -170,24 +180,33 @@ proc skipRandomNumbers*(s: var Rand) = ## **See also:** ## * `next proc<#next,Rand>`_ runnableExamples("--threads:on"): - import std/[random, threadpool] + import std/random - const spawns = 4 const numbers = 100000 - proc randomSum(r: Rand): int = - var r = r + var + thr: array[0..3, Thread[(Rand, int)]] + vals: array[0..3, int] + + proc randomSum(params: tuple[r: Rand, index: int]) {.thread.} = + var r = params.r + var s = 0 # avoid cache thrashing for i in 1..numbers: - result += r.rand(0..10) + s += r.rand(0..10) + vals[params.index] = s var r = initRand(2019) - var vals: array[spawns, FlowVar[int]] - for val in vals.mitems: - val = spawn randomSum(r) + for i in 0..<thr.len: + createThread(thr[i], randomSum, (r, i)) r.skipRandomNumbers() + joinThreads(thr) + for val in vals: - doAssert abs(^val - numbers * 5) / numbers < 0.1 + doAssert abs(val - numbers * 5) / numbers < 0.1 + + doAssert vals == [501737, 497901, 500683, 500157] + when defined(js): const helper = [0xbeac0467u32, 0xd86b048bu32] @@ -205,6 +224,22 @@ proc skipRandomNumbers*(s: var Rand) = s.a0 = s0 s.a1 = s1 +proc rand[T: uint | uint64](r: var Rand; max: T): T = + # xxx export in future work + if max == 0: return + else: + let max = uint64(max) + when T.high.uint64 == uint64.high: + if max == uint64.high: return T(next(r)) + var iters = 0 + while true: + let x = next(r) + # avoid `mod` bias + if x <= randMax - (randMax mod max) or iters > 20: + return T(x mod (max + 1)) + else: + inc iters + proc rand*(r: var Rand; max: Natural): int {.benign.} = ## Returns a random integer in the range `0..max` using the given state. ## @@ -216,15 +251,11 @@ proc rand*(r: var Rand; max: Natural): int {.benign.} = ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: var r = initRand(123) - doAssert r.rand(100) == 0 - doAssert r.rand(100) == 96 - doAssert r.rand(100) == 66 - - if max == 0: return - while true: - let x = next(r) - if x <= randMax - (randMax mod Ui(max)): - return int(x mod (uint64(max) + 1u64)) + if false: + assert r.rand(100) == 96 # implementation defined + # bootstrap: can't use `runnableExamples("-r:off")` + cast[int](rand(r, uint64(max))) + # xxx toUnsigned pending https://github.com/nim-lang/Nim/pull/18445 proc rand*(max: int): int {.benign.} = ## Returns a random integer in the range `0..max`. @@ -241,11 +272,9 @@ proc rand*(max: int): int {.benign.} = ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type - runnableExamples: + runnableExamples("-r:off"): randomize(123) - doAssert rand(100) == 0 - doAssert rand(100) == 96 - doAssert rand(100) == 66 + assert [rand(100), rand(100)] == [96, 63] # implementation defined rand(state, max) @@ -305,15 +334,16 @@ proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T = ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: var r = initRand(345) - doAssert r.rand(1..6) == 4 - doAssert r.rand(1..6) == 4 - doAssert r.rand(1..6) == 6 - let f = r.rand(-1.0 .. 1.0) # 0.8741183448756229 + assert r.rand(1..5) <= 5 + assert r.rand(-1.1 .. 1.2) >= -1.1 assert x.a <= x.b when T is SomeFloat: result = rand(r, x.b - x.a) + x.a else: # Integers and Enum types - result = T(rand(r, int(x.b) - int(x.a)) + int(x.a)) + whenJsNoBigInt64: + result = cast[T](rand(r, cast[uint](x.b) - cast[uint](x.a)) + cast[uint](x.a)) + do: + result = cast[T](rand(r, cast[uint64](x.b) - cast[uint64](x.a)) + cast[uint64](x.a)) proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = ## For a slice `a..b`, returns a value in the range `a..b`. @@ -333,14 +363,33 @@ proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: randomize(345) - doAssert rand(1..6) == 4 - doAssert rand(1..6) == 4 - doAssert rand(1..6) == 6 + assert rand(1..6) <= 6 result = rand(state, x) -proc rand*[T: SomeInteger](t: typedesc[T]): T = - ## Returns a random integer in the range `low(T)..high(T)`. +proc rand*[T: Ordinal](r: var Rand; t: typedesc[T]): T {.since: (1, 7, 1).} = + ## Returns a random Ordinal in the range `low(T)..high(T)`. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## **See also:** + ## * `rand proc<#rand,int>`_ that returns an integer + ## * `rand proc<#rand,float>`_ that returns a floating point number + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + when T is range or T is enum: + result = rand(r, low(T)..high(T)) + elif T is bool: + result = r.next < randMax div 2 + else: + whenJsNoBigInt64: + result = cast[T](r.next shr (sizeof(uint)*8 - sizeof(T)*8)) + do: + result = cast[T](r.next shr (sizeof(uint64)*8 - sizeof(T)*8)) + +proc rand*[T: Ordinal](t: typedesc[T]): T = + ## Returns a random Ordinal in the range `low(T)..high(T)`. ## ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. @@ -354,20 +403,15 @@ proc rand*[T: SomeInteger](t: typedesc[T]): T = ## that accepts a slice runnableExamples: randomize(567) - doAssert rand(int8) == 55 - doAssert rand(int8) == -42 - doAssert rand(int8) == 43 - doAssert rand(uint32) == 578980729'u32 - doAssert rand(uint32) == 4052940463'u32 - doAssert rand(uint32) == 2163872389'u32 - doAssert rand(range[1..16]) == 11 - doAssert rand(range[1..16]) == 4 - doAssert rand(range[1..16]) == 16 + type E = enum a, b, c, d - when T is range: - result = rand(state, low(T)..high(T)) - else: - result = cast[T](state.next) + assert rand(E) in a..d + assert rand(char) in low(char)..high(char) + assert rand(int8) in low(int8)..high(int8) + assert rand(uint32) in low(uint32)..high(uint32) + assert rand(range[1..16]) in 1..16 + + result = rand(state, t) proc sample*[T](r: var Rand; s: set[T]): T = ## Returns a random element from the set `s` using the given state. @@ -380,9 +424,7 @@ proc sample*[T](r: var Rand; s: set[T]): T = runnableExamples: var r = initRand(987) let s = {1, 3, 5, 7, 9} - doAssert r.sample(s) == 5 - doAssert r.sample(s) == 7 - doAssert r.sample(s) == 1 + assert r.sample(s) in s assert card(s) != 0 var i = rand(r, card(s) - 1) @@ -406,9 +448,7 @@ proc sample*[T](s: set[T]): T = runnableExamples: randomize(987) let s = {1, 3, 5, 7, 9} - doAssert sample(s) == 5 - doAssert sample(s) == 7 - doAssert sample(s) == 1 + assert sample(s) in s sample(state, s) @@ -423,13 +463,11 @@ proc sample*[T](r: var Rand; a: openArray[T]): T = runnableExamples: let marbles = ["red", "blue", "green", "yellow", "purple"] var r = initRand(456) - doAssert r.sample(marbles) == "blue" - doAssert r.sample(marbles) == "yellow" - doAssert r.sample(marbles) == "red" + assert r.sample(marbles) in marbles result = a[r.rand(a.low..a.high)] -proc sample*[T](a: openArray[T]): T = +proc sample*[T](a: openArray[T]): lent T = ## Returns a random element from `a`. ## ## If `randomize <#randomize>`_ has not been called, the order of outcomes @@ -445,9 +483,7 @@ proc sample*[T](a: openArray[T]): T = runnableExamples: let marbles = ["red", "blue", "green", "yellow", "purple"] randomize(456) - doAssert sample(marbles) == "blue" - doAssert sample(marbles) == "yellow" - doAssert sample(marbles) == "red" + assert sample(marbles) in marbles result = a[rand(a.low..a.high)] @@ -476,9 +512,7 @@ proc sample*[T, U](r: var Rand; a: openArray[T]; cdf: openArray[U]): T = let count = [1, 6, 8, 3, 4] let cdf = count.cumsummed var r = initRand(789) - doAssert r.sample(marbles, cdf) == "red" - doAssert r.sample(marbles, cdf) == "green" - doAssert r.sample(marbles, cdf) == "blue" + assert r.sample(marbles, cdf) in marbles assert(cdf.len == a.len) # Two basic sanity checks. assert(float(cdf[^1]) > 0.0) @@ -512,9 +546,7 @@ proc sample*[T, U](a: openArray[T]; cdf: openArray[U]): T = let count = [1, 6, 8, 3, 4] let cdf = count.cumsummed randomize(789) - doAssert sample(marbles, cdf) == "red" - doAssert sample(marbles, cdf) == "green" - doAssert sample(marbles, cdf) == "blue" + assert sample(marbles, cdf) in marbles state.sample(a, cdf) @@ -547,10 +579,10 @@ proc gauss*(mu = 0.0, sigma = 1.0): float {.since: (1, 3).} = proc initRand*(seed: int64): Rand = ## Initializes a new `Rand <#Rand>`_ state using the given seed. ## - ## `seed` must not be zero. Providing a specific seed will produce - ## the same results for that seed each time. + ## Providing a specific seed will produce the same results for that seed each time. ## - ## The resulting state is independent of the default RNG's state. + ## The resulting state is independent of the default RNG's state. When `seed == 0`, + ## we internally set the seed to an implementation defined non-zero value. ## ## **See also:** ## * `initRand proc<#initRand>`_ that uses the current time @@ -560,20 +592,22 @@ proc initRand*(seed: int64): Rand = from std/times import getTime, toUnix, nanosecond var r1 = initRand(123) - let now = getTime() var r2 = initRand(now.toUnix * 1_000_000_000 + now.nanosecond) - - doAssert seed != 0 # 0 causes `rand(int)` to always return 0 for example. + const seedFallback0 = int32.high # arbitrary + let seed = if seed != 0: seed else: seedFallback0 # because 0 is a fixed point result.a0 = Ui(seed shr 16) result.a1 = Ui(seed and 0xffff) + when not defined(nimLegacyRandomInitRand): + # calling `discard next(result)` (even a few times) would still produce + # skewed numbers for the 1st call to `rand()`. + skipRandomNumbers(result) discard next(result) proc randomize*(seed: int64) {.benign.} = ## Initializes the default random number generator with the given seed. ## - ## `seed` must not be zero. Providing a specific seed will produce - ## the same results for that seed each time. + ## Providing a specific seed will produce the same results for that seed each time. ## ## **See also:** ## * `initRand proc<#initRand,int64>`_ that initializes a Rand state @@ -600,7 +634,8 @@ proc shuffle*[T](r: var Rand; x: var openArray[T]) = var cards = ["Ace", "King", "Queen", "Jack", "Ten"] var r = initRand(678) r.shuffle(cards) - doAssert cards == ["King", "Ace", "Queen", "Ten", "Jack"] + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] for i in countdown(x.high, 1): let j = r.rand(i) @@ -620,19 +655,33 @@ proc shuffle*[T](x: var openArray[T]) = var cards = ["Ace", "King", "Queen", "Jack", "Ten"] randomize(678) shuffle(cards) - doAssert cards == ["King", "Ace", "Queen", "Ten", "Jack"] + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] shuffle(state, x) -when not defined(nimscript) and not defined(standalone): - import std/times +when not defined(standalone): + when defined(js): + import std/times + else: + when defined(nimscript): + import std/hashes + else: + import std/[hashes, os, sysrand, monotimes] + + when compileOption("threads"): + import std/locks + var baseSeedLock: Lock + baseSeedLock.initLock + + var baseState: Rand proc initRand(): Rand = - ## Initializes a new Rand state with a seed based on the current time. + ## Initializes a new Rand state. ## ## The resulting state is independent of the default RNG's state. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## See also: ## * `initRand proc<#initRand,int64>`_ that accepts a seed for a new Rand state @@ -642,20 +691,50 @@ when not defined(nimscript) and not defined(standalone): let time = int64(times.epochTime() * 1000) and 0x7fff_ffff result = initRand(time) else: - let now = times.getTime() - result = initRand(convert(Seconds, Nanoseconds, now.toUnix) + now.nanosecond) + proc getRandomState(): Rand = + when defined(nimscript): + result = Rand( + a0: CompileTime.hash.Ui, + a1: CompileDate.hash.Ui) + if not result.isValid: + result = DefaultRandSeed + else: + var urand: array[sizeof(Rand), byte] + + for i in 0 .. 7: + if sysrand.urandom(urand): + copyMem(result.addr, urand[0].addr, sizeof(Rand)) + if result.isValid: + break + + if not result.isValid: + # Don't try to get alternative random values from other source like time or process/thread id, + # because such code would be never tested and is a liability for security. + quit("Failed to initializes baseState in random module as sysrand.urandom doesn't work.") + + when compileOption("threads"): + baseSeedLock.withLock: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + else: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers since (1, 5, 1): export initRand proc randomize*() {.benign.} = ## Initializes the default random number generator with a seed based on - ## the current time. + ## random number source. ## ## This proc only needs to be called once, and it should be called before ## the first usage of procs from this module that use the default RNG. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## **See also:** ## * `randomize proc<#randomize,int64>`_ that accepts a seed diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index a059651bb0..5f806bd701 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -22,6 +22,8 @@ runnableExamples: doAssert r1 / r2 == -2 // 3 import std/[math, hashes] +when defined(nimPreviewSlimSystem): + import std/assertions type Rational*[T] = object ## A rational number, consisting of a numerator `num` and a denominator `den`. @@ -38,16 +40,16 @@ func reduce*[T: SomeInteger](x: var Rational[T]) = reduce(r) doAssert r.num == 1 doAssert r.den == 2 - + if x.den == 0: + raise newException(DivByZeroDefect, "division by zero") let common = gcd(x.num, x.den) if x.den > 0: x.num = x.num div common x.den = x.den div common - elif x.den < 0: - x.num = -x.num div common - x.den = -x.den div common - else: - raise newException(DivByZeroDefect, "division by zero") + when T isnot SomeUnsignedInt: + if x.den < 0: + x.num = -x.num div common + x.den = -x.den div common func initRational*[T: SomeInteger](num, den: T): Rational[T] = ## Creates a new rational number with numerator `num` and denominator `den`. @@ -316,3 +318,23 @@ func hash*[T](x: Rational[T]): Hash = h = h !& hash(copy.num) h = h !& hash(copy.den) result = !$h + +func `^`*[T: SomeInteger](x: Rational[T], y: T): Rational[T] = + ## Computes `x` to the power of `y`. + ## + ## The exponent `y` must be an integer. Negative exponents are supported + ## but floating point exponents are not. + runnableExamples: + doAssert (-3 // 5) ^ 0 == (1 // 1) + doAssert (-3 // 5) ^ 1 == (-3 // 5) + doAssert (-3 // 5) ^ 2 == (9 // 25) + doAssert (-3 // 5) ^ -2 == (25 // 9) + + if y >= 0: + result.num = x.num ^ y + result.den = x.den ^ y + else: + result.num = x.den ^ -y + result.den = x.num ^ -y + # Note that all powers of reduced rationals are already reduced, + # so we don't need to call reduce() here diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim index 232a2b3838..ffa0128dc8 100644 --- a/lib/pure/reservedmem.nim +++ b/lib/pure/reservedmem.nim @@ -9,7 +9,7 @@ ## :Authors: Zahary Karadjov ## -## This module provides utilities for reserving a portions of the +## This module provides utilities for reserving portions of the ## address space of a program without consuming physical memory. ## It can be used to implement a dynamically resizable buffer that ## is guaranteed to remain in the same memory location. The buffer @@ -18,7 +18,10 @@ ## ## Unstable API. -from os import raiseOSError, osLastError +from std/oserrors import raiseOSError, osLastError + +when defined(nimPreviewSlimSystem): + import std/assertions template distance*(lhs, rhs: pointer): int = cast[int](rhs) - cast[int](lhs) @@ -41,26 +44,11 @@ type mem: ReservedMem when defined(windows): - import winlean - - type - SYSTEM_INFO {.final, pure.} = object - u1: uint32 - dwPageSize: uint32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr uint32 - dwNumberOfProcessors: uint32 - dwProcessorType: uint32 - dwAllocationGranularity: uint32 - wProcessorLevel: uint16 - wProcessorRevision: uint16 - - proc getSystemInfo(lpSystemInfo: ptr SYSTEM_INFO) {.stdcall, - dynlib: "kernel32", importc: "GetSystemInfo".} + import std/winlean + import std/private/win_getsysteminfo proc getAllocationGranularity: uint = - var sysInfo: SYSTEM_INFO + var sysInfo: SystemInfo getSystemInfo(addr sysInfo) return uint(sysInfo.dwAllocationGranularity) @@ -80,7 +68,7 @@ when defined(windows): raiseOSError(osLastError()) else: - import posix + import std/posix let allocationGranularity = sysconf(SC_PAGESIZE) diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 42550af1d8..8750aca874 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -19,6 +19,9 @@ include system/inclrtl import std/streams +when defined(nimPreviewSlimSystem): + import std/[syncio, formatfloat, assertions] + {.push debugger: off.} # the user does not want to trace a part # of the standard library! diff --git a/lib/pure/segfaults.nim b/lib/pure/segfaults.nim index e0da8b81d1..65b059e86d 100644 --- a/lib/pure/segfaults.nim +++ b/lib/pure/segfaults.nim @@ -26,11 +26,11 @@ se.msg = "Could not access value because it is nil." when defined(windows): include "../system/ansi_c" - import winlean + import std/winlean const EXCEPTION_ACCESS_VIOLATION = DWORD(0xc0000005'i32) - EXCEPTION_CONTINUE_SEARCH = Long(0) + EXCEPTION_CONTINUE_SEARCH = LONG(0) type PEXCEPTION_RECORD = ptr object @@ -65,7 +65,7 @@ when defined(windows): c_signal(SIGSEGV, segfaultHandler) else: - import posix + import std/posix var sa: Sigaction diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index e78219aec0..ac180e2bda 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -27,13 +27,17 @@ ## ## TODO: `/dev/poll`, `event ports` and filesystem events. -import os, nativesockets +import std/nativesockets +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/assertions const hasThreadSupport = compileOption("threads") and defined(threadsafe) const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly) or + defined(dragonfly) or defined(nuttx) or (defined(linux) and not defined(android) and not defined(emscripten)) ## This constant is used to determine whether the destination platform is ## fully supported by `ioselectors` module. @@ -201,12 +205,11 @@ when defined(nimdoc): ## to `value`. This `value` can be modified in the scope of ## the `withData` call. ## - ## .. code-block:: nim - ## + ## ```nim ## s.withData(fd, value) do: ## # block is executed only if `fd` registered in selector `s` ## value.uid = 1000 - ## + ## ``` template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2: untyped) = @@ -214,15 +217,14 @@ when defined(nimdoc): ## to `value`. This `value` can be modified in the scope of ## the `withData` call. ## - ## .. code-block:: nim - ## + ## ```nim ## s.withData(fd, value) do: ## # block is executed only if `fd` registered in selector `s`. ## value.uid = 1000 ## do: ## # block is executed if `fd` not registered in selector `s`. ## raise - ## + ## ``` proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. @@ -233,9 +235,9 @@ when defined(nimdoc): ## For *poll* and *select* selectors `-1` is returned. else: - import strutils + import std/strutils when hasThreadSupport: - import locks + import std/locks type SharedArray[T] = UncheckedArray[T] @@ -243,8 +245,8 @@ else: proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) - proc reallocSharedArray[T](sa: ptr SharedArray[T], nsize: int): ptr SharedArray[T] = - result = cast[ptr SharedArray[T]](reallocShared(sa, sizeof(T) * nsize)) + proc reallocSharedArray[T](sa: ptr SharedArray[T], oldsize, nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](reallocShared0(sa, oldsize * sizeof(T), sizeof(T) * nsize)) proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) @@ -286,7 +288,7 @@ else: setBlocking(fd.SocketHandle, false) when not defined(windows): - import posix + import std/posix template setKey(s, pident, pevents, pparam, pdata: untyped) = var skey = addr(s.fds[pident]) @@ -321,9 +323,37 @@ else: proc verifySelectParams(timeout: int) = # Timeout of -1 means: wait forever # Anything higher is the time to wait in milliseconds. - doAssert(timeout >= -1, "Cannot select with a negative value, got " & $timeout) + doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) - when defined(linux) and not defined(emscripten): + when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): + template maxDescriptors*(): int = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. + when defined(windows): + 16_700_000 + elif defined(zephyr) or defined(freertos): + FD_MAX + else: + var fdLim: RLimit + var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) + if res >= 0: + res = int(fdLim.rlim_cur) - 1 + res + + when defined(nimIoselector): + when nimIoselector == "epoll": + include ioselects/ioselectors_epoll + elif nimIoselector == "kqueue": + include ioselects/ioselectors_kqueue + elif nimIoselector == "poll": + include ioselects/ioselectors_poll + elif nimIoselector == "select": + include ioselects/ioselectors_select + else: + {.fatal: "Unknown nimIoselector specified by define.".} + elif defined(linux) and not defined(emscripten): include ioselects/ioselectors_epoll elif bsdPlatform: include ioselects/ioselectors_kqueue @@ -337,5 +367,9 @@ else: include ioselects/ioselectors_select elif defined(freertos) or defined(lwip): include ioselects/ioselectors_select + elif defined(zephyr): + include ioselects/ioselectors_poll + elif defined(nuttx): + include ioselects/ioselectors_epoll else: include ioselects/ioselectors_poll diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim deleted file mode 100644 index 5ba0486084..0000000000 --- a/lib/pure/smtp.nim +++ /dev/null @@ -1,343 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements the SMTP client protocol as specified by RFC 5321, -## this can be used to send mail to any SMTP Server. -## -## This module also implements the protocol used to format messages, -## as specified by RFC 2822. -## -## Example gmail use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(useSsl = true, debug=true) -## smtpConn.connect("smtp.gmail.com", Port 465) -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## Example for startTls use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(debug=true) -## smtpConn.connect("smtp.mailtrap.io", Port 2525) -## smtpConn.startTls() -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## For SSL support this module relies on OpenSSL. If you want to -## enable SSL, compile with `-d:ssl`. - -import net, strutils, strtabs, base64, os, strutils -import asyncnet, asyncdispatch - -export Port - -type - Message* = object - msgTo: seq[string] - msgCc: seq[string] - msgSubject: string - msgOtherHeaders: StringTableRef - msgBody: string - - ReplyError* = object of IOError - - SmtpBase[SocketType] = ref object - sock: SocketType - address: string - debug: bool - - Smtp* = SmtpBase[Socket] - AsyncSmtp* = SmtpBase[AsyncSocket] - -proc containsNewline(xs: seq[string]): bool = - for x in xs: - if x.contains({'\c', '\L'}): - return true - -proc debugSend*(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = - ## Sends `cmd` on the socket connected to the SMTP server. - ## - ## If the `smtp` object was created with `debug` enabled, - ## debugSend will invoke `echo("C:" & cmd)` before sending. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - - if smtp.debug: - echo("C:" & cmd) - await smtp.sock.send(cmd) - -proc debugRecv*(smtp: Smtp | AsyncSmtp): Future[string] {.multisync.} = - ## Receives a line of data from the socket connected to the - ## SMTP server. - ## - ## If the `smtp` object was created with `debug` enabled, - ## debugRecv will invoke `echo("S:" & result.string)` after - ## the data is received. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - ## - ## See `checkReply(reply)<#checkReply,AsyncSmtp,string>`_. - - result = await smtp.sock.recvLine() - if smtp.debug: - echo("S:" & result) - -proc quitExcpt(smtp: Smtp, msg: string) = - smtp.debugSend("QUIT") - raise newException(ReplyError, msg) - -const compiledWithSsl = defined(ssl) - -when not defined(ssl): - let defaultSSLContext: SslContext = nil -else: - var defaultSSLContext {.threadvar.}: SslContext - - proc getSSLContext(): SslContext = - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) - result = defaultSSLContext - -proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string], - otherHeaders: openArray[tuple[name, value: string]]): Message = - ## Creates a new MIME compliant message. - ## - ## You need to make sure that `mSubject`, `mTo` and `mCc` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not mSubject.contains({'\c', '\L'}), - "'mSubject' shouldn't contain any newline characters") - doAssert(not (mTo.containsNewline() or mCc.containsNewline()), - "'mTo' and 'mCc' shouldn't contain any newline characters") - - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - for n, v in items(otherHeaders): - result.msgOtherHeaders[n] = v - -proc createMessage*(mSubject, mBody: string, mTo, - mCc: seq[string] = @[]): Message = - ## Alternate version of the above. - ## - ## You need to make sure that `mSubject`, `mTo` and `mCc` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not mSubject.contains({'\c', '\L'}), - "'mSubject' shouldn't contain any newline characters") - doAssert(not (mTo.containsNewline() or mCc.containsNewline()), - "'mTo' and 'mCc' shouldn't contain any newline characters") - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - -proc `$`*(msg: Message): string = - ## stringify for `Message`. - result = "" - if msg.msgTo.len() > 0: - result = "TO: " & msg.msgTo.join(", ") & "\c\L" - if msg.msgCc.len() > 0: - result.add("CC: " & msg.msgCc.join(", ") & "\c\L") - # TODO: Folding? i.e when a line is too long, shorten it... - result.add("Subject: " & msg.msgSubject & "\c\L") - for key, value in pairs(msg.msgOtherHeaders): - result.add(key & ": " & value & "\c\L") - - result.add("\c\L") - result.add(msg.msgBody) - -proc newSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): Smtp = - ## Creates a new `Smtp` instance. - new result - result.debug = debug - result.sock = newSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc newAsyncSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): AsyncSmtp = - ## Creates a new `AsyncSmtp` instance. - new result - result.debug = debug - - result.sock = newAsyncSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = - var retFuture = newFuture[void]() - var sendFut = smtp.debugSend("QUIT") - sendFut.callback = - proc () = - retFuture.fail(newException(ReplyError, msg)) - return retFuture - -proc checkReply*(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = - ## Calls `debugRecv<#debugRecv,AsyncSmtp>`_ and checks that the received - ## data starts with `reply`. If the received data does not start - ## with `reply`, then a `QUIT` command will be sent to the SMTP - ## server and a `ReplyError` exception will be raised. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - - var line = await smtp.debugRecv() - if not line.startsWith(reply): - await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) - -proc helo*(smtp: Smtp | AsyncSmtp) {.multisync.} = - # Sends the HELO request - await smtp.debugSend("HELO " & smtp.address & "\c\L") - await smtp.checkReply("250") - -proc connect*(smtp: Smtp | AsyncSmtp, - address: string, port: Port) {.multisync.} = - ## Establishes a connection with a SMTP server. - ## May fail with ReplyError or with a socket error. - smtp.address = address - await smtp.sock.connect(address, port) - await smtp.checkReply("220") - await smtp.helo() - -proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SslContext = nil) {.multisync.} = - ## Put the SMTP connection in TLS (Transport Layer Security) mode. - ## May fail with ReplyError - await smtp.debugSend("STARTTLS\c\L") - await smtp.checkReply("220") - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapConnectedSocket(smtp.sock, handshakeAsClient) - else: - sslContext.wrapConnectedSocket(smtp.sock, handshakeAsClient) - await smtp.helo() - else: - {.error: "SMTP module compiled without SSL support".} - -proc auth*(smtp: Smtp | AsyncSmtp, username, password: string) {.multisync.} = - ## Sends an AUTH command to the server to login as the `username` - ## using `password`. - ## May fail with ReplyError. - - await smtp.debugSend("AUTH LOGIN\c\L") - await smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" - # i.e "334 VXNlcm5hbWU6" - await smtp.debugSend(encode(username) & "\c\L") - await smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) - - await smtp.debugSend(encode(password) & "\c\L") - await smtp.checkReply("235") # Check whether the authentication was successful. - -proc sendMail*(smtp: Smtp | AsyncSmtp, fromAddr: string, - toAddrs: seq[string], msg: string) {.multisync.} = - ## Sends `msg` from `fromAddr` to the addresses specified in `toAddrs`. - ## Messages may be formed using `createMessage` by converting the - ## Message into a string. - ## - ## You need to make sure that `fromAddr` and `toAddrs` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not (toAddrs.containsNewline() or fromAddr.contains({'\c', '\L'})), - "'toAddrs' and 'fromAddr' shouldn't contain any newline characters") - - await smtp.debugSend("MAIL FROM:<" & fromAddr & ">\c\L") - await smtp.checkReply("250") - for address in items(toAddrs): - await smtp.debugSend("RCPT TO:<" & address & ">\c\L") - await smtp.checkReply("250") - - # Send the message - await smtp.debugSend("DATA " & "\c\L") - await smtp.checkReply("354") - await smtp.sock.send(msg & "\c\L") - await smtp.debugSend(".\c\L") - await smtp.checkReply("250") - -proc close*(smtp: Smtp | AsyncSmtp) {.multisync.} = - ## Disconnects from the SMTP server and closes the socket. - await smtp.debugSend("QUIT\c\L") - smtp.sock.close() - -when not defined(testing) and isMainModule: - # To test with a real SMTP service, create a smtp.ini file, e.g.: - # username = "" - # password = "" - # smtphost = "smtp.gmail.com" - # port = 465 - # use_tls = true - # sender = "" - # recipient = "" - - import parsecfg - - proc `[]`(c: Config, key: string): string = c.getSectionValue("", key) - - let - conf = loadConfig("smtp.ini") - msg = createMessage("Hello from Nim's SMTP!", - "Hello!\n Is this awesome or what?", @[conf["recipient"]]) - - assert conf["smtphost"] != "" - - proc async_test() {.async.} = - let client = newAsyncSmtp( - conf["use_tls"].parseBool, - debug = true - ) - await client.connect(conf["smtphost"], conf["port"].parseInt.Port) - await client.auth(conf["username"], conf["password"]) - await client.sendMail(conf["sender"], @[conf["recipient"]], $msg) - await client.close() - echo "async email sent" - - proc sync_test() = - var smtpConn = newSmtp( - conf["use_tls"].parseBool, - debug = true - ) - smtpConn.connect(conf["smtphost"], conf["port"].parseInt.Port) - smtpConn.auth(conf["username"], conf["password"]) - smtpConn.sendMail(conf["sender"], @[conf["recipient"]], $msg) - smtpConn.close() - echo "sync email sent" - - waitFor async_test() - sync_test() diff --git a/lib/pure/smtp.nim.cfg b/lib/pure/smtp.nim.cfg deleted file mode 100644 index 521e21de41..0000000000 --- a/lib/pure/smtp.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:ssl diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim index 2d2644ebe8..d60cd22eb6 100644 --- a/lib/pure/ssl_certs.nim +++ b/lib/pure/ssl_certs.nim @@ -10,7 +10,7 @@ ## The default locations can be overridden using the SSL_CERT_FILE and ## SSL_CERT_DIR environment variables. -import os, strutils +import std/[os, strutils] # FWIW look for files before scanning entire dirs. @@ -34,6 +34,7 @@ elif defined(linux): # Fedora/RHEL "/etc/pki/tls/certs", # Android + "/data/data/com.termux/files/usr/etc/tls/cert.pem", "/system/etc/security/cacerts", ] elif defined(bsd): @@ -87,7 +88,7 @@ when defined(haiku): proc find_paths_etc(architecture: cstring, baseDirectory: cint, subPath: cstring, flags: uint32, paths: var ptr UncheckedArray[cstring], - pathCount: var csize): int32 + pathCount: var csize_t): int32 {.importc, header: "<FindDirectory.h>".} proc free(p: pointer) {.importc, header: "<stdlib.h>".} @@ -125,12 +126,18 @@ iterator scanSSLCertificates*(useEnvVars = false): string = if fileExists(p): yield p elif dirExists(p): + # check if it's a dir where each cert is one file + # named by it's hasg + for fn in joinPath(p, "*.0").walkFiles: + yield p.normalizePathEnd(true) + break for fn in joinPath(p, "*").walkFiles(): + yield fn else: var paths: ptr UncheckedArray[cstring] - size: csize + size: csize_t let err = find_paths_etc( nil, B_FIND_PATH_DATA_DIRECTORY, "ssl/CARootCertificates.pem", B_FIND_PATH_EXISTING_ONLY, paths, size @@ -143,7 +150,7 @@ iterator scanSSLCertificates*(useEnvVars = false): string = # Certificates management on windows # when defined(windows) or defined(nimdoc): # -# import openssl +# import std/openssl # # type # PCCertContext {.final, pure.} = pointer diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 6e2d5fecdd..6a4fd8f013 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -42,7 +42,7 @@ runnableExamples: template `~=`(a, b: float): bool = almostEqual(a, b) - var statistics: RunningStat ## Must be var + var statistics: RunningStat # must be var statistics.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) doAssert statistics.n == 8 doAssert statistics.mean() ~= 2.0 @@ -55,6 +55,9 @@ runnableExamples: from std/math import FloatClass, sqrt, pow, round +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + {.push debugger: off.} # the user does not want to trace a part # of the standard library! {.push checks: off, line_dir: off, stack_trace: off.} @@ -76,7 +79,7 @@ type proc clear*(s: var RunningStat) = ## Resets `s`. s.n = 0 - s.min = toBiggestFloat(int.high) + s.min = 0.0 s.max = 0.0 s.sum = 0.0 s.mom1 = 0.0 @@ -86,11 +89,14 @@ proc clear*(s: var RunningStat) = proc push*(s: var RunningStat, x: float) = ## Pushes a value `x` for processing. - if s.n == 0: s.min = x + if s.n == 0: + s.min = x + s.max = x + else: + if s.min > x: s.min = x + if s.max < x: s.max = x inc(s.n) # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.min > x: s.min = x - if s.max < x: s.max = x s.sum += x let n = toFloat(s.n) let delta = x - s.mom1 diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index eb1c9cc146..8ae7fb3c1d 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -27,75 +27,79 @@ ## StringStream example ## -------------------- ## -## .. code-block:: Nim +## ```Nim +## import std/streams ## -## import std/streams +## var strm = newStringStream("""The first line +## the second line +## the third line""") ## -## var strm = newStringStream("""The first line -## the second line -## the third line""") +## var line = "" ## -## var line = "" +## while strm.readLine(line): +## echo line ## -## while strm.readLine(line): -## echo line +## # Output: +## # The first line +## # the second line +## # the third line ## -## # Output: -## # The first line -## # the second line -## # the third line -## -## strm.close() +## strm.close() +## ``` ## ## FileStream example ## ------------------ ## ## Write file stream example: ## -## .. code-block:: Nim +## ```Nim +## import std/streams ## -## import std/streams +## var strm = newFileStream("somefile.txt", fmWrite) +## var line = "" ## -## var strm = newFileStream("somefile.txt", fmWrite) -## var line = "" +## if not isNil(strm): +## strm.writeLine("The first line") +## strm.writeLine("the second line") +## strm.writeLine("the third line") +## strm.close() ## -## if not isNil(strm): -## strm.writeLine("The first line") -## strm.writeLine("the second line") -## strm.writeLine("the third line") -## strm.close() -## -## # Output (somefile.txt): -## # The first line -## # the second line -## # the third line +## # Output (somefile.txt): +## # The first line +## # the second line +## # the third line +## ``` ## ## Read file stream example: ## -## .. code-block:: Nim +## ```Nim +## import std/streams ## -## import std/streams +## var strm = newFileStream("somefile.txt", fmRead) +## var line = "" ## -## var strm = newFileStream("somefile.txt", fmRead) -## var line = "" +## if not isNil(strm): +## while strm.readLine(line): +## echo line +## strm.close() ## -## if not isNil(strm): -## while strm.readLine(line): -## echo line -## strm.close() -## -## # Output: -## # The first line -## # the second line -## # the third line +## # Output: +## # The first line +## # the second line +## # the third line +## ``` ## ## See also ## ======== ## * `asyncstreams module <asyncstreams.html>`_ -## * `io module <io.html>`_ for `FileMode enum <io.html#FileMode>`_ +## * `io module <syncio.html>`_ for `FileMode enum <syncio.html#FileMode>`_ import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + export FileMode + proc newEIO(msg: string): owned(ref IOError) = new(result) result.msg = msg @@ -111,7 +115,7 @@ type ## * That these fields here shouldn't be used directly. ## They are accessible so that a stream implementation can override them. closeImpl*: proc (s: Stream) - {.nimcall, raises: [Exception, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + {.nimcall, raises: [IOError, OSError], tags: [WriteIOEffect], gcsafe.} atEndImpl*: proc (s: Stream): bool {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} setPositionImpl*: proc (s: Stream, pos: int) @@ -170,10 +174,20 @@ proc close*(s: Stream) = ## See also: ## * `flush proc <#flush,Stream>`_ runnableExamples: - var strm = newStringStream("The first line\nthe second line\nthe third line") - ## do something... - strm.close() - if not isNil(s.closeImpl): s.closeImpl(s) + block: + let strm = newStringStream("The first line\nthe second line\nthe third line") + ## do something... + strm.close() + + block: + let strm = newFileStream("amissingfile.txt") + # deferring works even if newFileStream fails + defer: strm.close() + if not isNil(strm): + ## do something... + + if not isNil(s) and not isNil(s.closeImpl): + s.closeImpl(s) proc atEnd*(s: Stream): bool = ## Checks if more data can be read from `s`. Returns ``true`` if all data has @@ -240,6 +254,9 @@ proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int = result = s.readDataStrImpl(s, buffer, slice) else: # fallback + when declared(prepareMutation): + # buffer might potentially be a CoW literal with ARC + prepareMutation(buffer) result = s.readData(addr buffer[slice.a], slice.b + 1 - slice.a) template jsOrVmBlock(caseJsOrVm, caseElse: untyped): untyped = @@ -331,9 +348,9 @@ proc write*[T](s: Stream, x: T) = ## **Note:** Not available for JS backend. Use `write(Stream, string) ## <#write,Stream,string>`_ for now. ## - ## .. code-block:: Nim - ## - ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ```Nim + ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ``` runnableExamples: var strm = newStringStream("") strm.write("abcde") @@ -1191,6 +1208,11 @@ else: # after 1.3 or JS not defined proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = var s = StringStream(s) + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(buffer) # buffer might potentially be a CoW literal with ARC result = min(slice.b + 1 - slice.a, s.data.len - s.pos) if result > 0: jsOrVmBlock: @@ -1252,7 +1274,7 @@ else: # after 1.3 or JS not defined var s = StringStream(s) s.data = "" - proc newStringStream*(s: string = ""): owned StringStream = + proc newStringStream*(s: sink string = ""): owned StringStream = ## Creates a new stream from the string `s`. ## ## See also: @@ -1271,6 +1293,11 @@ else: # after 1.3 or JS not defined new(result) result.data = s + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose result.atEndImpl = ssAtEnd @@ -1331,7 +1358,7 @@ proc newFileStream*(f: File): owned FileStream = ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream ## from string. ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ is the same - ## as using `open proc <io.html#open,File,string,FileMode,int>`_ + ## as using `open proc <syncio.html#open,File,string,FileMode,int>`_ ## on Examples. ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a ## file stream from the file name and the mode. @@ -1370,7 +1397,7 @@ proc newFileStream*(filename: string, mode: FileMode = fmRead, ## Creates a new stream from the file named `filename` with the mode `mode`. ## ## If the file cannot be opened, `nil` is returned. See the `io module - ## <io.html>`_ for a list of available `FileMode enums <io.html#FileMode>`_. + ## <syncio.html>`_ for a list of available `FileMode enums <syncio.html#FileMode>`_. ## ## **Note:** ## * **This function returns nil in case of failure.** @@ -1455,7 +1482,7 @@ when false: # do not import windows as this increases compile times: discard else: - import posix + import std/posix proc hsSetPosition(s: FileHandleStream, pos: int) = discard lseek(s.handle, pos, SEEK_SET) @@ -1503,6 +1530,7 @@ when false: of fmReadWrite: flags = O_RDWR or int(O_CREAT) of fmReadWriteExisting: flags = O_RDWR of fmAppend: flags = O_WRONLY or int(O_CREAT) or O_APPEND + static: raiseAssert "unreachable" # handle bug #17888 var handle = open(filename, flags) if handle < 0: raise newEOS("posix.open() call failed") result = newFileHandleStream(handle) diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim index 7a501760be..99752a9ab0 100644 --- a/lib/pure/streamwrapper.nim +++ b/lib/pure/streamwrapper.nim @@ -11,7 +11,11 @@ ## ## **Since** version 1.2. -import deques, streams +import std/[deques, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions + type PipeOutStream*[T] = ref object of T @@ -87,14 +91,14 @@ proc newPipeOutStream*[T](s: sink (ref T)): owned PipeOutStream[T] = ## when setPosition/getPosition is called or write operation is performed. ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## import std/[osproc, streamwrapper] ## var ## p = startProcess(exePath) ## outStream = p.outputStream().newPipeOutStream() ## echo outStream.peekChar ## p.close() + ## ``` assert s.readDataImpl != nil diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 7ab3590388..7d093ebb37 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -74,6 +74,20 @@ runnableExamples: assert fmt"{123.456:13e}" == " 1.234560e+02" ##[ +# Expressions +]## +runnableExamples: + let x = 3.14 + assert fmt"{(if x!=0: 1.0/x else: 0):.5}" == "0.31847" + assert fmt"""{(block: + var res: string + for i in 1..15: + res.add (if i mod 15 == 0: "FizzBuzz" + elif i mod 5 == 0: "Buzz" + elif i mod 3 == 0: "Fizz" + else: $i) & " " + res)}""" == "1 2 Fizz 4 Buzz Fizz 7 8 Fizz Buzz 11 Fizz 13 14 FizzBuzz " +##[ # Debugging strings `fmt"{expr=}"` expands to `fmt"expr={expr}"` namely the text of the expression, @@ -119,17 +133,31 @@ runnableExamples: An expression like `&"{key} is {value:arg} {{z}}"` is transformed into: -.. code-block:: nim + ```nim var temp = newStringOfCap(educatedCapGuess) temp.formatValue(key, "") temp.add(" is ") temp.formatValue(value, arg) temp.add(" {z}") temp + ``` Parts of the string that are enclosed in the curly braces are interpreted -as Nim code, to escape a `{` or `}`, double it. +as Nim code. To escape a `{` or `}`, double it. +Within a curly expression, however, `{`, `}`, must be escaped with a backslash. + +To enable evaluating Nim expressions within curlies, colons inside parentheses +do not need to be escaped. +]## + +runnableExamples: + let x = "hello" + assert fmt"""{ "\{(" & x & ")\}" }""" == "{(hello)}" + assert fmt"""{{({ x })}}""" == "{(hello)}" + assert fmt"""{ $(\{x:1,"world":2\}) }""" == """[("hello", 1), ("world", 2)]""" + +##[ `&` delegates most of the work to an open overloaded set of `formatValue` procs. The required signature for a type `T` that supports formatting is usually `proc formatValue(result: var string; x: T; specifier: string)`. @@ -144,34 +172,34 @@ For strings and numeric types the optional argument is a so-called # Standard format specifiers for strings, integers and floats -The general form of a standard format specifier is:: +The general form of a standard format specifier is: - [[fill]align][sign][#][0][minimumwidth][.precision][type] + [[fill]align][sign][#][0][minimumwidth][.precision][type] The square brackets `[]` indicate an optional element. -The optional 'align' flag can be one of the following: +The optional `align` flag can be one of the following: -'<' - Forces the field to be left-aligned within the available +`<` +: Forces the field to be left-aligned within the available space. (This is the default for strings.) -'>' - Forces the field to be right-aligned within the available space. +`>` +: Forces the field to be right-aligned within the available space. (This is the default for numbers.) -'^' - Forces the field to be centered within the available space. +`^` +: Forces the field to be centered within the available space. Note that unless a minimum field width is defined, the field width will always be the same size as the data to fill it, so that the alignment option has no meaning in this case. -The optional 'fill' character defines the character to be used to pad +The optional `fill` character defines the character to be used to pad the field to the minimum width. The fill character, if present, must be followed by an alignment flag. -The 'sign' option is only valid for numeric types, and can be one of the following: +The `sign` option is only valid for numeric types, and can be one of the following: ================= ==================================================== Sign Meaning @@ -184,22 +212,22 @@ The 'sign' option is only valid for numeric types, and can be one of the followi positive numbers. ================= ==================================================== -If the '#' character is present, integers use the 'alternate form' for formatting. +If the `#` character is present, integers use the 'alternate form' for formatting. This means that binary, octal and hexadecimal output will be prefixed -with '0b', '0o' and '0x', respectively. +with `0b`, `0o` and `0x`, respectively. -'width' is a decimal integer defining the minimum field width. If not specified, +`width` is a decimal integer defining the minimum field width. If not specified, then the field width will be determined by the content. -If the width field is preceded by a zero ('0') character, this enables +If the width field is preceded by a zero (`0`) character, this enables zero-padding. -The 'precision' is a decimal number indicating how many digits should be displayed +The `precision` is a decimal number indicating how many digits should be displayed after the decimal point in a floating point conversion. For non-numeric types the field indicates the maximum field size - in other words, how many characters will be used from the field content. The precision is ignored for integer conversions. -Finally, the 'type' determines how the data should be presented. +Finally, the `type` determines how the data should be presented. The available integer presentation types are: @@ -213,7 +241,7 @@ The available integer presentation types are: lower-case letters for the digits above 9. `X` Hex format. Outputs the number in base 16, using uppercase letters for the digits above 9. -(None) The same as 'd'. +(None) The same as `d`. ================= ==================================================== The available floating point presentation types are: @@ -222,21 +250,27 @@ The available floating point presentation types are: Type Result ================= ==================================================== `e` Exponent notation. Prints the number in scientific - notation using the letter 'e' to indicate the + notation using the letter `e` to indicate the exponent. -`E` Exponent notation. Same as 'e' except it converts +`E` Exponent notation. Same as `e` except it converts the number to uppercase. `f` Fixed point. Displays the number as a fixed-point number. -`F` Fixed point. Same as 'f' except it converts the +`F` Fixed point. Same as `f` except it converts the number to uppercase. `g` General format. This prints the number as a fixed-point number, unless the number is too - large, in which case it switches to 'e' + large, in which case it switches to `e` exponent notation. -`G` General format. Same as 'g' except it switches to 'E' +`G` General format. Same as `g` except it switches to `E` if the number gets to large. -(None) Similar to 'g', except that it prints at least one +`i` Complex General format. This is only supported for + complex numbers, which it prints using the mathematical + (RE+IMj) format. The real and imaginary parts are printed + using the general format `g` by default, but it is + possible to combine this format with one of the other + formats (e.g `jf`). +(None) Similar to `g`, except that it prints at least one digit after the decimal point. ================= ==================================================== @@ -245,13 +279,14 @@ The available floating point presentation types are: Because of the well defined order how templates and macros are expanded, strformat cannot expand template arguments: -.. code-block:: nim + ```nim template myTemplate(arg: untyped): untyped = echo "arg is: ", arg echo &"--- {arg} ---" let x = "abc" myTemplate(x) + ``` First the template `myTemplate` is expanded, where every identifier `arg` is substituted with its argument. The `arg` inside the @@ -262,12 +297,13 @@ identifier that cannot be resolved anymore. The workaround for this is to bind the template argument to a new local variable. -.. code-block:: nim + ```nim template myTemplate(arg: untyped): untyped = block: let arg1 {.inject.} = arg echo "arg is: ", arg1 echo &"--- {arg1} ---" + ``` The use of `{.inject.}` here is necessary again because of template expansion order and hygienic templates. But since we generally want to @@ -275,6 +311,7 @@ keep the hygiene of `myTemplate`, and we do not want `arg1` to be injected into the context where `myTemplate` is expanded, everything is wrapped in a `block`. + # Future directions A curly expression with commas in it like `{x, argA, argB}` could be @@ -288,6 +325,10 @@ single letter DSLs. import std/[macros, parseutils, unicode] import std/strutils except format +when defined(nimPreviewSlimSystem): + import std/assertions + + proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) if v < 10: @@ -391,9 +432,9 @@ proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string proc parseStandardFormatSpecifier*(s: string; start = 0; ignoreUnknownSuffix = false): StandardFormatSpecifier = ## An exported helper proc that parses the "standard format specifiers", - ## as specified by the grammar:: + ## as specified by the grammar: ## - ## [[fill]align][sign][#][0][minimumwidth][.precision][type] + ## [[fill]align][sign][#][0][minimumwidth][.precision][type] ## ## This is only of interest if you want to write a custom `format` proc that ## should support the standard format specifiers. If `ignoreUnknownSuffix` is true, @@ -440,6 +481,31 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; raise newException(ValueError, "invalid format string, cannot parse: " & s[i..^1]) +proc toRadix(typ: char): int = + case typ + of 'x', 'X': 16 + of 'd', '\0': 10 + of 'o': 8 + of 'b': 2 + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'x', 'X', 'b', 'd', 'o' but got: " & typ) + +proc formatValue*[T: SomeInteger](result: var string; value: T; + specifier: static string) = + ## Standard format implementation for `SomeInteger`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + when specifier.len == 0: + result.add $value + else: + const + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) + + result.add formatInt(value, radix, spec) + proc formatValue*[T: SomeInteger](result: var string; value: T; specifier: string) = ## Standard format implementation for `SomeInteger`. It makes little @@ -447,43 +513,16 @@ proc formatValue*[T: SomeInteger](result: var string; value: T; ## by the `&` macro. if specifier.len == 0: result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - var radix = 10 - case spec.typ - of 'x', 'X': radix = 16 - of 'd', '\0': discard - of 'b': radix = 2 - of 'o': radix = 8 else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) - result.add formatInt(value, radix, spec) + let + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) -proc formatValue*(result: var string; value: SomeFloat; specifier: string) = - ## Standard format implementation for `SomeFloat`. It makes little - ## sense to call this directly, but it is required to exist - ## by the `&` macro. - if specifier.len == 0: - result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - - var fmode = ffDefault - case spec.typ - of 'e', 'E': - fmode = ffScientific - of 'f', 'F': - fmode = ffDecimal - of 'g', 'G': - fmode = ffDefault - of '\0': discard - else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) + result.add formatInt(value, radix, spec) +proc formatFloat( + result: var string, value: SomeFloat, fmode: FloatFormatMode, + spec: StandardFormatSpecifier) = var f = formatBiggestFloat(value, fmode, spec.precision) var sign = false if value >= 0.0: @@ -518,23 +557,83 @@ proc formatValue*(result: var string; value: SomeFloat; specifier: string) = else: result.add res +proc toFloatFormatMode(typ: char): FloatFormatMode = + case typ + of 'e', 'E': ffScientific + of 'f', 'F': ffDecimal + of 'g', 'G': ffDefault + of '\0': ffDefault + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & typ) + +proc formatValue*(result: var string; value: SomeFloat; specifier: static string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + when specifier.len == 0: + result.add $value + else: + const + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: SomeFloat; specifier: string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + if specifier.len == 0: + result.add $value + else: + let + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: string; specifier: static string) = + ## Standard format implementation for `string`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + const spec = parseStandardFormatSpecifier(specifier) + var value = + when spec.typ in {'s', '\0'}: value + else: static: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + when spec.precision != -1: + if spec.precision < runeLen(value): + const precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + proc formatValue*(result: var string; value: string; specifier: string) = ## Standard format implementation for `string`. It makes little ## sense to call this directly, but it is required to exist ## by the `&` macro. let spec = parseStandardFormatSpecifier(specifier) - var value = value - case spec.typ - of 's', '\0': discard - else: - raise newException(ValueError, - "invalid type in format string for string, expected 's', but got " & - spec.typ) + var value = + if spec.typ in {'s', '\0'}: value + else: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) if spec.precision != -1: if spec.precision < runeLen(value): - setLen(value, runeOffset(value, spec.precision)) + let precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) +proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: static string) = + mixin `$` + formatValue(result, $value, specifier) + proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: string) = mixin `$` formatValue(result, $value, specifier) @@ -545,19 +644,20 @@ template formatValue(result: var string; value: char; specifier: string) = template formatValue(result: var string; value: cstring; specifier: string) = result.add value -proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = - if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "string formatting (fmt(), &) only works with string literals", pattern +proc strformatImpl(f: string; openChar, closeChar: char, + lineInfoNode: NimNode = nil): NimNode = + template missingCloseChar = + error("invalid format string: missing closing character '" & closeChar & "'") + if openChar == ':' or closeChar == ':': error "openChar and closeChar must not be ':'" - let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") - result = newNimNode(nnkStmtListExpr, lineInfoFrom = pattern) + result = newNimNode(nnkStmtListExpr, lineInfoNode) # XXX: https://github.com/nim-lang/Nim/issues/8405 # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils # module are not accessible at compile-time: - let expectedGrowth = when defined(useNimRtl): 0 else: count(f, '{') * 10 + let expectedGrowth = when defined(useNimRtl): 0 else: count(f, openChar) * 10 result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + expectedGrowth))) var strlit = "" @@ -573,28 +673,46 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = strlit = "" var subexpr = "" - while i < f.len and f[i] != closeChar and f[i] != ':': - if f[i] == '=': + var inParens = 0 + var inSingleQuotes = false + var inDoubleQuotes = false + template notEscaped:bool = f[i-1]!='\\' + while i < f.len and f[i] != closeChar and (f[i] != ':' or inParens != 0): + case f[i] + of '\\': + if i < f.len-1 and f[i+1] in {openChar,closeChar,':'}: inc i + of '\'': + if not inDoubleQuotes and notEscaped: inSingleQuotes = not inSingleQuotes + of '\"': + if notEscaped: inDoubleQuotes = not inDoubleQuotes + of '(': + if not (inSingleQuotes or inDoubleQuotes): inc inParens + of ')': + if not (inSingleQuotes or inDoubleQuotes): dec inParens + of '=': let start = i inc i i += f.skipWhitespace(i) + if i == f.len: + missingCloseChar if f[i] == closeChar or f[i] == ':': result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) else: subexpr.add f[start ..< i] - else: - subexpr.add f[i] - inc i + continue + else: discard + subexpr.add f[i] + inc i + + if i == f.len: + missingCloseChar var x: NimNode try: x = parseExpr(subexpr) - except ValueError: - when declared(getCurrentExceptionMsg): - let msg = getCurrentExceptionMsg() - error("could not parse `" & subexpr & "`.\n" & msg, pattern) - else: - error("could not parse `" & subexpr & "`.\n", pattern) + except ValueError as e: + error("could not parse `$#` in `$#`.\n$#" % [subexpr, f, e.msg]) + x.copyLineInfo(lineInfoNode) let formatSym = bindSym("formatValue", brOpen) var options = "" if f[i] == ':': @@ -602,40 +720,71 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = while i < f.len and f[i] != closeChar: options.add f[i] inc i + if i == f.len: + missingCloseChar if f[i] == closeChar: inc i - else: - doAssert false, "invalid format string: missing '}'" result.add newCall(formatSym, res, x, newLit(options)) elif f[i] == closeChar: - if f[i+1] == closeChar: + if i<f.len-1 and f[i+1] == closeChar: strlit.add closeChar inc i, 2 else: - doAssert false, "invalid format string: '}' instead of '}}'" - inc i + raiseAssert "invalid format string: '$1' instead of '$1$1'" % $closeChar else: strlit.add f[i] inc i if strlit.len > 0: result.add newCall(bindSym"add", res, newLit(strlit)) result.add res + # workaround for #20381 + var blockExpr = newNimNode(nnkBlockExpr, lineInfoNode) + blockExpr.add(newEmptyNode()) + blockExpr.add(result) + result = blockExpr when defined(debugFmtDsl): echo repr result -macro `&`*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## For a specification of the `&` macro, see the module level documentation. +macro fmt(pattern: static string; openChar: static char, closeChar: static char, lineInfoNode: untyped): string = + ## version of `fmt` with dummy untyped param for line info + strformatImpl(pattern, openChar, closeChar, lineInfoNode) -macro fmt*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## An alias for `& <#&.m,string>`_. +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} -macro fmt*(pattern: string; openChar, closeChar: char): untyped = - ## The same as `fmt <#fmt.m,string>`_, but uses `openChar` instead of `'{'` - ## and `closeChar` instead of `'}'`. +template fmt*(pattern: static string; openChar: static char, closeChar: static char): string {.callsite.} = + ## Interpolates `pattern` using symbols in scope. runnableExamples: - let testInt = 123 - assert "<testInt>".fmt('<', '>') == "123" - assert """(()"foo" & "bar"())""".fmt(')', '(') == "(foobar)" - assert """ ""{"123+123"}"" """.fmt('"', '"') == " \"{246}\" " + let x = 7 + assert "var is {x * 2}".fmt == "var is 14" + assert "var is {{x}}".fmt == "var is {x}" # escape via doubling + const s = "foo: {x}" + assert s.fmt == "foo: 7" # also works with const strings - strformatImpl(pattern, openChar.intVal.char, closeChar.intVal.char) + assert fmt"\n" == r"\n" # raw string literal + assert "\n".fmt == "\n" # regular literal (likewise with `fmt("\n")` or `fmt "\n"`) + runnableExamples: + # custom `openChar`, `closeChar` + let x = 7 + assert "<x>".fmt('<', '>') == "7" + assert "<<<x>>>".fmt('<', '>') == "<7>" + assert "`x`".fmt('`', '`') == "7" + fmt(pattern, openChar, closeChar, dummyForLineInfo) + +template fmt*(pattern: static string): untyped {.callsite.} = + ## Alias for `fmt(pattern, '{', '}')`. + fmt(pattern, '{', '}', dummyForLineInfo) + +template `&`*(pattern: string{lit}): string {.callsite.} = + ## `&pattern` is the same as `pattern.fmt`. + ## For a specification of the `&` macro, see the module level documentation. + # pending bug #18275, bug #18278, use `pattern: static string` + # consider deprecating this, it's redundant with `fmt` and `fmt` is strictly + # more flexible, readable (no confusion with the binary `&`), self-documenting, + # not to mention #18275, bug #18278. + runnableExamples: + let x = 7 + assert &"{x}\n" == "7\n" # regular string literal + assert &"{x}\n" == "{x}\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"{x}\n" # see `fmt` docs, this would use a raw string literal + fmt(pattern, '{', '}', dummyForLineInfo) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 6bedf2de25..16ef9e6428 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -12,7 +12,7 @@ This module contains a `scanf`:idx: macro that can be used for extracting substrings from an input string. This is often easier than regular expressions. Some examples as an appetizer: -.. code-block:: nim + ```nim # check if input string matches a triple of integers: const input = "(1,2,4)" var x, y, z: int @@ -26,6 +26,7 @@ Some examples as an appetizer: var myfloat: float if scanf(input, "$i-$i-$i $w$s$f", year, month, day, identifier, myfloat): echo "yes, we have a match!" + ``` As can be seen from the examples, strings are matched verbatim except for substrings starting with ``$``. These constructions are available: @@ -35,7 +36,7 @@ substrings starting with ``$``. These constructions are available: ``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. ``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. ``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. -``$f`` Matches a floating pointer number. Uses ``parseFloat``. +``$f`` Matches a floating-point number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Za-z_][A-Za-z_0-9]*``. ``$c`` Matches a single ASCII character. ``$s`` Skips optional whitespace. @@ -52,7 +53,7 @@ substrings starting with ``$``. These constructions are available: ================= ======================================================== Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*`` -and ``.+`` they work quite differently, there is no non-deterministic +and ``.+``, they work quite differently. There is no non-deterministic state machine involved and the matches are non-greedy. ``[$*]`` matches ``[xyz]`` via ``parseutils.parseUntil``. @@ -83,8 +84,7 @@ matches optional tokens without any result binding. In this example, we define a helper proc ``someSep`` that skips some separators which we then use in our scanf pattern to help us in the matching process: -.. code-block:: nim - + ```nim proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = # Note: The parameters and return value must match to what ``scanf`` requires result = 0 @@ -92,11 +92,11 @@ which we then use in our scanf pattern to help us in the matching process: if scanf(input, "$w$[someSep]$w", key, value): ... + ``` -It also possible to pass arguments to a user definable matcher: - -.. code-block:: nim +It is also possible to pass arguments to a user definable matcher: + ```nim proc ndigits(input: string; intVal: var int; start: int; n: int): int = # matches exactly ``n`` digits. Matchers need to return 0 if nothing # matched or otherwise the number of processed chars. @@ -115,6 +115,7 @@ It also possible to pass arguments to a user definable matcher: var year, month, day: int if scanf("2013-01-03", "${ndigits(4)}-${ndigits(2)}-${ndigits(2)}$.", year, month, day): ... + ``` The scanp macro @@ -145,8 +146,7 @@ not implemented. Simple example that parses the ``/etc/passwd`` file line by line: -.. code-block:: nim - + ```nim const etc_passwd = """root:x:0:0:root:/root:/bin/bash daemon:x:1:1:daemon:/usr/sbin:/bin/sh @@ -165,16 +165,16 @@ Simple example that parses the ``/etc/passwd`` file line by line: result.add entry else: break + ``` The ``scanp`` maps the grammar code into Nim code that performs the parsing. The parsing is performed with the help of 3 helper templates that that can be implemented for a custom type. These templates need to be named ``atom`` and ``nxt``. ``atom`` should be -overloaded to handle both single characters and sets of character. - -.. code-block:: nim +overloaded to handle both `char` and `set[char]`. + ```nim import std/streams template atom(input: Stream; idx: int; c: char): bool = @@ -190,11 +190,11 @@ overloaded to handle both single characters and sets of character. if scanp(content, idx, +( ~{'\L', '\0'} -> entry.add(peekChar($input))), '\L'): result.add entry + ``` Calling ordinary Nim procs inside the macro is possible: -.. code-block:: nim - + ```nim proc digits(s: string; intVal: var int; start: int): int = var x = 0 while result+start < s.len and s[result+start] in {'0'..'9'} and s[result+start] != ':': @@ -220,12 +220,12 @@ Calling ordinary Nim procs inside the macro is possible: result.add login & " " & homedir else: break + ``` When used for matching, keep in mind that likewise scanf, no backtracking is performed. -.. code-block:: nim - + ```nim proc skipUntil(s: string; until: string; unless = '\0'; start: int): int = # Skips all characters until the string `until` is found. Returns 0 # if the char `unless` is found first or the end is reached. @@ -256,12 +256,12 @@ is performed. for r in collectLinks(body): echo r + ``` In this example both macros are combined seamlessly in order to maximise efficiency and perform different checks. -.. code-block:: nim - + ```nim iterator parseIps*(soup: string): string = ## ipv4 only! const digits = {'0'..'9'} @@ -279,13 +279,17 @@ efficiency and perform different checks. yield buf buf.setLen(0) # need to clear `buf` each time, cause it might contain garbage idx.inc - + ``` ]## -import macros, parseutils +import std/[macros, parseutils] import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + proc conditionsToIfChain(n, idx, res: NimNode; start: int): NimNode = assert n.kind == nnkStmtList if start >= n.len: return newAssignment(res, newLit true) @@ -468,7 +472,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b macro scanTuple*(input: untyped; pattern: static[string]; matcherTypes: varargs[untyped]): untyped {.since: (1, 5).}= ## Works identically as scanf, but instead of predeclaring variables it returns a tuple. - ## Tuple is started with a bool which indicates if the scan was successful + ## Tuple is started with a bool which indicates if the scan was successful ## followed by the requested data. ## If using a user defined matcher, provide the types in order they appear after pattern: ## `line.scanTuple("${yourMatcher()}", int)` @@ -508,7 +512,7 @@ macro scanTuple*(input: untyped; pattern: static[string]; matcherTypes: varargs[ inc userMatches else: discard inc p - result.add newPar(newCall(ident("scanf"), input, newStrLitNode(pattern))) + result.add nnkTupleConstr.newTree(newCall(ident("scanf"), input, newStrLitNode(pattern))) for arg in arguments: result[^1][0].add arg result[^1].add arg @@ -577,13 +581,14 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = of nnkCallKinds: # *{'A'..'Z'} !! s.add(!_) template buildWhile(input, idx, init, cond, action): untyped = + mixin hasNxt while hasNxt(input, idx): init if not cond: break action # (x) a # bind action a to (x) - if it[0].kind == nnkPar and it.len == 2: + if it[0].kind in {nnkPar, nnkTupleConstr} and it.len == 2: result = atm(it[0], input, idx, placeholder(it[1], input, idx)) elif it.kind == nnkInfix and it[0].eqIdent"->": # bind matching to some action: @@ -688,4 +693,4 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = result.add toIfChain(conds, idx, res, 0) result.add res when defined(debugScanp): - echo repr result \ No newline at end of file + echo repr result diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 3b90fea509..4b07aca5a3 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -51,13 +51,17 @@ runnableExamples: import std/private/since import - hashes, strutils + std/[hashes, strutils] + +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(js) or defined(nimscript) or defined(Standalone): {.pragma: rtlFunc.} else: {.pragma: rtlFunc, rtl.} - import os + import std/envvars include "system/inclrtl" @@ -257,10 +261,7 @@ proc newStringTable*(mode: StringTableMode): owned(StringTableRef) {. ## See also: ## * `newStringTable(keyValuePairs) proc ## <#newStringTable,varargs[tuple[string,string]],StringTableMode>`_ - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) + result = StringTableRef(mode: mode, counter: 0, data: newSeq[KeyValuePair](startSize)) proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): owned(StringTableRef) {. diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 36a2115456..a9e3df53e4 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -70,17 +70,21 @@ runnableExamples: ## easier substring extraction than regular expressions -import parseutils -from math import pow, floor, log10 -from algorithm import reverse +import std/parseutils +from std/math import pow, floor, log10 +from std/algorithm import fill, reverse import std/enumutils -from unicode import toLower, toUpper +from std/unicode import toLower, toUpper export toLower, toUpper include "system/inclrtl" -import std/private/since -from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, startsWithImpl, endsWithImpl +import std/private/[since, jsutils] +from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, + startsWithImpl, endsWithImpl + +when defined(nimPreviewSlimSystem): + import std/assertions const @@ -91,6 +95,15 @@ const Letters* = {'A'..'Z', 'a'..'z'} ## The set of letters. + UppercaseLetters* = {'A'..'Z'} + ## The set of uppercase ASCII letters. + + LowercaseLetters* = {'a'..'z'} + ## The set of lowercase ASCII letters. + + PunctuationChars* = {'!'..'/', ':'..'@', '['..'`', '{'..'~'} + ## The set of all ASCII punctuation characters. + Digits* = {'0'..'9'} ## The set of digits. @@ -107,17 +120,20 @@ const ## The set of characters a newline terminator can start with (carriage ## return, line feed). + PrintableChars* = Letters + Digits + PunctuationChars + Whitespace + ## The set of all printable ASCII characters (letters, digits, whitespace, and punctuation characters). + AllChars* = {'\x00'..'\xFF'} ## A set with all the possible characters. ## ## Not very useful by its own, you can use it to create *inverted* sets to ## make the `find func<#find,string,set[char],Natural,int>`_ ## find **invalid** characters in strings. Example: - ## - ## .. code-block:: nim + ## ```nim ## let invalid = AllChars - Digits ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 + ## ``` func isAlphaAscii*(c: char): bool {.rtl, extern: "nsuIsAlphaAsciiChar".} = ## Checks whether or not character `c` is alphabetical. @@ -169,7 +185,7 @@ func isLowerAscii*(c: char): bool {.rtl, extern: "nsuIsLowerAsciiChar".} = doAssert isLowerAscii('e') == true doAssert isLowerAscii('E') == false doAssert isLowerAscii('7') == false - return c in {'a'..'z'} + return c in LowercaseLetters func isUpperAscii*(c: char): bool {.rtl, extern: "nsuIsUpperAsciiChar".} = ## Checks whether or not `c` is an upper case character. @@ -183,8 +199,7 @@ func isUpperAscii*(c: char): bool {.rtl, extern: "nsuIsUpperAsciiChar".} = doAssert isUpperAscii('e') == false doAssert isUpperAscii('E') == true doAssert isUpperAscii('7') == false - return c in {'A'..'Z'} - + return c in UppercaseLetters func toLowerAscii*(c: char): char {.rtl, extern: "nsuToLowerAsciiChar".} = ## Returns the lower case version of character `c`. @@ -199,7 +214,7 @@ func toLowerAscii*(c: char): char {.rtl, extern: "nsuToLowerAsciiChar".} = runnableExamples: doAssert toLowerAscii('A') == 'a' doAssert toLowerAscii('e') == 'e' - if c in {'A'..'Z'}: + if c in UppercaseLetters: result = char(uint8(c) xor 0b0010_0000'u8) else: result = c @@ -236,7 +251,7 @@ func toUpperAscii*(c: char): char {.rtl, extern: "nsuToUpperAsciiChar".} = runnableExamples: doAssert toUpperAscii('a') == 'A' doAssert toUpperAscii('E') == 'E' - if c in {'a'..'z'}: + if c in LowercaseLetters: result = char(uint8(c) xor 0b0010_0000'u8) else: result = c @@ -273,14 +288,20 @@ func nimIdentNormalize*(s: string): string = ## ## That means to convert to lower case and remove any '_' on all characters ## except first one. + ## + ## .. Warning:: Backticks (`) are not handled: they remain *as is* and + ## spaces are preserved. See `nimIdentBackticksNormalize + ## <dochelpers.html#nimIdentBackticksNormalize,string>`_ for + ## an alternative approach. runnableExamples: doAssert nimIdentNormalize("Foo_bar") == "Foobar" result = newString(s.len) - if s.len > 0: - result[0] = s[0] + if s.len == 0: + return + result[0] = s[0] var j = 1 for i in 1..len(s) - 1: - if s[i] in {'A'..'Z'}: + if s[i] in UppercaseLetters: result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) inc j elif s[i] != '_': @@ -302,7 +323,7 @@ func normalize*(s: string): string {.rtl, extern: "nsuNormalize".} = result = newString(s.len) var j = 0 for i in 0..len(s) - 1: - if s[i] in {'A'..'Z'}: + if s[i] in UppercaseLetters: result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) inc j elif s[i] != '_': @@ -313,9 +334,9 @@ func normalize*(s: string): string {.rtl, extern: "nsuNormalize".} = func cmpIgnoreCase*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreCase("FooBar", "foobar") == 0 doAssert cmpIgnoreCase("bar", "Foo") < 0 @@ -333,9 +354,9 @@ func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = ## ## Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreStyle("foo_bar", "FooBar") == 0 doAssert cmpIgnoreStyle("foo_bar_5", "FooBar4") > 0 @@ -345,11 +366,14 @@ func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = # --------- Private templates for different split separators ----------- func substrEq(s: string, pos: int, substr: string): bool = - var i = 0 + # Always returns false for empty `substr` var length = substr.len - while i < length and pos+i < s.len and s[pos+i] == substr[i]: - inc i - return i == length + if length > 0: + var i = 0 + while i < length and pos+i < s.len and s[pos+i] == substr[i]: + inc i + i == length + else: false template stringHasSep(s: string, index: int, seps: set[char]): bool = s[index] in seps @@ -399,14 +423,12 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## ## Substrings are separated by the character `sep`. ## The code: - ## - ## .. code-block:: nim + ## ```nim ## for word in split(";;this;is;an;;example;;;", ';'): ## writeLine(stdout, word) - ## + ## ``` ## Results in: - ## - ## .. code-block:: + ## ``` ## "" ## "" ## "this" @@ -417,6 +439,7 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## "" ## "" ## "" + ## ``` ## ## See also: ## * `rsplit iterator<#rsplit.i,string,char,int>`_ @@ -431,41 +454,49 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## ## Substrings are separated by a substring containing only `seps`. ## - ## .. code-block:: nim + ## ```nim ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) + ## ``` ## ## ...generates this output: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "an" ## "example" + ## ``` ## ## And the following code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) + ## ``` ## ## ...produces the same output as the first example. The code: ## - ## .. code-block:: nim + ## ```nim ## let date = "2012-11-20T22:08:08.398990" ## let separators = {' ', '-', ':', 'T'} ## for number in split(date, separators): ## writeLine(stdout, number) + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## "2012" ## "11" ## "20" ## "22" ## "08" ## "08.398990" + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,set[char],int>`_ @@ -480,23 +511,30 @@ iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Substrings are separated by the string `sep`. ## The code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("thisDATAisDATAcorrupted", "DATA"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "corrupted" + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,string,int,bool>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ ## * `split func<#split,string,string,int>`_ - splitCommon(s, sep, maxsplit, sep.len) + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + splitCommon(s, sep, maxsplit, sepLen) template rsplitCommon(s, sep, maxsplit, sepLen) = @@ -527,17 +565,19 @@ iterator rsplit*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo:bar".rsplit(':'): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the char `sep`. ## @@ -552,20 +592,25 @@ iterator rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo bar".rsplit(WhiteSpace): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the set of chars `seps` ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,set[char],int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ @@ -577,44 +622,52 @@ iterator rsplit*(s: string, sep: string, maxsplit: int = -1, keepSeparators: bool = false): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,string,int>`_ except in reverse order. + ## <#split.i,string,string,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foothebar".rsplit("the"): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the string `sep` ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,string,int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ ## * `rsplit func<#rsplit,string,string,int>`_ - rsplitCommon(s, sep, maxsplit, sep.len) + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + rsplitCommon(s, sep, maxsplit, sepLen) iterator splitLines*(s: string, keepEol = false): string = ## Splits the string `s` into its containing lines. ## ## Every `character literal <manual.html#lexical-analysis-character-literals>`_ ## newline combination (CR, LF, CR-LF) is supported. The result strings - ## contain no trailing end of line characters unless parameter `keepEol` + ## contain no trailing end of line characters unless the parameter `keepEol` ## is set to `true`. ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## for line in splitLines("\nthis\nis\nan\n\nexample\n"): ## writeLine(stdout, line) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "" ## "this" ## "is" @@ -622,6 +675,7 @@ iterator splitLines*(s: string, keepEol = false): string = ## "" ## "example" ## "" + ## ``` ## ## See also: ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ @@ -654,16 +708,17 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## ## The following code: ## - ## .. code-block:: nim + ## ```nim ## let s = " foo \t bar baz " ## for ms in [-1, 1, 2, 3]: ## echo "------ maxsplit = ", ms, ":" ## for item in s.splitWhitespace(maxsplit=ms): ## echo '"', item, '"' + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## ------ maxsplit = -1: ## "foo" ## "bar" @@ -679,6 +734,7 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## "foo" ## "bar" ## "baz" + ## ``` ## ## See also: ## * `splitLines iterator<#splitLines.i,string>`_ @@ -707,6 +763,9 @@ func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ ## The same as the `split iterator <#split.i,string,set[char],int>`_ (see its ## documentation), but is a func that returns a sequence of substrings. ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator <#split.i,string,set[char],int>`_ ## * `rsplit func<#rsplit,string,set[char],int>`_ @@ -715,6 +774,7 @@ func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ runnableExamples: doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] doAssert "".split({' '}) == @[""] + doAssert "empty seps return unsplit s".split({}) == @["empty seps return unsplit s"] accResult(split(s, seps, maxsplit)) func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, @@ -724,6 +784,9 @@ func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string,int>`_. ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator <#split.i,string,string,int>`_ ## * `rsplit func<#rsplit,string,string,int>`_ @@ -736,14 +799,13 @@ func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] doAssert "a largely spaced sentence".split(" ", maxsplit = 1) == @["a", " largely spaced sentence"] - doAssert(sep.len > 0) - + doAssert "empty sep returns unsplit s".split("") == @["empty sep returns unsplit s"] accResult(split(s, sep, maxsplit)) func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitChar".} = ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a func - ## that returns a sequence of substrings. + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -751,13 +813,15 @@ func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` ## ## See also: ## * `rsplit iterator <#rsplit.i,string,char,int>`_ @@ -771,7 +835,7 @@ func rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitCharSet".} = ## The same as the `rsplit iterator <#rsplit.i,string,set[char],int>`_, but is a - ## func that returns a sequence of substrings. + ## func that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -779,13 +843,18 @@ func rsplit*(s: string, seps: set[char] = Whitespace, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", {'#'}, maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,set[char],int>`_ @@ -798,7 +867,7 @@ func rsplit*(s: string, seps: set[char] = Whitespace, func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitString".} = ## The same as the `rsplit iterator <#rsplit.i,string,string,int,bool>`_, but is a func - ## that returns a sequence of substrings. + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -806,13 +875,18 @@ func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", "#", maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,string,int,bool>`_ @@ -828,6 +902,7 @@ func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, doAssert "".rsplit("Elon Musk") == @[""] doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] + doAssert "empty sep returns unsplit s".rsplit("") == @["empty sep returns unsplit s"] accResult(rsplit(s, sep, maxsplit)) result.reverse() @@ -923,14 +998,26 @@ func toHex*[T: SomeInteger](x: T, len: Positive): string = doAssert b.toHex(4) == "1001" doAssert toHex(62, 3) == "03E" doAssert toHex(-8, 6) == "FFFFF8" - toHexImpl(cast[BiggestUInt](x), len, x < 0) + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), len, x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), len, x < 0) + else: + toHexImpl(BiggestUInt(x), len, x < 0) func toHex*[T: SomeInteger](x: T): string = ## Shortcut for `toHex(x, T.sizeof * 2)` runnableExamples: doAssert toHex(1984'i64) == "00000000000007C0" doAssert toHex(1984'i16) == "07C0" - toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), 2*sizeof(T), x < 0) + else: + toHexImpl(BiggestUInt(x), 2*sizeof(T), x < 0) func toHex*(s: string): string {.rtl.} = ## Converts a bytes string to its hexadecimal representation. @@ -1476,12 +1563,40 @@ func dedent*(s: string, count: Natural = indentation(s)): string {.rtl, doAssert x == "Hello\n There\n" unindent(s, count, " ") -func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} = - ## Deletes in `s` (must be declared as `var`) the characters at positions - ## `first .. last` (both ends included). +func delete*(s: var string, slice: Slice[int]) = + ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains + ## elements out of range. ## - ## This modifies `s` itself, it does not return a copy. + ## This operation moves all elements after `s[slice]` in linear time, and + ## is the string analog to `sequtils.delete`. runnableExamples: + var a = "abcde" + doAssertRaises(IndexDefect): a.delete(4..5) + assert a == "abcde" + a.delete(4..4) + assert a == "abcd" + a.delete(1..2) + assert a == "ad" + a.delete(1..<1) # empty slice + assert a == "ad" + when compileOption("boundChecks"): + if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + # if j < s.len: moveMem(addr s[i], addr s[j], s.len - j) # pending benchmark + while i < newLen: + s[i] = s[j] + inc(i) + inc(j) + setLen(s, newLen) + +func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete", + deprecated: "use `delete(s, first..last)`".} = + ## Deletes in `s` the characters at positions `first .. last` (both ends included). + runnableExamples("--warning:deprecated:off"): var a = "abracadabra" a.delete(4, 5) @@ -1502,7 +1617,6 @@ func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} = inc(j) setLen(s, newLen) - func startsWith*(s: string, prefix: char): bool {.inline.} = ## Returns true if `s` starts with character `prefix`. ## @@ -1600,7 +1714,7 @@ func removePrefix*(s: var string, chars: set[char] = Newlines) {.rtl, var start = 0 while start < s.len and s[start] in chars: start += 1 - if start > 0: s.delete(0, start - 1) + if start > 0: s.delete(0..start - 1) func removePrefix*(s: var string, c: char) {.rtl, extern: "nsuRemovePrefixChar".} = @@ -1627,8 +1741,8 @@ func removePrefix*(s: var string, prefix: string) {.rtl, var answers = "yesyes" answers.removePrefix("yes") doAssert answers == "yes" - if s.startsWith(prefix): - s.delete(0, prefix.len - 1) + if s.startsWith(prefix) and prefix.len > 0: + s.delete(0..prefix.len - 1) func removeSuffix*(s: var string, chars: set[char] = Newlines) {.rtl, extern: "nsuRemoveSuffixCharSet".} = @@ -1694,8 +1808,9 @@ func addSep*(dest: var string, sep = ", ", startLen: Natural = 0) {.inline.} = ## ## A shorthand for: ## - ## .. code-block:: nim + ## ```nim ## if dest.len > startLen: add(dest, sep) + ## ``` ## ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than @@ -1760,7 +1875,7 @@ func join*(a: openArray[string], sep: string = ""): string {.rtl, else: result = "" -func join*[T: not string](a: openArray[T], sep: string = ""): string {.rtl.} = +proc join*[T: not string](a: openArray[T], sep: string = ""): string = ## Converts all elements in the container `a` to strings using `$`, ## and concatenates them with `sep`. runnableExamples: @@ -1773,36 +1888,44 @@ func join*[T: not string](a: openArray[T], sep: string = ""): string {.rtl.} = add(result, $x) type - SkipTable* = array[char, int] + SkipTable* = array[char, int] ## Character table for efficient substring search. func initSkipTable*(a: var SkipTable, sub: string) {.rtl, extern: "nsuInitSkipTable".} = - ## Preprocess table `a` for `sub`. + ## Initializes table `a` for efficient search of substring `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + # TODO: this should be the `default()` initializer for the type. let m = len(sub) - var i = 0 - while i <= 0xff-7: - a[chr(i + 0)] = m - a[chr(i + 1)] = m - a[chr(i + 2)] = m - a[chr(i + 3)] = m - a[chr(i + 4)] = m - a[chr(i + 5)] = m - a[chr(i + 6)] = m - a[chr(i + 7)] = m - i += 8 + fill(a, m) for i in 0 ..< m - 1: a[sub[i]] = m - 1 - i -func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. +func initSkipTable*(sub: string): SkipTable {.noinit, rtl, + extern: "nsuInitNewSkipTable".} = + ## Returns a new table initialized for `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + initSkipTable(result, sub) + +func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindStrA".} = ## Searches for `sub` in `s` inside range `start..last` using preprocessed ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last ## element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ let - last = if last == 0: s.high else: last + last = if last < 0: s.high else: last subLast = sub.len - 1 if subLast == -1: @@ -1812,6 +1935,7 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. # This is an implementation of the Boyer-Moore Horspool algorithms # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + result = -1 var skip = start while last - skip >= subLast: @@ -1821,7 +1945,6 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. return skip dec i inc skip, a[s[skip + subLast]] - return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. @@ -1830,68 +1953,97 @@ when not (defined(js) or defined(nimdoc) or defined(nimscript)): else: const hasCStringBuiltin = false -func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, +func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindChar".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,char,Natural>`_ + ## * `rfind func<#rfind,string,char,Natural,int>`_ ## * `replace func<#replace,string,char,char>`_ - let last = if last == 0: s.high else: last - when nimvm: + result = -1 + let last = if last < 0: s.high else: last + + template findImpl = for i in int(start)..last: - if sub == s[i]: return i + if s[i] == sub: + return i + + when nimvm: + findImpl() else: when hasCStringBuiltin: - let L = last-start+1 - if L > 0: - let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + let length = last-start+1 + if length > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + return cast[int](found) -% cast[int](s.cstring) else: - for i in int(start)..last: - if sub == s[i]: return i - return -1 + findImpl() -func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. +func find*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindCharSet".} = ## Searches for `chars` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,set[char],Natural>`_ + ## * `rfind func<#rfind,string,set[char],Natural,int>`_ ## * `multiReplace func<#multiReplace,string,varargs[]>`_ - let last = if last == 0: s.high else: last + result = -1 + let last = if last < 0: s.high else: last for i in int(start)..last: - if s[i] in chars: return i - return -1 + if s[i] in chars: + return i -func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, +when defined(linux): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: """#define _GNU_SOURCE +#include <string.h>""".} +elif defined(bsd) or (defined(macosx) and not defined(ios)): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: "#include <string.h>".} + +func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindStr".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,string,Natural>`_ + ## * `rfind func<#rfind,string,string,Natural,int>`_ ## * `replace func<#replace,string,string,string>`_ - if sub.len > s.len: return -1 + if sub.len > s.len - start: return -1 if sub.len == 1: return find(s, sub[0], start, last) - var a {.noinit.}: SkipTable - initSkipTable(a, sub) - result = find(a, s, sub, start, last) + + template useSkipTable = + result = find(initSkipTable(sub), s, sub, start, last) + + when nimvm: + useSkipTable() + else: + when declared(memmem): + let subLen = sub.len + if last < 0 and start < s.len and subLen != 0: + let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) + result = if not found.isNil: + cast[int](found) -% cast[int](s.cstring) + else: + -1 + else: + useSkipTable() + else: + useSkipTable() func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuRFindChar".} = @@ -1902,7 +2054,7 @@ func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,char,Natural,int>`_ @@ -1920,7 +2072,7 @@ func rfind*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,set[char],Natural,int>`_ @@ -1938,11 +2090,14 @@ func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,string,Natural,int>`_ if sub.len == 0: + let rightIndex: Natural = if last < 0: s.len else: last + return max(start, rightIndex) + if sub.len > s.len - start: return -1 let last = if last == -1: s.high else: last result = 0 @@ -1998,7 +2153,7 @@ func countLines*(s: string): int {.rtl, extern: "nsuCountLines".} = ## Returns the number of lines in the string `s`. ## ## This is the same as `len(splitLines(s))`, but much more efficient - ## because it doesn't modify the string creating temporal objects. Every + ## because it doesn't modify the string creating temporary objects. Every ## `character literal <manual.html#lexical-analysis-character-literals>`_ ## newline combination (CR, LF, CR-LF) is supported. ## @@ -2037,7 +2192,7 @@ func contains*(s: string, chars: set[char]): bool = func replace*(s, sub: string, by = ""): string {.rtl, extern: "nsuReplaceStr".} = - ## Replaces every occurence of `sub` in `s` by the string `by`. + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. ## ## See also: ## * `find func<#find,string,string,Natural,int>`_ @@ -2064,8 +2219,7 @@ func replace*(s, sub: string, by = ""): string {.rtl, # copy the rest: add result, substr(s, i) else: - var a {.noinit.}: SkipTable - initSkipTable(a, sub) + var a = initSkipTable(sub) let last = s.high var i = 0 while true: @@ -2079,7 +2233,8 @@ func replace*(s, sub: string, by = ""): string {.rtl, func replace*(s: string, sub, by: char): string {.rtl, extern: "nsuReplaceChar".} = - ## Replaces `sub` in `s` by the character `by`. + ## Replaces every occurrence of the character `sub` in `s` with the character + ## `by`. ## ## Optimized version of `replace <#replace,string,string,string>`_ for ## characters. @@ -2097,16 +2252,15 @@ func replace*(s: string, sub, by: char): string {.rtl, func replaceWord*(s, sub: string, by = ""): string {.rtl, extern: "nsuReplaceWord".} = - ## Replaces `sub` in `s` by the string `by`. + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. ## ## Each occurrence of `sub` has to be surrounded by word boundaries ## (comparable to `\b` in regular expressions), otherwise it is not ## replaced. if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} - var a {.noinit.}: SkipTable result = "" - initSkipTable(a, sub) + var a = initSkipTable(sub) var i = 0 let last = s.high let sublen = sub.len @@ -2127,17 +2281,31 @@ func replaceWord*(s, sub: string, by = ""): string {.rtl, add result, substr(s, i) func multiReplace*(s: string, replacements: varargs[(string, string)]): string = - ## Same as replace, but specialized for doing multiple replacements in a single - ## pass through the input string. + ## Same as `replace<#replace,string,string,string>`_, but specialized for + ## doing multiple replacements in a single pass through the input string. ## - ## `multiReplace` performs all replacements in a single pass, this means it - ## can be used to swap the occurrences of "a" and "b", for instance. + ## `multiReplace` scans the input string from left to right and replaces the + ## matching substrings in the same order as passed in the argument list. + ## + ## The implications of the order of scanning the string and matching the + ## replacements: + ## - In case of multiple matches at a given position, the earliest + ## replacement is applied. + ## - Overlaps are not handled. After performing a replacement, the scan + ## continues from the character after the matched substring. If the + ## resulting string then contains a possible match starting in a newly + ## placed substring, the additional replacement is not performed. ## ## If the resulting string is not longer than the original input string, ## only a single memory allocation is required. ## - ## The order of the replacements does matter. Earlier replacements are - ## preferred over later replacements in the argument list. + runnableExamples: + # Swapping occurrences of 'a' and 'b': + doAssert multireplace("abba", [("a", "b"), ("b", "a")]) == "baab" + + # The second replacement ("ab") is matched and performed first, the scan then + # continues from 'c', so the "bc" replacement is never matched and thus skipped. + doAssert multireplace("abc", [("bc", "x"), ("ab", "_b")]) == "_bc" result = newStringOfCap(s.len) var i = 0 var fastChk: set[char] = {} @@ -2172,7 +2340,7 @@ func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, doAssert insertSep("1000000") == "1_000_000" result = newStringOfCap(s.len) let hasPrefix = isDigit(s[s.low]) == false - var idx:int + var idx: int if hasPrefix: result.add s[s.low] for i in (s.low + 1)..s.high: @@ -2186,7 +2354,7 @@ func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, result.setLen(L + idx) var j = 0 dec(L) - for i in countdown(partsLen-1,0): + for i in countdown(partsLen-1, 0): if j == digits: result[L + idx] = sep dec(L) @@ -2197,13 +2365,21 @@ func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, func escape*(s: string, prefix = "\"", suffix = "\""): string {.rtl, extern: "nsuEscape".} = - ## Escapes a string `s`. See `system.addEscapedChar - ## <system.html#addEscapedChar,string,char>`_ for the escaping scheme. + ## Escapes a string `s`. + ## + ## .. note:: The escaping scheme is different from + ## `system.addEscapedChar`. + ## + ## * replaces `'\0'..'\31'` and `'\127'..'\255'` by `\xHH` where `HH` is its hexadecimal value + ## * replaces ``\`` by `\\` + ## * replaces `'` by `\'` + ## * replaces `"` by `\"` ## ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. ## Both may be empty strings. ## ## See also: + ## * `addEscapedChar proc<system.html#addEscapedChar,string,char>`_ ## * `unescape func<#unescape,string,string,string>`_ for the opposite ## operation result = newStringOfCap(s.len + s.len shr 2) @@ -2278,8 +2454,8 @@ func validIdentifier*(s: string): bool {.rtl, extern: "nsuValidIdentifier".} = # floating point formatting: when not defined(js): - func c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs} + func c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs.} type FloatFormatMode* = enum @@ -2306,60 +2482,63 @@ func formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, doAssert x.formatBiggestFloat() == "123.4560000000000" doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" - when defined(js): - var precision = precision - if precision == -1: - # use the same default precision as c_sprintf - precision = 6 - var res: cstring - case format - of ffDefault: - {.emit: "`res` = `f`.toString();".} - of ffDecimal: - {.emit: "`res` = `f`.toFixed(`precision`);".} - of ffScientific: - {.emit: "`res` = `f`.toExponential(`precision`);".} - result = $res - if 1.0 / f == -Inf: - # JavaScript removes the "-" from negative Zero, add it back here - result = "-" & $res - for i in 0 ..< result.len: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if result[i] in {'.', ','}: result[i] = decimalSep + when nimvm: + discard "implemented in the vmops" else: - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision >= 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - L = c_sprintf(addr buf, addr frmtstr, precision, f) + when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_snprintf + precision = 6 + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalSep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - L = c_sprintf(addr buf, addr frmtstr, f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalSep - else: result[i] = buf[i] - when defined(windows): - # VS pre 2015 violates the C standard: "The exponent always contains at - # least two digits, and only as many more digits as necessary to - # represent the exponent." [C11 §7.21.6.1] - # The following post-processing fixes this behavior. - if result.len > 4 and result[^4] == '+' and result[^3] == '0': - result[^3] = result[^2] - result[^2] = result[^1] - result.setLen(result.len - 1) + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision >= 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), precision, f) + else: + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalSep + else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) func formatFloat*(f: float, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; decimalSep = '.'): string {. @@ -2399,7 +2578,8 @@ func trimZeros*(x: var string; decimalSep = '.') = var pos = last while pos >= 0 and x[pos] == '0': dec(pos) if pos > sPos: inc(pos) - x.delete(pos, last) + if last >= pos: + x.delete(pos..last) type BinaryPrefixMode* = enum ## The different names for binary prefixes. @@ -2491,13 +2671,13 @@ func formatEng*(f: BiggestFloat, ## decimal point or (if `trim` is true) the maximum number of digits to be ## shown. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(0, 2, trim=false) == "0.00" ## formatEng(0, 2) == "0" ## formatEng(0.053, 0) == "53e-3" ## formatEng(52731234, 2) == "52.73e6" ## formatEng(-52731234, 2) == "-52.73e6" + ## ``` ## ## If `siPrefix` is set to true, the number will be displayed with the SI ## prefix corresponding to the exponent. For example 4100 will be displayed @@ -2512,8 +2692,7 @@ func formatEng*(f: BiggestFloat, ## different to appending the unit to the result as the location of the space ## is altered depending on whether there is an exponent. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" ## formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" ## formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space @@ -2523,6 +2702,7 @@ func formatEng*(f: BiggestFloat, ## formatEng(4100) == "4.1e3" ## formatEng(4100, unit="V") == "4.1e3 V" ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true + ## ``` ## ## `decimalSep` is used as the decimal separator. ## @@ -2610,8 +2790,8 @@ func findNormalized(x: string, inArray: openArray[string]): int = # security hole... return -1 -func invalidFormatString() {.noinline.} = - raise newException(ValueError, "invalid format string") +func invalidFormatString(formatstr: string) {.noinline.} = + raise newException(ValueError, "invalid format string: " & formatstr) func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, extern: "nsuAddf".} = @@ -2623,7 +2803,7 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, if formatstr[i] == '$' and i+1 < len(formatstr): case formatstr[i+1] of '#': - if num > a.high: invalidFormatString() + if num > a.high: invalidFormatString(formatstr) add s, a[num] inc i, 2 inc num @@ -2639,7 +2819,7 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, j = j * 10 + ord(formatstr[i]) - ord('0') inc(i) let idx = if not negative: j-1 else: a.len-j - if idx < 0 or idx > a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString(formatstr) add s, a[idx] of '{': var j = i+2 @@ -2656,27 +2836,27 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, inc(j) if isNumber == 1: let idx = if not negative: k-1 else: a.len-k - if idx < 0 or idx > a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString(formatstr) add s, a[idx] else: var x = findNormalized(substr(formatstr, i+2, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + else: invalidFormatString(formatstr) i = j+1 of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': var j = i+1 while j < formatstr.len and formatstr[j] in PatternChars: inc(j) var x = findNormalized(substr(formatstr, i+1, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + else: invalidFormatString(formatstr) i = j else: - invalidFormatString() + invalidFormatString(formatstr) else: add s, formatstr[i] inc(i) -func `%` *(formatstr: string, a: openArray[string]): string {.rtl, +func `%`*(formatstr: string, a: openArray[string]): string {.rtl, extern: "nsuFormatOpenArray".} = ## Interpolates a format string with the values from `a`. ## @@ -2686,13 +2866,15 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, ## ## This is best explained by an example: ## - ## .. code-block:: nim + ## ```nim ## "$1 eats $2." % ["The cat", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The substitution variables (the thing after the `$`) are enumerated ## from 1 to `a.len`. @@ -2700,21 +2882,24 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, ## The notation `$#` can be used to refer to the next substitution ## variable: ## - ## .. code-block:: nim + ## ```nim ## "$# eats $#." % ["The cat", "fish"] + ## ``` ## ## Substitution variables can also be words (that is ## `[A-Za-z_]+[A-Za-z0-9_]*`) in which case the arguments in `a` with even ## indices are keys and with odd indices are the corresponding values. ## An example: ## - ## .. code-block:: nim + ## ```nim ## "$animal eats $food." % ["animal", "The cat", "food", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The variables are compared with `cmpIgnoreStyle`. `ValueError` is ## raised if an ill-formed format string has been passed to the `%` operator. @@ -2724,7 +2909,7 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, result = newStringOfCap(formatstr.len + a.len shl 4) addf(result, formatstr, a) -func `%` *(formatstr, a: string): string {.rtl, +func `%`*(formatstr, a: string): string {.rtl, extern: "nsuFormatSingleElem".} = ## This is the same as `formatstr % [a]` (see ## `% func<#%25,string,openArray[string]>`_). @@ -2780,7 +2965,7 @@ func strip*(s: string, leading = true, trailing = true, result = substr(s, first, last) func stripLineEnd*(s: var string) = - ## Returns `s` stripped from one of these suffixes: + ## Strips one of these suffixes from `s` in-place: ## `\r, \n, \r\n, \f, \v` (at most once instance). ## For example, can be useful in conjunction with `osproc.execCmdEx`. ## aka: `chomp`:idx: @@ -2812,13 +2997,14 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## Substrings are separated by a substring containing only `seps`. ## Example: ## - ## .. code-block:: nim + ## ```nim ## for word in tokenize(" this is an example "): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## (" ", true) ## ("this", false) ## (" ", true) @@ -2828,6 +3014,7 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## (" ", true) ## ("example", false) ## (" ", true) + ## ``` var i = 0 while true: var j = i diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index 9654a06785..7833ed0633 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -19,7 +19,7 @@ proc checkPragma(ex, prag: var NimNode) = prag = ex[1] ex = ex[0] -proc createProcType(p, b: NimNode): NimNode {.compileTime.} = +proc createProcType(p, b: NimNode): NimNode = result = newNimNode(nnkProcTy) var formalParams = newNimNode(nnkFormalParams).add(b) @@ -54,6 +54,8 @@ proc createProcType(p, b: NimNode): NimNode {.compileTime.} = macro `=>`*(p, b: untyped): untyped = ## Syntax sugar for anonymous procedures. It also supports pragmas. + ## + ## .. warning:: Semicolons can not be used to separate procedure arguments. runnableExamples: proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) @@ -119,9 +121,9 @@ macro `=>`*(p, b: untyped): untyped = else: error("Incorrect procedure parameter.", c) params.add(identDefs) - of nnkIdent: + of nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym: var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) + identDefs.add(ident $p) identDefs.add(ident"auto") identDefs.add(newEmptyNode()) params.add(identDefs) @@ -133,6 +135,8 @@ macro `=>`*(p, b: untyped): untyped = macro `->`*(p, b: untyped): untyped = ## Syntax sugar for procedure types. It also supports pragmas. + ## + ## .. warning:: Semicolons can not be used to separate procedure arguments. runnableExamples: proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) # is the same as: @@ -186,7 +190,7 @@ macro dumpToString*(x: untyped): string = assert dumpToString(a + x) == "a + x: 1 + x = 11" template square(x): untyped = x * x assert dumpToString(square(x)) == "square(x): x * x = 100" - assert not compiles dumpToString(1 + nonexistant) + assert not compiles dumpToString(1 + nonexistent) import std/strutils assert "failedAssertImpl" in dumpToString(assert true) # example with a statement result = newCall(bindSym"dumpToStringImpl") @@ -227,9 +231,19 @@ macro capture*(locals: varargs[typed], body: untyped): untyped {.since: (1, 1).} let locals = if locals.len == 1 and locals[0].kind == nnkBracket: locals[0] else: locals for arg in locals: - if arg.strVal == "result": - error("The variable name cannot be `result`!", arg) - params.add(newIdentDefs(ident(arg.strVal), freshIdentNodes getTypeInst arg)) + proc getIdent(n: NimNode): NimNode = + case n.kind + of nnkIdent, nnkSym: + let nStr = n.strVal + if nStr == "result": + error("The variable name cannot be `result`!", n) + result = ident(nStr) + of nnkHiddenDeref: result = n[0].getIdent() + else: + error("The argument to be captured `" & n.repr & "` is not a pure identifier. " & + "It is an unsupported `" & $n.kind & "` node.", n) + let argName = getIdent(arg) + params.add(newIdentDefs(argName, freshIdentNodes getTypeInst arg)) result = newNimNode(nnkCall) result.add(newProc(newEmptyNode(), params, body, nnkLambda)) for arg in locals: result.add(arg) @@ -331,9 +345,10 @@ proc collectImpl(init, body: NimNode): NimNode {.since: (1, 1).} = let res = genSym(nskVar, "collectResult") var bracketExpr: NimNode if init != nil: - expectKind init, {nnkCall, nnkIdent, nnkSym} + expectKind init, {nnkCall, nnkIdent, nnkSym, nnkClosedSymChoice, nnkOpenSymChoice} bracketExpr = newTree(nnkBracketExpr, - if init.kind == nnkCall: freshIdentNodes(init[0]) else: freshIdentNodes(init)) + if init.kind in {nnkCall, nnkClosedSymChoice, nnkOpenSymChoice}: + freshIdentNodes(init[0]) else: freshIdentNodes(init)) else: bracketExpr = newTree(nnkBracketExpr) let (resBody, keyType, valueType) = trans(body, res, bracketExpr) @@ -387,6 +402,10 @@ macro collect*(init, body: untyped): untyped {.since: (1, 1).} = macro collect*(body: untyped): untyped {.since: (1, 5).} = ## Same as `collect` but without an `init` parameter. + ## + ## **See also:** + ## * `sequtils.toSeq proc<sequtils.html#toSeq.t%2Cuntyped>`_ + ## * `sequtils.mapIt template<sequtils.html#mapIt.t%2Ctyped%2Cuntyped>`_ runnableExamples: import std/[sets, tables] let data = @["bird", "word"] @@ -407,10 +426,4 @@ macro collect*(body: untyped): untyped {.since: (1, 5).} = for i, d in data.pairs: {i: d} assert m == {0: "bird", 1: "word"}.toTable - # avoid `collect` when `sequtils.toSeq` suffices: - assert collect(for i in 1..3: i*i) == @[1, 4, 9] # ok in this case - assert collect(for i in 1..3: i) == @[1, 2, 3] # overkill in this case - from std/sequtils import toSeq - assert toSeq(1..3) == @[1, 2, 3] # simpler - result = collectImpl(nil, body) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index cada721967..53b3d61da2 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -15,14 +15,59 @@ ## code `exitprocs.addExitProc(resetAttributes)` to restore the defaults. ## Similarly, if you hide the cursor, make sure to unhide it with ## `showCursor` before quitting. +## +## Progress bar +## ============ +## +## Basic progress bar example: +runnableExamples("-r:off"): + import std/[os, strutils] -import macros -import strformat -from strutils import toLowerAscii, `%` -import colors + for i in 0..100: + stdout.styledWriteLine(fgRed, "0% ", fgWhite, '#'.repeat i, if i > 50: fgGreen else: fgYellow, "\t", $i , "%") + sleep 42 + cursorUp 1 + eraseLine() + + stdout.resetAttributes() + +##[ +## Playing with colorful and styled text +]## + +## Procs like `styledWriteLine`, `styledEcho` etc. have a temporary effect on +## text parameters. Style parameters only affect the text parameter right after them. +## After being called, these procs will reset the default style of the terminal. +## While `setBackGroundColor`, `setForeGroundColor` etc. have a lasting +## influence on the terminal, you can use `resetAttributes` to +## reset the default style of the terminal. +runnableExamples("-r:off"): + stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") + stdout.styledWriteLine(fgRed, "red text ") + stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") + stdout.styledWriteLine(" ordinary text without style ") + + stdout.setBackGroundColor(bgCyan, true) + stdout.setForeGroundColor(fgBlue) + stdout.write("blue text in cyan background") + stdout.resetAttributes() + + # You can specify multiple text parameters. Style parameters + # only affect the text parameter right after them. + styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" + + stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") + +import std/macros +import std/strformat +from std/strutils import toLowerAscii, `%`, parseInt +import std/colors when defined(windows): - import winlean + import std/winlean + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] type PTerminal = ref object @@ -51,10 +96,11 @@ const fgPrefix = "\e[38;2;" bgPrefix = "\e[48;2;" ansiResetCode* = "\e[0m" + getPos = "\e[6n" stylePrefix = "\e[" when defined(windows): - import winlean, os + import std/[winlean, os] const DUPLICATE_SAME_ACCESS = 2 @@ -128,6 +174,7 @@ when defined(windows): return 0 proc terminalWidth*(): int = + ## Returns the terminal width in columns. var w: int = 0 w = terminalWidthIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -136,6 +183,7 @@ when defined(windows): return 80 proc terminalHeight*(): int = + ## Returns the terminal height in rows. var h: int = 0 h = terminalHeightIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -173,6 +221,9 @@ when defined(windows): raiseOSError(osLastError()) return (int(c.dwCursorPosition.x), int(c.dwCursorPosition.y)) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError, OSError].} = + return getCursorPos(getStdHandle(STD_OUTPUT_HANDLE)) + proc setCursorPos(h: Handle, x, y: int) = var c: COORD c.x = int16(x) @@ -206,7 +257,7 @@ when defined(windows): if f == stderr: term.hStderr else: term.hStdout else: - import termios, posix, os, parseutils + import std/[termios, posix, os, parseutils] proc setRaw(fd: FileHandle, time: cint = TCSAFLUSH) = var mode: Termios @@ -220,6 +271,48 @@ else: mode.c_cc[VTIME] = 0.cuchar discard fd.tcSetAttr(time, addr mode) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError].} = + ## Returns cursor position (x, y) + ## writes to stdout and expects the terminal to respond via stdin + var + xStr = "" + yStr = "" + ch: char + ct: int + readX = false + + # use raw mode to ask terminal for cursor position + let fd = getFileHandle(stdin) + var oldMode: Termios + discard fd.tcGetAttr(addr oldMode) + fd.setRaw() + stdout.write(getPos) + flushFile(stdout) + + try: + # parse response format: [yyy;xxxR + while true: + let n = readBuffer(stdin, addr ch, 1) + if n == 0 or ch == 'R': + if xStr == "" or yStr == "": + raise newException(ValueError, "Got character position message that was missing data") + break + ct += 1 + if ct > 16: + raise newException(ValueError, "Got unterminated character position message from terminal") + if ch == ';': + readX = true + elif ch in {'0'..'9'}: + if readX: + xStr.add(ch) + else: + yStr.add(ch) + finally: + # restore previous terminal mode + discard fd.tcSetAttr(TCSADRAIN, addr oldMode) + + return (parseInt(xStr), parseInt(yStr)) + proc terminalWidthIoctl*(fds: openArray[int]): int = ## Returns terminal width from first fd that supports the ioctl. @@ -244,25 +337,57 @@ else: ## Returns some reasonable terminal width from either standard file ## descriptors, controlling terminal, environment variables or tradition. - var w = terminalWidthIoctl([0, 1, 2]) #Try standard file descriptors + # POSIX environment variable takes precendence. + # _COLUMNS_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred width in column positions for + # the terminal screen or window. If this variable is unset or null, + # the implementation determines the number of columns, appropriate + # for the terminal or window, in an unspecified manner. + # When COLUMNS is set, any terminal-width information implied by TERM + # is overridden. Users and conforming applications should not set COLUMNS + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var w: int + var s = getEnv("COLUMNS") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, w) > 0 and w > 0: + return w + w = terminalWidthIoctl([0, 1, 2]) # Try standard file descriptors if w > 0: return w - var cterm = newString(L_ctermid) #Try controlling tty + var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) if fd != -1: w = terminalWidthIoctl([int(fd)]) discard close(fd) if w > 0: return w - var s = getEnv("COLUMNS") #Try standard env var - if len(s) > 0 and parseInt(s, w) > 0 and w > 0: - return w - return 80 #Finally default to venerable value + return 80 # Finally default to venerable value proc terminalHeight*(): int = ## Returns some reasonable terminal height from either standard file ## descriptors, controlling terminal, environment variables or tradition. ## Zero is returned if the height could not be determined. - var h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors + # POSIX environment variable takes precendence. + # _LINES_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred number of lines on a page or + # the vertical screen or window size in lines. A line in this case + # is a vertical measure large enough to hold the tallest character + # in the character set being displayed. If this variable is unset or null, + # the implementation determines the number of lines, appropriate + # for the terminal or window (size, terminal baud rate, and so on), + # in an unspecified manner. + # When LINES is set, any terminal-height information implied by TERM + # is overridden. Users and conforming applications should not set LINES + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var h: int + var s = getEnv("LINES") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, h) > 0 and h > 0: + return h + h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors if h > 0: return h var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) @@ -270,9 +395,6 @@ else: h = terminalHeightIoctl([int(fd)]) discard close(fd) if h > 0: return h - var s = getEnv("LINES") # Try standard env var - if len(s) > 0 and parseInt(s, h) > 0 and h > 0: - return h return 0 # Could not determine height proc terminalSize*(): tuple[w, h: int] = @@ -347,6 +469,9 @@ when defined(windows): proc cursorUp*(f: File, count = 1) = ## Moves the cursor up by `count` rows. + runnableExamples("-r:off"): + stdout.cursorUp(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -357,6 +482,9 @@ proc cursorUp*(f: File, count = 1) = proc cursorDown*(f: File, count = 1) = ## Moves the cursor down by `count` rows. + runnableExamples("-r:off"): + stdout.cursorDown(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -367,6 +495,9 @@ proc cursorDown*(f: File, count = 1) = proc cursorForward*(f: File, count = 1) = ## Moves the cursor forward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorForward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -377,6 +508,9 @@ proc cursorForward*(f: File, count = 1) = proc cursorBackward*(f: File, count = 1) = ## Moves the cursor backward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorBackward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -418,6 +552,9 @@ else: proc eraseLine*(f: File) = ## Erases the entire current line. + runnableExamples("-r:off"): + write(stdout, "never mind") + stdout.eraseLine() # nothing will be printed on the screen when defined(windows): let h = conHandle(f) var scrbuf: CONSOLE_SCREEN_BUFFER_INFO @@ -480,7 +617,7 @@ proc resetAttributes*(f: File) = gBG = 0 type - Style* = enum ## different styles for text output + Style* = enum ## Different styles for text output. styleBright = 1, ## bright text styleDim, ## dim text styleItalic, ## italic (or reverse on terminals not supporting) @@ -534,7 +671,7 @@ proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = stdout.write(ansiStyleCode(gBG)) type - ForegroundColor* = enum ## terminal's foreground colors + ForegroundColor* = enum ## Terminal's foreground colors. fgBlack = 30, ## black fgRed, ## red fgGreen, ## green @@ -546,7 +683,7 @@ type fg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) fgDefault ## default terminal foreground color - BackgroundColor* = enum ## terminal's background colors + BackgroundColor* = enum ## Terminal's background colors. bgBlack = 40, ## black bgRed, ## red bgGreen, ## green @@ -582,9 +719,9 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright = false) = 0, # fg8Bit not supported, see `enableTrueColors` instead. 0] # unused if fg == fgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultForegroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultForegroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[fg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[fg]))) else: gFG = ord(fg) if bright: inc(gFG, 60) @@ -611,9 +748,9 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright = false) = 0, # bg8Bit not supported, see `enableTrueColors` instead. 0] # unused if bg == bgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultBackgroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultBackgroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[bg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[bg]))) else: gBG = ord(bg) if bright: inc(gBG, 60) @@ -701,14 +838,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = ## When some argument is `Style`, `set[Style]`, `ForegroundColor`, ## `BackgroundColor` or `TerminalCmd` then it is not sent directly to ## `f`, but instead corresponding terminal style proc is called. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## stdout.styledWrite(fgRed, "red text ") - ## stdout.styledWrite(fgGreen, "green text") - ## + runnableExamples("-r:off"): + stdout.styledWrite(fgRed, "red text ") + stdout.styledWrite(fgGreen, "green text") + var reset = false result = newNimNode(nnkStmtList) @@ -731,14 +864,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = template styledWriteLine*(f: File, args: varargs[untyped]) = ## Calls `styledWrite` and appends a newline at the end. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## proc error(msg: string) = - ## styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) - ## + runnableExamples: + proc error(msg: string) = + styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + styledWrite(f, args) write(f, "\n") @@ -747,7 +876,7 @@ template styledEcho*(args: varargs[untyped]) = stdout.styledWriteLine(args) proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. + ## Reads a single character from the terminal, blocking until it is entered. ## The character is not printed to the terminal. when defined(windows): let fd = getStdHandle(STD_INPUT_HANDLE) @@ -793,7 +922,7 @@ when defined(windows): stdout.write "\n" else: - import termios + import std/termios proc readPasswordFromStdin*(prompt: string, password: var string): bool {.tags: [ReadIOEffect, WriteIOEffect].} = @@ -849,10 +978,10 @@ proc isTrueColorSupported*(): bool = return getTerminal().trueColorIsSupported when defined(windows): - import os + import std/os proc enableTrueColors*() = - ## Enable true color. + ## Enables true color. var term = getTerminal() when defined(windows): var @@ -885,7 +1014,7 @@ proc enableTrueColors*() = term.trueColorIsEnabled = term.trueColorIsSupported proc disableTrueColors*() = - ## Disable true color. + ## Disables true color. var term = getTerminal() when defined(windows): if term.trueColorIsSupported: @@ -902,51 +1031,3 @@ proc newTerminal(): owned(PTerminal) = new result when defined(windows): initTerminal(result) - -when not defined(testing) and isMainModule: - assert ansiStyleCode(styleBright) == "\e[1m" - assert ansiStyleCode(styleStrikethrough) == "\e[9m" - # exitprocs.addExitProc(resetAttributes) - write(stdout, "never mind") - stdout.eraseLine() - stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") - stdout.styledWriteLine("italic text ", {styleItalic}) - stdout.setBackGroundColor(bgCyan, true) - stdout.setForeGroundColor(fgBlue) - stdout.write("blue text in cyan background") - stdout.resetAttributes() - echo "" - stdout.writeLine("ordinary text") - echo "more ordinary text" - styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" - echo "ordinary text again" - styledEcho styleBright, fgRed, "[FAIL]", resetStyle, fgRed, " Nay :(" - echo "ordinary text again" - setForeGroundColor(fgGreen) - echo "green text" - echo "more green text" - setForeGroundColor(fgBlue) - echo "blue text" - resetAttributes() - echo "ordinary text" - - stdout.styledWriteLine(fgRed, "red text ") - stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") - stdout.styledWriteLine(" ordinary text ") - stdout.styledWriteLine(fgGreen, "green text") - - writeStyled("underscored text", {styleUnderscore}) - stdout.styledWrite(fgRed, " red text ") - writeStyled("bright text ", {styleBright}) - echo "ordinary text" - - stdout.styledWrite(fgRed, "red text ") - stdout.styledWrite(fgWhite, bgRed, "white text in red background") - stdout.styledWrite(" ordinary text ") - stdout.styledWrite(fgGreen, "green text") - echo "" - echo "ordinary text" - stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") - stdout.styledWriteLine(bgYellow, "text in yellow bg", styleBright, - " bold text in yellow bg", bgDefault, " bold text") - echo "ordinary text" diff --git a/lib/pure/times.nim b/lib/pure/times.nim index e763b17bbd..a8e24313ff 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -20,7 +20,7 @@ Examples ======== - .. code-block:: nim + ```nim import std/[times, os] # Simple benchmarking let time = cpuTime() @@ -37,6 +37,7 @@ # Arithmetic using TimeInterval echo "One year from now : ", now() + 1.years echo "One month from now : ", now() + 1.months + ``` Parsing and Formatting Dates ============================ @@ -44,10 +45,10 @@ The `DateTime` type can be parsed and formatted using the different `parse` and `format` procedures. - .. code-block:: nim - + ```nim let dt = parse("2000-01-01", "yyyy-MM-dd") echo dt.format("yyyy-MM-dd") + ``` The different format patterns that are supported are documented below. @@ -62,6 +63,8 @@ | `Monday -> Mon` `dddd` Full string for the day of the week. | `Saturday -> Saturday` | `Monday -> Monday` + `GG` The last two digits of the Iso Week-Year | `30/12/2012 -> 13` + `GGGG` The Iso week-calendar year padded to four digits | `30/12/2012 -> 2013` `h` The hours in one digit if possible. Ranging from 1-12. | `5pm -> 5` | `2am -> 2` `hh` The hours in two digits always. If the hour is one digit, 0 is prepended. | `5pm -> 05` @@ -104,6 +107,10 @@ | `24 AD -> 24` | `24 BC -> -23` | `12345 AD -> 12345` + `V` The Iso Week-Number as one or two digits | `3/2/2012 -> 5` + | `1/4/2012 -> 13` + `VV` The Iso Week-Number as two digits always. 0 is prepended if one digit. | `3/2/2012 -> 05` + | `1/4/2012 -> 13` `z` Displays the timezone offset from UTC. | `UTC+7 -> +7` | `UTC-5 -> -5` `zz` Same as above but with leading 0. | `UTC+7 -> +07` @@ -196,13 +203,17 @@ * `monotimes module <monotimes.html>`_ ]## -import strutils, math, options +import std/[strutils, math, options] import std/private/since include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(js): - import jscore + import std/jscore # This is really bad, but overflow checks are broken badly for # ints on the JS backend. See #6752. @@ -226,7 +237,7 @@ when defined(js): {.pop.} elif defined(posix): - import posix + import std/posix type CTime = posix.Time @@ -235,7 +246,7 @@ elif defined(posix): {.importc: "gettimeofday", header: "<sys/time.h>", sideEffect.} elif defined(windows): - import winlean, std/time_t + import std/winlean, std/time_t type CTime = time_t.Time @@ -288,6 +299,13 @@ type YeardayRange* = range[0..365] NanosecondRange* = range[0..999_999_999] + IsoWeekRange* = range[1 .. 53] + ## An ISO 8601 calendar week number. + IsoYear* = distinct int + ## An ISO 8601 calendar year number. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + Time* = object ## Represents a point in time. seconds: int64 nanosecond: NanosecondRange @@ -313,8 +331,11 @@ type Duration* = object ## Represents a fixed duration of time, meaning a duration ## that has constant length independent of the context. ## - ## To create a new `Duration`, use `initDuration proc + ## To create a new `Duration`, use `initDuration ## <#initDuration,int64,int64,int64,int64,int64,int64,int64,int64>`_. + ## Instead of trying to access the private attributes, use + ## `inSeconds <#inSeconds,Duration>`_ for converting to seconds and + ## `inNanoseconds <#inNanoseconds,Duration>`_ for converting to nanoseconds. seconds: int64 nanosecond: NanosecondRange @@ -356,7 +377,7 @@ type Timezone* = ref object ## \ ## Timezone interface for supporting `DateTime <#DateTime>`_\s of arbitrary ## timezones. The `times` module only supplies implementations for the - ## systems local time and UTC. + ## system's local time and UTC. zonedTimeFromTimeImpl: proc (x: Time): ZonedTime {.tags: [], raises: [], benign.} zonedTimeFromAdjTimeImpl: proc (x: Time): ZonedTime @@ -394,6 +415,16 @@ const unitWeights: array[FixedTimeUnit, int64] = [ 7 * secondsInDay * 1e9.int64, ] +when (NimMajor, NimMinor) >= (1, 4): + # Newer versions of Nim don't track defects + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError].} + {.pragma: parseRaises, raises: [TimeParseError].} +else: + # Still track when using older versions + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError, Defect].} + {.pragma: parseRaises, raises: [TimeParseError, Defect].} + + # # Helper procs # @@ -485,7 +516,7 @@ proc fromEpochDay(epochday: int64): proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign.} = ## Returns the day of the year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).yearday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).yearday`. runnableExamples: doAssert getDayOfYear(1, mJan, 2000) == 0 doAssert getDayOfYear(10, mJan, 2000) == 9 @@ -505,7 +536,7 @@ proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign.} = ## Returns the day of the week enum from day, month and year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).weekday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).weekday`. runnableExamples: doAssert getDayOfWeek(13, mJun, 1990) == dWed doAssert $getDayOfWeek(13, mJun, 1990) == "Wednesday" @@ -513,7 +544,7 @@ proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay assertValidDate monthday, month, year # 1970-01-01 is a Thursday, we adjust to the previous Monday let days = toEpochDay(monthday, month, year) - 3 - let weeks = floorDiv(days, 7) + let weeks = floorDiv(days, 7'i64) let wd = days - weeks * 7 # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. # so we must correct for the WeekDay type. @@ -526,12 +557,60 @@ proc getDaysInYear*(year: int): int = doAssert getDaysInYear(2001) == 365 result = 365 + (if isLeapYear(year): 1 else: 0) +proc `==`*(a, b: IsoYear): bool {.borrow.} +proc `$`*(p: IsoYear): string {.borrow.} + +proc getWeeksInIsoYear*(y: IsoYear): IsoWeekRange {.since: (1, 5).} = + ## Returns the number of weeks in the specified ISO 8601 week-based year, which can be + ## either 53 or 52. + runnableExamples: + assert getWeeksInIsoYear(IsoYear(2019)) == 52 + assert getWeeksInIsoYear(IsoYear(2020)) == 53 + + var y = int(y) + + # support negative years + y = if y < 0: 400 + y mod 400 else: y + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let p = (y + (y div 4) - (y div 100) + (y div 400)) mod 7 + let y1 = y - 1 + let p1 = (y1 + (y1 div 4) - (y1 div 100) + (y1 div 400)) mod 7 + if p == 4 or p1 == 3: 53 else: 52 + +proc getIsoWeekAndYear*(dt: DateTime): + tuple[isoweek: IsoWeekRange, isoyear: IsoYear] {.since: (1, 5).} = + ## Returns the ISO 8601 week and year. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert getIsoWeekAndYear(initDateTime(21, mApr, 2018, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2018.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) + assert w == 01.IsoWeekRange + assert y == 2020.IsoYear + assert getIsoWeekAndYear(initDateTime(13, mSep, 2020, 00, 00, 00)) == (isoweek: 37.IsoWeekRange, isoyear: 2020.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(2, mJan, 2021, 00, 00, 00)) + assert w.int > 52 + assert w.int < 54 + assert y.int mod 100 == 20 + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + var w = (dt.yearday.int - dt.weekday.int + 10) div 7 + if w < 1: + (isoweek: getWeeksInIsoYear(IsoYear(dt.year - 1)), isoyear: IsoYear(dt.year - 1)) + elif (w > getWeeksInIsoYear(IsoYear(dt.year))): + (isoweek: IsoWeekRange(1), isoyear: IsoYear(dt.year + 1)) + else: + (isoweek: IsoWeekRange(w), isoyear: IsoYear(dt.year)) + proc stringifyUnit(value: int | int64, unit: TimeUnit): string = ## Stringify time unit with it's name, lowercased let strUnit = $unit result = "" - result.add($value) - result.add(" ") + result.addInt value + result.add ' ' if abs(value) != 1: result.add(strUnit.toLowerAscii()) else: @@ -574,11 +653,10 @@ template eqImpl(a: Duration|Time, b: Duration|Time): bool = const DurationZero* = Duration() ## \ ## Zero value for durations. Useful for comparisons. - ## - ## .. code-block:: nim - ## + ## ```nim ## doAssert initDuration(seconds = 1) > DurationZero ## doAssert initDuration(seconds = 0) == DurationZero + ## ``` proc initDuration*(nanoseconds, microseconds, milliseconds, seconds, minutes, hours, days, weeks: int64 = 0): Duration = @@ -622,56 +700,56 @@ template convert(dur: Duration, unit: static[FixedTimeUnit]): int64 = convert(Nanoseconds, unit, dur.nanosecond) proc inWeeks*(dur: Duration): int64 = - ## Convert the duration to the number of whole weeks. + ## Converts the duration to the number of whole weeks. runnableExamples: let dur = initDuration(days = 8) doAssert dur.inWeeks == 1 dur.convert(Weeks) proc inDays*(dur: Duration): int64 = - ## Convert the duration to the number of whole days. + ## Converts the duration to the number of whole days. runnableExamples: let dur = initDuration(hours = -50) doAssert dur.inDays == -2 dur.convert(Days) proc inHours*(dur: Duration): int64 = - ## Convert the duration to the number of whole hours. + ## Converts the duration to the number of whole hours. runnableExamples: let dur = initDuration(minutes = 60, days = 2) doAssert dur.inHours == 49 dur.convert(Hours) proc inMinutes*(dur: Duration): int64 = - ## Convert the duration to the number of whole minutes. + ## Converts the duration to the number of whole minutes. runnableExamples: let dur = initDuration(hours = 2, seconds = 10) doAssert dur.inMinutes == 120 dur.convert(Minutes) proc inSeconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole seconds. + ## Converts the duration to the number of whole seconds. runnableExamples: let dur = initDuration(hours = 2, milliseconds = 10) doAssert dur.inSeconds == 2 * 60 * 60 dur.convert(Seconds) proc inMilliseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole milliseconds. + ## Converts the duration to the number of whole milliseconds. runnableExamples: let dur = initDuration(seconds = -2) doAssert dur.inMilliseconds == -2000 dur.convert(Milliseconds) proc inMicroseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole microseconds. + ## Converts the duration to the number of whole microseconds. runnableExamples: let dur = initDuration(seconds = -2) doAssert dur.inMicroseconds == -2000000 dur.convert(Microseconds) proc inNanoseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole nanoseconds. + ## Converts the duration to the number of whole nanoseconds. runnableExamples: let dur = initDuration(seconds = -2) doAssert dur.inNanoseconds == -2000000000 @@ -877,6 +955,7 @@ since((1, 1)): export fromUnixFloat export toUnixFloat + proc fromWinTime*(win: int64): Time = ## Convert a Windows file time (100-nanosecond intervals since ## `1601-01-01T00:00:00Z`) to a `Time`. @@ -890,27 +969,33 @@ proc toWinTime*(t: Time): int64 = ## since `1601-01-01T00:00:00Z`). result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 +proc getTimeImpl(typ: typedesc[Time]): Time = + discard "implemented in the vm" + proc getTime*(): Time {.tags: [TimeEffect], benign.} = ## Gets the current time as a `Time` with up to nanosecond resolution. - when defined(js): - let millis = newDate().getTime() - let seconds = convert(Milliseconds, Seconds, millis) - let nanos = convert(Milliseconds, Nanoseconds, - millis mod convert(Seconds, Milliseconds, 1).int) - result = initTime(seconds, nanos) - elif defined(macosx): - var a {.noinit.}: Timeval - gettimeofday(a) - result = initTime(a.tv_sec.int64, - convert(Microseconds, Nanoseconds, a.tv_usec.int)) - elif defined(posix): - var ts {.noinit.}: Timespec - discard clock_gettime(CLOCK_REALTIME, ts) - result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) - elif defined(windows): - var f {.noinit.}: FILETIME - getSystemTimeAsFileTime(f) - result = fromWinTime(rdFileTime(f)) + when nimvm: + result = getTimeImpl(Time) + else: + when defined(js): + let millis = newDate().getTime() + let seconds = convert(Milliseconds, Seconds, millis) + let nanos = convert(Milliseconds, Nanoseconds, + millis mod convert(Seconds, Milliseconds, 1).int) + result = initTime(seconds, nanos) + elif defined(macosx): + var a {.noinit.}: Timeval + gettimeofday(a) + result = initTime(a.tv_sec.int64, + convert(Microseconds, Nanoseconds, a.tv_usec.int)) + elif defined(posix): + var ts {.noinit.}: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + elif defined(windows): + var f {.noinit.}: FILETIME + getSystemTimeAsFileTime(f) + result = fromWinTime(rdFileTime(f)) proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = ## Computes the duration between two points in time. @@ -955,7 +1040,7 @@ proc high*(typ: typedesc[Time]): Time = initTime(high(int64), high(NanosecondRange)) proc low*(typ: typedesc[Time]): Time = - initTime(low(int64), 0) + initTime(0, 0) # # DateTime & Timezone @@ -1056,13 +1141,13 @@ proc isLeapDay*(dt: DateTime): bool {.since: (1, 1).} = ## Returns whether `t` is a leap day, i.e. Feb 29 in a leap year. This matters ## as it affects time offset calculations. runnableExamples: - let dt = initDateTime(29, mFeb, 2020, 00, 00, 00, utc()) + let dt = dateTime(2020, mFeb, 29, 00, 00, 00, 00, utc()) doAssert dt.isLeapDay doAssert dt+1.years-1.years != dt - let dt2 = initDateTime(28, mFeb, 2020, 00, 00, 00, utc()) + let dt2 = dateTime(2020, mFeb, 28, 00, 00, 00, 00, utc()) doAssert not dt2.isLeapDay doAssert dt2+1.years-1.years == dt2 - doAssertRaises(Exception): discard initDateTime(29, mFeb, 2021, 00, 00, 00, utc()) + doAssertRaises(Exception): discard dateTime(2021, mFeb, 29, 00, 00, 00, 00, utc()) assertDateTimeInitialized dt dt.year.isLeapYear and dt.month == mFeb and dt.monthday == 29 @@ -1141,7 +1226,7 @@ proc name*(zone: Timezone): string = ## If the timezone doesn't exist in the tz database, or if the timezone ## name is unknown, then any string that describes the timezone ## unambiguously might be used. For example, the string "LOCAL" is used - ## for the systems local timezone. + ## for the system's local timezone. ## ## See also: https://en.wikipedia.org/wiki/Tz_database zone.name @@ -1329,14 +1414,13 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## `cpuTime` instead, depending on the use case. getTime().local -proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, - hour: HourRange, minute: MinuteRange, second: SecondRange, - nanosecond: NanosecondRange, - zone: Timezone = local()): DateTime = +proc dateTime*(year: int, month: Month, monthday: MonthdayRange, + hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0, + nanosecond: NanosecondRange = 0, + zone: Timezone = local()): DateTime = ## Create a new `DateTime <#DateTime>`_ in the specified timezone. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) - doAssert $dt1 == "2017-03-30T00:00:00Z" + assert $dateTime(2017, mMar, 30, zone = utc()) == "2017-03-30T00:00:00Z" assertValidDate monthday, month, year let dt = DateTime( @@ -1352,16 +1436,24 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, hour: HourRange, minute: MinuteRange, second: SecondRange, - zone: Timezone = local()): DateTime = + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = ## Create a new `DateTime <#DateTime>`_ in the specified timezone. - runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - doAssert $dt1 == "2017-03-30T00:00:00Z" - initDateTime(monthday, month, year, hour, minute, second, 0, zone) + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = + ## Create a new `DateTime <#DateTime>`_ in the specified timezone. + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, 0, zone) proc `+`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(hours = 5) doAssert $(dt + dur) == "2017-03-30T05:00:00Z" @@ -1369,7 +1461,7 @@ proc `+`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(days = 5) doAssert $(dt - dur) == "2017-03-25T00:00:00Z" @@ -1378,8 +1470,8 @@ proc `-`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt1, dt2: DateTime): Duration = ## Compute the duration between `dt1` and `dt2`. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - let dt2 = initDateTime(25, mMar, 2017, 00, 00, 00, utc()) + let dt1 = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dt2 = dateTime(2017, mMar, 25, 00, 00, 00, 00, utc()) doAssert dt1 - dt2 == initDuration(days = 5) @@ -1410,16 +1502,38 @@ proc getDateStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} runnableExamples: echo getDateStr(now() - 1.months) assertDateTimeInitialized dt - result = $dt.year & '-' & intToStr(dt.monthZero, 2) & - '-' & intToStr(dt.monthday, 2) + result = newStringOfCap(10) # len("YYYY-MM-DD") == 10 + result.addInt dt.year + result.add '-' + result.add intToStr(dt.monthZero, 2) + result.add '-' + result.add intToStr(dt.monthday, 2) proc getClockStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = ## Gets the current local clock time as a string of the format `HH:mm:ss`. runnableExamples: echo getClockStr(now() - 1.hours) assertDateTimeInitialized dt - result = intToStr(dt.hour, 2) & ':' & intToStr(dt.minute, 2) & - ':' & intToStr(dt.second, 2) + result = newStringOfCap(8) # len("HH:mm:ss") == 8 + result.add intToStr(dt.hour, 2) + result.add ':' + result.add intToStr(dt.minute, 2) + result.add ':' + result.add intToStr(dt.second, 2) + + +# +# Iso week forward declarations +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} # # TimeFormat @@ -1451,6 +1565,9 @@ type year: Option[int] month: Option[int] monthday: Option[int] + isoyear: Option[int] + yearweek: Option[int] + weekday: Option[WeekDay] utcOffset: Option[int] # '0' as default for these work fine @@ -1465,6 +1582,7 @@ type FormatPattern {.pure.} = enum d, dd, ddd, dddd + GG, GGGG h, hh, H, HH m, mm, M, MM, MMM, MMMM s, ss @@ -1474,6 +1592,7 @@ type YYYY uuuu UUUU + V, VV z, zz, zzz, zzzz ZZZ, ZZZZ g @@ -1602,6 +1721,8 @@ proc stringToPattern(str: string): FormatPattern = of "dd": result = dd of "ddd": result = ddd of "dddd": result = dddd + of "GG": result = GG + of "GGGG": result = GGGG of "h": result = h of "hh": result = hh of "H": result = H @@ -1624,6 +1745,8 @@ proc stringToPattern(str: string): FormatPattern = of "YYYY": result = YYYY of "uuuu": result = uuuu of "UUUU": result = UUUU + of "V": result = V + of "VV": result = VV of "z": result = z of "zz": result = zz of "zzz": result = zzz @@ -1673,6 +1796,10 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add loc.ddd[dt.weekday] of dddd: result.add loc.dddd[dt.weekday] + of GG: + result.add (dt.getIsoWeekAndYear.isoyear.int mod 100).intToStr(2) + of GGGG: + result.add $dt.getIsoWeekAndYear.isoyear of h: result.add( if dt.hour == 0: "12" @@ -1736,6 +1863,10 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add '+' & $year of UUUU: result.add $dt.year + of V: + result.add $dt.getIsoWeekAndYear.isoweek + of VV: + result.add dt.getIsoWeekAndYear.isoweek.intToStr(2) of z, zz, zzz, zzzz, ZZZ, ZZZZ: if dt.timezone != nil and dt.timezone.name == "Etc/UTC": result.add 'Z' @@ -1790,18 +1921,30 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, result = monthday in MonthdayRange of ddd: result = false - for v in loc.ddd: + for d, v in loc.ddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break of dddd: result = false - for v in loc.dddd: + for d, v in loc.dddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break + of GG: + # Assumes current century + var isoyear = takeInt(2..2) + var thisCen = now().year div 100 + parsed.isoyear = some(thisCen*100 + isoyear) + result = isoyear > 0 + of GGGG: + let isoyear = takeInt(1..high(int)) + parsed.isoyear = some(isoyear) + result = isoyear > 0 of h, H: parsed.hour = takeInt(1..2) result = parsed.hour in HourRange @@ -1892,6 +2035,14 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, parsed.year = some(year) of UUUU: parsed.year = some(takeInt(1..high(int), allowSign = true)) + of V: + let yearweek = takeInt(1..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange + of VV: + let yearweek = takeInt(2..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange of z, zz, zzz, zzzz, ZZZ, ZZZZ: case input[i] of '+', '-': @@ -1938,7 +2089,7 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, i.inc 2 else: result = false - of Lit: doAssert false, "Can't happen" + of Lit: raiseAssert "Can't happen" proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, input: string): DateTime = @@ -1987,10 +2138,42 @@ proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, if p.utcOffset.isNone: # No timezone parsed - assume timezone is `zone` - result = initDateTime(monthday, month, year, hour, minute, second, nanosecond, zone) + result = dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) else: # Otherwise convert to `zone` - result = (initDateTime(monthday, month, year, hour, minute, second, nanosecond, utc()).toTime + + result = (dateTime(year, month, monthday, hour, minute, second, nanosecond, utc()).toTime + + initDuration(seconds = p.utcOffset.get())).inZone(zone) + +proc toDateTimeByWeek(p: ParsedTime, zone: Timezone, f: TimeFormat, + input: string): DateTime = + var isoyear = p.isoyear.get(0) + var yearweek = p.yearweek.get(1) + var weekday = p.weekday.get(dMon) + + if p.amPm != apUnknown: + raiseParseException(f, input, "Parsing iso weekyear dates does not support am/pm") + + if p.year.isSome: + raiseParseException(f, input, "Use iso-year GG or GGGG as year with iso week number") + + if p.month.isSome: + raiseParseException(f, input, "Use either iso week number V or VV or month") + + if p.monthday.isSome: + raiseParseException(f, input, "Use weekday ddd or dddd as day with with iso week number") + + if p.isoyear.isNone: + raiseParseException(f, input, "Need iso-year with week number") + + let hour = p.hour + let minute = p.minute + let second = p.second + let nanosecond = p.nanosecond + + if p.utcOffset.isNone: + result = initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone) + else: + result = (initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone).toTime + initDuration(seconds = p.utcOffset.get())).inZone(zone) proc format*(dt: DateTime, f: TimeFormat, @@ -1998,7 +2181,7 @@ proc format*(dt: DateTime, f: TimeFormat, ## Format `dt` using the format specified by `f`. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == dt.format(f) assertDateTimeInitialized dt result = "" @@ -2023,7 +2206,7 @@ proc format*(dt: DateTime, f: string, loc: DateTimeLocale = DefaultLocale): stri ## See `Parsing and formatting dates`_ for documentation of the ## `format` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == format(dt, "yyyy-MM-dd") let dtFormat = initTimeFormat(f) result = dt.format(dtFormat, loc) @@ -2033,7 +2216,7 @@ proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = const f2 = initTimeFormat(f) result = dt.format(f2) -proc formatValue*(result: var string; value: DateTime, specifier: string) = +proc formatValue*(result: var string; value: DateTime | Time, specifier: string) = ## adapter for strformat. Not intended to be called directly. result.add format(value, if specifier.len == 0: "yyyy-MM-dd'T'HH:mm:sszzz" else: specifier) @@ -2046,7 +2229,7 @@ proc format*(time: Time, f: string, zone: Timezone = local()): string ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - var dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc()) + var dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, utc()) var tm = dt.toTime() doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00" time.inZone(zone).format(f) @@ -2057,13 +2240,8 @@ proc format*(time: Time, f: static[string], zone: Timezone = local()): string const f2 = initTimeFormat(f) result = time.inZone(zone).format(f2) -template formatValue*(result: var string; value: Time, specifier: string) = - ## adapter for `strformat`. Not intended to be called directly. - result.add format(value, specifier) - proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Parses `input` as a `DateTime` using the format specified by `f`. ## If no UTC offset was parsed, then `input` is assumed to be specified in ## the `zone` timezone. If a UTC offset was parsed, the result will be @@ -2072,7 +2250,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), ## Month and day names from the passed in `loc` are used. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == "2000-01-01".parse(f, utc()) var inpIdx = 0 # Input index var patIdx = 0 # Pattern index @@ -2103,31 +2281,33 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), raiseParseException(f, input, "Parsing ended but there was still patterns remaining") - result = toDateTime(parsed, zone, f, input) + if parsed.yearweek.isSome: + result = toDateTimeByWeek(parsed, zone, f, input) + elif parsed.isoyear.isSome: + raiseParseException(f, input, "Iso year GG or GGGG require iso week V or VV") + else: + result = toDateTime(parsed, zone, f, input) proc parse*(input, f: string, tz: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`. ## ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == parse("2000-01-01", "yyyy-MM-dd", utc()) let dtFormat = initTimeFormat(f) result = input.parse(dtFormat, tz, loc = loc) proc parse*(input: string, f: static[string], zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): - DateTime {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Overload that validates `f` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone, loc = loc) -proc parseTime*(input, f: string, zone: Timezone): Time - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = +proc parseTime*(input, f: string, zone: Timezone): Time {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`, then converting it a `Time`. ## @@ -2139,7 +2319,7 @@ proc parseTime*(input, f: string, zone: Timezone): Time parse(input, f, zone).toTime() proc parseTime*(input: string, f: static[string], zone: Timezone): Time - {.raises: [TimeParseError, Defect].} = + {.parseRaises.} = ## Overload that validates `format` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone).toTime() @@ -2148,7 +2328,7 @@ proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = ## Converts a `DateTime` object to a string representation. ## It uses the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $dt == "2000-01-01T12:00:00Z" doAssert $default(DateTime) == "Uninitialized DateTime" if not dt.isInitialized: @@ -2160,7 +2340,7 @@ proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## Converts a `Time` value to a string representation. It will use the local ## time zone and use the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + let dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, local()) let tm = dt.toTime() doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") $time.local @@ -2182,7 +2362,7 @@ proc initTimeInterval*(nanoseconds, microseconds, milliseconds, ## `seconds`, `minutes`, `hours`, `days`, `months`, and `years`. runnableExamples: let day = initTimeInterval(hours = 24) - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $(dt + day) == "2000-01-02T12:00:00Z" doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) result.nanoseconds = nanoseconds @@ -2268,8 +2448,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = ## - If `startDt.timezone != endDt.timezone`, then the result will be ## equivalent to `between(startDt.utc, endDt.utc)`. runnableExamples: - var a = initDateTime(25, mMar, 2015, 12, 0, 0, utc()) - var b = initDateTime(1, mApr, 2017, 15, 0, 15, utc()) + var a = dateTime(2015, mMar, 25, 12, 0, 0, 00, utc()) + var b = dateTime(2017, mApr, 1, 15, 0, 15, 00, utc()) var ti = initTimeInterval(years = 2, weeks = 1, hours = 3, seconds = 15) doAssert between(a, b) == ti doAssert between(a, b) == -between(b, a) @@ -2349,8 +2529,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = startDate = endDate # Handle hours, minutes, seconds, milliseconds, microseconds and nanoseconds - let newStartDt = initDateTime(startDate.monthday, startDate.month.Month, - startDate.year, startDt.hour, startDt.minute, startDt.second, + let newStartDt = dateTime(startDate.year, startDate.month.Month, + startDate.monthday, startDt.hour, startDt.minute, startDt.second, startDt.nanosecond, startDt.timezone) let dur = endDt - newStartDt let parts = toParts(dur) @@ -2501,7 +2681,7 @@ proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## So adding one month to `31 October` will result in `31 November`, which ## will overflow and result in `1 December`. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" # This is correct and happens due to monthday overflow. doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" @@ -2524,7 +2704,7 @@ proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = ## then the `months` component and so on. The returned `DateTime` will ## have the same timezone as the input. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" dt + (-interval) @@ -2567,6 +2747,33 @@ proc `+=`*(t: var Time, b: TimeInterval) = proc `-=`*(t: var Time, b: TimeInterval) = t = t - b +# +# Iso week +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + ## Create a new `DateTime <#DateTime>`_ from a weekday and an ISO 8601 week number and year + ## in the specified timezone. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert initDateTime(21, mApr, 2018, 00, 00, 00) == initDateTime(dSat, 16, 2018.IsoYear, 00, 00, 00) + assert initDateTime(30, mDec, 2019, 00, 00, 00) == initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) + assert initDateTime(13, mSep, 2020, 00, 00, 00) == initDateTime(dSun, 37, 2020.IsoYear, 00, 00, 00) + assert initDateTime(2, mJan, 2021, 00, 00, 00) == initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) + + # source https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00, zone).weekday.int - 4 + initDateTime(1, mJan, isoyear.int, hour, minute, second, nanosecond, zone) + initTimeInterval(days=d) + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + initDateTime(weekday, isoweek, isoyear, hour, minute, second, 0, zone) + # # Other # @@ -2580,7 +2787,9 @@ proc epochTime*(): float {.tags: [TimeEffect].} = ## ## .. warning:: Unsuitable for benchmarking (but still better than `now`), ## use `monotimes.getMonoTime` or `cpuTime` instead, depending on the use case. - when defined(macosx): + when defined(js): + result = newDate().getTime() / 1000 + elif defined(macosx): var a {.noinit.}: Timeval gettimeofday(a) result = toBiggestFloat(a.tv_sec.int64) + toBiggestFloat( @@ -2597,8 +2806,6 @@ proc epochTime*(): float {.tags: [TimeEffect].} = var secs = i64 div rateDiff var subsecs = i64 mod rateDiff result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 - elif defined(js): - result = newDate().getTime() / 1000 else: {.error: "unknown OS".} diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 6827e23b1a..7c58d1ddc3 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -15,6 +15,10 @@ import std/private/since export system.`$` # for backward compatibility +when defined(nimPreviewSlimSystem): + import std/assertions + + type HoleyEnum* = (not Ordinal) and enum ## Enum with holes. type OrdinalEnum* = Ordinal and enum ## Enum without holes. @@ -31,7 +35,7 @@ runnableExamples: assert C[float] is HoleyEnum proc name*(t: typedesc): string {.magic: "TypeTrait".} = - ## Returns the name of the given type. + ## Returns the name of `t`. ## ## Alias for `system.\`$\`(t) <dollars.html#$,typedesc>`_ since Nim v0.20. runnableExamples: @@ -39,7 +43,7 @@ proc name*(t: typedesc): string {.magic: "TypeTrait".} = doAssert name(seq[string]) == "seq[string]" proc arity*(t: typedesc): int {.magic: "TypeTrait".} = - ## Returns the arity of the given type. This is the number of "type" + ## Returns the arity of `t`. This is the number of "type" ## components or the number of generic parameters a given type `t` has. runnableExamples: doAssert arity(int) == 0 @@ -88,11 +92,27 @@ proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} = doAssert stripGenericParams(int) is int proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} - ## This trait returns true if the type `t` is safe to use for - ## `copyMem`:idx:. + ## Returns true if `t` is safe to use for `copyMem`:idx:. ## ## Other languages name a type like these `blob`:idx:. +proc hasDefaultValue*(t: typedesc): bool {.magic: "TypeTrait".} = + ## Returns true if `t` has a valid default value. + runnableExamples: + {.experimental: "strictNotNil".} + type + NilableObject = ref object + a: int + Object = NilableObject not nil + RequiresInit[T] = object + a {.requiresInit.}: T + + assert hasDefaultValue(NilableObject) + assert not hasDefaultValue(Object) + assert hasDefaultValue(string) + assert not hasDefaultValue(var string) + assert not hasDefaultValue(RequiresInit[int]) + proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = ## Returns true for named tuples, false for any other type. runnableExamples: @@ -100,25 +120,42 @@ proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = doAssert not isNamedTuple((string, int)) doAssert isNamedTuple(tuple[name: string, age: int]) -proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} = +template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = + ## Returns `T` for `ref T | ptr T`. + runnableExamples: + assert (ref int).pointerBase is int + type A = ptr seq[float] + assert A.pointerBase is seq[float] + assert (ref A).pointerBase is A # not seq[float] + assert (var s = "abc"; s[0].addr).typeof.pointerBase is char + T + +proc distinctBase*(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} = ## Returns the base type for distinct types, or the type itself otherwise. + ## If `recursive` is false, only the immediate distinct base will be returned. ## ## **See also:** - ## * `distinctBase template <#distinctBase.t,T>`_ + ## * `distinctBase template <#distinctBase.t,T,static[bool]>`_ runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert distinctBase(MyInt) is int + doAssert distinctBase(MyOtherInt) is int + doAssert distinctBase(MyOtherInt, false) is MyInt doAssert distinctBase(int) is int since (1, 1): - template distinctBase*[T](a: T): untyped = - ## Overload of `distinctBase <#distinctBase,typedesc>`_ for values. + template distinctBase*[T](a: T, recursive: static bool = true): untyped = + ## Overload of `distinctBase <#distinctBase,typedesc,static[bool]>`_ for values. runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert 12.MyInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase(false) is MyInt doAssert 12.distinctBase == 12 when T is distinct: - distinctBase(typeof(a))(a) + distinctBase(typeof(a), recursive)(a) else: # avoids hint ConvFromXtoItselfNotNeeded a @@ -205,7 +242,7 @@ macro genericParamsImpl(T: typedesc): untyped = case ai.typeKind of ntyTypeDesc: ret = ai - of ntyStatic: doAssert false + of ntyStatic: raiseAssert "unreachable" else: # getType from a resolved symbol might return a typedesc symbol. # If so, use it directly instead of wrapping it in StaticParam. @@ -263,3 +300,43 @@ since (1, 1): type T2 = T genericParamsImpl(T2) + + +proc hasClosureImpl(n: NimNode): bool = discard "see compiler/vmops.nim" + +proc hasClosure*(fn: NimNode): bool {.since: (1, 5, 1).} = + ## Returns true if the func/proc/etc `fn` has `closure`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = hasClosureImpl(fn) + +template toUnsigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns an unsigned type with same bit size as `T`. + runnableExamples: + assert int8.toUnsigned is uint8 + assert uint.toUnsigned is uint + assert int.toUnsigned is uint + # range types are currently unsupported: + assert not compiles(toUnsigned(range[0..7])) + when T is int8: uint8 + elif T is int16: uint16 + elif T is int32: uint32 + elif T is int64: uint64 + elif T is int: uint + else: T + +template toSigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns a signed type with same bit size as `T`. + runnableExamples: + assert int8.toSigned is int8 + assert uint16.toSigned is int16 + # range types are currently unsupported: + assert not compiles(toSigned(range[0..7])) + when T is uint8: int8 + elif T is uint16: int16 + elif T is uint32: int32 + elif T is uint64: int64 + elif T is uint: int + else: T diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 903f01fb41..8cbe117bb7 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -21,6 +21,16 @@ ## * `encodings module <encodings.html>`_ include "system/inclrtl" +import std/strbasics +template toOa(s: string): auto = s.toOpenArray(0, s.high) + +proc substr(s: openArray[char] , first, last: int): string = + # Copied substr from system + let first = max(first, 0) + let L = max(min(last, high(s)) - first + 1, 0) + result = newString(L) + for i in 0 .. L-1: + result[i] = s[i+first] type RuneImpl = int32 # underlying type of Rune @@ -32,7 +42,7 @@ type template ones(n: untyped): untyped = ((1 shl n)-1) -proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = +proc runeLen*(s: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Returns the number of runes of the string ``s``. runnableExamples: let a = "añyóng" @@ -51,7 +61,7 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = else: inc i inc(result) -proc runeLenAt*(s: string, i: Natural): int = +proc runeLenAt*(s: openArray[char], i: Natural): int = ## Returns the number of bytes the rune starting at ``s[i]`` takes. ## ## See also: @@ -71,7 +81,7 @@ proc runeLenAt*(s: string, i: Natural): int = const replRune = Rune(0xFFFD) -template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = +template fastRuneAt*(s: openArray[char] or string, i: int, result: untyped, doInc = true) = ## Returns the rune ``s[i]`` in ``result``. ## ## If ``doInc == true`` (default), ``i`` is incremented by the number @@ -149,7 +159,7 @@ template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = result = Rune(uint(s[i])) when doInc: inc(i) -proc runeAt*(s: string, i: Natural): Rune = +proc runeAt*(s: openArray[char], i: Natural): Rune = ## Returns the rune in ``s`` at **byte index** ``i``. ## ## See also: @@ -163,7 +173,7 @@ proc runeAt*(s: string, i: Natural): Rune = doAssert a.runeAt(3) == "y".runeAt(0) fastRuneAt(s, i, result, false) -proc validateUtf8*(s: string): int = +proc validateUtf8*(s: openArray[char]): int = ## Returns the position of the invalid byte in ``s`` if the string ``s`` does ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. ## @@ -300,7 +310,7 @@ proc `$`*(runes: seq[Rune]): string = for rune in runes: result.add rune -proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = +proc runeOffset*(s: openArray[char], pos: Natural, start: Natural = 0): int = ## Returns the byte position of rune ## at position ``pos`` in ``s`` with an optional start byte position. ## Returns the special value -1 if it runs out of the string. @@ -327,13 +337,13 @@ proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = inc i return o -proc runeReverseOffset*(s: string, rev: Positive): (int, int) = +proc runeReverseOffset*(s: openArray[char], rev: Positive): (int, int) = ## Returns a tuple with the byte offset of the ## rune at position ``rev`` in ``s``, counting ## from the end (starting with 1) and the total ## number of runes in the string. ## - ## Returns a negative value for offset if there are to few runes in + ## Returns a negative value for offset if there are too few runes in ## the string to satisfy the request. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -346,18 +356,16 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = a = rev.int o = 0 x = 0 + let times = 2*rev.int-s.runeLen # transformed from rev.int - a < s.runeLen - rev.int while o < s.len: let r = runeLenAt(s, o) o += r - if a < 0: + if a > times: x += r dec a + result = if a > 0: (-a, rev.int-a) else: (x, -a+rev.int) - if a > 0: - return (-a, rev.int-a) - return (x, -a+rev.int) - -proc runeAtPos*(s: string, pos: int): Rune = +proc runeAtPos*(s: openArray[char], pos: int): Rune = ## Returns the rune at position ``pos``. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -370,7 +378,7 @@ proc runeAtPos*(s: string, pos: int): Rune = ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ fastRuneAt(s, runeOffset(s, pos), result, false) -proc runeStrAtPos*(s: string, pos: Natural): string = +proc runeStrAtPos*(s: openArray[char], pos: Natural): string = ## Returns the rune at position ``pos`` as UTF8 String. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -382,9 +390,9 @@ proc runeStrAtPos*(s: string, pos: Natural): string = ## * `runeAtPos proc <#runeAtPos,string,int>`_ ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ let o = runeOffset(s, pos) - s[o .. (o+runeLenAt(s, o)-1)] + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) -proc runeSubStr*(s: string, pos: int, len: int = int.high): string = +proc runeSubStr*(s: openArray[char], pos: int, len: int = int.high): string = ## Returns the UTF-8 substring starting at code point ``pos`` ## with ``len`` code points. ## @@ -403,7 +411,7 @@ proc runeSubStr*(s: string, pos: int, len: int = int.high): string = if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: - result = s.substr(o, s.len-1) + result = s.substr(o, s.high) elif len < 0: let e = rl + len if e < 0: @@ -456,7 +464,7 @@ proc `==`*(a, b: Rune): bool = include "includes/unicode_ranges" -proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = +proc binarySearch(c: RuneImpl, tab: openArray[int32], len, stride: int): int = var n = len var t = 0 while n > 1: @@ -628,7 +636,7 @@ template runeCheck(s, runeProc) = fastRuneAt(s, i, rune, doInc = true) result = runeProc(rune) and result -proc isAlpha*(s: string): bool {.noSideEffect, +proc isAlpha*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all alphabetic runes. runnableExamples: @@ -636,7 +644,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, doAssert a.isAlpha runeCheck(s, isAlpha) -proc isSpace*(s: string): bool {.noSideEffect, +proc isSpace*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all whitespace runes. runnableExamples: @@ -657,21 +665,21 @@ template convertRune(s, runeProc) = rune = runeProc(rune) fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc toUpper*(s: string): string {.noSideEffect, +proc toUpper*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into upper-case runes. runnableExamples: doAssert toUpper("abγ") == "ABΓ" convertRune(s, toUpper) -proc toLower*(s: string): string {.noSideEffect, +proc toLower*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into lower-case runes. runnableExamples: doAssert toLower("ABΓ") == "abγ" convertRune(s, toLower) -proc swapCase*(s: string): string {.noSideEffect, +proc swapCase*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Swaps the case of runes in ``s``. ## @@ -693,7 +701,7 @@ proc swapCase*(s: string): string {.noSideEffect, rune = rune.toUpper() fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc capitalize*(s: string): string {.noSideEffect, +proc capitalize*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts the first character of ``s`` into an upper-case rune. runnableExamples: @@ -705,10 +713,13 @@ proc capitalize*(s: string): string {.noSideEffect, rune: Rune i = 0 fastRuneAt(s, i, rune, doInc = true) - result = $toUpper(rune) & substr(s, i) + result = $toUpper(rune) & substr(s.toOpenArray(i, s.high)) -proc translate*(s: string, replacements: proc(key: string): string): string {. - rtl, extern: "nuc$1".} = +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +proc translate*(s: openArray[char], replacements: proc(key: string): string): string {. + rtl, extern: "nuc$1", effectsOf: replacements.} = ## Translates words in a string using the ``replacements`` proc to substitute ## words inside ``s`` with their replacements. ## @@ -742,7 +753,7 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if whiteSpace and inWord: # If we've reached the end of a word - let word = s[wordStart ..< lastIndex] + let word = substr(s.toOpenArray(wordStart, lastIndex - 1)) result.add(replacements(word)) result.add($rune) inWord = false @@ -757,10 +768,10 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if wordStart < len(s) and inWord: # Get the trailing word at the end - let word = s[wordStart .. ^1] + let word = substr(s.toOpenArray(wordStart, s.high)) result.add(replacements(word)) -proc title*(s: string): string {.noSideEffect, +proc title*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts ``s`` to a unicode title. ## @@ -786,7 +797,7 @@ proc title*(s: string): string {.noSideEffect, fastToUTF8Copy(rune, result, resultIndex, doInc = true) -iterator runes*(s: string): Rune = +iterator runes*(s: openArray[char]): Rune = ## Iterates over any rune of the string ``s`` returning runes. var i = 0 @@ -795,7 +806,7 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result -iterator utf8*(s: string): string = +iterator utf8*(s: openArray[char]): string = ## Iterates over any rune of the string ``s`` returning utf8 values. ## ## See also: @@ -806,10 +817,10 @@ iterator utf8*(s: string): string = var o = 0 while o < s.len: let n = runeLenAt(s, o) - yield s[o .. (o+n-1)] + yield substr(s.toOpenArray(o, (o+n-1))) o += n -proc toRunes*(s: string): seq[Rune] = +proc toRunes*(s: openArray[char]): seq[Rune] = ## Obtains a sequence containing the Runes in ``s``. ## ## See also: @@ -822,12 +833,12 @@ proc toRunes*(s: string): seq[Rune] = for r in s.runes: result.add(r) -proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1".} = +proc cmpRunesIgnoreCase*(a, b: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Compares two UTF-8 strings and ignores the case. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b var i = 0 var j = 0 var ar, br: Rune @@ -835,11 +846,16 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1".} = # slow path: fastRuneAt(a, i, ar) fastRuneAt(b, j, br) - result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) + when sizeof(int) < 4: + const lo = low(int).int32 + const hi = high(int).int32 + result = clamp(RuneImpl(toLower(ar)) - RuneImpl(toLower(br)), lo, hi).int + else: + result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) if result != 0: return result = a.len - b.len -proc reversed*(s: string): string = +proc reversed*(s: openArray[char]): string = ## Returns the reverse of ``s``, interpreting it as runes. ## ## Unicode combining characters are correctly interpreted as well. @@ -874,9 +890,9 @@ proc reversed*(s: string): string = reverseUntil(len(s)) -proc graphemeLen*(s: string; i: Natural): Natural = +proc graphemeLen*(s: openArray[char]; i: Natural): Natural = ## The number of bytes belonging to byte index ``s[i]``, - ## including following combining code unit. + ## including following combining code units. runnableExamples: let a = "añyóng" doAssert a.graphemeLen(1) == 2 ## ñ @@ -893,7 +909,7 @@ proc graphemeLen*(s: string; i: Natural): Natural = if not isCombining(r2): break result = j-i -proc lastRune*(s: string; last: int): (Rune, int) = +proc lastRune*(s: openArray[char]; last: int): (Rune, int) = ## Length of the last rune in ``s[0..last]``. Returns the rune and its length ## in bytes. if s[last] <= chr(127): @@ -922,12 +938,12 @@ proc size*(r: Rune): int {.noSideEffect.} = else: result = 1 # --------- Private templates for different split separators ----------- -proc stringHasSep(s: string, index: int, seps: openArray[Rune]): bool = +proc stringHasSep(s: openArray[char], index: int, seps: openArray[Rune]): bool = var rune: Rune fastRuneAt(s, index, rune, false) return seps.contains(rune) -proc stringHasSep(s: string, index: int, sep: Rune): bool = +proc stringHasSep(s: openArray[char], index: int, sep: Rune): bool = var rune: Rune fastRuneAt(s, index, rune, false) return sep == rune @@ -945,12 +961,12 @@ template splitCommon(s, sep, maxsplit: untyped) = while last < sLen and not stringHasSep(s, last, sep): inc(last, runeLenAt(s, last)) if splits == 0: last = sLen - yield s[first .. (last - 1)] + yield substr(s.toOpenArray(first, (last - 1))) if splits == 0: break dec(splits) inc(last, if last < sLen: runeLenAt(s, last) else: 1) -iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, +iterator split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a group of separators. ## @@ -976,7 +992,7 @@ iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, splitCommon(s, seps, maxsplit) -iterator splitWhitespace*(s: string): string = +iterator splitWhitespace*(s: openArray[char]): string = ## Splits a unicode string at whitespace runes. splitCommon(s, unicodeSpaces, -1) @@ -984,13 +1000,13 @@ template accResult(iter: untyped) = result = @[] for x in iter: add(result, x) -proc splitWhitespace*(s: string): seq[string] {.noSideEffect, +proc splitWhitespace*(s: openArray[char]): seq[string] {.noSideEffect, rtl, extern: "ncuSplitWhitespace".} = ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ ## iterator, but is a proc that returns a sequence of substrings. accResult(splitWhitespace(s)) -iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = +iterator split*(s: openArray[char], sep: Rune, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a single separator. ## Substrings are separated by the rune ``sep``. runnableExamples: @@ -1001,19 +1017,19 @@ iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = splitCommon(s, sep, maxsplit) -proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): +proc split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRunes".} = ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, ## but is a proc that returns a sequence of substrings. accResult(split(s, seps, maxsplit)) -proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, +proc split*(s: openArray[char], sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRune".} = ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc ## that returns a sequence of substrings. accResult(split(s, sep, maxsplit)) -proc strip*(s: string, leading = true, trailing = true, +proc strip*(s: openArray[char], leading = true, trailing = true, runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, rtl, extern: "nucStrip".} = ## Strips leading or trailing ``runes`` from ``s`` and returns @@ -1068,7 +1084,7 @@ proc strip*(s: string, leading = true, trailing = true, let newLen = eI - sI + 1 result = newStringOfCap(newLen) if newLen > 0: - result.add s[sI .. eI] + result.add substr(s.toOpenArray(sI, eI)) proc repeat*(c: Rune, count: Natural): string {.noSideEffect, rtl, extern: "nucRepeatRune".} = @@ -1084,7 +1100,7 @@ proc repeat*(c: Rune, count: Natural): string {.noSideEffect, for i in 0 ..< count: result.add s -proc align*(s: string, count: Natural, padding = ' '.Rune): string {. +proc align*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect, rtl, extern: "nucAlignString".} = ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length ## of ``count``. @@ -1109,9 +1125,9 @@ proc align*(s: string, count: Natural, padding = ' '.Rune): string {. for i in 0 ..< spaces: result.add padStr result.add s else: - result = s + result = s.substr -proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. +proc alignLeft*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect.} = ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a ## rune-length of ``count``. @@ -1135,4 +1151,365 @@ proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. for i in sLen ..< count: result.add padStr else: - result = s + result = s.substr + + +proc runeLen*(s: string): int {.inline.} = + ## Returns the number of runes of the string ``s``. + runnableExamples: + let a = "añyóng" + doAssert a.runeLen == 6 + ## note: a.len == 8 + runeLen(toOa(s)) + +proc runeLenAt*(s: string, i: Natural): int {.inline.} = + ## Returns the number of bytes the rune starting at ``s[i]`` takes. + ## + ## See also: + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeLenAt(0) == 1 + doAssert a.runeLenAt(1) == 2 + runeLenAt(toOa(s), i) + +proc runeAt*(s: string, i: Natural): Rune {.inline.} = + ## Returns the rune in ``s`` at **byte index** ``i``. + ## + ## See also: + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1) == "ñ".runeAt(0) + doAssert a.runeAt(2) == "ñ".runeAt(1) + doAssert a.runeAt(3) == "y".runeAt(0) + fastRuneAt(s, i, result, false) + +proc validateUtf8*(s: string): int {.inline.} = + ## Returns the position of the invalid byte in ``s`` if the string ``s`` does + ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. + ## + ## See also: + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + validateUtf8(toOa(s)) + +proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int {.inline.} = + ## Returns the byte position of rune + ## at position ``pos`` in ``s`` with an optional start byte position. + ## Returns the special value -1 if it runs out of the string. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeReverseOffset proc <#runeReverseOffset,string,Positive>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeOffset(1) == 1 + doAssert a.runeOffset(3) == 4 + doAssert a.runeOffset(4) == 6 + runeOffset(toOa(s), pos, start) + +proc runeReverseOffset*(s: string, rev: Positive): (int, int) {.inline.} = + ## Returns a tuple with the byte offset of the + ## rune at position ``rev`` in ``s``, counting + ## from the end (starting with 1) and the total + ## number of runes in the string. + ## + ## Returns a negative value for offset if there are too few runes in + ## the string to satisfy the request. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeOffset proc <#runeOffset,string,Natural,Natural>`_ + runeReverseOffset(toOa(s), rev) + +proc runeAtPos*(s: string, pos: int): Rune {.inline.} = + ## Returns the rune at position ``pos``. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + fastRuneAt(toOa(s), runeOffset(s, pos), result, false) + +proc runeStrAtPos*(s: string, pos: Natural): string {.inline.} = + ## Returns the rune at position ``pos`` as UTF8 String. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + let o = runeOffset(s, pos) + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) + +proc runeSubStr*(s: string, pos: int, len: int = int.high): string {.inline.} = + ## Returns the UTF-8 substring starting at code point ``pos`` + ## with ``len`` code points. + ## + ## If ``pos`` or ``len`` is negative they count from + ## the end of the string. If ``len`` is not given it means the longest + ## possible string. + runnableExamples: + let s = "Hänsel ««: 10,00€" + doAssert(runeSubStr(s, 0, 2) == "Hä") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeSubStr(s, -6) == "10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, -6, 3) == "10,") + runeSubStr(toOa(s), pos, len) + + +proc isAlpha*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all alphabetic runes. + runnableExamples: + let a = "añyóng" + doAssert a.isAlpha + isAlpha(toOa(s)) + +proc isSpace*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all whitespace runes. + runnableExamples: + let a = "\t\l \v\r\f" + doAssert a.isSpace + isSpace(toOa(s)) + + +proc toUpper*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into upper-case runes. + runnableExamples: + doAssert toUpper("abγ") == "ABΓ" + toUpper(toOa(s)) + +proc toLower*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into lower-case runes. + runnableExamples: + doAssert toLower("ABΓ") == "abγ" + toLower(toOa(s)) + +proc swapCase*(s: string): string {.noSideEffect, inline.} = + ## Swaps the case of runes in ``s``. + ## + ## Returns a new string such that the cases of all runes + ## are swapped if possible. + runnableExamples: + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + swapCase(toOa(s)) + +proc capitalize*(s: string): string {.noSideEffect.} = + ## Converts the first character of ``s`` into an upper-case rune. + runnableExamples: + doAssert capitalize("βeta") == "Βeta" + capitalize(toOa(s)) + + +proc translate*(s: string, replacements: proc(key: string): string): string {.effectsOf: replacements, inline.} = + ## Translates words in a string using the ``replacements`` proc to substitute + ## words inside ``s`` with their replacements. + ## + ## ``replacements`` is any proc that takes a word and returns + ## a new word to fill it's place. + runnableExamples: + proc wordToNumber(s: string): string = + case s + of "one": "1" + of "two": "2" + else: s + let a = "one two three four" + doAssert a.translate(wordToNumber) == "1 2 three four" + translate(toOa(s), replacements) + +proc title*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside ``s`` is capitalized. + runnableExamples: + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + title(toOa(s)) + + +iterator runes*(s: string): Rune = + ## Iterates over any rune of the string ``s`` returning runes. + for rune in runes(toOa(s)): + yield rune + +iterator utf8*(s: string): string = + ## Iterates over any rune of the string ``s`` returning utf8 values. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + for str in utf8(toOa(s)): + yield str + +proc toRunes*(s: string): seq[Rune] {.inline.} = + ## Obtains a sequence containing the Runes in ``s``. + ## + ## See also: + ## * `$ proc <#$,Rune>`_ for a reverse operation + runnableExamples: + let a = toRunes("aáä") + doAssert a == @["a".runeAt(0), "á".runeAt(0), "ä".runeAt(0)] + toRunes(toOa(s)) + +proc cmpRunesIgnoreCase*(a, b: string): int {.inline.} = + ## Compares two UTF-8 strings and ignores the case. Returns: + ## + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b + cmpRunesIgnoreCase(a.toOa(), b.toOa()) + +proc reversed*(s: string): string {.inline.} = + ## Returns the reverse of ``s``, interpreting it as runes. + ## + ## Unicode combining characters are correctly interpreted as well. + runnableExamples: + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + reversed(toOa(s)) + +proc graphemeLen*(s: string; i: Natural): Natural {.inline.} = + ## The number of bytes belonging to byte index ``s[i]``, + ## including following combining code unit. + runnableExamples: + let a = "añyóng" + doAssert a.graphemeLen(1) == 2 ## ñ + doAssert a.graphemeLen(2) == 1 + doAssert a.graphemeLen(4) == 2 ## ó + graphemeLen(toOa(s), i) + +proc lastRune*(s: string; last: int): (Rune, int) {.inline.} = + ## Length of the last rune in ``s[0..last]``. Returns the rune and its length + ## in bytes. + lastRune(toOa(s), last) + +iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, + maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a group of separators. + ## + ## Substrings are separated by a substring containing only ``seps``. + runnableExamples: + import std/sequtils + + assert toSeq("hÃllo\lthis\lis an\texample\l是".split) == + @["hÃllo", "this", "is", "an", "example", "是"] + + # And the following code splits the same string using a sequence of Runes. + assert toSeq(split("añyóng:hÃllo;是$example", ";:$".toRunes)) == + @["añyóng", "hÃllo", "是", "example"] + + # example with a `Rune` separator and unused one `;`: + assert toSeq(split("ab是de:f:", ";:是".toRunes)) == @["ab", "de", "f", ""] + + # Another example that splits a string containing a date. + let date = "2012-11-20T22:08:08.398990" + + assert toSeq(split(date, " -:T".toRunes)) == + @["2012", "11", "20", "22", "08", "08.398990"] + + splitCommon(toOa(s), seps, maxsplit) + +iterator splitWhitespace*(s: string): string = + ## Splits a unicode string at whitespace runes. + splitCommon(s.toOa(), unicodeSpaces, -1) + + +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, inline.}= + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accResult(splitWhitespace(toOa(s))) + +iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a single separator. + ## Substrings are separated by the rune ``sep``. + runnableExamples: + import std/sequtils + + assert toSeq(split(";;hÃllo;this;is;an;;example;;;是", ";".runeAt(0))) == + @["", "", "hÃllo", "this", "is", "an", "", "example", "", "", "是"] + + splitCommon(toOa(s), sep, maxsplit) + +proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): + seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, + ## but is a proc that returns a sequence of substrings. + accResult(split(toOa(s), seps, maxsplit)) + +proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc + ## that returns a sequence of substrings. + accResult(split(toOa(s), sep, maxsplit)) + +proc strip*(s: string, leading = true, trailing = true, + runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, inline.} = + ## Strips leading or trailing ``runes`` from ``s`` and returns + ## the resulting string. + ## + ## If ``leading`` is true (default), leading ``runes`` are stripped. + ## If ``trailing`` is true (default), trailing ``runes`` are stripped. + ## If both are false, the string is returned unchanged. + runnableExamples: + let a = "\táñyóng " + doAssert a.strip == "áñyóng" + doAssert a.strip(leading = false) == "\táñyóng" + doAssert a.strip(trailing = false) == "áñyóng " + strip(toOa(s), leading, trailing, runes) + + +proc align*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length + ## of ``count``. + ## + ## ``padding`` characters (by default spaces) are added before ``s`` resulting in + ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to left align a string use the `alignLeft + ## proc <#alignLeft,string,Natural>`_. + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#'.Rune) == "##1232" + assert align("Åge", 5) == " Åge" + assert align("×", 4, '_'.Rune) == "___×" + align(toOa(s), count, padding) + +proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a + ## rune-length of ``count``. + ## + ## ``padding`` characters (by default spaces) are added after ``s`` resulting in + ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to right align a string use the `align + ## proc <#align,string,Natural>`_. + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#'.Rune) == "1232##" + assert alignLeft("Åge", 5) == "Åge " + assert alignLeft("×", 4, '_'.Rune) == "×___" + alignLeft(toOa(s), count, padding) diff --git a/lib/pure/unidecode/gen.py b/lib/pure/unidecode/gen.py index 0b180a3817..2fb69f7b20 100644 --- a/lib/pure/unidecode/gen.py +++ b/lib/pure/unidecode/gen.py @@ -1,4 +1,4 @@ -#! usr/bin/env python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Generates the unidecode.dat module diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 3cff833e4b..cfb7622585 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -34,9 +34,9 @@ ## ## Specify the test name as a command line argument. ## -## .. code:: -## +## ```cmd ## nim c -r test "my test name" "another test" +## ``` ## ## Multiple arguments can be used. ## @@ -45,9 +45,9 @@ ## ## Specify the suite name delimited by `"::"`. ## -## .. code:: -## +## ```cmd ## nim c -r test "my test name::" +## ``` ## ## Selecting tests by pattern ## ========================== @@ -58,19 +58,18 @@ ## ## Tests matching **any** of the arguments are executed. ## -## .. code:: -## +## ```cmd ## nim c -r test fast_suite::mytest1 fast_suite::mytest2 ## nim c -r test "fast_suite::mytest*" ## nim c -r test "auth*::" "crypto::hashing*" ## # Run suites starting with 'bug #' and standalone tests starting with '#' ## nim c -r test 'bug #*::' '::#*' +## ``` ## ## Examples ## ======== ## -## .. code:: nim -## +## ```nim ## suite "description for this stuff": ## echo "suite setup: run once before the tests" ## @@ -96,6 +95,7 @@ ## discard v[4] ## ## echo "suite teardown: run once after the tests" +## ``` ## ## Limitations/Bugs ## ================ @@ -108,6 +108,9 @@ import std/private/since import std/exitprocs +when defined(nimPreviewSlimSystem): + import std/assertions + import std/[macros, strutils, streams, times, sets, sequtils] when declared(stdout): @@ -218,7 +221,7 @@ proc resetOutputFormatters* {.since: (1, 1).} = formatters = @[] proc newConsoleOutputFormatter*(outputLevel: OutputLevel = outputLevelDefault, - colorOutput = true): <//>ConsoleOutputFormatter = + colorOutput = true): ConsoleOutputFormatter = ConsoleOutputFormatter( outputLevel: outputLevel, colorOutput: colorOutput @@ -232,7 +235,7 @@ proc colorOutput(): bool = else: result = false of "on": result = true of "off": result = false - else: doAssert false, $color + else: raiseAssert $color when declared(stdout): if existsEnv("NIMTEST_COLOR"): @@ -246,7 +249,7 @@ proc colorOutput(): bool = deprecateEnvVarHere() result = false -proc defaultConsoleFormatter*(): <//>ConsoleOutputFormatter = +proc defaultConsoleFormatter*(): ConsoleOutputFormatter = var colorOutput = colorOutput() var outputLevel = nimUnittestOutputLevel.parseEnum[:OutputLevel] when declared(stdout): @@ -315,7 +318,7 @@ proc xmlEscape(s: string): string = else: result.add(c) -proc newJUnitOutputFormatter*(stream: Stream): <//>JUnitOutputFormatter = +proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter = ## Creates a formatter that writes report to the specified stream in ## JUnit format. ## The ``stream`` is NOT closed automatically when the test are finished, @@ -430,8 +433,6 @@ proc matchFilter(suiteName, testName, filter: string): bool = return glob(suiteName, suiteAndTestFilters[0]) and glob(testName, suiteAndTestFilters[1]) -when defined(testing): export matchFilter - proc shouldRun(currentSuiteName, testName: string): bool = ## Check if a test should be run by matching suiteName and testName against ## test filters. @@ -472,27 +473,26 @@ template suite*(name, body) {.dirty.} = ## common fixture (``setup``, ``teardown``). The fixture is executed ## for EACH test. ## - ## .. code-block:: nim - ## suite "test suite for addition": - ## setup: - ## let result = 4 + ## ```nim + ## suite "test suite for addition": + ## setup: + ## let result = 4 ## - ## test "2 + 2 = 4": - ## check(2+2 == result) + ## test "2 + 2 = 4": + ## check(2+2 == result) ## - ## test "(2 + -2) != 4": - ## check(2 + -2 != result) + ## test "(2 + -2) != 4": + ## check(2 + -2 != result) ## - ## # No teardown needed + ## # No teardown needed + ## ``` ## ## The suite will run the individual test cases in the order in which ## they were listed. With default global settings the above code prints: ## - ## .. code-block:: - ## - ## [Suite] test suite for addition - ## [OK] 2 + 2 = 4 - ## [OK] (2 + -2) != 4 + ## [Suite] test suite for addition + ## [OK] 2 + 2 = 4 + ## [OK] (2 + -2) != 4 bind formatters, ensureInitialized, suiteEnded block: @@ -518,20 +518,24 @@ proc exceptionTypeName(e: ref Exception): string {.inline.} = if e == nil: "<foreign exception>" else: $e.name +when not declared(setProgramResult): + {.warning: "setProgramResult not available on platform, unittest will not" & + " give failing exit code on test failure".} + template setProgramResult(a: int) = + discard + template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## - ## .. code-block:: nim - ## - ## test "roses are red": - ## let roses = "red" - ## check(roses == "red") + ## ```nim + ## test "roses are red": + ## let roses = "red" + ## check(roses == "red") + ## ``` ## ## The above code outputs: ## - ## .. code-block:: - ## - ## [OK] roses are red + ## [OK] roses are red bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName, setProgramResult ensureInitialized() @@ -543,11 +547,14 @@ template test*(name, body) {.dirty.} = for formatter in formatters: formatter.testStarted(name) + {.push warning[BareExcept]:off.} try: when declared(testSetupIMPLFlag): testSetupIMPL() when declared(testTeardownIMPLFlag): defer: testTeardownIMPL() + {.push warning[BareExcept]:on.} body + {.pop.} except: let e = getCurrentException() @@ -569,16 +576,17 @@ template test*(name, body) {.dirty.} = ) testEnded(testResult) checkpoints = @[] + {.pop.} proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all ## checkpoints encountered so far are printed out. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## check((42, "the Answer to life and everything") == (1, "a")) - ## checkpoint("Checkpoint B") + ## ```nim + ## checkpoint("Checkpoint A") + ## check((42, "the Answer to life and everything") == (1, "a")) + ## checkpoint("Checkpoint B") + ## ``` ## ## outputs "Checkpoint A" once it fails. checkpoints.add(msg) @@ -590,11 +598,11 @@ template fail* = ## failed (change exit code and test status). This template is useful ## for debugging, but is otherwise mostly used internally. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## complicatedProcInThread() - ## fail() + ## ```nim + ## checkpoint("Checkpoint A") + ## complicatedProcInThread() + ## fail() + ## ``` ## ## outputs "Checkpoint A" before quitting. bind ensureInitialized, setProgramResult @@ -622,11 +630,10 @@ template skip* = ## for reasons depending on outer environment, ## or certain application logic conditions or configurations. ## The test code is still executed. - ## - ## .. code-block:: nim - ## - ## if not isGLContextCreated(): - ## skip() + ## ```nim + ## if not isGLContextCreated(): + ## skip() + ## ``` bind checkpoints testStatusIMPL = TestStatus.SKIPPED @@ -704,7 +711,9 @@ macro check*(conditions: untyped): untyped = result = quote do: block: `assigns` - if not `check`: + if `check`: + discard + else: checkpoint(`lineinfo` & ": Check failed: " & `callLit`) `printOuts` fail() @@ -720,7 +729,9 @@ macro check*(conditions: untyped): untyped = let callLit = checked.toStrLit result = quote do: - if not `checked`: + if `checked`: + discard + else: checkpoint(`lineinfo` & ": Check failed: " & `callLit`) fail() @@ -747,20 +758,25 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped = of 2: discard parseInt("Hello World!") of 3: raise newException(IOError, "I can't do that Dave.") else: assert 2 + 2 == 5 - + expect IOError, OSError, ValueError, AssertionDefect: defectiveRobot() template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} = + {.push warning[BareExcept]:off.} try: + {.push warning[BareExcept]:on.} body + {.pop.} checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.") fail() except errorTypes: discard except: - checkpoint(lineInfoLit & ": Expect Failed, unexpected exception was thrown.") + let err = getCurrentException() + checkpoint(lineInfoLit & ": Expect Failed, " & $err.name & " was thrown.") fail() + {.pop.} var errorTypes = newNimNode(nnkBracket) for exp in exceptions: diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 920529ecff..725d5bbd95 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -14,34 +14,36 @@ ## as a locator, a name, or both. The term "Uniform Resource Locator" ## (URL) refers to the subset of URIs. ## +## .. warning:: URI parsers in this module do not perform security validation. +## ## # Basic usage ## ## Combine URIs runnableExamples: let host = parseUri("https://nim-lang.org") - let blog = "/blog.html" - let bloguri = host / blog assert $host == "https://nim-lang.org" - assert $bloguri == "https://nim-lang.org/blog.html" + assert $(host / "/blog.html") == "https://nim-lang.org/blog.html" + assert $(host / "blog2.html") == "https://nim-lang.org/blog2.html" ## ## Access URI item runnableExamples: let res = parseUri("sftp://127.0.0.1:4343") - if isAbsolute(res): - assert res.port == "4343" - else: - echo "Wrong format" + assert isAbsolute(res) + assert res.port == "4343" ## ## Data URI Base64 runnableExamples: - doAssert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" - doAssert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + assert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" + assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" import std/[strutils, parseutils, base64] import std/private/[since, decode_helpers] +when defined(nimPreviewSlimSystem): + import std/assertions + type Url* = distinct string @@ -50,7 +52,7 @@ type scheme*, username*, password*: string hostname*, port*, path*, query*, anchor*: string opaque*: bool - isIpv6: bool # not expose it for compatibility. + isIpv6*: bool UriParseError* = object of ValueError @@ -126,13 +128,13 @@ func decodeUrl*(s: string, decodePlus = true): string = setLen(result, j) func encodeQuery*(query: openArray[(string, string)], usePlus = true, - omitEq = true): string = + omitEq = true, sep = '&'): string = ## Encodes a set of (key, value) parameters into a URL query string. ## ## Every (key, value) pair is URL-encoded and written as `key=value`. If the ## value is an empty string then the `=` is omitted, unless `omitEq` is ## false. - ## The pairs are joined together by a `&` character. + ## The pairs are joined together by the `sep` character. ## ## The `usePlus` parameter is passed down to the `encodeUrl` function that ## is used for the URL encoding of the string values. @@ -143,9 +145,10 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, assert encodeQuery({: }) == "" assert encodeQuery({"a": "1", "b": "2"}) == "a=1&b=2" assert encodeQuery({"a": "1", "b": ""}) == "a=1&b" + assert encodeQuery({"a": "1", "b": ""}, omitEq = false, sep = ';') == "a=1;b=" for elem in query: - # Encode the `key = value` pairs and separate them with a '&' - if result.len > 0: result.add('&') + # Encode the `key = value` pairs and separate them with 'sep' + if result.len > 0: result.add(sep) let (key, val) = elem result.add(encodeUrl(key, usePlus)) # Omit the '=' if the value string is empty @@ -153,15 +156,16 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, result.add('=') result.add(encodeUrl(val, usePlus)) -iterator decodeQuery*(data: string): tuple[key, value: string] = - ## Reads and decodes query string `data` and yields the `(key, value)` pairs - ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, an - ## error is raised when there is an unencoded `=` character in a decoded - ## value, which was the behavior in Nim < 1.5.1 +iterator decodeQuery*(data: string, sep = '&'): tuple[key, value: string] = + ## Reads and decodes the query string `data` and yields the `(key, value)` pairs + ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, + ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded + ## value, which was the behavior in Nim < 1.5.1. runnableExamples: import std/sequtils - doAssert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] - doAssert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] + assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("foo=1;bar=2=3", ';')) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] proc parseData(data: string, i: int, field: var string, sep: char): int = result = i @@ -189,7 +193,7 @@ iterator decodeQuery*(data: string): tuple[key, value: string] = when defined(nimLegacyParseQueryStrict): i = parseData(data, i, value, '=') else: - i = parseData(data, i, value, '&') + i = parseData(data, i, value, sep) yield (name, value) if i < data.len: when defined(nimLegacyParseQueryStrict): @@ -227,7 +231,6 @@ func parseAuthority(authority: string, result: var Uri) = i.inc func parsePath(uri: string, i: var int, result: var Uri) = - i.inc parseUntil(uri, result.path, {'?', '#'}, i) # The 'mailto' scheme's PATH actually contains the hostname/username @@ -243,19 +246,7 @@ func parsePath(uri: string, i: var int, result: var Uri) = i.inc # Skip '#' i.inc parseUntil(uri, result.anchor, {}, i) -func initUri*(): Uri = - ## Initializes a URI with `scheme`, `username`, `password`, - ## `hostname`, `port`, `path`, `query` and `anchor`. - ## - ## **See also:** - ## * `Uri type <#Uri>`_ for available fields in the URI type - runnableExamples: - var uri2: Uri - assert initUri() == uri2 - result = Uri(scheme: "", username: "", password: "", hostname: "", port: "", - path: "", query: "", anchor: "") - -func initUri*(isIpv6: bool): Uri {.since: (1, 3, 5).} = +func initUri*(isIpv6 = false): Uri = ## Initializes a URI with `scheme`, `username`, `password`, ## `hostname`, `port`, `path`, `query`, `anchor` and `isIpv6`. ## @@ -294,7 +285,7 @@ func parseUri*(uri: string, result: var Uri) = var i = 0 # Check if this is a reference URI (relative URI) - let doubleSlash = uri.len > 1 and uri[1] == '/' + let doubleSlash = uri.len > 1 and uri[0] == '/' and uri[1] == '/' if i < uri.len and uri[i] == '/': # Make sure `uri` doesn't begin with '//'. if not doubleSlash: @@ -455,10 +446,8 @@ func combine*(uris: varargs[Uri]): Uri = func isAbsolute*(uri: Uri): bool = ## Returns true if URI is absolute, false otherwise. runnableExamples: - let foo = parseUri("https://nim-lang.org") - assert isAbsolute(foo) == true - let bar = parseUri("nim-lang") - assert isAbsolute(bar) == false + assert parseUri("https://nim-lang.org").isAbsolute + assert not parseUri("nim-lang").isAbsolute return uri.scheme != "" and (uri.hostname != "" or uri.path != "") func `/`*(x: Uri, path: string): Uri = @@ -506,44 +495,62 @@ func `?`*(u: Uri, query: openArray[(string, string)]): Uri = func `$`*(u: Uri): string = ## Returns the string representation of the specified URI object. runnableExamples: - let foo = parseUri("https://nim-lang.org") - assert $foo == "https://nim-lang.org" - result = "" - if u.scheme.len > 0: - result.add(u.scheme) - if u.opaque: - result.add(":") - else: - result.add("://") - if u.username.len > 0: - result.add(u.username) - if u.password.len > 0: - result.add(":") - result.add(u.password) - result.add("@") + assert $parseUri("https://nim-lang.org") == "https://nim-lang.org" + # Get the len of all the parts. + let schemeLen = u.scheme.len + let usernameLen = u.username.len + let passwordLen = u.password.len + let hostnameLen = u.hostname.len + let portLen = u.port.len + let pathLen = u.path.len + let queryLen = u.query.len + let anchorLen = u.anchor.len + # Prepare a string that fits all the parts and all punctuation chars. + # 12 is the max len required by all possible punctuation chars. + result = newStringOfCap( + schemeLen + usernameLen + passwordLen + hostnameLen + portLen + pathLen + queryLen + anchorLen + 12 + ) + # Insert to result. + if schemeLen > 0: + result.add u.scheme + result.add ':' + if not u.opaque: + result.add '/' + result.add '/' + if usernameLen > 0: + result.add u.username + if passwordLen > 0: + result.add ':' + result.add u.password + result.add '@' if u.hostname.endsWith('/'): if u.isIpv6: - result.add("[" & u.hostname[0 .. ^2] & "]") + result.add '[' + result.add u.hostname[0 .. ^2] + result.add ']' else: - result.add(u.hostname[0 .. ^2]) + result.add u.hostname[0 .. ^2] else: if u.isIpv6: - result.add("[" & u.hostname & "]") + result.add '[' + result.add u.hostname + result.add ']' else: - result.add(u.hostname) - if u.port.len > 0: - result.add(":") - result.add(u.port) - if u.path.len > 0: - if u.hostname.len > 0 and u.path[0] != '/': - result.add('/') - result.add(u.path) - if u.query.len > 0: - result.add("?") - result.add(u.query) - if u.anchor.len > 0: - result.add("#") - result.add(u.anchor) + result.add u.hostname + if portLen > 0: + result.add ':' + result.add u.port + if pathLen > 0: + if hostnameLen > 0 and u.path[0] != '/': + result.add '/' + result.add u.path + if queryLen > 0: + result.add '?' + result.add u.query + if anchorLen > 0: + result.add '#' + result.add u.anchor + proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3).} = ## Convenience proc for `base64.encode` returns a standard Base64 Data URI (RFC-2397) @@ -552,28 +559,14 @@ proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3) ## * `mimetypes <mimetypes.html>`_ for `mime` argument ## * https://tools.ietf.org/html/rfc2397 ## * https://en.wikipedia.org/wiki/Data_URI_scheme - runnableExamples: static: doAssert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + runnableExamples: static: assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" assert encoding.len > 0 and mime.len > 0 # Must *not* be URL-Safe, see RFC-2397 - result = "data:" & mime & ";charset=" & encoding & ";base64," & base64.encode(data) - -when isMainModule and defined(testing): - # needed (pending https://github.com/nim-lang/Nim/pull/11865) because - # `removeDotSegments` is private, the other tests are in `turi`. - block: # removeDotSegments - # `removeDotSegments` is exported for -d:testing only - doAssert removeDotSegments("/foo/bar/baz") == "/foo/bar/baz" - doAssert removeDotSegments("") == "" # empty test - doAssert removeDotSegments(".") == "." # trailing period - doAssert removeDotSegments("a1/a2/../a3/a4/a5/./a6/a7/././") == "a1/a3/a4/a5/a6/a7/" - doAssert removeDotSegments("https://a1/a2/../a3/a4/a5/./a6/a7/././") == "https://a1/a3/a4/a5/a6/a7/" - doAssert removeDotSegments("http://a1/a2") == "http://a1/a2" - doAssert removeDotSegments("http://www.ai.") == "http://www.ai." - when false: # xxx these cases are buggy - # this should work, refs https://webmasters.stackexchange.com/questions/73934/how-can-urls-have-a-dot-at-the-end-e-g-www-bla-de - doAssert removeDotSegments("http://www.ai./") == "http://www.ai./" # fails - echo removeDotSegments("http://www.ai./") # http://www.ai/ - echo removeDotSegments("a/b.../c") # b.c - echo removeDotSegments("a/b../c") # bc - echo removeDotSegments("a/.../c") # .c - echo removeDotSegments("a//../b") # a/b - echo removeDotSegments("a/b/c//") # a/b/c// + let base64encoded: string = base64.encode(data) + # ("data:".len + ";charset=".len + ";base64,".len) == 22 + result = newStringOfCap(22 + mime.len + encoding.len + base64encoded.len) + result.add "data:" + result.add mime + result.add ";charset=" + result.add encoding + result.add ";base64," + result.add base64encoded diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 3d9c288ede..2c1e4e37ca 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -9,7 +9,10 @@ ## This module parses an XML document and creates its XML tree representation. -import streams, parsexml, strtabs, xmltree +import std/[streams, parsexml, strtabs, xmltree] + +when defined(nimPreviewSlimSystem): + import std/syncio type XmlError* = object of ValueError ## Exception that is raised @@ -148,7 +151,7 @@ proc loadXml*(path: string, options: set[XmlParseOption] = {reportComments}): Xm when isMainModule: when not defined(testing): - import os + import std/os var errors: seq[string] = @[] var x = loadXml(paramStr(1), errors) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 62a62eb4b7..5c0cbc5e43 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -31,7 +31,11 @@ runnableExamples: ## * `htmlgen module <htmlgen.html>`_ for html code generator import std/private/since -import macros, strtabs, strutils +import std/[macros, strtabs, strutils, sequtils] + +when defined(nimPreviewSlimSystem): + import std/assertions + type XmlNode* = ref XmlNodeObj ## An XML tree consisting of XML nodes. @@ -66,11 +70,19 @@ const xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" ## Header to use for complete XML output. +template expect(node: XmlNode, kind: set[XmlNodeKind]) = + ## Check the node's kind is within a set of values + assert node.k in kind, "Got " & $node.k + +template expect(node: XmlNode, kind: XmlNodeKind) = + ## Check the node's kind equals a value + assert node.k == kind, "Got " & $node.k + proc newXmlNode(kind: XmlNodeKind): XmlNode = ## Creates a new ``XmlNode``. result = XmlNode(k: kind) -proc newElement*(tag: string): XmlNode = +proc newElement*(tag: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnElement`` with the given `tag`. ## ## See also: @@ -89,7 +101,7 @@ proc newElement*(tag: string): XmlNode = result.s = @[] # init attributes lazily to save memory -proc newText*(text: string): XmlNode = +proc newText*(text: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnText`` with the text `text`. runnableExamples: var b = newText("my text") @@ -99,13 +111,13 @@ proc newText*(text: string): XmlNode = result = newXmlNode(xnText) result.fText = text -proc newVerbatimText*(text: string): XmlNode {.since: (1, 3).} = +proc newVerbatimText*(text: sink string): XmlNode {.since: (1, 3).} = ## Creates a new ``XmlNode`` of kind ``xnVerbatimText`` with the text `text`. ## **Since**: Version 1.3. result = newXmlNode(xnVerbatimText) result.fText = text -proc newComment*(comment: string): XmlNode = +proc newComment*(comment: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnComment`` with the text `comment`. runnableExamples: var c = newComment("my comment") @@ -115,7 +127,7 @@ proc newComment*(comment: string): XmlNode = result = newXmlNode(xnComment) result.fText = comment -proc newCData*(cdata: string): XmlNode = +proc newCData*(cdata: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnCData`` with the text `cdata`. runnableExamples: var d = newCData("my cdata") @@ -135,7 +147,7 @@ proc newEntity*(entity: string): XmlNode = result = newXmlNode(xnEntity) result.fText = entity -proc newXmlTree*(tag: string, children: openArray[XmlNode], +proc newXmlTree*(tag: sink string, children: openArray[XmlNode], attributes: XmlAttributes = nil): XmlNode = ## Creates a new XML tree with `tag`, `children` and `attributes`. ## @@ -151,7 +163,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], h.add newEntity("some entity") let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes let k = newXmlTree("treeTag", [g, h], att) - + doAssert $k == """<treeTag key1="first value" key2="second value"> <myTag>some text<!-- this is comment --></myTag> <secondTag>&some entity;</secondTag> @@ -163,7 +175,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], for i in 0..children.len-1: result.s[i] = children[i] result.fAttr = attributes -proc text*(n: XmlNode): string {.inline.} = +proc text*(n: XmlNode): lent string {.inline.} = ## Gets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -178,10 +190,10 @@ proc text*(n: XmlNode): string {.inline.} = assert $c == "<!-- my comment -->" assert c.text == "my comment" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} result = n.fText -proc `text=`*(n: XmlNode, text: string){.inline.} = +proc `text=`*(n: XmlNode, text: sink string) {.inline.} = ## Sets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -196,10 +208,10 @@ proc `text=`*(n: XmlNode, text: string){.inline.} = e.text = "a new entity text" assert $e == "&a new entity text;" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} n.fText = text -proc tag*(n: XmlNode): string {.inline.} = +proc tag*(n: XmlNode): lent string {.inline.} = ## Gets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -217,10 +229,10 @@ proc tag*(n: XmlNode): string {.inline.} = </firstTag>""" assert a.tag == "firstTag" - assert n.k == xnElement + n.expect xnElement result = n.fTag -proc `tag=`*(n: XmlNode, tag: string) {.inline.} = +proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = ## Sets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -240,7 +252,7 @@ proc `tag=`*(n: XmlNode, tag: string) {.inline.} = <childTag /> </newTag>""" - assert n.k == xnElement + n.expect xnElement n.fTag = tag proc rawText*(n: XmlNode): string {.inline.} = @@ -294,26 +306,60 @@ proc innerText*(n: XmlNode): string = proc add*(father, son: XmlNode) {.inline.} = ## Adds the child `son` to `father`. + ## `father` must be of `xnElement` type ## ## See also: + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newText("my text") f.add newElement("sonTag") f.add newEntity("my entity") assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement add(father.s, son) +proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = + ## Adds the children `sons` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) + assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement + add(father.s, sons) + + proc insert*(father, son: XmlNode, index: int) {.inline.} = ## Inserts the child `son` to a given position in `father`. ## - ## `father` and `son` must be of `xnElement` kind. + ## `father` must be of `xnElement` kind. ## ## See also: + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -323,18 +369,52 @@ proc insert*(father, son: XmlNode, index: int) {.inline.} = <first /> </myTag>""" - assert father.k == xnElement and son.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) +proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = + ## Inserts the children openArray[`sons`] to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + assert $f == """<myTag> + <second /> + <third /> + <first /> +</myTag>""" + + father.expect xnElement + if len(father.s) > index: + insert(father.s, sons, index) + else: + insert(father.s, sons, len(father.s)) + proc delete*(n: XmlNode, i: Natural) = ## Deletes the `i`'th child of `n`. ## ## See also: + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -344,9 +424,88 @@ proc delete*(n: XmlNode, i: Natural) = <first /> </myTag>""" - assert n.k == xnElement + n.expect xnElement n.s.delete(i) +proc delete*(n: XmlNode, slice: Slice[int]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + f.delete(0..1) + assert $f == """<myTag> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + +proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = + ## Replaces the `i`'th child of `n` with `replacement` openArray. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.replace(0, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(i) + n.s.insert(replacement, i) + +proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("fifth")], 0) + f.replace(0..1, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + n.s.insert(replacement, slice.a) + proc len*(n: XmlNode): int {.inline.} = ## Returns the number of `n`'s children. runnableExamples: @@ -374,12 +533,12 @@ proc `[]`*(n: XmlNode, i: int): XmlNode {.inline.} = assert $f[1] == "<first />" assert $f[0] == "<second />" - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc `[]`*(n: var XmlNode, i: int): var XmlNode {.inline.} = ## Returns the `i`'th child of `n` so that it can be modified. - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc clear*(n: var XmlNode) = @@ -389,13 +548,13 @@ proc clear*(n: var XmlNode) = var g = newElement("myTag") g.add newText("some text") g.add newComment("this is comment") - + var h = newElement("secondTag") h.add newEntity("some entity") - + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes var k = newXmlTree("treeTag", [g, h], att) - + doAssert $k == """<treeTag key1="first value" key2="second value"> <myTag>some text<!-- this is comment --></myTag> <secondTag>&some entity;</secondTag> @@ -431,12 +590,12 @@ iterator items*(n: XmlNode): XmlNode {.inline.} = # <!-- this is comment --> # <secondTag>&some entity;<![CDATA[some cdata]]></secondTag> - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = ## Iterates over all direct children of `n` so that they can be modified. - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, @@ -447,7 +606,7 @@ proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes var j = newElement("myTag") j.attrs = att - + doAssert $j == """<myTag key1="first value" key2="second value" />""" newStringTable(keyValuePairs) @@ -468,7 +627,7 @@ proc attrs*(n: XmlNode): XmlAttributes {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = @@ -485,7 +644,7 @@ proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = @@ -502,7 +661,7 @@ proc attrsLen*(n: XmlNode): int {.inline.} = j.attrs = att assert j.attrsLen == 2 - assert n.k == xnElement + n.expect xnElement if not isNil(n.fAttr): result = len(n.fAttr) proc attr*(n: XmlNode, name: string): string = @@ -520,7 +679,7 @@ proc attr*(n: XmlNode, name: string): string = assert j.attr("key1") == "first value" assert j.attr("key2") == "second value" - assert n.kind == xnElement + n.expect xnElement if n.attrs == nil: return "" return n.attrs.getOrDefault(name) @@ -552,15 +711,15 @@ proc escape*(s: string): string = ## ## Escapes these characters: ## - ## ------------ ------------------- + ## ============ =================== ## char is converted to - ## ------------ ------------------- + ## ============ =================== ## ``<`` ``<`` ## ``>`` ``>`` ## ``&`` ``&`` ## ``"`` ``"`` ## ``'`` ``'`` - ## ------------ ------------------- + ## ============ =================== ## ## You can also use `addEscaped proc <#addEscaped,string,string>`_. result = newStringOfCap(s.len) @@ -572,23 +731,11 @@ proc addIndent(result: var string, indent: int, addNewLines: bool) = for i in 1 .. indent: result.add(' ') -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, - addNewLines = true) = - ## Adds the textual representation of `n` to string `result`. - runnableExamples: - var - a = newElement("firstTag") - b = newText("my text") - c = newComment("my comment") - s = "" - s.add(c) - s.add(a) - s.add(b) - assert s == "<!-- my comment --><firstTag />my text" - +proc addImpl(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true, lastNodeIsText = false) = proc noWhitespace(n: XmlNode): bool = for i in 0 ..< n.len: - if n[i].kind in {xnText, xnEntity}: return true + if n[i].kind in {xnText, xnVerbatimText, xnEntity}: return true proc addEscapedAttr(result: var string, s: string) = # `addEscaped` alternative with less escaped characters. @@ -605,7 +752,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, case n.k of xnElement: - if indent > 0: + if indent > 0 and not lastNodeIsText: result.addIndent(indent, addNewLines) let @@ -634,8 +781,10 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, else: indent+indWidth result.add('>') + var lastNodeIsText = false for i in 0 ..< n.len: - result.add(n[i], indentNext, indWidth, addNewLines) + result.addImpl(n[i], indentNext, indWidth, addNewLines, lastNodeIsText) + lastNodeIsText = (n[i].kind == xnText) or (n[i].kind == xnVerbatimText) if not n.noWhitespace(): result.addIndent(indent, addNewLines) @@ -660,6 +809,21 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add(n.fText) result.add(';') +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true) {.inline.} = + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "<!-- my comment --><firstTag />my text" + result.addImpl(n, indent, indWidth, addNewLines) + proc `$`*(n: XmlNode): string = ## Converts `n` into its string representation. ## @@ -678,7 +842,7 @@ proc child*(n: XmlNode, name: string): XmlNode = f.add newElement("thirdSon") assert $(f.child("secondSon")) == "<secondSon />" - assert n.kind == xnElement + n.expect xnElement for i in items(n): if i.kind == xnElement: if i.tag == name: @@ -714,7 +878,7 @@ proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode], a.findAll("BAD", s, caseInsensitive = true) assert $s == "@[<bad>c text</bad>, <BAD>d text</BAD>]" - assert n.k == xnElement + n.expect xnElement for child in n.items(): if child.k != xnElement: continue @@ -744,7 +908,7 @@ proc findAll*(n: XmlNode, tag: string, caseInsensitive = false): seq[XmlNode] = newSeq(result, 0) findAll(n, tag, result, caseInsensitive) -proc xmlConstructor(a: NimNode): NimNode {.compileTime.} = +proc xmlConstructor(a: NimNode): NimNode = if a.kind == nnkCall: result = newCall("newXmlTree", toStrLit(a[0])) var attrs = newNimNode(nnkBracket, a) @@ -771,11 +935,12 @@ proc xmlConstructor(a: NimNode): NimNode {.compileTime.} = macro `<>`*(x: untyped): untyped = ## Constructor macro for XML. Example usage: ## - ## .. code-block:: nim + ## ```nim ## <>a(href="http://nim-lang.org", newText("Nim rules.")) + ## ``` ## - ## Produces an XML tree for:: + ## Produces an XML tree for: ## - ## <a href="http://nim-lang.org">Nim rules.</a> + ## <a href="http://nim-lang.org">Nim rules.</a> ## result = xmlConstructor(x) diff --git a/lib/std/appdirs.nim b/lib/std/appdirs.nim new file mode 100644 index 0000000000..963451efe2 --- /dev/null +++ b/lib/std/appdirs.nim @@ -0,0 +1,94 @@ +## This module implements helpers for determining special directories used by apps. + +## .. importdoc:: paths.nim + +from std/private/osappdirs import nil +import std/paths +import std/envvars + +proc getHomeDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getHomeDir()) + +proc getDataDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + result = Path(osappdirs.getDataDir()) + +proc getConfigDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getConfigDir()) + +proc getCacheDir*(): Path {.inline.} = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + # follows https://crates.io/crates/platform-dirs + result = Path(osappdirs.getCacheDir()) + +proc getCacheDir*(app: Path): Path {.inline.} = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + result = Path(osappdirs.getCacheDir(app.string)) + +proc getTempDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## .. Note:: This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + result = Path(osappdirs.getTempDir()) diff --git a/lib/system/assertions.nim b/lib/std/assertions.nim similarity index 63% rename from lib/system/assertions.nim rename to lib/std/assertions.nim index cf705dd6ed..56c37d2057 100644 --- a/lib/system/assertions.nim +++ b/lib/std/assertions.nim @@ -1,47 +1,24 @@ -## This module provides various assertion utilities. -## -## **Note:** This module is reexported by `system` and thus does not need to be -## imported directly (with `system/assertions`). +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# -runnableExamples: - # assert - assert 1 == 1 - - # onFailedAssert - block: - type MyError = object of CatchableError - lineinfo: tuple[filename: string, line: int, column: int] - - # block-wide policy to change the failed assert - # exception type in order to include a lineinfo - onFailedAssert(msg): - var e = new(MyError) - e.msg = msg - e.lineinfo = instantiationInfo(-2) - raise e - - # doAssert - doAssert 1 == 1 # generates code even when built with `-d:danger` or `--assertions:off` - - # doAssertRaises - doAssertRaises(ValueError): - raise newException(ValueError, "Hello World") - -runnableExamples("--assertions:off"): - assert 1 == 2 # no code generated - -# xxx pending bug #16993: move runnableExamples to respective templates - -when not declared(sysFatal): +when not defined(nimPreviewSlimSystem) and not declared(sysFatal): + include "system/rawquits" include "system/fatal" +## This module implements assertion handling. + import std/private/miscdollars # --------------------------------------------------------------------------- # helpers type InstantiationInfo = tuple[filename: string, line: int, column: int] -proc `$`(x: int): string {.magic: "IntToStr", noSideEffect.} proc `$`(info: InstantiationInfo): string = # The +1 is needed here # instead of overriding `$` (and changing its meaning), consider explicit name. @@ -50,22 +27,18 @@ proc `$`(info: InstantiationInfo): string = # --------------------------------------------------------------------------- -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = ## Raises an `AssertionDefect` with `msg`. - sysFatal(AssertionDefect, msg) + when defined(nimPreviewSlimSystem): + raise newException(AssertionDefect, msg) + else: + sysFatal(AssertionDefect, msg) proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = ## Raises an `AssertionDefect` with `msg`, but this is hidden ## from the effect system. Called when an assertion failed. - # trick the compiler to not list `AssertionDefect` when called - # by `assert`. - # xxx simplify this pending bootstrap >= 1.4.0, after which cast not needed - # anymore since `Defect` can't be raised. - type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, tags: [].} - cast[Hide](raiseAssert)(msg) + raiseAssert(msg) template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = when enabled: @@ -86,25 +59,46 @@ template assert*(cond: untyped, msg = "") = ## ## No code will be generated for `assert` when passing `-d:danger` (implied by `--assertions:off`). ## See `command line switches <nimc.html#compiler-usage-commandminusline-switches>`_. + runnableExamples: assert 1 == 1 + runnableExamples("--assertions:off"): + assert 1 == 2 # no code generated, no failure here + runnableExamples("-d:danger"): assert 1 == 2 # ditto assertImpl(cond, msg, astToStr(cond), compileOption("assertions")) template doAssert*(cond: untyped, msg = "") = ## Similar to `assert <#assert.t,untyped,string>`_ but is always turned on regardless of `--assertions`. + runnableExamples: + doAssert 1 == 1 # generates code even when built with `-d:danger` or `--assertions:off` assertImpl(cond, msg, astToStr(cond), true) template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = ## Sets an assertion failure handler that will intercept any assert ## statements following `onFailedAssert` in the current scope. - template failedAssertImpl(msgIMPL: string): untyped {.dirty.} = + runnableExamples: + type MyError = object of CatchableError + lineinfo: tuple[filename: string, line: int, column: int] + # block-wide policy to change the failed assert exception type in order to + # include a lineinfo + onFailedAssert(msg): + raise (ref MyError)(msg: msg, lineinfo: instantiationInfo(-2)) + doAssertRaises(MyError): doAssert false + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template failedAssertImpl(msgIMPL: string): untyped {.dirty, redefine.} = let msg = msgIMPL code template doAssertRaises*(exception: typedesc, code: untyped) = ## Raises `AssertionDefect` if specified `code` does not raise `exception`. + runnableExamples: + doAssertRaises(ValueError): raise newException(ValueError, "Hello World") + doAssertRaises(CatchableError): raise newException(ValueError, "Hello World") + doAssertRaises(AssertionDefect): doAssert false var wrong = false const begin = "expected raising '" & astToStr(exception) & "', instead" const msgEnd = " by: " & astToStr(code) - template raisedForeign = raiseAssert(begin & " raised foreign exception" & msgEnd) + template raisedForeign {.gensym.} = raiseAssert(begin & " raised foreign exception" & msgEnd) + {.push warning[BareExcept]:off.} when Exception is exception: try: if true: @@ -119,7 +113,10 @@ template doAssertRaises*(exception: typedesc, code: untyped) = wrong = true except exception: discard - except Exception as e: raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) + except Exception as e: + mixin `$` # alternatively, we could define $cstring in this module + raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) except: raisedForeign() + {.pop.} if wrong: raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/lib/std/channels.nim b/lib/std/channels.nim deleted file mode 100644 index 3bbc8a6b67..0000000000 --- a/lib/std/channels.nim +++ /dev/null @@ -1,498 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Prell, Mamy André-Ratsimbazafy & Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# Based on https://github.com/mratsim/weave/blob/5696d94e6358711e840f8c0b7c684fcc5cbd4472/unused/channels/channels_legacy.nim -# Those are translations of @aprell (Andreas Prell) original channels from C to Nim -# (https://github.com/aprell/tasking-2.0/blob/master/src/channel_shm/channel.c) -# And in turn they are an implementation of Michael & Scott lock-based queues -# (note the paper has 2 channels: lock-free and lock-based) with additional caching: -# Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue Algorithms -# Maged M. Michael, Michael L. Scott, 1996 -# https://www.cs.rochester.edu/~scott/papers/1996_PODC_queues.pdf - -## This module only works with `--gc:arc` or `--gc:orc`. -## -## .. warning:: This module is experimental and its interface may change. -## -## The following is a simple example of two different ways to use channels: -## blocking and non-blocking. -## - -runnableExamples("--threads:on --gc:orc"): - import std/os - - # In this example a channel is declared at module scope. - # Channels are generic, and they include support for passing objects between - # threads. - # Note that isolated data passed through channels is moved around. - var chan = newChannel[string]() - - # This proc will be run in another thread using the threads module. - proc firstWorker() = - chan.send("Hello World!") - - # This is another proc to run in a background thread. This proc takes a while - # to send the message since it sleeps for 2 seconds (or 2000 milliseconds). - proc secondWorker() = - sleep(2000) - chan.send("Another message") - - # Launch the worker. - var worker1: Thread[void] - createThread(worker1, firstWorker) - - # Block until the message arrives, then print it out. - let dest = chan.recv() - assert dest == "Hello World!" - - # Wait for the thread to exit before moving on to the next example. - worker1.joinThread() - - # Launch the other worker. - var worker2: Thread[void] - createThread(worker2, secondWorker) - # This time, use a non-blocking approach with tryRecv. - # Since the main thread is not blocked, it could be used to perform other - # useful work while it waits for data to arrive on the channel. - var messages: seq[string] - while true: - var msg = "" - if chan.tryRecv(msg): - messages.add msg # "Another message" - break - - messages.add "Pretend I'm doing useful work..." - # For this example, sleep in order not to flood stdout with the above - # message. - sleep(400) - - # Wait for the second thread to exit before cleaning up the channel. - worker2.joinThread() - - # Clean up the channel. - assert chan.close() - - assert messages[^1] == "Another message" - assert messages.len >= 2 - - -when not defined(gcArc) and not defined(gcOrc) and not defined(nimdoc): - {.error: "This channel implementation requires --gc:arc or --gc:orc".} - -import std/[locks, atomics, isolation] -import system/ansi_c - -# Channel (Shared memory channels) -# ---------------------------------------------------------------------------------- - -const - cacheLineSize {.intdefine.} = 64 # TODO: some Samsung phone have 128 cache-line - nimChannelCacheSize* {.intdefine.} = 100 - -type - ChannelRaw = ptr ChannelObj - ChannelObj = object - headLock, tailLock: Lock - notFullCond, notEmptyCond: Cond - closed: Atomic[bool] - size: int - itemsize: int # up to itemsize bytes can be exchanged over this channel - head {.align: cacheLineSize.} : int # Items are taken from head and new items are inserted at tail - tail: int - buffer: ptr UncheckedArray[byte] - atomicCounter: Atomic[int] - - ChannelCache = ptr ChannelCacheObj - ChannelCacheObj = object - next: ChannelCache - chanSize: int - chanN: int - numCached: int - cache: array[nimChannelCacheSize, ChannelRaw] - -# ---------------------------------------------------------------------------------- - -proc numItems(chan: ChannelRaw): int {.inline.} = - result = chan.tail - chan.head - if result < 0: - inc(result, 2 * chan.size) - - assert result <= chan.size - -template isFull(chan: ChannelRaw): bool = - abs(chan.tail - chan.head) == chan.size - -template isEmpty(chan: ChannelRaw): bool = - chan.head == chan.tail - -# Unbuffered / synchronous channels -# ---------------------------------------------------------------------------------- - -template numItemsUnbuf(chan: ChannelRaw): int = - chan.head - -template isFullUnbuf(chan: ChannelRaw): bool = - chan.head == 1 - -template isEmptyUnbuf(chan: ChannelRaw): bool = - chan.head == 0 - -# ChannelRaw kinds -# ---------------------------------------------------------------------------------- - -func isUnbuffered(chan: ChannelRaw): bool = - chan.size - 1 == 0 - -# ChannelRaw status and properties -# ---------------------------------------------------------------------------------- - -proc isClosed(chan: ChannelRaw): bool {.inline.} = load(chan.closed, moRelaxed) - -proc peek(chan: ChannelRaw): int {.inline.} = - (if chan.isUnbuffered: numItemsUnbuf(chan) else: numItems(chan)) - -# Per-thread channel cache -# ---------------------------------------------------------------------------------- - -var channelCache {.threadvar.}: ChannelCache -var channelCacheLen {.threadvar.}: int - -proc allocChannelCache(size, n: int): bool = - ## Allocate a free list for storing channels of a given type - var p = channelCache - - # Avoid multiple free lists for the exact same type of channel - while not p.isNil: - if size == p.chanSize and n == p.chanN: - return false - p = p.next - - p = cast[ptr ChannelCacheObj](c_malloc(csize_t sizeof(ChannelCacheObj))) - if p.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - p.chanSize = size - p.chanN = n - p.numCached = 0 - - p.next = channelCache - channelCache = p - inc channelCacheLen - result = true - -proc freeChannelCache*() = - ## Frees the entire channel cache, including all channels - var p = channelCache - var q: ChannelCache - - while not p.isNil: - q = p.next - for i in 0 ..< p.numCached: - let chan = p.cache[i] - if not chan.buffer.isNil: - c_free(chan.buffer) - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - c_free(chan) - c_free(p) - dec channelCacheLen - p = q - - assert(channelCacheLen == 0) - channelCache = nil - -# Channels memory ops -# ---------------------------------------------------------------------------------- - -proc allocChannel(size, n: int): ChannelRaw = - when nimChannelCacheSize > 0: - var p = channelCache - - while not p.isNil: - if size == p.chanSize and n == p.chanN: - # Check if free list contains channel - if p.numCached > 0: - dec p.numCached - result = p.cache[p.numCached] - assert(result.isEmpty) - return - else: - # All the other lists in cache won't match - break - p = p.next - - result = cast[ChannelRaw](c_malloc(csize_t sizeof(ChannelObj))) - if result.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - # To buffer n items, we allocate for n - result.buffer = cast[ptr UncheckedArray[byte]](c_malloc(csize_t n*size)) - if result.buffer.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - initLock(result.headLock) - initLock(result.tailLock) - initCond(result.notFullCond) - initCond(result.notEmptyCond) - - result.closed.store(false, moRelaxed) # We don't need atomic here, how to? - result.size = n - result.itemsize = size - result.head = 0 - result.tail = 0 - result.atomicCounter.store(0, moRelaxed) - - when nimChannelCacheSize > 0: - # Allocate a cache as well if one of the proper size doesn't exist - discard allocChannelCache(size, n) - -proc freeChannel(chan: ChannelRaw) = - if chan.isNil: - return - - when nimChannelCacheSize > 0: - var p = channelCache - while not p.isNil: - if chan.itemsize == p.chanSize and - chan.size == p.chanN: - if p.numCached < nimChannelCacheSize: - # If space left in cache, cache it - p.cache[p.numCached] = chan - inc p.numCached - return - else: - # All the other lists in cache won't match - break - p = p.next - - if not chan.buffer.isNil: - c_free(chan.buffer) - - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - - c_free(chan) - -# MPMC Channels (Multi-Producer Multi-Consumer) -# ---------------------------------------------------------------------------------- - -proc sendUnbufferedMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isFullUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isFullUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isFullUnbuf: - wait(chan.notFullcond, chan.headLock) - - assert chan.isEmptyUnbuf - assert size <= chan.itemsize - copyMem(chan.buffer, data, size) - - chan.head = 1 - - release(chan.headLock) - signal(chan.notEmptyCond) - result = true - -proc sendMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return sendUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isFull: - return false - - acquire(chan.tailLock) - - if nonBlocking and chan.isFull: - # Another thread was faster - release(chan.tailLock) - return false - - while chan.isFull: - wait(chan.notFullcond, chan.tailLock) - - assert not chan.isFull - assert size <= chan.itemsize - - let writeIdx = if chan.tail < chan.size: chan.tail - else: chan.tail - chan.size - - copyMem(chan.buffer[writeIdx * chan.itemsize].addr, data, size) - - inc chan.tail - if chan.tail == 2 * chan.size: - chan.tail = 0 - - release(chan.tailLock) - signal(chan.notEmptyCond) - result = true - -proc recvUnbufferedMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isEmptyUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmptyUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isEmptyUnbuf: - wait(chan.notEmptyCond, chan.headLock) - - assert chan.isFullUnbuf - assert size <= chan.itemsize - - copyMem(data, chan.buffer, size) - - chan.head = 0 - - release(chan.headLock) - signal(chan.notFullCond) - result = true - -proc recvMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return recvUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isEmpty: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmpty: - # Another thread took the last data - release(chan.headLock) - return false - - while chan.isEmpty: - wait(chan.notEmptyCond, chan.headLock) - - assert not chan.isEmpty - assert size <= chan.itemsize - - let readIdx = if chan.head < chan.size: chan.head - else: chan.head - chan.size - - copyMem(data, chan.buffer[readIdx * chan.itemsize].addr, size) - - inc chan.head - if chan.head == 2 * chan.size: - chan.head = 0 - - release(chan.headLock) - signal(chan.notFullCond) - result = true - -proc channelCloseMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if chan.isClosed: - # ChannelRaw already closed - return false - - store(chan.closed, true, moRelaxed) - result = true - -proc channelOpenMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if not chan.isClosed: - # ChannelRaw already open - return false - - store(chan.closed, false, moRelaxed) - result = true - -# Public API -# ---------------------------------------------------------------------------------- - -type - Channel*[T] = object ## Typed channels - d: ChannelRaw - -proc `=destroy`*[T](c: var Channel[T]) = - if c.d != nil: - if load(c.d.atomicCounter, moAcquire) == 0: - if c.d.buffer != nil: - freeChannel(c.d) - else: - atomicDec(c.d.atomicCounter) - -proc `=`*[T](dest: var Channel[T], src: Channel[T]) = - ## Shares `Channel` by reference counting. - if src.d != nil: - atomicInc(src.d.atomicCounter) - - if dest.d != nil: - `=destroy`(dest) - dest.d = src.d - -func trySend*[T](c: Channel[T], src: var Isolated[T]): bool {.inline.} = - ## Sends item to the channel(non blocking). - var data = src.extract - result = sendMpmc(c.d, data.addr, sizeof(T), true) - if result: - wasMoved(data) - -template trySend*[T](c: Channel[T], src: T): bool = - ## Helper templates for `trySend`. - trySend(c, isolate(src)) - -func tryRecv*[T](c: Channel[T], dst: var T): bool {.inline.} = - ## Receives item from the channel(non blocking). - recvMpmc(c.d, dst.addr, sizeof(T), true) - -func send*[T](c: Channel[T], src: sink Isolated[T]) {.inline.} = - ## Sends item to the channel(blocking). - var data = src.extract - discard sendMpmc(c.d, data.addr, sizeof(T), false) - wasMoved(data) - -template send*[T](c: Channel[T]; src: T) = - ## Helper templates for `send`. - send(c, isolate(src)) - -func recv*[T](c: Channel[T]): T {.inline.} = - ## Receives item from the channel(blocking). - discard recvMpmc(c.d, result.addr, sizeof(result), false) - -func open*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelOpenMpmc() - -func close*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelCloseMpmc() - -func peek*[T](c: Channel[T]): int {.inline.} = peek(c.d) - -proc newChannel*[T](elements = 30): Channel[T] = - ## Returns a new `Channel`. `elements` should be positive. - ## `elements` is used to specify whether a channel is buffered or not. - ## If `elements` = 1, the channel is unbuffered. If `elements` > 1, the - ## channel is buffered. - assert elements >= 1, "Elements must be positive!" - result = Channel[T](d: allocChannel(sizeof(T), elements)) diff --git a/lib/std/cmdline.nim b/lib/std/cmdline.nim new file mode 100644 index 0000000000..0ba4619e53 --- /dev/null +++ b/lib/std/cmdline.nim @@ -0,0 +1,313 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains system facilities for reading command +## line parameters. + +## **See also:** +## * `parseopt module <parseopt.html>`_ for command-line parser beyond +## `parseCmdLine proc`_ + + +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/widestrs + +when defined(nodejs): + from std/private/oscommon import ReadDirEffect + + +const weirdTarget = defined(nimscript) or defined(js) + + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix +else: + {.error: "The cmdline module has not been implemented for the target platform.".} + + +# Needed by windows in order to obtain the command line for targets +# other than command line targets +when defined(windows) and not weirdTarget: + template getCommandLine*(): untyped = getCommandLineW() + + +proc parseCmdLine*(c: string): seq[string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a `command line`:idx: into several components. + ## + ## **Note**: This proc is only occasionally useful, better use the + ## `parseopt module <parseopt.html>`_. + ## + ## On Windows, it uses the `following parsing rules + ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: + ## + ## * Arguments are delimited by white space, which is either a space or a tab. + ## * The caret character (^) is not recognized as an escape character or + ## delimiter. The character is handled completely by the command-line parser + ## in the operating system before being passed to the argv array in the + ## program. + ## * A string surrounded by double quotation marks ("string") is interpreted + ## as a single argument, regardless of white space contained within. A + ## quoted string can be embedded in an argument. + ## * A double quotation mark preceded by a backslash (\") is interpreted as a + ## literal double quotation mark character ("). + ## * Backslashes are interpreted literally, unless they immediately precede + ## a double quotation mark. + ## * If an even number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is interpreted as a string delimiter. + ## * If an odd number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is "escaped" by the remaining backslash, + ## causing a literal double quotation mark (") to be placed in argv. + ## + ## On Posix systems, it uses the following parsing rules: + ## Components are separated by whitespace unless the whitespace + ## occurs within ``"`` or ``'`` quotes. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + + result = @[] + var i = 0 + var a = "" + while true: + setLen(a, 0) + # eat all delimiting whitespace + while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) + if i >= c.len: break + when defined(windows): + # parse a single argument according to the above rules: + var inQuote = false + while i < c.len: + case c[i] + of '\\': + var j = i + while j < c.len and c[j] == '\\': inc(j) + if j < c.len and c[j] == '"': + for k in 1..(j-i) div 2: a.add('\\') + if (j-i) mod 2 == 0: + i = j + else: + a.add('"') + i = j+1 + else: + a.add(c[i]) + inc(i) + of '"': + inc(i) + if not inQuote: inQuote = true + elif i < c.len and c[i] == '"': + a.add(c[i]) + inc(i) + else: + inQuote = false + break + of ' ', '\t': + if not inQuote: break + a.add(c[i]) + inc(i) + else: + a.add(c[i]) + inc(i) + else: + case c[i] + of '\'', '\"': + var delim = c[i] + inc(i) # skip ' or " + while i < c.len and c[i] != delim: + add a, c[i] + inc(i) + if i < c.len: inc(i) + else: + while i < c.len and c[i] > ' ': + add(a, c[i]) + inc(i) + add(result, move a) + +when defined(nimdoc): + # Common forward declaration docstring block for parameter retrieval procs. + proc paramCount*(): int {.tags: [ReadIOEffect].} = + ## Returns the number of `command line arguments`:idx: given to the + ## application. + ## + ## Unlike `argc`:idx: in C, if your binary was called without parameters this + ## will return zero. + ## You can query each individual parameter with `paramStr proc`_ + ## or retrieve all of them in one go with `commandLineParams proc`_. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramCount): + ## # Use paramCount() here + ## else: + ## # Do something else! + ## ``` + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + ## Returns the `i`-th `command line argument`:idx: given to the application. + ## + ## `i` should be in the range `1..paramCount()`, the `IndexDefect` + ## exception will be raised for invalid values. Instead of iterating + ## over `paramCount()`_ with this proc you can + ## call the convenience `commandLineParams()`_. + ## + ## Similarly to `argv`:idx: in C, + ## it is possible to call `paramStr(0)` but this will return OS specific + ## contents (usually the name of the invoked executable). You should avoid + ## this and call `getAppFilename() <os.html#getAppFilename>`_ instead. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `commandLineParams proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramStr): + ## # Use paramStr() here + ## else: + ## # Do something else! + ## ``` + +elif defined(nimscript): discard +elif defined(nodejs): + type Argv = object of JsRoot + let argv {.importjs: "process.argv".} : Argv + proc len(argv: Argv): int {.importjs: "#.length".} + proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} + + proc paramCount*(): int {.tags: [ReadDirEffect].} = + result = argv.len - 2 + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + let i = i + 1 + if i < argv.len and i >= 0: + result = $argv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) +elif defined(windows): + # Since we support GUI applications with Nim, we sometimes generate + # a WinMain entry proc. But a WinMain proc has no access to the parsed + # command line arguments. The way to get them differs. Thus we parse them + # ourselves. This has the additional benefit that the program's behaviour + # is always the same -- independent of the used C compiler. + var + ownArgv {.threadvar.}: seq[string] + ownParsedArgv {.threadvar.}: bool + + proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + result = ownArgv.len-1 + + proc paramStr*(i: int): string {.rtl, extern: "nos$1", + tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + if i < ownArgv.len and i >= 0: + result = ownArgv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) + +elif defined(genode): + proc paramStr*(i: int): string = + raise newException(OSError, "paramStr is not implemented on Genode") + + proc paramCount*(): int = + raise newException(OSError, "paramCount is not implemented on Genode") +elif weirdTarget or (defined(posix) and appType == "lib"): + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramStr is not implemented on current platform") + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramCount is not implemented on current platform") +elif not defined(createNimRtl) and + not(defined(posix) and appType == "lib"): + # On Posix, there is no portable way to get the command line from a DLL. + var + cmdCount {.importc: "cmdCount".}: cint + cmdLine {.importc: "cmdLine".}: cstringArray + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if i < cmdCount and i >= 0: + result = $cmdLine[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + result = cmdCount-1 + +when declared(paramCount) or defined(nimdoc): + proc commandLineParams*(): seq[string] = + ## Convenience proc which returns the command line parameters. + ## + ## This returns **only** the parameters. If you want to get the application + ## executable filename, call `getAppFilename() <os.html#getAppFilename>`_. + ## + ## **Availability**: On Posix there is no portable way to get the command + ## line from a DLL and thus the proc isn't defined in this environment. You + ## can test for its availability with `declared() + ## <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(commandLineParams): + ## # Use commandLineParams() here + ## else: + ## # Do something else! + ## ``` + result = @[] + for i in 1..paramCount(): + result.add(paramStr(i)) +else: + proc commandLineParams*(): seq[string] {.error: + "commandLineParams() unsupported by dynamic libraries".} = + discard diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index 725b68acd2..6d8bd22f49 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this @@ -8,7 +8,7 @@ # ## This module allows querying the compiler about -## diverse configuration settings. +## diverse configuration settings. See also `compileOption`. # Note: Only add new enum values at the end to ensure binary compatibility with # other Nim compiler versions! @@ -32,6 +32,8 @@ type backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` ## and `nim js` would imply backend=js libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 + gc {.deprecated.} ## gc selected + mm ## memory management selected MultipleValueSetting* {.pure.} = enum ## \ ## settings resulting in a seq of string values diff --git a/lib/std/decls.nim b/lib/std/decls.nim index dd7d19da7d..bb7ec35931 100644 --- a/lib/std/decls.nim +++ b/lib/std/decls.nim @@ -1,19 +1,31 @@ -# see `semLowerLetVarCustomPragma` for compiler support that enables these -# lowerings +## This module implements syntax sugar for some declarations. -template byaddr*(lhs, typ, ex) = - ## Allows a syntax for lvalue reference, exact analog to - ## `auto& a = ex;` in C++ +import std/macros + +macro byaddr*(sect) = + ## Allows a syntax for l-value references, being an exact analog to + ## `auto& a = ex;` in C++. + ## + ## .. warning:: This makes use of 2 experimental features, namely nullary + ## templates instantiated as symbols and variable macro pragmas. + ## For this reason, its behavior is not stable. The current implementation + ## allows redefinition, but this is not an intended consequence. runnableExamples: - var s = @[10,11,12] + var s = @[10, 11, 12] var a {.byaddr.} = s[0] - a+=100 - doAssert s == @[110,11,12] - doAssert a is int + a += 100 + assert s == @[110, 11, 12] + assert a is int var b {.byaddr.}: int = s[0] - doAssert a.addr == b.addr - when typ is typeof(nil): - let tmp = addr(ex) - else: - let tmp: ptr typ = addr(ex) - template lhs: untyped = tmp[] + assert a.addr == b.addr + expectLen sect, 1 + let def = sect[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = addr(`ex`) + template `lhs`: untyped = tmp[] + result.copyLineInfo(def) diff --git a/lib/std/dirs.nim b/lib/std/dirs.nim new file mode 100644 index 0000000000..380d6d08f8 --- /dev/null +++ b/lib/std/dirs.nim @@ -0,0 +1,135 @@ +## This module implements directory handling. + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osdirs import dirExists, createDir, existsOrCreateDir, removeDir, + moveDir, walkDir, setCurrentDir, + walkDirRec, PathComponent + +export PathComponent + +proc dirExists*(dir: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + result = dirExists(dir.string) + +proc createDir*(dir: Path) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `moveDir proc`_ + createDir(dir.string) + +proc existsOrCreateDir*(dir: Path): bool {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + result = existsOrCreateDir(dir.string) + +proc removeDir*(dir: Path, checkDir = false + ) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `removeFile proc <files.html#removeFile>`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + removeDir(dir.string, checkDir) + +proc moveDir*(source, dest: Path) {.inline, tags: [ReadIOEffect, WriteIOEffect].} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc <files.html#moveFile>`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + moveDir(source.string, dest.string) + +iterator walkDir*(dir: Path; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: Path] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + for (k, p) in walkDir(dir.string, relative, checkDir, skipSpecial): + yield (k, Path(p)) + +iterator walkDirRec*(dir: Path, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + Path {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkDir iterator`_ + for p in walkDirRec(dir.string, yieldFilter, followFilter, relative, + checkDir, skipSpecial): + yield Path(p) + +proc setCurrentDir*(newDir: Path) {.inline, tags: [].} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getCurrentDir proc <paths.html#getCurrentDir>`_ + osdirs.setCurrentDir(newDir.string) diff --git a/lib/std/editdistance.nim b/lib/std/editdistance.nim index 9f29c5c05f..40c0017ae0 100644 --- a/lib/std/editdistance.nim +++ b/lib/std/editdistance.nim @@ -10,7 +10,7 @@ ## This module implements an algorithm to compute the ## `edit distance`:idx: between two Unicode strings. -import unicode +import std/unicode proc editDistance*(a, b: string): int {.noSideEffect.} = ## Returns the **unicode-rune** edit distance between `a` and `b`. @@ -18,7 +18,7 @@ proc editDistance*(a, b: string): int {.noSideEffect.} = ## This uses the `Levenshtein`:idx: distance algorithm with only a linear ## memory overhead. runnableExamples: static: doAssert editdistance("Kitten", "Bitten") == 1 - if len(a) > len(b): + if runeLen(a) > runeLen(b): # make `b` the longer string return editDistance(b, a) # strip common prefix diff --git a/lib/std/effecttraits.nim b/lib/std/effecttraits.nim index 358280db0e..3d1b4ffd32 100644 --- a/lib/std/effecttraits.nim +++ b/lib/std/effecttraits.nim @@ -11,13 +11,14 @@ ## for Nim's macro system. ## **Since**: Version 1.4. ## -## One can test for the existance of this standard module +## One can test for the existence of this standard module ## via `defined(nimHasEffectTraitsModule)`. -import macros +import std/macros proc getRaisesListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" proc getTagsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc getForbidsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" proc isGcSafeImpl(n: NimNode): bool = discard "see compiler/vmops.nim" proc hasNoSideEffectsImpl(n: NimNode): bool = discard "see compiler/vmops.nim" @@ -37,6 +38,14 @@ proc getTagsList*(fn: NimNode): NimNode = expectKind fn, nnkSym result = getTagsListImpl(fn) +proc getForbidsList*(fn: NimNode): NimNode = + ## Extracts the `.forbids` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getForbidsListImpl(fn) + proc isGcSafe*(fn: NimNode): bool = ## Return true if the func/proc/etc `fn` is `gcsafe`. ## `fn` has to be a resolved symbol of kind `nnkSym`. This diff --git a/lib/std/enumerate.nim b/lib/std/enumerate.nim index a8f0e1ba7f..beb65ed30e 100644 --- a/lib/std/enumerate.nim +++ b/lib/std/enumerate.nim @@ -11,7 +11,7 @@ ## macro system. import std/private/since -import macros +import std/macros macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = @@ -21,7 +21,7 @@ macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = ## The default starting count `0` can be manually overridden if needed. runnableExamples: let a = [10, 20, 30] - var b: seq[(int, int)] + var b: seq[(int, int)] = @[] for i, x in enumerate(a): b.add((i, x)) assert b == @[(0, 10), (1, 20), (2, 30)] diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 16dab9d1aa..bcfb2d5d7b 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -10,48 +10,63 @@ import std/macros from std/typetraits import OrdinalEnum, HoleyEnum +when defined(nimPreviewSlimSystem): + import std/assertions + + # xxx `genEnumCaseStmt` needs tests and runnableExamples -macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, +macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, userMin, userMax: static[int], normalizer: static[proc(s :string): string]): untyped = - # generates a case stmt, which assigns the correct enum field given + # Generates a case stmt, which assigns the correct enum field given # a normalized string comparison to the `argSym` input. - # string normalization is done using passed normalizer. - # NOTE: for an enum with fields Foo, Bar, ... we cannot generate - # `of "Foo".nimIdentNormalize: Foo`. - # This will fail, if the enum is not defined at top level (e.g. in a block). - # Thus we check for the field value of the (possible holed enum) and convert - # the integer value to the generic argument `typ`. + # string normalization is done using passed normalizer. let typ = typ.getTypeInst[1] - let impl = typ.getImpl[2] + let typSym = typ.getTypeImpl.getTypeInst # skip aliases etc to get type sym + let impl = typSym.getImpl[2] expectKind impl, nnkEnumTy let normalizerNode = quote: `normalizer` expectKind normalizerNode, nnkSym result = nnkCaseStmt.newTree(newCall(normalizerNode, argSym)) # stores all processed field strings to give error msg for ambiguous enums var foundFields: seq[string] = @[] + var fVal = "" var fStr = "" # string of current field var fNum = BiggestInt(0) # int value of current field for f in impl: case f.kind of nnkEmpty: continue # skip first node of `enumTy` - of nnkSym, nnkIdent: fStr = f.strVal + of nnkSym, nnkIdent: + fVal = f.strVal + fStr = fVal + of nnkAccQuoted: + fVal = "" + for ch in f: + fVal.add ch.strVal + fStr = fVal of nnkEnumFieldDef: + fVal = f[0].strVal case f[1].kind - of nnkStrLit: fStr = f[1].strVal + of nnkStrLit: + fStr = f[1].strVal of nnkTupleConstr: fStr = f[1][1].strVal fNum = f[1][0].intVal of nnkIntLit: fStr = f[0].strVal fNum = f[1].intVal - else: error("Invalid tuple syntax!", f[1]) - else: error("Invalid node for enum type!", f) + else: + let fAst = f[0].getImpl + if fAst.kind == nnkStrLit: + fStr = fAst.strVal + else: + error("Invalid tuple syntax!", f[1]) + else: error("Invalid node for enum type `" & $f.kind & "`!", f) # add field if string not already added if fNum >= userMin and fNum <= userMax: fStr = normalizer(fStr) if fStr notin foundFields: - result.add nnkOfBranch.newTree(newLit fStr, nnkCall.newTree(typ, newLit fNum)) + result.add nnkOfBranch.newTree(newLit fStr, newDotExpr(typ, ident fVal)) foundFields.add fStr else: error("Ambiguous enums cannot be parsed, field " & $fStr & @@ -67,7 +82,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, result.add nnkElse.newTree(default) macro enumFullRange(a: typed): untyped = - newNimNode(nnkCurly).add(a.getType[1][1..^1]) + newNimNode(nnkBracket).add(a.getType[1][1..^1]) macro enumNames(a: typed): untyped = # this could be exported too; in particular this could be useful for enum with holes. @@ -79,15 +94,90 @@ macro enumNames(a: typed): untyped = iterator items*[T: HoleyEnum](E: typedesc[T]): T = ## Iterates over an enum with holes. runnableExamples: - type A = enum a0 = 2, a1 = 4, a2 - type B[T] = enum b0 = 2, b1 = 4 + type + A = enum + a0 = 2 + a1 = 4 + a2 + B[T] = enum + b0 = 2 + b1 = 4 from std/sequtils import toSeq assert A.toSeq == [a0, a1, a2] assert B[float].toSeq == [B[float].b0, B[float].b1] for a in enumFullRange(E): yield a -func symbolName*[T: OrdinalEnum](a: T): string = +func span(T: typedesc[HoleyEnum]): int = + (T.high.ord - T.low.ord) + 1 + +const invalidSlot = uint8.high + +proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = + const n = span(T) + var i = 0 + assert n <= invalidSlot.int + var ret {.noinit.}: array[n, uint8] + for ai in mitems(ret): ai = invalidSlot + for ai in items(T): + ret[ai.ord - T.low.ord] = uint8(i) + inc(i) + return ret + +func symbolRankImpl[T](a: T): int {.inline.} = + const n = T.span + const thres = 255 # must be <= `invalidSlot`, but this should be tuned. + when n <= thres: + const lookup = genLookup(T) + let lookup2 {.global.} = lookup # xxx improve pending https://github.com/timotheecour/Nim/issues/553 + #[ + This could be optimized using a hash adapted to `T` (possible since it's known at CT) + to get better key distribution before indexing into the lookup table table. + ]# + {.noSideEffect.}: # because it's immutable + let ret = lookup2[ord(a) - T.low.ord] + if ret != invalidSlot: return ret.int + else: + var i = 0 + # we could also generate a case statement as optimization + for ai in items(T): + if ai == a: return i + inc(i) + raise newException(IndexDefect, $ord(a) & " invalid for " & $T) + +template symbolRank*[T: enum](a: T): int = + ## Returns the index in which `a` is listed in `T`. + ## + ## The cost for a `HoleyEnum` is implementation defined, currently optimized + ## for small enums, otherwise is `O(T.enumLen)`. + runnableExamples: + type + A = enum # HoleyEnum + a0 = -3 + a1 = 10 + a2 + a3 = (20, "f3Alt") + B = enum # OrdinalEnum + b0 + b1 + b2 + C = enum # OrdinalEnum + c0 = 10 + c1 + c2 + assert a2.symbolRank == 2 + assert b2.symbolRank == 2 + assert c2.symbolRank == 2 + assert c2.ord == 12 + assert a2.ord == 11 + var invalid = 7.A + doAssertRaises(IndexDefect): discard invalid.symbolRank + when T is Ordinal: ord(a) - T.low.ord.static + else: symbolRankImpl(a) + +func symbolName*[T: enum](a: T): string = ## Returns the symbol name of an enum. + ## + ## This uses `symbolRank`. runnableExamples: type B = enum b0 = (10, "kb0") @@ -97,5 +187,10 @@ func symbolName*[T: OrdinalEnum](a: T): string = assert b.symbolName == "b0" assert $b == "kb0" static: assert B.high.symbolName == "b2" + type C = enum # HoleyEnum + c0 = -3 + c1 = 4 + c2 = 20 + assert c1.symbolName == "c1" const names = enumNames(T) - names[a.ord - T.low.ord] + names[a.symbolRank] diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim new file mode 100644 index 0000000000..a955077ea9 --- /dev/null +++ b/lib/std/envvars.nim @@ -0,0 +1,221 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/envvars` module implements environment variable handling. +import std/oserrors + +type + ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read + ## from an environment variable. + WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write + ## to an environment variable. + + +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + var num: int + var keys: RootObj + {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} + for i in 0..<num: + var key, value: cstring + {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".} + yield ($key, $value) + + # commented because it must keep working with js+VM + # elif defined(js): + # {.error: "requires -d:nodejs".} + + else: + + when defined(windows): + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "<stdlib.h>".} + from std/private/win_setenv import setEnvImpl + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t", header: "<stdlib.h>".} = int16 + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", + header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): WideCString = + let r: WideCString = env.newWideCString + cast[WideCString](c_wgetenv(cast[ptr wchar_t](r))) + else: + proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "<stdlib.h>".} + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "<stdlib.h>".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): cstring = c_getenv(env) + + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc`_. + ## + ## See also: + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" + + let env = getEnvImpl(key) + if env == nil: + result = default + else: + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + result = getEnvImpl(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + else: + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "<string.h>".} + else: + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "<string.h>".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # <crt_externs.h>, can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "<crt_externs.h>".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + let env = getEnvironmentStringsW() + var e = env + if e != nil: + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[WideCString](cast[ByteAddress](eend)+2) + if int(eend[1]) == 0: break + discard freeEnvironmentStringsW(env) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index c6537f7f89..f26368f424 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -7,7 +7,11 @@ # distribution, for details about the copyright. # -import locks +## This module allows adding hooks to program exit. + +import std/locks +when defined(js) and not defined(nodejs): + import std/assertions type FunKind = enum kClosure, kNoconv # extend as needed @@ -18,20 +22,20 @@ type var gFunsLock: Lock - gFuns: seq[Fun] + gFuns {.cursor.}: seq[Fun] #Intentionally use the cursor to break up the lifetime trace and make it compatible with JS. initLock(gFunsLock) when defined(js): proc addAtExit(quitProc: proc() {.noconv.}) = when defined(nodejs): - asm """ + {.emit: """ process.on('exit', `quitProc`); - """ + """.} elif defined(js): - asm """ + {.emit: """ window.onbeforeunload = `quitProc`; - """ + """.} else: proc addAtExit(quitProc: proc() {.noconv.}) {. importc: "atexit", header: "<stdlib.h>".} @@ -43,6 +47,7 @@ proc callClosures() {.noconv.} = case fun.kind of kClosure: fun.fun1() of kNoconv: fun.fun2() + gFuns.setLen(0) template fun() = if gFuns.len == 0: @@ -64,24 +69,19 @@ proc addExitProc*(cl: proc() {.noconv.}) = fun() gFuns.add Fun(kind: kNoconv, fun2: cl) -when not defined(nimscript): +when not defined(nimscript) and (not defined(js) or defined(nodejs)): proc getProgramResult*(): int = when defined(js) and defined(nodejs): - asm """ + {.emit: """ `result` = process.exitCode; -""" - elif not defined(js): - result = programResult +""".} else: - doAssert false + result = programResult proc setProgramResult*(a: int) = - # pending https://github.com/nim-lang/Nim/issues/14674 when defined(js) and defined(nodejs): - asm """ + {.emit: """ process.exitCode = `a`; -""" - elif not defined(js): - programResult = a +""".} else: - doAssert false + programResult = a diff --git a/lib/std/files.nim b/lib/std/files.nim new file mode 100644 index 0000000000..c4e0491c99 --- /dev/null +++ b/lib/std/files.nim @@ -0,0 +1,46 @@ +## This module implements file handling. +## +## **See also:** +## * `paths module <paths.html>`_ for path manipulation + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osfiles import fileExists, removeFile, + moveFile + + +proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + result = fileExists(filename.string) + +proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc <dirs.html#removeDir>`_ + ## * `moveFile proc`_ + removeFile(file.string) + +proc moveFile*(source, dest: Path) {.inline, + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect].} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc <dirs.html#moveDir>`_ + ## * `removeFile proc`_ + moveFile(source.string, dest.string) diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim new file mode 100644 index 0000000000..63d3442154 --- /dev/null +++ b/lib/std/formatfloat.nim @@ -0,0 +1,143 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements formatting floats as strings. + +when defined(nimPreviewSlimSystem): + import std/assertions + +proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "<string.h>", discardable.} + +proc addCstringN(result: var string, buf: cstring; buflen: int) = + # no nimvm support needed, so it doesn't need to be fast here either + let oldLen = result.len + let newLen = oldLen + buflen + result.setLen newLen + c_memcpy(result[oldLen].addr, buf, buflen.csize_t) + +import std/private/[dragonbox, schubfach] + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + result = toChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: float32): int = + result = float32ToChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs, noSideEffect.} + +proc writeToBuffer(buf: var array[65, char]; value: cstring) = + var i = 0 + while value[i] != '\0': + buf[i] = value[i] + inc i + +proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + var n = c_snprintf(cast[cstring](addr buf), 65, "%.16g", value).int + var hasDot = false + for i in 0..n-1: + if buf[i] == ',': + buf[i] = '.' + hasDot = true + elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: + hasDot = true + if not hasDot: + buf[n] = '.' + buf[n+1] = '0' + buf[n+2] = '\0' + result = n + 2 + else: + result = n + # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' + # of '-1.#IND' are produced. + # We want to get rid of these here: + if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: + writeToBuffer(buf, "nan") + result = 3 + elif buf[n-1] == 'F': + if buf[0] == '-': + writeToBuffer(buf, "-inf") + result = 4 + else: + writeToBuffer(buf, "inf") + result = 3 + +proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + writeFloatToBufferRoundtrip(buf, value) + else: + writeFloatToBufferSprintf(buf, value) + +proc addFloatRoundtrip*(result: var string; x: float | float32) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferRoundtrip(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +proc addFloatSprintf*(result: var string; x: float) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferSprintf(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +when defined(js): + proc nimFloatToString(a: float): cstring = + ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2 + # print `-0.0` properly + {.emit: """ + function nimOnlyDigitsOrMinus(n) { + return n.toString().match(/^-?\d+$/); + } + if (Number.isSafeInteger(`a`)) + `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0" + else { + `result` = `a`+"" + if(nimOnlyDigitsOrMinus(`result`)){ + `result` = `a`+".0" + } + } + """.} + +proc addFloat*(result: var string; x: float | float32) {.inline.} = + ## Converts float to its string representation and appends it to `result`. + runnableExamples: + var + s = "foo:" + b = 45.67 + s.addFloat(45.67) + assert s == "foo:45.67" + template impl = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + addFloatRoundtrip(result, x) + else: + addFloatSprintf(result, x) + when defined(js): + when nimvm: impl() + else: + result.add nimFloatToString(x) + else: impl() + +when defined(nimPreviewSlimSystem): + func `$`*(x: float | float32): string = + ## Outplace version of `addFloat`. + result.addFloat(x) diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim new file mode 100644 index 0000000000..d0f07c5273 --- /dev/null +++ b/lib/std/genasts.nim @@ -0,0 +1,89 @@ +## This module implements AST generation using captured variables for macros. + +import std/macros + +type GenAstOpt* = enum + kDirtyTemplate, + # When set, uses a dirty template in implementation of `genAst`. This + # is occasionally useful as workaround for issues such as #8220, see + # `strformat limitations <strformat.html#limitations>`_ for details. + # Default is unset, to avoid hijacking of uncaptured local symbols by + # symbols in caller scope. + kNoNewLit, + # don't call call newLit automatically in `genAst` capture parameters + +macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped = + ## Accepts a list of captured variables `a=b` or `a` and a block and returns the + ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured + ## unless `kDirtyTemplate in options`. + runnableExamples: + # This example shows how one could write a simplified version of `unittest.check`. + import std/[macros, strutils] + macro check2(cond: bool): untyped = + assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind + result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): + # each local symbol we access must be explicitly captured + if not cond: + raiseAssert "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] + let a = 3 + check2 a*2 == a+3 + if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' + + runnableExamples: + # This example goes in more details about the capture semantics. + macro fun(a: string, b: static bool): untyped = + let c = 'z' + var d = 11 # implicitly {.gensym.} and needs to be captured for use in `genAst`. + proc localFun(): auto = 12 # implicitly {.inject.}, doesn't need to be captured. + genAst(a, b, c = true): + # `a`, `b` are captured explicitly, `c` is a local definition masking `c = 'z'`. + const b2 = b # macro static param `b` is forwarded here as a static param. + # `echo d` would give: `var not init` because `d` is not captured. + (a & a, b, c, localFun()) # localFun can be called without capture. + assert fun("ab", false) == ("abab", false, true, 12) + + let params = newTree(nnkFormalParams, newEmptyNode()) + let pragmas = + if kDirtyTemplate in options: + nnkPragma.newTree(ident"dirty") + else: + newEmptyNode() + + template newLitMaybe(a): untyped = + when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)): + a # `proc` actually also covers template, macro + else: newLit(a) + + # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669 + let name = genSym(nskTemplate, "_fun") + let call = newCall(name) + for a in args[0..^2]: + var varName: NimNode + var varVal: NimNode + case a.kind + of nnkExprEqExpr: + varName = a[0] + varVal = a[1] + of nnkIdent: + varName = a + varVal = a + else: error("invalid argument kind: " & $a.kind, a) + if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal) + + params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode()) + call.add varVal + + result = newStmtList() + result.add nnkTemplateDef.newTree( + name, + newEmptyNode(), + newEmptyNode(), + params, + pragmas, + newEmptyNode(), + args[^1]) + result.add newCall(bindSym"getAst", call) + +template genAst*(args: varargs[untyped]): untyped = + ## Convenience wrapper around `genAstOpt`. + genAstOpt({}, args) diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim new file mode 100644 index 0000000000..d2da76ea87 --- /dev/null +++ b/lib/std/importutils.nim @@ -0,0 +1,44 @@ +##[ +Utilities related to import and symbol resolution. + +Experimental API, subject to change. +]## + +#[ +Possible future APIs: +* module symbols (https://github.com/nim-lang/Nim/pull/9560) +* whichModule (subsumes canImport / moduleExists) (https://github.com/timotheecour/Nim/issues/376) +* getCurrentPkgDir (https://github.com/nim-lang/Nim/pull/10530) +* import from a computed string + related APIs (https://github.com/nim-lang/Nim/pull/10527) +]# + +when defined(nimImportutilsExample): + type + Foo = object + f0: int # private + Goo*[T] = object + g0: int # private + proc initFoo*(): auto = Foo() + +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = + ## Enables access to private fields of `t` in current scope. + runnableExamples("-d:nimImportutilsExample"): + # here we're importing a module containing: + # type + # Foo = object + # f0: int # private + # Goo*[T] = object + # g0: int # private + # proc initFoo*(): auto = Foo() + var f = initFoo() + block: + assert not compiles(f.f0) + privateAccess(f.type) + f.f0 = 1 # accessible in this scope + block: + assert f.f0 == 1 # still in scope + assert not compiles(f.f0) + + # this also works with generics + privateAccess(Goo) + assert Goo[float](g0: 1).g0 == 1 diff --git a/lib/std/isolation.nim b/lib/std/isolation.nim index 8daca233bf..b03e00651e 100644 --- a/lib/std/isolation.nim +++ b/lib/std/isolation.nim @@ -15,10 +15,10 @@ ## type - Isolated*[T] = object ## Isolated data can only be moved, not copied. + Isolated*[T] {.sendable.} = object ## Isolated data can only be moved, not copied. value: T -proc `=`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} +proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} proc `=sink`*[T](dest: var Isolated[T]; src: Isolated[T]) {.inline.} = # delegate to value's sink operation @@ -38,9 +38,9 @@ func isolate*[T](value: sink T): Isolated[T] {.magic: "Isolate".} = func unsafeIsolate*[T](value: sink T): Isolated[T] = ## Creates an isolated subgraph from the expression `value`. - ## + ## ## .. warning:: The proc doesn't check whether `value` is isolated. - ## + ## Isolated[T](value: value) func extract*[T](src: var Isolated[T]): T = diff --git a/lib/std/jsbigints.nim b/lib/std/jsbigints.nim index 21bb9e1a05..4e996ea7b9 100644 --- a/lib/std/jsbigints.nim +++ b/lib/std/jsbigints.nim @@ -3,27 +3,36 @@ when not defined(js): {.fatal: "Module jsbigints is designed to be used with the JavaScript backend.".} -type JsBigIntImpl {.importc: "bigint".} = int # https://github.com/nim-lang/Nim/pull/16606 -type JsBigInt* = distinct JsBigIntImpl ## Arbitrary precision integer for JavaScript target. +when defined(nimPreviewSlimSystem): + import std/assertions + +type JsBigIntImpl {.importjs: "bigint".} = int # https://github.com/nim-lang/Nim/pull/16606 +type JsBigInt* = distinct JsBigIntImpl ## Arbitrary precision integer for JavaScript target. func big*(integer: SomeInteger): JsBigInt {.importjs: "BigInt(#)".} = ## Constructor for `JsBigInt`. - when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard runnableExamples: doAssert big(1234567890) == big"1234567890" doAssert 0b1111100111.big == 0o1747.big and 0o1747.big == 999.big + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".} = + ## Constructor for `JsBigInt`. + runnableExamples: + doAssert -1'big == 1'big - 2'big + # supports decimal, binary, octal, hex: + doAssert -12'big == big"-12" + doAssert 12'big == 12.big + doAssert 0b101'big == 0b101.big + doAssert 0o701'big == 0o701.big + doAssert 0xdeadbeaf'big == 0xdeadbeaf.big + doAssert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big + doAssert not compiles(static(12'big)) + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = - ## Constructor for `JsBigInt`. - when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard - runnableExamples: - doAssert big"-1" == big"1" - big"2" - # supports decimal, binary, octal, hex: - doAssert big"12" == 12.big - doAssert big"0b101" == 0b101.big - doAssert big"0o701" == 0o701.big - doAssert big"0xdeadbeaf" == 0xdeadbeaf.big - doAssert big"0xffffffffffffffff" == (1.big shl 64.big) - 1.big + ## Alias for `'big` + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard func toCstring*(this: JsBigInt; radix: 2..36): cstring {.importjs: "#.toString(#)".} = ## Converts from `JsBigInt` to `cstring` representation. @@ -55,10 +64,10 @@ func wrapToUint*(this: JsBigInt; bits: Natural): JsBigInt {.importjs: runnableExamples: doAssert (big("3") + big("2") ** big("66")).wrapToUint(66) == big("3") -func toNumber*(this: JsBigInt): BiggestInt {.importjs: "Number(#)".} = +func toNumber*(this: JsBigInt): int {.importjs: "Number(#)".} = ## Does not do any bounds check and may or may not return an inexact representation. runnableExamples: - doAssert toNumber(big"2147483647") == 2147483647.BiggestInt + doAssert toNumber(big"2147483647") == 2147483647.int func `+`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = runnableExamples: @@ -101,7 +110,7 @@ func `==`*(x, y: JsBigInt): bool {.importjs: "(# == #)".} = doAssert big"42" == big"42" func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = - # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 + # (#) needed due to unary minus runnableExamples: doAssert big"2" ** big"64" == big"18446744073709551616" doAssert big"-2" ** big"3" == big"-8" @@ -111,8 +120,6 @@ func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = try: discard big"2" ** big"-1" # raises foreign `RangeError` except: ok = true doAssert ok - # pending https://github.com/nim-lang/Nim/pull/15940, simplify to: - # doAssertRaises: discard big"2" ** big"-1" # raises foreign `RangeError` func `and`*(x, y: JsBigInt): JsBigInt {.importjs: "(# & #)".} = runnableExamples: diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 36bc772d5d..2195946197 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -1,122 +1,123 @@ ## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API -## .. Note:: jsfetch is Experimental. jsfetch module requires `-d:nimExperimentalJsfetch` when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} -when defined(nimExperimentalJsfetch) or defined(nimdoc): - import std/[asyncjs, jsheaders, jsformdata] - from std/httpcore import HttpMethod - from std/jsffi import JsObject +import std/[asyncjs, jsformdata, jsheaders] +export jsformdata, jsheaders +from std/httpcore import HttpMethod +from std/jsffi import JsObject - type - FetchOptions* = ref object of JsRoot ## Options for Fetch API. - keepalive*: bool - metod* {.importjs: "method".}: cstring - body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring +type + FetchOptions* = ref object of JsRoot ## Options for Fetch API. + keepalive*: bool + metod* {.importjs: "method".}: cstring + body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring + headers*: Headers - FetchModes* = enum ## Mode options. - fmCors = "cors" - fmNoCors = "no-cors" - fmSameOrigin = "same-origin" + FetchModes* = enum ## Mode options. + fmCors = "cors" + fmNoCors = "no-cors" + fmSameOrigin = "same-origin" - FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials - fcInclude = "include" - fcSameOrigin = "same-origin" - fcOmit = "omit" + FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials + fcInclude = "include" + fcSameOrigin = "same-origin" + fcOmit = "omit" - FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache - fchDefault = "default" - fchNoStore = "no-store" - fchReload = "reload" - fchNoCache = "no-cache" - fchForceCache = "force-cache" + FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache + fchDefault = "default" + fchNoStore = "no-store" + fchReload = "reload" + fchNoCache = "no-cache" + fchForceCache = "force-cache" - FetchRedirects* = enum ## Redirects options. - frFollow = "follow" - frError = "error" - frManual = "manual" + FetchRedirects* = enum ## Redirects options. + frFollow = "follow" + frError = "error" + frManual = "manual" - FetchReferrerPolicies* = enum ## Referrer Policy options. - frpNoReferrer = "no-referrer" - frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" - frpOrigin = "origin" - frpOriginWhenCrossOrigin = "origin-when-cross-origin" - frpUnsafeUrl = "unsafe-url" + FetchReferrerPolicies* = enum ## Referrer Policy options. + frpNoReferrer = "no-referrer" + frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" + frpOrigin = "origin" + frpOriginWhenCrossOrigin = "origin-when-cross-origin" + frpUnsafeUrl = "unsafe-url" - Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body - bodyUsed*: bool + Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring - Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body + Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring - Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body +func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} + ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. +func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} + ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. - func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} - ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. +func newRequest*(url: cstring; fetchOptions: FetchOptions): Request {.importjs: "(new Request(#, #))".} + ## Constructor for `Request` with `fetchOptions`. Same as `fetch(url, fetchOptions)`. - func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} - ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. +func clone*(self: Response | Request): Response {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone - func clone*(self: Response | Request): Response {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone +proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/text - proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text +proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/json - proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json +proc formData*(self: Response): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/formData - proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData +proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring; headers: Headers = newHeaders()): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #, headers: #}".} + ## .. warning:: Unsafe `newfetchOptions`. - proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; - keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: - "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} - ## .. warning:: Unsafe `newfetchOptions`. - - func newfetchOptions*(metod: HttpMethod; body: cstring; - mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; - keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = - ## Constructor for `FetchOptions`. - result = FetchOptions( - body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, - keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, - metod: (case metod - of HttpHead: "HEAD".cstring - of HttpGet: "GET".cstring - of HttpPost: "POST".cstring - of HttpPut: "PUT".cstring - of HttpDelete: "DELETE".cstring - of HttpPatch: "PATCH".cstring - else: "GET".cstring - ) +func newfetchOptions*(metod = HttpGet; body: cstring = nil; + mode = fmCors; credentials = fcSameOrigin; cache = fchDefault; referrerPolicy = frpNoReferrerWhenDowngrade; + keepalive = false; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, + headers: Headers = newHeaders()): FetchOptions = + ## Constructor for `FetchOptions`. + result = FetchOptions( + body: if metod notin {HttpHead, HttpGet}: body else: nil, + mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, headers: headers, + metod: (case metod + of HttpHead: "HEAD".cstring + of HttpGet: "GET".cstring + of HttpPost: "POST".cstring + of HttpPut: "PUT".cstring + of HttpDelete: "DELETE".cstring + of HttpPatch: "PATCH".cstring + else: "GET".cstring ) + ) - proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} - ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. +proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} + ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. - proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} - ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. +proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} + ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. - func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} +func toCstring*(self: Request | Response | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} - func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) +func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self) -runnableExamples("-d:nimExperimentalJsfetch -r:off"): - import std/[asyncjs, jsconsole, jsheaders, jsformdata] +runnableExamples("-r:off"): + import std/[asyncjs, jsconsole, jsformdata, jsheaders] from std/httpcore import HttpMethod from std/jsffi import JsObject from std/sugar import `=>` @@ -132,7 +133,8 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): keepalive = false, redirect = "follow".cstring, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options0.keepalive == false assert options0.metod == "POST".cstring @@ -144,6 +146,7 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): assert options0.redirect == "follow".cstring assert options0.referrer == "client".cstring assert options0.integrity == "".cstring + assert options0.headers.len == 0 block: let options1: FetchOptions = newFetchOptions( @@ -156,7 +159,8 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): keepalive = false, redirect = frFollow, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options1.keepalive == false assert options1.metod == $HttpPost @@ -168,6 +172,7 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): assert options1.redirect == $frFollow assert options1.referrer == "client".cstring assert options1.integrity == "".cstring + assert options1.headers.len == 0 block: let response: Response = newResponse(body = "-. .. --".cstring) @@ -183,16 +188,15 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): assert response.ok assert response.status == 200.cint assert response.headers is Headers - assert response.body is Body + assert response.body is cstring discard example() - when defined(nimExperimentalAsyncjsThen): - block: - proc example2 {.async.} = - await fetch("https://api.github.com/users/torvalds".cstring) - .then((response: Response) => response.json()) - .then((json: JsObject) => console.log(json)) - .catch((err: Error) => console.log("Request Failed", err)) + block: + proc example2 {.async.} = + await fetch("https://api.github.com/users/torvalds".cstring) + .then((response: Response) => response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) - discard example2() + discard example2() diff --git a/lib/std/jsformdata.nim b/lib/std/jsformdata.nim index 120f8742d2..61dcc39a3b 100644 --- a/lib/std/jsformdata.nim +++ b/lib/std/jsformdata.nim @@ -2,17 +2,21 @@ when not defined(js): {.fatal: "Module jsformdata is designed to be used with the JavaScript backend.".} +from std/dom import Blob + type FormData* = ref object of JsRoot ## FormData API. func newFormData*(): FormData {.importjs: "new FormData()".} -func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring) {.importjs: "#.append(#, #)".} +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.append(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append - ## Duplicate keys are allowed and order is preserved. + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. -func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring, filename: cstring) {.importjs: "#.append(#, #, #)".} +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.append(#, #, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append - ## Duplicate keys are allowed and order is preserved. + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. func delete*(self: FormData; name: cstring) {.importjs: "#.$1(#)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/delete @@ -34,10 +38,10 @@ func values*(self: FormData): seq[cstring] {.importjs: "Array.from(#.$1())".} func pairs*(self: FormData): seq[tuple[key, val: cstring]] {.importjs: "Array.from(#.entries())".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/entries -func put*(self: FormData; name, value, filename: cstring) {.importjs: "#.set(#, #, #)".} +func put*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.set(#, #, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set -func `[]=`*(self: FormData; name, value: cstring) {.importjs: "#.set(#, #)".} +func `[]=`*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.set(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set func `[]`*(self: FormData; name: cstring): cstring {.importjs: "#.get(#)".} diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index fa61d79dbf..2d28748ce8 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -1,5 +1,5 @@ ##[ -This module implements a hookable (de)serialization for arbitrary types. +This module implements a hookable (de)serialization for arbitrary types using JSON. Design goal: avoid importing modules where a custom serialization is needed; see strtabs.fromJsonHook,toJsonHook for an example. ]## @@ -11,9 +11,12 @@ runnableExamples: z1: int8 let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable]) let j = a.toJson - doAssert j.jsonTo(typeof(a)).toJson == j + assert j.jsonTo(typeof(a)).toJson == j + assert $[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2].toJson == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert 0.0.toJson.kind == JFloat + assert Inf.toJson.kind == JString -import std/[json,strutils,tables,sets,strtabs,options] +import std/[json, strutils, tables, sets, strtabs, options, strformat] #[ Future directions: @@ -28,9 +31,17 @@ add a way to customize serialization, for e.g.: ]# import std/macros +from std/enumutils import symbolName +from std/typetraits import OrdinalEnum, tupleLen + +when defined(nimPreviewSlimSystem): + import std/assertions + + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} type - Joptions* = object + Joptions* = object # xxx rename FromJsonOptions ## Options controlling the behavior of `fromJson`. allowExtraKeys*: bool ## If `true` Nim's object to which the JSON is parsed is not required to @@ -39,10 +50,25 @@ type ## If `true` Nim's object to which JSON is parsed is allowed to have ## fields without corresponding JSON keys. # in future work: a key rename could be added + EnumMode* = enum + joptEnumOrd + joptEnumSymbol + joptEnumString + JsonNodeMode* = enum ## controls `toJson` for JsonNode types + joptJsonNodeAsRef ## returns the ref as is + joptJsonNodeAsCopy ## returns a deep copy of the JsonNode + joptJsonNodeAsObject ## treats JsonNode as a regular ref object + ToJsonOptions* = object + enumMode*: EnumMode + jsonNodeMode*: JsonNodeMode + # xxx charMode, etc -proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} -template distinctBase[T](a: T): untyped = distinctBase(typeof(a))(a) +proc initToJsonOptions*(): ToJsonOptions = + ## initializes `ToJsonOptions` with sane options. + ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) + +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T, recursive: static bool = true): untyped = distinctBase(typeof(a), recursive)(a) macro getDiscriminants(a: typedesc): seq[string] = ## return the discriminant keys @@ -52,19 +78,24 @@ macro getDiscriminants(a: typedesc): seq[string] = let sym = a[1] let t = sym.getTypeImpl let t2 = t[2] - doAssert t2.kind == nnkRecList - result = newTree(nnkBracket) - for ti in t2: - if ti.kind == nnkRecCase: - let key = ti[0][0] - let typ = ti[0][1] - result.add newLit key.strVal - if result.len > 0: + case t2.kind + of nnkEmpty: # allow empty objects result = quote do: - @`result` + seq[string].default + of nnkRecList: + result = newTree(nnkBracket) + for ti in t2: + if ti.kind == nnkRecCase: + let key = ti[0][0] + result.add newLit key.strVal + if result.len > 0: + result = quote do: + @`result` + else: + result = quote do: + seq[string].default else: - result = quote do: - seq[string].default + raiseAssert "unexpected kind: " & $t2.kind macro initCaseObject(T: typedesc, fun: untyped): untyped = ## does the minimum to construct a valid case object, only initializing @@ -78,7 +109,7 @@ macro initCaseObject(T: typedesc, fun: untyped): untyped = case t.kind of nnkObjectTy: t2 = t[2] of nnkRefTy: t2 = t[0].getTypeImpl[2] - else: doAssert false, $t.kind # xxx `nnkPtrTy` could be handled too + else: raiseAssert $t.kind # xxx `nnkPtrTy` could be handled too doAssert t2.kind == nnkRecList result = newTree(nnkObjConstr) result.add sym @@ -91,14 +122,14 @@ macro initCaseObject(T: typedesc, fun: untyped): untyped = `fun`(`key2`, typedesc[`typ`]) result.add newTree(nnkExprColonExpr, key, val) -proc checkJsonImpl(cond: bool, condStr: string, msg = "") = - if not cond: - # just pick 1 exception type for simplicity; other choices would be: - # JsonError, JsonParser, JsonKindError - raise newException(ValueError, msg) +proc raiseJsonException(condStr: string, msg: string) {.noinline.} = + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, condStr & " failed: " & msg) template checkJson(cond: untyped, msg = "") = - checkJsonImpl(cond, astToStr(cond), msg) + if not cond: + raiseJsonException(astToStr(cond), msg) proc hasField[T](obj: T, field: string): bool = for k, _ in fieldPairs(obj): @@ -106,7 +137,7 @@ proc hasField[T](obj: T, field: string): bool = return true return false -macro accessField(obj: typed, name: static string): untyped = +macro accessField(obj: typed, name: static string): untyped = newDotExpr(obj, ident(name)) template fromJsonFields(newObj, oldObj, json, discKeys, opt) = @@ -128,7 +159,7 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = if discKeys.len == 0 or hasField(oldObj, key): val = accessField(oldObj, key) else: - checkJson false, $($T, key, json) + checkJson false, "key '$1' for $2 not in $3" % [key, $T, json.pretty()] else: if json.hasKey key: numMatched.inc @@ -146,8 +177,8 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = json.len == numMatched else: json.len == num and num == numMatched - - checkJson ok, $(json.len, num, numMatched, $T, json) + + checkJson ok, "There were $1 keys (expecting $2) for $3 with $4" % [$json.len, $num, $T, json.pretty()] proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) @@ -180,23 +211,24 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = adding "json path" leading to `b` can be added in future work. ]# checkJson b != nil, $($T, b) - when compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + when compiles(fromJsonHook(a, b, opt)): fromJsonHook(a, b, opt) + elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b) elif T is bool: a = to(b,T) elif T is enum: case b.kind of JInt: a = T(b.getBiggestInt()) of JString: a = parseEnum[T](b.getStr()) - else: checkJson false, $($T, " ", b) + else: checkJson false, fmt"Expecting int/string for {$T} got {b.pretty()}" elif T is uint|uint64: a = T(to(b, uint64)) elif T is Ordinal: a = cast[T](to(b, int)) elif T is pointer: a = cast[pointer](to(b, int)) - elif T is distinct: - when nimvm: - # bug, potentially related to https://github.com/nim-lang/Nim/issues/12282 - a = T(jsonTo(b, distinctBase(T))) - else: - a.distinctBase.fromJson(b) + elif T is distinct: a.distinctBase.fromJson(b) elif T is string|SomeNumber: a = to(b,T) + elif T is cstring: + case b.kind + of JNull: a = nil + of JString: a = b.str + else: checkJson false, fmt"Expecting null/string for {$T} got {b.pretty()}" elif T is JsonNode: a = b elif T is ref | ptr: if b.kind == JNull: a = nil @@ -204,11 +236,15 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = a = T() fromJson(a[], b, opt) elif T is array: - checkJson a.len == b.len, $(a.len, b.len, $T) + checkJson a.len == b.len, fmt"Json array size doesn't match for {$T}" var i = 0 for ai in mitems(a): fromJson(ai, b[i], opt) i.inc + elif T is set: + type E = typeof(for ai in a: ai) + for val in b.getElems: + incl a, jsonTo(val, E) elif T is seq: a.setLen b.len for i, val in b.getElems: @@ -236,49 +272,83 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = fromJsonFields(a, nil, b, seq[string].default, opt) else: checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + + when compiles(tupleLen(T)): + let tupleSize = tupleLen(T) + else: + # Tuple len isn't in csources_v1 so using tupleLen would fail. + # Else branch basically never runs (tupleLen added in 1.1 and jsonutils in 1.4), but here for consistency + var tupleSize = 0 + for val in fields(a): + tupleSize.inc + + checkJson b.len == tupleSize, fmt"Json doesn't match expected length of {tupleSize}, got {b.pretty()}" var i = 0 for val in fields(a): fromJson(val, b[i], opt) i.inc - checkJson b.len == i, $(b.len, i, $T, b) # could customize else: # checkJson not appropriate here - static: doAssert false, "not yet implemented: " & $T + static: raiseAssert "not yet implemented: " & $T proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = ## reverse of `toJson` fromJson(result, b, opt) -proc toJson*[T](a: T): JsonNode = +proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to ## customize serialization, see strtabs.toJsonHook for an example. - when compiles(toJsonHook(a)): result = toJsonHook(a) + ## + ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can + ## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. + ## It is expected that this behavior becomes the new default in upcoming versions. + when compiles(toJsonHook(a, opt)): result = toJsonHook(a, opt) + elif compiles(toJsonHook(a)): result = toJsonHook(a) elif T is object | tuple: when T is object or isNamedTuple(T): result = newJObject() - for k, v in a.fieldPairs: result[k] = toJson(v) + for k, v in a.fieldPairs: result[k] = toJson(v, opt) else: result = newJArray() - for v in a.fields: result.add toJson(v) + for v in a.fields: result.add toJson(v, opt) elif T is ref | ptr: - if system.`==`(a, nil): result = newJNull() - else: result = toJson(a[]) - elif T is array | seq: + template impl = + if system.`==`(a, nil): result = newJNull() + else: result = toJson(a[], opt) + when T is JsonNode: + case opt.jsonNodeMode + of joptJsonNodeAsRef: result = a + of joptJsonNodeAsCopy: result = copy(a) + of joptJsonNodeAsObject: impl() + else: impl() + elif T is array | seq | set: result = newJArray() - for ai in a: result.add toJson(ai) - elif T is pointer: result = toJson(cast[int](a)) + for ai in a: result.add toJson(ai, opt) + elif T is pointer: result = toJson(cast[int](a), opt) # edge case: `a == nil` could've also led to `newJNull()`, but this results # in simpler code for `toJson` and `fromJson`. - elif T is distinct: result = toJson(a.distinctBase) + elif T is distinct: result = toJson(a.distinctBase, opt) elif T is bool: result = %(a) elif T is SomeInteger: result = %a + elif T is enum: + case opt.enumMode + of joptEnumOrd: + when T is Ordinal or defined(nimPreviewJsonutilsHoleyEnum): %(a.ord) + else: toJson($a, opt) + of joptEnumSymbol: + when T is OrdinalEnum: + toJson(symbolName(a), opt) + else: + toJson($a, opt) + of joptEnumString: toJson($a, opt) elif T is Ordinal: result = %(a.ord) + elif T is cstring: (if a == nil: result = newJNull() else: result = % $a) else: result = %a -proc fromJsonHook*[K, V](t: var (Table[K, V] | OrderedTable[K, V]), - jsonNode: JsonNode) = +proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]), + jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `Table` and `OrderedTable` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook>`_ runnableExamples: @@ -294,27 +364,32 @@ proc fromJsonHook*[K, V](t: var (Table[K, V] | OrderedTable[K, V]), "type is `" & $jsonNode.kind & "`." clear(t) for k, v in jsonNode: - t[k] = jsonTo(v, V) + t[k] = jsonTo(v, V, opt) -proc toJsonHook*[K, V](t: (Table[K, V] | OrderedTable[K, V])): JsonNode = +proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V]), opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `Table` and `OrderedTable` types. ## ## See also: ## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_ runnableExamples: - import std/[tables, json] + import std/[tables, json, sugar] let foo = ( t: [("two", 2)].toTable, ot: [("one", 1), ("three", 3)].toOrderedTable) assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}""" + # if keys are not string|cstring, you can use this: + let a = {10: "foo", 11: "bar"}.newOrderedTable + let a2 = collect: (for k,v in a: (k,v)) + assert $toJson(a2) == """[[10,"foo"],[11,"bar"]]""" result = newJObject() for k, v in pairs(t): - result[k] = toJson(v) + # not sure if $k has overhead for string + result[(when K is string: k else: $k)] = toJson(v, opt) -proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) = +proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `HashSet` and `OrderedSet` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_ runnableExamples: @@ -330,9 +405,9 @@ proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) = "type is `" & $jsonNode.kind & "`." clear(s) for v in jsonNode: - incl(s, jsonTo(v, A)) + incl(s, jsonTo(v, A, opt)) -proc toJsonHook*[A](s: SomeSet[A]): JsonNode = +proc toJsonHook*[A](s: SomeSet[A], opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `HashSet` and `OrderedSet` types. ## ## See also: @@ -344,11 +419,11 @@ proc toJsonHook*[A](s: SomeSet[A]): JsonNode = result = newJArray() for k in s: - add(result, toJson(k)) + add(result, toJson(k, opt)) -proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) = +proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `Option` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,Option[T]>`_ runnableExamples: @@ -360,11 +435,11 @@ proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) = assert isNone(opt) if jsonNode.kind != JNull: - self = some(jsonTo(jsonNode, T)) + self = some(jsonTo(jsonNode, T, opt)) else: self = none[T]() -proc toJsonHook*[T](self: Option[T]): JsonNode = +proc toJsonHook*[T](self: Option[T], opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `Option` types. ## ## See also: @@ -377,13 +452,13 @@ proc toJsonHook*[T](self: Option[T]): JsonNode = assert $toJson(optNone) == "null" if isSome(self): - toJson(get(self)) + toJson(get(self), opt) else: newJNull() proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = ## Enables `fromJson` for `StringTableRef` type. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,StringTableRef>`_ runnableExamples: @@ -401,7 +476,7 @@ proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = proc toJsonHook*(a: StringTableRef): JsonNode = ## Enables `toJson` for `StringTableRef` type. - ## + ## ## See also: ## * `fromJsonHook proc<#fromJsonHook,StringTableRef,JsonNode>`_ runnableExamples: diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index 78736d719e..bf6dc776b9 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -10,17 +10,14 @@ ##[ The `std/monotimes` module implements monotonic timestamps. A monotonic timestamp represents the time that has passed since some system defined -point in time. The monotonic timestamps are guaranteed to always increase, +point in time. The monotonic timestamps are guaranteed not to decrease, meaning that that the following is guaranteed to work: ]## runnableExamples: - import std/os - let a = getMonoTime() - sleep(10) let b = getMonoTime() - assert a < b + assert a <= b ##[ This is not guaranteed for the `times.Time` type! This means that the @@ -79,6 +76,10 @@ when defined(js): elif defined(posix) and not defined(osx): import std/posix +when defined(zephyr): + proc k_uptime_ticks(): int64 {.importc: "k_uptime_ticks", header: "<kernel.h>".} + proc k_ticks_to_ns_floor64(ticks: int64): int64 {.importc: "k_ticks_to_ns_floor64", header: "<kernel.h>".} + elif defined(windows): proc QueryPerformanceCounter(res: var uint64) {. importc: "QueryPerformanceCounter", stdcall, dynlib: "kernel32".} @@ -101,6 +102,9 @@ proc getMonoTime*(): MonoTime {.tags: [TimeEffect].} = mach_timebase_info(machAbsoluteTimeFreq) result = MonoTime(ticks: ticks * machAbsoluteTimeFreq.numer div machAbsoluteTimeFreq.denom) + elif defined(zephyr): + let ticks = k_ticks_to_ns_floor64(k_uptime_ticks()) + result = MonoTime(ticks: ticks) elif defined(posix): var ts: Timespec discard clock_gettime(CLOCK_MONOTONIC, ts) diff --git a/lib/std/objectdollar.nim b/lib/std/objectdollar.nim new file mode 100644 index 0000000000..86ce9afc82 --- /dev/null +++ b/lib/std/objectdollar.nim @@ -0,0 +1,13 @@ +## This module implements a generic `$` operator to convert objects to strings. + +import std/private/miscdollars + +proc `$`*[T: object](x: T): string = + ## Generic `$` operator for objects with similar output to + ## `$` for named tuples. + runnableExamples: + type Foo = object + a, b: int + let x = Foo(a: 23, b: 45) + assert $x == "(a: 23, b: 45)" + tupleObjectDollar(result, x) diff --git a/lib/pure/includes/oserr.nim b/lib/std/oserrors.nim similarity index 57% rename from lib/pure/includes/oserr.nim rename to lib/std/oserrors.nim index b0740cbe60..7b11c5e8e4 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/std/oserrors.nim @@ -1,16 +1,28 @@ -# Include file that implements 'osErrorMsg' and friends. Do not import it! +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# -when not declared(os) and not declared(ospaths): - {.error: "This is an include file for os.nim!".} + +## The `std/oserrors` module implements OS error reporting. + +type + OSErrorCode* = distinct int32 ## Specifies an OS Error Code. when not defined(nimscript): - var errno {.importc, header: "<errno.h>".}: cint - - proc c_strerror(errnum: cint): cstring {. - importc: "strerror", header: "<string.h>".} - when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + else: + var errno {.importc, header: "<errno.h>".}: cint + + proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "<string.h>".} proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} proc `$`*(err: OSErrorCode): string {.borrow.} @@ -18,18 +30,14 @@ proc `$`*(err: OSErrorCode): string {.borrow.} proc osErrorMsg*(errorCode: OSErrorCode): string = ## Converts an OS error code into a human readable string. ## - ## The error code can be retrieved using the `osLastError proc <#osLastError>`_. + ## The error code can be retrieved using the `osLastError proc`_. ## ## If conversion fails, or `errorCode` is `0` then `""` will be ## returned. ## - ## On Windows, the `-d:useWinAnsi` compilation flag can be used to - ## make this procedure use the non-unicode Win API calls to retrieve the - ## message. - ## ## See also: - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ - ## * `osLastError proc <#osLastError>`_ + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ runnableExamples: when defined(linux): assert osErrorMsg(OSErrorCode(0)) == "" @@ -41,18 +49,11 @@ proc osErrorMsg*(errorCode: OSErrorCode): string = discard elif defined(windows): if errorCode != OSErrorCode(0'i32): - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: + result = $msgbuf + if msgbuf != nil: localFree(cast[pointer](msgbuf)) else: if errorCode != OSErrorCode(0'i32): result = $c_strerror(errorCode.int32) @@ -63,33 +64,30 @@ proc newOSError*( ## Creates a new `OSError exception <system.html#OSError>`_. ## ## The `errorCode` will determine the - ## message, `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ will be used + ## message, `osErrorMsg proc`_ will be used ## to get this message. ## - ## The error code can be retrieved using the `osLastError proc - ## <#osLastError>`_. + ## The error code can be retrieved using the `osLastError proc`_. ## ## If the error code is `0` or an error message could not be retrieved, ## the message `unknown OS error` will be used. ## ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `osLastError proc <#osLastError>`_ - var e: owned(ref OSError); new(e) - e.errorCode = errorCode.int32 - e.msg = osErrorMsg(errorCode) + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ + result = (ref OSError)(errorCode: errorCode.int32, msg: osErrorMsg(errorCode)) if additionalInfo.len > 0: - if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.addQuoted additionalInfo - if e.msg == "": - e.msg = "unknown OS error" - return e + if result.msg.len > 0 and result.msg[^1] != '\n': result.msg.add '\n' + result.msg.add "Additional info: " + result.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. + if result.msg == "": + result.msg = "unknown OS error" proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = ## Raises an `OSError exception <system.html#OSError>`_. ## - ## Read the description of the `newOSError proc <#newOSError,OSErrorCode,string>`_ to learn + ## Read the description of the `newOSError proc`_ to learn ## how the exception object is created. raise newOSError(errorCode, additionalInfo) @@ -108,8 +106,8 @@ proc osLastError*(): OSErrorCode {.sideEffect.} = ## immediately after an OS call fails. On POSIX systems this is not a problem. ## ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ when defined(nimscript): discard elif defined(windows): diff --git a/lib/std/outparams.nim b/lib/std/outparams.nim new file mode 100644 index 0000000000..a471fbaa72 --- /dev/null +++ b/lib/std/outparams.nim @@ -0,0 +1,38 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## `outParamsAt` macro for easy writing code that works with both 2.0 and 1.x. + +import std/macros + +macro outParamsAt*(positions: static openArray[int]; n: untyped): untyped = + ## Use this macro to annotate `out` parameters in a portable way. + runnableExamples: + proc p(x: var int) {.outParamsAt: [1].} = + discard "x is really an 'out int' if the Nim compiler supports 'out' parameters" + + result = n + when defined(nimHasOutParams): + var p = n.params + for po in positions: + p[po][^2].expectKind nnkVarTy + p[po][^2] = newTree(nnkOutTy, p[po][^2][0]) + +when isMainModule: + {.experimental: "strictDefs".} + + proc main(x: var int) {.outParamsAt: [1].} = + x = 3 + + proc us = + var x: int + main x + echo x + + us() diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim index 2b5896be09..3320558f2a 100644 --- a/lib/std/packedsets.nim +++ b/lib/std/packedsets.nim @@ -12,17 +12,15 @@ ## ## Supports any Ordinal type. ## -## **Note**: Currently the assignment operator `=` for `PackedSet[A]` -## performs some rather meaningless shallow copy. Since Nim currently does -## not allow the assignment operator to be overloaded, use the `assign proc -## <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy. -## ## See also ## ======== ## * `sets module <sets.html>`_ for more general hash sets import std/private/since -import hashes +import std/hashes + +when defined(nimPreviewSlimSystem): + import std/assertions type BitScalar = uint @@ -38,7 +36,7 @@ const IntMask = 1 shl IntShift - 1 type - Trunk = ref object + Trunk {.acyclic.} = ref object next: Trunk # all nodes are connected with this pointer key: int # start address at bit 0 bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector @@ -111,7 +109,6 @@ proc intSetPut[A](t: var PackedSet[A], key: int): Trunk = t.data[h] = result proc bitincl[A](s: var PackedSet[A], key: int) {.inline.} = - var ret: Trunk var t = intSetPut(s, key shr TrunkShift) var u = key and TrunkMask t.bits[u shr IntShift] = t.bits[u shr IntShift] or @@ -201,6 +198,7 @@ proc contains*[A](s: PackedSet[A], key: A): bool = assert B notin letters if s.elems <= s.a.len: + result = false for i in 0..<s.elems: if s.a[i] == ord(key): return true else: @@ -409,18 +407,9 @@ proc isNil*[A](x: PackedSet[A]): bool {.inline.} = x.head.isNil and x.elems == 0 -proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = +proc `=copy`*[A](dest: var PackedSet[A], src: PackedSet[A]) = ## Copies `src` to `dest`. ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. - runnableExamples: - var - a = initPackedSet[int]() - b = initPackedSet[int]() - b.incl(5) - b.incl(7) - a.assign(b) - assert len(a) == 2 - if src.elems <= src.a.len: dest.data = @[] dest.max = 0 @@ -449,6 +438,19 @@ proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = dest.data[h] = n it = it.next +proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) {.inline, deprecated.} = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + runnableExamples: + var + a = initPackedSet[int]() + b = initPackedSet[int]() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + `=copy`(dest, src) + proc union*[A](s1, s2: PackedSet[A]): PackedSet[A] = ## Returns the union of the sets `s1` and `s2`. ## diff --git a/lib/std/paths.nim b/lib/std/paths.nim new file mode 100644 index 0000000000..664dedd31f --- /dev/null +++ b/lib/std/paths.nim @@ -0,0 +1,302 @@ +## This module implements path handling. +## +## **See also:** +## * `files module <files.html>`_ for file access + +import std/private/osseps +export osseps + +import std/envvars +import std/private/osappdirs + +import std/[pathnorm, hashes, sugar, strutils] + +from std/private/ospaths2 import joinPath, splitPath, + ReadDirEffect, WriteDirEffect, + isAbsolute, relativePath, + normalizePathEnd, isRelativeTo, parentDir, + tailDir, isRootDir, parentDirs, `/../`, + extractFilename, lastPathPart, + changeFileExt, addFileExt, cmpPaths, splitFile, + unixToNativePath, absolutePath, normalizeExe, + normalizePath +export ReadDirEffect, WriteDirEffect + +type + Path* = distinct string + +func hash*(x: Path): Hash = + let x = x.string.dup(normalizePath) + if FileSystemCaseSensitive: + result = x.hash + else: + result = x.toLowerAscii.hash + +template `$`*(x: Path): string = + string(x) + +func `==`*(x, y: Path): bool {.inline.} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. + result = cmpPaths(x.string, y.string) == 0 + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +func add(x: var string, tail: string) = + var state = 0 + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and x.endsWith({DirSep, AltSep}) + normalizePathEnd(x, trailingSep=false) + addNormalizePath(tail, x, state, DirSep) + normalizePathEnd(x, trailingSep=trailingSep) + +func add*(x: var Path, y: Path) {.borrow.} + +func `/`*(head, tail: Path): Path {.inline.} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + Path(joinPath(head.string, tail.string)) + +func splitPath*(path: Path): tuple[head, tail: Path] {.inline.} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `add proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + let res = splitPath(path.string) + result = (Path(res.head), Path(res.tail)) + +func splitFile*(path: Path): tuple[dir, name: Path, ext: string] {.inline.} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + let res = splitFile(path.string) + result = (Path(res.dir), Path(res.name), res.ext) + +func isAbsolute*(path: Path): bool {.inline, raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + result = isAbsolute(path.string) + +proc relativePath*(path, base: Path, sep = DirSep): Path {.inline.} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + result = Path(relativePath(path.string, base.string, sep)) + +proc isRelativeTo*(path: Path, base: Path): bool {.inline.} = + ## Returns true if `path` is relative to `base`. + result = isRelativeTo(path.string, base.string) + + +func parentDir*(path: Path): Path {.inline.} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + result = Path(parentDir(path.string)) + +func tailDir*(path: Path): Path {.inline.} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + result = Path(tailDir(path.string)) + +func isRootDir*(path: Path): bool {.inline.} = + ## Checks whether a given `path` is a root directory. + result = isRootDir(path.string) + +iterator parentDirs*(path: Path, fromRoot=false, inclusive=true): Path = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + for p in parentDirs(path.string, fromRoot, inclusive): + yield Path(p) + +func `/../`*(head, tail: Path): Path {.inline.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + Path(`/../`(head.string, tail.string)) + +func extractFilename*(path: Path): Path {.inline.} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(extractFilename(path.string)) + +func lastPathPart*(path: Path): Path {.inline.} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(lastPathPart(path.string)) + +func changeFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + result = Path(changeFileExt(filename.string, ext)) + +func addFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + result = Path(addFileExt(filename.string, ext)) + +func unixToNativePath*(path: Path, drive=Path("")): Path {.inline.} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + result = Path(unixToNativePath(path.string, drive.string)) + +proc getCurrentDir*(): Path {.inline, tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc <appdirs.html#getHomeDir>`_ + ## * `getConfigDir proc <appdirs.html#getConfigDir>`_ + ## * `getTempDir proc <appdirs.html#getTempDir>`_ + ## * `setCurrentDir proc <dirs.html#setCurrentDir>`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + result = Path(ospaths2.getCurrentDir()) + +proc normalizeExe*(file: var Path) {.borrow.} + +proc normalizePath*(path: var Path) {.borrow.} + +proc normalizePathEnd*(path: var Path, trailingSep = false) {.borrow.} + +proc absolutePath*(path: Path, root = getCurrentDir()): Path = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizePath proc`_ + result = Path(absolutePath(path.string, root.string)) + +proc expandTildeImpl(path: string): string {. + tags: [ReadEnvEffect, ReadIOEffect].} = + if len(path) == 0 or path[0] != '~': + result = path + elif len(path) == 1: + result = getHomeDir() + elif (path[1] in {DirSep, AltSep}): + result = joinPath(getHomeDir(), path.substr(2)) + else: + # TODO: handle `~bob` and `~bob/` which means home of bob + result = path + +proc expandTilde*(path: Path): Path {.inline, + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing + ## ``~`` with `getHomeDir() <appdirs.html#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## + ## Windows: this is still supported despite the Windows platform not having this + ## convention; also, both ``~/`` and ``~\`` are handled. + runnableExamples: + import std/appdirs + assert expandTilde(Path("~") / Path("appname.cfg")) == getHomeDir() / Path("appname.cfg") + assert expandTilde(Path("~/foo/bar")) == getHomeDir() / Path("foo/bar") + assert expandTilde(Path("/foo/bar")) == Path("/foo/bar") + result = Path(expandTildeImpl(path.string)) diff --git a/lib/std/private/bitops_utils.nim b/lib/std/private/bitops_utils.nim new file mode 100644 index 0000000000..0b94844167 --- /dev/null +++ b/lib/std/private/bitops_utils.nim @@ -0,0 +1,22 @@ +template forwardImpl*(impl, arg) {.dirty.} = + when sizeof(x) <= 4: + when x is SomeSignedInt: + impl(cast[uint32](x.int32)) + else: + impl(x.uint32) + else: + when x is SomeSignedInt: + impl(cast[uint64](x.int64)) + else: + impl(x.uint64) + +# this could also be implemented via: +# import std/typetraits +# template castToUnsigned*(x: SomeInteger): auto = cast[toUnsigned(typeof(x))](x) + +template castToUnsigned*(x: int8): uint8 = cast[uint8](x) +template castToUnsigned*(x: int16): uint16 = cast[uint16](x) +template castToUnsigned*(x: int32): uint32 = cast[uint32](x) +template castToUnsigned*(x: int64): uint64 = cast[uint64](x) +template castToUnsigned*(x: int): uint = cast[uint](x) +template castToUnsigned*[T: SomeUnsignedInt](x: T): T = x diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim new file mode 100644 index 0000000000..f2d0d25cba --- /dev/null +++ b/lib/std/private/digitsutils.nim @@ -0,0 +1,116 @@ +const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + +# Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c +# Generates: +# ```nim +# var res = "" +# for i in 0 .. 99: +# if i < 10: +# res.add "0" & $i +# else: +# res.add $i +# doAssert res == digits100 +# ``` + +proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) + +proc trailingZeros2Digits*(digits: uint32): int {.inline.} = + trailingZeros100[digits] + +when defined(js): + proc numToString(a: SomeInteger): cstring {.importjs: "((#) + \"\")".} + +func addChars[T](result: var string, x: T, start: int, n: int) {.inline.} = + let old = result.len + result.setLen old + n + template impl = + for i in 0..<n: result[old + i] = x[start + i] + when nimvm: impl + else: + when defined(js) or defined(nimscript): impl + else: + {.noSideEffect.}: + copyMem result[old].addr, x[start].unsafeAddr, n + +func addChars[T](result: var string, x: T) {.inline.} = + addChars(result, x, 0, x.len) + +func addIntImpl(result: var string, x: uint64) {.inline.} = + var tmp {.noinit.}: array[24, char] + var num = x + var next = tmp.len - 1 + const nbatch = 100 + + while num >= nbatch: + let originNum = num + num = num div nbatch + let index = int16((originNum - num * nbatch) shl 1) + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec(next, 2) + + # process last 1-2 digits + if num < 10: + tmp[next] = chr(ord('0') + num.uint8) + else: + let index = num * 2 + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec next + addChars(result, tmp, next, tmp.len - next) + +when not defined(nimHasEnforceNoRaises): + {.pragma: enforceNoRaises.} + +func addInt*(result: var string, x: uint64) {.enforceNoRaises.} = + when nimvm: addIntImpl(result, x) + else: + when not defined(js): addIntImpl(result, x) + else: + addChars(result, numToString(x)) + +proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = + ## Converts integer to its string representation and appends it to `result`. + runnableExamples: + var s = "foo" + s.addInt(45) + assert s == "foo45" + template impl = + var num: uint64 + if x < 0: + if x == low(int64): + num = cast[uint64](x) + else: + num = uint64(-x) + result.add '-' + else: + num = uint64(x) + addInt(result, num) + when nimvm: impl() + else: + when defined(js): + addChars(result, numToString(x)) + else: impl() + +proc addInt*(result: var string; x: int) {.inline, enforceNoRaises.} = + addInt(result, int64(x)) diff --git a/lib/std/private/dragonbox.nim b/lib/std/private/dragonbox.nim new file mode 100644 index 0000000000..85ffea84a2 --- /dev/null +++ b/lib/std/private/dragonbox.nim @@ -0,0 +1,1325 @@ +## Copyright 2020 Junekey Jeon +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +## char* output_end = Dtoa(buffer, value); +## +## Converts the given double-precision number into decimal form and stores the result in the given +## buffer. +## +## The buffer must be large enough, i.e. >= DtoaMinBufferLength. +## The output format is similar to printf("%g"). +## The output is _not_ null-terminted. +## +## The output is optimal, i.e. the output string +## 1. rounds back to the input number when read in (using round-to-nearest-even) +## 2. is as short as possible, +## 3. is as close to the input number as possible. +## +## Note: +## This function may temporarily write up to DtoaMinBufferLength characters into the buffer. + + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + +const + dtoaMinBufferLength*: cint = 64 + +## This file contains an implementation of Junekey Jeon's Dragonbox algorithm. +## +## It is a simplified version of the reference implementation found here: +## https://github.com/jk-jeon/dragonbox +## +## The reference implementation also works with single-precision floating-point numbers and +## has options to configure the rounding mode. + +template dragonbox_Assert*(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType* = float + BitsType* = uint64 + +type + Double* = object + bits*: BitsType + +const ## = p (includes the hidden bit) + significandSize*: int32 = 53 + +const ## static constexpr int32_t MaxExponent = 1024 - 1 - (SignificandSize - 1); + ## static constexpr int32_t MinExponent = std::numeric_limits<value_type>::min_exponent - 1 - (SignificandSize - 1); + exponentBias*: int32 = 1024 - 1 + (significandSize - 1) + +const + maxIeeeExponent*: BitsType = BitsType(2 * 1024 - 1) + +const ## = 2^(p-1) + hiddenBit*: BitsType = BitsType(1) shl (significandSize - 1) + +const ## = 2^(p-1) - 1 + significandMask*: BitsType = hiddenBit - 1 + +const + exponentMask*: BitsType = maxIeeeExponent shl (significandSize - 1) + +const + signMask*: BitsType = not (not BitsType(0) shr 1) + +proc constructDouble*(bits: BitsType): Double = + result.bits = bits + +proc constructDouble*(value: ValueType): Double = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand*(this: Double): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent*(this: Double): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero*(this: Double): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit*(this: Double): int {.noSideEffect.} = + return ord((this.bits and signMask) != 0) + + +# ================================================================================================== +# +# ================================================================================================== +## namespace +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2*(x: int32; n: int32): int32 {.inline.} = + return x shr n + +proc floorLog2Pow10*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1233) + dragonbox_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +proc floorLog10Pow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611, 22) + +proc floorLog10ThreeQuartersPow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611 - 524031, 22) + +# ================================================================================================== +# +# ================================================================================================== + +type + uint64x2* {.bycopy.} = object + hi*: uint64 + lo*: uint64 + + +proc computePow10*(k: int32): uint64x2 {.inline.} = + const + kMin: int32 = -292 + const + kMax: int32 = 326 + const + pow10: array[kMax - kMin + 1, uint64x2] = [ + uint64x2(hi: 0xFF77B1FCBEBCDC4F'u, lo: 0x25E8E89C13BB0F7B'u), + uint64x2(hi: 0x9FAACF3DF73609B1'u, lo: 0x77B191618C54E9AD'u), + uint64x2(hi: 0xC795830D75038C1D'u, lo: 0xD59DF5B9EF6A2418'u), + uint64x2(hi: 0xF97AE3D0D2446F25'u, lo: 0x4B0573286B44AD1E'u), + uint64x2(hi: 0x9BECCE62836AC577'u, lo: 0x4EE367F9430AEC33'u), + uint64x2(hi: 0xC2E801FB244576D5'u, lo: 0x229C41F793CDA740'u), + uint64x2(hi: 0xF3A20279ED56D48A'u, lo: 0x6B43527578C11110'u), + uint64x2(hi: 0x9845418C345644D6'u, lo: 0x830A13896B78AAAA'u), + uint64x2(hi: 0xBE5691EF416BD60C'u, lo: 0x23CC986BC656D554'u), + uint64x2(hi: 0xEDEC366B11C6CB8F'u, lo: 0x2CBFBE86B7EC8AA9'u), + uint64x2(hi: 0x94B3A202EB1C3F39'u, lo: 0x7BF7D71432F3D6AA'u), + uint64x2(hi: 0xB9E08A83A5E34F07'u, lo: 0xDAF5CCD93FB0CC54'u), + uint64x2(hi: 0xE858AD248F5C22C9'u, lo: 0xD1B3400F8F9CFF69'u), + uint64x2(hi: 0x91376C36D99995BE'u, lo: 0x23100809B9C21FA2'u), + uint64x2(hi: 0xB58547448FFFFB2D'u, lo: 0xABD40A0C2832A78B'u), + uint64x2(hi: 0xE2E69915B3FFF9F9'u, lo: 0x16C90C8F323F516D'u), + uint64x2(hi: 0x8DD01FAD907FFC3B'u, lo: 0xAE3DA7D97F6792E4'u), + uint64x2(hi: 0xB1442798F49FFB4A'u, lo: 0x99CD11CFDF41779D'u), + uint64x2(hi: 0xDD95317F31C7FA1D'u, lo: 0x40405643D711D584'u), + uint64x2(hi: 0x8A7D3EEF7F1CFC52'u, lo: 0x482835EA666B2573'u), + uint64x2(hi: 0xAD1C8EAB5EE43B66'u, lo: 0xDA3243650005EED0'u), + uint64x2(hi: 0xD863B256369D4A40'u, lo: 0x90BED43E40076A83'u), + uint64x2(hi: 0x873E4F75E2224E68'u, lo: 0x5A7744A6E804A292'u), + uint64x2(hi: 0xA90DE3535AAAE202'u, lo: 0x711515D0A205CB37'u), + uint64x2(hi: 0xD3515C2831559A83'u, lo: 0x0D5A5B44CA873E04'u), + uint64x2(hi: 0x8412D9991ED58091'u, lo: 0xE858790AFE9486C3'u), + uint64x2(hi: 0xA5178FFF668AE0B6'u, lo: 0x626E974DBE39A873'u), + uint64x2(hi: 0xCE5D73FF402D98E3'u, lo: 0xFB0A3D212DC81290'u), + uint64x2(hi: 0x80FA687F881C7F8E'u, lo: 0x7CE66634BC9D0B9A'u), + uint64x2(hi: 0xA139029F6A239F72'u, lo: 0x1C1FFFC1EBC44E81'u), + uint64x2(hi: 0xC987434744AC874E'u, lo: 0xA327FFB266B56221'u), + uint64x2(hi: 0xFBE9141915D7A922'u, lo: 0x4BF1FF9F0062BAA9'u), + uint64x2(hi: 0x9D71AC8FADA6C9B5'u, lo: 0x6F773FC3603DB4AA'u), + uint64x2(hi: 0xC4CE17B399107C22'u, lo: 0xCB550FB4384D21D4'u), + uint64x2(hi: 0xF6019DA07F549B2B'u, lo: 0x7E2A53A146606A49'u), + uint64x2(hi: 0x99C102844F94E0FB'u, lo: 0x2EDA7444CBFC426E'u), + uint64x2(hi: 0xC0314325637A1939'u, lo: 0xFA911155FEFB5309'u), + uint64x2(hi: 0xF03D93EEBC589F88'u, lo: 0x793555AB7EBA27CB'u), + uint64x2(hi: 0x96267C7535B763B5'u, lo: 0x4BC1558B2F3458DF'u), + uint64x2(hi: 0xBBB01B9283253CA2'u, lo: 0x9EB1AAEDFB016F17'u), + uint64x2(hi: 0xEA9C227723EE8BCB'u, lo: 0x465E15A979C1CADD'u), + uint64x2(hi: 0x92A1958A7675175F'u, lo: 0x0BFACD89EC191ECA'u), + uint64x2(hi: 0xB749FAED14125D36'u, lo: 0xCEF980EC671F667C'u), + uint64x2(hi: 0xE51C79A85916F484'u, lo: 0x82B7E12780E7401B'u), + uint64x2(hi: 0x8F31CC0937AE58D2'u, lo: 0xD1B2ECB8B0908811'u), + uint64x2(hi: 0xB2FE3F0B8599EF07'u, lo: 0x861FA7E6DCB4AA16'u), + uint64x2(hi: 0xDFBDCECE67006AC9'u, lo: 0x67A791E093E1D49B'u), + uint64x2(hi: 0x8BD6A141006042BD'u, lo: 0xE0C8BB2C5C6D24E1'u), + uint64x2(hi: 0xAECC49914078536D'u, lo: 0x58FAE9F773886E19'u), + uint64x2(hi: 0xDA7F5BF590966848'u, lo: 0xAF39A475506A899F'u), + uint64x2(hi: 0x888F99797A5E012D'u, lo: 0x6D8406C952429604'u), + uint64x2(hi: 0xAAB37FD7D8F58178'u, lo: 0xC8E5087BA6D33B84'u), + uint64x2(hi: 0xD5605FCDCF32E1D6'u, lo: 0xFB1E4A9A90880A65'u), + uint64x2(hi: 0x855C3BE0A17FCD26'u, lo: 0x5CF2EEA09A550680'u), + uint64x2(hi: 0xA6B34AD8C9DFC06F'u, lo: 0xF42FAA48C0EA481F'u), + uint64x2(hi: 0xD0601D8EFC57B08B'u, lo: 0xF13B94DAF124DA27'u), + uint64x2(hi: 0x823C12795DB6CE57'u, lo: 0x76C53D08D6B70859'u), + uint64x2(hi: 0xA2CB1717B52481ED'u, lo: 0x54768C4B0C64CA6F'u), + uint64x2(hi: 0xCB7DDCDDA26DA268'u, lo: 0xA9942F5DCF7DFD0A'u), + uint64x2(hi: 0xFE5D54150B090B02'u, lo: 0xD3F93B35435D7C4D'u), + uint64x2(hi: 0x9EFA548D26E5A6E1'u, lo: 0xC47BC5014A1A6DB0'u), + uint64x2(hi: 0xC6B8E9B0709F109A'u, lo: 0x359AB6419CA1091C'u), + uint64x2(hi: 0xF867241C8CC6D4C0'u, lo: 0xC30163D203C94B63'u), + uint64x2(hi: 0x9B407691D7FC44F8'u, lo: 0x79E0DE63425DCF1E'u), + uint64x2(hi: 0xC21094364DFB5636'u, lo: 0x985915FC12F542E5'u), + uint64x2(hi: 0xF294B943E17A2BC4'u, lo: 0x3E6F5B7B17B2939E'u), + uint64x2(hi: 0x979CF3CA6CEC5B5A'u, lo: 0xA705992CEECF9C43'u), + uint64x2(hi: 0xBD8430BD08277231'u, lo: 0x50C6FF782A838354'u), + uint64x2(hi: 0xECE53CEC4A314EBD'u, lo: 0xA4F8BF5635246429'u), + uint64x2(hi: 0x940F4613AE5ED136'u, lo: 0x871B7795E136BE9A'u), + uint64x2(hi: 0xB913179899F68584'u, lo: 0x28E2557B59846E40'u), + uint64x2(hi: 0xE757DD7EC07426E5'u, lo: 0x331AEADA2FE589D0'u), + uint64x2(hi: 0x9096EA6F3848984F'u, lo: 0x3FF0D2C85DEF7622'u), + uint64x2(hi: 0xB4BCA50B065ABE63'u, lo: 0x0FED077A756B53AA'u), + uint64x2(hi: 0xE1EBCE4DC7F16DFB'u, lo: 0xD3E8495912C62895'u), + uint64x2(hi: 0x8D3360F09CF6E4BD'u, lo: 0x64712DD7ABBBD95D'u), + uint64x2(hi: 0xB080392CC4349DEC'u, lo: 0xBD8D794D96AACFB4'u), + uint64x2(hi: 0xDCA04777F541C567'u, lo: 0xECF0D7A0FC5583A1'u), + uint64x2(hi: 0x89E42CAAF9491B60'u, lo: 0xF41686C49DB57245'u), + uint64x2(hi: 0xAC5D37D5B79B6239'u, lo: 0x311C2875C522CED6'u), + uint64x2(hi: 0xD77485CB25823AC7'u, lo: 0x7D633293366B828C'u), + uint64x2(hi: 0x86A8D39EF77164BC'u, lo: 0xAE5DFF9C02033198'u), + uint64x2(hi: 0xA8530886B54DBDEB'u, lo: 0xD9F57F830283FDFD'u), + uint64x2(hi: 0xD267CAA862A12D66'u, lo: 0xD072DF63C324FD7C'u), + uint64x2(hi: 0x8380DEA93DA4BC60'u, lo: 0x4247CB9E59F71E6E'u), + uint64x2(hi: 0xA46116538D0DEB78'u, lo: 0x52D9BE85F074E609'u), + uint64x2(hi: 0xCD795BE870516656'u, lo: 0x67902E276C921F8C'u), + uint64x2(hi: 0x806BD9714632DFF6'u, lo: 0x00BA1CD8A3DB53B7'u), + uint64x2(hi: 0xA086CFCD97BF97F3'u, lo: 0x80E8A40ECCD228A5'u), + uint64x2(hi: 0xC8A883C0FDAF7DF0'u, lo: 0x6122CD128006B2CE'u), + uint64x2(hi: 0xFAD2A4B13D1B5D6C'u, lo: 0x796B805720085F82'u), + uint64x2(hi: 0x9CC3A6EEC6311A63'u, lo: 0xCBE3303674053BB1'u), + uint64x2(hi: 0xC3F490AA77BD60FC'u, lo: 0xBEDBFC4411068A9D'u), + uint64x2(hi: 0xF4F1B4D515ACB93B'u, lo: 0xEE92FB5515482D45'u), + uint64x2(hi: 0x991711052D8BF3C5'u, lo: 0x751BDD152D4D1C4B'u), + uint64x2(hi: 0xBF5CD54678EEF0B6'u, lo: 0xD262D45A78A0635E'u), + uint64x2(hi: 0xEF340A98172AACE4'u, lo: 0x86FB897116C87C35'u), + uint64x2(hi: 0x9580869F0E7AAC0E'u, lo: 0xD45D35E6AE3D4DA1'u), + uint64x2(hi: 0xBAE0A846D2195712'u, lo: 0x8974836059CCA10A'u), + uint64x2(hi: 0xE998D258869FACD7'u, lo: 0x2BD1A438703FC94C'u), + uint64x2(hi: 0x91FF83775423CC06'u, lo: 0x7B6306A34627DDD0'u), + uint64x2(hi: 0xB67F6455292CBF08'u, lo: 0x1A3BC84C17B1D543'u), + uint64x2(hi: 0xE41F3D6A7377EECA'u, lo: 0x20CABA5F1D9E4A94'u), + uint64x2(hi: 0x8E938662882AF53E'u, lo: 0x547EB47B7282EE9D'u), + uint64x2(hi: 0xB23867FB2A35B28D'u, lo: 0xE99E619A4F23AA44'u), + uint64x2(hi: 0xDEC681F9F4C31F31'u, lo: 0x6405FA00E2EC94D5'u), + uint64x2(hi: 0x8B3C113C38F9F37E'u, lo: 0xDE83BC408DD3DD05'u), + uint64x2(hi: 0xAE0B158B4738705E'u, lo: 0x9624AB50B148D446'u), + uint64x2(hi: 0xD98DDAEE19068C76'u, lo: 0x3BADD624DD9B0958'u), + uint64x2(hi: 0x87F8A8D4CFA417C9'u, lo: 0xE54CA5D70A80E5D7'u), + uint64x2(hi: 0xA9F6D30A038D1DBC'u, lo: 0x5E9FCF4CCD211F4D'u), + uint64x2(hi: 0xD47487CC8470652B'u, lo: 0x7647C32000696720'u), + uint64x2(hi: 0x84C8D4DFD2C63F3B'u, lo: 0x29ECD9F40041E074'u), + uint64x2(hi: 0xA5FB0A17C777CF09'u, lo: 0xF468107100525891'u), + uint64x2(hi: 0xCF79CC9DB955C2CC'u, lo: 0x7182148D4066EEB5'u), + uint64x2(hi: 0x81AC1FE293D599BF'u, lo: 0xC6F14CD848405531'u), + uint64x2(hi: 0xA21727DB38CB002F'u, lo: 0xB8ADA00E5A506A7D'u), + uint64x2(hi: 0xCA9CF1D206FDC03B'u, lo: 0xA6D90811F0E4851D'u), + uint64x2(hi: 0xFD442E4688BD304A'u, lo: 0x908F4A166D1DA664'u), + uint64x2(hi: 0x9E4A9CEC15763E2E'u, lo: 0x9A598E4E043287FF'u), + uint64x2(hi: 0xC5DD44271AD3CDBA'u, lo: 0x40EFF1E1853F29FE'u), + uint64x2(hi: 0xF7549530E188C128'u, lo: 0xD12BEE59E68EF47D'u), + uint64x2(hi: 0x9A94DD3E8CF578B9'u, lo: 0x82BB74F8301958CF'u), + uint64x2(hi: 0xC13A148E3032D6E7'u, lo: 0xE36A52363C1FAF02'u), + uint64x2(hi: 0xF18899B1BC3F8CA1'u, lo: 0xDC44E6C3CB279AC2'u), + uint64x2(hi: 0x96F5600F15A7B7E5'u, lo: 0x29AB103A5EF8C0BA'u), + uint64x2(hi: 0xBCB2B812DB11A5DE'u, lo: 0x7415D448F6B6F0E8'u), + uint64x2(hi: 0xEBDF661791D60F56'u, lo: 0x111B495B3464AD22'u), + uint64x2(hi: 0x936B9FCEBB25C995'u, lo: 0xCAB10DD900BEEC35'u), + uint64x2(hi: 0xB84687C269EF3BFB'u, lo: 0x3D5D514F40EEA743'u), + uint64x2(hi: 0xE65829B3046B0AFA'u, lo: 0x0CB4A5A3112A5113'u), + uint64x2(hi: 0x8FF71A0FE2C2E6DC'u, lo: 0x47F0E785EABA72AC'u), + uint64x2(hi: 0xB3F4E093DB73A093'u, lo: 0x59ED216765690F57'u), + uint64x2(hi: 0xE0F218B8D25088B8'u, lo: 0x306869C13EC3532D'u), + uint64x2(hi: 0x8C974F7383725573'u, lo: 0x1E414218C73A13FC'u), + uint64x2(hi: 0xAFBD2350644EEACF'u, lo: 0xE5D1929EF90898FB'u), + uint64x2(hi: 0xDBAC6C247D62A583'u, lo: 0xDF45F746B74ABF3A'u), + uint64x2(hi: 0x894BC396CE5DA772'u, lo: 0x6B8BBA8C328EB784'u), + uint64x2(hi: 0xAB9EB47C81F5114F'u, lo: 0x066EA92F3F326565'u), + uint64x2(hi: 0xD686619BA27255A2'u, lo: 0xC80A537B0EFEFEBE'u), + uint64x2(hi: 0x8613FD0145877585'u, lo: 0xBD06742CE95F5F37'u), + uint64x2(hi: 0xA798FC4196E952E7'u, lo: 0x2C48113823B73705'u), + uint64x2(hi: 0xD17F3B51FCA3A7A0'u, lo: 0xF75A15862CA504C6'u), + uint64x2(hi: 0x82EF85133DE648C4'u, lo: 0x9A984D73DBE722FC'u), + uint64x2(hi: 0xA3AB66580D5FDAF5'u, lo: 0xC13E60D0D2E0EBBB'u), + uint64x2(hi: 0xCC963FEE10B7D1B3'u, lo: 0x318DF905079926A9'u), + uint64x2(hi: 0xFFBBCFE994E5C61F'u, lo: 0xFDF17746497F7053'u), + uint64x2(hi: 0x9FD561F1FD0F9BD3'u, lo: 0xFEB6EA8BEDEFA634'u), + uint64x2(hi: 0xC7CABA6E7C5382C8'u, lo: 0xFE64A52EE96B8FC1'u), + uint64x2(hi: 0xF9BD690A1B68637B'u, lo: 0x3DFDCE7AA3C673B1'u), + uint64x2(hi: 0x9C1661A651213E2D'u, lo: 0x06BEA10CA65C084F'u), + uint64x2(hi: 0xC31BFA0FE5698DB8'u, lo: 0x486E494FCFF30A63'u), + uint64x2(hi: 0xF3E2F893DEC3F126'u, lo: 0x5A89DBA3C3EFCCFB'u), + uint64x2(hi: 0x986DDB5C6B3A76B7'u, lo: 0xF89629465A75E01D'u), + uint64x2(hi: 0xBE89523386091465'u, lo: 0xF6BBB397F1135824'u), + uint64x2(hi: 0xEE2BA6C0678B597F'u, lo: 0x746AA07DED582E2D'u), + uint64x2(hi: 0x94DB483840B717EF'u, lo: 0xA8C2A44EB4571CDD'u), + uint64x2(hi: 0xBA121A4650E4DDEB'u, lo: 0x92F34D62616CE414'u), + uint64x2(hi: 0xE896A0D7E51E1566'u, lo: 0x77B020BAF9C81D18'u), + uint64x2(hi: 0x915E2486EF32CD60'u, lo: 0x0ACE1474DC1D122F'u), + uint64x2(hi: 0xB5B5ADA8AAFF80B8'u, lo: 0x0D819992132456BB'u), + uint64x2(hi: 0xE3231912D5BF60E6'u, lo: 0x10E1FFF697ED6C6A'u), + uint64x2(hi: 0x8DF5EFABC5979C8F'u, lo: 0xCA8D3FFA1EF463C2'u), + uint64x2(hi: 0xB1736B96B6FD83B3'u, lo: 0xBD308FF8A6B17CB3'u), + uint64x2(hi: 0xDDD0467C64BCE4A0'u, lo: 0xAC7CB3F6D05DDBDF'u), + uint64x2(hi: 0x8AA22C0DBEF60EE4'u, lo: 0x6BCDF07A423AA96C'u), + uint64x2(hi: 0xAD4AB7112EB3929D'u, lo: 0x86C16C98D2C953C7'u), + uint64x2(hi: 0xD89D64D57A607744'u, lo: 0xE871C7BF077BA8B8'u), + uint64x2(hi: 0x87625F056C7C4A8B'u, lo: 0x11471CD764AD4973'u), + uint64x2(hi: 0xA93AF6C6C79B5D2D'u, lo: 0xD598E40D3DD89BD0'u), + uint64x2(hi: 0xD389B47879823479'u, lo: 0x4AFF1D108D4EC2C4'u), + uint64x2(hi: 0x843610CB4BF160CB'u, lo: 0xCEDF722A585139BB'u), + uint64x2(hi: 0xA54394FE1EEDB8FE'u, lo: 0xC2974EB4EE658829'u), + uint64x2(hi: 0xCE947A3DA6A9273E'u, lo: 0x733D226229FEEA33'u), + uint64x2(hi: 0x811CCC668829B887'u, lo: 0x0806357D5A3F5260'u), + uint64x2(hi: 0xA163FF802A3426A8'u, lo: 0xCA07C2DCB0CF26F8'u), + uint64x2(hi: 0xC9BCFF6034C13052'u, lo: 0xFC89B393DD02F0B6'u), + uint64x2(hi: 0xFC2C3F3841F17C67'u, lo: 0xBBAC2078D443ACE3'u), + uint64x2(hi: 0x9D9BA7832936EDC0'u, lo: 0xD54B944B84AA4C0E'u), + uint64x2(hi: 0xC5029163F384A931'u, lo: 0x0A9E795E65D4DF12'u), + uint64x2(hi: 0xF64335BCF065D37D'u, lo: 0x4D4617B5FF4A16D6'u), + uint64x2(hi: 0x99EA0196163FA42E'u, lo: 0x504BCED1BF8E4E46'u), + uint64x2(hi: 0xC06481FB9BCF8D39'u, lo: 0xE45EC2862F71E1D7'u), + uint64x2(hi: 0xF07DA27A82C37088'u, lo: 0x5D767327BB4E5A4D'u), + uint64x2(hi: 0x964E858C91BA2655'u, lo: 0x3A6A07F8D510F870'u), + uint64x2(hi: 0xBBE226EFB628AFEA'u, lo: 0x890489F70A55368C'u), + uint64x2(hi: 0xEADAB0ABA3B2DBE5'u, lo: 0x2B45AC74CCEA842F'u), + uint64x2(hi: 0x92C8AE6B464FC96F'u, lo: 0x3B0B8BC90012929E'u), + uint64x2(hi: 0xB77ADA0617E3BBCB'u, lo: 0x09CE6EBB40173745'u), + uint64x2(hi: 0xE55990879DDCAABD'u, lo: 0xCC420A6A101D0516'u), + uint64x2(hi: 0x8F57FA54C2A9EAB6'u, lo: 0x9FA946824A12232E'u), + uint64x2(hi: 0xB32DF8E9F3546564'u, lo: 0x47939822DC96ABFA'u), + uint64x2(hi: 0xDFF9772470297EBD'u, lo: 0x59787E2B93BC56F8'u), + uint64x2(hi: 0x8BFBEA76C619EF36'u, lo: 0x57EB4EDB3C55B65B'u), + uint64x2(hi: 0xAEFAE51477A06B03'u, lo: 0xEDE622920B6B23F2'u), + uint64x2(hi: 0xDAB99E59958885C4'u, lo: 0xE95FAB368E45ECEE'u), + uint64x2(hi: 0x88B402F7FD75539B'u, lo: 0x11DBCB0218EBB415'u), + uint64x2(hi: 0xAAE103B5FCD2A881'u, lo: 0xD652BDC29F26A11A'u), + uint64x2(hi: 0xD59944A37C0752A2'u, lo: 0x4BE76D3346F04960'u), + uint64x2(hi: 0x857FCAE62D8493A5'u, lo: 0x6F70A4400C562DDC'u), + uint64x2(hi: 0xA6DFBD9FB8E5B88E'u, lo: 0xCB4CCD500F6BB953'u), + uint64x2(hi: 0xD097AD07A71F26B2'u, lo: 0x7E2000A41346A7A8'u), + uint64x2(hi: 0x825ECC24C873782F'u, lo: 0x8ED400668C0C28C9'u), + uint64x2(hi: 0xA2F67F2DFA90563B'u, lo: 0x728900802F0F32FB'u), + uint64x2(hi: 0xCBB41EF979346BCA'u, lo: 0x4F2B40A03AD2FFBA'u), + uint64x2(hi: 0xFEA126B7D78186BC'u, lo: 0xE2F610C84987BFA9'u), + uint64x2(hi: 0x9F24B832E6B0F436'u, lo: 0x0DD9CA7D2DF4D7CA'u), + uint64x2(hi: 0xC6EDE63FA05D3143'u, lo: 0x91503D1C79720DBC'u), + uint64x2(hi: 0xF8A95FCF88747D94'u, lo: 0x75A44C6397CE912B'u), + uint64x2(hi: 0x9B69DBE1B548CE7C'u, lo: 0xC986AFBE3EE11ABB'u), + uint64x2(hi: 0xC24452DA229B021B'u, lo: 0xFBE85BADCE996169'u), + uint64x2(hi: 0xF2D56790AB41C2A2'u, lo: 0xFAE27299423FB9C4'u), + uint64x2(hi: 0x97C560BA6B0919A5'u, lo: 0xDCCD879FC967D41B'u), + uint64x2(hi: 0xBDB6B8E905CB600F'u, lo: 0x5400E987BBC1C921'u), + uint64x2(hi: 0xED246723473E3813'u, lo: 0x290123E9AAB23B69'u), + uint64x2(hi: 0x9436C0760C86E30B'u, lo: 0xF9A0B6720AAF6522'u), + uint64x2(hi: 0xB94470938FA89BCE'u, lo: 0xF808E40E8D5B3E6A'u), + uint64x2(hi: 0xE7958CB87392C2C2'u, lo: 0xB60B1D1230B20E05'u), + uint64x2(hi: 0x90BD77F3483BB9B9'u, lo: 0xB1C6F22B5E6F48C3'u), + uint64x2(hi: 0xB4ECD5F01A4AA828'u, lo: 0x1E38AEB6360B1AF4'u), + uint64x2(hi: 0xE2280B6C20DD5232'u, lo: 0x25C6DA63C38DE1B1'u), + uint64x2(hi: 0x8D590723948A535F'u, lo: 0x579C487E5A38AD0F'u), + uint64x2(hi: 0xB0AF48EC79ACE837'u, lo: 0x2D835A9DF0C6D852'u), + uint64x2(hi: 0xDCDB1B2798182244'u, lo: 0xF8E431456CF88E66'u), + uint64x2(hi: 0x8A08F0F8BF0F156B'u, lo: 0x1B8E9ECB641B5900'u), + uint64x2(hi: 0xAC8B2D36EED2DAC5'u, lo: 0xE272467E3D222F40'u), + uint64x2(hi: 0xD7ADF884AA879177'u, lo: 0x5B0ED81DCC6ABB10'u), + uint64x2(hi: 0x86CCBB52EA94BAEA'u, lo: 0x98E947129FC2B4EA'u), + uint64x2(hi: 0xA87FEA27A539E9A5'u, lo: 0x3F2398D747B36225'u), + uint64x2(hi: 0xD29FE4B18E88640E'u, lo: 0x8EEC7F0D19A03AAE'u), + uint64x2(hi: 0x83A3EEEEF9153E89'u, lo: 0x1953CF68300424AD'u), + uint64x2(hi: 0xA48CEAAAB75A8E2B'u, lo: 0x5FA8C3423C052DD8'u), + uint64x2(hi: 0xCDB02555653131B6'u, lo: 0x3792F412CB06794E'u), + uint64x2(hi: 0x808E17555F3EBF11'u, lo: 0xE2BBD88BBEE40BD1'u), + uint64x2(hi: 0xA0B19D2AB70E6ED6'u, lo: 0x5B6ACEAEAE9D0EC5'u), + uint64x2(hi: 0xC8DE047564D20A8B'u, lo: 0xF245825A5A445276'u), + uint64x2(hi: 0xFB158592BE068D2E'u, lo: 0xEED6E2F0F0D56713'u), + uint64x2(hi: 0x9CED737BB6C4183D'u, lo: 0x55464DD69685606C'u), + uint64x2(hi: 0xC428D05AA4751E4C'u, lo: 0xAA97E14C3C26B887'u), + uint64x2(hi: 0xF53304714D9265DF'u, lo: 0xD53DD99F4B3066A9'u), + uint64x2(hi: 0x993FE2C6D07B7FAB'u, lo: 0xE546A8038EFE402A'u), + uint64x2(hi: 0xBF8FDB78849A5F96'u, lo: 0xDE98520472BDD034'u), + uint64x2(hi: 0xEF73D256A5C0F77C'u, lo: 0x963E66858F6D4441'u), + uint64x2(hi: 0x95A8637627989AAD'u, lo: 0xDDE7001379A44AA9'u), + uint64x2(hi: 0xBB127C53B17EC159'u, lo: 0x5560C018580D5D53'u), + uint64x2(hi: 0xE9D71B689DDE71AF'u, lo: 0xAAB8F01E6E10B4A7'u), + uint64x2(hi: 0x9226712162AB070D'u, lo: 0xCAB3961304CA70E9'u), + uint64x2(hi: 0xB6B00D69BB55C8D1'u, lo: 0x3D607B97C5FD0D23'u), + uint64x2(hi: 0xE45C10C42A2B3B05'u, lo: 0x8CB89A7DB77C506B'u), + uint64x2(hi: 0x8EB98A7A9A5B04E3'u, lo: 0x77F3608E92ADB243'u), + uint64x2(hi: 0xB267ED1940F1C61C'u, lo: 0x55F038B237591ED4'u), + uint64x2(hi: 0xDF01E85F912E37A3'u, lo: 0x6B6C46DEC52F6689'u), + uint64x2(hi: 0x8B61313BBABCE2C6'u, lo: 0x2323AC4B3B3DA016'u), + uint64x2(hi: 0xAE397D8AA96C1B77'u, lo: 0xABEC975E0A0D081B'u), + uint64x2(hi: 0xD9C7DCED53C72255'u, lo: 0x96E7BD358C904A22'u), + uint64x2(hi: 0x881CEA14545C7575'u, lo: 0x7E50D64177DA2E55'u), + uint64x2(hi: 0xAA242499697392D2'u, lo: 0xDDE50BD1D5D0B9EA'u), + uint64x2(hi: 0xD4AD2DBFC3D07787'u, lo: 0x955E4EC64B44E865'u), + uint64x2(hi: 0x84EC3C97DA624AB4'u, lo: 0xBD5AF13BEF0B113F'u), + uint64x2(hi: 0xA6274BBDD0FADD61'u, lo: 0xECB1AD8AEACDD58F'u), + uint64x2(hi: 0xCFB11EAD453994BA'u, lo: 0x67DE18EDA5814AF3'u), + uint64x2(hi: 0x81CEB32C4B43FCF4'u, lo: 0x80EACF948770CED8'u), + uint64x2(hi: 0xA2425FF75E14FC31'u, lo: 0xA1258379A94D028E'u), + uint64x2(hi: 0xCAD2F7F5359A3B3E'u, lo: 0x096EE45813A04331'u), + uint64x2(hi: 0xFD87B5F28300CA0D'u, lo: 0x8BCA9D6E188853FD'u), + uint64x2(hi: 0x9E74D1B791E07E48'u, lo: 0x775EA264CF55347E'u), + uint64x2(hi: 0xC612062576589DDA'u, lo: 0x95364AFE032A819E'u), + uint64x2(hi: 0xF79687AED3EEC551'u, lo: 0x3A83DDBD83F52205'u), + uint64x2(hi: 0x9ABE14CD44753B52'u, lo: 0xC4926A9672793543'u), + uint64x2(hi: 0xC16D9A0095928A27'u, lo: 0x75B7053C0F178294'u), + uint64x2(hi: 0xF1C90080BAF72CB1'u, lo: 0x5324C68B12DD6339'u), + uint64x2(hi: 0x971DA05074DA7BEE'u, lo: 0xD3F6FC16EBCA5E04'u), + uint64x2(hi: 0xBCE5086492111AEA'u, lo: 0x88F4BB1CA6BCF585'u), + uint64x2(hi: 0xEC1E4A7DB69561A5'u, lo: 0x2B31E9E3D06C32E6'u), + uint64x2(hi: 0x9392EE8E921D5D07'u, lo: 0x3AFF322E62439FD0'u), + uint64x2(hi: 0xB877AA3236A4B449'u, lo: 0x09BEFEB9FAD487C3'u), + uint64x2(hi: 0xE69594BEC44DE15B'u, lo: 0x4C2EBE687989A9B4'u), + uint64x2(hi: 0x901D7CF73AB0ACD9'u, lo: 0x0F9D37014BF60A11'u), + uint64x2(hi: 0xB424DC35095CD80F'u, lo: 0x538484C19EF38C95'u), + uint64x2(hi: 0xE12E13424BB40E13'u, lo: 0x2865A5F206B06FBA'u), + uint64x2(hi: 0x8CBCCC096F5088CB'u, lo: 0xF93F87B7442E45D4'u), + uint64x2(hi: 0xAFEBFF0BCB24AAFE'u, lo: 0xF78F69A51539D749'u), + uint64x2(hi: 0xDBE6FECEBDEDD5BE'u, lo: 0xB573440E5A884D1C'u), + uint64x2(hi: 0x89705F4136B4A597'u, lo: 0x31680A88F8953031'u), + uint64x2(hi: 0xABCC77118461CEFC'u, lo: 0xFDC20D2B36BA7C3E'u), + uint64x2(hi: 0xD6BF94D5E57A42BC'u, lo: 0x3D32907604691B4D'u), + uint64x2(hi: 0x8637BD05AF6C69B5'u, lo: 0xA63F9A49C2C1B110'u), + uint64x2(hi: 0xA7C5AC471B478423'u, lo: 0x0FCF80DC33721D54'u), + uint64x2(hi: 0xD1B71758E219652B'u, lo: 0xD3C36113404EA4A9'u), + uint64x2(hi: 0x83126E978D4FDF3B'u, lo: 0x645A1CAC083126EA'u), + uint64x2(hi: 0xA3D70A3D70A3D70A'u, lo: 0x3D70A3D70A3D70A4'u), + uint64x2(hi: 0xCCCCCCCCCCCCCCCC'u, lo: 0xCCCCCCCCCCCCCCCD'u), + uint64x2(hi: 0x8000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC800000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xFA00000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9C40000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC350000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xF424000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9896800000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBEBC200000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xEE6B280000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9502F90000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBA43B74000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE8D4A51000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9184E72A00000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB5E620F480000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE35FA931A0000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8E1BC9BF04000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB1A2BC2EC5000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xDE0B6B3A76400000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8AC7230489E80000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xAD78EBC5AC620000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD8D726B7177A8000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x878678326EAC9000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA968163F0A57B400'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD3C21BCECCEDA100'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x84595161401484A0'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA56FA5B99019A5C8'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xCECB8F27F4200F3A'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x813F3978F8940984'u, lo: 0x4000000000000000'u), + uint64x2(hi: 0xA18F07D736B90BE5'u, lo: 0x5000000000000000'u), + uint64x2(hi: 0xC9F2C9CD04674EDE'u, lo: 0xA400000000000000'u), + uint64x2(hi: 0xFC6F7C4045812296'u, lo: 0x4D00000000000000'u), + uint64x2(hi: 0x9DC5ADA82B70B59D'u, lo: 0xF020000000000000'u), + uint64x2(hi: 0xC5371912364CE305'u, lo: 0x6C28000000000000'u), + uint64x2(hi: 0xF684DF56C3E01BC6'u, lo: 0xC732000000000000'u), + uint64x2(hi: 0x9A130B963A6C115C'u, lo: 0x3C7F400000000000'u), + uint64x2(hi: 0xC097CE7BC90715B3'u, lo: 0x4B9F100000000000'u), + uint64x2(hi: 0xF0BDC21ABB48DB20'u, lo: 0x1E86D40000000000'u), + uint64x2(hi: 0x96769950B50D88F4'u, lo: 0x1314448000000000'u), + uint64x2(hi: 0xBC143FA4E250EB31'u, lo: 0x17D955A000000000'u), + uint64x2(hi: 0xEB194F8E1AE525FD'u, lo: 0x5DCFAB0800000000'u), + uint64x2(hi: 0x92EFD1B8D0CF37BE'u, lo: 0x5AA1CAE500000000'u), + uint64x2(hi: 0xB7ABC627050305AD'u, lo: 0xF14A3D9E40000000'u), + uint64x2(hi: 0xE596B7B0C643C719'u, lo: 0x6D9CCD05D0000000'u), + uint64x2(hi: 0x8F7E32CE7BEA5C6F'u, lo: 0xE4820023A2000000'u), + uint64x2(hi: 0xB35DBF821AE4F38B'u, lo: 0xDDA2802C8A800000'u), + uint64x2(hi: 0xE0352F62A19E306E'u, lo: 0xD50B2037AD200000'u), + uint64x2(hi: 0x8C213D9DA502DE45'u, lo: 0x4526F422CC340000'u), + uint64x2(hi: 0xAF298D050E4395D6'u, lo: 0x9670B12B7F410000'u), + uint64x2(hi: 0xDAF3F04651D47B4C'u, lo: 0x3C0CDD765F114000'u), + uint64x2(hi: 0x88D8762BF324CD0F'u, lo: 0xA5880A69FB6AC800'u), + uint64x2(hi: 0xAB0E93B6EFEE0053'u, lo: 0x8EEA0D047A457A00'u), + uint64x2(hi: 0xD5D238A4ABE98068'u, lo: 0x72A4904598D6D880'u), + uint64x2(hi: 0x85A36366EB71F041'u, lo: 0x47A6DA2B7F864750'u), + uint64x2(hi: 0xA70C3C40A64E6C51'u, lo: 0x999090B65F67D924'u), + uint64x2(hi: 0xD0CF4B50CFE20765'u, lo: 0xFFF4B4E3F741CF6D'u), + uint64x2(hi: 0x82818F1281ED449F'u, lo: 0xBFF8F10E7A8921A4'u), + uint64x2(hi: 0xA321F2D7226895C7'u, lo: 0xAFF72D52192B6A0D'u), + uint64x2(hi: 0xCBEA6F8CEB02BB39'u, lo: 0x9BF4F8A69F764490'u), + uint64x2(hi: 0xFEE50B7025C36A08'u, lo: 0x02F236D04753D5B4'u), + uint64x2(hi: 0x9F4F2726179A2245'u, lo: 0x01D762422C946590'u), + uint64x2(hi: 0xC722F0EF9D80AAD6'u, lo: 0x424D3AD2B7B97EF5'u), + uint64x2(hi: 0xF8EBAD2B84E0D58B'u, lo: 0xD2E0898765A7DEB2'u), + uint64x2(hi: 0x9B934C3B330C8577'u, lo: 0x63CC55F49F88EB2F'u), + uint64x2(hi: 0xC2781F49FFCFA6D5'u, lo: 0x3CBF6B71C76B25FB'u), + uint64x2(hi: 0xF316271C7FC3908A'u, lo: 0x8BEF464E3945EF7A'u), + uint64x2(hi: 0x97EDD871CFDA3A56'u, lo: 0x97758BF0E3CBB5AC'u), + uint64x2(hi: 0xBDE94E8E43D0C8EC'u, lo: 0x3D52EEED1CBEA317'u), + uint64x2(hi: 0xED63A231D4C4FB27'u, lo: 0x4CA7AAA863EE4BDD'u), + uint64x2(hi: 0x945E455F24FB1CF8'u, lo: 0x8FE8CAA93E74EF6A'u), + uint64x2(hi: 0xB975D6B6EE39E436'u, lo: 0xB3E2FD538E122B44'u), + uint64x2(hi: 0xE7D34C64A9C85D44'u, lo: 0x60DBBCA87196B616'u), + uint64x2(hi: 0x90E40FBEEA1D3A4A'u, lo: 0xBC8955E946FE31CD'u), + uint64x2(hi: 0xB51D13AEA4A488DD'u, lo: 0x6BABAB6398BDBE41'u), + uint64x2(hi: 0xE264589A4DCDAB14'u, lo: 0xC696963C7EED2DD1'u), + uint64x2(hi: 0x8D7EB76070A08AEC'u, lo: 0xFC1E1DE5CF543CA2'u), + uint64x2(hi: 0xB0DE65388CC8ADA8'u, lo: 0x3B25A55F43294BCB'u), + uint64x2(hi: 0xDD15FE86AFFAD912'u, lo: 0x49EF0EB713F39EBE'u), + uint64x2(hi: 0x8A2DBF142DFCC7AB'u, lo: 0x6E3569326C784337'u), + uint64x2(hi: 0xACB92ED9397BF996'u, lo: 0x49C2C37F07965404'u), + uint64x2(hi: 0xD7E77A8F87DAF7FB'u, lo: 0xDC33745EC97BE906'u), + uint64x2(hi: 0x86F0AC99B4E8DAFD'u, lo: 0x69A028BB3DED71A3'u), + uint64x2(hi: 0xA8ACD7C0222311BC'u, lo: 0xC40832EA0D68CE0C'u), + uint64x2(hi: 0xD2D80DB02AABD62B'u, lo: 0xF50A3FA490C30190'u), + uint64x2(hi: 0x83C7088E1AAB65DB'u, lo: 0x792667C6DA79E0FA'u), + uint64x2(hi: 0xA4B8CAB1A1563F52'u, lo: 0x577001B891185938'u), + uint64x2(hi: 0xCDE6FD5E09ABCF26'u, lo: 0xED4C0226B55E6F86'u), + uint64x2(hi: 0x80B05E5AC60B6178'u, lo: 0x544F8158315B05B4'u), + uint64x2(hi: 0xA0DC75F1778E39D6'u, lo: 0x696361AE3DB1C721'u), + uint64x2(hi: 0xC913936DD571C84C'u, lo: 0x03BC3A19CD1E38E9'u), + uint64x2(hi: 0xFB5878494ACE3A5F'u, lo: 0x04AB48A04065C723'u), + uint64x2(hi: 0x9D174B2DCEC0E47B'u, lo: 0x62EB0D64283F9C76'u), + uint64x2(hi: 0xC45D1DF942711D9A'u, lo: 0x3BA5D0BD324F8394'u), + uint64x2(hi: 0xF5746577930D6500'u, lo: 0xCA8F44EC7EE36479'u), + uint64x2(hi: 0x9968BF6ABBE85F20'u, lo: 0x7E998B13CF4E1ECB'u), + uint64x2(hi: 0xBFC2EF456AE276E8'u, lo: 0x9E3FEDD8C321A67E'u), + uint64x2(hi: 0xEFB3AB16C59B14A2'u, lo: 0xC5CFE94EF3EA101E'u), + uint64x2(hi: 0x95D04AEE3B80ECE5'u, lo: 0xBBA1F1D158724A12'u), + uint64x2(hi: 0xBB445DA9CA61281F'u, lo: 0x2A8A6E45AE8EDC97'u), + uint64x2(hi: 0xEA1575143CF97226'u, lo: 0xF52D09D71A3293BD'u), + uint64x2(hi: 0x924D692CA61BE758'u, lo: 0x593C2626705F9C56'u), + uint64x2(hi: 0xB6E0C377CFA2E12E'u, lo: 0x6F8B2FB00C77836C'u), + uint64x2(hi: 0xE498F455C38B997A'u, lo: 0x0B6DFB9C0F956447'u), + uint64x2(hi: 0x8EDF98B59A373FEC'u, lo: 0x4724BD4189BD5EAC'u), + uint64x2(hi: 0xB2977EE300C50FE7'u, lo: 0x58EDEC91EC2CB657'u), + uint64x2(hi: 0xDF3D5E9BC0F653E1'u, lo: 0x2F2967B66737E3ED'u), + uint64x2(hi: 0x8B865B215899F46C'u, lo: 0xBD79E0D20082EE74'u), + uint64x2(hi: 0xAE67F1E9AEC07187'u, lo: 0xECD8590680A3AA11'u), + uint64x2(hi: 0xDA01EE641A708DE9'u, lo: 0xE80E6F4820CC9495'u), + uint64x2(hi: 0x884134FE908658B2'u, lo: 0x3109058D147FDCDD'u), + uint64x2(hi: 0xAA51823E34A7EEDE'u, lo: 0xBD4B46F0599FD415'u), + uint64x2(hi: 0xD4E5E2CDC1D1EA96'u, lo: 0x6C9E18AC7007C91A'u), + uint64x2(hi: 0x850FADC09923329E'u, lo: 0x03E2CF6BC604DDB0'u), + uint64x2(hi: 0xA6539930BF6BFF45'u, lo: 0x84DB8346B786151C'u), + uint64x2(hi: 0xCFE87F7CEF46FF16'u, lo: 0xE612641865679A63'u), + uint64x2(hi: 0x81F14FAE158C5F6E'u, lo: 0x4FCB7E8F3F60C07E'u), + uint64x2(hi: 0xA26DA3999AEF7749'u, lo: 0xE3BE5E330F38F09D'u), + uint64x2(hi: 0xCB090C8001AB551C'u, lo: 0x5CADF5BFD3072CC5'u), + uint64x2(hi: 0xFDCB4FA002162A63'u, lo: 0x73D9732FC7C8F7F6'u), + uint64x2(hi: 0x9E9F11C4014DDA7E'u, lo: 0x2867E7FDDCDD9AFA'u), + uint64x2(hi: 0xC646D63501A1511D'u, lo: 0xB281E1FD541501B8'u), + uint64x2(hi: 0xF7D88BC24209A565'u, lo: 0x1F225A7CA91A4226'u), + uint64x2(hi: 0x9AE757596946075F'u, lo: 0x3375788DE9B06958'u), + uint64x2(hi: 0xC1A12D2FC3978937'u, lo: 0x0052D6B1641C83AE'u), + uint64x2(hi: 0xF209787BB47D6B84'u, lo: 0xC0678C5DBD23A49A'u), + uint64x2(hi: 0x9745EB4D50CE6332'u, lo: 0xF840B7BA963646E0'u), + uint64x2(hi: 0xBD176620A501FBFF'u, lo: 0xB650E5A93BC3D898'u), + uint64x2(hi: 0xEC5D3FA8CE427AFF'u, lo: 0xA3E51F138AB4CEBE'u), + uint64x2(hi: 0x93BA47C980E98CDF'u, lo: 0xC66F336C36B10137'u), + uint64x2(hi: 0xB8A8D9BBE123F017'u, lo: 0xB80B0047445D4184'u), + uint64x2(hi: 0xE6D3102AD96CEC1D'u, lo: 0xA60DC059157491E5'u), + uint64x2(hi: 0x9043EA1AC7E41392'u, lo: 0x87C89837AD68DB2F'u), + uint64x2(hi: 0xB454E4A179DD1877'u, lo: 0x29BABE4598C311FB'u), + uint64x2(hi: 0xE16A1DC9D8545E94'u, lo: 0xF4296DD6FEF3D67A'u), + uint64x2(hi: 0x8CE2529E2734BB1D'u, lo: 0x1899E4A65F58660C'u), + uint64x2(hi: 0xB01AE745B101E9E4'u, lo: 0x5EC05DCFF72E7F8F'u), + uint64x2(hi: 0xDC21A1171D42645D'u, lo: 0x76707543F4FA1F73'u), + uint64x2(hi: 0x899504AE72497EBA'u, lo: 0x6A06494A791C53A8'u), + uint64x2(hi: 0xABFA45DA0EDBDE69'u, lo: 0x0487DB9D17636892'u), + uint64x2(hi: 0xD6F8D7509292D603'u, lo: 0x45A9D2845D3C42B6'u), + uint64x2(hi: 0x865B86925B9BC5C2'u, lo: 0x0B8A2392BA45A9B2'u), + uint64x2(hi: 0xA7F26836F282B732'u, lo: 0x8E6CAC7768D7141E'u), + uint64x2(hi: 0xD1EF0244AF2364FF'u, lo: 0x3207D795430CD926'u), + uint64x2(hi: 0x8335616AED761F1F'u, lo: 0x7F44E6BD49E807B8'u), + uint64x2(hi: 0xA402B9C5A8D3A6E7'u, lo: 0x5F16206C9C6209A6'u), + uint64x2(hi: 0xCD036837130890A1'u, lo: 0x36DBA887C37A8C0F'u), + uint64x2(hi: 0x802221226BE55A64'u, lo: 0xC2494954DA2C9789'u), + uint64x2(hi: 0xA02AA96B06DEB0FD'u, lo: 0xF2DB9BAA10B7BD6C'u), + uint64x2(hi: 0xC83553C5C8965D3D'u, lo: 0x6F92829494E5ACC7'u), + uint64x2(hi: 0xFA42A8B73ABBF48C'u, lo: 0xCB772339BA1F17F9'u), + uint64x2(hi: 0x9C69A97284B578D7'u, lo: 0xFF2A760414536EFB'u), + uint64x2(hi: 0xC38413CF25E2D70D'u, lo: 0xFEF5138519684ABA'u), + uint64x2(hi: 0xF46518C2EF5B8CD1'u, lo: 0x7EB258665FC25D69'u), + uint64x2(hi: 0x98BF2F79D5993802'u, lo: 0xEF2F773FFBD97A61'u), + uint64x2(hi: 0xBEEEFB584AFF8603'u, lo: 0xAAFB550FFACFD8FA'u), + uint64x2(hi: 0xEEAABA2E5DBF6784'u, lo: 0x95BA2A53F983CF38'u), + uint64x2(hi: 0x952AB45CFA97A0B2'u, lo: 0xDD945A747BF26183'u), + uint64x2(hi: 0xBA756174393D88DF'u, lo: 0x94F971119AEEF9E4'u), + uint64x2(hi: 0xE912B9D1478CEB17'u, lo: 0x7A37CD5601AAB85D'u), + uint64x2(hi: 0x91ABB422CCB812EE'u, lo: 0xAC62E055C10AB33A'u), + uint64x2(hi: 0xB616A12B7FE617AA'u, lo: 0x577B986B314D6009'u), + uint64x2(hi: 0xE39C49765FDF9D94'u, lo: 0xED5A7E85FDA0B80B'u), + uint64x2(hi: 0x8E41ADE9FBEBC27D'u, lo: 0x14588F13BE847307'u), + uint64x2(hi: 0xB1D219647AE6B31C'u, lo: 0x596EB2D8AE258FC8'u), + uint64x2(hi: 0xDE469FBD99A05FE3'u, lo: 0x6FCA5F8ED9AEF3BB'u), + uint64x2(hi: 0x8AEC23D680043BEE'u, lo: 0x25DE7BB9480D5854'u), + uint64x2(hi: 0xADA72CCC20054AE9'u, lo: 0xAF561AA79A10AE6A'u), + uint64x2(hi: 0xD910F7FF28069DA4'u, lo: 0x1B2BA1518094DA04'u), + uint64x2(hi: 0x87AA9AFF79042286'u, lo: 0x90FB44D2F05D0842'u), + uint64x2(hi: 0xA99541BF57452B28'u, lo: 0x353A1607AC744A53'u), + uint64x2(hi: 0xD3FA922F2D1675F2'u, lo: 0x42889B8997915CE8'u), + uint64x2(hi: 0x847C9B5D7C2E09B7'u, lo: 0x69956135FEBADA11'u), + uint64x2(hi: 0xA59BC234DB398C25'u, lo: 0x43FAB9837E699095'u), + uint64x2(hi: 0xCF02B2C21207EF2E'u, lo: 0x94F967E45E03F4BB'u), + uint64x2(hi: 0x8161AFB94B44F57D'u, lo: 0x1D1BE0EEBAC278F5'u), + uint64x2(hi: 0xA1BA1BA79E1632DC'u, lo: 0x6462D92A69731732'u), + uint64x2(hi: 0xCA28A291859BBF93'u, lo: 0x7D7B8F7503CFDCFE'u), + uint64x2(hi: 0xFCB2CB35E702AF78'u, lo: 0x5CDA735244C3D43E'u), + uint64x2(hi: 0x9DEFBF01B061ADAB'u, lo: 0x3A0888136AFA64A7'u), + uint64x2(hi: 0xC56BAEC21C7A1916'u, lo: 0x088AAA1845B8FDD0'u), + uint64x2(hi: 0xF6C69A72A3989F5B'u, lo: 0x8AAD549E57273D45'u), + uint64x2(hi: 0x9A3C2087A63F6399'u, lo: 0x36AC54E2F678864B'u), + uint64x2(hi: 0xC0CB28A98FCF3C7F'u, lo: 0x84576A1BB416A7DD'u), + uint64x2(hi: 0xF0FDF2D3F3C30B9F'u, lo: 0x656D44A2A11C51D5'u), + uint64x2(hi: 0x969EB7C47859E743'u, lo: 0x9F644AE5A4B1B325'u), + uint64x2(hi: 0xBC4665B596706114'u, lo: 0x873D5D9F0DDE1FEE'u), + uint64x2(hi: 0xEB57FF22FC0C7959'u, lo: 0xA90CB506D155A7EA'u), + uint64x2(hi: 0x9316FF75DD87CBD8'u, lo: 0x09A7F12442D588F2'u), + uint64x2(hi: 0xB7DCBF5354E9BECE'u, lo: 0x0C11ED6D538AEB2F'u), + uint64x2(hi: 0xE5D3EF282A242E81'u, lo: 0x8F1668C8A86DA5FA'u), + uint64x2(hi: 0x8FA475791A569D10'u, lo: 0xF96E017D694487BC'u), + uint64x2(hi: 0xB38D92D760EC4455'u, lo: 0x37C981DCC395A9AC'u), + uint64x2(hi: 0xE070F78D3927556A'u, lo: 0x85BBE253F47B1417'u), + uint64x2(hi: 0x8C469AB843B89562'u, lo: 0x93956D7478CCEC8E'u), + uint64x2(hi: 0xAF58416654A6BABB'u, lo: 0x387AC8D1970027B2'u), + uint64x2(hi: 0xDB2E51BFE9D0696A'u, lo: 0x06997B05FCC0319E'u), + uint64x2(hi: 0x88FCF317F22241E2'u, lo: 0x441FECE3BDF81F03'u), + uint64x2(hi: 0xAB3C2FDDEEAAD25A'u, lo: 0xD527E81CAD7626C3'u), + uint64x2(hi: 0xD60B3BD56A5586F1'u, lo: 0x8A71E223D8D3B074'u), + uint64x2(hi: 0x85C7056562757456'u, lo: 0xF6872D5667844E49'u), + uint64x2(hi: 0xA738C6BEBB12D16C'u, lo: 0xB428F8AC016561DB'u), + uint64x2(hi: 0xD106F86E69D785C7'u, lo: 0xE13336D701BEBA52'u), + uint64x2(hi: 0x82A45B450226B39C'u, lo: 0xECC0024661173473'u), + uint64x2(hi: 0xA34D721642B06084'u, lo: 0x27F002D7F95D0190'u), + uint64x2(hi: 0xCC20CE9BD35C78A5'u, lo: 0x31EC038DF7B441F4'u), + uint64x2(hi: 0xFF290242C83396CE'u, lo: 0x7E67047175A15271'u), + uint64x2(hi: 0x9F79A169BD203E41'u, lo: 0x0F0062C6E984D386'u), + uint64x2(hi: 0xC75809C42C684DD1'u, lo: 0x52C07B78A3E60868'u), + uint64x2(hi: 0xF92E0C3537826145'u, lo: 0xA7709A56CCDF8A82'u), + uint64x2(hi: 0x9BBCC7A142B17CCB'u, lo: 0x88A66076400BB691'u), + uint64x2(hi: 0xC2ABF989935DDBFE'u, lo: 0x6ACFF893D00EA435'u), + uint64x2(hi: 0xF356F7EBF83552FE'u, lo: 0x0583F6B8C4124D43'u), + uint64x2(hi: 0x98165AF37B2153DE'u, lo: 0xC3727A337A8B704A'u), + uint64x2(hi: 0xBE1BF1B059E9A8D6'u, lo: 0x744F18C0592E4C5C'u), + uint64x2(hi: 0xEDA2EE1C7064130C'u, lo: 0x1162DEF06F79DF73'u), + uint64x2(hi: 0x9485D4D1C63E8BE7'u, lo: 0x8ADDCB5645AC2BA8'u), + uint64x2(hi: 0xB9A74A0637CE2EE1'u, lo: 0x6D953E2BD7173692'u), + uint64x2(hi: 0xE8111C87C5C1BA99'u, lo: 0xC8FA8DB6CCDD0437'u), + uint64x2(hi: 0x910AB1D4DB9914A0'u, lo: 0x1D9C9892400A22A2'u), + uint64x2(hi: 0xB54D5E4A127F59C8'u, lo: 0x2503BEB6D00CAB4B'u), + uint64x2(hi: 0xE2A0B5DC971F303A'u, lo: 0x2E44AE64840FD61D'u), + uint64x2(hi: 0x8DA471A9DE737E24'u, lo: 0x5CEAECFED289E5D2'u), + uint64x2(hi: 0xB10D8E1456105DAD'u, lo: 0x7425A83E872C5F47'u), + uint64x2(hi: 0xDD50F1996B947518'u, lo: 0xD12F124E28F77719'u), + uint64x2(hi: 0x8A5296FFE33CC92F'u, lo: 0x82BD6B70D99AAA6F'u), + uint64x2(hi: 0xACE73CBFDC0BFB7B'u, lo: 0x636CC64D1001550B'u), + uint64x2(hi: 0xD8210BEFD30EFA5A'u, lo: 0x3C47F7E05401AA4E'u), + uint64x2(hi: 0x8714A775E3E95C78'u, lo: 0x65ACFAEC34810A71'u), + uint64x2(hi: 0xA8D9D1535CE3B396'u, lo: 0x7F1839A741A14D0D'u), + uint64x2(hi: 0xD31045A8341CA07C'u, lo: 0x1EDE48111209A050'u), + uint64x2(hi: 0x83EA2B892091E44D'u, lo: 0x934AED0AAB460432'u), + uint64x2(hi: 0xA4E4B66B68B65D60'u, lo: 0xF81DA84D5617853F'u), + uint64x2(hi: 0xCE1DE40642E3F4B9'u, lo: 0x36251260AB9D668E'u), + uint64x2(hi: 0x80D2AE83E9CE78F3'u, lo: 0xC1D72B7C6B426019'u), + uint64x2(hi: 0xA1075A24E4421730'u, lo: 0xB24CF65B8612F81F'u), + uint64x2(hi: 0xC94930AE1D529CFC'u, lo: 0xDEE033F26797B627'u), + uint64x2(hi: 0xFB9B7CD9A4A7443C'u, lo: 0x169840EF017DA3B1'u), + uint64x2(hi: 0x9D412E0806E88AA5'u, lo: 0x8E1F289560EE864E'u), + uint64x2(hi: 0xC491798A08A2AD4E'u, lo: 0xF1A6F2BAB92A27E2'u), + uint64x2(hi: 0xF5B5D7EC8ACB58A2'u, lo: 0xAE10AF696774B1DB'u), + uint64x2(hi: 0x9991A6F3D6BF1765'u, lo: 0xACCA6DA1E0A8EF29'u), + uint64x2(hi: 0xBFF610B0CC6EDD3F'u, lo: 0x17FD090A58D32AF3'u), + uint64x2(hi: 0xEFF394DCFF8A948E'u, lo: 0xDDFC4B4CEF07F5B0'u), + uint64x2(hi: 0x95F83D0A1FB69CD9'u, lo: 0x4ABDAF101564F98E'u), + uint64x2(hi: 0xBB764C4CA7A4440F'u, lo: 0x9D6D1AD41ABE37F1'u), + uint64x2(hi: 0xEA53DF5FD18D5513'u, lo: 0x84C86189216DC5ED'u), + uint64x2(hi: 0x92746B9BE2F8552C'u, lo: 0x32FD3CF5B4E49BB4'u), + uint64x2(hi: 0xB7118682DBB66A77'u, lo: 0x3FBC8C33221DC2A1'u), + uint64x2(hi: 0xE4D5E82392A40515'u, lo: 0x0FABAF3FEAA5334A'u), + uint64x2(hi: 0x8F05B1163BA6832D'u, lo: 0x29CB4D87F2A7400E'u), + uint64x2(hi: 0xB2C71D5BCA9023F8'u, lo: 0x743E20E9EF511012'u), + uint64x2(hi: 0xDF78E4B2BD342CF6'u, lo: 0x914DA9246B255416'u), + uint64x2(hi: 0x8BAB8EEFB6409C1A'u, lo: 0x1AD089B6C2F7548E'u), + uint64x2(hi: 0xAE9672ABA3D0C320'u, lo: 0xA184AC2473B529B1'u), + uint64x2(hi: 0xDA3C0F568CC4F3E8'u, lo: 0xC9E5D72D90A2741E'u), + uint64x2(hi: 0x8865899617FB1871'u, lo: 0x7E2FA67C7A658892'u), + uint64x2(hi: 0xAA7EEBFB9DF9DE8D'u, lo: 0xDDBB901B98FEEAB7'u), + uint64x2(hi: 0xD51EA6FA85785631'u, lo: 0x552A74227F3EA565'u), + uint64x2(hi: 0x8533285C936B35DE'u, lo: 0xD53A88958F87275F'u), + uint64x2(hi: 0xA67FF273B8460356'u, lo: 0x8A892ABAF368F137'u), + uint64x2(hi: 0xD01FEF10A657842C'u, lo: 0x2D2B7569B0432D85'u), + uint64x2(hi: 0x8213F56A67F6B29B'u, lo: 0x9C3B29620E29FC73'u), + uint64x2(hi: 0xA298F2C501F45F42'u, lo: 0x8349F3BA91B47B8F'u), + uint64x2(hi: 0xCB3F2F7642717713'u, lo: 0x241C70A936219A73'u), + uint64x2(hi: 0xFE0EFB53D30DD4D7'u, lo: 0xED238CD383AA0110'u), + uint64x2(hi: 0x9EC95D1463E8A506'u, lo: 0xF4363804324A40AA'u), + uint64x2(hi: 0xC67BB4597CE2CE48'u, lo: 0xB143C6053EDCD0D5'u), + uint64x2(hi: 0xF81AA16FDC1B81DA'u, lo: 0xDD94B7868E94050A'u), + uint64x2(hi: 0x9B10A4E5E9913128'u, lo: 0xCA7CF2B4191C8326'u), + uint64x2(hi: 0xC1D4CE1F63F57D72'u, lo: 0xFD1C2F611F63A3F0'u), + uint64x2(hi: 0xF24A01A73CF2DCCF'u, lo: 0xBC633B39673C8CEC'u), + uint64x2(hi: 0x976E41088617CA01'u, lo: 0xD5BE0503E085D813'u), + uint64x2(hi: 0xBD49D14AA79DBC82'u, lo: 0x4B2D8644D8A74E18'u), + uint64x2(hi: 0xEC9C459D51852BA2'u, lo: 0xDDF8E7D60ED1219E'u), + uint64x2(hi: 0x93E1AB8252F33B45'u, lo: 0xCABB90E5C942B503'u), + uint64x2(hi: 0xB8DA1662E7B00A17'u, lo: 0x3D6A751F3B936243'u), + uint64x2(hi: 0xE7109BFBA19C0C9D'u, lo: 0x0CC512670A783AD4'u), + uint64x2(hi: 0x906A617D450187E2'u, lo: 0x27FB2B80668B24C5'u), + uint64x2(hi: 0xB484F9DC9641E9DA'u, lo: 0xB1F9F660802DEDF6'u), + uint64x2(hi: 0xE1A63853BBD26451'u, lo: 0x5E7873F8A0396973'u), + uint64x2(hi: 0x8D07E33455637EB2'u, lo: 0xDB0B487B6423E1E8'u), + uint64x2(hi: 0xB049DC016ABC5E5F'u, lo: 0x91CE1A9A3D2CDA62'u), + uint64x2(hi: 0xDC5C5301C56B75F7'u, lo: 0x7641A140CC7810FB'u), + uint64x2(hi: 0x89B9B3E11B6329BA'u, lo: 0xA9E904C87FCB0A9D'u), + uint64x2(hi: 0xAC2820D9623BF429'u, lo: 0x546345FA9FBDCD44'u), + uint64x2(hi: 0xD732290FBACAF133'u, lo: 0xA97C177947AD4095'u), + uint64x2(hi: 0x867F59A9D4BED6C0'u, lo: 0x49ED8EABCCCC485D'u), + uint64x2(hi: 0xA81F301449EE8C70'u, lo: 0x5C68F256BFFF5A74'u), + uint64x2(hi: 0xD226FC195C6A2F8C'u, lo: 0x73832EEC6FFF3111'u), + uint64x2(hi: 0x83585D8FD9C25DB7'u, lo: 0xC831FD53C5FF7EAB'u), + uint64x2(hi: 0xA42E74F3D032F525'u, lo: 0xBA3E7CA8B77F5E55'u), + uint64x2(hi: 0xCD3A1230C43FB26F'u, lo: 0x28CE1BD2E55F35EB'u), + uint64x2(hi: 0x80444B5E7AA7CF85'u, lo: 0x7980D163CF5B81B3'u), + uint64x2(hi: 0xA0555E361951C366'u, lo: 0xD7E105BCC332621F'u), + uint64x2(hi: 0xC86AB5C39FA63440'u, lo: 0x8DD9472BF3FEFAA7'u), + uint64x2(hi: 0xFA856334878FC150'u, lo: 0xB14F98F6F0FEB951'u), + uint64x2(hi: 0x9C935E00D4B9D8D2'u, lo: 0x6ED1BF9A569F33D3'u), + uint64x2(hi: 0xC3B8358109E84F07'u, lo: 0x0A862F80EC4700C8'u), + uint64x2(hi: 0xF4A642E14C6262C8'u, lo: 0xCD27BB612758C0FA'u), + uint64x2(hi: 0x98E7E9CCCFBD7DBD'u, lo: 0x8038D51CB897789C'u), + uint64x2(hi: 0xBF21E44003ACDD2C'u, lo: 0xE0470A63E6BD56C3'u), + uint64x2(hi: 0xEEEA5D5004981478'u, lo: 0x1858CCFCE06CAC74'u), + uint64x2(hi: 0x95527A5202DF0CCB'u, lo: 0x0F37801E0C43EBC8'u), + uint64x2(hi: 0xBAA718E68396CFFD'u, lo: 0xD30560258F54E6BA'u), + uint64x2(hi: 0xE950DF20247C83FD'u, lo: 0x47C6B82EF32A2069'u), + uint64x2(hi: 0x91D28B7416CDD27E'u, lo: 0x4CDC331D57FA5441'u), + uint64x2(hi: 0xB6472E511C81471D'u, lo: 0xE0133FE4ADF8E952'u), + uint64x2(hi: 0xE3D8F9E563A198E5'u, lo: 0x58180FDDD97723A6'u), + uint64x2(hi: 0x8E679C2F5E44FF8F'u, lo: 0x570F09EAA7EA7648'u), + uint64x2(hi: 0xB201833B35D63F73'u, lo: 0x2CD2CC6551E513DA'u), + uint64x2(hi: 0xDE81E40A034BCF4F'u, lo: 0xF8077F7EA65E58D1'u), + uint64x2(hi: 0x8B112E86420F6191'u, lo: 0xFB04AFAF27FAF782'u), + uint64x2(hi: 0xADD57A27D29339F6'u, lo: 0x79C5DB9AF1F9B563'u), + uint64x2(hi: 0xD94AD8B1C7380874'u, lo: 0x18375281AE7822BC'u), + uint64x2(hi: 0x87CEC76F1C830548'u, lo: 0x8F2293910D0B15B5'u), + uint64x2(hi: 0xA9C2794AE3A3C69A'u, lo: 0xB2EB3875504DDB22'u), + uint64x2(hi: 0xD433179D9C8CB841'u, lo: 0x5FA60692A46151EB'u), + uint64x2(hi: 0x849FEEC281D7F328'u, lo: 0xDBC7C41BA6BCD333'u), + uint64x2(hi: 0xA5C7EA73224DEFF3'u, lo: 0x12B9B522906C0800'u), + uint64x2(hi: 0xCF39E50FEAE16BEF'u, lo: 0xD768226B34870A00'u), + uint64x2(hi: 0x81842F29F2CCE375'u, lo: 0xE6A1158300D46640'u), + uint64x2(hi: 0xA1E53AF46F801C53'u, lo: 0x60495AE3C1097FD0'u), + uint64x2(hi: 0xCA5E89B18B602368'u, lo: 0x385BB19CB14BDFC4'u), + uint64x2(hi: 0xFCF62C1DEE382C42'u, lo: 0x46729E03DD9ED7B5'u), + uint64x2(hi: 0x9E19DB92B4E31BA9'u, lo: 0x6C07A2C26A8346D1'u), + uint64x2(hi: 0xC5A05277621BE293'u, lo: 0xC7098B7305241885'u), + uint64x2(hi: 0xF70867153AA2DB38'u, lo: 0xB8CBEE4FC66D1EA7'u)] + dragonbox_Assert(k >= kMin) + dragonbox_Assert(k <= kMax) + return pow10[k - kMin] + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2*(value: uint64; e2: int32): bool {.inline.} = + dragonbox_Assert(e2 >= 0) + return e2 < 64 and (value and ((uint64(1) shl e2) - 1)) == 0 + +## Returns whether value is divisible by 5^e5 + +proc multipleOfPow5*(value: uint64; e5: int32): bool {.inline.} = + type + MulCmp {.bycopy.} = object + mul: uint64 + cmp: uint64 + + const + mod5 = [MulCmp(mul: 0x0000000000000001'u, cmp: 0xFFFFFFFFFFFFFFFF'u), + MulCmp(mul: 0xCCCCCCCCCCCCCCCD'u, cmp: 0x3333333333333333'u), + MulCmp(mul: 0x8F5C28F5C28F5C29'u, cmp: 0x0A3D70A3D70A3D70'u), + MulCmp(mul: 0x1CAC083126E978D5'u, cmp: 0x020C49BA5E353F7C'u), + MulCmp(mul: 0xD288CE703AFB7E91'u, cmp: 0x0068DB8BAC710CB2'u), + MulCmp(mul: 0x5D4E8FB00BCBE61D'u, cmp: 0x0014F8B588E368F0'u), + MulCmp(mul: 0x790FB65668C26139'u, cmp: 0x000431BDE82D7B63'u), + MulCmp(mul: 0xE5032477AE8D46A5'u, cmp: 0x0000D6BF94D5E57A'u), + MulCmp(mul: 0xC767074B22E90E21'u, cmp: 0x00002AF31DC46118'u), + MulCmp(mul: 0x8E47CE423A2E9C6D'u, cmp: 0x0000089705F4136B'u), + MulCmp(mul: 0x4FA7F60D3ED61F49'u, cmp: 0x000001B7CDFD9D7B'u), + MulCmp(mul: 0x0FEE64690C913975'u, cmp: 0x00000057F5FF85E5'u), + MulCmp(mul: 0x3662E0E1CF503EB1'u, cmp: 0x000000119799812D'u), + MulCmp(mul: 0xA47A2CF9F6433FBD'u, cmp: 0x0000000384B84D09'u), + MulCmp(mul: 0x54186F653140A659'u, cmp: 0x00000000B424DC35'u), + MulCmp(mul: 0x7738164770402145'u, cmp: 0x0000000024075F3D'u), + MulCmp(mul: 0xE4A4D1417CD9A041'u, cmp: 0x000000000734ACA5'u), + MulCmp(mul: 0xC75429D9E5C5200D'u, cmp: 0x000000000170EF54'u), + MulCmp(mul: 0xC1773B91FAC10669'u, cmp: 0x000000000049C977'u), + MulCmp(mul: 0x26B172506559CE15'u, cmp: 0x00000000000EC1E4'u), + MulCmp(mul: 0xD489E3A9ADDEC2D1'u, cmp: 0x000000000002F394'u), + MulCmp(mul: 0x90E860BB892C8D5D'u, cmp: 0x000000000000971D'u), + MulCmp(mul: 0x502E79BF1B6F4F79'u, cmp: 0x0000000000001E39'u), + MulCmp(mul: 0xDCD618596BE30FE5'u, cmp: 0x000000000000060B'u), + MulCmp(mul: 0x2C2AD1AB7BFA3661'u, cmp: 0x0000000000000135'u)] + dragonbox_Assert(e5 >= 0) + dragonbox_Assert(e5 <= 24) + let m5: MulCmp = mod5[e5] + return value * m5.mul <= m5.cmp + +type + FloatingDecimal64* {.bycopy.} = object + significand*: uint64 + exponent*: int32 + + +proc toDecimal64AsymmetricInterval*(e2: int32): FloatingDecimal64 {.inline.} = + ## NB: + ## accept_lower_endpoint = true + ## accept_upper_endpoint = true + const + P: int32 = significandSize + ## Compute k and beta + let minusK: int32 = floorLog10ThreeQuartersPow2(e2) + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + ## Compute xi and zi + let pow10: uint64x2 = computePow10(-minusK) + let lowerEndpoint: uint64 = (pow10.hi - (pow10.hi shr (P + 1))) shr + (64 - P - betaMinus1) + let upperEndpoint: uint64 = (pow10.hi + (pow10.hi shr (P + 0))) shr + (64 - P - betaMinus1) + ## If we don't accept the left endpoint (but we do!) or + ## if the left endpoint is not an integer, increase it + let lowerEndpointIsInteger: bool = (2 <= e2 and e2 <= 3) + let xi: uint64 = lowerEndpoint + uint64(not lowerEndpointIsInteger) + let zi: uint64 = upperEndpoint + ## Try bigger divisor + var q: uint64 = zi div 10 + if q * 10 >= xi: + return FloatingDecimal64(significand: q, exponent: minusK + 1) + q = ((pow10.hi shr (64 - (P + 1) - betaMinus1)) + 1) div 2 + ## When tie occurs, choose one of them according to the rule + if e2 == -77: + dec(q, ord(q mod 2 != 0)) + ## Round to even. + else: + inc(q, ord(q < xi)) + return FloatingDecimal64(significand: q, exponent: minusK) + +proc computeDelta*(pow10: uint64x2; betaMinus1: int32): uint32 {.inline.} = + dragonbox_Assert(betaMinus1 >= 0) + dragonbox_Assert(betaMinus1 <= 63) + return cast[uint32](pow10.hi shr (64 - 1 - betaMinus1)) + +when defined(sizeof_Int128): + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + ## 1 mulx + type + uint128T = uint128 + let p: uint128T = uint128T(x) * y + let hi: uint64 = cast[uint64](p shr 64) + let lo: uint64 = cast[uint64](p) + return (hi, lo) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + var hi: uint64 = 0 + var lo: uint64 = umul128(x, y, addr(hi)) + return uint64x2(hi: hi, lo: lo) + +else: + proc lo32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + + proc hi32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + + proc mul128*(a: uint64; b: uint64): uint64x2 {.inline.} = + let b00: uint64 = uint64(lo32(a)) * lo32(b) + let b01: uint64 = uint64(lo32(a)) * hi32(b) + let b10: uint64 = uint64(hi32(a)) * lo32(b) + let b11: uint64 = uint64(hi32(a)) * hi32(b) + let mid1: uint64 = b10 + hi32(b00) + let mid2: uint64 = b01 + lo32(mid1) + let hi: uint64 = b11 + hi32(mid1) + hi32(mid2) + let lo: uint64 = lo32(b00) or uint64(lo32(mid2)) shl 32 + return uint64x2(hi: hi, lo: lo) + +## Returns (x * y) / 2^128 + +proc mulShift*(x: uint64; y: uint64x2): uint64 {.inline.} = + ## 2 mulx + var p1: uint64x2 = mul128(x, y.hi) + var p0: uint64x2 = mul128(x, y.lo) + p1.lo += p0.hi + inc(p1.hi, ord(p1.lo < p0.hi)) + return p1.hi + +proc mulParity*(twoF: uint64; pow10: uint64x2; betaMinus1: int32): bool {.inline.} = + ## 1 mulx, 1 mul + dragonbox_Assert(betaMinus1 >= 1) + dragonbox_Assert(betaMinus1 <= 63) + let p01: uint64 = twoF * pow10.hi + let p10: uint64 = mul128(twoF, pow10.lo).hi + let mid: uint64 = p01 + p10 + return (mid and (uint64(1) shl (64 - betaMinus1))) != 0 + +proc isIntegralEndpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -2: + return false + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc isIntegralMidpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -4: + return multipleOfPow2(twoF, minusK - e2 + 1) + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecimal64 {. + inline.} = + const + kappa: int32 = 2 + const + bigDivisor: uint32 = 1000 + ## 10^(kappa + 1) + const + smallDivisor: uint32 = 100 + ## 10^(kappa) + ## + ## Step 1: + ## integer promotion & Schubfach multiplier calculation. + ## + var m2: uint64 + var e2: int32 + if ieeeExponent != 0: + m2 = hiddenBit or ieeeSignificand + e2 = cast[int32](ieeeExponent) - exponentBias + if 0 <= -e2 and -e2 < significandSize and multipleOfPow2(m2, -e2): + ## Small integer. + return FloatingDecimal64(significand: m2 shr -e2, exponent: 0) + if ieeeSignificand == 0 and ieeeExponent > 1: + ## Shorter interval case; proceed like Schubfach. + return toDecimal64AsymmetricInterval(e2) + else: + ## Subnormal case; interval is always regular. + m2 = ieeeSignificand + e2 = 1 - exponentBias + let isEven: bool = (m2 mod 2 == 0) + let acceptLower: bool = isEven + let acceptUpper: bool = isEven + ## Compute k and beta. + let minusK: int32 = floorLog10Pow2(e2) - kappa + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + dragonbox_Assert(betaMinus1 >= 6) + dragonbox_Assert(betaMinus1 <= 9) + let pow10: uint64x2 = computePow10(-minusK) + ## Compute delta + ## 10^kappa <= delta < 10^(kappa + 1) + ## 100 <= delta < 1000 + let delta: uint32 = computeDelta(pow10, betaMinus1) + dragonbox_Assert(delta >= smallDivisor) + dragonbox_Assert(delta < bigDivisor) + let twoFl: uint64 = 2 * m2 - 1 + let twoFc: uint64 = 2 * m2 + let twoFr: uint64 = 2 * m2 + 1 + ## (54 bits) + ## Compute zi + ## (54 + 9 = 63 bits) + let zi: uint64 = mulShift(twoFr shl betaMinus1, pow10) + ## 2 mulx + ## + ## Step 2: + ## Try larger divisor. + ## + var q: uint64 = zi div bigDivisor + ## uint64_t q = Mul128(zi, 0x83126E978D4FDF3Cu).hi >> 9; // 1 mulx + var r: uint32 = cast[uint32](zi) - bigDivisor * cast[uint32](q) + ## r = zi % BigDivisor + ## 0 <= r < 1000 + if r < delta: ## likely ~50% ?! + ## (r > deltai) + ## Exclude the right endpoint if necessary + if r != 0 or acceptUpper or not isIntegralEndpoint(twoFr, e2, minusK): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + dragonbox_Assert(q != 0) + dec(q) + r = bigDivisor + elif r == delta: ## unlikely + ## Compare fractional parts. + ## Check conditions in the order different from the paper + ## to take advantage of short-circuiting + if (acceptLower and isIntegralEndpoint(twoFl, e2, minusK)) or + mulParity(twoFl, pow10, betaMinus1): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + else: + discard + ## + ## Step 3: + ## Find the significand with the smaller divisor + ## + q = q * 10 + ## 1 hmul + ## 0 <= r <= 1000 + let dist: uint32 = r - (delta div 2) + (smallDivisor div 2) + let distQ: uint32 = dist div 100 + ## 1 mul + ## const uint32_t dist_r = dist % 100; + q += distQ + ## if /*likely*/ (dist_r == 0) + if dist == distQ * 100: + ## const bool approx_y_parity = ((dist ^ (SmallDivisor / 2)) & 1) != 0; + let approxYParity: bool = (dist and 1) != 0 + ## Check z^(f) >= epsilon^(f) + ## We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1, + ## where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f) + ## Since there are only 2 possibilities, we only need to care about the + ## parity. Also, zi and r should have the same parity since the divisor + ## is an even number + if mulParity(twoFc, pow10, betaMinus1) != approxYParity: + dec(q) + elif q mod 2 != 0 and isIntegralMidpoint(twoFc, e2, minusK): + dec(q) + return FloatingDecimal64(significand: q, exponent: minusK + kappa) + +# ================================================================================================== +# ToChars +# ================================================================================================== + +when false: + template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset)) + + template dec(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) - uint(offset)) + template inc(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) + uint(offset)) + + proc memset(x: cstring; ch: char; L: int) {.importc, nodecl.} + proc memmove(a, b: cstring; L: int) {.importc, nodecl.} + +proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int {.inline.} = + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999'u32) + let q: uint32 = digits div 10000 + let r: uint32 = digits mod 10000 + let qH: uint32 = q div 100 + let qL: uint32 = q mod 100 + utoa2Digits(buf, pos, qH) + utoa2Digits(buf, pos + 2, qL) + if r == 0: + return trailingZeros2Digits(if qL == 0: qH else: qL) + (if qL == 0: 6 else: 4) + else: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos + 4, rH) + utoa2Digits(buf, pos + 6, rL) + return trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + +proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int {.inline.} = + var pos = pos + var output64 = output64 + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 17 digits remaining + if output64 >= 100000000'u64: + let q: uint64 = output64 div 100000000'u64 + let r: uint32 = cast[uint32](output64 mod 100000000'u64) + output64 = q + dec(pos, 8) + if r != 0: + tz = utoa8DigitsSkipTrailingZeros(buf, pos, r) + dragonbox_Assert(tz >= 0) + dragonbox_Assert(tz <= 7) + else: + tz = 8 + nd = 8 + dragonbox_Assert(output64 <= high(uint32)) + var output = cast[uint32](output64) + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + if tz == nd: + inc(tz, trailingZeros2Digits(if rL == 0: rH else: rL) + + (if rL == 0: 2 else: 0)) + else: + if tz == nd: + inc(tz, 4) + else: + for i in 0..3: buf[pos+i] = '0' + ## (actually not required...) + inc(nd, 4) + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + dragonbox_Assert(output >= 1) + dragonbox_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + dragonbox_Assert(q >= 1) + dragonbox_Assert(q <= 9) + dec(pos) + buf[pos] = chr(ord('0') + q) + return tz + +proc decimalLength*(v: uint64): int {.inline.} = + dragonbox_Assert(v >= 1) + dragonbox_Assert(v <= 99999999999999999'u64) + if cast[uint32](v shr 32) != 0: + if v >= 10000000000000000'u64: + return 17 + if v >= 1000000000000000'u64: + return 16 + if v >= 100000000000000'u64: + return 15 + if v >= 10000000000000'u64: + return 14 + if v >= 1000000000000'u64: + return 13 + if v >= 100000000000'u64: + return 12 + if v >= 10000000000'u64: + return 11 + return 10 + let v32: uint32 = cast[uint32](v) + if v32 >= 1000000000'u32: + return 10 + if v32 >= 100000000'u32: + return 9 + if v32 >= 10000000'u32: + return 8 + if v32 >= 1000000'u32: + return 7 + if v32 >= 100000'u32: + return 6 + if v32 >= 10000'u32: + return 5 + if v32 >= 1000'u32: + return 4 + if v32 >= 100'u32: + return 3 + if v32 >= 10'u32: + return 2 + return 1 + +proc formatDigits*[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint64; decimalExponent: int; + forceTrailingDotZero = false): int {.inline.} = + const + minFixedDecimalPoint = -6 + const + maxFixedDecimalPoint = 17 + var pos:int = pos.int + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 17, "internal error") + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999999999999'u64) + dragonbox_Assert(decimalExponent >= -999) + dragonbox_Assert(decimalExponent <= 999) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + int(decimalDigitsPosition + numDigits) + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + when true: #defined(vcc) and not defined(clang): + ## VC does not inline the memmove call below. (Even if compiled with /arch:AVX2.) + ## However, memcpy will be inlined. + var tmp: array[16, char] + for i in 0..<16: tmp[i] = buffer[i+pos+decimalPoint] + for i in 0..<16: buffer[i+pos+decimalPoint+1] = tmp[i] + else: + memmove(buffer +! (decimalPoint + 1), buffer +! decimalPoint, 16) + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + ## Copy the first digit one place to the left. + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent: int = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(ord('0') + k) + inc(pos) + elif k < 100: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + else: + let q: uint32 = k div 100 + let r: uint32 = k mod 100 + buffer[pos] = chr(ord('0') + q) + inc(pos) + utoa2Digits(buffer, pos, r) + inc(pos, 2) + return pos + +proc toChars*(buffer: var openArray[char]; v: float; forceTrailingDotZero = false): int {. + inline.} = + var pos = 0 + let significand: uint64 = physicalSignificand(constructDouble(v)) + let exponent: uint64 = physicalExponent(constructDouble(v)) + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec = toDecimal64(significand, exponent) + return formatDigits(buffer, pos, dec.significand, dec.exponent.int, + forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 + +when false: + proc toString*(value: float): string = + var buffer: array[dtoaMinBufferLength, char] + let last = toChars(addr buffer, value) + let L = cast[int](last) - cast[int](addr(buffer)) + result = newString(L) + copyMem(addr result[0], addr buffer[0], L) + diff --git a/lib/std/private/gitutils.nim b/lib/std/private/gitutils.nim index bf5e7cb1ff..6dc9c8f3be 100644 --- a/lib/std/private/gitutils.nim +++ b/lib/std/private/gitutils.nim @@ -4,7 +4,10 @@ internal API for now, API subject to change # xxx move other git utilities here; candidate for stdlib. -import std/[os, osproc, strutils] +import std/[os, paths, osproc, strutils, tempfiles] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] const commitHead* = "HEAD" @@ -29,12 +32,43 @@ template retryCall*(maxRetry = 3, backoffDuration = 1.0, call: untyped): bool = result proc isGitRepo*(dir: string): bool = - ## This command is used to get the relative path to the root of the repository. - ## Using this, we can verify whether a folder is a git repository by checking - ## whether the command success and if the output is empty. - let (output, status) = execCmdEx("git rev-parse --show-cdup", workingDir = dir) - # On Windows there will be a trailing newline on success, remove it. - # The value of a successful call typically won't have a whitespace (it's - # usually a series of ../), so we know that it's safe to unconditionally - # remove trailing whitespaces from the result. - result = status == 0 and output.strip() == "" + ## Avoid calling git since it depends on /bin/sh existing and fails in Nix. + return fileExists(dir/".git/HEAD") + +proc diffFiles*(path1, path2: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of files `path1`, `path2`, the exact form of + ## which is implementation defined. + # This could be customized, e.g. non-git diff with `diff -uNdr`, or with + # git diff options (e.g. --color-moved, --word-diff). + # in general, `git diff` has more options than `diff`. + var status = 0 + (result.output, status) = execCmdEx("git diff --no-index $1 $2" % [path1.quoteShell, path2.quoteShell]) + doAssert (status == 0) or (status == 1) + result.same = status == 0 + +proc diffStrings*(a, b: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of `a`, `b`, the exact form of which is + ## implementation defined. + ## See also `experimental.diff`. + runnableExamples: + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + let (c, same) = diffStrings(a, b) + doAssert not same + let (c2, same2) = diffStrings(a, a) + doAssert same2 + runnableExamples("-r:off"): + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + echo diffStrings(a, b).output + + template tmpFileImpl(prefix, str): auto = + let path = genTempPath(prefix, "") + writeFile(path, str) + path + let patha = tmpFileImpl("diffStrings_a_", a) + let pathb = tmpFileImpl("diffStrings_b_", b) + defer: + removeFile(patha) + removeFile(pathb) + result = diffFiles(patha, pathb) diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index b98a7808b9..a6d0885582 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -4,9 +4,18 @@ this can eventually be moved to std/os and `walkDirRec` can be implemented in te to avoid duplication ]## -import std/[os] +import std/os when defined(windows): - from strutils import replace + from std/strutils import replace + +when defined(nimPreviewSlimSystem): + import std/[assertions, objectdollar] + + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} type PathEntry* = object @@ -14,7 +23,7 @@ type path*: string iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = + relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect], effectsOf: follow.} = ## Improved `os.walkDirRec`. #[ note: a yieldFilter isn't needed because caller can filter at call site, without @@ -45,12 +54,17 @@ iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = n proc nativeToUnixPath*(path: string): string = # pending https://github.com/nim-lang/Nim/pull/13265 - doAssert not path.isAbsolute # not implemented here; absolute files need more care for the drive + result = path + when defined(windows): + if path.len >= 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':': + result[0] = '/' + result[1] = path[0] + if path.len > 2 and path[2] != '\\': + raiseAssert "paths like `C:foo` are currently unsupported, path: " & path when DirSep == '\\': - result = replace(path, '\\', '/') - else: result = path + result = replace(result, '\\', '/') when isMainModule: - import sugar - for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]): + import std/sugar + for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]): echo a diff --git a/lib/std/private/jsutils.nim b/lib/std/private/jsutils.nim index 32c717c561..5f79eab278 100644 --- a/lib/std/private/jsutils.nim +++ b/lib/std/private/jsutils.nim @@ -1,4 +1,4 @@ -when defined(js) or defined(nimdoc): +when defined(js): import std/jsbigints type @@ -14,7 +14,6 @@ when defined(js) or defined(nimdoc): func newUint32Array*(buffer: ArrayBuffer): Uint32Array {.importjs: "new Uint32Array(#)".} func newBigUint64Array*(buffer: ArrayBuffer): BigUint64Array {.importjs: "new BigUint64Array(#)".} - func newUint8Array*(n: int): Uint8Array {.importjs: "new Uint8Array(#)".} func `[]`*(arr: Uint32Array, i: int): uint32 {.importjs: "#[#]".} @@ -22,30 +21,76 @@ when defined(js) or defined(nimdoc): func `[]`*(arr: BigUint64Array, i: int): JsBigInt {.importjs: "#[#]".} func `[]=`*(arr: Float64Array, i: int, v: float) {.importjs: "#[#] = #".} - - proc jsTypeOf*[T](x: T): cstring {.importjs: "typeof(#)".} - ## Returns the name of the JsObject's JavaScript type as a cstring. - # xxx replace jsffi.jsTypeOf with this definition and add tests + proc jsTypeOf*[T](x: T): cstring {.importjs: "typeof(#)".} = + ## Returns the name of the JsObject's JavaScript type as a cstring. + # xxx replace jsffi.jsTypeOf with this definition and add tests + runnableExamples: + import std/[jsffi, jsbigints] + assert jsTypeOf(1.toJs) == "number" + assert jsTypeOf(false.toJs) == "boolean" + assert [1].toJs.jsTypeOf == "object" # note the difference with `getProtoName` + assert big"1".toJs.jsTypeOf == "bigint" proc jsConstructorName*[T](a: T): cstring = - asm """`result` = `a`.constructor.name""" + runnableExamples: + import std/jsffi + let a = array[2, float64].default + assert jsConstructorName(a) == "Float64Array" + assert jsConstructorName(a.toJs) == "Float64Array" + {.emit: """`result` = `a`.constructor.name;""".} proc hasJsBigInt*(): bool = - asm """`result` = typeof BigInt != 'undefined'""" + {.emit: """`result` = typeof BigInt != 'undefined';""".} proc hasBigUint64Array*(): bool = - asm """`result` = typeof BigUint64Array != 'undefined'""" + {.emit: """`result` = typeof BigUint64Array != 'undefined';""".} - proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} + proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} = + runnableExamples: + import std/[jsffi, jsbigints] + type A = ref object + assert 1.toJs.getProtoName == "[object Number]" + assert "a".toJs.getProtoName == "[object String]" + assert big"1".toJs.getProtoName == "[object BigInt]" + assert false.toJs.getProtoName == "[object Boolean]" + assert (a: 1).toJs.getProtoName == "[object Object]" + assert A.default.toJs.getProtoName == "[object Null]" + assert [1].toJs.getProtoName == "[object Int32Array]" # implementation defined + assert @[1].toJs.getProtoName == "[object Array]" # ditto - proc isInteger*[T](x: T): bool {.importjs: "Number.isInteger(#)".} + const maxSafeInteger* = 9007199254740991 + ## The same as `Number.MAX_SAFE_INTEGER` or `2^53 - 1`. + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER + runnableExamples: + let a {.importjs: "Number.MAX_SAFE_INTEGER".}: int64 + assert a == maxSafeInteger + + proc isInteger*[T](x: T): bool {.importjs: "Number.isInteger(#)".} = + runnableExamples: + import std/jsffi + assert 1.isInteger + assert not 1.5.isInteger + assert 1.toJs.isInteger + assert not 1.5.toJs.isInteger proc isSafeInteger*[T](x: T): bool {.importjs: "Number.isSafeInteger(#)".} = runnableExamples: import std/jsffi assert not "123".toJs.isSafeInteger + assert 123.isSafeInteger assert 123.toJs.isSafeInteger - assert 9007199254740991.toJs.isSafeInteger - assert not 9007199254740992.toJs.isSafeInteger + when false: + assert 9007199254740991.toJs.isSafeInteger + assert not 9007199254740992.toJs.isSafeInteger - let maxSafeInteger* {.importjs: "Number.MAX_SAFE_INTEGER".} : int64 +template whenJsNoBigInt64*(no64, yes64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + no64 diff --git a/lib/std/private/miscdollars.nim b/lib/std/private/miscdollars.nim index a41cf1bc17..06fda6fa1a 100644 --- a/lib/std/private/miscdollars.nim +++ b/lib/std/private/miscdollars.nim @@ -1,3 +1,5 @@ +from std/private/digitsutils import addInt + template toLocation*(result: var string, file: string | cstring, line: int, col: int) = ## avoids spurious allocations # Hopefully this can be re-used everywhere so that if a user needs to customize, @@ -5,11 +7,33 @@ template toLocation*(result: var string, file: string | cstring, line: int, col: result.add file if line > 0: result.add "(" - # simplify this after moving moving `include strmantle` above import assertions` - when declared(addInt): result.addInt line - else: result.add $line + addInt(result, line) if col > 0: result.add ", " - when declared(addInt): result.addInt col - else: result.add $col + addInt(result, col) result.add ")" + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +template tupleObjectDollar*[T: tuple | object](result: var string, x: T) = + result = "(" + const isNamed = T is object or isNamedTuple(typeof(T)) + var count {.used.} = 0 + for name, value in fieldPairs(x): + if count > 0: result.add(", ") + when isNamed: + result.add(name) + result.add(": ") + count.inc + when compiles($value): + when value isnot string and value isnot seq and compiles(value.isNil): + if value.isNil: result.add "nil" + else: result.addQuoted(value) + else: + result.addQuoted(value) + else: + result.add("...") + when not isNamed: + if count == 1: + result.add(",") # $(1,) should print as the semantically legal (1,) + result.add(")") diff --git a/lib/std/private/ntpath.nim b/lib/std/private/ntpath.nim new file mode 100644 index 0000000000..7c8661bb7f --- /dev/null +++ b/lib/std/private/ntpath.nim @@ -0,0 +1,61 @@ +# This module is inspired by Python's `ntpath.py` module. + +import std/[ + strutils, +] + +# Adapted `splitdrive` function from the following commits in Python source +# code: +# 5a607a3ee5e81bdcef3f886f9d20c1376a533df4 (2009): Initial UNC handling (by Mark Hammond) +# 2ba0fd5767577954f331ecbd53596cd8035d7186 (2022): Support for "UNC"-device paths (by Barney Gale) +# +# FAQ: Why use `strip` below? `\\?\UNC` is the start of a "UNC symbolic link", +# which is a special UNC form. Running `strip` differentiates `\\?\UNC\` (a UNC +# symbolic link) from e.g. `\\?\UNCD` (UNCD is the server in the UNC path). +func splitDrive*(p: string): tuple[drive, path: string] = + ## Splits a Windows path into a drive and path part. The drive can be e.g. + ## `C:`. It can also be a UNC path (`\\server\drive\path`). + ## + ## The equivalent `splitDrive` for POSIX systems always returns empty drive. + ## Therefore this proc is only necessary on DOS-like file systems (together + ## with Nim's `doslikeFileSystem` conditional variable). + ## + ## This proc's use case is to extract `path` such that it can be manipulated + ## like a POSIX path. + runnableExamples: + doAssert splitDrive("C:") == ("C:", "") + doAssert splitDrive(r"C:\") == (r"C:", r"\") + doAssert splitDrive(r"\\server\drive\foo\bar") == (r"\\server\drive", r"\foo\bar") + doAssert splitDrive(r"\\?\UNC\server\share\dir") == (r"\\?\UNC\server\share", r"\dir") + + result = ("", p) + if p.len < 2: + return + const sep = '\\' + let normp = p.replace('/', sep) + if p.len > 2 and normp[0] == sep and normp[1] == sep and normp[2] != sep: + + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path + # \\machine\mountpoint\directory\etc\... + # directory ^^^^^^^^^^^^^^^ + let start = block: + const unc = "\\\\?\\UNC" # Length is 7 + let idx = min(8, normp.len) + if unc == normp[0..<idx].strip(chars = {sep}, leading = false).toUpperAscii: + 8 + else: + 2 + let index = normp.find(sep, start) + if index == -1: + return + var index2 = normp.find(sep, index + 1) + + # a UNC path can't have two slashes in a row (after the initial two) + if index2 == index + 1: + return + if index2 == -1: + index2 = p.len + return (p[0..<index2], p[index2..^1]) + if p[1] == ':': + return (p[0..1], p[2..^1]) diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim new file mode 100644 index 0000000000..07a6809bb3 --- /dev/null +++ b/lib/std/private/osappdirs.nim @@ -0,0 +1,176 @@ +## .. importdoc:: paths.nim, dirs.nim + +include system/inclrtl +import std/envvars +import std/private/ospaths2 + +proc getHomeDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getDataDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + runnableExamples: + import std/os + assert getHomeDir() == expandTilde("~") + + when defined(windows): return getEnv("USERPROFILE") & "\\" + else: return getEnv("HOME") & "/" + +proc getDataDir*(): string {.rtl, extern: "nos$1" + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + elif defined(macosx): + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / "Library" / "Application Support") + else: + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / ".local" / "share") + result.normalizePathEnd(trailingSep = true) + +proc getConfigDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getDataDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + else: + result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") + result.normalizePathEnd(trailingSep = true) + +proc getCacheDir*(): string = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + ## * `getDataDir proc`_ + # follows https://crates.io/crates/platform-dirs + when defined(windows): + result = getEnv("LOCALAPPDATA") + elif defined(osx): + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches") + else: + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache") + result.normalizePathEnd(false) + +proc getCacheDir*(app: string): string = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + when defined(windows): + getCacheDir() / app / "cache" + else: + getCacheDir() / app + + +when defined(windows): + type DWORD = uint32 + + when defined(nimPreviewSlimSystem): + import std/widestrs + + proc getTempPath( + nBufferLength: DWORD, lpBuffer: WideCString + ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = + ## Retrieves the path of the directory designated for temporary files. + +template getEnvImpl(result: var string, tempDirList: openArray[string]) = + for dir in tempDirList: + if existsEnv(dir): + result = getEnv(dir) + break + +template getTempDirImpl(result: var string) = + when defined(windows): + getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) + else: + getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) + +proc getTempDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## **Note:** This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + const tempDirDefault = "/tmp" + when defined(tempDir): + const tempDir {.strdefine.}: string = tempDirDefault + result = tempDir + else: + when nimvm: + getTempDirImpl(result) + else: + when defined(windows): + let size = getTempPath(0, nil) + # If the function fails, the return value is zero. + if size > 0: + let buffer = newWideCString(size.int) + if getTempPath(size, buffer) > 0: + result = $buffer + elif defined(android): result = "/data/local/tmp" + else: + getTempDirImpl(result) + if result.len == 0: + result = tempDirDefault + normalizePathEnd(result, trailingSep=true) diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim new file mode 100644 index 0000000000..c49d52ef29 --- /dev/null +++ b/lib/std/private/oscommon.nim @@ -0,0 +1,186 @@ +include system/inclrtl + +import std/[oserrors] + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osdirs.nim, os.nim + +const weirdTarget* = defined(nimscript) or defined(js) + + +type + ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read + ## operation from the directory + ## structure. + WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write + ## operation to + ## the directory structure. + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix + proc c_rename(oldname, newname: cstring): cint {. + importc: "rename", header: "<stdio.h>".} +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +when defined(windows) and not weirdTarget: + template wrapUnary*(varname, winApiProc, arg: untyped) = + var varname = winApiProc(newWideCString(arg)) + + template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = + var varname = winApiProc(newWideCString(arg), arg2) + proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = + result = findFirstFileW(newWideCString(a), b) + template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) + + template getFilename*(f: untyped): untyped = + $cast[WideCString](addr(f.cFileName[0])) + + proc skipFindData*(f: WIN32_FIND_DATA): bool {.inline.} = + # Note - takes advantage of null delimiter in the cstring + const dot = ord('.') + result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or + f.cFileName[1].int == dot and f.cFileName[2].int == 0) + + +type + PathComponent* = enum ## Enumeration specifying a path component. + ## + ## See also: + ## * `walkDirRec iterator`_ + ## * `FileInfo object`_ + pcFile, ## path refers to a file + pcLinkToFile, ## path refers to a symbolic link to a file + pcDir, ## path refers to a directory + pcLinkToDir ## path refers to a symbolic link to a directory + + +when defined(posix) and not weirdTarget: + proc getSymlinkFileKind*(path: string): + tuple[pc: PathComponent, isSpecial: bool] = + # Helper function. + var s: Stat + assert(path != "") + result = (pcLinkToFile, false) + if stat(path, s) == 0'i32: + if S_ISDIR(s.st_mode): + result = (pcLinkToDir, false) + elif not S_ISREG(s.st_mode): + result = (pcLinkToFile, true) + +proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = + ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. + ## + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). + ## In case of other errors `OSError` is raised. + ## Returns true in case of success. + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 + else: + result = c_rename(source, dest) == 0'i32 + + if not result: + let err = osLastError() + let isAccessDeniedError = + when defined(windows): + const AccessDeniedError = OSErrorCode(5) + isDir and err == AccessDeniedError + else: + err == EXDEV.OSErrorCode + if not isAccessDeniedError: + raiseOSError(err, $(source, dest)) + +when not defined(windows): + const maxSymlinkLen* = 1024 + +proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noNimJs, sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + ## + ## See also: + ## * `dirExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, filename) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 + else: + var res: Stat + return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) + + +proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], + noNimJs, sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + ## + ## See also: + ## * `fileExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, dir) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + else: + var res: Stat + result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) + + +proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], + noWeirdTarget, sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + ## + ## See also: + ## * `fileExists proc`_ + ## * `dirExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, link) + if a != -1'i32: + # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` + # may also be needed. + result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 + else: + var res: Stat + result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + +when defined(windows) and not weirdTarget: + proc openHandle*(path: string, followSymlink=true, writeAccess=false): Handle = + var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + if not followSymlink: + flags = flags or FILE_FLAG_OPEN_REPARSE_POINT + let access = if writeAccess: GENERIC_WRITE else: 0'i32 + + result = createFileW( + newWideCString(path), access, + FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, + nil, OPEN_EXISTING, flags, 0 + ) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim new file mode 100644 index 0000000000..b89a59c8dd --- /dev/null +++ b/lib/std/private/osdirs.nim @@ -0,0 +1,561 @@ +## .. importdoc:: osfiles.nim, appdirs.nim, paths.nim + +include system/inclrtl +import std/oserrors + + +import ospaths2, osfiles +import oscommon +export dirExists, PathComponent + + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/[posix, times] + +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +# Templates for filtering directories and files +when defined(windows) and not weirdTarget: + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool {.dirty.} = + dirExists(f) + template isFile(f: string): bool {.dirty.} = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` + when defined(windows): + var + f: WIN32_FIND_DATA + res: int + res = findFirstFile(pattern, f) + if res != -1: + defer: findClose(res) + let dotPos = searchExtPos(pattern) + while true: + if not skipFindData(f) and filter(f): + # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check + # that the file extensions have the same length ... + let ff = getFilename(f) + let idx = ff.len - pattern.len + dotPos + if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or + (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): + yield splitFile(pattern).dir / extractFilename(ff) + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: # here we use glob + var + f: Glob + res: int + f.gl_offs = 0 + f.gl_pathc = 0 + f.gl_pathv = nil + res = glob(pattern, 0, nil, addr(f)) + defer: globfree(addr(f)) + if res == 0: + for i in 0.. f.gl_pathc - 1: + assert(f.gl_pathv[i] != nil) + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files and directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + assert "lib/pure/os.nim".unixToNativePath in paths + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + walkCommon(pattern, isDir) + +proc staticWalkDir(dir: string; relative: bool): seq[ + tuple[kind: PathComponent, path: string]] = + discard + +iterator walkDir*(dir: string; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + ## + ## **Example:** + ## + ## This directory structure: + ## + ## dirA / dirB / fileB1.txt + ## / dirC + ## / fileA1.txt + ## / fileA2.txt + ## + ## and this code: + runnableExamples("-r:off"): + import std/[strutils, sugar] + # note: order is not guaranteed + # this also works at compile time + assert collect(for k in walkDir("dirA"): k.path).join(" ") == + "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDirRec iterator`_ + + when nimvm: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + else: + when weirdTarget: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + elif defined(windows): + var f: WIN32_FIND_DATA + var h = findFirstFile(dir / "*", f) + if h == -1: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: findClose(h) + while true: + var k = pcFile + if not skipFindData(f): + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + yield (k, xx) + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: + var d = opendir(dir) + if d == nil: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: discard closedir(d) + while true: + var x = readdir(d) + if x == nil: break + var y = $cast[cstring](addr x.d_name) + if y != "." and y != "..": + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + template resolveSymlink() = + var isSpecial: bool + (k, isSpecial) = getSymlinkFileKind(path) + if skipSpecial and isSpecial: continue + + template kSetGeneric() = # pure Posix component `k` resolution + if lstat(path.cstring, s) < 0'i32: continue # don't yield + elif S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + resolveSymlink() + elif skipSpecial and not S_ISREG(s.st_mode): continue + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + case x.d_type + of DT_DIR: k = pcDir + of DT_LNK: + resolveSymlink() + of DT_UNKNOWN: + kSetGeneric() + else: # DT_REG or special "files" like FIFOs + if skipSpecial and x.d_type != DT_REG: continue + else: discard # leave it as pcFile + else: # assuming that field `d_type` is not present + kSetGeneric() + + yield (k, y) + +iterator walkDirRec*(dir: string, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + string {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + + var stack = @[""] + var checkDir = checkDir + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir, + skipSpecial = skipSpecial): + let rel = d / p + if k in {pcDir, pcLinkToDir} and k in followFilter: + stack.add rel + if k in yieldFilter: + yield if relative: rel else: dir / rel + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. + # Future work can provide a way to customize this and do error reporting. + + +proc rawRemoveDir(dir: string) {.noWeirdTarget.} = + when defined(windows): + wrapUnary(res, removeDirectoryW, dir) + let lastError = osLastError() + if res == 0'i32 and lastError.int32 != 3'i32 and + lastError.int32 != 18'i32 and lastError.int32 != 2'i32: + raiseOSError(lastError, dir) + else: + if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) + +proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ + WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + for kind, path in walkDir(dir, checkDir = checkDir): + case kind + of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) + of pcDir: removeDir(path, true) + # for subdirectories there is no benefit in `checkDir = false` + # (unless perhaps for edge case of concurrent processes also deleting + # the same files) + rawRemoveDir(dir) + +proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = + # Try to create one directory (not the whole path). + # returns `true` for success, `false` if the path has previously existed + # + # This is a thin wrapper over mkDir (or alternatives on other systems), + # so in case of a pre-existing path we don't check that it is a directory. + when defined(solaris): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno in {EEXIST, ENOSYS}: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(haiku): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST or errno == EROFS: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(posix): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST: + result = false + else: + #echo res + raiseOSError(osLastError(), dir) + else: + wrapUnary(res, createDirectoryW, dir) + + if res != 0'i32: + result = true + elif getLastError() == 183'i32: + result = false + else: + raiseOSError(osLastError(), dir) + +proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + result = not rawCreateDir(dir) + if result: + # path already exists - need to check that it is indeed a directory + if not dirExists(dir): + raise newException(IOError, "Failed to create '" & dir & "'") + +proc createDir*(dir: string) {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + if dir == "": + return + var omitNext = isAbsolute(dir) + for p in parentDirs(dir, fromRoot=true): + if omitNext: + omitNext = false + else: + discard existsOrCreateDir(p) + +proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest`. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will copy the attributes from + ## `source` into `dest`. + ## + ## On other platforms created files and directories will inherit the + ## default permissions of a newly created file/directory for the user. + ## Use `copyDirWithPermissions proc`_ + ## to preserve attributes recursively on these platforms. + ## + ## See also: + ## * `copyDirWithPermissions proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + createDir(dest) + for kind, path in walkDir(source): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDir(path, dest / noSource) + else: + copyFile(path, dest / noSource, {cfSymlinkAsIs}) + + +proc copyDirWithPermissions*(source, dest: string, + ignorePermissionErrors = true) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], + benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs + ## on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file or directory + ## itself has been copied, which won't happen atomically and could lead to a + ## race condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `moveDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + createDir(dest) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + false) + except: + if not ignorePermissionErrors: + raise + for kind, path in walkDir(source): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) + else: + copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) + +proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + if not tryMoveFSObject(source, dest, isDir = true): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) + +proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + when defined(windows): + if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: + raiseOSError(osLastError(), newDir) + else: + if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim new file mode 100644 index 0000000000..a1d7079c5a --- /dev/null +++ b/lib/std/private/osfiles.nim @@ -0,0 +1,416 @@ +include system/inclrtl +import std/private/since +import std/oserrors + +import oscommon +export fileExists + +import ospaths2, ossymlinks + +## .. importdoc:: osdirs.nim, os.nim + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/[posix, times] + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +type + FilePermission* = enum ## File access permission, modelled after UNIX. + ## + ## See also: + ## * `getFilePermissions`_ + ## * `setFilePermissions`_ + ## * `FileInfo object`_ + fpUserExec, ## execute access for the file owner + fpUserWrite, ## write access for the file owner + fpUserRead, ## read access for the file owner + fpGroupExec, ## execute access for the group + fpGroupWrite, ## write access for the group + fpGroupRead, ## read access for the group + fpOthersExec, ## execute access for others + fpOthersWrite, ## write access for others + fpOthersRead ## read access for others + +proc getFilePermissions*(filename: string): set[FilePermission] {. + rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = + ## Retrieves file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is checked, every other + ## permission is available in any case. + ## + ## See also: + ## * `setFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var a: Stat + if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) + result = {} + if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) + if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) + if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) + + if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) + if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) + if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) + + if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) + if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) + if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: + result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, + fpOthersExec, fpOthersRead} + else: + result = {fpUserExec..fpOthersRead} + +proc setFilePermissions*(filename: string, permissions: set[FilePermission], + followSymlinks = true) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], + noWeirdTarget.} = + ## Sets the file permissions for `filename`. + ## + ## If `followSymlinks` set to true (default) and ``filename`` points to a + ## symlink, permissions are set to the file symlink points to. + ## `followSymlinks` set to false is a noop on Windows and some POSIX + ## systems (including Linux) on which `lchmod` is either unavailable or always + ## fails, given that symlinks permissions there are not observed. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is changed, depending on + ## ``fpUserWrite`` permission. + ## + ## See also: + ## * `getFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var p = 0.Mode + if fpUserRead in permissions: p = p or S_IRUSR.Mode + if fpUserWrite in permissions: p = p or S_IWUSR.Mode + if fpUserExec in permissions: p = p or S_IXUSR.Mode + + if fpGroupRead in permissions: p = p or S_IRGRP.Mode + if fpGroupWrite in permissions: p = p or S_IWGRP.Mode + if fpGroupExec in permissions: p = p or S_IXGRP.Mode + + if fpOthersRead in permissions: p = p or S_IROTH.Mode + if fpOthersWrite in permissions: p = p or S_IWOTH.Mode + if fpOthersExec in permissions: p = p or S_IXOTH.Mode + + if not followSymlinks and filename.symlinkExists: + when declared(lchmod): + if lchmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + if chmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if fpUserWrite in permissions: + res = res and not FILE_ATTRIBUTE_READONLY + else: + res = res or FILE_ATTRIBUTE_READONLY + wrapBinary(res2, setFileAttributesW, filename, res) + if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) + + +const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) + # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` + +when hasCCopyfile: + # `copyfile` API available since osx 10.5. + {.push nodecl, header: "<copyfile.h>".} + type + copyfile_state_t {.nodecl.} = pointer + copyfile_flags_t = cint + proc copyfile_state_alloc(): copyfile_state_t + proc copyfile_state_free(state: copyfile_state_t): cint + proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} + when (NimMajor, NimMinor) >= (1, 4): + let + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + else: + var + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + {.pop.} + +type + CopyFlag* = enum ## Copy options. + cfSymlinkAsIs, ## Copy symlinks as symlinks + cfSymlinkFollow, ## Copy the files symlinks point to + cfSymlinkIgnore ## Ignore symlinks + +const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} + +proc copyFile*(source, dest: string, options = {cfSymlinkFollow}; bufferSize = 16_384) {.rtl, + extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], + noWeirdTarget.} = + ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will + ## copy the source file's attributes into dest. + ## + ## On other platforms you need + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ + ## procs + ## to copy them by hand (or use the convenience `copyFileWithPermissions + ## proc`_), + ## otherwise `dest` will inherit the default permissions of a newly + ## created file for the user. + ## + ## If `dest` already exists, the file attributes + ## will be preserved and the content overwritten. + ## + ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless + ## `-d:nimLegacyCopyFile` is used. + ## + ## `copyFile` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyDir proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + + doAssert card(copyFlagSymlink * options) == 1, "There should be exactly one cfSymlink* in options" + let isSymlink = source.symlinkExists + if isSymlink and (cfSymlinkIgnore in options or defined(windows)): + return + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + if copyFileW(s, d, 0'i32) == 0'i32: + raiseOSError(osLastError(), $(source, dest)) + else: + if isSymlink and cfSymlinkAsIs in options: + createSymlink(expandSymlink(source), dest) + else: + when hasCCopyfile: + let state = copyfile_state_alloc() + # xxx `COPYFILE_STAT` could be used for one-shot + # `copyFileWithPermissions`. + let status = c_copyfile(source.cstring, dest.cstring, state, + COPYFILE_DATA) + if status != 0: + let err = osLastError() + discard copyfile_state_free(state) + raiseOSError(err, $(source, dest)) + let status2 = copyfile_state_free(state) + if status2 != 0: raiseOSError(osLastError(), $(source, dest)) + else: + # generic version of copyFile which works for any platform: + var d, s: File + if not open(s, source):raiseOSError(osLastError(), source) + if not open(d, dest, fmWrite): + close(s) + raiseOSError(osLastError(), dest) + + # Hints for kernel-level aggressive sequential low-fragmentation read-aheads: + # https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_fadvise.html + when defined(linux) or defined(osx): + discard posix_fadvise(getFileHandle(d), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + discard posix_fadvise(getFileHandle(s), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + + var buf = alloc(bufferSize) + while true: + var bytesread = readBuffer(s, buf, bufferSize) + if bytesread > 0: + var byteswritten = writeBuffer(d, buf, bytesread) + if bytesread != byteswritten: + dealloc(buf) + close(s) + close(d) + raiseOSError(osLastError(), dest) + if bytesread != bufferSize: break + dealloc(buf) + close(s) + flushFile(d) + close(d) + +proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}; bufferSize = 16_384) + {.noWeirdTarget, since: (1,3,7).} = + ## Copies a file `source` into directory `dir`, which must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## `copyFileToDir` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + if dir.len == 0: # treating "" as "." is error prone + raise newException(ValueError, "dest is empty") + copyFile(source, dir / source.lastPathPart, options, bufferSize) + + +proc copyFileWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + options = {cfSymlinkFollow}) {.noWeirdTarget.} = + ## Copies a file from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ + ## procs on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file itself has + ## been copied, which won't happen atomically and could lead to a race + ## condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + ## * `copyDir proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + ## * `copyDirWithPermissions proc`_ + copyFile(source, dest, options) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + (cfSymlinkFollow in options)) + except: + if not ignorePermissionErrors: + raise + +when not declared(ENOENT) and not defined(windows): + when defined(nimscript): + when not defined(haiku): + const ENOENT = cint(2) # 2 on most systems including Solaris + else: + const ENOENT = cint(-2147459069) + else: + var ENOENT {.importc, header: "<errno.h>".}: cint + +when defined(windows) and not weirdTarget: + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = + setFileAttributesW(file, attrs) + +proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, returns `false`. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + result = true + when defined(windows): + let f = newWideCString(file) + if deleteFile(f) == 0: + result = false + let err = getLastError() + if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: + result = true + elif err == ERROR_ACCESS_DENIED and + setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and + deleteFile(f) != 0: + result = true + else: + if unlink(file) != 0'i32 and errno != ENOENT: + result = false + +proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `moveFile proc`_ + if not tryRemoveFile(file): + raiseOSError(osLastError(), file) + +proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `tryRemoveFile proc`_ + + if not tryMoveFSObject(source, dest, isDir = false): + when defined(windows): + raiseAssert "unreachable" + else: + # Fallback to copy & del + copyFileWithPermissions(source, dest, options={cfSymlinkAsIs}) + try: + removeFile(source) + except: + discard tryRemoveFile(dest) + raise diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim new file mode 100644 index 0000000000..bc69ff7256 --- /dev/null +++ b/lib/std/private/ospaths2.nim @@ -0,0 +1,1030 @@ +include system/inclrtl +import std/private/since + +import std/[strutils, pathnorm] +import std/oserrors + +import oscommon +export ReadDirEffect, WriteDirEffect + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osappdirs.nim, osdirs.nim, osseps.nim, os.nim + +const weirdTarget = defined(nimscript) or defined(js) + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix, system/ansi_c +else: + {.error: "OS module not ported to your operating system!".} + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} + + +import std/private/osseps +export osseps + +proc absolutePathInternal(path: string): string {.gcsafe.} + +proc normalizePathEnd*(path: var string, trailingSep = false) = + ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on + ## ``trailingSep``, and taking care of edge cases: it preservers whether + ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, + ## not `AltSep`. Trailing `/.` are compressed, see examples. + if path.len == 0: return + var i = path.len + while i >= 1: + if path[i-1] in {DirSep, AltSep}: dec(i) + elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) + else: break + if trailingSep: + # foo// => foo + path.setLen(i) + # foo => foo/ + path.add DirSep + elif i > 0: + # foo// => foo + path.setLen(i) + else: + # // => / (empty case was already taken care of) + path = $DirSep + +proc normalizePathEnd*(path: string, trailingSep = false): string = + ## outplace overload + runnableExamples: + when defined(posix): + assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" + assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" + assert normalizePathEnd(".//./.", trailingSep = false) == "." + assert normalizePathEnd("", trailingSep = true) == "" # not / ! + assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! + result = path + result.normalizePathEnd(trailingSep) + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +proc joinPathImpl(result: var string, state: var int, tail: string) = + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) + normalizePathEnd(result, trailingSep=false) + addNormalizePath(tail, result, state, DirSep) + normalizePathEnd(result, trailingSep=trailingSep) + +proc joinPath*(head, tail: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert joinPath("usr", "lib") == "usr/lib" + assert joinPath("usr", "lib/") == "usr/lib/" + assert joinPath("usr", "") == "usr" + assert joinPath("usr/", "") == "usr/" + assert joinPath("", "") == "" + assert joinPath("", "lib") == "lib" + assert joinPath("", "/lib") == "/lib" + assert joinPath("usr/", "/lib") == "usr/lib" + assert joinPath("usr/lib", "../bin") == "usr/bin" + + result = newStringOfCap(head.len + tail.len) + var state = 0 + joinPathImpl(result, state, head) + joinPathImpl(result, state, tail) + when false: + if len(head) == 0: + result = tail + elif head[len(head)-1] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & substr(tail, 1) + else: + result = head & tail + else: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & tail + else: + result = head & DirSep & tail + +proc joinPath*(parts: varargs[string]): string {.noSideEffect, + rtl, extern: "nos$1OpenArray".} = + ## The same as `joinPath(head, tail) proc`_, + ## but works with any number of directory parts. + ## + ## You need to pass at least one element or the proc + ## will assert in debug builds and crash on release builds. + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `splitPath proc`_ + runnableExamples: + when defined(posix): + assert joinPath("a") == "a" + assert joinPath("a", "b", "c") == "a/b/c" + assert joinPath("usr/lib", "../../var", "log") == "var/log" + + var estimatedLen = 0 + for p in parts: estimatedLen += p.len + result = newStringOfCap(estimatedLen) + var state = 0 + for i in 0..high(parts): + joinPathImpl(result, state, parts[i]) + +proc `/`*(head, tail: string): string {.noSideEffect, inline.} = + ## The same as `joinPath(head, tail) proc`_. + ## + ## See also: + ## * `/../ proc`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert "usr" / "" == "usr" + assert "" / "lib" == "lib" + assert "" / "/lib" == "/lib" + assert "usr/" / "/lib/" == "usr/lib/" + assert "usr" / "lib" / "../bin" == "usr/bin" + + result = joinPath(head, tail) + +when doslikeFileSystem: + import std/private/ntpath + +proc splitPath*(path: string): tuple[head, tail: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + runnableExamples: + assert splitPath("usr/local/bin") == ("usr/local", "bin") + assert splitPath("usr/local/bin/") == ("usr/local/bin", "") + assert splitPath("/bin/") == ("/bin", "") + when (NimMajor, NimMinor) <= (1, 0): + assert splitPath("/bin") == ("", "bin") + else: + assert splitPath("/bin") == ("/", "bin") + assert splitPath("bin") == ("", "bin") + assert splitPath("") == ("", "") + + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(path) + let stop = drive.len + else: + const stop = 0 + + var sepPos = -1 + for i in countdown(len(path)-1, stop): + if path[i] in {DirSep, AltSep}: + sepPos = i + break + if sepPos >= 0: + result.head = substr(path, 0, + when (NimMajor, NimMinor) <= (1, 0): + sepPos-1 + else: + if likely(sepPos >= 1): sepPos-1 else: 0 + ) + result.tail = substr(path, sepPos+1) + else: + when doslikeFileSystem: + result.head = drive + result.tail = splitpath + else: + result.head = "" + result.tail = path + +proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + runnableExamples: + assert not "".isAbsolute + assert not ".".isAbsolute + when defined(posix): + assert "/".isAbsolute + assert not "a/".isAbsolute + assert "/a/".isAbsolute + + if len(path) == 0: return false + + when doslikeFileSystem: + var len = len(path) + result = (path[0] in {'/', '\\'}) or + (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') + elif defined(macos): + # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path + result = path[0] != ':' + elif defined(RISCOS): + result = path[0] == '$' + elif defined(posix): + result = path[0] == '/' + elif defined(nodejs): + {.emit: [result," = require(\"path\").isAbsolute(",path.cstring,");"].} + else: + raiseAssert "unreachable" # if ever hits here, adapt as needed + +when FileSystemCaseSensitive: + template `!=?`(a, b: char): bool = a != b +else: + template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) + +when doslikeFileSystem: + proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = + ## An absolute path from the root of the current drive (e.g. "\foo") + path.len > 0 and + (path[0] == AltSep or + (path[0] == DirSep and + (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) + + proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = + ## Return true if path1 and path2 have a same root. + ## + ## Detail of Windows path formats: + ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats + + assert(isAbsolute(path1)) + assert(isAbsolute(path2)) + + if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): + result = true + elif cmpIgnoreCase(splitDrive(path1).drive, splitDrive(path2).drive) == 0: + result = true + else: + result = false + +proc relativePath*(path, base: string, sep = DirSep): string {. + rtl, extern: "nos$1".} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep_) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + runnableExamples: + assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" + when not doslikeFileSystem: # On Windows, UNC-paths start with `//` + assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" + assert relativePath("", "/users/moo", '/') == "" + assert relativePath("foo", ".", '/') == "foo" + assert relativePath("foo", "foo", '/') == "." + + if path.len == 0: return "" + var base = if base == ".": "" else: base + var path = path + path.normalizePathAux + base.normalizePathAux + let a1 = isAbsolute(path) + let a2 = isAbsolute(base) + if a1 and not a2: + base = absolutePathInternal(base) + elif a2 and not a1: + path = absolutePathInternal(path) + + when doslikeFileSystem: + if isAbsolute(path) and isAbsolute(base): + if not sameRoot(path, base): + return path + + var f = default PathIter + var b = default PathIter + var ff = (0, -1) + var bb = (0, -1) # (int, int) + result = newStringOfCap(path.len) + # skip the common prefix: + while f.hasNext(path) and b.hasNext(base): + ff = next(f, path) + bb = next(b, base) + let diff = ff[1] - ff[0] + if diff != bb[1] - bb[0]: break + var same = true + for i in 0..diff: + if path[i + ff[0]] !=? base[i + bb[0]]: + same = false + break + if not same: break + ff = (0, -1) + bb = (0, -1) + # for i in 0..diff: + # result.add base[i + bb[0]] + + # /foo/bar/xxx/ -- base + # /foo/bar/baz -- path path + # ../baz + # every directory that is in 'base', needs to add '..' + while true: + if bb[1] >= bb[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + result.add ".." + if not b.hasNext(base): break + bb = b.next(base) + + # add the rest of 'path': + while true: + if ff[1] >= ff[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + for i in 0..ff[1] - ff[0]: + result.add path[i + ff[0]] + if not f.hasNext(path): break + ff = f.next(path) + + when not defined(nimOldRelativePathBehavior): + if result.len == 0: result.add "." + +proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = + ## Returns true if `path` is relative to `base`. + runnableExamples: + doAssert isRelativeTo("./foo//bar", "foo") + doAssert isRelativeTo("foo/bar", ".") + doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") + doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") + let path = path.normalizePath + let base = base.normalizePath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." + +proc parentDirPos(path: string): int = + var q = 1 + if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 + for i in countdown(len(path)-q, 0): + if path[i] in {DirSep, AltSep}: return i + result = -1 + +proc parentDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + runnableExamples: + assert parentDir("") == "" + when defined(posix): + assert parentDir("/usr/local/bin") == "/usr/local" + assert parentDir("foo/bar//") == "foo" + assert parentDir("//foo//bar//.") == "/foo" + assert parentDir("./foo") == "." + assert parentDir("/./foo//./") == "/" + assert parentDir("a//./") == "." + assert parentDir("a/b/c/..") == "a" + result = pathnorm.normalizePath(path) + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(result) + result = splitpath + var sepPos = parentDirPos(result) + if sepPos >= 0: + result = substr(result, 0, sepPos) + normalizePathEnd(result) + elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: + # `.` => `..` and .. => `../..`(etc) would be a sensible alternative + # `/` => `/` (as done with splitFile) would be a sensible alternative + result = "" + else: + result = "." + when doslikeFileSystem: + if result.len == 0: + discard + elif drive.len > 0 and result.len == 1 and result[0] in {DirSep, AltSep}: + result = drive + else: + result = drive & result + +proc tailDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + runnableExamples: + assert tailDir("/bin") == "bin" + assert tailDir("bin") == "" + assert tailDir("bin/") == "" + assert tailDir("/usr/local/bin") == "usr/local/bin" + assert tailDir("//usr//local//bin//") == "usr//local//bin//" + assert tailDir("./usr/local/bin") == "usr/local/bin" + assert tailDir("usr/local/bin") == "local/bin" + + var i = 0 + when doslikeFileSystem: + let (drive, splitpath) = path.splitDrive + if drive != "": + return splitpath.strip(chars = {DirSep, AltSep}, trailing = false) + while i < len(path): + if path[i] in {DirSep, AltSep}: + while i < len(path) and path[i] in {DirSep, AltSep}: inc i + return substr(path, i) + inc i + result = "" + +proc isRootDir*(path: string): bool {. + noSideEffect, rtl, extern: "nos$1".} = + ## Checks whether a given `path` is a root directory. + runnableExamples: + assert isRootDir("") + assert isRootDir(".") + assert isRootDir("/") + assert isRootDir("a") + assert not isRootDir("/a") + assert not isRootDir("a/b/c") + + when doslikeFileSystem: + if splitDrive(path).path == "": + return true + result = parentDirPos(path) < 0 + +iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + runnableExamples: + let g = "a/b/c" + + for p in g.parentDirs: + echo p + # a/b/c + # a/b + # a + + for p in g.parentDirs(fromRoot=true): + echo p + # a/ + # a/b/ + # a/b/c + + for p in g.parentDirs(inclusive=false): + echo p + # a/b + # a + + if not fromRoot: + var current = path + if inclusive: yield path + while true: + if current.isRootDir: break + current = current.parentDir + yield current + else: + when doslikeFileSystem: + let start = path.splitDrive.drive.len + else: + const start = 0 + for i in countup(start, path.len - 2): # ignore the last / + # deal with non-normalized paths such as /foo//bar//baz + if path[i] in {DirSep, AltSep} and + (i == 0 or path[i-1] notin {DirSep, AltSep}): + yield path.substr(0, i) + + if inclusive: yield path + +proc `/../`*(head, tail: string): string {.noSideEffect.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + runnableExamples: + when defined(posix): + assert "a/b/c" /../ "d/e" == "a/b/d/e" + assert "a" /../ "d/e" == "a/d/e" + + when doslikeFileSystem: + let (drive, head) = splitDrive(head) + let sepPos = parentDirPos(head) + if sepPos >= 0: + result = substr(head, 0, sepPos-1) / tail + else: + result = head / tail + when doslikeFileSystem: + result = drive / result + +proc normExt(ext: string): string = + if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here + else: result = ExtSep & ext + +proc searchExtPos*(path: string): int = + ## Returns index of the `'.'` char in `path` if it signifies the beginning + ## of the file extension. Returns -1 otherwise. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert searchExtPos("a/b/c") == -1 + assert searchExtPos("c.nim") == 1 + assert searchExtPos("a/b/c.nim") == 5 + assert searchExtPos("a.b.c.nim") == 5 + assert searchExtPos(".nim") == -1 + assert searchExtPos("..nim") == -1 + assert searchExtPos("a..nim") == 2 + + # Unless there is any char that is not `ExtSep` before last `ExtSep` in the file name, + # it is not a file extension. + const DirSeps = when doslikeFileSystem: {DirSep, AltSep, ':'} else: {DirSep, AltSep} + result = -1 + var i = path.high + while i >= 1: + if path[i] == ExtSep: + break + elif path[i] in DirSeps: + return -1 # do not skip over path + dec i + + for j in countdown(i - 1, 0): + if path[j] in DirSeps: + return -1 + elif path[j] != ExtSep: + result = i + break + +proc splitFile*(path: string): tuple[dir, name, ext: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep_ unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + var (dir, name, ext) = splitFile("usr/local/nimc.html") + assert dir == "usr/local" + assert name == "nimc" + assert ext == ".html" + (dir, name, ext) = splitFile("/usr/local/os") + assert dir == "/usr/local" + assert name == "os" + assert ext == "" + (dir, name, ext) = splitFile("/usr/local/") + assert dir == "/usr/local" + assert name == "" + assert ext == "" + (dir, name, ext) = splitFile("/tmp.txt") + assert dir == "/" + assert name == "tmp" + assert ext == ".txt" + + var namePos = 0 + var dotPos = 0 + when doslikeFileSystem: + let (drive, _) = splitDrive(path) + let stop = len(drive) + result.dir = drive + else: + const stop = 0 + for i in countdown(len(path) - 1, stop): + if path[i] in {DirSep, AltSep} or i == 0: + if path[i] in {DirSep, AltSep}: + result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) + namePos = i + 1 + if dotPos > i: + result.name = substr(path, namePos, dotPos - 1) + result.ext = substr(path, dotPos) + else: + result.name = substr(path, namePos) + break + elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and + path[i - 1] notin {DirSep, AltSep} and + path[i + 1] != ExtSep and dotPos == 0: + dotPos = i + +proc extractFilename*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert extractFilename("foo/bar/") == "" + assert extractFilename("foo/bar") == "bar" + assert extractFilename("foo/bar.baz") == "bar.baz" + + if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: + result = "" + else: + result = splitPath(path).tail + +proc lastPathPart*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert lastPathPart("foo/bar/") == "bar" + assert lastPathPart("foo/bar") == "bar" + + let path = path.normalizePathEnd(trailingSep = false) + result = extractFilename(path) + +proc changeFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert changeFileExt("foo.bar", "baz") == "foo.baz" + assert changeFileExt("foo.bar", "") == "foo" + assert changeFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = substr(filename, 0, extPos-1) & normExt(ext) + +proc addFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + runnableExamples: + assert addFileExt("foo.bar", "baz") == "foo.bar" + assert addFileExt("foo.bar", "") == "foo.bar" + assert addFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = filename + +proc cmpPaths*(pathA, pathB: string): int {. + noSideEffect, rtl, extern: "nos$1".} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. Returns: + ## + ## | `0` if pathA == pathB + ## | `< 0` if pathA < pathB + ## | `> 0` if pathA > pathB + runnableExamples: + when defined(macosx): + assert cmpPaths("foo", "Foo") == 0 + elif defined(posix): + assert cmpPaths("foo", "Foo") > 0 + + let a = normalizePath(pathA) + let b = normalizePath(pathB) + if FileSystemCaseSensitive: + result = cmp(a, b) + else: + when defined(nimscript): + result = cmpic(a, b) + elif defined(nimdoc): discard + else: + result = cmpIgnoreCase(a, b) + +proc unixToNativePath*(path: string, drive=""): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + when defined(unix): + result = path + else: + if path.len == 0: return "" + + var start: int + if path[0] == '/': + # an absolute path + when doslikeFileSystem: + if drive != "": + result = drive & ":" & DirSep + else: + result = $DirSep + elif defined(macos): + result = "" # must not start with ':' + else: + result = $DirSep + start = 1 + elif path[0] == '.' and (path.len == 1 or path[1] == '/'): + # current directory + result = $CurDir + start = when doslikeFileSystem: 1 else: 2 + else: + result = "" + start = 0 + + var i = start + while i < len(path): # ../../../ --> :::: + if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': + # parent directory + when defined(macos): + if result[high(result)] == ':': + add result, ':' + else: + add result, ParDir + else: + add result, ParDir & DirSep + inc(i, 3) + elif path[i] == '/': + add result, DirSep + inc(i) + else: + add result, path[i] + inc(i) + + +when not defined(nimscript): + proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `setCurrentDir proc`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + when defined(nodejs): + var ret: cstring + {.emit: "`ret` = process.cwd();".} + return $ret + elif defined(js): + raiseAssert "use -d:nodejs to have `getCurrentDir` defined" + elif defined(windows): + var bufsize = MAX_PATH.int32 + var res = newWideCString(bufsize) + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break + else: + var bufsize = 1024 # should be enough + result = newString(bufsize) + while true: + if getcwd(result.cstring, bufsize) != nil: + setLen(result, c_strlen(result.cstring)) + break + else: + let err = osLastError() + if err.int32 == ERANGE: + bufsize = bufsize shl 1 + doAssert(bufsize >= 0) + result = newString(bufsize) + else: + raiseOSError(osLastError()) + +proc absolutePath*(path: string, root = getCurrentDir()): string = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizedPath proc`_ + ## * `normalizePath proc`_ + runnableExamples: + assert absolutePath("a") == getCurrentDir() / "a" + + if isAbsolute(path): path + else: + if not root.isAbsolute: + raise newException(ValueError, "The specified root is not absolute: " & root) + joinPath(root, path) + +proc absolutePathInternal(path: string): string = + absolutePath(path, getCurrentDir()) + + +proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = + ## Normalize a path. + ## + ## Consecutive directory separators are collapsed, including an initial double slash. + ## + ## On relative paths, double dot (`..`) sequences are collapsed if possible. + ## On absolute paths they are always collapsed. + ## + ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. + ## Triple dot is not handled. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizedPath proc`_ for outplace version + ## * `normalizeExe proc`_ + runnableExamples: + when defined(posix): + var a = "a///b//..//c///d" + a.normalizePath() + assert a == "a/c/d" + + path = pathnorm.normalizePath(path) + when false: + let isAbs = isAbsolute(path) + var stack: seq[string] = @[] + for p in split(path, {DirSep}): + case p + of "", ".": + continue + of "..": + if stack.len == 0: + if isAbs: + discard # collapse all double dots on absoluta paths + else: + stack.add(p) + elif stack[^1] == "..": + stack.add(p) + else: + discard stack.pop() + else: + stack.add(p) + + if isAbs: + path = DirSep & join(stack, $DirSep) + elif stack.len > 0: + path = join(stack, $DirSep) + else: + path = "." + +proc normalizePathAux(path: var string) = normalizePath(path) + +proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns a normalized path for the current OS. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizePath proc`_ for the in-place version + runnableExamples: + when defined(posix): + assert normalizedPath("a///b//..//c///d") == "a/c/d" + result = pathnorm.normalizePath(path) + +proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = + ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. + runnableExamples: + import std/sugar + when defined(posix): + doAssert "foo".dup(normalizeExe) == "./foo" + doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" + doAssert "".dup(normalizeExe) == "" + when defined(posix): + if file.len > 0 and DirSep notin file and file != "." and file != "..": + file = "./" & file + +proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noWeirdTarget.} = + ## Returns true if both pathname arguments refer to the same physical + ## file or directory. + ## + ## Raises `OSError` if any of the files does not + ## exist or information about it can not be obtained. + ## + ## This proc will return true if given two alternative hard-linked or + ## sym-linked paths to the same file or directory. + ## + ## See also: + ## * `sameFileContent proc`_ + when defined(windows): + var success = true + var f1 = openHandle(path1) + var f2 = openHandle(path2) + + var lastErr: OSErrorCode + if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: + var fi1, fi2: BY_HANDLE_FILE_INFORMATION + + if getFileInformationByHandle(f1, addr(fi1)) != 0 and + getFileInformationByHandle(f2, addr(fi2)) != 0: + result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and + fi1.nFileIndexHigh == fi2.nFileIndexHigh and + fi1.nFileIndexLow == fi2.nFileIndexLow + else: + lastErr = osLastError() + success = false + else: + lastErr = osLastError() + success = false + + discard closeHandle(f1) + discard closeHandle(f2) + + if not success: raiseOSError(lastErr, $(path1, path2)) + else: + var a, b: Stat + if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: + raiseOSError(osLastError(), $(path1, path2)) + else: + result = a.st_dev == b.st_dev and a.st_ino == b.st_ino diff --git a/lib/pure/includes/osseps.nim b/lib/std/private/osseps.nim similarity index 94% rename from lib/pure/includes/osseps.nim rename to lib/std/private/osseps.nim index 10c85047bc..f2d49d8861 100644 --- a/lib/pure/includes/osseps.nim +++ b/lib/std/private/osseps.nim @@ -3,6 +3,8 @@ # Improved based on info in 'compiler/platform.nim' +## .. importdoc:: ospaths2.nim + const doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) @@ -37,9 +39,9 @@ const when doslikeFileSystem: '/' else: DirSep ## An alternative character used by the operating system to separate - ## pathname components, or the same as `DirSep <#DirSep>`_ if only one separator + ## pathname components, or the same as DirSep_ if only one separator ## character exists. This is set to `'/'` on Windows systems - ## where `DirSep <#DirSep>`_ is a backslash (`'\\'`). + ## where DirSep_ is a backslash (`'\\'`). PathSep* = when defined(macos) or defined(RISCOS): ',' @@ -55,7 +57,7 @@ const defined(PalmOS) or defined(MorphOS): false else: true ## True if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths proc <#cmpPaths,string,string>`_ to compare filenames properly. + ## `cmpPaths proc`_ to compare filenames properly. ExeExt* = when doslikeFileSystem: "exe" diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim new file mode 100644 index 0000000000..c1760c42ec --- /dev/null +++ b/lib/std/private/ossymlinks.nim @@ -0,0 +1,78 @@ +include system/inclrtl +import std/oserrors + +import oscommon +export symlinkExists + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +## .. importdoc:: os.nim + +proc createSymlink*(src, dest: string) {.noWeirdTarget.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + + when defined(windows): + const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 + # allows anyone with developer mode on to create a link + let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + raiseOSError(osLastError(), $(src, dest)) + else: + if symlink(src, dest) != 0: + raiseOSError(osLastError(), $(src, dest)) + +proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + when defined(windows) or defined(nintendoswitch): + result = symlinkPath + else: + var bufLen = 1024 + while true: + result = newString(bufLen) + let len = readlink(symlinkPath.cstring, result.cstring, bufLen) + if len < 0: + raiseOSError(osLastError(), symlinkPath) + if len < bufLen: + result.setLen(len) + break + bufLen = bufLen shl 1 diff --git a/lib/std/private/schubfach.nim b/lib/std/private/schubfach.nim new file mode 100644 index 0000000000..b8c85d2bc7 --- /dev/null +++ b/lib/std/private/schubfach.nim @@ -0,0 +1,436 @@ +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +# -------------------------------------------------------------------------------------------------- +## This file contains an implementation of the Schubfach algorithm as described in +## +## \[1] Raffaello Giulietti, "The Schubfach way to render doubles", +## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN +# -------------------------------------------------------------------------------------------------- + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + + +template sf_Assert(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType = float32 + BitsType = uint32 + Single {.bycopy.} = object + bits: BitsType + +const + significandSize: int32 = 24 + MaxExponent = 128 + exponentBias: int32 = MaxExponent - 1 + (significandSize - 1) + maxIeeeExponent: BitsType = BitsType(2 * MaxExponent - 1) + hiddenBit: BitsType = BitsType(1) shl (significandSize - 1) + significandMask: BitsType = hiddenBit - 1 + exponentMask: BitsType = maxIeeeExponent shl (significandSize - 1) + signMask: BitsType = not (not BitsType(0) shr 1) + +proc constructSingle(bits: BitsType): Single = + result.bits = bits + +proc constructSingle(value: ValueType): Single = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand(this: Single): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent(this: Single): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero(this: Single): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit(this: Single): int {.noSideEffect.} = + return int((this.bits and signMask) != 0) + +# ================================================================================================== +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2(x: int32; n: int32): int32 {.inline.} = + return x shr n + +## Returns floor(log_10(2^e)) +## ```c +## static inline int32_t FloorLog10Pow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611, 22); +## } +## ``` +## Returns floor(log_10(3/4 2^e)) +## ```c +## static inline int32_t FloorLog10ThreeQuartersPow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611 - 524031, 22); +## } +## ``` +## Returns floor(log_2(10^e)) + +proc floorLog2Pow10(e: int32): int32 {.inline.} = + sf_Assert(e >= -1233) + sf_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + +proc computePow10Single(k: int32): uint64 {.inline.} = + ## There are unique beta and r such that 10^k = beta 2^r and + ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and + ## beta = 2^-r 10^k. + ## Let g = ceil(beta), so (g-1) 2^r < 10^k <= g 2^r, with the latter + ## value being a pretty good overestimate for 10^k. + ## NB: Since for all the required exponents k, we have g < 2^64, + ## all constants can be stored in 128-bit integers. + sf_Assert(k >= kMin) + sf_Assert(k <= kMax) + return g[k - kMin] + +proc lo32(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + +proc hi32(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + +when defined(sizeof_Int128): + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let p: uint128 = uint128(g) * cp + let y1: uint32 = lo32(cast[uint64](p shr 64)) + let y0: uint32 = hi32(cast[uint64](p)) + return y1 or uint32(y0 > 1) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = + var p1: uint64 = 0 + var p0: uint64 = umul128(g, cpHi, addr(p1)) + let y1: uint32 = lo32(p1) + let y0: uint32 = hi32(p0) + return y1 or uint32(y0 > 1) + +else: + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let b01: uint64 = uint64(lo32(g)) * cp + let b11: uint64 = uint64(hi32(g)) * cp + let hi: uint64 = b11 + hi32(b01) + let y1: uint32 = hi32(hi) + let y0: uint32 = lo32(hi) + return y1 or uint32(y0 > 1) + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2(value: uint32; e2: int32): bool {.inline.} = + sf_Assert(e2 >= 0) + sf_Assert(e2 <= 31) + return (value and ((uint32(1) shl e2) - 1)) == 0 + +type + FloatingDecimal32 {.bycopy.} = object + digits: uint32 ## num_digits <= 9 + exponent: int32 + +proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal32 {. + inline.} = + var c: uint32 + var q: int32 + if ieeeExponent != 0: + c = hiddenBit or ieeeSignificand + q = cast[int32](ieeeExponent) - exponentBias + if 0 <= -q and -q < significandSize and multipleOfPow2(c, -q): + return FloatingDecimal32(digits: c shr -q, exponent: 0'i32) + else: + c = ieeeSignificand + q = 1 - exponentBias + let isEven: bool = (c mod 2 == 0) + let lowerBoundaryIsCloser: bool = (ieeeSignificand == 0 and ieeeExponent > 1) + ## const int32_t qb = q - 2; + let cbl: uint32 = 4 * c - 2 + uint32(lowerBoundaryIsCloser) + let cb: uint32 = 4 * c + let cbr: uint32 = 4 * c + 2 + ## (q * 1262611 ) >> 22 == floor(log_10( 2^q)) + ## (q * 1262611 - 524031) >> 22 == floor(log_10(3/4 2^q)) + sf_Assert(q >= -1500) + sf_Assert(q <= 1500) + let k: int32 = floorDivPow2(q * 1262611 - (if lowerBoundaryIsCloser: 524031 else: 0), 22) + let h: int32 = q + floorLog2Pow10(-k) + 1 + sf_Assert(h >= 1) + sf_Assert(h <= 4) + let pow10: uint64 = computePow10Single(-k) + let vbl: uint32 = roundToOdd(pow10, cbl shl h) + let vb: uint32 = roundToOdd(pow10, cb shl h) + let vbr: uint32 = roundToOdd(pow10, cbr shl h) + let lower: uint32 = vbl + uint32(not isEven) + let upper: uint32 = vbr - uint32(not isEven) + ## See Figure 4 in [1]. + ## And the modifications in Figure 6. + let s: uint32 = vb div 4 + ## NB: 4 * s == vb & ~3 == vb & -4 + if s >= 10: + let sp: uint32 = s div 10 + ## = vb / 40 + let upInside: bool = lower <= 40 * sp + let wpInside: bool = 40 * sp + 40 <= upper + ## if (up_inside || wp_inside) // NB: At most one of u' and w' is in R_v. + if upInside != wpInside: + return FloatingDecimal32(digits: sp + uint32(wpInside), exponent: k + 1) + let uInside: bool = lower <= 4 * s + let wInside: bool = 4 * s + 4 <= upper + if uInside != wInside: + return FloatingDecimal32(digits: s + uint32(wInside), exponent: k) + let mid: uint32 = 4 * s + 2 + ## = 2(s + t) + let roundUp: bool = vb > mid or (vb == mid and (s and 1) != 0) + return FloatingDecimal32(digits: s + uint32(roundUp), exponent: k) + +## ================================================================================================== +## ToChars +## ================================================================================================== + +proc printDecimalDigitsBackwards[T: Ordinal](buf: var openArray[char]; pos: T; output: uint32): int {.inline.} = + var output = output + var pos = pos + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 9 digits remaining + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + tz = trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + else: + tz = 4 + nd = 4 + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + sf_Assert(output >= 1) + sf_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + sf_Assert(q >= 1) + sf_Assert(q <= 9) + dec(pos) + buf[pos] = chr(uint32('0') + q) + return tz + +proc decimalLength(v: uint32): int {.inline.} = + sf_Assert(v >= 1) + sf_Assert(v <= 999999999'u) + if v >= 100000000'u: + return 9 + if v >= 10000000'u: + return 8 + if v >= 1000000'u: + return 7 + if v >= 100000'u: + return 6 + if v >= 10000'u: + return 5 + if v >= 1000'u: + return 4 + if v >= 100'u: + return 3 + if v >= 10'u: + return 2 + return 1 + +proc formatDigits[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint32; decimalExponent: int; + forceTrailingDotZero: bool = false): int {.inline.} = + const + minFixedDecimalPoint: int32 = -4 + maxFixedDecimalPoint: int32 = 9 + var pos = pos + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 1, "internal error") + sf_Assert(digits >= 1) + sf_Assert(digits <= 999999999'u) + sf_Assert(decimalExponent >= -99) + sf_Assert(decimalExponent <= 99) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + ## Avoid calling memset/memcpy with variable arguments below... + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + decimalDigitsPosition + numDigits + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + for i in countdown(7, 0): + buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(uint32('0') + k) + inc pos + else: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + return pos + +proc float32ToChars*(buffer: var openArray[char]; v: float32; forceTrailingDotZero = false): int {. + inline.} = + let significand: uint32 = physicalSignificand(constructSingle(v)) + let exponent: uint32 = physicalExponent(constructSingle(v)) + var pos = 0 + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec: auto = toDecimal32(significand, exponent) + return formatDigits(buffer, pos, dec.digits, dec.exponent.int, forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 diff --git a/lib/std/private/since.nim b/lib/std/private/since.nim index 5b22b63917..720120f117 100644 --- a/lib/std/private/since.nim +++ b/lib/std/private/since.nim @@ -1,5 +1,5 @@ ##[ -`since` is used to emulate older versions of nim stdlib with `--useVersion`, +`since` is used to emulate older versions of nim stdlib, see `tuse_version.nim`. If a symbol `foo` is added in version `(1,3,5)`, use `{.since: (1.3.5).}`, not @@ -15,19 +15,19 @@ The emulation cannot be 100% faithful and to avoid adding too much complexity, template since*(version: (int, int), body: untyped) {.dirty.} = ## Evaluates `body` if the ``(NimMajor, NimMinor)`` is greater than ## or equal to `version`. Usage: - ## - ## .. code-block:: Nim + ## ```Nim ## proc fun*() {.since: (1, 3).} ## since (1, 3): fun() + ## ``` when (NimMajor, NimMinor) >= version: body template since*(version: (int, int, int), body: untyped) {.dirty.} = ## Evaluates `body` if ``(NimMajor, NimMinor, NimPatch)`` is greater than ## or equal to `version`. Usage: - ## - ## .. code-block:: Nim + ## ```Nim ## proc fun*() {.since: (1, 3, 1).} ## since (1, 3, 1): fun() + ## ``` when (NimMajor, NimMinor, NimPatch) >= version: body diff --git a/lib/std/private/strimpl.nim b/lib/std/private/strimpl.nim index 7d42a7cf83..f8c9236a5e 100644 --- a/lib/std/private/strimpl.nim +++ b/lib/std/private/strimpl.nim @@ -74,3 +74,40 @@ template endsWithImpl*[T: string | cstring](s, suffix: T) = func cmpNimIdentifier*[T: string | cstring](a, b: T): int = cmpIgnoreStyleImpl(a, b, true) + +func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} +func c_strstr(haystack, needle: cstring): cstring {. + importc: "strstr", header: "<string.h>".} + + +func find*(s: cstring, sub: char, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].rfind` for a `start`-origin index. + let last = if last == 0: s.high else: last + let L = last-start+1 + if L > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + if not found.isNil: + return cast[int](found) -% cast[int](s) + return -1 + +func find*(s, sub: cstring, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].find` for a `start`-origin index. + if sub.len > s.len - start: return -1 + if sub.len == 1: return find(s, sub[0], start, last) + if last == 0 and s.len > start: + let found = c_strstr(cast[cstring](s[start].unsafeAddr), sub) + if not found.isNil: + result = cast[int](found) -% cast[int](s) + else: + result = -1 diff --git a/lib/std/private/syslocks.nim b/lib/std/private/syslocks.nim new file mode 100644 index 0000000000..e19ec2c04d --- /dev/null +++ b/lib/std/private/syslocks.nim @@ -0,0 +1,234 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level system locks and condition vars. + +{.push stackTrace: off.} + +when defined(windows): + type + Handle = int + + SysLock* {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure, byref.} = object # CRITICAL_SECTION in WinApi + DebugInfo: pointer + LockCount: int32 + RecursionCount: int32 + OwningThread: int + LockSemaphore: int + SpinCount: int + + SysCond* {.importc: "RTL_CONDITION_VARIABLE", header: "<windows.h>", byref.} = object + thePtr {.importc: "Ptr".} : Handle + + proc initSysLock*(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} + ## Initializes the lock `L`. + + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} + ## Tries to acquire the lock `L`. + + proc tryAcquireSys*(L: var SysLock): bool {.inline.} = + result = tryAcquireSysAux(L) != 0'i32 + + proc acquireSys*(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} + ## Acquires the lock `L`. + + proc releaseSys*(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} + ## Releases the lock `L`. + + proc deinitSys*(L: SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} + + proc initializeConditionVariable( + conditionVariable: var SysCond + ) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "InitializeConditionVariable".} + + proc sleepConditionVariableCS( + conditionVariable: var SysCond, + PCRITICAL_SECTION: var SysLock, + dwMilliseconds: int + ): int32 {.stdcall, noSideEffect, dynlib: "kernel32", importc: "SleepConditionVariableCS".} + + + proc signalSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeConditionVariable".} + + proc broadcastSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeAllConditionVariable".} + + proc initSysCond*(cond: var SysCond) {.inline.} = + initializeConditionVariable(cond) + proc deinitSysCond*(cond: SysCond) {.inline.} = + discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard sleepConditionVariableCS(cond, lock, -1'i32) + +elif defined(genode): + const + Header = "genode_cpp/syslocks.h" + type + SysLock* {.importcpp: "Nim::SysLock", pure, final, + header: Header.} = object + SysCond* {.importcpp: "Nim::SysCond", pure, final, + header: Header.} = object + + proc initSysLock*(L: var SysLock) = discard + proc deinitSys*(L: SysLock) = discard + proc acquireSys*(L: var SysLock) {.noSideEffect, importcpp.} + proc tryAcquireSys*(L: var SysLock): bool {.noSideEffect, importcpp.} + proc releaseSys*(L: var SysLock) {.noSideEffect, importcpp.} + + proc initSysCond*(L: var SysCond) = discard + proc deinitSysCond*(L: SysCond) = discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) {. + noSideEffect, importcpp.} + proc signalSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + proc broadcastSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + +else: + type + SysLockObj {.importc: "pthread_mutex_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[40 div sizeof(clong), clong] + + SysLockAttr* {.importc: "pthread_mutexattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysCondObj {.importc: "pthread_cond_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[48 div sizeof(clonglong), clonglong] + + SysCondAttr {.importc: "pthread_condattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysLockType = distinct cint + + proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. + importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysAux(L: SysLockObj) {.noSideEffect, + importc: "pthread_mutex_destroy", header: "<pthread.h>".} + + proc acquireSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_lock", header: "<pthread.h>".} + proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, + importc: "pthread_mutex_trylock", header: "<pthread.h>".} + + proc releaseSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_unlock", header: "<pthread.h>".} + + when defined(ios): + # iOS will behave badly if sync primitives are moved in memory. In order + # to prevent this once and for all, we're doing an extra malloc when + # initializing the primitive. + type + SysLock* = ptr SysLockObj + SysCond* = ptr SysCondObj + + when not declared(c_malloc): + proc c_malloc(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + + proc initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + L = cast[SysLock](c_malloc(csize_t(sizeof(SysLockObj)))) + initSysLockAux(L[], attr) + + proc deinitSys*(L: SysLock) = + deinitSysAux(L[]) + c_free(L) + + template acquireSys*(L: var SysLock) = + acquireSysAux(L[]) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L[]) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L[]) + else: + type + SysLock* = SysLockObj + SysCond* = SysCondObj + + template initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + initSysLockAux(L, attr) + template deinitSys*(L: SysLock) = + deinitSysAux(L) + template acquireSys*(L: var SysLock) = + acquireSysAux(L) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L) + + # rlocks + var SysLockType_Reentrant* {.importc: "PTHREAD_MUTEX_RECURSIVE", + header: "<pthread.h>".}: SysLockType + proc initSysLockAttr*(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType*(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + + # locks + proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysCondAux(cond: SysCondObj) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} + + proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj): cint {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc broadcastSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_broadcast", header: "<pthread.h>", noSideEffect.} + + when defined(ios): + proc initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + cond = cast[SysCond](c_malloc(csize_t(sizeof(SysCondObj)))) + initSysCondAux(cond[], cond_attr) + + proc deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond[]) + c_free(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond[], lock[]) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond[]) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond[]) + else: + template initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + initSysCondAux(cond, cond_attr) + template deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond, lock) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond) + +{.pop.} diff --git a/lib/std/private/threadtypes.nim b/lib/std/private/threadtypes.nim new file mode 100644 index 0000000000..a1cdf21dc0 --- /dev/null +++ b/lib/std/private/threadtypes.nim @@ -0,0 +1,176 @@ +include system/inclrtl + +const hasSharedHeap* = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own + +when defined(windows): + type + Handle* = int + SysThread* = Handle + WinThreadProc* = proc (x: pointer): int32 {.stdcall.} + + proc createThread*(lpThreadAttributes: pointer, dwStackSize: int32, + lpStartAddress: WinThreadProc, + lpParameter: pointer, + dwCreationFlags: int32, + lpThreadId: var int32): SysThread {. + stdcall, dynlib: "kernel32", importc: "CreateThread".} + + proc winSuspendThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "SuspendThread".} + + proc winResumeThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "ResumeThread".} + + proc waitForSingleObject*(hHandle: SysThread, dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + + proc waitForMultipleObjects*(nCount: int32, + lpHandles: ptr SysThread, + bWaitAll: int32, + dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} + + proc terminateThread*(hThread: SysThread, dwExitCode: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "TerminateThread".} + + proc setThreadAffinityMask*(hThread: SysThread, dwThreadAffinityMask: uint) {. + importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} + +elif defined(genode): + const + GenodeHeader* = "genode_cpp/threads.h" + type + SysThread* {.importcpp: "Nim::SysThread", + header: GenodeHeader, final, pure.} = object + GenodeThreadProc* = proc (x: pointer) {.noconv.} + + proc initThread*(s: var SysThread, + env: GenodeEnv, + stackSize: culonglong, + entry: GenodeThreadProc, + arg: pointer, + affinity: cuint) {. + importcpp: "#.initThread(@)".} + + +else: + when not (defined(macosx) or defined(haiku)): + {.passl: "-pthread".} + + when not defined(haiku): + {.passc: "-pthread".} + + const + schedh = "#define _GNU_SOURCE\n#include <sched.h>" + pthreadh* = "#define _GNU_SOURCE\n#include <pthread.h>" + + when not declared(Time): + when defined(linux): + type Time = clong + else: + type Time = int + + when (defined(linux) or defined(nintendoswitch)) and defined(amd64): + type + SysThread* {.importc: "pthread_t", + header: "<sys/types.h>" .} = distinct culong + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + abi: array[56 div sizeof(clong), clong] + elif defined(openbsd) and defined(amd64): + type + SysThread* {.importc: "pthread_t", header: "<pthread.h>".} = object + Pthread_attr* {.importc: "pthread_attr_t", + header: "<pthread.h>".} = object + else: + type + SysThread* {.importc: "pthread_t", header: "<sys/types.h>".} = int + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + type + Timespec* {.importc: "struct timespec", header: "<time.h>".} = object + tv_sec*: Time + tv_nsec*: clong + + proc pthread_attr_init*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstacksize*(a1: var Pthread_attr, a2: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_destroy*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + + proc pthread_create*(a1: var SysThread, a2: var Pthread_attr, + a3: proc (x: pointer): pointer {.noconv.}, + a4: pointer): cint {.importc: "pthread_create", + header: pthreadh.} + proc pthread_join*(a1: SysThread, a2: ptr pointer): cint {. + importc, header: pthreadh.} + + proc pthread_cancel*(a1: SysThread): cint {. + importc: "pthread_cancel", header: pthreadh.} + + type CpuSet* {.importc: "cpu_set_t", header: schedh.} = object + when defined(linux) and defined(amd64): + abi: array[1024 div (8 * sizeof(culong)), culong] + + proc cpusetZero*(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} + proc cpusetIncl*(cpu: cint; s: var CpuSet) {. + importc: "CPU_SET", header: schedh.} + + when defined(android): + # libc of android doesn't implement pthread_setaffinity_np, + # it exposes pthread_gettid_np though, so we can use that in combination + # with sched_setaffinity to set the thread affinity. + type Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 # From posix_other.nim + + proc setAffinityTID*(tid: Pid; setsize: csize_t; s: var CpuSet) {. + importc: "sched_setaffinity", header: schedh.} + + proc pthread_gettid_np*(thread: SysThread): Pid {. + importc: "pthread_gettid_np", header: pthreadh.} + + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) = + setAffinityTID(pthread_gettid_np(thread), setsize, s) + else: + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) {. + importc: "pthread_setaffinity_np", header: pthreadh.} + + +const + emulatedThreadVars* = compileOption("tlsEmulation") +# we preallocate a fixed size for thread local storage, so that no heap +# allocations are needed. Currently less than 16K are used on a 64bit machine. +# We use `float` for proper alignment: +const nimTlsSize {.intdefine.} = 16000 +type + ThreadLocalStorage* = array[0..(nimTlsSize div sizeof(float)), float] + PGcThread* = ptr GcThread + GcThread* {.pure, inheritable.} = object + when emulatedThreadVars: + tls*: ThreadLocalStorage + else: + nil + when hasSharedHeap: + next*, prev*: PGcThread + stackBottom*, stackTop*: pointer + stackSize*: int + else: + nil + +const hasAllocStack* = defined(zephyr) # maybe freertos too? + +type + Thread*[TArg] = object + core*: PGcThread + sys*: SysThread + when TArg is void: + dataFn*: proc () {.nimcall, gcsafe.} + else: + dataFn*: proc (m: TArg) {.nimcall, gcsafe.} + data*: TArg + when hasAllocStack: + rawStack*: pointer + +proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} diff --git a/lib/std/private/underscored_calls.nim b/lib/std/private/underscored_calls.nim index 6d0a99ab50..f853572b51 100644 --- a/lib/std/private/underscored_calls.nim +++ b/lib/std/private/underscored_calls.nim @@ -10,7 +10,9 @@ ## This is an internal helper module. Do not use. -import macros +import std/macros + +proc underscoredCalls*(result, calls, arg0: NimNode) proc underscoredCall(n, arg0: NimNode): NimNode = proc underscorePos(n: NimNode): int = @@ -19,13 +21,19 @@ proc underscoredCall(n, arg0: NimNode): NimNode = return 0 if n.kind in nnkCallKinds: - result = copyNimNode(n) - result.add n[0] + if n[0].kind in {nnkIdent, nnkSym} and n[0].eqIdent("with"): + expectKind n[1], {nnkIdent, nnkSym} - let u = underscorePos(n) - for i in 1..u-1: result.add n[i] - result.add arg0 - for i in u+1..n.len-1: result.add n[i] + result = newStmtList() + underscoredCalls(result, n[2 .. ^1].newStmtList, newDotExpr(arg0, n[1])) + else: + result = copyNimNode(n) + result.add n[0] + + let u = underscorePos(n) + for i in 1..u-1: result.add n[i] + result.add arg0 + for i in u+1..n.len-1: result.add n[i] elif n.kind in {nnkAsgn, nnkExprEqExpr}: var field = n[0] if n[0].kind == nnkDotExpr and n[0][0].eqIdent("_"): @@ -39,7 +47,7 @@ proc underscoredCall(n, arg0: NimNode): NimNode = result.add arg0 proc underscoredCalls*(result, calls, arg0: NimNode) = - expectKind calls, {nnkArglist, nnkStmtList, nnkStmtListExpr} + expectKind calls, {nnkArgList, nnkStmtList, nnkStmtListExpr} for call in calls: if call.kind in {nnkStmtList, nnkStmtListExpr}: diff --git a/lib/std/private/win_getsysteminfo.nim b/lib/std/private/win_getsysteminfo.nim new file mode 100644 index 0000000000..b98478231b --- /dev/null +++ b/lib/std/private/win_getsysteminfo.nim @@ -0,0 +1,15 @@ +type + SystemInfo* = object + u1: uint32 + dwPageSize: uint32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr uint32 + dwNumberOfProcessors*: uint32 + dwProcessorType: uint32 + dwAllocationGranularity*: uint32 + wProcessorLevel: uint16 + wProcessorRevision: uint16 + +proc getSystemInfo*(lpSystemInfo: ptr SystemInfo) {.stdcall, + dynlib: "kernel32", importc: "GetSystemInfo".} diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim new file mode 100644 index 0000000000..66e199dfec --- /dev/null +++ b/lib/std/private/win_setenv.nim @@ -0,0 +1,106 @@ +#[ +Copyright (c) Facebook, Inc. and its affiliates. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Adapted `setenv` from https://github.com/facebook/folly/blob/master/folly/portability/Stdlib.cpp +translated from C to nim. +]# + +#[ +Introduced in https://github.com/facebook/folly/commit/5d8ca09a3f96afefb44e35808f03651a096ab9c7 + +TODO: +check errno_t vs cint +]# + +when not defined(windows): discard +else: + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t".} = int16 + + proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} + # same as winlean.setEnvironmentVariableA + + proc c_getenv(varname: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} + proc c_wputenv(envstring: ptr wchar_t): cint {.importc: "_wputenv", header: "<stdlib.h>".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "<stdlib.h>".} + + var errno {.importc, header: "<errno.h>".}: cint + var genviron {.importc: "_environ".}: ptr ptr char + # xxx `ptr UncheckedArray[WideCString]` did not work + + proc wcstombs(wcstr: ptr char, mbstr: ptr wchar_t, count: csize_t): csize_t {.importc, header: "<stdlib.h>".} + # xxx cint vs errno_t? + + proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = + const EINVAL = cint(22) + let wideName: WideCString = newWideCString(name) + if overwrite == 0 and c_wgetenv(cast[ptr wchar_t](wideName)) != nil: + return 0 + + if value != "": + let envstring: WideCString = newWideCString(name & "=" & value) + let e = c_wputenv(cast[ptr wchar_t](envstring)) + if e != 0: + errno = EINVAL + return -1 + return 0 + #[ + We are trying to set the value to an empty string, but `_putenv` deletes + entries if the value is an empty string, and just calling + SetEnvironmentVariableA doesn't update `_environ`, + so we have to do these terrible things. + ]# + let envstring: WideCString = newWideCString(name & "= ") + if c_wputenv(cast[ptr wchar_t](envstring)) != 0: + errno = EINVAL + return -1 + # Here lies the documentation we blatently ignore to make this work. + var s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[0] = Utf16Char('\0') + #[ + This would result in a double null termination, which normally signifies the + end of the environment variable list, so we stick a completely empty + environment variable into the list instead. + ]# + s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[1] = Utf16Char('=') + #[ + If genviron is null, the MBCS environment has not been initialized + yet, and we don't need to try to update it. We have to do this otherwise + we'd be forcing the initialization and maintenance of the MBCS environment + even though it's never actually used in most programs. + ]# + if genviron != nil: + + # wcstombs returns `high(csize_t)` if any characters cannot be represented + # in the current codepage. Skip updating MBCS environment in this case. + # For some reason, second `wcstombs` can find non-convertible characters + # that the first `wcstombs` cannot. + let requiredSizeS = wcstombs(nil, cast[ptr wchar_t](wideName), 0) + if requiredSizeS != high(csize_t): + let requiredSize = requiredSizeS.int + var buf = newSeq[char](requiredSize + 1) + let buf2 = buf[0].addr + if wcstombs(buf2, cast[ptr wchar_t](wideName), csize_t(requiredSize + 1)) != high(csize_t): + var ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[0] = '\0' + ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[1] = '=' + + # And now, we have to update the outer environment to have a proper empty value. + if setEnvironmentVariableW(wideName, value.newWideCString) == 0: + errno = EINVAL + return -1 + return 0 diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index c7fac0a549..8e7bc6a92d 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this @@ -36,7 +36,7 @@ template toSet*(iter: untyped): untyped = incl(result, x) result -macro enmRange(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) +macro enumElementsAsSet(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) # func fullSet*(T: typedesc): set[T] {.inline.} = # xxx would give: Error: ordinal type expected func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = @@ -49,7 +49,7 @@ func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = when T is Ordinal: {T.low..T.high} else: # Hole filled enum - enmRange(T) + enumElementsAsSet(T) func complement*[T](s: set[T]): set[T] {.inline.} = ## Returns the set complement of `a`. diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index b74b285f8c..213af42296 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -1,41 +1,40 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## **Note:** Import `std/sha1` to use this module. -## ## [SHA-1 (Secure Hash Algorithm 1)](https://en.wikipedia.org/wiki/SHA-1) ## is a cryptographic hash function which takes an input and produces ## a 160-bit (20-byte) hash value known as a message digest. ## -## Basic usage -## =========== -## +## See also +## ======== +## * `base64 module<base64.html>`_ for a Base64 encoder and decoder +## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types +## * `md5 module<md5.html>`_ for the MD5 checksum algorithm + runnableExamples: let accessName = secureHash("John Doe") assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" -## .. code-block:: -## let -## a = secureHashFile("myFile.nim") -## b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") -## -## if a == b: -## echo "Files match" -## -## See also -## ======== -## * `base64 module<base64.html>`_ implements a Base64 encoder and decoder -## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types -## * `md5 module<md5.html>`_ implements the MD5 checksum algorithm +runnableExamples("-r:off"): + let + a = secureHashFile("myFile.nim") + b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") + assert a == b, "files don't match" + + +{.deprecated: "use command `nimble install checksums` and import `checksums/sha1` instead".} import std/strutils from std/endians import bigEndian32, bigEndian64 +when defined(nimPreviewSlimSystem): + import std/syncio + const Sha1DigestSize = 20 type @@ -282,3 +281,7 @@ proc `==`*(a, b: SecureHash): bool = # Not a constant-time comparison, but that's acceptable in this context Sha1Digest(a) == Sha1Digest(b) + +proc isValidSha1Hash*(s: string): bool = + ## Checks if a string is a valid sha1 hash sum. + s.len == 40 and allCharsInSet(s, HexDigits) \ No newline at end of file diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim index 5c882858db..45e9067953 100644 --- a/lib/std/socketstreams.nim +++ b/lib/std/socketstreams.nim @@ -31,39 +31,40 @@ ## Examples ## ======== ## -## .. code-block:: Nim -## import std/socketstreams +## ```Nim +## import std/socketstreams ## -## var -## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## stream = newReadSocketStream(socket) -## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") -## echo stream.readLine() # Will call `recv` -## stream.setPosition(0) -## echo stream.readLine() # Will return the read line from the buffer -## stream.resetStream() # Buffer is now empty, position is 0 -## echo stream.readLine() # Will call `recv` again -## stream.close() # Closes the socket +## var +## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## stream = newReadSocketStream(socket) +## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") +## echo stream.readLine() # Will call `recv` +## stream.setPosition(0) +## echo stream.readLine() # Will return the read line from the buffer +## stream.resetStream() # Buffer is now empty, position is 0 +## echo stream.readLine() # Will call `recv` again +## stream.close() # Closes the socket +## ``` ## -## .. code-block:: Nim +## ```Nim +## import std/socketstreams ## -## import std/socketstreams -## -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.connect("127.0.0.1", Port(12345)) -## var sendStream = newWriteSocketStream(socket) -## sendStream.write "NOM" -## sendStream.setPosition(1) -## echo sendStream.peekStr(2) # OM -## sendStream.write "I" -## sendStream.setPosition(0) -## echo sendStream.readStr(3) # NIM -## echo sendStream.getPosition() # 3 -## sendStream.flush() # This actually performs the writing to the socket -## sendStream.setPosition(1) -## sendStream.write "I" # Throws an error as we can't write into an already sent buffer +## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## socket.connect("127.0.0.1", Port(12345)) +## var sendStream = newWriteSocketStream(socket) +## sendStream.write "NOM" +## sendStream.setPosition(1) +## echo sendStream.peekStr(2) # OM +## sendStream.write "I" +## sendStream.setPosition(0) +## echo sendStream.readStr(3) # NIM +## echo sendStream.getPosition() # 3 +## sendStream.flush() # This actually performs the writing to the socket +## sendStream.setPosition(1) +## sendStream.write "I" # Throws an error as we can't write into an already sent buffer +## ``` -import net, streams +import std/[net, streams] type ReadSocketStream* = ref ReadSocketStreamObj @@ -146,7 +147,7 @@ proc wsFlush(s: Stream) = s.lastFlush = s.buf.len proc rsClose(s: Stream) = - {.cast(tags: []).}: + {.cast(raises: [IOError, OSError]), cast(tags: []).}: # todo fixme maybe do something? var s = ReadSocketStream(s) s.data.close() diff --git a/lib/std/staticos.nim b/lib/std/staticos.nim new file mode 100644 index 0000000000..2617c69138 --- /dev/null +++ b/lib/std/staticos.nim @@ -0,0 +1,13 @@ +## This module implements path handling like os module but works at only compile-time. +## This module works even when cross compiling to OS that is not supported by os module. + +proc staticFileExists*(filename: string): bool {.compileTime.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + discard + +proc staticDirExists*(dir: string): bool {.compileTime.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + discard diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index 9232204abe..b2c36a4bef 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2021 Nim Contributors # # See the file "copying.txt", included in this @@ -11,6 +11,10 @@ ## ## Experimental API, subject to change. +when defined(nimPreviewSlimSystem): + import std/assertions + + const whitespaces = {' ', '\t', '\v', '\r', '\l', '\f'} proc add*(x: var string, y: openArray[char]) = @@ -19,8 +23,8 @@ proc add*(x: var string, y: openArray[char]) = # Use `{.noalias.}` ? let n = x.len x.setLen n + y.len - # pending https://github.com/nim-lang/Nim/issues/14655#issuecomment-643671397 - # use x.setLen(n + y.len, isInit = false) + # pending #19727 + # setLen unnecessarily zeros memory var i = 0 while i < y.len: x[n + i] = y[i] @@ -46,7 +50,7 @@ func setSlice*(s: var string, slice: Slice[int]) = import std/sugar var a = "Hello, Nim!" - doassert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" doAssert a.dup(setSlice(0 .. 0)) == "H" doAssert a.dup(setSlice(0 .. 1)) == "He" doAssert a.dup(setSlice(0 .. 10)) == a diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim new file mode 100644 index 0000000000..dbe9086121 --- /dev/null +++ b/lib/std/symlinks.nim @@ -0,0 +1,33 @@ +## This module implements symlink (symbolic link) handling. + +## .. importdoc:: os.nim + +from std/paths import Path, ReadDirEffect + +from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink + +proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + result = symlinkExists(link.string) + +proc createSymlink*(src, dest: Path) {.inline.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + createSymlink(src.string, dest.string) + +proc expandSymlink*(symlinkPath: Path): Path {.inline.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + result = Path(expandSymlink(symlinkPath.string)) diff --git a/lib/system/io.nim b/lib/std/syncio.nim similarity index 79% rename from lib/system/io.nim rename to lib/std/syncio.nim index 300e7ea3f7..c34a025af5 100644 --- a/lib/system/io.nim +++ b/lib/std/syncio.nim @@ -1,18 +1,19 @@ # # # Nim's Runtime Library -# (c) Copyright 2019 Nim contributors +# (c) Copyright 2022 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## This is a part of `system.nim`, you should not manually import it. +## This module implements various synchronized I/O operations. - -include inclrtl +include system/inclrtl import std/private/since -import formatfloat +import std/formatfloat +when defined(windows): + import std/widestrs # ----------------- IO Part ------------------------------------------------ type @@ -20,27 +21,38 @@ type incompleteStruct.} = object File* = ptr CFile ## The type representing a file handle. - FileMode* = enum ## The file mode when opening a file. - fmRead, ## Open the file for read access only. - fmWrite, ## Open the file for write access only. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWrite, ## Open the file for read and write access. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWriteExisting, ## Open the file for read and write access. - ## If the file does not exist, it will not be - ## created. The existing file will not be cleared. - fmAppend ## Open the file for writing only; append data - ## at the end. + FileMode* = enum ## The file mode when opening a file. + fmRead, ## Open the file for read access only. + ## If the file does not exist, it will not + ## be created. + fmWrite, ## Open the file for write access only. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWrite, ## Open the file for read and write access. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWriteExisting, ## Open the file for read and write access. + ## If the file does not exist, it will not be + ## created. The existing file will not be cleared. + fmAppend ## Open the file for writing only; append data + ## at the end. If the file does not exist, it + ## will be created. - FileHandle* = cint ## type that represents an OS file handle; this is - ## useful for low-level file access + FileHandle* = cint ## The type that represents an OS file handle; this is + ## useful for low-level file access. + + FileSeekPos* = enum ## Position relative to which seek should happen. + # The values are ordered so that they match with stdio + # SEEK_SET, SEEK_CUR and SEEK_END respectively. + fspSet ## Seek to absolute value + fspCur ## Seek relative to current position + fspEnd ## Seek relative to end # text file handling: when not defined(nimscript) and not defined(js): # duplicated between io and ansi_c - const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined(dragonfly)) and not defined(emscripten) + const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined( + dragonfly)) and not defined(emscripten) const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" @@ -78,7 +90,7 @@ proc c_fputs(c: cstring, f: File): cint {. proc c_fgets(c: cstring, n: cint, f: File): cstring {. importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].} proc c_fgetc(stream: File): cint {. - importc: "fgetc", header: "<stdio.h>", tags: [ReadIOEffect].} + importc: "fgetc", header: "<stdio.h>", tags: [].} proc c_ungetc(c: cint, f: File): cint {. importc: "ungetc", header: "<stdio.h>", tags: [].} proc c_putc(c: cint, stream: File): cint {. @@ -93,7 +105,7 @@ proc c_feof(f: File): cint {. importc: "feof", header: "<stdio.h>".} when not declared(c_fwrite): - proc c_fwrite(buf: pointer, size, n: csize_t, f: File): cint {. + proc c_fwrite(buf: pointer, size, n: csize_t, f: File): csize_t {. importc: "fwrite", header: "<stdio.h>".} # C routine that is used here: @@ -108,8 +120,22 @@ when defined(windows): else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "_fseeki64", header: "<stdio.h>", tags: [].} - proc c_ftell(f: File): int64 {. - importc: "_ftelli64", header: "<stdio.h>", tags: [].} + when defined(tcc): + proc c_fsetpos(f: File, pos: var int64): int32 {. + importc: "fsetpos", header: "<stdio.h>", tags: [].} + proc c_fgetpos(f: File, pos: var int64): int32 {. + importc: "fgetpos", header: "<stdio.h>", tags: [].} + proc c_telli64(f: cint): int64 {. + importc: "_telli64", header: "<io.h>", tags: [].} + proc c_ftell(f: File): int64 = + # Taken from https://pt.osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.4-trunk/mingwrt/mingwex/stdio/ftelli64.c + result = -1'i64 + var pos: int64 + if c_fgetpos(f, pos) == 0 and c_fsetpos(f, pos) == 0: + result = c_telli64(c_fileno(f)) + else: + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "<stdio.h>", tags: [].} else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "fseeko", header: "<stdio.h>", tags: [].} @@ -125,21 +151,11 @@ proc c_fprintf(f: File, frmt: cstring): cint {. proc c_fputc(c: char, f: File): cint {. importc: "fputc", header: "<stdio.h>".} -# When running nim in android app, stdout goes nowhere, so echo gets ignored -# To redirect echo to the android logcat, use -d:androidNDK -when defined(androidNDK): - const ANDROID_LOG_VERBOSE = 2.cint - proc android_log_print(prio: cint, tag: cstring, fmt: cstring): cint - {.importc: "__android_log_print", header: "<android/log.h>", varargs, discardable.} - -template sysFatal(exc, msg) = - raise newException(exc, msg) - proc raiseEIO(msg: string) {.noinline, noreturn.} = - sysFatal(IOError, msg) + raise newException(IOError, msg) proc raiseEOF() {.noinline, noreturn.} = - sysFatal(EOFError, "EOF reached") + raise newException(EOFError, "EOF reached") proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} @@ -158,24 +174,25 @@ proc checkErr(f: File) = # shouldn't happen quit(1) -{.push stackTrace:off, profiler:off.} +{.push stackTrace: off, profiler: off.} proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer pointed to by `buffer`. Returns + ## Reads `len` bytes into the buffer pointed to by `buffer`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) if result != len: checkErr(f) -proc readBytes*(f: File, a: var openArray[int8|uint8], start, len: Natural): int {. +proc readBytes*(f: File, a: var openArray[int8|uint8], start, + len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = readBuffer(f, addr(a[start]), len) proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], benign.} = - ## reads up to `a.len` bytes into the buffer `a`. Returns + ## Reads up to `a.len` bytes into the buffer `a`. Returns ## the actual number of bytes that have been read which may be less than ## `a.len` (if not as many bytes are remaining), but not greater. result = readBuffer(f, addr(a[0]), a.len) @@ -183,7 +200,7 @@ proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], be proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {. tags: [ReadIOEffect], benign, deprecated: "use other `readChars` overload, possibly via: readChars(toOpenArray(buf, start, len-1))".} = - ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. if (start + len) > len(a): @@ -197,7 +214,7 @@ proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} = proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of buffer pointed to by the parameter `buffer` to the + ## Writes the bytes of buffer pointed to by the parameter `buffer` to the ## file `f`. Returns the number of actual written bytes, which may be less ## than `len` in case of an error. result = cast[int](c_fwrite(buffer, 1, cast[csize_t](len), f)) @@ -205,7 +222,7 @@ proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -213,7 +230,7 @@ proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -226,10 +243,7 @@ when defined(windows): # machine. We also enable `setConsoleOutputCP(65001)` now by default. # But we cannot call printf directly as the string might contain \0. # So we have to loop over all the sections separated by potential \0s. - var i = c_fprintf(f, "%s", s) - if i < 0: - if doRaise: raiseEIO("cannot write string to file") - return + var i = int c_fprintf(f, "%s", s) while i < s.len: if s[i] == '\0': let w = c_fputc('\0', f) @@ -252,7 +266,7 @@ proc write*(f: File, s: string) {.tags: [WriteIOEffect], benign.} = raiseEIO("cannot write string to file") {.pop.} -when NoFakeVars: +when defined(nimscript): when defined(windows): const IOFBF = cint(0) @@ -306,7 +320,7 @@ elif defined(windows): const BufSize = 4000 -proc close*(f: File) {.tags: [], gcsafe.} = +proc close*(f: File) {.tags: [], gcsafe, sideEffect.} = ## Closes the file. if not f.isNil: discard c_fclose(f) @@ -325,7 +339,7 @@ proc flushFile*(f: File) {.tags: [WriteIOEffect].} = discard c_fflush(f) proc getFileHandle*(f: File): FileHandle = - ## returns the file handle of the file `f`. This is only useful for + ## Returns the file handle of the file `f`. This is only useful for ## platform specific programming. ## Note that on Windows this doesn't return the Windows-specific handle, ## but the C library's notion of a handle, whatever that means. @@ -333,7 +347,7 @@ proc getFileHandle*(f: File): FileHandle = c_fileno(f) proc getOsFileHandle*(f: File): FileHandle = - ## returns the OS file handle of the file `f`. This is only useful for + ## Returns the OS file handle of the file `f`. This is only useful for ## platform specific programming. when defined(windows): result = FileHandle getOsfhandle(cint getFileHandle(f)) @@ -342,15 +356,15 @@ proc getOsFileHandle*(f: File): FileHandle = when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(windows): proc setInheritable*(f: FileHandle, inheritable: bool): bool = - ## control whether a file handle can be inherited by child processes. Returns + ## Controls whether a file handle can be inherited by child processes. Returns ## `true` on success. This requires the OS file handle, which can be ## retrieved via `getOsFileHandle <#getOsFileHandle,File>`_. ## ## This procedure is not guaranteed to be available for all platforms. Test for - ## availability with `declared() <system.html#declared,untyped>`. + ## availability with `declared() <system.html#declared,untyped>`_. when SupportIoctlInheritCtl: result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1 - elif defined(freertos): + elif defined(freertos) or defined(zephyr): result = true elif defined(posix): var flags = c_fcntl(f, F_GETFD) @@ -364,7 +378,7 @@ when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(w proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f` into `line`. May throw an IO + ## Reads a line of text from the file `f` into `line`. May throw an IO ## exception. ## A line of text may be delimited by `LF` or `CRLF`. The newline ## character(s) are not part of the returned string. Returns `false` @@ -373,7 +387,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} - when defined(windows) and not defined(useWinAnsi): + when defined(windows): proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer, nNumberOfCharsToRead: int32, lpNumberOfCharsRead: ptr int32, @@ -405,7 +419,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], if f.isatty: const numberOfCharsToRead = 2048 var numberOfCharsRead = 0'i32 - var buffer = newWideCString("", numberOfCharsToRead) + var buffer = newWideCString(numberOfCharsToRead) if readConsole(getOsFileHandle(f), addr(buffer[0]), numberOfCharsToRead, addr(numberOfCharsRead), nil) == 0: var error = getLastError() @@ -421,8 +435,8 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], numberOfCharsRead -= 2 # handle Ctrl+Z as EOF for i in 0..<numberOfCharsRead: - if buffer[i].uint16 == 26: #Ctrl+Z - close(f) #has the same effect as setting EOF + if buffer[i].uint16 == 26: #Ctrl+Z + close(f) #has the same effect as setting EOF if i == 0: line = "" return false @@ -450,7 +464,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], while true: # fixes #9634; this pattern may need to be abstracted as a template if reused; # likely other io procs need this for correctness. - fgetsSuccess = c_fgets(addr line[pos], sp.cint, f) != nil + fgetsSuccess = c_fgets(cast[cstring](addr line[pos]), sp.cint, f) != nil if fgetsSuccess: break when not defined(nimscript): if errno == EINTR: @@ -463,15 +477,16 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp)) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done - var last = cast[ByteAddress](m) - cast[ByteAddress](addr line[0]) + var last = cast[int](m) - cast[int](addr line[0]) if last > 0 and line[last-1] == '\c': line.setLen(last-1) return last > 1 or fgetsSuccess - # We have to distinguish between two possible cases: + elif last > 0 and line[last-1] == '\0': + # We have to distinguish among three possible cases: # \0\l\0 => line ending in a null character. # \0\l\l => last line without newline, null was put there by fgets. - elif last > 0 and line[last-1] == '\0': - if last < pos + sp - 1 and line[last+1] != '\0': + # \0\l => last line without newline, null was put there by fgets. + if last >= pos + sp - 1 or line[last+1] != '\0': # bug #21273 dec last line.setLen(last) return last > 0 or fgetsSuccess @@ -483,8 +498,8 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], sp = 128 # read in 128 bytes at a time line.setLen(pos+sp) -proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f`. May throw an IO exception. +proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} = + ## Reads a line of text from the file `f`. May throw an IO exception. ## A line of text may be delimited by `LF` or `CRLF`. The newline ## character(s) are not part of the returned string. result = newStringOfCap(80) @@ -556,7 +571,7 @@ proc readAllFile(file: File, len: int64): string = result = newString(len) let bytes = readBuffer(file, addr(result[0]), len) if endOfFile(file): - if bytes < len: + if bytes.int64 < len: result.setLen(bytes) else: # We read all the bytes but did not reach the EOF @@ -586,7 +601,7 @@ proc readAll*(file: File): string {.tags: [ReadIOEffect], benign.} = proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, tags: [WriteIOEffect], benign.} = - ## writes the values `x` to `f` and then writes "\\n". + ## Writes the values `x` to `f` and then writes "\\n". ## May throw an IO exception. for i in items(x): write(f, i) @@ -594,7 +609,7 @@ proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, # interface to the C procs: -when defined(windows) and not defined(useWinAnsi): +when defined(windows): when defined(cpp): proc wfopen(filename, mode: WideCString): pointer {. importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} @@ -633,8 +648,11 @@ const "" else: "" - FormatOpen: array[FileMode, string] = [ - "rb" & NoInheritFlag, "wb" & NoInheritFlag, "w+b" & NoInheritFlag, + RawFormatOpen: array[FileMode, cstring] = [ + # used for open by FileHandle, which calls `fdopen` + cstring("rb"), "wb", "w+b", "r+b", "ab"] + FormatOpen: array[FileMode, cstring] = [ + cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag, "r+b" & NoInheritFlag, "ab" & NoInheritFlag ] #"rt", "wt", "w+t", "r+t", "at" @@ -650,7 +668,7 @@ when defined(posix) and not defined(nimscript): Stat {.importc: "struct stat", header: "<sys/stat.h>", final, pure.} = object ## struct stat filler_1: array[24, char] - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file filler_2: array[144 - 24 - 4, char] proc modeIsDir(m: Mode): bool = @@ -663,7 +681,7 @@ when defined(posix) and not defined(nimscript): Stat {.importc: "struct stat", header: "<sys/stat.h>", final, pure.} = object ## struct stat - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file proc modeIsDir(m: Mode): bool {.importc: "S_ISDIR", header: "<sys/stat.h>".} ## Test for a directory. @@ -681,7 +699,7 @@ proc open*(f: var File, filename: string, ## This throws no exception if the file could not be opened. ## ## The file handle associated with the resulting `File` is not inheritable. - var p = fopen(filename, FormatOpen[mode]) + var p = fopen(filename.cstring, FormatOpen[mode]) if p != nil: var f2 = cast[File](p) when defined(posix) and not defined(nimscript): @@ -700,21 +718,21 @@ proc open*(f: var File, filename: string, result = true f = cast[File](p) - if bufSize > 0 and bufSize <= high(cint).int: + if bufSize > 0 and bufSize.uint <= high(uint): discard c_setvbuf(f, nil, IOFBF, cast[csize_t](bufSize)) elif bufSize == 0: discard c_setvbuf(f, nil, IONBF, 0) proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {. tags: [], benign.} = - ## reopens the file `f` with given `filename` and `mode`. This + ## Reopens the file `f` with given `filename` and `mode`. This ## is often used to redirect the `stdin`, `stdout` or `stderr` ## file variables. ## ## Default mode is readonly. Returns true if the file could be reopened. ## ## The file handle associated with `f` won't be inheritable. - if freopen(filename, FormatOpen[mode], f) != nil: + if freopen(filename.cstring, FormatOpen[mode], f) != nil: when not defined(nimInheritHandles) and declared(setInheritable) and NoInheritFlag.len == 0: if not setInheritable(getOsFileHandle(f), false): @@ -730,10 +748,11 @@ proc open*(f: var File, filehandle: FileHandle, ## ## The passed file handle will no longer be inheritable. when not defined(nimInheritHandles) and declared(setInheritable): - let oshandle = when defined(windows): FileHandle getOsfhandle(filehandle) else: filehandle + let oshandle = when defined(windows): FileHandle getOsfhandle( + filehandle) else: filehandle if not setInheritable(oshandle, false): return false - f = c_fdopen(filehandle, FormatOpen[mode]) + f = c_fdopen(filehandle, RawFormatOpen[mode]) result = f != nil proc open*(filename: string, @@ -745,22 +764,22 @@ proc open*(filename: string, ## ## The file handle associated with the resulting `File` is not inheritable. if not open(result, filename, mode, bufSize): - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) -proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign.} = - ## sets the position of the file pointer that is used for read/write +proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign, sideEffect.} = + ## Sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. if c_fseek(f, pos, cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos*(f: File): int64 {.benign.} = - ## retrieves the current position of the file pointer that is used to + ## Retrieves the current position of the file pointer that is used to ## read from the file `f`. The file's first byte has the index zero. result = c_ftell(f) if result < 0: raiseEIO("cannot retrieve file position") proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} = - ## retrieves the file size (in bytes) of `f`. + ## Retrieves the file size (in bytes) of `f`. let oldPos = getFilePos(f) discard c_fseek(f, 0, 2) # seek the end of the file result = getFilePos(f) @@ -775,50 +794,6 @@ proc setStdIoUnbuffered*() {.tags: [], benign.} = when declared(stdin): discard c_setvbuf(stdin, nil, IONBF, 0) -when declared(stdout): - when defined(windows) and compileOption("threads"): - const insideRLocksModule = false - include "system/syslocks" - - var echoLock: SysLock - initSysLock echoLock - - const stdOutLock = not defined(windows) and not defined(android) and - not defined(nintendoswitch) and not defined(freertos) and - hostOS != "any" - - const echoDoRaise = not defined(nimLegacyEchoNoRaise) and not defined(guiapp) # see PR #16366 - - template checkErrMaybe(succeeded: bool): untyped = - if not succeeded: - when echoDoRaise: - checkErr(stdout) - - proc echoBinSafe(args: openArray[string]) {.compilerproc.} = - when defined(androidNDK): - var s = "" - for arg in args: - s.add arg - android_log_print(ANDROID_LOG_VERBOSE, "nim", s) - else: - # flockfile deadlocks some versions of Android 5.x.x - when stdOutLock: - proc flockfile(f: File) {.importc, nodecl.} - proc funlockfile(f: File) {.importc, nodecl.} - flockfile(stdout) - defer: funlockfile(stdout) - when defined(windows) and compileOption("threads"): - acquireSys echoLock - defer: releaseSys echoLock - for s in args: - when defined(windows): - writeWindows(stdout, s, doRaise = echoDoRaise) - else: - checkErrMaybe(c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout) == s.len) - const linefeed = "\n" - checkErrMaybe(c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) == linefeed.len) - checkErrMaybe(c_fflush(stdout) == 0) - when defined(windows) and not defined(nimscript) and not defined(js): # work-around C's sucking abstraction: @@ -836,14 +811,33 @@ when defined(windows) and not defined(nimscript) and not defined(js): when defined(windows) and appType == "console" and not defined(nimDontSetUtf8CodePage) and not defined(nimscript): + import std/exitprocs + proc setConsoleOutputCP(codepage: cuint): int32 {.stdcall, dynlib: "kernel32", importc: "SetConsoleOutputCP".} proc setConsoleCP(wCodePageID: cuint): int32 {.stdcall, dynlib: "kernel32", importc: "SetConsoleCP".} + proc getConsoleOutputCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleOutputCP".} + proc getConsoleCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleCP".} - const Utf8codepage = 65001 - discard setConsoleOutputCP(Utf8codepage) - discard setConsoleCP(Utf8codepage) + const Utf8codepage = 65001'u32 + + let + consoleOutputCP = getConsoleOutputCP() + consoleCP = getConsoleCP() + + proc restoreConsoleOutputCP() = discard setConsoleOutputCP(consoleOutputCP) + proc restoreConsoleCP() = discard setConsoleCP(consoleCP) + + if consoleOutputCP != Utf8codepage: + discard setConsoleOutputCP(Utf8codepage) + addExitProc(restoreConsoleOutputCP) + + if consoleCP != Utf8codepage: + discard setConsoleCP(Utf8codepage) + addExitProc(restoreConsoleCP) proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} = ## Opens a file named `filename` for reading, calls `readAll @@ -858,7 +852,7 @@ proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} = finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} = ## Opens a file named `filename` for writing. Then writes the @@ -871,7 +865,7 @@ proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} = finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = ## Opens a file named `filename` for writing. Then writes the @@ -880,14 +874,14 @@ proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = var f: File = nil if open(f, filename, fmWrite): try: - f.writeBuffer(unsafeAddr content[0], content.len) + discard f.writeBuffer(unsafeAddr content[0], content.len) finally: close(f) else: raise newException(IOError, "cannot open: " & filename) proc readLines*(filename: string, n: Natural): seq[string] = - ## read `n` lines from the file named `filename`. Raises an IO exception + ## Reads `n` lines from the file named `filename`. Raises an IO exception ## in case of an error. Raises EOF if file does not contain at least `n` lines. ## Available at compile time. A line of text may be delimited by `LF` or `CRLF`. ## The newline character(s) are not part of the returned strings. @@ -901,9 +895,10 @@ proc readLines*(filename: string, n: Natural): seq[string] = finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) -template readLines*(filename: string): seq[string] {.deprecated: "use readLines with two arguments".} = +template readLines*(filename: string): seq[ + string] {.deprecated: "use readLines with two arguments".} = readLines(filename, 1) iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = @@ -912,15 +907,15 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = ## If the file does not exist `IOError` is raised. The trailing newline ## character(s) are removed from the iterated lines. Example: ## - ## .. code-block:: nim - ## import std/strutils - ## - ## proc transformLetters(filename: string) = - ## var buffer = "" - ## for line in filename.lines: - ## buffer.add(line.replace("a", "0") & '\n') - ## writeFile(filename, buffer) - var f = open(filename, bufSize=8000) + runnableExamples: + import std/strutils + + proc transformLetters(filename: string) = + var buffer = "" + for line in filename.lines: + buffer.add(line.replace("a", "0") & '\n') + writeFile(filename, buffer) + var f = open(filename, bufSize = 8000) try: var res = newStringOfCap(80) while f.readLine(res): yield res @@ -928,17 +923,20 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = close(f) iterator lines*(f: File): string {.tags: [ReadIOEffect].} = - ## Iterate over any line in the file `f`. + ## Iterates over any line in the file `f`. ## ## The trailing newline character(s) are removed from the iterated lines. - ## Example: ## - ## .. code-block:: nim - ## proc countZeros(filename: File): tuple[lines, zeros: int] = - ## for line in filename.lines: - ## for letter in line: - ## if letter == '0': - ## result.zeros += 1 - ## result.lines += 1 + runnableExamples: + proc countZeros(filename: File): tuple[lines, zeros: int] = + for line in filename.lines: + for letter in line: + if letter == '0': + result.zeros += 1 + result.lines += 1 var res = newStringOfCap(80) while f.readLine(res): yield res + +template `&=`*(f: File, x: typed) = + ## An alias for `write`. + write(f, x) diff --git a/lib/system/atomics.nim b/lib/std/sysatomics.nim similarity index 80% rename from lib/system/atomics.nim rename to lib/std/sysatomics.nim index f69cb753d8..2f203b3eb7 100644 --- a/lib/system/atomics.nim +++ b/lib/std/sysatomics.nim @@ -7,17 +7,22 @@ # distribution, for details about the copyright. # +when defined(nimPreviewSlimSystem): + {.deprecated: "use `std/atomics` instead".} + # Atomic operations for Nim. {.push stackTrace:off, profiler:off.} -const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) +const + hasThreadSupport = compileOption("threads") and not defined(nimscript) +const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) or defined(nintendoswitch) const someVcc = defined(vcc) or defined(clang_cl) type AtomType* = SomeNumber|pointer|ptr|char|bool ## Type Class representing valid types for use with atomic procs -when someGcc and hasThreadSupport: +when someGcc: type AtomMemModel* = distinct cint var ATOMIC_RELAXED* {.importc: "__ATOMIC_RELAXED", nodecl.}: AtomMemModel @@ -73,7 +78,7 @@ when someGcc and hasThreadSupport: proc atomicCompareExchangeN*[T: AtomType](p, expected: ptr T, desired: T, weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. - importc: "__atomic_compare_exchange_n ", nodecl.} + importc: "__atomic_compare_exchange_n", nodecl.} ## This proc implements an atomic compare and exchange operation. This compares the ## contents at p with the contents at expected and if equal, writes desired at p. ## If they are not equal, the current contents at p is written into expected. @@ -100,13 +105,13 @@ when someGcc and hasThreadSupport: proc atomicSubFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_sub_fetch", nodecl.} proc atomicOrFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_or_fetch ", nodecl.} + importc: "__atomic_or_fetch", nodecl.} proc atomicAndFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_and_fetch", nodecl.} proc atomicXorFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_xor_fetch", nodecl.} proc atomicNandFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_nand_fetch ", nodecl.} + importc: "__atomic_nand_fetch", nodecl.} ## Perform the operation return the old value, all memory models are valid proc atomicFetchAdd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. @@ -164,23 +169,48 @@ when someGcc and hasThreadSupport: ## ignore this parameter. template fence*() = atomicThreadFence(ATOMIC_SEQ_CST) -elif someVcc and hasThreadSupport: +elif someVcc: type AtomMemModel* = distinct cint const - ATOMIC_RELAXED = 0.AtomMemModel - ATOMIC_CONSUME = 1.AtomMemModel - ATOMIC_ACQUIRE = 2.AtomMemModel - ATOMIC_RELEASE = 3.AtomMemModel - ATOMIC_ACQ_REL = 4.AtomMemModel - ATOMIC_SEQ_CST = 5.AtomMemModel + ATOMIC_RELAXED* = 0.AtomMemModel + ATOMIC_CONSUME* = 1.AtomMemModel + ATOMIC_ACQUIRE* = 2.AtomMemModel + ATOMIC_RELEASE* = 3.AtomMemModel + ATOMIC_ACQ_REL* = 4.AtomMemModel + ATOMIC_SEQ_CST* = 5.AtomMemModel proc `==`(x1, x2: AtomMemModel): bool {.borrow.} - proc readBarrier() {.importc: "_ReadBarrier", header: "<intrin.h>".} - proc writeBarrier() {.importc: "_WriteBarrier", header: "<intrin.h>".} + proc readBarrier() {.importc: "_ReadBarrier", header: "<intrin.h>".} + proc writeBarrier() {.importc: "_WriteBarrier", header: "<intrin.h>".} proc fence*() {.importc: "_ReadWriteBarrier", header: "<intrin.h>".} + when defined(cpp): + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importcpp: "_InterlockedCompareExchange64(static_cast<NI64 volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importcpp: "_InterlockedCompareExchange(static_cast<long volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importcpp: "_InterlockedCompareExchange8(static_cast<char volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importcpp: "_InterlockedExchange8(static_cast<NI8 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importcpp: "_InterlockedExchange16(static_cast<NI16 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importcpp: "_InterlockedExchange(static_cast<long volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importcpp: "_InterlockedExchange64(static_cast<NI64 volatile *>(#), #)", header: "<intrin.h>".} + else: + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importc: "_InterlockedCompareExchange8", header: "<intrin.h>".} + + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64", header: "<intrin.h>".} + + template barrier(mem: AtomMemModel) = when mem == ATOMIC_RELAXED: discard elif mem == ATOMIC_CONSUME: readBarrier() @@ -189,10 +219,28 @@ elif someVcc and hasThreadSupport: elif mem == ATOMIC_ACQ_REL: fence() elif mem == ATOMIC_SEQ_CST: fence() + proc atomicStoreN*[T: AtomType](p: ptr T, val: T, mem: static[AtomMemModel]) = + barrier(mem) + p[] = val + proc atomicLoadN*[T: AtomType](p: ptr T, mem: static[AtomMemModel]): T = result = p[] barrier(mem) + proc atomicCompareExchangeN*[T: ptr](p, expected: ptr T, desired: T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](desired), cast[int64](expected)) == + cast[int64](expected) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](desired), cast[int32](expected)) == + cast[int32](expected) + + proc atomicExchangeN*[T: ptr](p: ptr T, val: T, mem: AtomMemModel): T = + when sizeof(T) == 8: + cast[T](interlockedExchange64(p, cast[int64](val))) + elif sizeof(T) == 4: + cast[T](interlockedExchange32(p, cast[int32](val))) when defined(cpp): when sizeof(int) == 8: proc addAndFetch*(p: ptr int, val: int): int {. @@ -215,9 +263,11 @@ else: inc(p[], val) result = p[] -proc atomicInc*(memLoc: var int, x: int = 1): int = + +proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically increments the integer by some `x`. It returns the new value. when someGcc and hasThreadSupport: - result = atomicAddFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, x) inc(result, x) @@ -225,12 +275,13 @@ proc atomicInc*(memLoc: var int, x: int = 1): int = inc(memLoc, x) result = memLoc -proc atomicDec*(memLoc: var int, x: int = 1): int = +proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically decrements the integer by some `x`. It returns the new value. when someGcc and hasThreadSupport: when declared(atomicSubFetch): - result = atomicSubFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicSubFetch(memLoc.addr, x, ATOMIC_SEQ_CST) else: - result = atomicAddFetch(memLoc.addr, -x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, -x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, -x) dec(result, x) @@ -239,21 +290,6 @@ proc atomicDec*(memLoc: var int, x: int = 1): int = result = memLoc when someVcc: - when defined(cpp): - proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 - {.importcpp: "_InterlockedCompareExchange64(static_cast<NI64 volatile *>(#), #, #)", header: "<intrin.h>".} - proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 - {.importcpp: "_InterlockedCompareExchange(static_cast<NI volatile *>(#), #, #)", header: "<intrin.h>".} - proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte - {.importcpp: "_InterlockedCompareExchange8(static_cast<char volatile *>(#), #, #)", header: "<intrin.h>".} - else: - proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 - {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} - proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 - {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} - proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte - {.importc: "_InterlockedCompareExchange8", header: "<intrin.h>".} - proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = when sizeof(T) == 8: interlockedCompareExchange64(p, cast[int64](newValue), cast[int64](oldValue)) == @@ -327,7 +363,7 @@ elif someGcc or defined(tcc): elif defined(icl): proc cpuRelax* {.importc: "_mm_pause", header: "xmmintrin.h".} elif false: - from os import sleep + from std/os import sleep proc cpuRelax* {.inline.} = os.sleep(1) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index c9908143c8..6f2c6b0c16 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -10,23 +10,23 @@ ## .. warning:: This module was added in Nim 1.6. If you are using it for cryptographic purposes, ## keep in mind that so far this has not been audited by any security professionals, ## therefore may not be secure. -## +## ## `std/sysrand` generates random numbers from a secure source provided by the operating system. -## It is also called Cryptographically secure pseudorandom number generator. -## It should be unpredictable enough for cryptographic applications, +## It is a cryptographically secure pseudorandom number generator +## and should be unpredictable enough for cryptographic applications, ## though its exact quality depends on the OS implementation. ## -## | Targets | Implementation| -## | :--- | ----: | -## | Windows | `BCryptGenRandom`_ | -## | Linux | `getrandom`_ | -## | MacOSX | `getentropy`_ | -## | IOS | `SecRandomCopyBytes`_ | -## | OpenBSD | `getentropy openbsd`_ | -## | FreeBSD | `getrandom freebsd`_ | -## | JS(Web Browser) | `getRandomValues`_ | -## | Nodejs | `randomFillSync`_ | -## | Other Unix platforms | `/dev/urandom`_ | +## | Targets | Implementation | +## | :--- | ----: | +## | Windows | `BCryptGenRandom`_ | +## | Linux | `getrandom`_ | +## | MacOSX | `SecRandomCopyBytes`_ | +## | iOS | `SecRandomCopyBytes`_ | +## | OpenBSD | `getentropy openbsd`_ | +## | FreeBSD | `getrandom freebsd`_ | +## | JS (Web Browser) | `getRandomValues`_ | +## | Node.js | `randomFillSync`_ | +## | Other Unix platforms | `/dev/urandom`_ | ## ## .. _BCryptGenRandom: https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom ## .. _getrandom: https://man7.org/linux/man-pages/man2/getrandom.2.html @@ -38,6 +38,11 @@ ## .. _randomFillSync: https://nodejs.org/api/crypto.html#crypto_crypto_randomfillsync_buffer_offset_size ## .. _/dev/urandom: https://en.wikipedia.org/wiki//dev/random ## +## On a Linux target, a call to the `getrandom` syscall can be avoided (e.g. +## for targets running kernel version < 3.17) by passing a compile flag of +## `-d:nimNoGetRandom`. If this flag is passed, sysrand will use `/dev/urandom` +## as with any other POSIX compliant OS. +## runnableExamples: doAssert urandom(0).len == 0 @@ -52,13 +57,16 @@ runnableExamples: when not defined(js): - import std/os + import std/oserrors when defined(posix): import std/posix +when defined(nimPreviewSlimSystem): + import std/assertions + const - batchImplOS = defined(freebsd) or defined(openbsd) or (defined(macosx) and not defined(ios)) + batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) batchSize {.used.} = 256 when batchImplOS: @@ -132,7 +140,7 @@ elif defined(windows): type PVOID = pointer BCRYPT_ALG_HANDLE = PVOID - PUCHAR = ptr cuchar + PUCHAR = ptr uint8 NTSTATUS = clong ULONG = culong @@ -159,15 +167,16 @@ elif defined(windows): result = randomBytes(addr dest[0], size) -elif defined(linux): - # TODO using let, pending bootstrap >= 1.4.0 - var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong +elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): + when (NimMajor, NimMinor) >= (1, 4): + let SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + else: + var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong const syscallHeader = """#include <unistd.h> #include <sys/syscall.h>""" - proc syscall( - n: clong, buf: pointer, bufLen: cint, flags: cuint - ): clong {.importc: "syscall", header: syscallHeader.} + proc syscall(n: clong): clong {. + importc: "syscall", varargs, header: syscallHeader.} # When reading from the urandom source (GRND_RANDOM is not set), # getrandom() will block until the entropy pool has been # initialized (unless the GRND_NONBLOCK flag was specified). If a @@ -183,25 +192,34 @@ elif defined(linux): while result < size: let readBytes = syscall(SYS_getrandom, addr dest[result], cint(size - result), 0).int if readBytes == 0: - doAssert false + raiseAssert "unreachable" elif readBytes > 0: inc(result, readBytes) else: - if osLastError().int in {EINTR, EAGAIN}: - discard + if osLastError().cint in [EINTR, EAGAIN]: discard else: result = -1 break elif defined(openbsd): proc getentropy(p: pointer, size: cint): cint {.importc: "getentropy", header: "<unistd.h>".} - # fills a buffer with high-quality entropy, + # Fills a buffer with high-quality entropy, # which can be used as input for process-context pseudorandom generators like `arc4random`. # The maximum buffer size permitted is 256 bytes. proc getRandomImpl(p: pointer, size: int): int {.inline.} = result = getentropy(p, cint(size)).int +elif defined(zephyr): + proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "<random/rand32.h>".} + # Fill the destination buffer with cryptographically secure + # random data values + # + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + # 0 if success, -EIO if entropy reseed error + result = sys_csrand_get(p, csize_t(size)).int + elif defined(freebsd): type cssize_t {.importc: "ssize_t", header: "<sys/types.h>".} = int @@ -214,8 +232,8 @@ elif defined(freebsd): proc getRandomImpl(p: pointer, size: int): int {.inline.} = result = getrandom(p, csize_t(size), 0) -elif defined(ios): - {.passL: "-framework Security".} +elif defined(ios) or defined(macosx): + {.passl: "-framework Security".} const errSecSuccess = 0 ## No error. @@ -227,7 +245,7 @@ elif defined(ios): proc secRandomCopyBytes( rnd: SecRandomRef, count: csize_t, bytes: pointer - ): cint {.importc: "SecRandomCopyBytes", header: "<Security/SecRandom.h>".} + ): cint {.importc: "SecRandomCopyBytes", header: "<Security/SecRandom.h>".} ## https://developer.apple.com/documentation/security/1399291-secrandomcopybytes template urandomImpl(result: var int, dest: var openArray[byte]) = @@ -237,19 +255,6 @@ elif defined(ios): result = secRandomCopyBytes(nil, csize_t(size), addr dest[0]) -elif defined(macosx): - const sysrandomHeader = """#include <Availability.h> -#include <sys/random.h> -""" - - proc getentropy(p: pointer, size: csize_t): cint {.importc: "getentropy", header: sysrandomHeader.} - # getentropy() fills a buffer with random data, which can be used as input - # for process-context pseudorandom generators like arc4random(3). - # The maximum buffer size permitted is 256 bytes. - - proc getRandomImpl(p: pointer, size: int): int {.inline.} = - result = getentropy(p, csize_t(size)).int - else: template urandomImpl(result: var int, dest: var openArray[byte]) = let size = dest.len @@ -292,7 +297,7 @@ proc urandom*(dest: var openArray[byte]): bool = ## If the call succeeds, returns `true`. ## ## If `dest` is empty, `urandom` immediately returns success, - ## without calling underlying operating system api. + ## without calling the underlying operating system API. ## ## .. warning:: The code hasn't been audited by cryptography experts and ## is provided as-is without guarantees. Use at your own risks. For production @@ -310,7 +315,7 @@ proc urandom*(dest: var openArray[byte]): bool = proc urandom*(size: Natural): seq[byte] {.inline.} = ## Returns random bytes suitable for cryptographic use. - ## + ## ## .. warning:: The code hasn't been audited by cryptography experts and ## is provided as-is without guarantees. Use at your own risks. For production ## systems we advise you to request an external audit. diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim new file mode 100644 index 0000000000..7e59747f55 --- /dev/null +++ b/lib/std/tasks.nim @@ -0,0 +1,312 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides basic primitives for creating parallel programs. +## A `Task` should be only owned by a single Thread, it cannot be shared by threads. + +import std/[macros, isolation, typetraits] + +when defined(nimPreviewSlimSystem): + import std/assertions + +export isolation + + +when compileOption("threads"): + from std/effecttraits import isGcSafe + + +# +# proc hello(a: int, b: string) = +# echo $a & b +# +# let literal = "Nim" +# let t = toTask(hello(521, literal)) +# +# +# is roughly converted to +# +# type +# ScratchObj_369098780 = object +# a: int +# b: string +# +# let scratch_369098762 = cast[ptr ScratchObj_369098780](c_calloc(csize_t 1, +# csize_t sizeof(ScratchObj_369098780))) +# if scratch_369098762.isNil: +# raise newException(OutOfMemDefect, "Could not allocate memory") +# block: +# var isolate_369098776 = isolate(521) +# scratch_369098762.a = extract(isolate_369098776) +# var isolate_369098778 = isolate(literal) +# scratch_369098762.b = extract(isolate_369098778) +# proc hello_369098781(args`gensym3: pointer) {.nimcall.} = +# let objTemp_369098775 = cast[ptr ScratchObj_369098780](args`gensym3) +# let :tmp_369098777 = objTemp_369098775.a +# let :tmp_369098779 = objTemp_369098775.b +# hello(a = :tmp_369098777, b = :tmp_369098779) +# +# proc destroyScratch_369098782(args`gensym3: pointer) {.nimcall.} = +# let obj_369098783 = cast[ptr ScratchObj_369098780](args`gensym3) +# =destroy(obj_369098783[]) +# let t = Task(callback: hello_369098781, args: scratch_369098762, destroy: destroyScratch_369098782) +# + + +type + Task* = object ## `Task` contains the callback and its arguments. + callback: proc (args, res: pointer) {.nimcall, gcsafe.} + args: pointer + destroy: proc (args: pointer) {.nimcall, gcsafe.} + + +proc `=copy`*(x: var Task, y: Task) {.error.} + +const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) +when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`*(t: Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) +else: + proc `=destroy`*(t: var Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) + +proc invoke*(task: Task; res: pointer = nil) {.inline, gcsafe.} = + ## Invokes the `task`. + assert task.callback != nil + task.callback(task.args, res) + +template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) = + # block: + # var isoTempA = isolate(521) + # scratch.a = extract(isolateA) + # var isoTempB = isolate(literal) + # scratch.b = extract(isolateB) + let isolatedTemp = genSym(nskTemp, "isoTemp") + scratchAssignList.add newVarStmt(isolatedTemp, newCall(newIdentNode("isolate"), procParam)) + scratchAssignList.add newAssignment(scratchDotExpr, + newCall(newIdentNode("extract"), isolatedTemp)) + +template addAllNode(assignParam: NimNode, procParam: NimNode) = + let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0]) + + checkIsolate(scratchAssignList, procParam, scratchDotExpr) + + let tempNode = genSym(kind = nskTemp, ident = formalParams[i][0].strVal) + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], tempNode) + tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0])) + scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam) + +proc analyseRootSym(s: NimNode): NimNode = + result = s + while true: + case result.kind + of nnkBracketExpr, nnkDerefExpr, nnkHiddenDeref, + nnkAddr, nnkHiddenAddr, + nnkObjDownConv, nnkObjUpConv: + result = result[0] + of nnkDotExpr, nnkCheckedFieldExpr, nnkHiddenStdConv, nnkHiddenSubConv: + result = result[1] + else: + break + +macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task = + ## Converts the call and its arguments to `Task`. + runnableExamples: + proc hello(a: int) = echo a + + let b = toTask hello(13) + assert b is Task + + let retType = getTypeInst(e) + let returnsVoid = retType.typeKind == ntyVoid + + let rootSym = analyseRootSym(e[0]) + expectKind rootSym, nnkSym + + when compileOption("threads"): + if not isGcSafe(rootSym): + error("'toTask' takes a GC safe call expression", e) + + if hasClosure(rootSym): + error("closure call is not allowed", e) + + if e.len > 1: + let scratchIdent = genSym(kind = nskTemp, ident = "scratch") + let impl = e[0].getTypeInst + + when defined(nimTasksDebug): + echo impl.treeRepr + echo e.treeRepr + let formalParams = impl[0] + + var + scratchRecList = newNimNode(nnkRecList) + scratchAssignList: seq[NimNode] + tempAssignList: seq[NimNode] + callNode: seq[NimNode] + + let + objTemp = genSym(nskTemp, ident = "objTemp") + + for i in 1 ..< formalParams.len: + var param = formalParams[i][1] + + if param.kind == nnkBracketExpr and param[0].eqIdent("sink"): + param = param[0] + + if param.typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + + case param.kind + of nnkVarTy: + error("'toTask'ed function cannot have a 'var' parameter", e) + of nnkBracketExpr: + if param[0].typeKind == ntyTypeDesc: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + elif param[0].typeKind in {ntyVarargs, ntyOpenArray}: + if param[1].typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + let + seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) + seqCallNode = newCall("@", e[i]) + addAllNode(seqType, seqCallNode) + else: + addAllNode(param, e[i]) + of nnkBracket, nnkObjConstr: + # passing by static parameters + # so we pass them directly instead of passing by scratchObj + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + of nnkSym, nnkPtrTy, nnkProcTy, nnkTupleConstr: + addAllNode(param, e[i]) + of nnkCharLit..nnkNilLit: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + else: + error("'toTask'ed function cannot have a parameter of " & $param.kind & " kind", e) + + let scratchObjType = genSym(kind = nskType, ident = "ScratchObj") + let scratchObj = nnkTypeSection.newTree( + nnkTypeDef.newTree( + scratchObjType, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + scratchRecList + ) + ) + ) + + + let scratchObjPtrType = quote do: + cast[ptr `scratchObjType`](allocShared0(sizeof(`scratchObjType`))) + + let scratchLetSection = newLetStmt(scratchIdent, scratchObjPtrType) + + var stmtList = newStmtList() + stmtList.add(scratchObj) + stmtList.add(scratchLetSection) + stmtList.add(nnkBlockStmt.newTree(newEmptyNode(), newStmtList(scratchAssignList))) + + var functionStmtList = newStmtList() + let funcCall = newCall(e[0], callNode) + functionStmtList.add tempAssignList + + let funcName = genSym(nskProc, rootSym.strVal) + let destroyName = genSym(nskProc, "destroyScratch") + let objTemp2 = genSym(ident = "obj") + let tempNode = quote("@") do: + `=destroy`(@objTemp2[]) + + var funcDecl: NimNode + if returnsVoid: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + `funcCall` + else: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + cast[ptr `retType`](res)[] = `funcCall` + + result = quote do: + `stmtList` + + `funcDecl` + + proc `destroyName`(args: pointer) {.gcsafe, nimcall.} = + let `objTemp2` = cast[ptr `scratchObjType`](args) + `tempNode` + + Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`) + else: + let funcCall = newCall(e[0]) + let funcName = genSym(nskProc, rootSym.strVal) + + if returnsVoid: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + `funcCall` + + Task(callback: `funcName`, args: nil) + else: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + cast[ptr `retType`](res)[] = `funcCall` + + Task(callback: `funcName`, args: nil) + + + when defined(nimTasksDebug): + echo result.repr + +runnableExamples: + block: + var num = 0 + proc hello(a: int) = inc num, a + + let b = toTask hello(13) + b.invoke() + assert num == 13 + # A task can be invoked multiple times + b.invoke() + assert num == 26 + + block: + type + Runnable = ref object + data: int + + var data: int + proc hello(a: Runnable) {.nimcall.} = + a.data += 2 + data = a.data + + + when false: + # the parameters of call must be isolated. + let x = Runnable(data: 12) + let b = toTask hello(x) # error ----> expression cannot be isolated: x + b.invoke() + + let b = toTask(hello(Runnable(data: 12))) + b.invoke() + assert data == 14 + b.invoke() + assert data == 16 diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim new file mode 100644 index 0000000000..539305bde3 --- /dev/null +++ b/lib/std/tempfiles.nim @@ -0,0 +1,192 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module creates temporary files and directories. +## +## Experimental API, subject to change. + +#[ +See also: +* `GetTempFileName` (on windows), refs https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettempfilenamea +* `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html +]# + +import std / [os, random] + +when defined(nimPreviewSlimSystem): + import std/syncio + +const + maxRetry = 10000 + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + nimTempPathLength {.intdefine.} = 8 + + +when defined(windows): + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + var O_RDWR {.importc: "_O_RDWR", header: "<fcntl.h>".}: cint + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "_fdopen",header: "<stdio.h>".} + + proc open_osfhandle(osh: Handle, mode: cint): cint {. + importc: "_open_osfhandle", header: "<io.h>".} + + proc close_osfandle(fd: cint): cint {. + importc: "_close", header: "<io.h>".} +else: + import std/posix + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "fdopen",header: "<stdio.h>".} + + +proc safeOpen(filename: string): File = + ## Open files exclusively; returns `nil` if the file already exists. + # xxx this should be clarified; it doesn't in particular prevent other processes + # from opening the file, at least currently. + when defined(windows): + let dwShareMode = FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE + let dwCreation = CREATE_NEW + let dwFlags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + let handle = createFileW(newWideCString(filename), GENERIC_READ or GENERIC_WRITE, dwShareMode, + nil, dwCreation, dwFlags, Handle(0)) + + if handle == INVALID_HANDLE_VALUE: + if getLastError() == ERROR_FILE_EXISTS: + return nil + else: + raiseOSError(osLastError(), filename) + + let fileHandle = open_osfhandle(handle, O_RDWR) + if fileHandle == -1: + discard closeHandle(handle) + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard close_osfandle(fileHandle) + raiseOSError(osLastError(), filename) + else: + # xxx we need a `proc toMode(a: FilePermission): Mode`, possibly by + # exposing fusion/filepermissions.fromFilePermissions to stdlib; then we need + # to expose a `perm` param so users can customize this (e.g. the temp file may + # need execute permissions), and figure out how to make the API cross platform. + let mode = Mode(S_IRUSR or S_IWUSR) + let flags = posix.O_RDWR or posix.O_CREAT or posix.O_EXCL + let fileHandle = posix.open(filename, flags, mode) + if fileHandle == -1: + if errno == EEXIST: + # xxx `getLastError()` should be defined on posix too and resolve to `errno`? + return nil + else: + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard posix.close(fileHandle) # TODO handles failure when closing file + raiseOSError(osLastError(), filename) + + +type + NimTempPathState = object + state: Rand + isInit: bool + +var nimTempPathState {.threadvar.}: NimTempPathState + +template randomPathName(length: Natural): string = + var res = newString(length) + if not nimTempPathState.isInit: + nimTempPathState.isInit = true + nimTempPathState.state = initRand() + + for i in 0 ..< length: + res[i] = nimTempPathState.state.sample(letters) + res + +proc getTempDirImpl(dir: string): string {.inline.} = + result = dir + if result.len == 0: + result = getTempDir() + +proc genTempPath*(prefix, suffix: string, dir = ""): string = + ## Generates a path name in `dir`. + ## + ## The path begins with `prefix` and ends with `suffix`. + ## + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + let dir = getTempDirImpl(dir) + result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + +proc createTempFile*(prefix, suffix: string, dir = ""): tuple[cfile: File, path: string] = + ## Creates a new temporary file in the directory `dir`. + ## + ## This generates a path name using `genTempPath(prefix, suffix, dir)` and + ## returns a file handle to an open file and the path of that file, possibly after + ## retrying to ensure it doesn't already exist. + ## + ## If failing to create a temporary file, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to close `result.cfile` and + ## remove `result.file` when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempFile("", "", "nonexistent") + let (cfile, path) = createTempFile("tmpprefix_", "_end.tmp") + # path looks like: getTempDir() / "tmpprefix_FDCIRZA0_end.tmp" + cfile.write "foo" + cfile.setFilePos 0 + assert readAll(cfile) == "foo" + close cfile + assert readFile(path) == "foo" + removeFile(path) + # xxx why does above work without `cfile.flushFile` ? + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result.path = genTempPath(prefix, suffix, dir) + result.cfile = safeOpen(result.path) + if result.cfile != nil: + return + + raise newException(OSError, "Failed to create a temporary file under directory " & dir) + +proc createTempDir*(prefix, suffix: string, dir = ""): string = + ## Creates a new temporary directory in the directory `dir`. + ## + ## This generates a dir name using `genTempPath(prefix, suffix, dir)`, creates + ## the directory and returns it, possibly after retrying to ensure it doesn't + ## already exist. + ## + ## If failing to create a temporary directory, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempDir("", "", "nonexistent") + let dir = createTempDir("tmpprefix_", "_end") + # dir looks like: getTempDir() / "tmpprefix_YEl9VuVj_end" + assert dirExists(dir) + removeDir(dir) + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result = genTempPath(prefix, suffix, dir) + if not existsOrCreateDir(result): + return + + raise newException(OSError, "Failed to create a temporary directory under directory " & dir) diff --git a/lib/std/time_t.nim b/lib/std/time_t.nim index 7fb6e6d468..5fa95fff38 100644 --- a/lib/std/time_t.nim +++ b/lib/std/time_t.nim @@ -19,5 +19,5 @@ elif defined(windows): # newest version of Visual C++ defines time_t to be of 64 bits type Time* {.importc: "time_t", header: "<time.h>".} = distinct int64 elif defined(posix): - import posix + import std/posix export posix.Time \ No newline at end of file diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim new file mode 100644 index 0000000000..7b0b81968f --- /dev/null +++ b/lib/std/typedthreads.nim @@ -0,0 +1,305 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +Thread support for Nim. Threads allow multiple functions to execute concurrently. + +In Nim, threads are a low-level construct and using a library like `malebolgia`, `taskpools` or `weave` is recommended. + +When creating a thread, you can pass arguments to it. As Nim's garbage collector does not use atomic references, sharing +`ref` and other variables managed by the garbage collector between threads is not supported. +Use global variables to do so, or pointers. + +Memory allocated using [`sharedAlloc`](./system.html#allocShared.t%2CNatural) can be used and shared between threads. + +To communicate between threads, consider using [channels](./system.html#Channel) + +Examples +======== + +```Nim +import std/locks + +var + thr: array[0..4, Thread[tuple[a,b: int]]] + L: Lock + +proc threadFunc(interval: tuple[a,b: int]) {.thread.} = + for i in interval.a..interval.b: + acquire(L) # lock stdout + echo i + release(L) + +initLock(L) + +for i in 0..high(thr): + createThread(thr[i], threadFunc, (i*10, i*10+5)) +joinThreads(thr) + +deinitLock(L) +``` + +When using a memory management strategy that supports shared heaps like `arc` or `boehm`, +you can pass pointer to threads and share memory between them, but the memory must outlive the thread. +The default memory management strategy, `orc`, supports this. +The example below is **not valid** for memory management strategies that use local heaps like `refc`! + +```Nim +import locks + +var l: Lock + +proc threadFunc(obj: ptr seq[int]) {.thread.} = + withLock l: + for i in 0..<100: + obj[].add(obj[].len * obj[].len) + +proc threadHandler() = + var thr: array[0..4, Thread[ptr seq[int]]] + var s = newSeq[int]() + + for i in 0..high(thr): + createThread(thr[i], threadFunc, s.addr) + joinThreads(thr) + echo s + +initLock(l) +threadHandler() +deinitLock(l) +``` +]## + + +import std/private/[threadtypes] +export Thread + +import system/ansi_c + +when defined(nimPreviewSlimSystem): + import std/assertions + +when defined(genode): + import genode/env + +when hostOS == "any": + {.error: "Threads not implemented for os:any. Please compile with --threads:off.".} + +when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or + defined(cpu16) or defined(cpu8): + const + nimThreadStackSize {.intdefine.} = 8192 + nimThreadStackGuard {.intdefine.} = 128 + + StackGuardSize = nimThreadStackGuard + ThreadStackSize = nimThreadStackSize - nimThreadStackGuard +else: + const + StackGuardSize = 4096 + ThreadStackMask = + when defined(genode): + 1024*64*sizeof(int)-1 + else: + 1024*256*sizeof(int)-1 + + ThreadStackSize = ThreadStackMask+1 - StackGuardSize + + +when defined(gcDestructors): + proc allocThreadStorage(size: int): pointer = + result = c_malloc(csize_t size) + zeroMem(result, size) +else: + template allocThreadStorage(size: untyped): untyped = allocShared0(size) + +#const globalsSlot = ThreadVarSlot(0) +#sysAssert checkSlot.int == globalsSlot.int + +# Zephyr doesn't include this properly without some help +when defined(zephyr): + {.emit: """/*INCLUDESECTION*/ + #include <pthread.h> + """.} + + +# We jump through some hops here to ensure that Nim thread procs can have +# the Nim calling convention. This is needed because thread procs are +# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just +# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. + + + +{.push stack_trace:off.} +when defined(windows): + proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = + nimThreadProcWrapperBody(closure) + # implicitly return 0 +elif defined(genode): + proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = + nimThreadProcWrapperBody(closure) +else: + proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = + nimThreadProcWrapperBody(closure) +{.pop.} + +proc running*[TArg](t: Thread[TArg]): bool {.inline.} = + ## Returns true if `t` is running. + result = t.dataFn != nil + +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## Returns the thread handle of `t`. + result = t.sys + +when hostOS == "windows": + const MAXIMUM_WAIT_OBJECTS = 64 + + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard waitForSingleObject(t.sys, -1'i32) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + var a: array[MAXIMUM_WAIT_OBJECTS, SysThread] + var k = 0 + while k < len(t): + var count = min(len(t) - k, MAXIMUM_WAIT_OBJECTS) + for i in 0..(count - 1): a[i] = t[i + k].sys + discard waitForMultipleObjects(int32(count), + cast[ptr SysThread](addr(a)), 1, -1) + inc(k, MAXIMUM_WAIT_OBJECTS) + +elif defined(genode): + proc joinThread*[TArg](t: Thread[TArg]) {.importcpp.} + ## Waits for the thread `t` to finish. + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +else: + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard pthread_join(t.sys, nil) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +when false: + # XXX a thread should really release its heap here somehow: + proc destroyThread*[TArg](t: var Thread[TArg]) = + ## Forces the thread `t` to terminate. This is potentially dangerous if + ## you don't have full control over `t` and its acquired resources. + when hostOS == "windows": + discard TerminateThread(t.sys, 1'i32) + else: + discard pthread_cancel(t.sys) + when declared(registerThread): unregisterThread(addr(t)) + t.dataFn = nil + ## if thread `t` already exited, `t.core` will be `null`. + if not isNil(t.core): + deallocThreadStorage(t.core) + t.core = nil + +when hostOS == "windows": + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. + ## `param` is passed to `tp`. `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var dummyThreadId: int32 + t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], + addr(t), 0'i32, dummyThreadId) + if t.sys <= 0: + raise newException(ResourceExhaustedError, "cannot create thread") + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + setThreadAffinityMask(t.sys, uint(1 shl cpu)) + +elif defined(genode): + var affinityOffset: cuint = 1 + ## CPU affinity offset for next thread, safe to roll-over. + + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.stackSize = ThreadStackSize + t.sys.initThread( + runtimeEnv, + ThreadStackSize.culonglong, + threadProcWrapper[TArg], addr(t), affinityOffset) + inc affinityOffset + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + {.hint: "cannot change Genode thread CPU affinity after initialization".} + discard + +else: + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. `param` is passed to `tp`. + ## `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var a {.noinit.}: Pthread_attr + doAssert pthread_attr_init(a) == 0 + when hasAllocStack: + var + rawstk = allocThreadStorage(ThreadStackSize + StackGuardSize) + stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) + let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + t.rawStack = rawstk + else: + let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + + when not defined(ios): + # This fails on iOS + doAssert(setstacksizeResult == 0) + if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: + raise newException(ResourceExhaustedError, "cannot create thread") + doAssert pthread_attr_destroy(a) == 0 + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + when not defined(macosx): + var s {.noinit.}: CpuSet + cpusetZero(s) + cpusetIncl(cpu.cint, s) + setAffinity(t.sys, csize_t(sizeof(s)), s) + +proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = + createThread[void](t, tp) + +when not defined(gcOrc): + include system/threadids diff --git a/lib/std/vmutils.nim b/lib/std/vmutils.nim new file mode 100644 index 0000000000..e16912a3c4 --- /dev/null +++ b/lib/std/vmutils.nim @@ -0,0 +1,11 @@ +##[ +Experimental API, subject to change. +]## + +proc vmTrace*(on: bool) {.compileTime.} = + runnableExamples: + static: vmTrace(true) + proc fn = + var a = 1 + vmTrace(false) + static: fn() diff --git a/lib/std/widestrs.nim b/lib/std/widestrs.nim new file mode 100644 index 0000000000..2ddf80d14e --- /dev/null +++ b/lib/std/widestrs.nim @@ -0,0 +1,239 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim support for C/C++'s `wide strings`:idx:. + +#when not declared(ThisIsSystem): +# {.error: "You must not import this module explicitly".} + +type + Utf16Char* = distinct int16 + +when not (defined(cpu16) or defined(cpu8)): + when defined(nimv2): + + type + WideCString* = ptr UncheckedArray[Utf16Char] + + WideCStringObj* = object + bytes: int + data: WideCString + + const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`(a: WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + else: + proc `=destroy`(a: var WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + + proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.} + + proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = + a.bytes = b.bytes + a.data = b.data + + proc createWide(a: var WideCStringObj; bytes: int) = + a.bytes = bytes + when compileOption("threads"): + a.data = cast[typeof(a.data)](allocShared0(bytes)) + else: + a.data = cast[typeof(a.data)](alloc0(bytes)) + + template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx] + template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val + + template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil) + + converter toWideCString*(x: WideCStringObj): WideCString {.inline.} = + result = x.data + + else: + template nullWide(): untyped = nil + + type + WideCString* = ref UncheckedArray[Utf16Char] + WideCStringObj* = WideCString + + template createWide(a; L) = + unsafeNew(a, L) + + proc ord(arg: Utf16Char): int = int(cast[uint16](arg)) + + proc len*(w: WideCString): int = + ## returns the length of a widestring. This traverses the whole string to + ## find the binary zero end marker! + result = 0 + while int16(w[result]) != 0'i16: inc result + + const + UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) + UNI_MAX_BMP = 0x0000FFFF + UNI_MAX_UTF16 = 0x0010FFFF + # UNI_MAX_UTF32 = 0x7FFFFFFF + # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF + + halfShift = 10 + halfBase = 0x0010000 + halfMask = 0x3FF + + UNI_SUR_HIGH_START = 0xD800 + UNI_SUR_HIGH_END = 0xDBFF + UNI_SUR_LOW_START = 0xDC00 + UNI_SUR_LOW_END = 0xDFFF + UNI_REPL = 0xFFFD + + template ones(n: untyped): untyped = ((1 shl n)-1) + + template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) = + ## Returns the unicode character `s[i]` in `result`. If `doInc == true` + ## `i` is incremented by the number of bytes that have been processed. + bind ones + + if ord(s[i]) <= 127: + result = ord(s[i]) + when doInc: inc(i) + elif ord(s[i]) shr 5 == 0b110: + #assert(ord(s[i+1]) shr 6 == 0b10) + if i <= L - 2: + result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) + when doInc: inc(i, 2) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 4 == 0b1110: + if i <= L - 3: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + result = (ord(s[i]) and ones(4)) shl 12 or + (ord(s[i+1]) and ones(6)) shl 6 or + (ord(s[i+2]) and ones(6)) + when doInc: inc(i, 3) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 3 == 0b11110: + if i <= L - 4: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + #assert(ord(s[i+3]) shr 6 == 0b10) + result = (ord(s[i]) and ones(3)) shl 18 or + (ord(s[i+1]) and ones(6)) shl 12 or + (ord(s[i+2]) and ones(6)) shl 6 or + (ord(s[i+3]) and ones(6)) + when doInc: inc(i, 4) + else: + result = UNI_REPL + when doInc: inc(i) + else: + result = 0xFFFD + when doInc: inc(i) + + iterator runes(s: cstring, L: int): int = + var + i = 0 + result: int + while i < L: + fastRuneAt(s, i, L, result, true) + yield result + + proc newWideCString*(size: int): WideCStringObj = + createWide(result, size * 2 + 2) + + proc newWideCString*(source: cstring, L: int): WideCStringObj = + ## Warning:: `source` needs to be preallocated with the length `L` + createWide(result, L * 2 + 2) + var d = 0 + for ch in runes(source, L): + + if ch <= UNI_MAX_BMP: + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END: + result[d] = UNI_REPLACEMENT_CHAR + else: + result[d] = cast[Utf16Char](uint16(ch)) + elif ch > UNI_MAX_UTF16: + result[d] = UNI_REPLACEMENT_CHAR + else: + let ch = ch - halfBase + result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START)) + inc d + result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START)) + inc d + result[d] = Utf16Char(0) + + proc newWideCString*(s: cstring): WideCStringObj = + if s.isNil: return nullWide + + result = newWideCString(s, s.len) + + proc newWideCString*(s: string): WideCStringObj = + result = newWideCString(cstring s, s.len) + + proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = + result = newStringOfCap(estimate + estimate shr 2) + + var i = 0 + while w[i].int16 != 0'i16: + var ch = ord(w[i]) + inc i + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: + # If the 16 bits following the high surrogate are in the source buffer... + let ch2 = ord(w[i]) + + # If it's a low surrogate, convert to UTF32: + if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: + ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase + inc i + else: + #invalid UTF-16 + ch = replacement + elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: + #invalid UTF-16 + ch = replacement + + if ch < 0x80: + result.add chr(ch) + elif ch < 0x800: + result.add chr((ch shr 6) or 0xc0) + result.add chr((ch and 0x3f) or 0x80) + elif ch < 0x10000: + result.add chr((ch shr 12) or 0xe0) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + elif ch <= 0x10FFFF: + result.add chr((ch shr 18) or 0xf0) + result.add chr(((ch shr 12) and 0x3f) or 0x80) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + else: + # replacement char(in case user give very large number): + result.add chr(0xFFFD shr 12 or 0b1110_0000) + result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) + result.add chr(0xFFFD and ones(6) or 0b10_0000_00) + + proc `$`*(s: WideCString): string = + result = s $ 80 + + when defined(nimv2): + proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string = + `$`(s.data, estimate, replacement) + + proc `$`*(s: WideCStringObj): string = + $(s.data) + + proc len*(w: WideCStringObj): int {.inline.} = + len(w.data) diff --git a/lib/std/with.nim b/lib/std/with.nim index 79afd61a4b..c2eaa4befc 100644 --- a/lib/std/with.nim +++ b/lib/std/with.nim @@ -12,15 +12,16 @@ ## and https://github.com/nim-lang/RFCs/issues/192 for details leading to this ## particular design. ## -## **Since** version 1.2. +## **Since:** version 1.2. -import macros, private / underscored_calls +import std/[macros, private / underscored_calls] macro with*(arg: typed; calls: varargs[untyped]): untyped = - ## This macro provides the `chaining`:idx: of function calls. + ## This macro provides `chaining`:idx: of function calls. ## It does so by patching every call in `calls` to ## use `arg` as the first argument. - ## **This evaluates `arg` multiple times!** + ## + ## .. caution:: This evaluates `arg` multiple times! runnableExamples: var x = "yay" with x: @@ -34,5 +35,14 @@ macro with*(arg: typed; calls: varargs[untyped]): untyped = -= 5 doAssert a == 43 + # Nesting works for object types too! + var foo = (bar: 1, qux: (baz: 2)) + with foo: + bar = 2 + with qux: + baz = 3 + doAssert foo.bar == 2 + doAssert foo.qux.baz == 3 + result = newNimNode(nnkStmtList, arg) underscoredCalls(result, calls, arg) diff --git a/lib/std/wordwrap.nim b/lib/std/wordwrap.nim index 7dcfc7f599..9333f880be 100644 --- a/lib/std/wordwrap.nim +++ b/lib/std/wordwrap.nim @@ -9,7 +9,7 @@ ## This module contains an algorithm to wordwrap a Unicode string. -import strutils, unicode +import std/[strutils, unicode] proc olen(s: string; start, lastExclusive: int): int = var i = start diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index 3ff48fbfea..0b75c270e9 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -1,9 +1,20 @@ -## This module allows chains of field-access and indexing where the LHS can be nil. -## This simplifies code by reducing need for if-else branches around intermediate values -## that may be nil. +## This module allows evaluating expressions safely against the following conditions: +## * nil dereferences +## * field accesses with incorrect discriminant in case objects +## +## `default(T)` is returned in those cases when evaluating an expression of type `T`. +## This simplifies code by reducing need for if-else branches. ## ## Note: experimental module, unstable API. +#[ +TODO: +consider handling indexing operations, eg: +doAssert ?.default(seq[int])[3] == default(int) +]# + +import std/macros + runnableExamples: type Foo = ref object x1: string @@ -24,8 +35,125 @@ runnableExamples: assert (?.f2.x2.x2).x3 == nil # this terminates ?. early +runnableExamples: + # ?. also allows case object + type B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + var b = B(cond: false, b0: 3) + doAssertRaises(FieldDefect): discard b.b1 # wrong discriminant + doAssert ?.b.b1 == 0.0 # safe + b = B(cond: true, b1: 4.5) + doAssert ?.b.b1 == 4.5 + + # lvalue semantics are preserved: + if (let p = ?.b.b1.addr; p != nil): p[] = 4.7 + doAssert b.b1 == 4.7 + +proc finalize(n: NimNode, lhs: NimNode, level: int): NimNode = + if level == 0: + result = quote: `lhs` = `n` + else: + result = quote: (let `lhs` = `n`) + +proc process(n: NimNode, lhs: NimNode, label: NimNode, level: int): NimNode = + var n = n.copyNimTree + var it = n + let addr2 = bindSym"addr" + var old: tuple[n: NimNode, index: int] + while true: + if it.len == 0: + result = finalize(n, lhs, level) + break + elif it.kind == nnkCheckedFieldExpr: + let dot = it[0] + let obj = dot[0] + let objRef = quote do: `addr2`(`obj`) + # avoids a copy and preserves lvalue semantics, see tests + let check = it[1] + let okSet = check[1] + let kind1 = check[2] + let tmp = genSym(nskLet, "tmpCase") + let body = process(objRef, tmp, label, level + 1) + let tmp3 = nnkDerefExpr.newTree(tmp) + it[0][0] = tmp3 + let dot2 = nnkDotExpr.newTree(@[tmp, dot[1]]) + if old.n != nil: old.n[old.index] = dot2 + else: n = dot2 + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp3`.`kind1` notin `okSet`: break `label` + `assgn` + break + elif it.kind in {nnkHiddenDeref, nnkDerefExpr}: + let tmp = genSym(nskLet, "tmp") + let body = process(it[0], tmp, label, level + 1) + it[0] = tmp + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp` == nil: break `label` + `assgn` + break + elif it.kind == nnkCall: # consider extending to `nnkCallKinds` + # `copyNimTree` needed to avoid `typ = nil` issues + old = (it, 1) + it = it[1].copyNimTree + else: + old = (it, 0) + it = it[0] + +macro `?.`*(a: typed): auto = + ## Transforms `a` into an expression that can be safely evaluated even in + ## presence of intermediate nil pointers/references, in which case a default + ## value is produced. + let lhs = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs, label, 0) + result = quote do: + var `lhs`: type(`a`) + block `label`: + `body` + `lhs` + +# the code below is not needed for `?.` from std/options import Option, isSome, get, option, unsafeGet, UnpackDefect -export options.get, options.isSome, options.isNone + +macro `??.`*(a: typed): Option = + ## Same as `?.` but returns an `Option`. + runnableExamples: + import std/options + type Foo = ref object + x1: ref int + x2: int + # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: + var f1 = Foo(x1: int.new, x2: 2) + doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. + doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain + doAssert (??.f1.x2).get == 2 + + var f2: Foo + doAssert not (??.f2.x1[]).isSome # f2 was nil + + doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get + doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) + + let lhs = genSym(nskVar, "lhs") + let lhs2 = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs2, label, 0) + result = quote do: + var `lhs`: Option[type(`a`)] + block `label`: + var `lhs2`: type(`a`) + `body` + `lhs` = option(`lhs2`) + `lhs` template fakeDot*(a: Option, b): untyped = ## See top-level example. @@ -59,50 +187,7 @@ func `[]`*[U](a: Option[U]): auto {.inline.} = if a2 != nil: result = option(a2[]) -import std/macros - -func replace(n: NimNode): NimNode = - if n.kind == nnkDotExpr: - result = newCall(bindSym"fakeDot", replace(n[0]), n[1]) - elif n.kind == nnkPar: - doAssert n.len == 1 - result = newCall(bindSym"option", n[0]) - elif n.kind in {nnkCall, nnkObjConstr}: - result = newCall(bindSym"option", n) - elif n.len == 0: - result = newCall(bindSym"option", n) - else: - n[0] = replace(n[0]) - result = n - -proc safeGet[T](a: Option[T]): T {.inline.} = - get(a, default(T)) - -macro `?.`*(a: untyped): auto = - ## Transforms `a` into an expression that can be safely evaluated even in - ## presence of intermediate nil pointers/references, in which case a default - ## value is produced. - result = replace(a) - result = quote do: - # `result`.val # TODO: expose a way to do this directly in std/options, e.g.: `getAsIs` - safeGet(`result`) - -macro `??.`*(a: untyped): Option = - ## Same as `?.` but returns an `Option`. - runnableExamples: - type Foo = ref object - x1: ref int - x2: int - # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: - var f1 = Foo(x1: int.new, x2: 2) - doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. - doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain - doAssert (??.f1.x2).get == 2 - - var f2: Foo - doAssert not (??.f2.x1[]).isSome # f2 was nil - from std/options import UnpackDefect - doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get - doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) - - result = replace(a) +when false: + # xxx: expose a way to do this directly in std/options, e.g.: `getAsIs` + proc safeGet[T](a: Option[T]): T {.inline.} = + get(a, default(T)) diff --git a/lib/system.nim b/lib/system.nim index 5740431259..ce4ee75363 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -22,201 +22,19 @@ ## .. include:: ./system_overview.rst -type - float* {.magic: Float.} ## Default floating point type. - float32* {.magic: Float32.} ## 32 bit floating point type. - float64* {.magic: Float.} ## 64 bit floating point type. - -# 'float64' is now an alias to 'float'; this solves many problems - -type - char* {.magic: Char.} ## Built-in 8 bit character type (unsigned). - string* {.magic: String.} ## Built-in string type. - cstring* {.magic: Cstring.} ## Built-in cstring (*compatible string*) type. - pointer* {.magic: Pointer.} ## Built-in pointer type, use the `addr` - ## operator to get a pointer to a variable. - - typedesc* {.magic: TypeDesc.} ## Meta type to denote a type description. - -type - `ptr`*[T] {.magic: Pointer.} ## Built-in generic untraced pointer type. - `ref`*[T] {.magic: Pointer.} ## Built-in generic traced pointer type. - - `nil` {.magic: "Nil".} - - void* {.magic: "VoidType".} ## Meta type to denote the absence of any type. - auto* {.magic: Expr.} ## Meta type for automatic type determination. - any* {.deprecated: "Deprecated since v1.5; Use auto instead.".} = distinct auto ## Deprecated; Use `auto` instead. See https://github.com/nim-lang/RFCs/issues/281 - untyped* {.magic: Expr.} ## Meta type to denote an expression that - ## is not resolved (for templates). - typed* {.magic: Stmt.} ## Meta type to denote an expression that - ## is resolved (for templates). - include "system/basic_types" - -proc compileOption*(option: string): bool {. - magic: "CompileOption", noSideEffect.} - ## Can be used to determine an `on|off` compile-time option. +func zeroDefault*[T](_: typedesc[T]): T {.magic: "ZeroDefault".} = + ## Returns the binary zeros representation of the type `T`. It ignores + ## default fields of an object. ## ## See also: - ## * `compileOption <#compileOption,string,string>`_ for enum options - ## * `defined <#defined,untyped>`_ - ## * `std/compilesettings module <compilesettings.html>`_ - ## - ## Example: - ## - ## .. code-block:: Nim - ## when compileOption("floatchecks"): - ## echo "compiled with floating point NaN and Inf checks" + ## * `default <#default,typedesc[T]>`_ -proc compileOption*(option, arg: string): bool {. - magic: "CompileOptionArg", noSideEffect.} - ## Can be used to determine an enum compile-time option. - ## - ## See also: - ## * `compileOption <#compileOption,string>`_ for `on|off` options - ## * `defined <#defined,untyped>`_ - ## * `std/compilesettings module <compilesettings.html>`_ - ## - ## Example: - ## - ## .. code-block:: Nim - ## when compileOption("opt", "size") and compileOption("gc", "boehm"): - ## echo "compiled with optimization for size and uses Boehm's GC" +include "system/compilation" {.push warning[GcMem]: off, warning[Uninit]: off.} -{.push hints: off.} - -proc `or`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} - ## Constructs an `or` meta class. - -proc `and`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} - ## Constructs an `and` meta class. - -proc `not`*(a: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} - ## Constructs an `not` meta class. - - -type - SomeFloat* = float|float32|float64 - ## Type class matching all floating point number types. - - SomeNumber* = SomeInteger|SomeFloat - ## Type class matching all number types. - -proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## defined. - ## - ## See also: - ## * `compileOption <#compileOption,string>`_ for `on|off` options - ## * `compileOption <#compileOption,string,string>`_ for enum options - ## * `define pragmas <manual.html#implementation-specific-pragmas-compileminustime-define-pragmas>`_ - ## - ## `x` is an external symbol introduced through the compiler's - ## `-d:x switch <nimc.html#compiler-usage-compileminustime-symbols>`_ to enable - ## build time conditionals: - ## - ## .. code-block:: Nim - ## when not defined(release): - ## # Do here programmer friendly expensive sanity checks. - ## # Put here the normal code - -when defined(nimHashOrdinalFixed): - type - Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, - ## bool, character, and enumeration types - ## as well as their subtypes. See also - ## `SomeOrdinal`. -else: - # bootstrap <= 0.20.0 - type - OrdinalImpl[T] {.magic: Ordinal.} - Ordinal* = OrdinalImpl | uint | uint64 - -proc runnableExamples*(rdoccmd = "", body: untyped) {.magic: "RunnableExamples".} - ## A section you should use to mark `runnable example`:idx: code with. - ## - ## - In normal debug and release builds code within - ## a `runnableExamples` section is ignored. - ## - The documentation generator is aware of these examples and considers them - ## part of the `##` doc comment. As the last step of documentation - ## generation each runnableExample is put in its own file `$file_examples$i.nim`, - ## compiled and tested. The collected examples are - ## put into their own module to ensure the examples do not refer to - ## non-exported symbols. - ## - ## Usage: - ## - ## .. code-block:: Nim - ## proc double*(x: int): int = - ## ## This proc doubles a number. - ## runnableExamples: - ## ## at module scope - ## assert double(5) == 10 - ## block: ## at block scope - ## defer: echo "done" - ## result = 2 * x - ## runnableExamples "-d:foo -b:cpp": - ## import std/compilesettings - ## doAssert querySetting(backend) == "cpp" - ## runnableExamples "-r:off": ## this one is only compiled - ## import std/browsers - ## openDefaultBrowser "https://forum.nim-lang.org/" - -when defined(nimHasDeclaredMagic): - proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared. `x` has to be an identifier or a qualified identifier. - ## - ## See also: - ## * `declaredInScope <#declaredInScope,untyped>`_ - ## - ## This can be used to check whether a library provides a certain - ## feature or not: - ## - ## .. code-block:: Nim - ## when not declared(strutils.toUpper): - ## # provide our own toUpper proc here, because strutils is - ## # missing it. -else: - proc declared*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} - -when defined(nimHasDeclaredMagic): - proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared in the current scope. `x` has to be an identifier. -else: - proc declaredInScope*(x: untyped): bool {.magic: "DefinedInScope", noSideEffect, compileTime.} - -proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = - ## Builtin `addr` operator for taking the address of a memory location. - ## Cannot be overloaded. - ## - ## See also: - ## * `unsafeAddr <#unsafeAddr,T>`_ - ## - ## .. code-block:: Nim - ## var - ## buf: seq[char] = @['a','b','c'] - ## p = buf[1].addr - ## echo p.repr # ref 0x7faa35c40059 --> 'b' - ## echo p[] # b - discard - -proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = - ## Builtin `addr` operator for taking the address of a memory - ## location. This works even for `let` variables or parameters - ## for better interop with C and so it is considered even more - ## unsafe than the ordinary `addr <#addr,T>`_. - ## - ## **Note**: When you use it to write a wrapper for a C library, you should - ## always check that the original library does never write to data behind the - ## pointer that is returned from this procedure. - ## - ## Cannot be overloaded. - discard +# {.push hints: off.} type `static`*[T] {.magic: "Static".} @@ -248,44 +66,104 @@ proc typeof*(x: untyped; mode = typeOfIter): typedesc {. doAssert type(myFoo()) is string doAssert typeof(myFoo()) is string doAssert typeof(myFoo(), typeOfIter) is string - doAssert typeof(myFoo3) is "iterator" + doAssert typeof(myFoo3) is iterator doAssert typeof(myFoo(), typeOfProc) is float doAssert typeof(0.0, typeOfProc) is float - doAssert typeof(myFoo3, typeOfProc) is "iterator" + doAssert typeof(myFoo3, typeOfProc) is iterator doAssert not compiles(typeof(myFoo2(), typeOfProc)) # this would give: Error: attempting to call routine: 'myFoo2' # since `typeOfProc` expects a typed expression and `myFoo2()` can # only be used in a `for` context. +proc `or`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} + ## Constructs an `or` meta class. + +proc `and`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} + ## Constructs an `and` meta class. + +proc `not`*(a: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} + ## Constructs an `not` meta class. + +when defined(nimHasIterable): + type + iterable*[T] {.magic: IterableType.} ## Represents an expression that yields `T` + +type + Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, + ## bool, character, and enumeration types + ## as well as their subtypes. See also + ## `SomeOrdinal`. + + +proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = + ## Builtin `addr` operator for taking the address of a memory location. + ## + ## .. note:: This works for `let` variables or parameters + ## for better interop with C. When you use it to write a wrapper + ## for a C library and take the address of `let` variables or parameters, + ## you should always check that the original library + ## does never write to data behind the pointer that is returned from + ## this procedure. + ## + ## Cannot be overloaded. + ## + ## ```nim + ## var + ## buf: seq[char] = @['a','b','c'] + ## p = buf[1].addr + ## echo p.repr # ref 0x7faa35c40059 --> 'b' + ## echo p[] # b + ## ``` + discard + +proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = + ## .. warning:: `unsafeAddr` is a deprecated alias for `addr`, + ## use `addr` instead. + discard + + const ThisIsSystem = true -proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.} - ## Leaked implementation detail. Do not use. +proc new*[T](a: var ref T, finalizer: proc (x: ref T) {.nimcall.}) {. + magic: "NewFinalize", noSideEffect.} + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. + ## + ## When the garbage collector frees the object, `finalizer` is called. + ## The `finalizer` may not keep a reference to the + ## object pointed to by `x`. The `finalizer` cannot prevent the GC from + ## freeing the object. + ## + ## **Note**: The `finalizer` refers to the type `T`, not to the object! + ## This means that for each object of type `T` the finalizer will be called! -when true: - proc new*[T](a: var ref T, finalizer: proc (x: ref T) {.nimcall.}) {. - magic: "NewFinalize", noSideEffect.} - ## Creates a new object of type `T` and returns a safe (traced) - ## reference to it in `a`. - ## - ## When the garbage collector frees the object, `finalizer` is called. - ## The `finalizer` may not keep a reference to the - ## object pointed to by `x`. The `finalizer` cannot prevent the GC from - ## freeing the object. - ## - ## **Note**: The `finalizer` refers to the type `T`, not to the object! - ## This means that for each object of type `T` the finalizer will be called! +proc `=wasMoved`*[T](obj: var T) {.magic: "WasMoved", noSideEffect.} = + ## Generic `wasMoved`:idx: implementation that can be overridden. -proc wasMoved*[T](obj: var T) {.magic: "WasMoved", noSideEffect.} = +proc wasMoved*[T](obj: var T) {.inline, noSideEffect.} = ## Resets an object `obj` to its initial (binary zero) value to signify ## it was "moved" and to signify its destructor should do nothing and ## ideally be optimized away. - discard + {.cast(raises: []), cast(tags: []).}: + `=wasMoved`(obj) proc move*[T](x: var T): T {.magic: "Move", noSideEffect.} = result = x - wasMoved(x) + {.cast(raises: []), cast(tags: []).}: + `=wasMoved`(x) + +when defined(nimHasEnsureMove): + proc ensureMove*[T](x: T): T {.magic: "EnsureMove", noSideEffect.} = + ## Ensures that `x` is moved to the new location, otherwise it gives + ## an error at the compile time. + runnableExamples: + proc foo = + var x = "Hello" + let y = ensureMove(x) + doAssert y == "Hello" + foo() + discard "implemented in injectdestructors" type range*[T]{.magic: "Range".} ## Generic type to construct range types. @@ -315,52 +193,57 @@ proc high*[T: Ordinal|enum|range](x: T): T {.magic: "High", noSideEffect, ## **This proc is deprecated**, use this one instead: ## * `high(typedesc) <#high,typedesc[T]>`_ ## - ## .. code-block:: Nim - ## high(2) # => 9223372036854775807 + ## ```nim + ## high(2) # => 9223372036854775807 + ## ``` proc high*[T: Ordinal|enum|range](x: typedesc[T]): T {.magic: "High", noSideEffect.} ## Returns the highest possible value of an ordinal or enum type. ## ## `high(int)` is Nim's way of writing `INT_MAX`:idx: or `MAX_INT`:idx:. + ## ```nim + ## high(int) # => 9223372036854775807 + ## ``` ## ## See also: ## * `low(typedesc) <#low,typedesc[T]>`_ - ## - ## .. code-block:: Nim - ## high(int) # => 9223372036854775807 proc high*[T](x: openArray[T]): int {.magic: "High", noSideEffect.} ## Returns the highest possible index of a sequence `x`. + ## ```nim + ## var s = @[1, 2, 3, 4, 5, 6, 7] + ## high(s) # => 6 + ## for i in low(s)..high(s): + ## echo s[i] + ## ``` ## ## See also: ## * `low(openArray) <#low,openArray[T]>`_ - ## - ## .. code-block:: Nim - ## var s = @[1, 2, 3, 4, 5, 6, 7] - ## high(s) # => 6 - ## for i in low(s)..high(s): - ## echo s[i] proc high*[I, T](x: array[I, T]): I {.magic: "High", noSideEffect.} ## Returns the highest possible index of an array `x`. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## var arr = [1, 2, 3, 4, 5, 6, 7] + ## high(arr) # => 6 + ## for i in low(arr)..high(arr): + ## echo arr[i] + ## ``` + ## ## See also: ## * `low(array) <#low,array[I,T]>`_ - ## - ## .. code-block:: Nim - ## var arr = [1, 2, 3, 4, 5, 6, 7] - ## high(arr) # => 6 - ## for i in low(arr)..high(arr): - ## echo arr[i] proc high*[I, T](x: typedesc[array[I, T]]): I {.magic: "High", noSideEffect.} ## Returns the highest possible index of an array type. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## high(array[7, int]) # => 6 + ## ``` + ## ## See also: ## * `low(typedesc[array]) <#low,typedesc[array[I,T]]>`_ - ## - ## .. code-block:: Nim - ## high(array[7, int]) # => 6 proc high*(x: cstring): int {.magic: "High", noSideEffect.} ## Returns the highest possible index of a compatible string `x`. @@ -371,13 +254,13 @@ proc high*(x: cstring): int {.magic: "High", noSideEffect.} proc high*(x: string): int {.magic: "High", noSideEffect.} ## Returns the highest possible index of a string `x`. + ## ```nim + ## var str = "Hello world!" + ## high(str) # => 11 + ## ``` ## ## See also: ## * `low(string) <#low,string>`_ - ## - ## .. code-block:: Nim - ## var str = "Hello world!" - ## high(str) # => 11 proc low*[T: Ordinal|enum|range](x: T): T {.magic: "Low", noSideEffect, deprecated: "Deprecated since v1.4; there should not be `low(value)`. Use `low(type)`.".} @@ -387,52 +270,57 @@ proc low*[T: Ordinal|enum|range](x: T): T {.magic: "Low", noSideEffect, ## **This proc is deprecated**, use this one instead: ## * `low(typedesc) <#low,typedesc[T]>`_ ## - ## .. code-block:: Nim - ## low(2) # => -9223372036854775808 + ## ```nim + ## low(2) # => -9223372036854775808 + ## ``` proc low*[T: Ordinal|enum|range](x: typedesc[T]): T {.magic: "Low", noSideEffect.} ## Returns the lowest possible value of an ordinal or enum type. ## ## `low(int)` is Nim's way of writing `INT_MIN`:idx: or `MIN_INT`:idx:. + ## ```nim + ## low(int) # => -9223372036854775808 + ## ``` ## ## See also: ## * `high(typedesc) <#high,typedesc[T]>`_ - ## - ## .. code-block:: Nim - ## low(int) # => -9223372036854775808 proc low*[T](x: openArray[T]): int {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of a sequence `x`. + ## ```nim + ## var s = @[1, 2, 3, 4, 5, 6, 7] + ## low(s) # => 0 + ## for i in low(s)..high(s): + ## echo s[i] + ## ``` ## ## See also: ## * `high(openArray) <#high,openArray[T]>`_ - ## - ## .. code-block:: Nim - ## var s = @[1, 2, 3, 4, 5, 6, 7] - ## low(s) # => 0 - ## for i in low(s)..high(s): - ## echo s[i] proc low*[I, T](x: array[I, T]): I {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of an array `x`. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## var arr = [1, 2, 3, 4, 5, 6, 7] + ## low(arr) # => 0 + ## for i in low(arr)..high(arr): + ## echo arr[i] + ## ``` + ## ## See also: ## * `high(array) <#high,array[I,T]>`_ - ## - ## .. code-block:: Nim - ## var arr = [1, 2, 3, 4, 5, 6, 7] - ## low(arr) # => 0 - ## for i in low(arr)..high(arr): - ## echo arr[i] proc low*[I, T](x: typedesc[array[I, T]]): I {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of an array type. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## low(array[7, int]) # => 0 + ## ``` + ## ## See also: ## * `high(typedesc[array]) <#high,typedesc[array[I,T]]>`_ - ## - ## .. code-block:: Nim - ## low(array[7, int]) # => 0 proc low*(x: cstring): int {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of a compatible string `x`. @@ -442,23 +330,24 @@ proc low*(x: cstring): int {.magic: "Low", noSideEffect.} proc low*(x: string): int {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of a string `x`. + ## ```nim + ## var str = "Hello world!" + ## low(str) # => 0 + ## ``` ## ## See also: ## * `high(string) <#high,string>`_ - ## - ## .. code-block:: Nim - ## var str = "Hello world!" - ## low(str) # => 0 -proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} - ## Use this instead of `=` for a `shallow copy`:idx:. - ## - ## The shallow copy only changes the semantics for sequences and strings - ## (and types which contain those). - ## - ## Be careful with the changed semantics though! - ## There is a reason why the default assignment does a deep copy of sequences - ## and strings. +when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} + ## Use this instead of `=` for a `shallow copy`:idx:. + ## + ## The shallow copy only changes the semantics for sequences and strings + ## (and types which contain those). + ## + ## Be careful with the changed semantics though! + ## There is a reason why the default assignment does a deep copy of sequences + ## and strings. # :array|openArray|string|seq|cstring|tuple proc `[]`*[I: Ordinal;T](a: T; i: I): T {. @@ -466,18 +355,51 @@ proc `[]`*[I: Ordinal;T](a: T; i: I): T {. proc `[]=`*[I: Ordinal;T,S](a: T; i: I; x: sink S) {.noSideEffect, magic: "ArrPut".} proc `=`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} +proc `=copy`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} proc arrGet[I: Ordinal;T](a: T; i: I): T {. noSideEffect, magic: "ArrGet".} proc arrPut[I: Ordinal;T,S](a: T; i: I; x: S) {.noSideEffect, magic: "ArrPut".} -proc `=destroy`*[T](x: var T) {.inline, magic: "Destroy".} = - ## Generic `destructor`:idx: implementation that can be overridden. - discard -proc `=sink`*[T](x: var T; y: T) {.inline, magic: "Asgn".} = +const arcLikeMem = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + + +when defined(nimAllowNonVarDestructor) and arcLikeMem and defined(nimPreviewNonVarDestructor): + proc `=destroy`*[T](x: T) {.inline, magic: "Destroy".} = + ## Generic `destructor`:idx: implementation that can be overridden. + discard +else: + proc `=destroy`*[T](x: var T) {.inline, magic: "Destroy".} = + ## Generic `destructor`:idx: implementation that can be overridden. + discard + + when defined(nimAllowNonVarDestructor) and arcLikeMem: + proc `=destroy`*(x: string) {.inline, magic: "Destroy", enforceNoRaises.} = + discard + + proc `=destroy`*[T](x: seq[T]) {.inline, magic: "Destroy".} = + discard + + proc `=destroy`*[T](x: ref T) {.inline, magic: "Destroy".} = + discard + +when defined(nimHasDup): + proc `=dup`*[T](x: T): T {.inline, magic: "Dup".} = + ## Generic `dup`:idx: implementation that can be overridden. + discard + +proc `=sink`*[T](x: var T; y: T) {.inline, nodestroy, magic: "Asgn".} = ## Generic `sink`:idx: implementation that can be overridden. - shallowCopy(x, y) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + x = y + else: + shallowCopy(x, y) + +when defined(nimHasTrace): + proc `=trace`*[T](x: var T; env: pointer) {.inline, magic: "Trace".} = + ## Generic `trace`:idx: implementation that can be overridden. + discard type HSlice*[T, U] = object ## "Heterogeneous" slice type. @@ -491,18 +413,19 @@ proc `..`*[T, U](a: sink T, b: sink U): HSlice[T, U] {.noSideEffect, inline, mag ## ## Slices can also be used in the set constructor and in ordinal case ## statements, but then they are special-cased by the compiler. - ## - ## .. code-block:: Nim + ## ```nim ## let a = [10, 20, 30, 40, 50] ## echo a[2 .. 3] # @[30, 40] + ## ``` result = HSlice[T, U](a: a, b: b) -proc `..`*[T](b: sink T): HSlice[int, T] {.noSideEffect, inline, magic: "DotDot".} = +proc `..`*[T](b: sink T): HSlice[int, T] + {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} = ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`. - ## - ## .. code-block:: Nim + ## ```nim ## let a = [10, 20, 30, 40, 50] ## echo a[.. 2] # @[10, 20, 30] + ## ``` result = HSlice[int, T](a: 0, b: b) when defined(hotCodeReloading): @@ -510,23 +433,17 @@ when defined(hotCodeReloading): else: {.pragma: hcrInline.} -{.push profiler: off.} -let nimvm* {.magic: "Nimvm", compileTime.}: bool = false - ## May be used only in `when` expression. - ## It is true in Nim VM context and false otherwise. -{.pop.} - include "system/arithmetics" include "system/comparisons" const - appType* {.magic: "AppType"}: string = "" + appType* {.magic: "AppType".}: string = "" ## A string that describes the application type. Possible values: ## `"console"`, `"gui"`, `"lib"`. include "system/inclrtl" -const NoFakeVars* = defined(nimscript) ## `true` if the backend doesn't support \ +const NoFakeVars = defined(nimscript) ## `true` if the backend doesn't support \ ## "fake variables" like `var EBADF {.importc.}: cint`. const notJSnotNims = not defined(js) and not defined(nimscript) @@ -544,10 +461,6 @@ when not defined(js) and not defined(nimSeqsV2): data: UncheckedArray[char] NimString = ptr NimStringDesc -when notJSnotNims and not defined(nimSeqsV2): - template space(s: PGenericSeq): int {.dirty.} = - s.reserved and not (seqShallowFlag or strlitFlag) - when notJSnotNims: include "system/hti" @@ -563,6 +476,7 @@ type ## is an `int` type ranging from one to the maximum value ## of an `int`. This type is often useful for documentation and debugging. +type RootObj* {.compilerproc, inheritable.} = object ## The root of Nim's object hierarchy. ## @@ -570,8 +484,64 @@ type ## However, objects that have no ancestor are also allowed. RootRef* = ref RootObj ## Reference to `RootObj`. +const NimStackTraceMsgs = compileOption("stacktraceMsgs") -include "system/exceptions" +type + RootEffect* {.compilerproc.} = object of RootObj ## \ + ## Base effect class. + ## + ## Each effect should inherit from `RootEffect` unless you know what + ## you're doing. + +type + StackTraceEntry* = object ## In debug mode exceptions store the stack trace that led + ## to them. A `StackTraceEntry` is a single entry of the + ## stack trace. + procname*: cstring ## Name of the proc that is currently executing. + line*: int ## Line number of the proc that is currently executing. + filename*: cstring ## Filename of the proc that is currently executing. + when NimStackTraceMsgs: + frameMsg*: string ## When a stacktrace is generated in a given frame and + ## rendered at a later time, we should ensure the stacktrace + ## data isn't invalidated; any pointer into PFrame is + ## subject to being invalidated so shouldn't be stored. + when defined(nimStackTraceOverride): + programCounter*: uint ## Program counter - will be used to get the rest of the info, + ## when `$` is called on this type. We can't use + ## "cuintptr_t" in here. + procnameStr*, filenameStr*: string ## GC-ed alternatives to "procname" and "filename" + + Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \ + ## Base exception class. + ## + ## Each exception has to inherit from `Exception`. See the full `exception + ## hierarchy <manual.html#exception-handling-exception-hierarchy>`_. + parent*: ref Exception ## Parent exception (can be used as a stack). + name*: cstring ## The exception's name is its Nim identifier. + ## This field is filled automatically in the + ## `raise` statement. + msg* {.exportc: "message".}: string ## The exception's message. Not + ## providing an exception message + ## is bad style. + when defined(js): + trace*: string + else: + trace*: seq[StackTraceEntry] + up: ref Exception # used for stacking exceptions. Not exported! + + Defect* = object of Exception ## \ + ## Abstract base class for all exceptions that Nim's runtime raises + ## but that are strictly uncatchable as they can also be mapped to + ## a `quit` / `trap` / `exit` operation. + + CatchableError* = object of Exception ## \ + ## Abstract class for all exceptions that are catchable. + +when defined(nimIcIntegrityChecks): + include "system/exceptions" +else: + import system/exceptions + export exceptions when defined(js) or defined(nimdoc): type @@ -603,10 +573,10 @@ proc sizeof*[T](x: T): int {.magic: "SizeOf", noSideEffect.} ## sizeof should fallback to the `sizeof` in the C compiler. The ## result isn't available for the Nim compiler and therefore can't ## be used inside of macros. - ## - ## .. code-block:: Nim - ## sizeof('A') # => 1 - ## sizeof(2) # => 8 + ## ```nim + ## sizeof('A') # => 1 + ## sizeof(2) # => 8 + ## ``` proc alignof*[T](x: T): int {.magic: "AlignOf", noSideEffect.} proc alignof*(x: typedesc): int {.magic: "AlignOf", noSideEffect.} @@ -634,8 +604,7 @@ proc newSeq*[T](s: var seq[T], len: Natural) {.magic: "NewSeq", noSideEffect.} ## Note that the sequence will be filled with zeroed entries. ## After the creation of the sequence you should assign entries to ## the sequence instead of adding them. Example: - ## - ## .. code-block:: Nim + ## ```nim ## var inputStrings: seq[string] ## newSeq(inputStrings, 3) ## assert len(inputStrings) == 3 @@ -643,6 +612,7 @@ proc newSeq*[T](s: var seq[T], len: Natural) {.magic: "NewSeq", noSideEffect.} ## inputStrings[1] = "assignment" ## inputStrings[2] = "would crash" ## #inputStrings[3] = "out of bounds" + ## ``` proc newSeq*[T](len = 0.Natural): seq[T] = ## Creates a new sequence of type `seq[T]` with length `len`. @@ -650,51 +620,32 @@ proc newSeq*[T](len = 0.Natural): seq[T] = ## Note that the sequence will be filled with zeroed entries. ## After the creation of the sequence you should assign entries to ## the sequence instead of adding them. - ## - ## See also: - ## * `newSeqOfCap <#newSeqOfCap,Natural>`_ - ## * `newSeqUninitialized <#newSeqUninitialized,Natural>`_ - ## - ## .. code-block:: Nim + ## ```nim ## var inputStrings = newSeq[string](3) ## assert len(inputStrings) == 3 ## inputStrings[0] = "The fourth" ## inputStrings[1] = "assignment" ## inputStrings[2] = "would crash" ## #inputStrings[3] = "out of bounds" + ## ``` + ## + ## See also: + ## * `newSeqOfCap <#newSeqOfCap,Natural>`_ + ## * `newSeqUninit <#newSeqUninit,Natural>`_ newSeq(result, len) proc newSeqOfCap*[T](cap: Natural): seq[T] {. magic: "NewSeqOfCap", noSideEffect.} = ## Creates a new sequence of type `seq[T]` with length zero and capacity - ## `cap`. - ## - ## .. code-block:: Nim + ## `cap`. Example: + ## ```nim ## var x = newSeqOfCap[int](5) ## assert len(x) == 0 ## x.add(10) ## assert len(x) == 1 + ## ``` discard -when not defined(js): - proc newSeqUninitialized*[T: SomeNumber](len: Natural): seq[T] = - ## Creates a new sequence of type `seq[T]` with length `len`. - ## - ## Only available for numbers types. Note that the sequence will be - ## uninitialized. After the creation of the sequence you should assign - ## entries to the sequence instead of adding them. - ## - ## .. code-block:: Nim - ## var x = newSeqUninitialized[int](3) - ## assert len(x) == 3 - ## x[0] = 10 - result = newSeqOfCap[T](len) - when defined(nimSeqsV2): - cast[ptr int](addr result)[] = len - else: - var s = cast[PGenericSeq](result) - s.len = len - func len*[TOpenArray: openArray|varargs](x: TOpenArray): int {.magic: "LengthOpenArray".} = ## Returns the length of an openArray. runnableExamples: @@ -770,62 +721,42 @@ func chr*(u: range[0..255]): char {.magic: "Chr".} = doAssertRaises(RangeDefect): discard chr(x) doAssertRaises(RangeDefect): discard char(x) -# floating point operations: -proc `+`*(x: float32): float32 {.magic: "UnaryPlusF64", noSideEffect.} -proc `-`*(x: float32): float32 {.magic: "UnaryMinusF64", noSideEffect.} -proc `+`*(x, y: float32): float32 {.magic: "AddF64", noSideEffect.} -proc `-`*(x, y: float32): float32 {.magic: "SubF64", noSideEffect.} -proc `*`*(x, y: float32): float32 {.magic: "MulF64", noSideEffect.} -proc `/`*(x, y: float32): float32 {.magic: "DivF64", noSideEffect.} - -proc `+`*(x: float): float {.magic: "UnaryPlusF64", noSideEffect.} -proc `-`*(x: float): float {.magic: "UnaryMinusF64", noSideEffect.} -proc `+`*(x, y: float): float {.magic: "AddF64", noSideEffect.} -proc `-`*(x, y: float): float {.magic: "SubF64", noSideEffect.} -proc `*`*(x, y: float): float {.magic: "MulF64", noSideEffect.} -proc `/`*(x, y: float): float {.magic: "DivF64", noSideEffect.} - -proc `==`*(x, y: float32): bool {.magic: "EqF64", noSideEffect.} -proc `<=`*(x, y: float32): bool {.magic: "LeF64", noSideEffect.} -proc `<` *(x, y: float32): bool {.magic: "LtF64", noSideEffect.} - -proc `==`*(x, y: float): bool {.magic: "EqF64", noSideEffect.} -proc `<=`*(x, y: float): bool {.magic: "LeF64", noSideEffect.} -proc `<`*(x, y: float): bool {.magic: "LtF64", noSideEffect.} - include "system/setops" proc contains*[U, V, W](s: HSlice[U, V], value: W): bool {.noSideEffect, inline.} = ## Checks if `value` is within the range of `s`; returns true if - ## `value >= s.a and value <= s.b` - ## - ## .. code-block:: Nim + ## `value >= s.a and value <= s.b`. + ## ```nim ## assert((1..3).contains(1) == true) ## assert((1..3).contains(2) == true) ## assert((1..3).contains(4) == false) + ## ``` result = s.a <= value and value <= s.b -template `in`*(x, y: untyped): untyped {.dirty.} = contains(y, x) +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template `in`*(x, y: untyped): untyped {.dirty, callsite.} = contains(y, x) ## Sugar for `contains`. - ## - ## .. code-block:: Nim + ## ```nim ## assert(1 in (1..3) == true) ## assert(5 in (1..3) == false) -template `notin`*(x, y: untyped): untyped {.dirty.} = not contains(y, x) + ## ``` +template `notin`*(x, y: untyped): untyped {.dirty, callsite.} = not contains(y, x) ## Sugar for `not contains`. - ## - ## .. code-block:: Nim + ## ```nim ## assert(1 notin (1..3) == false) ## assert(5 notin (1..3) == true) + ## ``` proc `is`*[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## Checks if `T` is of the same type as `S`. ## ## For a negated version, use `isnot <#isnot.t,untyped,untyped>`_. ## - ## .. code-block:: Nim + ## ```nim ## assert 42 is int ## assert @[1, 2] is seq ## @@ -837,12 +768,13 @@ proc `is`*[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## ## assert(test[int](3) == 3) ## assert(test[string]("xyz") == 0) -template `isnot`*(x, y: untyped): untyped = not (x is y) + ## ``` +template `isnot`*(x, y: untyped): untyped {.callsite.} = not (x is y) ## Negated version of `is <#is,T,S>`_. Equivalent to `not(x is y)`. - ## - ## .. code-block:: Nim + ## ```nim ## assert 42 isnot float ## assert @[1, 2] isnot enum + ## ``` when (defined(nimOwnedEnabled) and not defined(nimscript)) or defined(nimFixedOwned): type owned*[T]{.magic: "BuiltinType".} ## type constructor to mark a ref/ptr or a closure as `owned`. @@ -870,8 +802,6 @@ when defined(nimOwnedEnabled) and not defined(nimscript): proc unown*[T](x: T): T {.magic: "Unown", noSideEffect.} ## Use the expression `x` ignoring its ownership attribute. - # This is only required to make 0.20 compile with the 0.19 line. - template `<//>`*(t: untyped): untyped = owned(t) else: template unown*(x: typed): untyped = x @@ -893,8 +823,6 @@ else: new(r) return r - # This is only required to make 0.20 compile with the 0.19 line. - template `<//>`*(t: untyped): untyped = t template disarm*(x: typed) = ## Useful for `disarming` dangling pointers explicitly for `--newruntime`. @@ -939,10 +867,10 @@ proc cmp*[T](x, y: T): int = ## ## This is useful for writing generic algorithms without performance loss. ## This generic implementation uses the `==` and `<` operators. - ## - ## .. code-block:: Nim - ## import std/algorithm - ## echo sorted(@[4, 2, 6, 5, 8, 7], cmp[int]) + ## ```nim + ## import std/algorithm + ## echo sorted(@[4, 2, 6, 5, 8, 7], cmp[int]) + ## ``` if x == y: return 0 if x < y: return -1 return 1 @@ -960,50 +888,56 @@ proc `@`* [IDX, T](a: sink array[IDX, T]): seq[T] {.magic: "ArrToSeq", noSideEff ## sequences with the array constructor: `@[1, 2, 3]` has the type ## `seq[int]`, while `[1, 2, 3]` has the type `array[0..2, int]`. ## - ## .. code-block:: Nim + ## ```nim ## let ## a = [1, 3, 5] ## b = "foo" ## ## echo @a # => @[1, 3, 5] ## echo @b # => @['f', 'o', 'o'] + ## ``` -proc default*(T: typedesc): T {.magic: "Default", noSideEffect.} = - ## returns the default value of the type `T`. - runnableExamples: +proc default*[T](_: typedesc[T]): T {.magic: "Default", noSideEffect.} = + ## Returns the default value of the type `T`. Contrary to `zeroDefault`, it takes default fields + ## of an object into consideration. + ## + ## See also: + ## * `zeroDefault <#zeroDefault,typedesc[T]>`_ + ## + runnableExamples("-d:nimPreviewRangeDefault"): assert (int, float).default == (0, 0.0) - # note: `var a = default(T)` is usually the same as `var a: T` and (currently) generates - # a value whose binary representation is all 0, regardless of whether this - # would violate type constraints such as `range`, `not nil`, etc. This - # property is required to implement certain algorithms efficiently which - # may require intermediate invalid states. type Foo = object a: range[2..6] - var a1: range[2..6] # currently, this compiles - # var a2: Foo # currently, this errors: Error: The Foo type doesn't have a default value. - # var a3 = Foo() # ditto - var a3 = Foo.default # this works, but generates a `UnsafeDefault` warning. - # note: the doc comment also explains why `default` can't be implemented - # via: `template default*[T](t: typedesc[T]): T = (var v: T; v)` + var x = Foo.default + assert x.a == 2 + proc reset*[T](obj: var T) {.noSideEffect.} = ## Resets an object `obj` to its default value. - obj = default(typeof(obj)) + when nimvm: + obj = default(typeof(obj)) + else: + when defined(gcDestructors): + {.cast(noSideEffect), cast(raises: []), cast(tags: []).}: + `=destroy`(obj) + `=wasMoved`(obj) + else: + obj = default(typeof(obj)) proc setLen*[T](s: var seq[T], newlen: Natural) {. - magic: "SetLengthSeq", noSideEffect.} + magic: "SetLengthSeq", noSideEffect, nodestroy.} ## Sets the length of seq `s` to `newlen`. `T` may be any sequence type. ## ## If the current length is greater than the new length, ## `s` will be truncated. - ## - ## .. code-block:: Nim + ## ```nim ## var x = @[10, 20] ## x.setLen(5) ## x[4] = 50 ## assert x == @[10, 20, 0, 0, 50] ## x.setLen(1) ## assert x == @[10] + ## ``` proc setLen*(s: var string, newlen: Natural) {. magic: "SetLengthStr", noSideEffect.} @@ -1011,16 +945,16 @@ proc setLen*(s: var string, newlen: Natural) {. ## ## If the current length is greater than the new length, ## `s` will be truncated. - ## - ## .. code-block:: Nim - ## var myS = "Nim is great!!" - ## myS.setLen(3) # myS <- "Nim" - ## echo myS, " is fantastic!!" + ## ```nim + ## var myS = "Nim is great!!" + ## myS.setLen(3) # myS <- "Nim" + ## echo myS, " is fantastic!!" + ## ``` proc newString*(len: Natural): string {. magic: "NewString", importc: "mnewString", noSideEffect.} - ## Returns a new string of length `len` but with uninitialized - ## content. One needs to fill the string character after character + ## Returns a new string of length `len`. + ## One needs to fill the string character after character ## with the index operator `s[i]`. ## ## This procedure exists only for optimization purposes; @@ -1034,41 +968,41 @@ proc newStringOfCap*(cap: Natural): string {. ## be achieved with the `&` operator or with `add`. proc `&`*(x: string, y: char): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates `x` with `y`. - ## - ## .. code-block:: Nim + ## ```nim ## assert("ab" & 'c' == "abc") + ## ``` proc `&`*(x, y: char): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates characters `x` and `y` into a string. - ## - ## .. code-block:: Nim + ## ```nim ## assert('a' & 'b' == "ab") + ## ``` proc `&`*(x, y: string): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates strings `x` and `y`. - ## - ## .. code-block:: Nim + ## ```nim ## assert("ab" & "cd" == "abcd") + ## ``` proc `&`*(x: char, y: string): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates `x` with `y`. - ## - ## .. code-block:: Nim + ## ```nim ## assert('a' & "bc" == "abc") + ## ``` # implementation note: These must all have the same magic value "ConStrStr" so # that the merge optimization works properly. proc add*(x: var string, y: char) {.magic: "AppendStrCh", noSideEffect.} ## Appends `y` to `x` in place. - ## - ## .. code-block:: Nim + ## ```nim ## var tmp = "" ## tmp.add('a') ## tmp.add('b') ## assert(tmp == "ab") + ## ``` proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} = ## Concatenates `x` and `y` in place. @@ -1085,19 +1019,7 @@ type littleEndian, bigEndian const - isMainModule* {.magic: "IsMainModule".}: bool = false - ## True only when accessed in the main module. This works thanks to - ## compiler magic. It is useful to embed testing code in a module. - - CompileDate* {.magic: "CompileDate"}: string = "0000-00-00" - ## The date (in UTC) of compilation as a string of the form - ## `YYYY-MM-DD`. This works thanks to compiler magic. - - CompileTime* {.magic: "CompileTime"}: string = "00:00:00" - ## The time (in UTC) of compilation as a string of the form - ## `HH:MM:SS`. This works thanks to compiler magic. - - cpuEndian* {.magic: "CpuEndian"}: Endianness = littleEndian + cpuEndian* {.magic: "CpuEndian".}: Endianness = littleEndian ## The endianness of the target CPU. This is a valuable piece of ## information for low-level code only. This works thanks to compiler ## magic. @@ -1115,7 +1037,7 @@ const ## Possible values: ## `"i386"`, `"alpha"`, `"powerpc"`, `"powerpc64"`, `"powerpc64el"`, ## `"sparc"`, `"amd64"`, `"mips"`, `"mipsel"`, `"arm"`, `"arm64"`, - ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`. + ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`, `"loongarch64"`. seqShallowFlag = low(int) strlitFlag = 1 shl (sizeof(int)*8 - 2) # later versions of the codegen \ @@ -1125,7 +1047,10 @@ const const hasThreadSupport = compileOption("threads") and not defined(nimscript) hasSharedHeap = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own - nimEnableCovariance* = defined(nimEnableCovariance) # or true + +when notJSnotNims and not defined(nimSeqsV2): + template space(s: PGenericSeq): int = + s.reserved and not (seqShallowFlag or strlitFlag) when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): # tcc doesn't support TLS @@ -1140,14 +1065,15 @@ when defined(boehmgc): elif defined(macosx): const boehmLib = "libgc.dylib" elif defined(openbsd): - const boehmLib = "libgc.so.4.0" + const boehmLib = "libgc.so.(4|5).0" elif defined(freebsd): const boehmLib = "libgc-threaded.so.1" else: const boehmLib = "libgc.so.1" {.pragma: boehmGC, noconv, dynlib: boehmLib.} -type TaintedString* {.deprecated: "Deprecated since 1.5".} = string +when not defined(nimPreviewSlimSystem): + type TaintedString* {.deprecated: "Deprecated since 1.5".} = string when defined(profiler) and not defined(nimscript): @@ -1171,6 +1097,8 @@ when not defined(js) and hostOS != "standalone": ## deprecated, prefer `quit` or `exitprocs.getProgramResult`, `exitprocs.setProgramResult`. import std/private/since +import system/ctypes +export ctypes proc align(address, alignment: int): int = if alignment == 0: # Actually, this is illegal. This branch exists to actively @@ -1179,49 +1107,9 @@ proc align(address, alignment: int): int = else: result = (address + (alignment - 1)) and not (alignment - 1) -when defined(nimdoc): - proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} - ## Stops the program immediately with an exit code. - ## - ## Before stopping the program the "exit procedures" are called in the - ## opposite order they were added with `addExitProc <exitprocs.html#addExitProc,proc)>`_. - ## `quit` never returns and ignores any exception that may have been raised - ## by the quit procedures. It does *not* call the garbage collector to free - ## all the memory, unless a quit procedure calls `GC_fullCollect - ## <#GC_fullCollect>`_. - ## - ## The proc `quit(QuitSuccess)` is called implicitly when your nim - ## program finishes without incident for platforms where this is the - ## expected behavior. A raised unhandled exception is - ## equivalent to calling `quit(QuitFailure)`. - ## - ## Note that this is a *runtime* call and using `quit` inside a macro won't - ## have any compile time effect. If you need to stop the compiler inside a - ## macro, use the `error <manual.html#pragmas-error-pragma>`_ or `fatal - ## <manual.html#pragmas-fatal-pragma>`_ pragmas. - -elif defined(genode): - include genode/env - - var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr - - type GenodeEnv* = GenodeEnvPtr - ## Opaque type representing Genode environment. - - proc quit*(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, - importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "<base/sleep.h>".} - - proc quit*(errorcode: int = QuitSuccess) = - systemEnv.quit(errorcode) - -elif defined(js) and defined(nodejs) and not defined(nimscript): - proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", - importc: "process.exit", noreturn.} - -else: - proc quit*(errorcode: int = QuitSuccess) {. - magic: "Exit", importc: "exit", header: "<stdlib.h>", noreturn.} - +include system/rawquits +when defined(genode): + export GenodeEnv template sysAssert(cond: bool, msg: string) = when defined(useSysAssert): @@ -1229,12 +1117,10 @@ template sysAssert(cond: bool, msg: string) = cstderr.rawWrite "[SYSASSERT] " cstderr.rawWrite msg cstderr.rawWrite "\n" - quit 1 + rawQuit 1 const hasAlloc = (hostOS != "standalone" or not defined(nogc)) and not defined(nimscript) -when notJSnotNims and hostOS != "standalone" and hostOS != "any": - include "system/cgprocs" when notJSnotNims and hasAlloc and not defined(nimSeqsV2): proc addChar(s: NimString, c: char): NimString {.compilerproc, benign.} @@ -1246,6 +1132,11 @@ when defined(nimscript) or not defined(nimSeqsV2): ## containers should also call their adding proc `add` for consistency. ## Generic code becomes much easier to write if the Nim naming scheme is ## respected. + ## ```nim + ## var s: seq[string] = @["test2","test2"] + ## s.add("test") + ## assert s == @["test2", "test2", "test"] + ## ``` when false: # defined(gcDestructors): proc add*[T](x: var seq[T], y: sink openArray[T]) {.noSideEffect.} = @@ -1255,13 +1146,13 @@ when false: # defined(gcDestructors): ## containers should also call their adding proc `add` for consistency. ## Generic code becomes much easier to write if the Nim naming scheme is ## respected. + ## ```nim + ## var s: seq[string] = @["test2","test2"] + ## s.add("test") # s <- @[test2, test2, test] + ## ``` ## ## See also: ## * `& proc <#&,seq[T],seq[T]>`_ - ## - ## .. code-block:: Nim - ## var s: seq[string] = @["test2","test2"] - ## s.add("test") # s <- @[test2, test2, test] {.noSideEffect.}: let xl = x.len setLen(x, xl + y.len) @@ -1283,10 +1174,14 @@ else: ## ## See also: ## * `& proc <#&,seq[T],seq[T]>`_ - ## - ## .. code-block:: Nim - ## var s: seq[string] = @["test2","test2"] - ## s.add("test") # s <- @[test2, test2, test] + runnableExamples: + var a = @["a1", "a2"] + a.add(["b1", "b2"]) + assert a == @["a1", "a2", "b1", "b2"] + var c = @["c0", "c1", "c2", "c3"] + a.add(c.toOpenArray(1, 2)) + assert a == @["a1", "a2", "b1", "b2", "c1", "c2"] + {.noSideEffect.}: let xl = x.len setLen(x, xl + y.len) @@ -1294,10 +1189,10 @@ else: when defined(nimSeqsV2): - template movingCopy(a, b) = + template movingCopy(a, b: typed) = a = move(b) else: - template movingCopy(a, b) = + template movingCopy(a, b: typed) = shallowCopy(a, b) proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} = @@ -1307,44 +1202,20 @@ proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} = ## ## See also: ## * `delete <#delete,seq[T],Natural>`_ for preserving the order - ## - ## .. code-block:: Nim - ## var i = @[1, 2, 3, 4, 5] - ## i.del(2) # => @[1, 2, 5, 4] + runnableExamples: + var a = @[10, 11, 12, 13, 14] + a.del(2) + assert a == @[10, 11, 14, 13] let xl = x.len - 1 movingCopy(x[i], x[xl]) setLen(x, xl) -proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = - ## Deletes the item at index `i` by moving all `x[i+1..]` items by one position. - ## - ## This is an `O(n)` operation. - ## - ## See also: - ## * `del <#delete,seq[T],Natural>`_ for O(1) operation - ## - ## .. code-block:: Nim - ## var i = @[1, 2, 3, 4, 5] - ## i.delete(2) # => @[1, 2, 4, 5] - template defaultImpl = - let xl = x.len - for j in i.int..xl-2: movingCopy(x[j], x[j+1]) - setLen(x, xl-1) - - when nimvm: - defaultImpl() - else: - when defined(js): - {.emit: "`x`.splice(`i`, 1);".} - else: - defaultImpl() - proc insert*[T](x: var seq[T], item: sink T, i = 0.Natural) {.noSideEffect.} = ## Inserts `item` into `x` at position `i`. - ## - ## .. code-block:: Nim - ## var i = @[1, 3, 5] - ## i.insert(99, 0) # i <- @[99, 1, 3, 5] + ## ```nim + ## var i = @[1, 3, 5] + ## i.insert(99, 0) # i <- @[99, 1, 3, 5] + ## ``` {.noSideEffect.}: template defaultImpl = let xl = x.len @@ -1372,163 +1243,17 @@ when not defined(nimV2): ## ## It works even for complex data graphs with cycles. This is a great ## debugging tool. - ## - ## .. code-block:: Nim - ## var s: seq[string] = @["test2", "test2"] - ## var i = @[1, 2, 3, 4, 5] - ## echo repr(s) # => 0x1055eb050[0x1055ec050"test2", 0x1055ec078"test2"] - ## echo repr(i) # => 0x1055ed050[1, 2, 3, 4, 5] + ## ```nim + ## var s: seq[string] = @["test2", "test2"] + ## var i = @[1, 2, 3, 4, 5] + ## echo repr(s) # => 0x1055eb050[0x1055ec050"test2", 0x1055ec078"test2"] + ## echo repr(i) # => 0x1055ed050[1, 2, 3, 4, 5] + ## ``` -type - ByteAddress* = int - ## is the signed integer type that should be used for converting - ## pointers to integer addresses for readability. - - BiggestFloat* = float64 - ## is an alias for the biggest floating point type the Nim - ## compiler supports. Currently this is `float64`, but it is - ## platform-dependent in general. - -when defined(js): - type BiggestUInt* = uint32 - ## is an alias for the biggest unsigned integer type the Nim compiler - ## supports. Currently this is `uint32` for JS and `uint64` for other - ## targets. -else: - type BiggestUInt* = uint64 - ## is an alias for the biggest unsigned integer type the Nim compiler - ## supports. Currently this is `uint32` for JS and `uint64` for other - ## targets. - -when defined(windows): +when not defined(nimPreviewSlimSystem): type - clong* {.importc: "long", nodecl.} = int32 - ## This is the same as the type `long` in *C*. - culong* {.importc: "unsigned long", nodecl.} = uint32 - ## This is the same as the type `unsigned long` in *C*. -else: - type - clong* {.importc: "long", nodecl.} = int - ## This is the same as the type `long` in *C*. - culong* {.importc: "unsigned long", nodecl.} = uint - ## This is the same as the type `unsigned long` in *C*. - -type # these work for most platforms: - cchar* {.importc: "char", nodecl.} = char - ## This is the same as the type `char` in *C*. - cschar* {.importc: "signed char", nodecl.} = int8 - ## This is the same as the type `signed char` in *C*. - cshort* {.importc: "short", nodecl.} = int16 - ## This is the same as the type `short` in *C*. - cint* {.importc: "int", nodecl.} = int32 - ## This is the same as the type `int` in *C*. - csize* {.importc: "size_t", nodecl, deprecated: "use `csize_t` instead".} = int - ## This isn't the same as `size_t` in *C*. Don't use it. - csize_t* {.importc: "size_t", nodecl.} = uint - ## This is the same as the type `size_t` in *C*. - clonglong* {.importc: "long long", nodecl.} = int64 - ## This is the same as the type `long long` in *C*. - cfloat* {.importc: "float", nodecl.} = float32 - ## This is the same as the type `float` in *C*. - cdouble* {.importc: "double", nodecl.} = float64 - ## This is the same as the type `double` in *C*. - clongdouble* {.importc: "long double", nodecl.} = BiggestFloat - ## This is the same as the type `long double` in *C*. - ## This C type is not supported by Nim's code generator. - - cuchar* {.importc: "unsigned char", nodecl.} = char - ## This is the same as the type `unsigned char` in *C*. - cushort* {.importc: "unsigned short", nodecl.} = uint16 - ## This is the same as the type `unsigned short` in *C*. - cuint* {.importc: "unsigned int", nodecl.} = uint32 - ## This is the same as the type `unsigned int` in *C*. - culonglong* {.importc: "unsigned long long", nodecl.} = uint64 - ## This is the same as the type `unsigned long long` in *C*. - - cstringArray* {.importc: "char**", nodecl.} = ptr UncheckedArray[cstring] - ## This is binary compatible to the type `char**` in *C*. The array's - ## high value is large enough to disable bounds checking in practice. - ## Use `cstringArrayToSeq proc <#cstringArrayToSeq,cstringArray,Natural>`_ - ## to convert it into a `seq[string]`. - - PFloat32* = ptr float32 ## An alias for `ptr float32`. - PFloat64* = ptr float64 ## An alias for `ptr float64`. - PInt64* = ptr int64 ## An alias for `ptr int64`. - PInt32* = ptr int32 ## An alias for `ptr int32`. - -proc toFloat*(i: int): float {.noSideEffect, inline.} = - ## Converts an integer `i` into a `float`. - ## - ## If the conversion fails, `ValueError` is raised. - ## However, on most platforms the conversion cannot fail. - ## - ## .. code-block:: Nim - ## let - ## a = 2 - ## b = 3.7 - ## - ## echo a.toFloat + b # => 5.7 - float(i) - -proc toBiggestFloat*(i: BiggestInt): BiggestFloat {.noSideEffect, inline.} = - ## Same as `toFloat <#toFloat,int>`_ but for `BiggestInt` to `BiggestFloat`. - BiggestFloat(i) - -proc toInt*(f: float): int {.noSideEffect.} = - ## Converts a floating point number `f` into an `int`. - ## - ## Conversion rounds `f` half away from 0, see - ## `Round half away from zero - ## <https://en.wikipedia.org/wiki/Rounding#Round_half_away_from_zero>`_. - ## - ## Note that some floating point numbers (e.g. infinity or even 1e19) - ## cannot be accurately converted. - ## - ## .. code-block:: Nim - ## doAssert toInt(0.49) == 0 - ## doAssert toInt(0.5) == 1 - ## doAssert toInt(-0.5) == -1 # rounding is symmetrical - if f >= 0: int(f+0.5) else: int(f-0.5) - -proc toBiggestInt*(f: BiggestFloat): BiggestInt {.noSideEffect.} = - ## Same as `toInt <#toInt,float>`_ but for `BiggestFloat` to `BiggestInt`. - if f >= 0: BiggestInt(f+0.5) else: BiggestInt(f-0.5) - -proc addQuitProc*(quitProc: proc() {.noconv.}) {. - importc: "atexit", header: "<stdlib.h>", deprecated: "use exitprocs.addExitProc".} - ## Adds/registers a quit procedure. - ## - ## Each call to `addQuitProc` registers another quit procedure. Up to 30 - ## procedures can be registered. They are executed on a last-in, first-out - ## basis (that is, the last function registered is the first to be executed). - ## `addQuitProc` raises an EOutOfIndex exception if `quitProc` cannot be - ## registered. - # Support for addQuitProc() is done by Ansi C's facilities here. - # In case of an unhandled exception the exit handlers should - # not be called explicitly! The user may decide to do this manually though. - -proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} - ## Swaps the values `a` and `b`. - ## - ## This is often more efficient than `tmp = a; a = b; b = tmp`. - ## Particularly useful for sorting algorithms. - ## - ## .. code-block:: Nim - ## var - ## a = 5 - ## b = 9 - ## - ## swap(a, b) - ## - ## assert a == 9 - ## assert b == 5 - -when not defined(js) and not defined(booting) and defined(nimTrMacros): - template swapRefsInArray*{swap(arr[a], arr[b])}(arr: openArray[ref], a, b: int) = - # Optimize swapping of array elements if they are refs. Default swap - # implementation will cause unsureAsgnRef to be emitted which causes - # unnecessary slow down in this case. - swap(cast[ptr pointer](addr arr[a])[], cast[ptr pointer](addr arr[b])[]) + csize* {.importc: "size_t", nodecl, deprecated: "use `csize_t` instead".} = int + ## This isn't the same as `size_t` in *C*. Don't use it. const Inf* = 0x7FF0000000000000'f64 @@ -1542,18 +1267,62 @@ const ## and expect a reasonable result - use the `isNaN` or `classify` procedure ## in the `math module <math.html>`_ for checking for NaN. +proc high*(T: typedesc[SomeFloat]): T = Inf +proc low*(T: typedesc[SomeFloat]): T = NegInf -include "system/memalloc" +proc toFloat*(i: int): float {.noSideEffect, inline.} = + ## Converts an integer `i` into a `float`. Same as `float(i)`. + ## + ## If the conversion fails, `ValueError` is raised. + ## However, on most platforms the conversion cannot fail. + ## + ## ```nim + ## let + ## a = 2 + ## b = 3.7 + ## + ## echo a.toFloat + b # => 5.7 + ## ``` + float(i) +proc toBiggestFloat*(i: BiggestInt): BiggestFloat {.noSideEffect, inline.} = + ## Same as `toFloat <#toFloat,int>`_ but for `BiggestInt` to `BiggestFloat`. + BiggestFloat(i) -proc `|`*(a, b: typedesc): typedesc = discard +proc toInt*(f: float): int {.noSideEffect.} = + ## Converts a floating point number `f` into an `int`. + ## + ## Conversion rounds `f` half away from 0, see + ## `Round half away from zero + ## <https://en.wikipedia.org/wiki/Rounding#Round_half_away_from_zero>`_, + ## as opposed to a type conversion which rounds towards zero. + ## + ## Note that some floating point numbers (e.g. infinity or even 1e19) + ## cannot be accurately converted. + ## ```nim + ## doAssert toInt(0.49) == 0 + ## doAssert toInt(0.5) == 1 + ## doAssert toInt(-0.5) == -1 # rounding is symmetrical + ## ``` + if f >= 0: int(f+0.5) else: int(f-0.5) -include "system/iterators_1" +proc toBiggestInt*(f: BiggestFloat): BiggestInt {.noSideEffect.} = + ## Same as `toInt <#toInt,float>`_ but for `BiggestFloat` to `BiggestInt`. + if f >= 0: BiggestInt(f+0.5) else: BiggestInt(f-0.5) +proc `/`*(x, y: int): float {.inline, noSideEffect.} = + ## Division of integers that results in a float. + ## ```nim + ## echo 7 / 5 # => 1.4 + ## ``` + ## + ## See also: + ## * `div <system.html#div,int,int>`_ + ## * `mod <system.html#mod,int,int>`_ + result = toFloat(x) / toFloat(y) {.push stackTrace: off.} - when defined(js): proc js_abs[T: SomeNumber](x: T): T {.importc: "Math.abs".} else: @@ -1573,62 +1342,114 @@ proc abs*[T: float64 | float32](x: T): T {.noSideEffect, inline.} = else: result = c_fabsf(x) -proc min*(x, y: float32): float32 {.noSideEffect, inline.} = - if x <= y or y != y: x else: y -proc min*(x, y: float64): float64 {.noSideEffect, inline.} = - if x <= y or y != y: x else: y -proc max*(x, y: float32): float32 {.noSideEffect, inline.} = - if y <= x or y != y: x else: y -proc max*(x, y: float64): float64 {.noSideEffect, inline.} = - if y <= x or y != y: x else: y -proc min*[T: not SomeFloat](x, y: T): T {.inline.} = - if x <= y: x else: y -proc max*[T: not SomeFloat](x, y: T): T {.inline.} = - if y <= x: x else: y +func abs*(x: int): int {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int8): int8 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int16): int16 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int32): int32 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int64): int64 {.magic: "AbsI", inline.} = + ## Returns the absolute value of `x`. + ## + ## If `x` is `low(x)` (that is -MININT for its type), + ## an overflow exception is thrown (if overflow checking is turned on). + result = if x < 0: -x else: x {.pop.} # stackTrace: off +when not defined(nimPreviewSlimSystem): + proc addQuitProc*(quitProc: proc() {.noconv.}) {. + importc: "atexit", header: "<stdlib.h>", deprecated: "use exitprocs.addExitProc".} + ## Adds/registers a quit procedure. + ## + ## Each call to `addQuitProc` registers another quit procedure. Up to 30 + ## procedures can be registered. They are executed on a last-in, first-out + ## basis (that is, the last function registered is the first to be executed). + ## `addQuitProc` raises an EOutOfIndex exception if `quitProc` cannot be + ## registered. + # Support for addQuitProc() is done by Ansi C's facilities here. + # In case of an unhandled exception the exit handlers should + # not be called explicitly! The user may decide to do this manually though. + +proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} + ## Swaps the values `a` and `b`. + ## + ## This is often more efficient than `tmp = a; a = b; b = tmp`. + ## Particularly useful for sorting algorithms. + ## + ## ```nim + ## var + ## a = 5 + ## b = 9 + ## + ## swap(a, b) + ## + ## assert a == 9 + ## assert b == 5 + ## ``` + +when not defined(js) and not defined(booting) and defined(nimTrMacros): + template swapRefsInArray*{swap(arr[a], arr[b])}(arr: openArray[ref], a, b: int) = + # Optimize swapping of array elements if they are refs. Default swap + # implementation will cause unsureAsgnRef to be emitted which causes + # unnecessary slow down in this case. + swap(cast[ptr pointer](addr arr[a])[], cast[ptr pointer](addr arr[b])[]) + +when not defined(nimscript): + {.push stackTrace: off, profiler: off.} + + when not defined(nimPreviewSlimSystem): + import std/sysatomics + export sysatomics + else: + import std/sysatomics + + {.pop.} + +include "system/memalloc" + + +proc `|`*(a, b: typedesc): typedesc = discard + +include "system/iterators_1" -proc high*(T: typedesc[SomeFloat]): T = Inf -proc low*(T: typedesc[SomeFloat]): T = NegInf proc len*[U: Ordinal; V: Ordinal](x: HSlice[U, V]): int {.noSideEffect, inline.} = ## Length of ordinal slice. When x.b < x.a returns zero length. - ## - ## .. code-block:: Nim + ## ```nim ## assert((0..5).len == 6) ## assert((5..2).len == 0) + ## ``` result = max(0, ord(x.b) - ord(x.a) + 1) -when true: # PRTEMP: remove? - proc isNil*[T](x: seq[T]): bool {.noSideEffect, magic: "IsNil", error.} - ## Seqs are no longer nil by default, but set and empty. - ## Check for zero length instead. - ## - ## See also: - ## * `isNil(string) <#isNil,string>`_ - - proc isNil*(x: string): bool {.noSideEffect, magic: "IsNil", error.} - ## See also: - ## * `isNil(seq[T]) <#isNil,seq[T]>`_ - proc isNil*[T](x: ref T): bool {.noSideEffect, magic: "IsNil".} proc isNil*[T](x: ptr T): bool {.noSideEffect, magic: "IsNil".} proc isNil*(x: pointer): bool {.noSideEffect, magic: "IsNil".} proc isNil*(x: cstring): bool {.noSideEffect, magic: "IsNil".} -proc isNil*[T: proc](x: T): bool {.noSideEffect, magic: "IsNil".} +proc isNil*[T: proc | iterator {.closure.}](x: T): bool {.noSideEffect, magic: "IsNil".} ## Fast check whether `x` is nil. This is sometimes more efficient than ## `== nil`. - -proc `@`*[T](a: openArray[T]): seq[T] = - ## Turns an *openArray* into a sequence. - ## - ## This is not as efficient as turning a fixed length array into a sequence - ## as it always copies every element of `a`. - newSeq(result, a.len) - for i in 0..a.len-1: result[i] = a[i] +when defined(nimHasTopDownInference): + # magic used for seq type inference + proc `@`*[T](a: openArray[T]): seq[T] {.magic: "OpenArrayToSeq".} = + ## Turns an *openArray* into a sequence. + ## + ## This is not as efficient as turning a fixed length array into a sequence + ## as it always copies every element of `a`. + newSeq(result, a.len) + for i in 0..a.len-1: result[i] = a[i] +else: + proc `@`*[T](a: openArray[T]): seq[T] = + ## Turns an *openArray* into a sequence. + ## + ## This is not as efficient as turning a fixed length array into a sequence + ## as it always copies every element of `a`. + newSeq(result, a.len) + for i in 0..a.len-1: result[i] = a[i] when defined(nimSeqsV2): @@ -1637,12 +1458,12 @@ when defined(nimSeqsV2): ## Concatenates two sequences. ## ## Requires copying of the sequences. + ## ```nim + ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) + ## ``` ## ## See also: ## * `add(var seq[T], openArray[T]) <#add,seq[T],openArray[T]>`_ - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) newSeq(result, x.len + y.len) for i in 0..x.len-1: result[i] = move(x[i]) @@ -1653,12 +1474,12 @@ when defined(nimSeqsV2): ## Appends element y to the end of the sequence. ## ## Requires copying of the sequence. + ## ```nim + ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) + ## ``` ## ## See also: ## * `add(var seq[T], T) <#add,seq[T],sinkT>`_ - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) newSeq(result, x.len + 1) for i in 0..x.len-1: result[i] = move(x[i]) @@ -1668,9 +1489,9 @@ when defined(nimSeqsV2): ## Prepends the element x to the beginning of the sequence. ## ## Requires copying of the sequence. - ## - ## .. code-block:: Nim + ## ```nim ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) + ## ``` newSeq(result, y.len + 1) result[0] = move(x) for i in 0..y.len-1: @@ -1682,12 +1503,12 @@ else: ## Concatenates two sequences. ## ## Requires copying of the sequences. + ## ```nim + ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) + ## ``` ## ## See also: ## * `add(var seq[T], openArray[T]) <#add,seq[T],openArray[T]>`_ - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) newSeq(result, x.len + y.len) for i in 0..x.len-1: result[i] = x[i] @@ -1698,12 +1519,12 @@ else: ## Appends element y to the end of the sequence. ## ## Requires copying of the sequence. + ## ```nim + ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) + ## ``` ## ## See also: ## * `add(var seq[T], T) <#add,seq[T],sinkT>`_ - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) newSeq(result, x.len + 1) for i in 0..x.len-1: result[i] = x[i] @@ -1713,19 +1534,15 @@ else: ## Prepends the element x to the beginning of the sequence. ## ## Requires copying of the sequence. - ## - ## .. code-block:: Nim + ## ```nim ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) + ## ``` newSeq(result, y.len + 1) result[0] = x for i in 0..y.len-1: result[i+1] = y[i] -proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} - ## Converts the AST of `x` into a string representation. This is very useful - ## for debugging. - proc instantiationInfo*(index = -1, fullPaths = false): tuple[ filename: string, line: int, column: int] {.magic: "InstantiationInfo", noSideEffect.} ## Provides access to the compiler's instantiation stack line information @@ -1738,7 +1555,7 @@ proc instantiationInfo*(index = -1, fullPaths = false): tuple[ ## to retrieve information about the current filename and line number. ## Example: ## - ## .. code-block:: nim + ## ```nim ## import std/strutils ## ## template testException(exception, code: untyped): typed = @@ -1760,16 +1577,8 @@ proc instantiationInfo*(index = -1, fullPaths = false): tuple[ ## testException(IndexDefect, tester(30)) ## testException(IndexDefect, tester(1)) ## # --> Test failure at example.nim:20 with 'tester(1)' + ## ``` -proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.} = - ## Special compile-time procedure that checks whether `x` can be compiled - ## without any semantic error. - ## This can be used to check whether a type supports some operation: - ## - ## .. code-block:: Nim - ## when compiles(3 + 4): - ## echo "'+' for integers is available" - discard when notJSnotNims: import system/ansi_c @@ -1779,8 +1588,7 @@ when notJSnotNims: {.push stackTrace: off.} when not defined(js) and hasThreadSupport and hostOS != "standalone": - const insideRLocksModule = false - include "system/syslocks" + import std/private/syslocks include "system/threadlocalstorage" when not defined(js) and defined(nimV2): @@ -1789,17 +1597,100 @@ when not defined(js) and defined(nimV2): TNimTypeV2 {.compilerproc.} = object destructor: pointer size: int - align: int - name: cstring + align: int16 + depth: int16 + display: ptr UncheckedArray[uint32] # classToken + when defined(nimTypeNames) or defined(nimArcIds): + name: cstring traceImpl: pointer - disposeImpl: pointer typeInfoV1: pointer # for backwards compat, usually nil + flags: int + when defined(gcDestructors): + when defined(cpp): + vTable: ptr UncheckedArray[pointer] # vtable for types + else: + vTable: UncheckedArray[pointer] # vtable for types PNimTypeV2 = ptr TNimTypeV2 +proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} + when notJSnotNims and defined(nimSeqsV2): include "system/strs_v2" include "system/seqs_v2" +when not defined(js): + template newSeqImpl(T, len) = + result = newSeqOfCap[T](len) + {.cast(noSideEffect).}: + when defined(nimSeqsV2): + cast[ptr int](addr result)[] = len + else: + var s = cast[PGenericSeq](result) + s.len = len + + proc newSeqUninitialized*[T: SomeNumber](len: Natural): seq[T] {.deprecated: "Use `newSeqUninit` instead".} = + ## Creates a new sequence of type `seq[T]` with length `len`. + ## + ## Only available for numbers types. Note that the sequence will be + ## uninitialized. After the creation of the sequence you should assign + ## entries to the sequence instead of adding them. + ## Example: + ## ```nim + ## var x = newSeqUninitialized[int](3) + ## assert len(x) == 3 + ## x[0] = 10 + ## ``` + result = newSeqOfCap[T](len) + when defined(nimSeqsV2): + cast[ptr int](addr result)[] = len + else: + var s = cast[PGenericSeq](result) + s.len = len + + func newSeqUninit*[T](len: Natural): seq[T] = + ## Creates a new sequence of type `seq[T]` with length `len`. + ## + ## Only available for types, which don't contain + ## managed memory or have destructors. + ## Note that the sequence will be uninitialized. + ## After the creation of the sequence you should assign + ## entries to the sequence instead of adding them. + runnableExamples: + var x = newSeqUninit[int](3) + assert len(x) == 3 + x[0] = 10 + when supportsCopyMem(T): + when nimvm: + result = newSeq[T](len) + else: + newSeqImpl(T, len) + else: + {.error: "The type T cannot contain managed memory or have destructors".} + + proc newStringUninit*(len: Natural): string = + ## Returns a new string of length `len` but with uninitialized + ## content. One needs to fill the string character after character + ## with the index operator `s[i]`. + ## + ## This procedure exists only for optimization purposes; + ## the same effect can be achieved with the `&` operator or with `add`. + when nimvm: + result = newString(len) + else: + result = newStringOfCap(len) + when defined(nimSeqsV2): + let s = cast[ptr NimStringV2](addr result) + if len > 0: + s.len = len + s.p.data[len] = '\0' + else: + let s = cast[NimString](result) + s.len = len + s.data[len] = '\0' +else: + proc newStringUninit*(len: Natural): string {. + magic: "NewString", importc: "mnewString", noSideEffect.} + {.pop.} when not defined(nimscript): @@ -1811,27 +1702,35 @@ when not defined(nimscript): when not declared(sysFatal): include "system/fatal" -when not defined(nimscript): - {.push stackTrace: off, profiler: off.} - - proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, - discardable, benign.} - ## Atomic increment of `memLoc`. Returns the value after the operation. - - proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, - discardable, benign.} - ## Atomic decrement of `memLoc`. Returns the value after the operation. - - include "system/atomics" - - {.pop.} - +type + PFrame* = ptr TFrame ## Represents a runtime frame of the call stack; + ## part of the debugger API. + # keep in sync with nimbase.h `struct TFrame_` + TFrame* {.importc, nodecl, final.} = object ## The frame itself. + prev*: PFrame ## Previous frame; used for chaining the call stack. + procname*: cstring ## Name of the proc that is currently executing. + line*: int ## Line number of the proc that is currently executing. + filename*: cstring ## Filename of the proc that is currently executing. + len*: int16 ## Length of the inspectable slots. + calldepth*: int16 ## Used for max call depth checking. + when NimStackTraceMsgs: + frameMsgLen*: int ## end position in frameMsgBuf for this frame. when defined(nimV2): + var + framePtr {.threadvar.}: PFrame + include system/arc -import system/assertions -export assertions +template newException*(exceptn: typedesc, message: string; + parentException: ref Exception = nil): untyped = + ## Creates an exception object of type `exceptn` and sets its `msg` field + ## to `message`. Returns the new exception object. + (ref exceptn)(msg: message, parent: parentException) + +when not defined(nimPreviewSlimSystem): + import std/assertions + export assertions import system/iterators export iterators @@ -1852,17 +1751,19 @@ proc contains*[T](a: openArray[T], item: T): bool {.inline.}= ## ## This allows the `in` operator: `a.contains(item)` is the same as ## `item in a`. - ## - ## .. code-block:: Nim + ## ```nim ## var a = @[1, 3, 5] ## assert a.contains(5) ## assert 3 in a ## assert 99 notin a + ## ``` return find(a, item) >= 0 proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = ## Returns the last item of `s` and decreases `s.len` by one. This treats ## `s` as a stack and implements the common *pop* operation. + ## + ## Raises `IndexDefect` if `s` is empty. runnableExamples: var a = @[1, 3, 5, 7] let b = pop(a) @@ -1908,24 +1809,7 @@ include "system/gc_interface" # we have to compute this here before turning it off in except.nim anyway ... const NimStackTrace = compileOption("stacktrace") -template coroutinesSupportedPlatform(): bool = - when defined(sparc) or defined(ELATE) or defined(boehmgc) or defined(gogc) or - defined(nogc) or defined(gcRegions) or defined(gcMarkAndSweep): - false - else: - true - -when defined(nimCoroutines): - # Explicit opt-in. - when not coroutinesSupportedPlatform(): - {.error: "Coroutines are not supported on this architecture and/or garbage collector.".} - const nimCoroutines* = true -elif defined(noNimCoroutines): - # Explicit opt-out. - const nimCoroutines* = false -else: - # Autodetect coroutine support. - const nimCoroutines* = false +import system/coro_detection {.push checks: off.} # obviously we cannot generate checking operations here :-) @@ -1961,8 +1845,7 @@ when notJSnotNims: ## ## `outOfMemHook` can be used to raise an exception in case of OOM like so: ## - ## .. code-block:: Nim - ## + ## ```nim ## var gOutOfMem: ref EOutOfMemory ## new(gOutOfMem) # need to be allocated *before* OOM really happened! ## gOutOfMem.msg = "out of memory" @@ -1971,6 +1854,7 @@ when notJSnotNims: ## raise gOutOfMem ## ## system.outOfMemHook = handleOOM + ## ``` ## ## If the handler does not raise an exception, ordinary control flow ## continues and the program is terminated. @@ -1980,20 +1864,6 @@ when notJSnotNims: ## writes an error message and terminates the program, except when ## using `--os:any` -type - PFrame* = ptr TFrame ## Represents a runtime frame of the call stack; - ## part of the debugger API. - # keep in sync with nimbase.h `struct TFrame_` - TFrame* {.importc, nodecl, final.} = object ## The frame itself. - prev*: PFrame ## Previous frame; used for chaining the call stack. - procname*: cstring ## Name of the proc that is currently executing. - line*: int ## Line number of the proc that is currently executing. - filename*: cstring ## Filename of the proc that is currently executing. - len*: int16 ## Length of the inspectable slots. - calldepth*: int16 ## Used for max call depth checking. - when NimStackTraceMsgs: - frameMsgLen*: int ## end position in frameMsgBuf for this frame. - when defined(js) or defined(nimdoc): proc add*(x: var string, y: cstring) {.asmNoStackFrame.} = ## Appends `y` to `x` in place. @@ -2002,14 +1872,14 @@ when defined(js) or defined(nimdoc): tmp.add(cstring("ab")) tmp.add(cstring("cd")) doAssert tmp == "abcd" - asm """ + {.emit: """ if (`x` === null) { `x` = []; } var off = `x`.length; `x`.length += `y`.length; for (var i = 0; i < `y`.length; ++i) { `x`[off+i] = `y`.charCodeAt(i); } - """ + """.} proc add*(x: var cstring, y: cstring) {.magic: "AppendStrStr".} = ## Appends `y` to `x` in place. ## Only implemented for JS backend. @@ -2030,8 +1900,7 @@ elif hasAlloc: inc(i) {.pop.} -proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", tags: [WriteIOEffect], - benign, sideEffect.} +proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", benign, sideEffect.} ## Writes and flushes the parameters to the standard output. ## ## Special built-in that takes a variable number of arguments. Each argument @@ -2053,16 +1922,10 @@ proc debugEcho*(x: varargs[typed, `$`]) {.magic: "Echo", noSideEffect, ## for debugging routines marked as `noSideEffect ## <manual.html#pragmas-nosideeffect-pragma>`_. -template newException*(exceptn: typedesc, message: string; - parentException: ref Exception = nil): untyped = - ## Creates an exception object of type `exceptn` and sets its `msg` field - ## to `message`. Returns the new exception object. - (ref exceptn)(msg: message, parent: parentException) - when hostOS == "standalone" and defined(nogc): proc nimToCStringConv(s: NimString): cstring {.compilerproc, inline.} = if s == nil or s.len == 0: result = cstring"" - else: result = cstring(addr s.data) + else: result = cast[cstring](addr s.data) proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.} ## Get type information for `x`. @@ -2070,22 +1933,6 @@ proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.} ## Ordinary code should not use this, but the `typeinfo module ## <typeinfo.html>`_ instead. -{.push stackTrace: off.} -func abs*(x: int): int {.magic: "AbsI", inline.} = - if x < 0: -x else: x -func abs*(x: int8): int8 {.magic: "AbsI", inline.} = - if x < 0: -x else: x -func abs*(x: int16): int16 {.magic: "AbsI", inline.} = - if x < 0: -x else: x -func abs*(x: int32): int32 {.magic: "AbsI", inline.} = - if x < 0: -x else: x -func abs*(x: int64): int64 {.magic: "AbsI", inline.} = - ## Returns the absolute value of `x`. - ## - ## If `x` is `low(x)` (that is -MININT for its type), - ## an overflow exception is thrown (if overflow checking is turned on). - result = if x < 0: -x else: x -{.pop.} when not defined(js): @@ -2098,13 +1945,13 @@ template likely*(val: bool): bool = ## You can use this template to decorate a branch condition. On certain ## platforms this can help the processor predict better which branch is ## going to be run. Example: - ## - ## .. code-block:: Nim + ## ```nim ## for value in inputValues: ## if likely(value <= 100): ## process(value) ## else: ## echo "Value too big!" + ## ``` ## ## On backends without branch prediction (JS and the nimscript VM), this ## template will not affect code execution. @@ -2122,13 +1969,13 @@ template unlikely*(val: bool): bool = ## You can use this proc to decorate a branch condition. On certain ## platforms this can help the processor predict better which branch is ## going to be run. Example: - ## - ## .. code-block:: Nim + ## ```nim ## for value in inputValues: ## if unlikely(value > 100): ## echo "Value too big!" ## else: ## process(value) + ## ``` ## ## On backends without branch prediction (JS and the nimscript VM), this ## template will not affect code execution. @@ -2140,39 +1987,50 @@ template unlikely*(val: bool): bool = else: unlikelyProc(val) -const - NimMajor* {.intdefine.}: int = 1 - ## is the major number of Nim's version. Example: - ## - ## .. code-block:: Nim - ## when (NimMajor, NimMinor, NimPatch) >= (1, 3, 1): discard - # see also std/private/since - - NimMinor* {.intdefine.}: int = 5 - ## is the minor number of Nim's version. - ## Odd for devel, even for releases. - - NimPatch* {.intdefine.}: int = 1 - ## is the patch number of Nim's version. - ## Odd for devel, even for releases. - import system/dollars export dollars +when defined(nimAuditDelete): + {.pragma: auditDelete, deprecated: "review this call for out of bounds behavior".} +else: + {.pragma: auditDelete.} + +proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect, systemRaisesDefect, auditDelete.} = + ## Deletes the item at index `i` by moving all `x[i+1..^1]` items by one position. + ## + ## This is an `O(n)` operation. + ## + ## See also: + ## * `del <#del,seq[T],Natural>`_ for O(1) operation + ## + runnableExamples: + var s = @[1, 2, 3, 4, 5] + s.delete(2) + doAssert s == @[1, 2, 4, 5] + + when not defined(nimAuditDelete): + if i > high(x): + # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring + raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed") + + template defaultImpl = + let xl = x.len + for j in i.int..xl-2: movingCopy(x[j], x[j+1]) + setLen(x, xl-1) + + when nimvm: + defaultImpl() + else: + when defined(js): + {.emit: "`x`.splice(`i`, 1);".} + else: + defaultImpl() + + const NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch ## is the version of Nim as a string. - -type - FileSeekPos* = enum ## Position relative to which seek should happen. - # The values are ordered so that they match with stdio - # SEEK_SET, SEEK_CUR and SEEK_END respectively. - fspSet ## Seek to absolute value - fspCur ## Seek relative to current position - fspEnd ## Seek relative to end - - when not defined(js): {.push stackTrace: off, profiler: off.} @@ -2225,7 +2083,7 @@ when notJSnotNims: proc equalMem(a, b: pointer, size: Natural): bool = nimCmpMem(a, b, size) == 0 proc cmpMem(a, b: pointer, size: Natural): int = - nimCmpMem(a, b, size) + nimCmpMem(a, b, size).int when not defined(js): proc cmp(x, y: string): int = @@ -2274,7 +2132,7 @@ when not defined(js) and declared(alloc0) and declared(dealloc): inc(i) dealloc(a) -when notJSnotNims: +when notJSnotNims and not gotoBasedExceptions: type PSafePoint = ptr TSafePoint TSafePoint {.compilerproc, final.} = object @@ -2287,7 +2145,12 @@ when not defined(js): when declared(initAllocator): initAllocator() when hasThreadSupport: - when hostOS != "standalone": include "system/threads" + when hostOS != "standalone": + include system/threadimpl + when not defined(nimPreviewSlimSystem): + import std/typedthreads + export typedthreads + elif not defined(nogc) and not defined(nimscript): when not defined(useNimRtl) and not defined(createNimRtl): initStackBottom() when declared(initGC): initGC() @@ -2296,6 +2159,16 @@ when notJSnotNims: proc setControlCHook*(hook: proc () {.noconv.}) ## Allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. + ## Example: + ## + ## ```nim + ## proc ctrlc() {.noconv.} = + ## echo "Ctrl+C fired!" + ## # do clean up stuff + ## quit() + ## + ## setControlCHook(ctrlc) + ## ``` when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook*() @@ -2321,10 +2194,7 @@ when notJSnotNims: # we cannot compile this with stack tracing on # as it would recurse endlessly! - when defined(nimNewIntegerOps): - include "system/integerops" - else: - include "system/arithm" + include "system/integerops" {.pop.} @@ -2337,6 +2207,16 @@ when not defined(js): when notJSnotNims: when hostOS != "standalone" and hostOS != "any": + type + LibHandle = pointer # private type + ProcAddr = pointer # library loading and loading of procs: + + proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} + proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} + + proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} + include "system/dyncalls" import system/countbits_impl @@ -2386,7 +2266,8 @@ when notJSnotNims and hasAlloc: include "system/repr" when notJSnotNims and hasThreadSupport and hostOS != "standalone": - include "system/channels_builtin" + when not defined(nimPreviewSlimSystem): + include "system/channels_builtin" when notJSnotNims and hostOS != "standalone": @@ -2418,45 +2299,125 @@ when notJSnotNims: include "system/profiler" {.pop.} - proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = + proc rawProc*[T: proc {.closure.} | iterator {.closure.}](x: T): pointer {.noSideEffect, inline.} = ## Retrieves the raw proc pointer of the closure `x`. This is - ## useful for interfacing closures with C. - when T is "closure": - {.emit: """ - `result` = `x`.ClP_0; - """.} - else: - {.error: "Only closure function and iterator are allowed!".} + ## useful for interfacing closures with C/C++, hash computations, etc. + ## If `rawEnv(x)` returns `nil`, the proc which the result points to + ## takes as many parameters as `x`, but with `{.nimcall.}` as its calling + ## convention instead of `{.closure.}`, otherwise it takes one more parameter + ## which is a `pointer`, and it still has `{.nimcall.}` as its calling convention. + ## To invoke the resulted proc, what this returns has to be casted into a `proc`, + ## not a `ptr proc`, and, in a case where `rawEnv(x)` returns non-`nil`, + ## the last and additional argument has to be the result of `rawEnv(x)`. + ## This is not available for the JS target. + #[ + The conversion from function pointer to `void*` is a tricky topic, but this + should work at least for c++ >= c++11, e.g. for `dlsym` support. + refs: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57869, + https://stackoverflow.com/questions/14125474/casts-between-pointer-to-function-and-pointer-to-object-in-c-and-c + ]# + runnableExamples: + proc makeClosure(x: int): (proc(y: int): int) = + var n = x + result = ( + proc(y: int): int = + n += y + return n + ) - proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} = - ## Retrieves the raw environment pointer of the closure `x`. This is - ## useful for interfacing closures with C. - when T is "closure": - {.emit: """ - `result` = `x`.ClE_0; - """.} - else: - {.error: "Only closure function and iterator are allowed!".} + var + c1 = makeClosure(10) + e = c1.rawEnv() + p = c1.rawProc() - proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} = + if e.isNil(): + let c2 = cast[proc(y: int): int {.nimcall.}](p) + echo c2(2) + else: + let c3 = cast[proc(y: int; env: pointer): int {.nimcall.}](p) + echo c3(3, e) + + {.emit: """ + `result` = (void*)`x`.ClP_0; + """.} + + proc rawEnv*[T: proc {.closure.} | iterator {.closure.}](x: T): pointer {.noSideEffect, inline.} = + ## Retrieves the raw environment pointer of the closure `x`. See also `rawProc`. + ## This is not available for the JS target. + {.emit: """ + `result` = `x`.ClE_0; + """.} + + proc finished*[T: iterator {.closure.}](x: T): bool {.noSideEffect, inline, magic: "Finished".} = ## It can be used to determine if a first class iterator has finished. - when T is "iterator": - {.emit: """ - `result` = ((NI*) `x`.ClE_0)[1] < 0; - """.} - else: - {.error: "Only closure iterator is allowed!".} + {.emit: """ + `result` = ((NI*) `x`.ClE_0)[1] < 0; + """.} + +from std/private/digitsutils import addInt +export addInt when defined(js): include "system/jssys" include "system/reprjs" -when defined(js) or defined(nimscript): - proc addInt*(result: var string; x: int64) = - result.add $x - proc addFloat*(result: var string; x: float) = - result.add $x +when defined(nimNoQuit): + proc quit*(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(nimdoc): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} + ## Stops the program immediately with an exit code. + ## + ## Before stopping the program the "exit procedures" are called in the + ## opposite order they were added with `addExitProc <exitprocs.html#addExitProc,proc)>`_. + ## + ## The proc `quit(QuitSuccess)` is called implicitly when your nim + ## program finishes without incident for platforms where this is the + ## expected behavior. A raised unhandled exception is + ## equivalent to calling `quit(QuitFailure)`. + ## + ## Note that this is a *runtime* call and using `quit` inside a macro won't + ## have any compile time effect. If you need to stop the compiler inside a + ## macro, use the `error <manual.html#pragmas-error-pragma>`_ or `fatal + ## <manual.html#pragmas-fatal-pragma>`_ pragmas. + ## + ## .. warning:: `errorcode` gets saturated when it exceeds the valid range + ## on the specific platform. On Posix, the valid range is `low(int8)..high(int8)`. + ## On Windows, the valid range is `low(int32)..high(int32)`. For instance, + ## `quit(int(0x100000000))` is equal to `quit(127)` on Linux. + ## + ## .. danger:: In almost all cases, in particular in library code, prefer + ## alternatives, e.g. `raiseAssert` or raise a `Defect`. + ## `quit` bypasses regular control flow in particular `defer`, + ## `try`, `catch`, `finally` and `destructors`, and exceptions that may have been + ## raised by an `addExitProc` proc, as well as cleanup code in other threads. + ## It does *not* call the garbage collector to free all the memory, + ## unless an `addExitProc` proc calls `GC_fullCollect <#GC_fullCollect>`_. + +elif defined(genode): + proc quit*(errorcode: int = QuitSuccess) {.inline, noreturn.} = + rawQuit(errorcode) + +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc quit*(errorcode: int = QuitSuccess) {.inline, noreturn.} = + when defined(posix): # posix uses low 8 bits + type ExitCodeRange = int8 + else: # win32 uses low 32 bits + type ExitCodeRange = cint + when sizeof(errorcode) > sizeof(ExitCodeRange): + if errorcode < low(ExitCodeRange): + rawQuit(low(ExitCodeRange).cint) + elif errorcode > high(ExitCodeRange): + rawQuit(high(ExitCodeRange).cint) + else: + rawQuit(errorcode.cint) + else: + rawQuit(errorcode.cint) proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} = ## A shorthand for `echo(errormsg); quit(errorcode)`. @@ -2471,281 +2432,22 @@ proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} = quit(errorcode) {.pop.} # checks: off -{.pop.} # hints: off +# {.pop.} # hints: off -proc `/`*(x, y: int): float {.inline, noSideEffect.} = - ## Division of integers that results in a float. - ## - ## See also: - ## * `div <#div,int,int>`_ - ## * `mod <#mod,int,int>`_ - ## - ## .. code-block:: Nim - ## echo 7 / 5 # => 1.4 - result = toFloat(x) / toFloat(y) - -type - BackwardsIndex* = distinct int ## Type that is constructed by `^` for - ## reversed array accesses. - ## (See `^ template <#^.t,int>`_) - -template `^`*(x: int): BackwardsIndex = BackwardsIndex(x) - ## Builtin `roof`:idx: operator that can be used for convenient array access. - ## `a[^x]` is a shortcut for `a[a.len-x]`. - ## - ## .. code-block:: Nim - ## let - ## a = [1, 3, 5, 7, 9] - ## b = "abcdefgh" - ## - ## echo a[^1] # => 9 - ## echo b[^2] # => g - -template `..^`*(a, b: untyped): untyped = - ## A shortcut for `.. ^` to avoid the common gotcha that a space between - ## '..' and '^' is required. - a .. ^b - -template `..<`*(a, b: untyped): untyped = - ## A shortcut for `a .. pred(b)`. - ## - ## .. code-block:: Nim - ## for i in 5 ..< 9: - ## echo i # => 5; 6; 7; 8 - a .. (when b is BackwardsIndex: succ(b) else: pred(b)) - -template spliceImpl(s, a, L, b: untyped): untyped = - # make room for additional elements or cut: - var shift = b.len - max(0,L) # ignore negative slice size - var newLen = s.len + shift - if shift > 0: - # enlarge: - setLen(s, newLen) - for i in countdown(newLen-1, a+b.len): movingCopy(s[i], s[i-shift]) - else: - for i in countup(a+b.len, newLen-1): movingCopy(s[i], s[i-shift]) - # cut down: - setLen(s, newLen) - # fill the hole: - for i in 0 ..< b.len: s[a+i] = b[i] - -template `^^`(s, i: untyped): untyped = - (when i is BackwardsIndex: s.len - int(i) else: int(i)) - -template `[]`*(s: string; i: int): char = arrGet(s, i) -template `[]=`*(s: string; i: int; val: char) = arrPut(s, i, val) - -proc `[]`*[T, U: Ordinal](s: string, x: HSlice[T, U]): string {.inline.} = - ## Slice operation for strings. - ## Returns the inclusive range `[s[x.a], s[x.b]]`: - ## - ## .. code-block:: Nim - ## var s = "abcdef" - ## assert s[1..3] == "bcd" - let a = s ^^ x.a - let L = (s ^^ x.b) - a + 1 - result = newString(L) - for i in 0 ..< L: result[i] = s[i + a] - -proc `[]=`*[T, U: Ordinal](s: var string, x: HSlice[T, U], b: string) = - ## Slice assignment for strings. - ## - ## If `b.len` is not exactly the number of elements that are referred to - ## by `x`, a `splice`:idx: is performed: - ## - runnableExamples: - var s = "abcdefgh" - s[1 .. ^2] = "xyz" - assert s == "axyzh" - - var a = s ^^ x.a - var L = (s ^^ x.b) - a + 1 - if L == b.len: - for i in 0..<L: s[i+a] = b[i] - else: - spliceImpl(s, a, L, b) - -proc `[]`*[Idx, T; U, V: Ordinal](a: array[Idx, T], x: HSlice[U, V]): seq[T] = - ## Slice operation for arrays. - ## Returns the inclusive range `[a[x.a], a[x.b]]`: - ## - ## .. code-block:: Nim - ## var a = [1, 2, 3, 4] - ## assert a[0..2] == @[1, 2, 3] - let xa = a ^^ x.a - let L = (a ^^ x.b) - xa + 1 - result = newSeq[T](L) - for i in 0..<L: result[i] = a[Idx(i + xa)] - -proc `[]=`*[Idx, T; U, V: Ordinal](a: var array[Idx, T], x: HSlice[U, V], b: openArray[T]) = - ## Slice assignment for arrays. - ## - ## .. code-block:: Nim - ## var a = [10, 20, 30, 40, 50] - ## a[1..2] = @[99, 88] - ## assert a == [10, 99, 88, 40, 50] - let xa = a ^^ x.a - let L = (a ^^ x.b) - xa + 1 - if L == b.len: - for i in 0..<L: a[Idx(i + xa)] = b[i] - else: - sysFatal(RangeDefect, "different lengths for slice assignment") - -proc `[]`*[T; U, V: Ordinal](s: openArray[T], x: HSlice[U, V]): seq[T] = - ## Slice operation for sequences. - ## Returns the inclusive range `[s[x.a], s[x.b]]`: - ## - ## .. code-block:: Nim - ## var s = @[1, 2, 3, 4] - ## assert s[0..2] == @[1, 2, 3] - let a = s ^^ x.a - let L = (s ^^ x.b) - a + 1 - newSeq(result, L) - for i in 0 ..< L: result[i] = s[i + a] - -proc `[]=`*[T; U, V: Ordinal](s: var seq[T], x: HSlice[U, V], b: openArray[T]) = - ## Slice assignment for sequences. - ## - ## If `b.len` is not exactly the number of elements that are referred to - ## by `x`, a `splice`:idx: is performed. - runnableExamples: - var s = @"abcdefgh" - s[1 .. ^2] = @"xyz" - assert s == @"axyzh" - - let a = s ^^ x.a - let L = (s ^^ x.b) - a + 1 - if L == b.len: - for i in 0 ..< L: s[i+a] = b[i] - else: - spliceImpl(s, a, L, b) - -proc `[]`*[T](s: openArray[T]; i: BackwardsIndex): T {.inline.} = - system.`[]`(s, s.len - int(i)) - -proc `[]`*[Idx, T](a: array[Idx, T]; i: BackwardsIndex): T {.inline.} = - a[Idx(a.len - int(i) + int low(a))] -proc `[]`*(s: string; i: BackwardsIndex): char {.inline.} = s[s.len - int(i)] - -proc `[]`*[T](s: var openArray[T]; i: BackwardsIndex): var T {.inline.} = - system.`[]`(s, s.len - int(i)) -proc `[]`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex): var T {.inline.} = - a[Idx(a.len - int(i) + int low(a))] -proc `[]`*(s: var string; i: BackwardsIndex): var char {.inline.} = s[s.len - int(i)] - -proc `[]=`*[T](s: var openArray[T]; i: BackwardsIndex; x: T) {.inline.} = - system.`[]=`(s, s.len - int(i), x) -proc `[]=`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex; x: T) {.inline.} = - a[Idx(a.len - int(i) + int low(a))] = x -proc `[]=`*(s: var string; i: BackwardsIndex; x: char) {.inline.} = - s[s.len - int(i)] = x - -proc slurp*(filename: string): string {.magic: "Slurp".} - ## This is an alias for `staticRead <#staticRead,string>`_. - -proc staticRead*(filename: string): string {.magic: "Slurp".} - ## Compile-time `readFile <io.html#readFile,string>`_ proc for easy - ## `resource`:idx: embedding: - ## - ## The maximum file size limit that `staticRead` and `slurp` can read is - ## near or equal to the *free* memory of the device you are using to compile. - ## - ## .. code-block:: Nim - ## const myResource = staticRead"mydatafile.bin" - ## - ## `slurp <#slurp,string>`_ is an alias for `staticRead`. - -proc gorge*(command: string, input = "", cache = ""): string {. - magic: "StaticExec".} = discard - ## This is an alias for `staticExec <#staticExec,string,string,string>`_. - -proc staticExec*(command: string, input = "", cache = ""): string {. - magic: "StaticExec".} = discard - ## Executes an external process at compile-time and returns its text output - ## (stdout + stderr). - ## - ## If `input` is not an empty string, it will be passed as a standard input - ## to the executed program. - ## - ## .. code-block:: Nim - ## const buildInfo = "Revision " & staticExec("git rev-parse HEAD") & - ## "\nCompiled on " & staticExec("uname -v") - ## - ## `gorge <#gorge,string,string,string>`_ is an alias for `staticExec`. - ## - ## Note that you can use this proc inside a pragma like - ## `passc <manual.html#implementation-specific-pragmas-passc-pragma>`_ or - ## `passl <manual.html#implementation-specific-pragmas-passl-pragma>`_. - ## - ## If `cache` is not empty, the results of `staticExec` are cached within - ## the `nimcache` directory. Use `--forceBuild` to get rid of this caching - ## behaviour then. `command & input & cache` (the concatenated string) is - ## used to determine whether the entry in the cache is still valid. You can - ## use versioning information for `cache`: - ## - ## .. code-block:: Nim - ## const stateMachine = staticExec("dfaoptimizer", "input", "0.8.0") - -proc gorgeEx*(command: string, input = "", cache = ""): tuple[output: string, - exitCode: int] = - ## Similar to `gorge <#gorge,string,string,string>`_ but also returns the - ## precious exit code. - discard - - -proc `+=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Increments in place a floating point number. - x = x + y - -proc `-=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Decrements in place a floating point number. - x = x - y - -proc `*=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Multiplies in place a floating point number. - x = x * y - -proc `/=`*(x: var float64, y: float64) {.inline, noSideEffect.} = - ## Divides in place a floating point number. - x = x / y - -proc `/=`*[T: float|float32](x: var T, y: T) {.inline, noSideEffect.} = - ## Divides in place a floating point number. - x = x / y +include "system/indices" proc `&=`*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} ## Appends in place to a string. - ## - ## .. code-block:: Nim + ## ```nim ## var a = "abc" ## a &= "de" # a <- "abcde" + ## ``` template `&=`*(x, y: typed) = ## Generic 'sink' operator for Nim. ## - ## For files an alias for `write`. ## If not specialized further, an alias for `add`. add(x, y) -when declared(File): - template `&=`*(f: File, x: typed) = write(f, x) - -template currentSourcePath*: string = instantiationInfo(-1, true).filename - ## Returns the full file-system path of the current source. - ## - ## To get the directory containing the current source, use it with - ## `os.parentDir() <os.html#parentDir%2Cstring>`_ as `currentSourcePath.parentDir()`. - ## - ## The path returned by this template is set at compile time. - ## - ## See the docstring of `macros.getProjectPath() <macros.html#getProjectPath>`_ - ## for an example to see the distinction between the `currentSourcePath` - ## and `getProjectPath`. - ## - ## See also: - ## * `getCurrentDir proc <os.html#getCurrentDir>`_ when compileOption("rangechecks"): template rangeCheck*(cond) = @@ -2756,28 +2458,31 @@ when compileOption("rangechecks"): else: template rangeCheck*(cond) = discard -proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} = - ## Marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not - ## perform deep copies of `s`. - ## - ## This is only useful for optimization purposes. - if s.len == 0: return - when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): - var s = cast[PGenericSeq](s) - s.reserved = s.reserved or seqShallowFlag +when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} = + ## Marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not + ## perform deep copies of `s`. + ## + ## This is only useful for optimization purposes. + if s.len == 0: return + when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): + var s = cast[PGenericSeq](s) + {.noSideEffect.}: + s.reserved = s.reserved or seqShallowFlag -proc shallow*(s: var string) {.noSideEffect, inline.} = - ## Marks a string `s` as `shallow`:idx:. Subsequent assignments will not - ## perform deep copies of `s`. - ## - ## This is only useful for optimization purposes. - when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): - var s = cast[PGenericSeq](s) - if s == nil: - s = cast[PGenericSeq](newString(0)) - # string literals cannot become 'shallow': - if (s.reserved and strlitFlag) == 0: - s.reserved = s.reserved or seqShallowFlag + proc shallow*(s: var string) {.noSideEffect, inline.} = + ## Marks a string `s` as `shallow`:idx:. Subsequent assignments will not + ## perform deep copies of `s`. + ## + ## This is only useful for optimization purposes. + when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): + var s = cast[PGenericSeq](s) + if s == nil: + s = cast[PGenericSeq](newString(0)) + # string literals cannot become 'shallow': + if (s.reserved and strlitFlag) == 0: + {.noSideEffect.}: + s.reserved = s.reserved or seqShallowFlag type NimNodeObj = object @@ -2785,36 +2490,47 @@ type NimNode* {.magic: "PNimrodNode".} = ref NimNodeObj ## Represents a Nim AST node. Macros operate on this type. -when defined(nimV2): - import system/repr_v2 - export repr_v2 +type + ForLoopStmt* {.compilerproc.} = object ## \ + ## A special type that marks a macro as a `for-loop macro`:idx:. + ## See `"For Loop Macro" <manual.html#macros-for-loop-macro>`_. macro varargsLen*(x: varargs[untyped]): int {.since: (1, 1).} = ## returns number of variadic arguments in `x` proc varargsLenImpl(x: NimNode): NimNode {.magic: "LengthOpenArray", noSideEffect.} varargsLenImpl(x) -when false: - template eval*(blk: typed): typed = - ## Executes a block of code at compile time just as if it was a macro. - ## - ## Optionally, the block can return an AST tree that will replace the - ## eval expression. - macro payload: typed {.gensym.} = blk - payload() +when defined(nimV2): + import system/repr_v2 + export repr_v2 + +proc repr*[T, U](x: HSlice[T, U]): string = + ## Generic `repr` operator for slices that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(1 .. 5) == "1 .. 5" + ## ``` + result = repr(x.a) + result.add(" .. ") + result.add(repr(x.b)) when hasAlloc or defined(nimscript): proc insert*(x: var string, item: string, i = 0.Natural) {.noSideEffect.} = ## Inserts `item` into `x` at position `i`. - ## - ## .. code-block:: Nim + ## ```nim ## var a = "abc" ## a.insert("zz", 0) # a <- "zzabc" + ## ``` + if item.len == 0: # prevents self-assignment + return var xl = x.len setLen(x, xl+item.len) var j = xl-1 while j >= i: - shallowCopy(x[j+item.len], x[j]) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + x[j+item.len] = move x[j] + else: + shallowCopy(x[j+item.len], x[j]) dec(j) j = 0 while j < item.len: @@ -2827,7 +2543,7 @@ when declared(initDebugger): proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = ## Adds a char to string `s` and applies the following escaping: ## - ## * replaces any `\` by `\\` + ## * replaces any ``\`` by `\\` ## * replaces any `'` by `\'` ## * replaces any `"` by `\"` ## * replaces any `\a` by `\\a` @@ -2838,13 +2554,14 @@ proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = ## * replaces any `\f` by `\\f` ## * replaces any `\r` by `\\r` ## * replaces any `\e` by `\\e` - ## * replaces any other character not in the set `{'\21..'\126'} - ## by `\xHH` where `HH` is its hexadecimal value. + ## * replaces any other character not in the set `{\21..\126}` + ## by `\xHH` where `HH` is its hexadecimal value ## ## The procedure has been designed so that its output is usable for many ## different common syntaxes. ## - ## **Note**: This is **not correct** for producing Ansi C code! + ## .. warning:: This is **not correct** for producing ANSI C code! + ## case c of '\a': s.add "\\a" # \x07 of '\b': s.add "\\b" # \x08 @@ -2881,7 +2598,7 @@ proc addQuoted*[T](s: var string, x: T) = ## Users may overload `addQuoted` for custom (string-like) types if ## they want to implement a customized element representation. ## - ## .. code-block:: Nim + ## ```nim ## var tmp = "" ## tmp.addQuoted(1) ## tmp.add(", ") @@ -2889,6 +2606,7 @@ proc addQuoted*[T](s: var string, x: T) = ## tmp.add(", ") ## tmp.addQuoted('c') ## assert(tmp == """1, "string", 'c'""") + ## ``` when T is string or T is cstring: s.add("\"") for c in x: @@ -2904,7 +2622,7 @@ proc addQuoted*[T](s: var string, x: T) = s.addEscapedChar(x) s.add("'") # prevent temporary string allocation - elif T is SomeSignedInt: + elif T is SomeInteger: s.addInt(x) elif T is SomeFloat: s.addFloat(x) @@ -2922,7 +2640,7 @@ proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} = ## the official signature says, the return type is *not* `RootObj` but a ## tuple of a structure that depends on the current scope. Example: ## - ## .. code-block:: Nim + ## ```nim ## proc testLocals() = ## var ## a = "something" @@ -2937,6 +2655,7 @@ proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} = ## # -> name a with value something ## # -> name b with value 4 ## # -> B is 1 + ## ``` discard when hasAlloc and notJSnotNims: @@ -2947,7 +2666,7 @@ when hasAlloc and notJSnotNims: ## This is also used by the code generator ## for the implementation of `spawn`. ## - ## For `--gc:arc` or `--gc:orc` deepcopy support has to be enabled + ## For `--mm:arc` or `--mm:orc` deepcopy support has to be enabled ## via `--deepcopy:on`. discard @@ -2960,10 +2679,10 @@ when hasAlloc and notJSnotNims: proc procCall*(x: untyped) {.magic: "ProcCall", compileTime.} = ## Special magic to prohibit dynamic binding for `method`:idx: calls. ## This is similar to `super`:idx: in ordinary OO languages. - ## - ## .. code-block:: Nim + ## ```nim ## # 'someMethod' will be resolved fully statically: ## procCall someMethod(a, b) + ## ``` discard @@ -2973,20 +2692,9 @@ proc `==`*(x, y: cstring): bool {.magic: "EqCString", noSideEffect, proc strcmp(a, b: cstring): cint {.noSideEffect, importc, header: "<string.h>".} if pointer(x) == pointer(y): result = true - elif x.isNil or y.isNil: result = false + elif pointer(x) == nil or pointer(y) == nil: result = false else: result = strcmp(x, y) == 0 -when true: # xxx PRTEMP remove - # bug #9149; ensure that 'typeof(nil)' does not match *too* well by using 'typeof(nil) | typeof(nil)', - # especially for converters, see tests/overload/tconverter_to_string.nim - # Eventually we will be able to remove this hack completely. - proc `==`*(x: string; y: typeof(nil) | typeof(nil)): bool {. - error: "'nil' is now invalid for 'string'".} = - discard - proc `==`*(x: typeof(nil) | typeof(nil); y: string): bool {. - error: "'nil' is now invalid for 'string'".} = - discard - template closureScope*(body: untyped): untyped = ## Useful when creating a closure in a loop to capture local loop variables by ## their current iteration values. @@ -2996,7 +2704,7 @@ template closureScope*(body: untyped): untyped = ## ## Example: ## - ## .. code-block:: Nim + ## ```nim ## var myClosure : proc() ## # without closureScope: ## for i in 0 .. 5: @@ -3011,20 +2719,19 @@ template closureScope*(body: untyped): untyped = ## if j == 3: ## myClosure = proc() = echo j ## myClosure() # outputs 3 + ## ``` (proc() = body)() template once*(body: untyped): untyped = ## Executes a block of code only once (the first time the block is reached). - ## - ## .. code-block:: Nim - ## - ## proc draw(t: Triangle) = - ## once: - ## graphicsInit() - ## line(t.p1, t.p2) - ## line(t.p2, t.p3) - ## line(t.p3, t.p1) - ## + ## ```nim + ## proc draw(t: Triangle) = + ## once: + ## graphicsInit() + ## line(t.p1, t.p2) + ## line(t.p2, t.p3) + ## line(t.p3, t.p1) + ## ``` var alreadyExecuted {.global.} = false if not alreadyExecuted: alreadyExecuted = true @@ -3032,7 +2739,19 @@ template once*(body: untyped): untyped = {.pop.} # warning[GcMem]: off, warning[Uninit]: off -proc substr*(s: string, first, last: int): string = +proc substr*(s: openArray[char]): string = + ## Copies a slice of `s` into a new string and returns this new + ## string. + runnableExamples: + let a = "abcdefgh" + assert a.substr(2, 5) == "cdef" + assert a.substr(2) == "cdefgh" + assert a.substr(5, 99) == "fgh" + result = newString(s.len) + for i, ch in s: + result[i] = ch + +proc substr*(s: string, first, last: int): string = # A bug with `magic: Slice` requires this to exist this way ## Copies a slice of `s` into a new string and returns this new ## string. ## @@ -3062,11 +2781,10 @@ when defined(nimconfig): when not defined(js): proc toOpenArray*[T](x: ptr UncheckedArray[T]; first, last: int): openArray[T] {. magic: "Slice".} - when defined(nimToOpenArrayCString): - proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. - magic: "Slice".} - proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. - magic: "Slice".} + proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. + magic: "Slice".} + proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. + magic: "Slice".} proc toOpenArray*[T](x: seq[T]; first, last: int): openArray[T] {. magic: "Slice".} @@ -3084,10 +2802,8 @@ proc toOpenArrayByte*(x: openArray[char]; first, last: int): openArray[byte] {. proc toOpenArrayByte*(x: seq[char]; first, last: int): openArray[byte] {. magic: "Slice".} -type - ForLoopStmt* {.compilerproc.} = object ## \ - ## A special type that marks a macro as a `for-loop macro`:idx:. - ## See `"For Loop Macro" <manual.html#macros-for-loop-macro>`_. +proc toOpenArrayChar*(x: openArray[byte]; first, last: int): openArray[char] {. + magic: "Slice".} when defined(genode): var componentConstructHook*: proc (env: GenodeEnv) {.nimcall.} @@ -3101,7 +2817,7 @@ when defined(genode): proc nim_component_construct(env: GenodeEnv) {.exportc.} = ## Procedure called during `Component::construct` by the loader. if componentConstructHook.isNil: - env.quit(programResult) + env.rawQuit(programResult) # No native Genode application initialization, # exit as would POSIX. else: @@ -3110,11 +2826,86 @@ when defined(genode): # and return to thread entrypoint. -import system/widestrs -export widestrs +when not defined(nimPreviewSlimSystem): + import std/widestrs + export widestrs -import system/io -export io +when notJSnotNims: + when defined(windows) and compileOption("threads"): + when not declared(addSysExitProc): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".} + var echoLock: SysLock + initSysLock echoLock + addSysExitProc(proc() {.noconv.} = deinitSys(echoLock)) + + const stdOutLock = compileOption("threads") and + not defined(windows) and + not defined(android) and + not defined(nintendoswitch) and + not defined(freertos) and + not defined(zephyr) and + not defined(nuttx) and + hostOS != "any" + + proc raiseEIO(msg: string) {.noinline, noreturn.} = + raise newException(IOError, msg) + + proc echoBinSafe(args: openArray[string]) {.compilerproc.} = + when defined(androidNDK): + # When running nim in android app, stdout goes nowhere, so echo gets ignored + # To redirect echo to the android logcat, use -d:androidNDK + const ANDROID_LOG_VERBOSE = 2.cint + proc android_log_print(prio: cint, tag: cstring, fmt: cstring): cint + {.importc: "__android_log_print", header: "<android/log.h>", varargs, discardable.} + var s = "" + for arg in args: + s.add arg + android_log_print(ANDROID_LOG_VERBOSE, "nim", s) + else: + # flockfile deadlocks some versions of Android 5.x.x + when stdOutLock: + proc flockfile(f: CFilePtr) {.importc, nodecl.} + proc funlockfile(f: CFilePtr) {.importc, nodecl.} + flockfile(cstdout) + when defined(windows) and compileOption("threads"): + acquireSys echoLock + for s in args: + when defined(windows): + # equivalent to syncio.writeWindows + proc writeWindows(f: CFilePtr; s: string; doRaise = false) = + # Don't ask why but the 'printf' family of function is the only thing + # that writes utf-8 strings reliably on Windows. At least on my Win 10 + # machine. We also enable `setConsoleOutputCP(65001)` now by default. + # But we cannot call printf directly as the string might contain \0. + # So we have to loop over all the sections separated by potential \0s. + var i = int c_fprintf(f, "%s", s) + while i < s.len: + if s[i] == '\0': + let w = c_fputc('\0', f) + if w != 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i + else: + let w = c_fprintf(f, "%s", unsafeAddr s[i]) + if w <= 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i, w + writeWindows(cstdout, s) + else: + discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, cstdout) + const linefeed = "\n" + discard c_fwrite(linefeed.cstring, linefeed.len, 1, cstdout) + discard c_fflush(cstdout) + when stdOutLock: + funlockfile(cstdout) + when defined(windows) and compileOption("threads"): + releaseSys echoLock + +when not defined(nimPreviewSlimSystem): + import std/syncio + export syncio when not defined(createNimHcr) and not defined(nimscript): include nimhcr @@ -3124,10 +2915,19 @@ when notJSnotNims and not defined(nimSeqsV2): ## String literals (e.g. "abc", etc) in the ARC/ORC mode are "copy on write", ## therefore you should call `prepareMutation` before modifying the strings ## via `addr`. - runnableExamples("--gc:arc"): + runnableExamples: var x = "abc" var y = "defgh" prepareMutation(y) # without this, you may get a `SIGBUS` or `SIGSEGV` moveMem(addr y[0], addr x[0], x.len) assert y == "abcgh" discard + +proc arrayWith*[T](y: T, size: static int): array[size, T] {.raises: [].} = + ## Creates a new array filled with `y`. + for i in 0..size-1: + when nimvm: + result[i] = y + else: + # TODO: fixme it should be `=dup` + result[i] = y diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 76c744f593..441dc17194 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -11,6 +11,8 @@ {.push profiler:off.} include osalloc +import std/private/syslocks +import std/sysatomics template track(op, address, size) = when defined(memTracker): @@ -22,7 +24,7 @@ template track(op, address, size) = const nimMinHeapPages {.intdefine.} = 128 # 0.5 MB SmallChunkSize = PageSize - MaxFli = 30 + MaxFli = when sizeof(int) > 2: 30 else: 14 MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32' # everywhere! MaxSli = 1 shl MaxLog2Sli @@ -30,7 +32,7 @@ const RealFli = MaxFli - FliOffset # size of chunks in last matrix bin - MaxBigChunkSize = 1 shl MaxFli - 1 shl (MaxFli-MaxLog2Sli-1) + MaxBigChunkSize = int(1'i32 shl MaxFli - 1'i32 shl (MaxFli-MaxLog2Sli-1)) HugeChunkSize = MaxBigChunkSize + 1 type @@ -44,44 +46,6 @@ type IntSet = object data: TrunkBuckets -type - FreeCell {.final, pure.} = object - next: ptr FreeCell # next free cell in chunk (overlaid with refcount) - when not defined(gcDestructors): - zeroField: int # 0 means cell is not used (overlaid with typ field) - # 1 means cell is manually managed pointer - # otherwise a PNimType is stored in there - else: - alignment: int - - PChunk = ptr BaseChunk - PBigChunk = ptr BigChunk - PSmallChunk = ptr SmallChunk - BaseChunk {.pure, inheritable.} = object - prevSize: int # size of previous chunk; for coalescing - # 0th bit == 1 if 'used - size: int # if < PageSize it is a small chunk - - SmallChunk = object of BaseChunk - next, prev: PSmallChunk # chunks of the same size - freeList: ptr FreeCell - free: int # how many bytes remain - acc: int # accumulator for small object allocation - when defined(nimAlignPragma): - data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory - else: - data: UncheckedArray[byte] - - BigChunk = object of BaseChunk # not necessarily > PageSize! - next, prev: PBigChunk # chunks of the same (or bigger) size - when defined(nimAlignPragma): - data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory - else: - data: UncheckedArray[byte] - -template smallChunkOverhead(): untyped = sizeof(SmallChunk) -template bigChunkOverhead(): untyped = sizeof(BigChunk) - # ------------- chunk table --------------------------------------------------- # We use a PtrSet of chunk starts and a table[Page, chunksize] for chunk # endings of big chunks. This is needed by the merging operation. The only @@ -102,29 +66,90 @@ type key, upperBound: int level: int +const + RegionHasLock = false # hasThreadSupport and defined(gcDestructors) + +type + FreeCell {.final, pure.} = object + next: ptr FreeCell # next free cell in chunk (overlaid with refcount) + when not defined(gcDestructors): + zeroField: int # 0 means cell is not used (overlaid with typ field) + # 1 means cell is manually managed pointer + # otherwise a PNimType is stored in there + else: + alignment: int + + PChunk = ptr BaseChunk + PBigChunk = ptr BigChunk + PSmallChunk = ptr SmallChunk + BaseChunk {.pure, inheritable.} = object + prevSize: int # size of previous chunk; for coalescing + # 0th bit == 1 if 'used + size: int # if < PageSize it is a small chunk + owner: ptr MemRegion + + SmallChunk = object of BaseChunk + next, prev: PSmallChunk # chunks of the same size + freeList: ptr FreeCell + free: int # how many bytes remain + acc: int # accumulator for small object allocation + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory + + BigChunk = object of BaseChunk # not necessarily > PageSize! + next, prev: PBigChunk # chunks of the same (or bigger) size + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory + HeapLinks = object len: int chunks: array[30, (PBigChunk, int)] next: ptr HeapLinks MemRegion = object - minLargeObj, maxLargeObj: int - freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] + when not defined(gcDestructors): + minLargeObj, maxLargeObj: int + freeSmallChunks: array[0..max(1, SmallChunkSize div MemAlign-1), PSmallChunk] + when defined(gcDestructors): + sharedFreeLists: array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell] flBitmap: uint32 slBitmap: array[RealFli, uint32] matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly + when RegionHasLock: + lock: SysLock + when defined(gcDestructors): + sharedFreeListBigChunks: PBigChunk # make no attempt at avoiding false sharing for now for this object field + chunkStarts: IntSet - root, deleted, last, freeAvlNodes: PAvlNode - locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages. + when not defined(gcDestructors): + root, deleted, last, freeAvlNodes: PAvlNode + lockActive, locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages. nextChunkSize: int - bottomData: AvlNode + when not defined(gcDestructors): + bottomData: AvlNode heapLinks: HeapLinks when defined(nimTypeNames): allocCounter, deallocCounter: int +template smallChunkOverhead(): untyped = sizeof(SmallChunk) +template bigChunkOverhead(): untyped = sizeof(BigChunk) + +when hasThreadSupport: + template loada(x: untyped): untyped = atomicLoadN(unsafeAddr x, ATOMIC_RELAXED) + template storea(x, y: untyped) = atomicStoreN(unsafeAddr x, y, ATOMIC_RELAXED) + + when false: + # not yet required + template atomicStatDec(x, diff: untyped) = discard atomicSubFetch(unsafeAddr x, diff, ATOMIC_RELAXED) + template atomicStatInc(x, diff: untyped) = discard atomicAddFetch(unsafeAddr x, diff, ATOMIC_RELAXED) +else: + template loada(x: untyped): untyped = x + template storea(x, y: untyped) = x = y + +template atomicStatDec(x, diff: untyped) = dec x, diff +template atomicStatInc(x, diff: untyped) = inc x, diff + const fsLookupTable: array[byte, int8] = [ -1'i8, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, @@ -165,7 +190,7 @@ proc mappingSearch(r, fl, sl: var int) {.inline.} = let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1 r = r + t r = r and not t - r = min(r, MaxBigChunkSize) + r = min(r, MaxBigChunkSize).int fl = msbit(uint32 r) sl = (r shr (fl - MaxLog2Sli)) - MaxSli dec fl, FliOffset @@ -231,11 +256,11 @@ proc addChunkToMatrix(a: var MemRegion; b: PBigChunk) = setBit(fl, a.flBitmap) proc incCurrMem(a: var MemRegion, bytes: int) {.inline.} = - inc(a.currMem, bytes) + atomicStatInc(a.currMem, bytes) proc decCurrMem(a: var MemRegion, bytes: int) {.inline.} = a.maxMem = max(a.maxMem, a.currMem) - dec(a.currMem, bytes) + atomicStatDec(a.currMem, bytes) proc getMaxMem(a: var MemRegion): int = # Since we update maxPagesCount only when freeing pages, @@ -243,6 +268,20 @@ proc getMaxMem(a: var MemRegion): int = # maximum of these both values here: result = max(a.currMem, a.maxMem) +const nimMaxHeap {.intdefine.} = 0 + +proc allocPages(a: var MemRegion, size: int): pointer = + when nimMaxHeap != 0: + if a.occ + size > nimMaxHeap * 1024 * 1024: + raiseOutOfMem() + osAllocPages(size) + +proc tryAllocPages(a: var MemRegion, size: int): pointer = + when nimMaxHeap != 0: + if a.occ + size > nimMaxHeap * 1024 * 1024: + raiseOutOfMem() + osTryAllocPages(size) + proc llAlloc(a: var MemRegion, size: int): pointer = # *low-level* alloc for the memory managers data structures. Deallocation # is done at the end of the allocator's life time. @@ -252,49 +291,50 @@ proc llAlloc(a: var MemRegion, size: int): pointer = # is one page: sysAssert roundup(size+sizeof(LLChunk), PageSize) == PageSize, "roundup 6" var old = a.llmem # can be nil and is correct with nil - a.llmem = cast[PLLChunk](osAllocPages(PageSize)) + a.llmem = cast[PLLChunk](allocPages(a, PageSize)) when defined(nimAvlcorruption): trackLocation(a.llmem, PageSize) incCurrMem(a, PageSize) a.llmem.size = PageSize - sizeof(LLChunk) a.llmem.acc = sizeof(LLChunk) a.llmem.next = old - result = cast[pointer](cast[ByteAddress](a.llmem) + a.llmem.acc) + result = cast[pointer](cast[int](a.llmem) + a.llmem.acc) dec(a.llmem.size, size) inc(a.llmem.acc, size) zeroMem(result, size) -proc getBottom(a: var MemRegion): PAvlNode = - result = addr(a.bottomData) - if result.link[0] == nil: - result.link[0] = result - result.link[1] = result +when not defined(gcDestructors): + proc getBottom(a: var MemRegion): PAvlNode = + result = addr(a.bottomData) + if result.link[0] == nil: + result.link[0] = result + result.link[1] = result -proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode = - if a.freeAvlNodes != nil: - result = a.freeAvlNodes - a.freeAvlNodes = a.freeAvlNodes.link[0] - else: - result = cast[PAvlNode](llAlloc(a, sizeof(AvlNode))) - when defined(nimAvlcorruption): - cprintf("tracking location: %p\n", result) - result.key = key - result.upperBound = upperBound - let bottom = getBottom(a) - result.link[0] = bottom - result.link[1] = bottom - result.level = 1 - #when defined(nimAvlcorruption): - # track("allocAvlNode", result, sizeof(AvlNode)) - sysAssert(bottom == addr(a.bottomData), "bottom data") - sysAssert(bottom.link[0] == bottom, "bottom link[0]") - sysAssert(bottom.link[1] == bottom, "bottom link[1]") + proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode = + if a.freeAvlNodes != nil: + result = a.freeAvlNodes + a.freeAvlNodes = a.freeAvlNodes.link[0] + else: + result = cast[PAvlNode](llAlloc(a, sizeof(AvlNode))) + when defined(nimAvlcorruption): + cprintf("tracking location: %p\n", result) + result.key = key + result.upperBound = upperBound + let bottom = getBottom(a) + result.link[0] = bottom + result.link[1] = bottom + result.level = 1 + #when defined(nimAvlcorruption): + # track("allocAvlNode", result, sizeof(AvlNode)) + sysAssert(bottom == addr(a.bottomData), "bottom data") + sysAssert(bottom.link[0] == bottom, "bottom link[0]") + sysAssert(bottom.link[1] == bottom, "bottom link[1]") -proc deallocAvlNode(a: var MemRegion, n: PAvlNode) {.inline.} = - n.link[0] = a.freeAvlNodes - a.freeAvlNodes = n + proc deallocAvlNode(a: var MemRegion, n: PAvlNode) {.inline.} = + n.link[0] = a.freeAvlNodes + a.freeAvlNodes = n -proc addHeapLink(a: var MemRegion; p: PBigChunk, size: int) = +proc addHeapLink(a: var MemRegion; p: PBigChunk, size: int): ptr HeapLinks = var it = addr(a.heapLinks) while it != nil and it.len >= it.chunks.len: it = it.next if it == nil: @@ -303,12 +343,15 @@ proc addHeapLink(a: var MemRegion; p: PBigChunk, size: int) = a.heapLinks.next = n n.chunks[0] = (p, size) n.len = 1 + result = n else: let L = it.len it.chunks[L] = (p, size) inc it.len + result = it -include "system/avltree" +when not defined(gcDestructors): + include "system/avltree" proc llDeallocAll(a: var MemRegion) = var it = a.llmem @@ -373,7 +416,7 @@ iterator elements(t: IntSet): int {.inline.} = r = r.next proc isSmallChunk(c: PChunk): bool {.inline.} = - return c.size <= SmallChunkSize-smallChunkOverhead() + result = c.size <= SmallChunkSize-smallChunkOverhead() proc chunkUnused(c: PChunk): bool {.inline.} = result = (c.prevSize and 1) == 0 @@ -389,7 +432,7 @@ iterator allObjects(m: var MemRegion): pointer {.inline.} = var c = cast[PSmallChunk](c) let size = c.size - var a = cast[ByteAddress](addr(c.data)) + var a = cast[int](addr(c.data)) let limit = a + c.acc while a <% limit: yield cast[pointer](a) @@ -408,13 +451,13 @@ when not defined(gcDestructors): # ------------- chunk management ---------------------------------------------- proc pageIndex(c: PChunk): int {.inline.} = - result = cast[ByteAddress](c) shr PageShift + result = cast[int](c) shr PageShift proc pageIndex(p: pointer): int {.inline.} = - result = cast[ByteAddress](p) shr PageShift + result = cast[int](p) shr PageShift proc pageAddr(p: pointer): PChunk {.inline.} = - result = cast[PChunk](cast[ByteAddress](p) and not PageMask) + result = cast[PChunk](cast[int](p) and not PageMask) #sysAssert(Contains(allocator.chunkStarts, pageIndex(result))) when false: @@ -426,49 +469,44 @@ when false: it, it.next, it.prev, it.size) it = it.next -const nimMaxHeap {.intdefine.} = 0 - proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when not defined(emscripten): if not a.blockChunkSizeIncrease: let usedMem = a.occ #a.currMem # - a.freeMem - when nimMaxHeap != 0: - if usedMem > nimMaxHeap * 1024 * 1024: - raiseOutOfMem() if usedMem < 64 * 1024: a.nextChunkSize = PageSize*4 else: a.nextChunkSize = min(roundup(usedMem shr 2, PageSize), a.nextChunkSize * 2) - a.nextChunkSize = min(a.nextChunkSize, MaxBigChunkSize) + a.nextChunkSize = min(a.nextChunkSize, MaxBigChunkSize).int var size = size if size > a.nextChunkSize: - result = cast[PBigChunk](osAllocPages(size)) + result = cast[PBigChunk](allocPages(a, size)) else: - result = cast[PBigChunk](osTryAllocPages(a.nextChunkSize)) + result = cast[PBigChunk](tryAllocPages(a, a.nextChunkSize)) if result == nil: - result = cast[PBigChunk](osAllocPages(size)) + result = cast[PBigChunk](allocPages(a, size)) a.blockChunkSizeIncrease = true else: size = a.nextChunkSize incCurrMem(a, size) inc(a.freeMem, size) - a.addHeapLink(result, size) + let heapLink = a.addHeapLink(result, size) when defined(debugHeapLinks): cprintf("owner: %p; result: %p; next pointer %p; size: %ld\n", addr(a), - result, result.heapLink, result.size) + result, heapLink, size) when defined(memtracker): trackLocation(addr result.size, sizeof(int)) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "requestOsChunks 1") + sysAssert((cast[int](result) and PageMask) == 0, "requestOsChunks 1") #zeroMem(result, size) result.next = nil result.prev = nil result.size = size # update next.prevSize: - var nxt = cast[ByteAddress](result) +% size + var nxt = cast[int](result) +% size sysAssert((nxt and PageMask) == 0, "requestOsChunks 2") var next = cast[PChunk](nxt) if pageIndex(next) in a.chunkStarts: @@ -476,7 +514,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = next.prevSize = size or (next.prevSize and 1) # set result.prevSize: var lastSize = if a.lastSize != 0: a.lastSize else: PageSize - var prv = cast[ByteAddress](result) -% lastSize + var prv = cast[int](result) -% lastSize sysAssert((nxt and PageMask) == 0, "requestOsChunks 3") var prev = cast[PChunk](prv) if pageIndex(prev) in a.chunkStarts and prev.size == lastSize: @@ -522,21 +560,22 @@ proc listRemove[T](head: var T, c: T) {.inline.} = proc updatePrevSize(a: var MemRegion, c: PBigChunk, prevSize: int) {.inline.} = - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "updatePrevSize") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "updatePrevSize") if isAccessible(a, ri): ri.prevSize = prevSize or (ri.prevSize and 1) proc splitChunk2(a: var MemRegion, c: PBigChunk, size: int): PBigChunk = - result = cast[PBigChunk](cast[ByteAddress](c) +% size) + result = cast[PBigChunk](cast[int](c) +% size) result.size = c.size - size track("result.size", addr result.size, sizeof(int)) - # XXX check if these two nil assignments are dead code given - # addChunkToMatrix's implementation: - result.next = nil - result.prev = nil + when not defined(nimOptimizedSplitChunk): + # still active because of weird codegen issue on some of our CIs: + result.next = nil + result.prev = nil # size and not used: result.prevSize = size + result.owner = addr a sysAssert((size and 1) == 0, "splitChunk 2") sysAssert((size and PageMask) == 0, "splitChunk: size is not a multiple of the PageSize") @@ -556,8 +595,8 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = when coalescLeft: let prevSize = c.prevSize if prevSize != 0: - var le = cast[PChunk](cast[ByteAddress](c) -% prevSize) - sysAssert((cast[ByteAddress](le) and PageMask) == 0, "freeBigChunk 4") + var le = cast[PChunk](cast[int](c) -% prevSize) + sysAssert((cast[int](le) and PageMask) == 0, "freeBigChunk 4") if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") if not isSmallChunk(le) and le.size < MaxBigChunkSize: @@ -567,11 +606,14 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = c = cast[PBigChunk](le) if c.size > MaxBigChunkSize: let rest = splitChunk2(a, c, MaxBigChunkSize) + when defined(nimOptimizedSplitChunk): + rest.next = nil + rest.prev = nil addChunkToMatrix(a, c) c = rest when coalescRight: - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "freeBigChunk 2") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "freeBigChunk 2") if isAccessible(a, ri) and chunkUnused(ri): sysAssert(not isSmallChunk(ri), "freeBigChunk 3") if not isSmallChunk(ri) and c.size < MaxBigChunkSize: @@ -591,6 +633,13 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = mappingSearch(size, fl, sl) sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") result = findSuitableBlock(a, fl, sl) + + when RegionHasLock: + if not a.lockActive: + a.lockActive = true + initSysLock(a.lock) + acquireSys a.lock + if result == nil: if size < nimMinHeapPages * PageSize: result = requestOsChunks(a, nimMinHeapPages * PageSize) @@ -600,6 +649,7 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # if we over allocated split the chunk: if result.size > size: splitChunk(a, result, size) + result.owner = addr a else: removeChunkFromMatrix2(a, result, fl, sl) if result.size >= size + PageSize: @@ -607,22 +657,33 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # set 'used' to to true: result.prevSize = 1 track("setUsedToFalse", addr result.size, sizeof(int)) + sysAssert result.owner == addr a, "getBigChunk: No owner set!" incl(a, a.chunkStarts, pageIndex(result)) dec(a.freeMem, size) + when RegionHasLock: + releaseSys a.lock proc getHugeChunk(a: var MemRegion; size: int): PBigChunk = - result = cast[PBigChunk](osAllocPages(size)) + result = cast[PBigChunk](allocPages(a, size)) + when RegionHasLock: + if not a.lockActive: + a.lockActive = true + initSysLock(a.lock) + acquireSys a.lock incCurrMem(a, size) # XXX add this to the heap links. But also remove it from it later. when false: a.addHeapLink(result, size) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "getHugeChunk") + sysAssert((cast[int](result) and PageMask) == 0, "getHugeChunk") result.next = nil result.prev = nil result.size = size # set 'used' to to true: result.prevSize = 1 + result.owner = addr a incl(a, a.chunkStarts, pageIndex(result)) + when RegionHasLock: + releaseSys a.lock proc freeHugeChunk(a: var MemRegion; c: PBigChunk) = let size = c.size @@ -686,76 +747,85 @@ else: template trackSize(x) = discard template untrackSize(x) = discard -when false: - # not yet used by the GCs - proc rawTryAlloc(a: var MemRegion; requestedSize: int): pointer = - sysAssert(allocInv(a), "rawAlloc: begin") - sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") - sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") - var size = roundup(requestedSize, MemAlign) - inc a.occ, size - trackSize(size) - sysAssert(size >= requestedSize, "insufficient allocated size!") - #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) - if size <= SmallChunkSize-smallChunkOverhead(): - # allocate a small block: for small chunks, we use only its next pointer - var s = size div MemAlign - var c = a.freeSmallChunks[s] - if c == nil: - result = nil - else: - sysAssert c.size == size, "rawAlloc 6" - if c.freeList == nil: - sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, - "rawAlloc 7") - result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) - inc(c.acc, size) - else: - result = c.freeList - sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") - c.freeList = c.freeList.next - dec(c.free, size) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") - if c.free < size: - listRemove(a.freeSmallChunks[s], c) - sysAssert(allocInv(a), "rawAlloc: end listRemove test") - sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% - size == 0, "rawAlloc 21") - sysAssert(allocInv(a), "rawAlloc: end small size") +proc deallocBigChunk(a: var MemRegion, c: PBigChunk) = + when RegionHasLock: + acquireSys a.lock + dec a.occ, c.size + untrackSize(c.size) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" + when not defined(gcDestructors): + a.deleted = getBottom(a) + del(a, a.root, cast[int](addr(c.data))) + if c.size >= HugeChunkSize: freeHugeChunk(a, c) + else: freeBigChunk(a, c) + when RegionHasLock: + releaseSys a.lock + +when defined(gcDestructors): + template atomicPrepend(head, elem: untyped) = + # see also https://en.cppreference.com/w/cpp/atomic/atomic_compare_exchange + when hasThreadSupport: + while true: + elem.next.storea head.loada + if atomicCompareExchangeN(addr head, addr elem.next, elem, weak = true, ATOMIC_RELEASE, ATOMIC_RELAXED): + break else: - inc size, bigChunkOverhead() - var fl, sl: int - mappingSearch(size, fl, sl) - sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") - let c = findSuitableBlock(a, fl, sl) - if c != nil: - removeChunkFromMatrix2(a, c, fl, sl) - if c.size >= size + PageSize: - splitChunk(a, c, size) - # set 'used' to to true: - c.prevSize = 1 - incl(a, a.chunkStarts, pageIndex(c)) - dec(a.freeMem, size) - result = addr(c.data) - sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") - sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") - if a.root == nil: a.root = getBottom(a) - add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) - else: - result = nil + elem.next.storea head.loada + head.storea elem + + proc addToSharedFreeListBigChunks(a: var MemRegion; c: PBigChunk) {.inline.} = + sysAssert c.next == nil, "c.next pointer must be nil" + atomicPrepend a.sharedFreeListBigChunks, c + + proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; size: int) {.inline.} = + atomicPrepend c.owner.sharedFreeLists[size], f + + const MaxSteps = 20 + + proc compensateCounters(a: var MemRegion; c: PSmallChunk; size: int) = + # rawDealloc did NOT do the usual: + # `inc(c.free, size); dec(a.occ, size)` because it wasn't the owner of these + # memory locations. We have to compensate here for these for the entire list. + # Well, not for the entire list, but for `max` elements of the list because + # we split the list in order to achieve bounded response times. + var it = c.freeList + var x = 0 + while it != nil: + inc x, size + let chunk = cast[PSmallChunk](pageAddr(it)) + inc(chunk.free, x) + it = it.next + dec(a.occ, x) + + proc freeDeferredObjects(a: var MemRegion; root: PBigChunk) = + var it = root + var maxIters = MaxSteps # make it time-bounded + while true: + let rest = it.next.loada + it.next.storea nil + deallocBigChunk(a, cast[PBigChunk](it)) + if maxIters == 0: + if rest != nil: + addToSharedFreeListBigChunks(a, rest) + sysAssert a.sharedFreeListBigChunks != nil, "re-enqueing failed" + break + it = rest + dec maxIters + if it == nil: break proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = when defined(nimTypeNames): inc(a.allocCounter) sysAssert(allocInv(a), "rawAlloc: begin") sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") - sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") var size = roundup(requestedSize, MemAlign) + sysAssert(size >= sizeof(FreeCell), "rawAlloc: requested size too small") sysAssert(size >= requestedSize, "insufficient allocated size!") #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + if size <= SmallChunkSize-smallChunkOverhead(): # allocate a small block: for small chunks, we use only its next pointer - var s = size div MemAlign + let s = size div MemAlign var c = a.freeSmallChunks[s] if c == nil: c = getSmallChunk(a) @@ -764,21 +834,31 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = c.size = size c.acc = size c.free = SmallChunkSize - smallChunkOverhead() - size + sysAssert c.owner == addr(a), "rawAlloc: No owner set!" c.next = nil c.prev = nil listAdd(a.freeSmallChunks[s], c) result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 4") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 4") else: sysAssert(allocInv(a), "rawAlloc: begin c != nil") sysAssert c.next != c, "rawAlloc 5" #if c.size != size: # c_fprintf(stdout, "csize: %lld; size %lld\n", c.size, size) sysAssert c.size == size, "rawAlloc 6" + when defined(gcDestructors): + if c.freeList == nil: + when hasThreadSupport: + # Steal the entire list from `sharedFreeList`: + c.freeList = atomicExchangeN(addr a.sharedFreeLists[s], nil, ATOMIC_RELAXED) + else: + c.freeList = a.sharedFreeLists[s] + a.sharedFreeLists[s] = nil + compensateCounters(a, c, size) if c.freeList == nil: sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, "rawAlloc 7") - result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + result = cast[pointer](cast[int](addr(c.data)) +% c.acc) inc(c.acc, size) else: result = c.freeList @@ -786,19 +866,28 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") c.freeList = c.freeList.next dec(c.free, size) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 9") sysAssert(allocInv(a), "rawAlloc: end c != nil") sysAssert(allocInv(a), "rawAlloc: before c.free < size") if c.free < size: sysAssert(allocInv(a), "rawAlloc: before listRemove test") listRemove(a.freeSmallChunks[s], c) sysAssert(allocInv(a), "rawAlloc: end listRemove test") - sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + sysAssert(((cast[int](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") inc a.occ, size trackSize(c.size) else: + when defined(gcDestructors): + when hasThreadSupport: + let deferredFrees = atomicExchangeN(addr a.sharedFreeListBigChunks, nil, ATOMIC_RELAXED) + else: + let deferredFrees = a.sharedFreeListBigChunks + a.sharedFreeListBigChunks = nil + if deferredFrees != nil: + freeDeferredObjects(a, deferredFrees) + size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block var c = if size >= HugeChunkSize: getHugeChunk(a, size) @@ -806,10 +895,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" result = addr(c.data) - sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") - sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") - if a.root == nil: a.root = getBottom(a) - add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + sysAssert((cast[int](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[int](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + when not defined(gcDestructors): + if a.root == nil: a.root = getBottom(a) + add(a, a.root, cast[int](result), cast[int](result)+%size) inc a.occ, c.size trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") @@ -826,51 +916,57 @@ proc rawDealloc(a: var MemRegion, p: pointer) = #sysAssert(isAllocatedPtr(a, p), "rawDealloc: no allocated pointer") sysAssert(allocInv(a), "rawDealloc: begin") var c = pageAddr(p) + sysAssert(c != nil, "rawDealloc: begin") if isSmallChunk(c): # `p` is within a small chunk: var c = cast[PSmallChunk](c) - var s = c.size - dec a.occ, s - untrackSize(s) - sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% - s == 0, "rawDealloc 3") + let s = c.size + # ^ We might access thread foreign storage here. + # The other thread cannot possibly free this block as it's still alive. var f = cast[ptr FreeCell](p) - when not defined(gcDestructors): - #echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField))) - sysAssert(f.zeroField != 0, "rawDealloc 1") - f.zeroField = 0 - f.next = c.freeList - c.freeList = f - when overwriteFree: - # set to 0xff to check for usage after free bugs: - nimSetMem(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32, - s -% sizeof(FreeCell)) - # check if it is not in the freeSmallChunks[s] list: - if c.free < s: - # add it to the freeSmallChunks[s] array: - listAdd(a.freeSmallChunks[s div MemAlign], c) - inc(c.free, s) + if c.owner == addr(a): + # We own the block, there is no foreign thread involved. + dec a.occ, s + untrackSize(s) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% + s == 0, "rawDealloc 3") + when not defined(gcDestructors): + #echo("setting to nil: ", $cast[int](addr(f.zeroField))) + sysAssert(f.zeroField != 0, "rawDealloc 1") + f.zeroField = 0 + f.next = c.freeList + c.freeList = f + when overwriteFree: + # set to 0xff to check for usage after free bugs: + nimSetMem(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32, + s -% sizeof(FreeCell)) + # check if it is not in the freeSmallChunks[s] list: + if c.free < s: + # add it to the freeSmallChunks[s] array: + listAdd(a.freeSmallChunks[s div MemAlign], c) + inc(c.free, s) + else: + inc(c.free, s) + if c.free == SmallChunkSize-smallChunkOverhead(): + listRemove(a.freeSmallChunks[s div MemAlign], c) + c.size = SmallChunkSize + freeBigChunk(a, cast[PBigChunk](c)) else: - inc(c.free, s) - if c.free == SmallChunkSize-smallChunkOverhead(): - listRemove(a.freeSmallChunks[s div MemAlign], c) - c.size = SmallChunkSize - freeBigChunk(a, cast[PBigChunk](c)) - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% + when defined(gcDestructors): + addToSharedFreeList(c, f, s div MemAlign) + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 2") else: # set to 0xff to check for usage after free bugs: when overwriteFree: nimSetMem(p, -1'i32, c.size -% bigChunkOverhead()) - # free big chunk - var c = cast[PBigChunk](c) - dec a.occ, c.size - untrackSize(c.size) - sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" - a.deleted = getBottom(a) - del(a, a.root, cast[int](addr(c.data))) - if c.size >= HugeChunkSize: freeHugeChunk(a, c) - else: freeBigChunk(a, c) + when defined(gcDestructors): + if c.owner == addr(a): + deallocBigChunk(a, cast[PBigChunk](c)) + else: + addToSharedFreeListBigChunks(c.owner[], cast[PBigChunk](c)) + else: + deallocBigChunk(a, cast[PBigChunk](c)) sysAssert(allocInv(a), "rawDealloc: end") when logAlloc: cprintf("dealloc(pointer_%p)\n", p) @@ -881,7 +977,7 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() result = (c.acc >% offset) and (offset %% c.size == 0) and (cast[ptr FreeCell](p).zeroField >% 1) @@ -899,12 +995,12 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() if c.acc >% offset: - sysAssert(cast[ByteAddress](addr(c.data)) +% offset == - cast[ByteAddress](p), "offset is not what you think it is") - var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +% + sysAssert(cast[int](addr(c.data)) +% offset == + cast[int](p), "offset is not what you think it is") + var d = cast[ptr FreeCell](cast[int](addr(c.data)) +% offset -% (offset %% c.size)) if d.zeroField >% 1: result = d @@ -931,7 +1027,7 @@ when not defined(gcDestructors): proc ptrSize(p: pointer): int = when not defined(gcDestructors): - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) var c = pageAddr(p) sysAssert(not chunkUnused(c), "ptrSize") result = c.size -% sizeof(FreeCell) @@ -949,7 +1045,7 @@ proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} = result = rawAlloc(allocator, size+sizeof(FreeCell)) cast[ptr FreeCell](result).zeroField = 1 # mark it as used sysAssert(not isAllocatedPtr(allocator, result), "alloc") - result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell)) + result = cast[pointer](cast[int](result) +% sizeof(FreeCell)) track("alloc", result, size) else: result = rawAlloc(allocator, size) @@ -961,7 +1057,7 @@ proc alloc0(allocator: var MemRegion, size: Natural): pointer = proc dealloc(allocator: var MemRegion, p: pointer) = when not defined(gcDestructors): sysAssert(p != nil, "dealloc: p is nil") - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) sysAssert(x != nil, "dealloc: x is nil") sysAssert(isAccessible(allocator, x), "is not accessible") sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") @@ -994,7 +1090,7 @@ proc deallocOsPages(a: var MemRegion) = let (p, size) = it.chunks[i] when defined(debugHeapLinks): cprintf("owner %p; dealloc A: %p size: %ld; next: %p\n", addr(a), - it, it.size, next) + it, size, next) sysAssert size >= PageSize, "origSize too small" osDeallocPages(p, size) it = next @@ -1022,7 +1118,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = result = interiorAllocatedPtr(allocator, p) proc isAllocatedPtr*(p: pointer): bool = - let p = cast[pointer](cast[ByteAddress](p)-%ByteAddress(sizeof(Cell))) + let p = cast[pointer](cast[int](p)-%ByteAddress(sizeof(Cell))) result = isAllocatedPtr(allocator, p) proc deallocOsPages = deallocOsPages(allocator) @@ -1042,7 +1138,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = proc realloc0Impl(p: pointer, oldSize, newSize: Natural): pointer = result = realloc(allocator, p, newSize) if newSize > oldSize: - zeroMem(cast[pointer](cast[int](result) + oldSize), newSize - oldSize) + zeroMem(cast[pointer](cast[uint](result) + uint(oldSize)), newSize - oldSize) when false: proc countFreeMem(): int = @@ -1052,25 +1148,34 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = inc(result, it.size) it = it.next - proc getFreeMem(): int = - result = allocator.freeMem - #sysAssert(result == countFreeMem()) + when hasThreadSupport and not defined(gcDestructors): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".} - proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem() - proc getMaxMem*(): int = return getMaxMem(allocator) + var sharedHeap: MemRegion + var heapLock: SysLock + initSysLock(heapLock) + addSysExitProc(proc() {.noconv.} = deinitSys(heapLock)) + + proc getFreeMem(): int = + #sysAssert(result == countFreeMem()) + result = allocator.freeMem + + proc getTotalMem(): int = + result = allocator.currMem + + proc getOccupiedMem(): int = + result = allocator.occ #getTotalMem() - getFreeMem() + + proc getMaxMem*(): int = + result = getMaxMem(allocator) when defined(nimTypeNames): proc getMemCounters*(): (int, int) = getMemCounters(allocator) # -------------------- shared heap region ---------------------------------- - when hasThreadSupport: - var sharedHeap: MemRegion - var heapLock: SysLock - initSysLock(heapLock) proc allocSharedImpl(size: Natural): pointer = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) result = alloc(sharedHeap, size) releaseSys(heapLock) @@ -1082,7 +1187,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = zeroMem(result, size) proc deallocSharedImpl(p: pointer) = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) dealloc(sharedHeap, p) releaseSys(heapLock) @@ -1090,7 +1195,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = deallocImpl(p) proc reallocSharedImpl(p: pointer, newSize: Natural): pointer = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) result = realloc(sharedHeap, p, newSize) releaseSys(heapLock) @@ -1098,7 +1203,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = result = reallocImpl(p, newSize) proc reallocShared0Impl(p: pointer, oldSize, newSize: Natural): pointer = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) result = realloc0(sharedHeap, p, oldSize, newSize) releaseSys(heapLock) @@ -1106,20 +1211,31 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = result = realloc0Impl(p, oldSize, newSize) when hasThreadSupport: - template sharedMemStatsShared(v: int) = - acquireSys(heapLock) - result = v - releaseSys(heapLock) + when defined(gcDestructors): + proc getFreeSharedMem(): int = + allocator.freeMem - proc getFreeSharedMem(): int = - sharedMemStatsShared(sharedHeap.freeMem) + proc getTotalSharedMem(): int = + allocator.currMem - proc getTotalSharedMem(): int = - sharedMemStatsShared(sharedHeap.currMem) + proc getOccupiedSharedMem(): int = + allocator.occ - proc getOccupiedSharedMem(): int = - sharedMemStatsShared(sharedHeap.occ) - #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) + else: + template sharedMemStatsShared(v: int) = + acquireSys(heapLock) + result = v + releaseSys(heapLock) + + proc getFreeSharedMem(): int = + sharedMemStatsShared(sharedHeap.freeMem) + + proc getTotalSharedMem(): int = + sharedMemStatsShared(sharedHeap.currMem) + + proc getOccupiedSharedMem(): int = + sharedMemStatsShared(sharedHeap.occ) + #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) {.pop.} {.pop.} diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 7e156eaab6..3098e17d69 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -31,7 +31,10 @@ proc c_abort*() {. importc: "abort", header: "<stdlib.h>", noSideEffect, noreturn.} -when defined(linux) and defined(amd64): +when defined(nimBuiltinSetjmp): + type + C_JmpBuf* = array[5, pointer] +elif defined(linux) and defined(amd64): type C_JmpBuf* {.importc: "jmp_buf", header: "<setjmp.h>", bycopy.} = object abi: array[200 div sizeof(clong), clong] @@ -62,7 +65,7 @@ elif defined(macosx) or defined(linux) or defined(freebsd) or SIGSEGV* = cint(11) SIGTERM* = cint(15) SIGPIPE* = cint(13) - SIG_DFL* = cast[CSighandlerT](0) + SIG_DFL* = CSighandlerT(nil) elif defined(haiku): const SIGABRT* = cint(6) @@ -72,9 +75,9 @@ elif defined(haiku): SIGSEGV* = cint(11) SIGTERM* = cint(15) SIGPIPE* = cint(7) - SIG_DFL* = cast[CSighandlerT](0) + SIG_DFL* = CSighandlerT(nil) else: - when NoFakeVars: + when defined(nimscript): {.error: "SIGABRT not ported to your platform".} else: var @@ -92,18 +95,58 @@ when defined(macosx): elif defined(haiku): const SIGBUS* = cint(30) -when defined(nimSigSetjmp) and not defined(nimStdSetjmp): +# "nimRawSetjmp" is defined by default for certain platforms, so we need the +# "nimStdSetjmp" escape hatch with it. +when defined(nimSigSetjmp): proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "<setjmp.h>", importc: "siglongjmp".} - template c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp*(jmpb: C_JmpBuf): cint = proc c_sigsetjmp(jmpb: C_JmpBuf, savemask: cint): cint {. header: "<setjmp.h>", importc: "sigsetjmp".} c_sigsetjmp(jmpb, 0) +elif defined(nimBuiltinSetjmp): + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) = + # Apple's Clang++ has trouble converting array names to pointers, so we need + # to be very explicit here. + proc c_builtin_longjmp(jmpb: ptr pointer, retval: cint) {. + importc: "__builtin_longjmp", nodecl.} + # The second parameter needs to be 1 and sometimes the C/C++ compiler checks it. + c_builtin_longjmp(unsafeAddr jmpb[0], 1) + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_builtin_setjmp(jmpb: ptr pointer): cint {. + importc: "__builtin_setjmp", nodecl.} + c_builtin_setjmp(unsafeAddr jmpb[0]) + elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. - header: "<setjmp.h>", importc: "_longjmp".} - proc c_setjmp*(jmpb: C_JmpBuf): cint {. - header: "<setjmp.h>", importc: "_setjmp".} + when defined(windows): + # No `_longjmp()` on Windows. + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "<setjmp.h>", importc: "longjmp".} + when defined(vcc) or defined(clangcl): + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "<setjmp.h>", importc: "setjmp".} + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + when defined(nimHasStyleChecks): + {.push styleChecks: off.} + + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. + header: "<setjmp.h>", importc: "_setjmp".} + c_setjmp_win(jmpb, nil) + + when defined(nimHasStyleChecks): + {.pop.} + else: + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "<setjmp.h>", importc: "_longjmp".} + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "<setjmp.h>", importc: "_setjmp".} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "<setjmp.h>", importc: "longjmp".} @@ -137,24 +180,44 @@ proc c_printf*(frmt: cstring): cint {. proc c_fputs*(c: cstring, f: CFilePtr): cint {. importc: "fputs", header: "<stdio.h>", discardable.} +proc c_fputc*(c: char, f: CFilePtr): cint {. + importc: "fputc", header: "<stdio.h>", discardable.} proc c_sprintf*(buf, frmt: cstring): cint {. importc: "sprintf", header: "<stdio.h>", varargs, noSideEffect.} # we use it only in a way that cannot lead to security issues -proc c_malloc*(size: csize_t): pointer {. - importc: "malloc", header: "<stdlib.h>".} -proc c_calloc*(nmemb, size: csize_t): pointer {. - importc: "calloc", header: "<stdlib.h>".} -proc c_free*(p: pointer) {. - importc: "free", header: "<stdlib.h>".} -proc c_realloc*(p: pointer, newsize: csize_t): pointer {. - importc: "realloc", header: "<stdlib.h>".} +proc c_snprintf*(buf: cstring, n: csize_t, frmt: cstring): cint {. + importc: "snprintf", header: "<stdio.h>", varargs, noSideEffect.} -proc c_fwrite*(buf: pointer, size, n: csize_t, f: CFilePtr): cint {. +when defined(zephyr) and not defined(zephyrUseLibcMalloc): + proc c_malloc*(size: csize_t): pointer {. + importc: "k_malloc", header: "<kernel.h>".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "k_calloc", header: "<kernel.h>".} + proc c_free*(p: pointer) {. + importc: "k_free", header: "<kernel.h>".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer = + # Zephyr's kernel malloc doesn't support realloc + result = c_malloc(newSize) + # match the ansi c behavior + if not result.isNil(): + copyMem(result, p, newSize) + c_free(p) +else: + proc c_malloc*(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "calloc", header: "<stdlib.h>".} + proc c_free*(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer {. + importc: "realloc", header: "<stdlib.h>".} + +proc c_fwrite*(buf: pointer, size, n: csize_t, f: CFilePtr): csize_t {. importc: "fwrite", header: "<stdio.h>".} -proc c_fflush(f: CFilePtr): cint {. +proc c_fflush*(f: CFilePtr): cint {. importc: "fflush", header: "<stdio.h>".} proc rawWriteString*(f: CFilePtr, s: cstring, length: int) {.compilerproc, nonReloadable, inline.} = @@ -164,7 +227,7 @@ proc rawWriteString*(f: CFilePtr, s: cstring, length: int) {.compilerproc, nonRe proc rawWrite*(f: CFilePtr, s: cstring) {.compilerproc, nonReloadable, inline.} = # we cannot throw an exception here! - discard c_fwrite(s, 1, cast[csize_t](s.len), f) + discard c_fwrite(s, 1, c_strlen(s), f) discard c_fflush(f) {.pop.} diff --git a/lib/system/arc.nim b/lib/system/arc.nim index ed5e9f5cae..d001fcaa5e 100644 --- a/lib/system/arc.nim +++ b/lib/system/arc.nim @@ -12,26 +12,6 @@ In this new runtime we simplify the object layouts a bit: The runtime type information is only accessed for the objects that have it and it's always at offset 0 then. The ``ref`` object header is independent from the runtime type and only contains a reference count. - -Object subtyping is checked via the generated 'name'. This should have -comparable overhead to the old pointer chasing approach but has the benefit -that it works across DLL boundaries. - -The generated name is a concatenation of the object names in the hierarchy -so that a subtype check becomes a substring check. For example:: - - type - ObjectA = object of RootObj - ObjectB = object of ObjectA - -ObjectA's ``name`` is "|ObjectA|RootObj|". -ObjectB's ``name`` is "|ObjectB|ObjectA|RootObj|". - -Now to check for ``x of ObjectB`` we need to check -for ``x.typ.name.hasSubstring("|ObjectB|")``. In the actual implementation, -however, we could also use a -hash of ``package & "." & module & "." & name`` to save space. - ]# when defined(gcOrc): @@ -46,6 +26,9 @@ else: rcMask = 0b111 rcShift = 3 # shift by rcShift to get the reference counter +const + orcLeakDetector = defined(nimOrcLeakDetector) + type RefHeader = object rc: int # the object header is now a single RC field. @@ -56,9 +39,21 @@ type # in O(1) without doubly linked lists when defined(nimArcDebug) or defined(nimArcIds): refId: int + when defined(gcOrc) and orcLeakDetector: + filename: cstring + line: int Cell = ptr RefHeader +template setFrameInfo(c: Cell) = + when orcLeakDetector: + if framePtr != nil and framePtr.prev != nil: + c.filename = framePtr.prev.filename + c.line = framePtr.prev.line + else: + c.filename = nil + c.line = 0 + template head(p: pointer): Cell = cast[Cell](cast[int](p) -% sizeof(RefHeader)) @@ -77,6 +72,21 @@ elif defined(nimArcIds): const traceId = -1 +when defined(gcAtomicArc) and hasThreadSupport: + template decrement(cell: Cell): untyped = + discard atomicDec(cell.rc, rcIncrement) + template increment(cell: Cell): untyped = + discard atomicInc(cell.rc, rcIncrement) + template count(x: Cell): untyped = + atomicLoadN(x.rc.addr, ATOMIC_ACQUIRE) shr rcShift +else: + template decrement(cell: Cell): untyped = + dec(cell.rc, rcIncrement) + template increment(cell: Cell): untyped = + inc(cell.rc, rcIncrement) + template count(x: Cell): untyped = + x.rc shr rcShift + proc nimNewObj(size, alignment: int): pointer {.compilerRtl.} = let hdrSize = align(sizeof(RefHeader), alignment) let s = size + hdrSize @@ -89,9 +99,10 @@ proc nimNewObj(size, alignment: int): pointer {.compilerRtl.} = atomicInc gRefId if head(result).refId == traceId: writeStackTrace() - cfprintf(cstderr, "[nimNewObj] %p %ld\n", result, head(result).rc shr rcShift) + cfprintf(cstderr, "[nimNewObj] %p %ld\n", result, head(result).count) when traceCollector: cprintf("[Allocated] %p result: %p\n", result -! sizeof(RefHeader), result) + setFrameInfo head(result) proc nimNewObjUninit(size, alignment: int): pointer {.compilerRtl.} = # Same as 'newNewObj' but do not initialize the memory to zero. @@ -110,21 +121,34 @@ proc nimNewObjUninit(size, alignment: int): pointer {.compilerRtl.} = atomicInc gRefId if head(result).refId == traceId: writeStackTrace() - cfprintf(cstderr, "[nimNewObjUninit] %p %ld\n", result, head(result).rc shr rcShift) + cfprintf(cstderr, "[nimNewObjUninit] %p %ld\n", result, head(result).count) when traceCollector: cprintf("[Allocated] %p result: %p\n", result -! sizeof(RefHeader), result) + setFrameInfo head(result) proc nimDecWeakRef(p: pointer) {.compilerRtl, inl.} = - dec head(p).rc, rcIncrement + decrement head(p) + +proc isUniqueRef*[T](x: ref T): bool {.inline.} = + ## Returns true if the object `x` points to is uniquely referenced. Such + ## an object can potentially be passed over to a different thread safely, + ## if great care is taken. This queries the internal reference count of + ## the object which is subject to lots of optimizations! In other words + ## the value of `isUniqueRef` can depend on the used compiler version and + ## optimizer setting. + ## Nevertheless it can be used as a very valuable debugging tool and can + ## be used to specify the constraints of a threading related API + ## via `assert isUniqueRef(x)`. + head(cast[pointer](x)).rc == 0 proc nimIncRef(p: pointer) {.compilerRtl, inl.} = when defined(nimArcDebug): if head(p).refId == traceId: writeStackTrace() - cfprintf(cstderr, "[IncRef] %p %ld\n", p, head(p).rc shr rcShift) + cfprintf(cstderr, "[IncRef] %p %ld\n", p, head(p).count) - inc head(p).rc, rcIncrement + increment head(p) when traceCollector: cprintf("[INCREF] %p\n", head(p)) @@ -154,7 +178,7 @@ proc nimRawDispose(p: pointer, alignment: int) {.compilerRtl.} = when defined(nimOwnedEnabled): if head(p).rc >= rcIncrement: cstderr.rawWrite "[FATAL] dangling references exist\n" - quit 1 + rawQuit 1 when defined(nimArcDebug): # we do NOT really free the memory here in order to reliably detect use-after-frees if freedCells.data == nil: init(freedCells) @@ -193,24 +217,29 @@ proc nimDecRefIsLast(p: pointer): bool {.compilerRtl, inl.} = when defined(nimArcDebug): if cell.refId == traceId: writeStackTrace() - cfprintf(cstderr, "[DecRef] %p %ld\n", p, cell.rc shr rcShift) + cfprintf(cstderr, "[DecRef] %p %ld\n", p, cell.count) - if (cell.rc and not rcMask) == 0: - result = true - when traceCollector: - cprintf("[ABOUT TO DESTROY] %p\n", cell) + when defined(gcAtomicArc) and hasThreadSupport: + # `atomicDec` returns the new value + if atomicDec(cell.rc, rcIncrement) == -rcIncrement: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) else: - dec cell.rc, rcIncrement - # According to Lins it's correct to do nothing else here. - when traceCollector: - cprintf("[DeCREF] %p\n", cell) + if cell.count == 0: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) + else: + decrement cell + # According to Lins it's correct to do nothing else here. + when traceCollector: + cprintf("[DECREF] %p\n", cell) proc GC_unref*[T](x: ref T) = ## New runtime only supports this operation for 'ref T'. - if nimDecRefIsLast(cast[pointer](x)): - # XXX this does NOT work for virtual destructors! - `=destroy`(x[]) - nimRawDispose(cast[pointer](x), T.alignOf) + var y {.cursor.} = x + `=destroy`(y) proc GC_ref*[T](x: ref T) = ## New runtime only supports this operation for 'ref T'. @@ -218,22 +247,21 @@ proc GC_ref*[T](x: ref T) = when not defined(gcOrc): template GC_fullCollect* = - ## Forces a full garbage collection pass. With `--gc:arc` a nop. + ## Forces a full garbage collection pass. With `--mm:arc` a nop. discard template setupForeignThreadGc* = - ## With `--gc:arc` a nop. + ## With `--mm:arc` a nop. discard template tearDownForeignThreadGc* = - ## With `--gc:arc` a nop. + ## With `--mm:arc` a nop. discard -proc isObj(obj: PNimTypeV2, subclass: cstring): bool {.compilerRtl, inl.} = - proc strstr(s, sub: cstring): cstring {.header: "<string.h>", importc.} +proc isObjDisplayCheck(source: PNimTypeV2, targetDepth: int16, token: uint32): bool {.compilerRtl, inl.} = + result = targetDepth <= source.depth and source.display[targetDepth] == token - result = strstr(obj.name, subclass) != nil - -proc chckObj(obj: PNimTypeV2, subclass: cstring) {.compilerRtl.} = - # checks if obj is of type subclass: - if not isObj(obj, subclass): sysFatal(ObjectConversionDefect, "invalid object conversion") +when defined(gcDestructors): + proc nimGetVTable(p: pointer, index: int): pointer + {.compilerRtl, inline, raises: [].} = + result = cast[ptr PNimTypeV2](p).vTable[index] diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim deleted file mode 100644 index 158f401779..0000000000 --- a/lib/system/arithm.nim +++ /dev/null @@ -1,425 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# simple integer arithmetic with overflow checking - -proc raiseOverflow {.compilerproc, noinline.} = - # a single proc to reduce code size to a minimum - sysFatal(OverflowDefect, "over- or underflow") - -proc raiseDivByZero {.compilerproc, noinline.} = - sysFatal(DivByZeroDefect, "division by zero") - -when defined(builtinOverflow): - # Builtin compiler functions for improved performance - when sizeof(clong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddl_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubl_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smull_overflow", nodecl, nosideeffect.} - - elif sizeof(clonglong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddll_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubll_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smulll_overflow", nodecl, nosideeffect.} - - when sizeof(int) == 8: - proc addIntOverflow(a, b: int, c: var int): bool {.inline.} = - addInt64Overflow(a, b, c) - - proc subIntOverflow(a, b: int, c: var int): bool {.inline.} = - subInt64Overflow(a, b, c) - - proc mulIntOverflow(a, b: int, c: var int): bool {.inline.} = - mulInt64Overflow(a, b, c) - - elif sizeof(int) == 4 and sizeof(cint) == 4: - proc addIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_sadd_overflow", nodecl, nosideeffect.} - - proc subIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_ssub_overflow", nodecl, nosideeffect.} - - proc mulIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_smul_overflow", nodecl, nosideeffect.} - - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - if addInt64Overflow(a, b, result): - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - if subInt64Overflow(a, b, result): - raiseOverflow() - - proc mulInt64(a, b: int64): int64 {.compilerproc, inline.} = - if mulInt64Overflow(a, b, result): - raiseOverflow() -else: - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= int64(0) or (result xor b) >= int64(0): - return result - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= int64(0) or (result xor not b) >= int64(0): - return result - raiseOverflow() - - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if they're - # approximately the same. Else, because the native int product is the only - # one that can lose catastrophic amounts of information, it's the native int - # product that must have overflowed. - # - proc mulInt64(a, b: int64): int64 {.compilerproc.} = - var - resAsFloat, floatProd: float64 - result = a *% b - floatProd = toBiggestFloat(a) # conversion - floatProd = floatProd * toBiggestFloat(b) - resAsFloat = toBiggestFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -proc negInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): return -a - raiseOverflow() - -proc absInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): - if a >= 0: return a - else: return -a - raiseOverflow() - -proc divInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - if a == low(int64) and b == int64(-1): - raiseOverflow() - return a div b - -proc modInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - return a mod b - -proc absInt(a: int): int {.compilerproc, inline.} = - if a != low(int): - if a >= 0: return a - else: return -a - raiseOverflow() - -const - asmVersion = defined(i386) and (defined(vcc) or defined(wcc) or - defined(dmc) or defined(gcc) or defined(llvm_gcc)) - # my Version of Borland C++Builder does not have - # tasm32, which is needed for assembler blocks - # this is why Borland is not included in the 'when' - -when asmVersion and not defined(gcc) and not defined(llvm_gcc): - # assembler optimized versions for compilers that - # have an intel syntax assembler: - proc addInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - # a in eax, and b in edx - asm """ - mov eax, ecx - add eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc subInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - sub eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc negInt(a: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - neg eax - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc divInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - ret - """ - - proc modInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - mov eax, edx - ret - """ - - proc mulInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - imul ecx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - -elif false: # asmVersion and (defined(gcc) or defined(llvm_gcc)): - proc addInt(a, b: int): int {.compilerproc, inline.} = - # don't use a pure proc here! - asm """ - "addl %%ecx, %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - #".intel_syntax noprefix" - #/* Intel syntax here */ - #".att_syntax" - - proc subInt(a, b: int): int {.compilerproc, inline.} = - asm """ "subl %%ecx,%%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - - proc mulInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "imull %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc negInt(a: int): int {.compilerproc, inline.} = - asm """ "negl %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`) - """ - - proc divInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc modInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - "movl %%edx, %%eax" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - -when not declared(addInt) and defined(builtinOverflow): - proc addInt(a, b: int): int {.compilerproc, inline.} = - if addIntOverflow(a, b, result): - raiseOverflow() - -when not declared(subInt) and defined(builtinOverflow): - proc subInt(a, b: int): int {.compilerproc, inline.} = - if subIntOverflow(a, b, result): - raiseOverflow() - -when not declared(mulInt) and defined(builtinOverflow): - proc mulInt(a, b: int): int {.compilerproc, inline.} = - if mulIntOverflow(a, b, result): - raiseOverflow() - -# Platform independent versions of the above (slower!) -when not declared(addInt): - proc addInt(a, b: int): int {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= 0 or (result xor b) >= 0: - return result - raiseOverflow() - -when not declared(subInt): - proc subInt(a, b: int): int {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= 0 or (result xor not b) >= 0: - return result - raiseOverflow() - -when not declared(negInt): - proc negInt(a: int): int {.compilerproc, inline.} = - if a != low(int): return -a - raiseOverflow() - -when not declared(divInt): - proc divInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - if a == low(int) and b == -1: - raiseOverflow() - return a div b - -when not declared(modInt): - proc modInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - return a mod b - -when not declared(mulInt): - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if - # they're approximately the same. Else, because the native int product is - # the only one that can lose catastrophic amounts of information, it's the - # native int product that must have overflowed. - # - proc mulInt(a, b: int): int {.compilerproc.} = - var - resAsFloat, floatProd: float - - result = a *% b - floatProd = toFloat(a) * toFloat(b) - resAsFloat = toFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -# We avoid setting the FPU control word here for compatibility with libraries -# written in other languages. - -proc raiseFloatInvalidOp {.compilerproc, noinline.} = - sysFatal(FloatInvalidOpDefect, "FPU operation caused a NaN result") - -proc nanCheck(x: float64) {.compilerproc, inline.} = - if x != x: raiseFloatInvalidOp() - -proc raiseFloatOverflow(x: float64) {.compilerproc, noinline.} = - if x > 0.0: - sysFatal(FloatOverflowDefect, "FPU operation caused an overflow") - else: - sysFatal(FloatUnderflowDefect, "FPU operations caused an underflow") - -proc infCheck(x: float64) {.compilerproc, inline.} = - if x != 0.0 and x*0.5 == x: raiseFloatOverflow(x) diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index 0dd329495f..fa7ca784d7 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -1,4 +1,4 @@ -proc succ*[T: Ordinal](x: T, y = 1): T {.magic: "Succ", noSideEffect.} = +proc succ*[T: Ordinal](x: T, y: int = 1): T {.magic: "Succ", noSideEffect.} = ## Returns the `y`-th successor (default: 1) of the value `x`. ## ## If such a value does not exist, `OverflowDefect` is raised @@ -7,7 +7,7 @@ proc succ*[T: Ordinal](x: T, y = 1): T {.magic: "Succ", noSideEffect.} = assert succ(5) == 6 assert succ(5, 3) == 8 -proc pred*[T: Ordinal](x: T, y = 1): T {.magic: "Pred", noSideEffect.} = +proc pred*[T: Ordinal](x: T, y: int = 1): T {.magic: "Pred", noSideEffect.} = ## Returns the `y`-th predecessor (default: 1) of the value `x`. ## ## If such a value does not exist, `OverflowDefect` is raised @@ -16,7 +16,7 @@ proc pred*[T: Ordinal](x: T, y = 1): T {.magic: "Pred", noSideEffect.} = assert pred(5) == 4 assert pred(5, 3) == 2 -proc inc*[T: Ordinal](x: var T, y = 1) {.magic: "Inc", noSideEffect.} = +proc inc*[T: Ordinal](x: var T, y: int = 1) {.magic: "Inc", noSideEffect.} = ## Increments the ordinal `x` by `y`. ## ## If such a value does not exist, `OverflowDefect` is raised or a compile @@ -28,7 +28,7 @@ proc inc*[T: Ordinal](x: var T, y = 1) {.magic: "Inc", noSideEffect.} = inc(i, 3) assert i == 6 -proc dec*[T: Ordinal](x: var T, y = 1) {.magic: "Dec", noSideEffect.} = +proc dec*[T: Ordinal](x: var T, y: int = 1) {.magic: "Dec", noSideEffect.} = ## Decrements the ordinal `x` by `y`. ## ## If such a value does not exist, `OverflowDefect` is raised or a compile @@ -45,109 +45,6 @@ proc dec*[T: Ordinal](x: var T, y = 1) {.magic: "Dec", noSideEffect.} = # -------------------------------------------------------------------------- # built-in operators -when defined(nimNoZeroExtendMagic): - proc ze*(x: int8): int {.deprecated.} = - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint8](x))) - - proc ze*(x: int16): int {.deprecated.} = - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint16](x))) - - proc ze64*(x: int8): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint8](x))) - - proc ze64*(x: int16): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint16](x))) - - proc ze64*(x: int32): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint32](x))) - - proc ze64*(x: int): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. Does nothing if the size of an `int` is the same as `int64`. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint](x))) - - proc toU8*(x: int): int8 {.deprecated.} = - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int8](x) - - proc toU16*(x: int): int16 {.deprecated.} = - ## treats `x` as unsigned and converts it to an `int16` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int16](x) - - proc toU32*(x: int64): int32 {.deprecated.} = - ## treats `x` as unsigned and converts it to an `int32` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int32](x) - -elif not defined(js): - proc ze*(x: int8): int {.magic: "Ze8ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze*(x: int16): int {.magic: "Ze16ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int8): int64 {.magic: "Ze8ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int16): int64 {.magic: "Ze16ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int32): int64 {.magic: "Ze32ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int): int64 {.magic: "ZeIToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. Does nothing if the size of an `int` is the same as `int64`. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU8*(x: int): int8 {.magic: "ToU8", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU16*(x: int): int16 {.magic: "ToU16", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an `int16` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU32*(x: int64): int32 {.magic: "ToU32", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an `int32` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - # integer calculations: proc `+`*(x: int): int {.magic: "UnaryPlusI", noSideEffect.} ## Unary `+` operator for an integer. Has no effect. @@ -196,7 +93,7 @@ proc `*`*(x, y: int16): int16 {.magic: "MulI", noSideEffect.} proc `*`*(x, y: int32): int32 {.magic: "MulI", noSideEffect.} proc `*`*(x, y: int64): int64 {.magic: "MulI", noSideEffect.} -proc `div`*(x, y: int): int {.magic: "DivI", noSideEffect.} = +proc `div`*(x, y: int): int {.magic: "DivI", noSideEffect.} = ## Computes the integer division. ## ## This is roughly the same as `math.trunc(x/y).int`. @@ -399,6 +296,59 @@ proc `mod`*(x, y: uint16): uint16 {.magic: "ModU", noSideEffect.} proc `mod`*(x, y: uint32): uint32 {.magic: "ModU", noSideEffect.} proc `mod`*(x, y: uint64): uint64 {.magic: "ModU", noSideEffect.} +proc `+=`*[T: SomeInteger](x: var T, y: T) {. + magic: "Inc", noSideEffect.} + ## Increments an integer. + +proc `-=`*[T: SomeInteger](x: var T, y: T) {. + magic: "Dec", noSideEffect.} + ## Decrements an integer. + +proc `*=`*[T: SomeInteger](x: var T, y: T) {. + inline, noSideEffect.} = + ## Binary `*=` operator for integers. + x = x * y + +# floating point operations: +proc `+`*(x: float32): float32 {.magic: "UnaryPlusF64", noSideEffect.} +proc `-`*(x: float32): float32 {.magic: "UnaryMinusF64", noSideEffect.} +proc `+`*(x, y: float32): float32 {.magic: "AddF64", noSideEffect.} +proc `-`*(x, y: float32): float32 {.magic: "SubF64", noSideEffect.} +proc `*`*(x, y: float32): float32 {.magic: "MulF64", noSideEffect.} +proc `/`*(x, y: float32): float32 {.magic: "DivF64", noSideEffect.} + +proc `+`*(x: float): float {.magic: "UnaryPlusF64", noSideEffect.} +proc `-`*(x: float): float {.magic: "UnaryMinusF64", noSideEffect.} +proc `+`*(x, y: float): float {.magic: "AddF64", noSideEffect.} +proc `-`*(x, y: float): float {.magic: "SubF64", noSideEffect.} +proc `*`*(x, y: float): float {.magic: "MulF64", noSideEffect.} +proc `/`*(x, y: float): float {.magic: "DivF64", noSideEffect.} + +proc `+=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Increments in place a floating point number. + x = x + y + +proc `-=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Decrements in place a floating point number. + x = x - y + +proc `*=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Multiplies in place a floating point number. + x = x * y + +proc `/=`*(x: var float64, y: float64) {.inline, noSideEffect.} = + ## Divides in place a floating point number. + x = x / y + +proc `/=`*[T: float|float32](x: var T, y: T) {.inline, noSideEffect.} = + ## Divides in place a floating point number. + x = x / y + +# the following have to be included in system, not imported for some reason: + proc `+%`*(x, y: int): int {.inline.} = ## Treats `x` and `y` as unsigned and adds them. ## @@ -453,16 +403,3 @@ proc `%%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) mod cast[u proc `%%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) mod cast[uint16](y)) proc `%%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) mod cast[uint32](y)) proc `%%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) mod cast[uint64](y)) - -proc `+=`*[T: SomeInteger](x: var T, y: T) {. - magic: "Inc", noSideEffect.} - ## Increments an integer. - -proc `-=`*[T: SomeInteger](x: var T, y: T) {. - magic: "Dec", noSideEffect.} - ## Decrements an integer. - -proc `*=`*[T: SomeInteger](x: var T, y: T) {. - inline, noSideEffect.} = - ## Binary `*=` operator for integers. - x = x * y diff --git a/lib/system/assign.nim b/lib/system/assign.nim index faf16fc902..9f4cbc0feb 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -15,8 +15,8 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) {.benign. proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, shallow: bool) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericAssignAux(cast[pointer](d +% n.offset), @@ -56,8 +56,8 @@ template deepSeqAssignImpl(operation, additionalArg) {.dirty.} = proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericAssignAux 2") case mt.kind of tyString: @@ -89,17 +89,17 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var ss = nimNewSeqOfCap(mt, seq.len) cast[PGenericSeq](ss).len = seq.len unsureAsgnRef(x, ss) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) copyMem(cast[pointer](dst +% align(GenericSeqSize, mt.base.align)), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align)), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align)), seq.len *% mt.base.size) else: unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericAssignAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), mt.base, shallow) of tyObject: var it = mt.base @@ -167,7 +167,7 @@ when false: of tyPointer: k = "range" of tyOpenArray: k = "openarray" of tyString: k = "string" - of tyCString: k = "cstring" + of tyCstring: k = "cstring" of tyInt: k = "int" of tyInt32: k = "int32" else: k = "other" @@ -181,15 +181,15 @@ proc genericSeqAssign(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericAssignOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericAssign(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) proc objectInit(dest: pointer, typ: PNimType) {.compilerproc, benign.} proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "objectInitAux") of nkSlot: objectInit(cast[pointer](d +% n.offset), n.typ) @@ -203,7 +203,7 @@ proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = proc objectInit(dest: pointer, typ: PNimType) = # the generic init proc that takes care of initialization of complex # objects on the stack or heap - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case typ.kind of tyObject: # iterate over any structural type @@ -226,7 +226,7 @@ proc objectInit(dest: pointer, typ: PNimType) = proc genericReset(dest: pointer, mt: PNimType) {.compilerproc, benign.} proc genericResetAux(dest: pointer, n: ptr TNimNode) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "genericResetAux") of nkSlot: genericReset(cast[pointer](d +% n.offset), n.typ) @@ -238,7 +238,7 @@ proc genericResetAux(dest: pointer, n: ptr TNimNode) = zeroMem(cast[pointer](d +% n.offset), n.typ.size) proc genericReset(dest: pointer, mt: PNimType) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) sysAssert(mt != nil, "genericReset 2") case mt.kind of tyRef: @@ -275,10 +275,12 @@ proc genericReset(dest: pointer, mt: PNimType) = proc selectBranch(discVal, L: int, a: ptr array[0x7fff, ptr TNimNode]): ptr TNimNode = - result = a[L] # a[L] contains the ``else`` part (but may be nil) if discVal <% L: - let x = a[discVal] - if x != nil: result = x + result = a[discVal] + if result == nil: + result = a[L] + else: + result = a[L] # a[L] contains the ``else`` part (but may be nil) proc FieldDiscriminantCheck(oldDiscVal, newDiscVal: int, a: ptr array[0x7fff, ptr TNimNode], diff --git a/lib/system/basic_types.nim b/lib/system/basic_types.nim index 7779e1ce90..bf81b9b6a3 100644 --- a/lib/system/basic_types.nim +++ b/lib/system/basic_types.nim @@ -1,15 +1,45 @@ type - int* {.magic: "Int".} ## Default integer type; bitwidth depends on + int* {.magic: Int.} ## Default integer type; bitwidth depends on ## architecture, but is always the same as a pointer. - int8* {.magic: "Int8".} ## Signed 8 bit integer type. - int16* {.magic: "Int16".} ## Signed 16 bit integer type. - int32* {.magic: "Int32".} ## Signed 32 bit integer type. - int64* {.magic: "Int64".} ## Signed 64 bit integer type. - uint* {.magic: "UInt".} ## Unsigned default integer type. - uint8* {.magic: "UInt8".} ## Unsigned 8 bit integer type. - uint16* {.magic: "UInt16".} ## Unsigned 16 bit integer type. - uint32* {.magic: "UInt32".} ## Unsigned 32 bit integer type. - uint64* {.magic: "UInt64".} ## Unsigned 64 bit integer type. + int8* {.magic: Int8.} ## Signed 8 bit integer type. + int16* {.magic: Int16.} ## Signed 16 bit integer type. + int32* {.magic: Int32.} ## Signed 32 bit integer type. + int64* {.magic: Int64.} ## Signed 64 bit integer type. + uint* {.magic: UInt.} ## Unsigned default integer type. + uint8* {.magic: UInt8.} ## Unsigned 8 bit integer type. + uint16* {.magic: UInt16.} ## Unsigned 16 bit integer type. + uint32* {.magic: UInt32.} ## Unsigned 32 bit integer type. + uint64* {.magic: UInt64.} ## Unsigned 64 bit integer type. + +type + float* {.magic: Float.} ## Default floating point type. + float32* {.magic: Float32.} ## 32 bit floating point type. + float64* {.magic: Float.} ## 64 bit floating point type. + +# 'float64' is now an alias to 'float'; this solves many problems + +type + char* {.magic: Char.} ## Built-in 8 bit character type (unsigned). + string* {.magic: String.} ## Built-in string type. + cstring* {.magic: Cstring.} ## Built-in cstring (*compatible string*) type. + pointer* {.magic: Pointer.} ## Built-in pointer type, use the `addr` + ## operator to get a pointer to a variable. + + typedesc* {.magic: TypeDesc.} ## Meta type to denote a type description. + +type + `ptr`*[T] {.magic: Pointer.} ## Built-in generic untraced pointer type. + `ref`*[T] {.magic: Pointer.} ## Built-in generic traced pointer type. + + `nil` {.magic: "Nil".} + + void* {.magic: "VoidType".} ## Meta type to denote the absence of any type. + auto* {.magic: Expr.} ## Meta type for automatic type determination. + any* {.deprecated: "Deprecated since v1.5; Use auto instead.".} = distinct auto ## Deprecated; Use `auto` instead. See https://github.com/nim-lang/RFCs/issues/281 + untyped* {.magic: Expr.} ## Meta type to denote an expression that + ## is not resolved (for templates). + typed* {.magic: Stmt.} ## Meta type to denote an expression that + ## is resolved (for templates). type # we need to start a new type section here, so that ``0`` can have a type bool* {.magic: "Bool".} = enum ## Built-in boolean type. @@ -29,15 +59,16 @@ type SomeInteger* = SomeSignedInt|SomeUnsignedInt ## Type class matching all integer types. + SomeFloat* = float|float32|float64 + ## Type class matching all floating point number types. + + SomeNumber* = SomeInteger|SomeFloat + ## Type class matching all number types. + SomeOrdinal* = int|int8|int16|int32|int64|bool|enum|uint|uint8|uint16|uint32|uint64 ## Type class matching all ordinal types; however this includes enums with ## holes. See also `Ordinal` - BiggestInt* = int64 - ## is an alias for the biggest signed integer type the Nim compiler - ## supports. Currently this is `int64`, but it is platform-dependent - ## in general. - {.push warning[GcMem]: off, warning[Uninit]: off.} {.push hints: off.} diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim index d7c55a4d9f..0663247c28 100644 --- a/lib/system/bitmasks.nim +++ b/lib/system/bitmasks.nim @@ -10,13 +10,18 @@ # Page size of the system; in most cases 4096 bytes. For exotic OS or # CPU this needs to be changed: const - PageShift = when defined(cpu16): 8 else: 12 # \ - # my tests showed no improvements for using larger page sizes. + PageShift = when defined(nimPage256) or defined(cpu16): 3 + elif defined(nimPage512): 9 + elif defined(nimPage1k): 10 + else: 12 # \ # my tests showed no improvements for using larger page sizes. + PageSize = 1 shl PageShift PageMask = PageSize-1 + MemAlign = # also minimal allocatable memory block - when defined(useMalloc): + when defined(nimMemAlignTiny): 4 + elif defined(useMalloc): when defined(amd64): 16 else: 8 else: 16 diff --git a/lib/system/cellseqs_v1.nim b/lib/system/cellseqs_v1.nim index 1952491b37..1a305aa428 100644 --- a/lib/system/cellseqs_v1.nim +++ b/lib/system/cellseqs_v1.nim @@ -16,18 +16,21 @@ type d: PCellArray proc contains(s: CellSeq, c: PCell): bool {.inline.} = - for i in 0 .. s.len-1: - if s.d[i] == c: return true + for i in 0 ..< s.len: + if s.d[i] == c: + return true return false +proc resize(s: var CellSeq) = + s.cap = s.cap * 3 div 2 + let d = cast[PCellArray](alloc(s.cap * sizeof(PCell))) + copyMem(d, s.d, s.len * sizeof(PCell)) + dealloc(s.d) + s.d = d + proc add(s: var CellSeq, c: PCell) {.inline.} = if s.len >= s.cap: - s.cap = s.cap * 3 div 2 - var d = cast[PCellArray](alloc(s.cap * sizeof(PCell))) - copyMem(d, s.d, s.len * sizeof(PCell)) - dealloc(s.d) - s.d = d - # XXX: realloc? + resize(s) s.d[s.len] = c inc(s.len) diff --git a/lib/system/cellseqs_v2.nim b/lib/system/cellseqs_v2.nim index 59cf67dcdf..c6c7b1a8e2 100644 --- a/lib/system/cellseqs_v2.nim +++ b/lib/system/cellseqs_v2.nim @@ -10,38 +10,35 @@ # Cell seqs for cyclebreaker and cyclicrefs_v2. type - CellTuple = (PT, PNimTypeV2) - CellArray = ptr UncheckedArray[CellTuple] - CellSeq = object + CellTuple[T] = (T, PNimTypeV2) + CellArray[T] = ptr UncheckedArray[CellTuple[T]] + CellSeq[T] = object len, cap: int - d: CellArray + d: CellArray[T] -proc add(s: var CellSeq, c: PT; t: PNimTypeV2) {.inline.} = +proc resize[T](s: var CellSeq[T]) = + s.cap = s.cap * 3 div 2 + var newSize = s.cap * sizeof(CellTuple[T]) + when compileOption("threads"): + s.d = cast[CellArray[T]](reallocShared(s.d, newSize)) + else: + s.d = cast[CellArray[T]](realloc(s.d, newSize)) + +proc add[T](s: var CellSeq[T], c: T, t: PNimTypeV2) {.inline.} = if s.len >= s.cap: - s.cap = s.cap * 3 div 2 - when compileOption("threads"): - var d = cast[CellArray](allocShared(uint(s.cap * sizeof(CellTuple)))) - else: - var d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) - copyMem(d, s.d, s.len * sizeof(CellTuple)) - when compileOption("threads"): - deallocShared(s.d) - else: - dealloc(s.d) - s.d = d - # XXX: realloc? + s.resize() s.d[s.len] = (c, t) inc(s.len) -proc init(s: var CellSeq, cap: int = 1024) = +proc init[T](s: var CellSeq[T], cap: int = 1024) = s.len = 0 s.cap = cap when compileOption("threads"): - s.d = cast[CellArray](allocShared(uint(s.cap * sizeof(CellTuple)))) + s.d = cast[CellArray[T]](allocShared(uint(s.cap * sizeof(CellTuple[T])))) else: - s.d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) + s.d = cast[CellArray[T]](alloc(s.cap * sizeof(CellTuple[T]))) -proc deinit(s: var CellSeq) = +proc deinit[T](s: var CellSeq[T]) = if s.d != nil: when compileOption("threads"): deallocShared(s.d) @@ -51,6 +48,6 @@ proc deinit(s: var CellSeq) = s.len = 0 s.cap = 0 -proc pop(s: var CellSeq): (PT, PNimTypeV2) = +proc pop[T](s: var CellSeq[T]): (T, PNimTypeV2) = result = s.d[s.len-1] dec s.len diff --git a/lib/system/cellsets.nim b/lib/system/cellsets.nim index ea00176b53..92036c2262 100644 --- a/lib/system/cellsets.nim +++ b/lib/system/cellsets.nim @@ -7,13 +7,47 @@ # distribution, for details about the copyright. # -# Efficient set of pointers for the GC (and repr) -when defined(gcOrc) or defined(gcArc): +#[ + +Efficient set of pointers for the GC (and repr) +----------------------------------------------- + +The GC depends on an extremely efficient datastructure for storing a +set of pointers - this is called a `CellSet` in the source code. +Inserting, deleting and searching are done in constant time. However, +modifying a `CellSet` during traversal leads to undefined behaviour. + +All operations on a CellSet have to perform efficiently. Because a Cellset can +become huge a hash table alone is not suitable for this. + +We use a mixture of bitset and hash table for this. The hash table maps *pages* +to a page descriptor. The page descriptor contains a bit for any possible cell +address within this page. So including a cell is done as follows: + +- Find the page descriptor for the page the cell belongs to. +- Set the appropriate bit in the page descriptor indicating that the + cell points to the start of a memory block. + +Removing a cell is analogous - the bit has to be set to zero. +Single page descriptors are never deleted from the hash table. This is not +needed as the data structures needs to be rebuilt periodically anyway. + +Complete traversal is done in this way:: + + for each page descriptor d: + for each bit in d: + if bit == 1: + traverse the pointer belonging to this bit + +]# + +when defined(gcOrc) or defined(gcArc) or defined(gcAtomicArc): type PCell = Cell - include bitmasks + when not declaredInScope(PageShift): + include bitmasks else: type @@ -44,7 +78,7 @@ type head: PPageDesc data: PPageDescArray -when defined(gcOrc) or defined(gcArc): +when defined(gcOrc) or defined(gcArc) or defined(gcAtomicArc): discard else: include cellseqs_v1 diff --git a/lib/system/cgprocs.nim b/lib/system/cgprocs.nim index 9d0d248c39..9a7645f9bf 100644 --- a/lib/system/cgprocs.nim +++ b/lib/system/cgprocs.nim @@ -8,13 +8,3 @@ # # Headers for procs that the code generator depends on ("compilerprocs") - -type - LibHandle = pointer # private type - ProcAddr = pointer # library loading and loading of procs: - -proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} -proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} -proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} - -proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} diff --git a/lib/system/channels_builtin.nim b/lib/system/channels_builtin.nim index c7a445766f..02b4d8cbfd 100644 --- a/lib/system/channels_builtin.nim +++ b/lib/system/channels_builtin.nim @@ -26,7 +26,7 @@ ## The following is a simple example of two different ways to use channels: ## blocking and non-blocking. ## -## .. code-block :: Nim +## ```Nim ## # Be sure to compile with --threads:on. ## # The channels and threads modules are part of system and should not be ## # imported. @@ -87,18 +87,20 @@ ## ## # Clean up the channel. ## chan.close() +## ``` ## ## Sample output ## ------------- ## The program should output something similar to this, but keep in mind that -## exact results may vary in the real world:: -## Hello World! -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Another message +## exact results may vary in the real world: +## +## Hello World! +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Another message ## ## Passing Channels Safely ## ----------------------- @@ -112,7 +114,7 @@ ## using e.g. `system.allocShared0` and pass these pointers through thread ## arguments: ## -## .. code-block :: Nim +## ```Nim ## proc worker(channel: ptr Channel[string]) = ## let greeting = channel[].recv() ## echo greeting @@ -134,10 +136,13 @@ ## deallocShared(channel) ## ## localChannelExample() # "Hello from the main thread!" +## ``` when not declared(ThisIsSystem): {.error: "You must not import this module explicitly".} +import std/private/syslocks + type pbytes = ptr UncheckedArray[byte] RawChannel {.pure, final.} = object ## msg queue for a thread @@ -182,8 +187,8 @@ when not usesDestructors: proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, mode: LoadStoreMode) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: storeAux(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), n.typ, t, mode) @@ -202,8 +207,8 @@ when not usesDestructors: cast[pointer](cast[int](p) +% x) var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "mt == nil") case mt.kind of tyString: @@ -242,14 +247,14 @@ when not usesDestructors: x[] = alloc0(t.region, align(GenericSeqSize, mt.base.align) +% seq.len *% mt.base.size) else: unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) var dstseq = cast[PGenericSeq](dst) dstseq.len = seq.len dstseq.reserved = seq.len for i in 0..seq.len-1: storeAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), mt.base, t, mode) if mode != mStore: dealloc(t.region, s2) @@ -360,8 +365,8 @@ proc sendImpl(q: PRawChannel, typ: PNimType, msg: pointer, noBlock: bool): bool rawSend(q, msg, typ) q.elemType = typ - releaseSys(q.lock) signalSysCond(q.cond) + releaseSys(q.lock) result = true proc send*[TMsg](c: var Channel[TMsg], msg: sink TMsg) {.inline.} = @@ -389,7 +394,7 @@ proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) = q.ready = false if typ != q.elemType: releaseSys(q.lock) - sysFatal(ValueError, "cannot receive message of wrong type") + raise newException(ValueError, "cannot receive message of wrong type") rawRecv(q, res, typ) if q.maxItems > 0 and q.count == q.maxItems - 1: # Parent thread is awaiting in send. Wake it up. diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index e0a8bd78ed..b488559644 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -9,6 +9,8 @@ # Implementation of some runtime checks. include system/indexerrors +when defined(nimPreviewSlimSystem): + import std/formatfloat proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} = when hostOS == "standalone": @@ -16,6 +18,9 @@ proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} = else: sysFatal(RangeDefect, "value out of range: ", $val) +proc raiseIndexError4(l1, h1, h2: int) {.compilerproc, noinline.} = + sysFatal(IndexDefect, "index out of bounds: " & $l1 & ".." & $h1 & " notin 0.." & $(h2 - 1)) + proc raiseIndexError3(i, a, b: int) {.compilerproc, noinline.} = sysFatal(IndexDefect, formatErrorIndexBound(i, a, b)) @@ -26,13 +31,33 @@ proc raiseIndexError() {.compilerproc, noinline.} = sysFatal(IndexDefect, "index out of bounds") proc raiseFieldError(f: string) {.compilerproc, noinline.} = + ## remove after bootstrap > 1.5.1 sysFatal(FieldDefect, f) +when defined(nimV2): + proc raiseFieldError2(f: string, discVal: int) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, f & $discVal & "'") + + proc raiseFieldErrorStr(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, formatFieldDefect(f, discVal)) +else: + proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, formatFieldDefect(f, discVal)) + proc raiseRangeErrorI(i, a, b: BiggestInt) {.compilerproc, noinline.} = - sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) + when defined(standalone): + sysFatal(RangeDefect, "value out of range") + else: + sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) proc raiseRangeErrorF(i, a, b: float) {.compilerproc, noinline.} = - sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) + when defined(standalone): + sysFatal(RangeDefect, "value out of range") + else: + sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) proc raiseRangeErrorU(i, a, b: uint64) {.compilerproc, noinline.} = # todo: better error reporting diff --git a/lib/system/comparisons.nim b/lib/system/comparisons.nim index 2e5664a958..ce5e486f71 100644 --- a/lib/system/comparisons.nim +++ b/lib/system/comparisons.nim @@ -9,7 +9,7 @@ proc `==`*[Enum: enum](x, y: Enum): bool {.magic: "EqEnum", noSideEffect.} = place1, place2 = 3 var e1 = field1 - e2 = Enum1(place2) + e2 = place2.ord.Enum1 assert e1 == e2 assert not compiles(e1 == place2) # raises error proc `==`*(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} = @@ -35,7 +35,7 @@ proc `==`*[T](x, y: ref T): bool {.magic: "EqRef", noSideEffect.} ## Checks that two `ref` variables refer to the same item. proc `==`*[T](x, y: ptr T): bool {.magic: "EqRef", noSideEffect.} ## Checks that two `ptr` variables refer to the same item. -proc `==`*[T: proc](x, y: T): bool {.magic: "EqProc", noSideEffect.} +proc `==`*[T: proc | iterator](x, y: T): bool {.magic: "EqProc", noSideEffect.} ## Checks that two `proc` variables refer to the same procedure. proc `<=`*[Enum: enum](x, y: Enum): bool {.magic: "LeEnum", noSideEffect.} @@ -125,15 +125,18 @@ proc `<`*[T](x, y: ref T): bool {.magic: "LtPtr", noSideEffect.} proc `<`*[T](x, y: ptr T): bool {.magic: "LtPtr", noSideEffect.} proc `<`*(x, y: pointer): bool {.magic: "LtPtr", noSideEffect.} -template `!=`*(x, y: untyped): untyped = +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template `!=`*(x, y: untyped): untyped {.callsite.} = ## Unequals operator. This is a shorthand for `not (x == y)`. not (x == y) -template `>=`*(x, y: untyped): untyped = +template `>=`*(x, y: untyped): untyped {.callsite.} = ## "is greater or equals" operator. This is the same as `y <= x`. y <= x -template `>`*(x, y: untyped): untyped = +template `>`*(x, y: untyped): untyped {.callsite.} = ## "is greater" operator. This is the same as `y < x`. y < x @@ -206,6 +209,14 @@ proc `==`*(x, y: uint16): bool {.magic: "EqI", noSideEffect.} proc `==`*(x, y: uint32): bool {.magic: "EqI", noSideEffect.} proc `==`*(x, y: uint64): bool {.magic: "EqI", noSideEffect.} +proc `<=`*(x, y: float32): bool {.magic: "LeF64", noSideEffect.} +proc `<=`*(x, y: float): bool {.magic: "LeF64", noSideEffect.} + +proc `<`*(x, y: float32): bool {.magic: "LtF64", noSideEffect.} +proc `<`*(x, y: float): bool {.magic: "LtF64", noSideEffect.} + +proc `==`*(x, y: float32): bool {.magic: "EqF64", noSideEffect.} +proc `==`*(x, y: float): bool {.magic: "EqF64", noSideEffect.} {.push stackTrace: off.} @@ -220,6 +231,13 @@ proc min*(x, y: int32): int32 {.magic: "MinI", noSideEffect.} = proc min*(x, y: int64): int64 {.magic: "MinI", noSideEffect.} = ## The minimum value of two integers. if x <= y: x else: y +proc min*(x, y: float32): float32 {.noSideEffect, inline.} = + if x <= y or y != y: x else: y +proc min*(x, y: float64): float64 {.noSideEffect, inline.} = + if x <= y or y != y: x else: y +proc min*[T: not SomeFloat](x, y: T): T {.inline.} = + ## Generic minimum operator of 2 values based on `<=`. + if x <= y: x else: y proc max*(x, y: int): int {.magic: "MaxI", noSideEffect.} = if y <= x: x else: y @@ -232,6 +250,13 @@ proc max*(x, y: int32): int32 {.magic: "MaxI", noSideEffect.} = proc max*(x, y: int64): int64 {.magic: "MaxI", noSideEffect.} = ## The maximum value of two integers. if y <= x: x else: y +proc max*(x, y: float32): float32 {.noSideEffect, inline.} = + if y <= x or y != y: x else: y +proc max*(x, y: float64): float64 {.noSideEffect, inline.} = + if y <= x or y != y: x else: y +proc max*[T: not SomeFloat](x, y: T): T {.inline.} = + ## Generic maximum operator of 2 values based on `<=`. + if y <= x: x else: y proc min*[T](x: openArray[T]): T = @@ -250,7 +275,7 @@ proc max*[T](x: openArray[T]): T = proc clamp*[T](x, a, b: T): T = - ## Limits the value `x` within the interval [a, b]. + ## Limits the value `x` within the interval \[a, b]. ## This proc is equivalent to but faster than `max(a, min(b, x))`. ## ## .. warning:: `a <= b` is assumed and will not be checked (currently). @@ -299,7 +324,7 @@ proc `==`*[T](x, y: seq[T]): bool {.noSideEffect.} = return true else: var sameObject = false - asm """`sameObject` = `x` === `y`""" + {.emit: """`sameObject` = `x` === `y`""".} if sameObject: return true if x.len != y.len: diff --git a/lib/system/compilation.nim b/lib/system/compilation.nim new file mode 100644 index 0000000000..cda16fae5c --- /dev/null +++ b/lib/system/compilation.nim @@ -0,0 +1,209 @@ +const + NimMajor* {.intdefine.}: int = 2 + ## is the major number of Nim's version. Example: + ## ```nim + ## when (NimMajor, NimMinor, NimPatch) >= (1, 3, 1): discard + ## ``` + # see also std/private/since + + NimMinor* {.intdefine.}: int = 1 + ## is the minor number of Nim's version. + ## Odd for devel, even for releases. + + NimPatch* {.intdefine.}: int = 1 + ## is the patch number of Nim's version. + ## Odd for devel, even for releases. + +{.push profiler: off.} +let nimvm* {.magic: "Nimvm", compileTime.}: bool = false + ## May be used only in `when` expression. + ## It is true in Nim VM context and false otherwise. +{.pop.} + +const + isMainModule* {.magic: "IsMainModule".}: bool = false + ## True only when accessed in the main module. This works thanks to + ## compiler magic. It is useful to embed testing code in a module. + + CompileDate* {.magic: "CompileDate".}: string = "0000-00-00" + ## The date (in UTC) of compilation as a string of the form + ## `YYYY-MM-DD`. This works thanks to compiler magic. + + CompileTime* {.magic: "CompileTime".}: string = "00:00:00" + ## The time (in UTC) of compilation as a string of the form + ## `HH:MM:SS`. This works thanks to compiler magic. + +proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## defined. + ## + ## `x` is an external symbol introduced through the compiler's + ## `-d:x switch <nimc.html#compiler-usage-compileminustime-symbols>`_ to enable + ## build time conditionals: + ## ```nim + ## when not defined(release): + ## # Do here programmer friendly expensive sanity checks. + ## # Put here the normal code + ## ``` + ## + ## See also: + ## * `compileOption <#compileOption,string>`_ for `on|off` options + ## * `compileOption <#compileOption,string,string>`_ for enum options + ## * `define pragmas <manual.html#implementation-specific-pragmas-compileminustime-define-pragmas>`_ + +proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared. `x` has to be an identifier or a qualified identifier. + ## + ## This can be used to check whether a library provides a certain + ## feature or not: + ## ```nim + ## when not declared(strutils.toUpper): + ## # provide our own toUpper proc here, because strutils is + ## # missing it. + ## ``` + ## + ## See also: + ## * `declaredInScope <#declaredInScope,untyped>`_ + +proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared in the current scope. `x` has to be an identifier. + +proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.} = + ## Special compile-time procedure that checks whether `x` can be compiled + ## without any semantic error. + ## This can be used to check whether a type supports some operation: + ## ```nim + ## when compiles(3 + 4): + ## echo "'+' for integers is available" + ## ``` + discard + +proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} + ## Converts the AST of `x` into a string representation. This is very useful + ## for debugging. + +proc runnableExamples*(rdoccmd = "", body: untyped) {.magic: "RunnableExamples".} = + ## A section you should use to mark `runnable example`:idx: code with. + ## + ## - In normal debug and release builds code within + ## a `runnableExamples` section is ignored. + ## - The documentation generator is aware of these examples and considers them + ## part of the `##` doc comment. As the last step of documentation + ## generation each runnableExample is put in its own file `$file_examples$i.nim`, + ## compiled and tested. The collected examples are + ## put into their own module to ensure the examples do not refer to + ## non-exported symbols. + runnableExamples: + proc timesTwo*(x: int): int = + ## This proc doubles a number. + runnableExamples: + # at module scope + const exported* = 123 + assert timesTwo(5) == 10 + block: # at block scope + defer: echo "done" + runnableExamples "-d:foo -b:cpp": + import std/compilesettings + assert querySetting(backend) == "cpp" + assert defined(foo) + runnableExamples "-r:off": ## this one is only compiled + import std/browsers + openDefaultBrowser "https://forum.nim-lang.org/" + 2 * x + +proc compileOption*(option: string): bool {. + magic: "CompileOption", noSideEffect.} = + ## Can be used to determine an `on|off` compile-time option. + ## + ## See also: + ## * `compileOption <#compileOption,string,string>`_ for enum options + ## * `defined <#defined,untyped>`_ + ## * `std/compilesettings module <compilesettings.html>`_ + runnableExamples("--floatChecks:off"): + static: doAssert not compileOption("floatchecks") + {.push floatChecks: on.} + static: doAssert compileOption("floatchecks") + # floating point NaN and Inf checks enabled in this scope + {.pop.} + +proc compileOption*(option, arg: string): bool {. + magic: "CompileOptionArg", noSideEffect.} = + ## Can be used to determine an enum compile-time option. + ## + ## See also: + ## * `compileOption <#compileOption,string>`_ for `on|off` options + ## * `defined <#defined,untyped>`_ + ## * `std/compilesettings module <compilesettings.html>`_ + runnableExamples: + when compileOption("opt", "size") and compileOption("gc", "boehm"): + discard "compiled with optimization for size and uses Boehm's GC" + +template currentSourcePath*: string = instantiationInfo(-1, true).filename + ## Returns the full file-system path of the current source. + ## + ## To get the directory containing the current source, use it with + ## `ospaths2.parentDir() <ospaths2.html#parentDir%2Cstring>`_ as + ## `currentSourcePath.parentDir()`. + ## + ## The path returned by this template is set at compile time. + ## + ## See the docstring of `macros.getProjectPath() <macros.html#getProjectPath>`_ + ## for an example to see the distinction between the `currentSourcePath()` + ## and `getProjectPath()`. + ## + ## See also: + ## * `ospaths2.getCurrentDir() proc <ospaths2.html#getCurrentDir>`_ + +proc slurp*(filename: string): string {.magic: "Slurp".} + ## This is an alias for `staticRead <#staticRead,string>`_. + +proc staticRead*(filename: string): string {.magic: "Slurp".} + ## Compile-time `readFile <syncio.html#readFile,string>`_ proc for easy + ## `resource`:idx: embedding: + ## + ## The maximum file size limit that `staticRead` and `slurp` can read is + ## near or equal to the *free* memory of the device you are using to compile. + ## ```nim + ## const myResource = staticRead"mydatafile.bin" + ## ``` + ## + ## `slurp <#slurp,string>`_ is an alias for `staticRead`. + +proc gorge*(command: string, input = "", cache = ""): string {. + magic: "StaticExec".} = discard + ## This is an alias for `staticExec <#staticExec,string,string,string>`_. + +proc staticExec*(command: string, input = "", cache = ""): string {. + magic: "StaticExec".} = discard + ## Executes an external process at compile-time and returns its text output + ## (stdout + stderr). + ## + ## If `input` is not an empty string, it will be passed as a standard input + ## to the executed program. + ## ```nim + ## const buildInfo = "Revision " & staticExec("git rev-parse HEAD") & + ## "\nCompiled on " & staticExec("uname -v") + ## ``` + ## + ## `gorge <#gorge,string,string,string>`_ is an alias for `staticExec`. + ## + ## Note that you can use this proc inside a pragma like + ## `passc <manual.html#implementation-specific-pragmas-passc-pragma>`_ or + ## `passl <manual.html#implementation-specific-pragmas-passl-pragma>`_. + ## + ## If `cache` is not empty, the results of `staticExec` are cached within + ## the `nimcache` directory. Use `--forceBuild` to get rid of this caching + ## behaviour then. `command & input & cache` (the concatenated string) is + ## used to determine whether the entry in the cache is still valid. You can + ## use versioning information for `cache`: + ## ```nim + ## const stateMachine = staticExec("dfaoptimizer", "input", "0.8.0") + ## ``` + +proc gorgeEx*(command: string, input = "", cache = ""): tuple[output: string, + exitCode: int] = + ## Similar to `gorge <#gorge,string,string,string>`_ but also returns the + ## precious exit code. + discard diff --git a/lib/system/coro_detection.nim b/lib/system/coro_detection.nim new file mode 100644 index 0000000000..f6c1b5c152 --- /dev/null +++ b/lib/system/coro_detection.nim @@ -0,0 +1,20 @@ +## Coroutine detection logic + +template coroutinesSupportedPlatform(): bool = + when defined(sparc) or defined(ELATE) or defined(boehmgc) or defined(gogc) or + defined(nogc) or defined(gcRegions) or defined(gcMarkAndSweep): + false + else: + true + +when defined(nimCoroutines): + # Explicit opt-in. + when not coroutinesSupportedPlatform(): + {.error: "Coroutines are not supported on this architecture and/or garbage collector.".} + const nimCoroutines* = true +elif defined(noNimCoroutines): + # Explicit opt-out. + const nimCoroutines* = false +else: + # Autodetect coroutine support. + const nimCoroutines* = false diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index 6c85612e25..34969cb328 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -9,17 +9,85 @@ ## Contains the used algorithms for counting bits. -proc countBits32*(n: uint32): int {.compilerproc.} = +from std/private/bitops_utils import forwardImpl, castToUnsigned + +const useBuiltins* = not defined(noIntrinsicsBitOpts) +const noUndefined* = defined(noUndefinedBitOpts) +const useGCC_builtins* = (defined(gcc) or defined(llvm_gcc) or + defined(clang)) and useBuiltins +const useICC_builtins* = defined(icc) and useBuiltins +const useVCC_builtins* = defined(vcc) and useBuiltins +const arch64* = sizeof(int) == 8 + +template countBitsImpl(n: uint32): int = # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel var v = uint32(n) v = v - ((v shr 1'u32) and 0x55555555'u32) v = (v and 0x33333333'u32) + ((v shr 2'u32) and 0x33333333'u32) - result = (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int + (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int -proc countBits64*(n: uint64): int {.compilerproc, inline.} = +template countBitsImpl(n: uint64): int = # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel var v = uint64(n) v = v - ((v shr 1'u64) and 0x5555555555555555'u64) v = (v and 0x3333333333333333'u64) + ((v shr 2'u64) and 0x3333333333333333'u64) v = (v + (v shr 4'u64) and 0x0F0F0F0F0F0F0F0F'u64) - result = ((v * 0x0101010101010101'u64) shr 56'u64).int + ((v * 0x0101010101010101'u64) shr 56'u64).int + + +when useGCC_builtins: + # Returns the number of set 1-bits in value. + proc builtin_popcount(x: cuint): cint {.importc: "__builtin_popcount", cdecl.} + proc builtin_popcountll(x: culonglong): cint {. + importc: "__builtin_popcountll", cdecl.} + +elif useVCC_builtins: + # Counts the number of one bits (population count) in a 16-, 32-, or 64-byte unsigned integer. + func builtin_popcnt16(a2: uint16): uint16 {. + importc: "__popcnt16", header: "<intrin.h>".} + func builtin_popcnt32(a2: uint32): uint32 {. + importc: "__popcnt", header: "<intrin.h>".} + func builtin_popcnt64(a2: uint64): uint64 {. + importc: "__popcnt64", header: "<intrin.h>".} + +elif useICC_builtins: + # Intel compiler intrinsics: http://fulla.fnal.gov/intel/compiler_c/main_cls/intref_cls/common/intref_allia_misc.htm + # see also: https://software.intel.com/en-us/node/523362 + # Count the number of bits set to 1 in an integer a, and return that count in dst. + func builtin_popcnt32(a: cint): cint {. + importc: "_popcnt", header: "<immintrin.h>".} + func builtin_popcnt64(a: uint64): cint {. + importc: "_popcnt64", header: "<immintrin.h>".} + + +func countSetBitsImpl*(x: SomeInteger): int {.inline.} = + ## Counts the set bits in an integer (also called `Hamming weight`:idx:). + # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. + # like GCC and MSVC + let x = x.castToUnsigned + when nimvm: + result = forwardImpl(countBitsImpl, x) + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int + else: result = builtin_popcountll(x.culonglong).int + elif useVCC_builtins: + when sizeof(x) <= 2: result = builtin_popcnt16(x.uint16).int + elif sizeof(x) <= 4: result = builtin_popcnt32(x.uint32).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).uint32).int + + builtin_popcnt32((x.uint64 shr 32'u64).uint32).int + elif useICC_builtins: + when sizeof(x) <= 4: result = builtin_popcnt32(x.cint).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).cint).int + + builtin_popcnt32((x.uint64 shr 32'u64).cint).int + else: + when sizeof(x) <= 4: result = countBitsImpl(x.uint32) + else: result = countBitsImpl(x.uint64) + +proc countBits32*(n: uint32): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) + +proc countBits64*(n: uint64): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) diff --git a/lib/system/ctypes.nim b/lib/system/ctypes.nim new file mode 100644 index 0000000000..b788274bd7 --- /dev/null +++ b/lib/system/ctypes.nim @@ -0,0 +1,84 @@ +## Some type definitions for compatibility between different +## backends and platforms. + +type + BiggestInt* = int64 + ## is an alias for the biggest signed integer type the Nim compiler + ## supports. Currently this is `int64`, but it is platform-dependent + ## in general. + + BiggestFloat* = float64 + ## is an alias for the biggest floating point type the Nim + ## compiler supports. Currently this is `float64`, but it is + ## platform-dependent in general. + + BiggestUInt* = uint64 + ## is an alias for the biggest unsigned integer type the Nim compiler + ## supports. Currently this is `uint64`, but it is platform-dependent + ## in general. + +when defined(windows): + type + clong* {.importc: "long", nodecl.} = int32 + ## This is the same as the type `long` in *C*. + culong* {.importc: "unsigned long", nodecl.} = uint32 + ## This is the same as the type `unsigned long` in *C*. +else: + type + clong* {.importc: "long", nodecl.} = int + ## This is the same as the type `long` in *C*. + culong* {.importc: "unsigned long", nodecl.} = uint + ## This is the same as the type `unsigned long` in *C*. + +type # these work for most platforms: + cchar* {.importc: "char", nodecl.} = char + ## This is the same as the type `char` in *C*. + cschar* {.importc: "signed char", nodecl.} = int8 + ## This is the same as the type `signed char` in *C*. + cshort* {.importc: "short", nodecl.} = int16 + ## This is the same as the type `short` in *C*. + cint* {.importc: "int", nodecl.} = int32 + ## This is the same as the type `int` in *C*. + csize_t* {.importc: "size_t", nodecl.} = uint + ## This is the same as the type `size_t` in *C*. + clonglong* {.importc: "long long", nodecl.} = int64 + ## This is the same as the type `long long` in *C*. + cfloat* {.importc: "float", nodecl.} = float32 + ## This is the same as the type `float` in *C*. + cdouble* {.importc: "double", nodecl.} = float64 + ## This is the same as the type `double` in *C*. + clongdouble* {.importc: "long double", nodecl.} = BiggestFloat + ## This is the same as the type `long double` in *C*. + ## This C type is not supported by Nim's code generator. + + cuchar* {.importc: "unsigned char", nodecl, deprecated: "use `char` or `uint8` instead".} = char + ## Deprecated: Use `uint8` instead. + cushort* {.importc: "unsigned short", nodecl.} = uint16 + ## This is the same as the type `unsigned short` in *C*. + cuint* {.importc: "unsigned int", nodecl.} = uint32 + ## This is the same as the type `unsigned int` in *C*. + culonglong* {.importc: "unsigned long long", nodecl.} = uint64 + ## This is the same as the type `unsigned long long` in *C*. + +type + ByteAddress* {.deprecated: "use `uint`".} = int + ## is the signed integer type that should be used for converting + ## pointers to integer addresses for readability. + + cstringArray* {.importc: "char**", nodecl.} = ptr UncheckedArray[cstring] + ## This is binary compatible to the type `char**` in *C*. The array's + ## high value is large enough to disable bounds checking in practice. + ## Use `cstringArrayToSeq proc <#cstringArrayToSeq,cstringArray,Natural>`_ + ## to convert it into a `seq[string]`. + +when not defined(nimPreviewSlimSystem): + # pollutes namespace + type + PFloat32* {.deprecated: "use `ptr float32`".} = ptr float32 + ## An alias for `ptr float32`. + PFloat64* {.deprecated: "use `ptr float64`".} = ptr float64 + ## An alias for `ptr float64`. + PInt64* {.deprecated: "use `ptr int64`".} = ptr int64 + ## An alias for `ptr int64`. + PInt32* {.deprecated: "use `ptr int32`".} = ptr int32 + ## An alias for `ptr int32`. diff --git a/lib/system/cyclebreaker.nim b/lib/system/cyclebreaker.nim index a3159f95cf..45b0a5a650 100644 --- a/lib/system/cyclebreaker.nim +++ b/lib/system/cyclebreaker.nim @@ -53,7 +53,6 @@ depth-first traversal suffices. ]# -type PT = ptr pointer include cellseqs_v2 const @@ -78,7 +77,7 @@ proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = discard type GcEnv = object - traceStack: CellSeq + traceStack: CellSeq[ptr pointer] proc trace(p: pointer; desc: PNimTypeV2; j: var GcEnv) {.inline.} = when false: @@ -139,7 +138,8 @@ proc breakCycles(s: Cell; desc: PNimTypeV2) = else: # anyhow as a link that the produced destructor does not have to follow: u[] = nil - cprintf("[Bug] %p %s RC %ld\n", t, desc.name, t.rc shr rcShift) + when traceCollector: + cprintf("[Bug] %p %s RC %ld\n", t, desc.name, t.rc shr rcShift) deinit j.traceStack proc thinout*[T](x: ref T) {.inline.} = @@ -149,7 +149,7 @@ proc thinout*[T](x: ref T) {.inline.} = ## and thus would keep the graph from being freed are `nil`'ed. ## This is a form of cycle collection that works well with Nim's ARC ## and its associated cost model. - proc getDynamicTypeInfo[T](x: T): PNimTypeV2 {.magic: "GetTypeInfoV2", noSideEffect, locks: 0.} + proc getDynamicTypeInfo[T](x: T): PNimTypeV2 {.magic: "GetTypeInfoV2", noSideEffect.} breakCycles(head(cast[pointer](x)), getDynamicTypeInfo(x[])) diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 1f30b8427c..72d35f5186 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -61,8 +61,8 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; tab: var PtrTable) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericDeepCopyAux(cast[pointer](d +% n.offset), @@ -85,8 +85,8 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericDeepCopyAux 2") case mt.kind of tyString: @@ -113,11 +113,11 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = return sysAssert(dest != nil, "genericDeepCopyAux 3") unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericDeepCopyAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), mt.base, tab) of tyObject: # we need to copy m_type field for tyObject, as it could be empty for @@ -199,8 +199,8 @@ proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericDeepCopyOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericDeepCopy(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index ce4e8e0cad..89a739d5a7 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -1,62 +1,38 @@ -proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. `$` is Nim's general way of - ## spelling `toString`:idx:. +## `$` is Nim's general way of spelling `toString`:idx:. +runnableExamples: + assert $0.1 == "0.1" + assert $(-2*3) == "-6" -template dollarImpl(x: uint | uint64, result: var string) = - type destTyp = typeof(x) - if x == 0: - result = "0" - else: - result = newString(60) - var i = 0 - var n = x - while n != 0: - let nn = n div destTyp(10) - result[i] = char(n - destTyp(10) * nn + ord('0')) - inc i - n = nn - result.setLen i +import std/private/[digitsutils, miscdollars] - let half = i div 2 - # Reverse - for t in 0 .. half-1: swap(result[t], result[i-t-1]) +when not defined(nimPreviewSlimSystem): + import std/formatfloat + export addFloat + func `$`*(x: float | float32): string = + ## Outplace version of `addFloat`. + result.addFloat(x) -when defined(js): - import std/private/since - since (1, 3): - proc `$`*(x: uint): string = - ## Caveat: currently implemented as $(cast[int](x)), tied to current - ## semantics of js' Number type. - # for c, see strmantle.`$` - when nimvm: - dollarImpl(x, result) - else: - result = $(int(x)) +proc `$`*(x: int): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) - proc `$`*(x: uint64): string = - ## Compatibility note: - ## the results may change in future releases if/when js target implements - ## 64bit ints. - # pending https://github.com/nim-lang/RFCs/issues/187 - when nimvm: - dollarImpl(x, result) - else: - result = $(cast[int](x)) -else: - proc `$`*(x: uint64): string {.noSideEffect, raises: [].} = - ## The stringify operator for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - dollarImpl(x, result) +proc `$`*(x: int64): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) -proc `$`*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. +proc `$`*(x: uint64): string {.raises: [].} = + ## Outplace version of `addInt`. + addInt(result, x) + +# same as old `ctfeWhitelist` behavior, whether or not this is a good idea. +template gen(T) = + # xxx simplify this by supporting this in compiler: int{lit} | uint64{lit} | int64{lit} + func `$`*(x: T{lit}): string {.compileTime.} = result.addInt(x) +gen(int) +gen(uint64) +gen(int64) -proc `$`*(x: float): string {.magic: "FloatToStr", noSideEffect.} - ## The stringify operator for a float argument. Returns `x` - ## converted to a decimal string. proc `$`*(x: bool): string {.magic: "BoolToStr", noSideEffect.} ## The stringify operator for a boolean argument. Returns `x` @@ -65,9 +41,9 @@ proc `$`*(x: bool): string {.magic: "BoolToStr", noSideEffect.} proc `$`*(x: char): string {.magic: "CharToStr", noSideEffect.} ## The stringify operator for a character argument. Returns `x` ## converted to a string. - ## - ## .. code-block:: Nim + ## ```Nim ## assert $'c' == "c" + ## ``` proc `$`*(x: cstring): string {.magic: "CStrToStr", noSideEffect.} ## The stringify operator for a CString argument. Returns `x` @@ -91,58 +67,25 @@ proc `$`*(t: typedesc): string {.magic: "TypeTrait".} ## For more procedures dealing with `typedesc`, see ## `typetraits module <typetraits.html>`_. ## - ## .. code-block:: Nim + ## ```Nim ## doAssert $(typeof(42)) == "int" ## doAssert $(typeof("Foo")) == "string" ## static: doAssert $(typeof(@['A', 'B'])) == "seq[char]" + ## ``` -when defined(nimHasIsNamedTuple): - proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -else: - # for bootstrap; remove after release 1.2 - proc isNamedTuple(T: typedesc): bool = - # Taken from typetraits. - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": - return compiles(t.Field0) - else: - return true - return false - - -proc `$`*[T: tuple|object](x: T): string = +proc `$`*[T: tuple](x: T): string = ## Generic `$` operator for tuples that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(23, 45) == "(23, 45)" ## $(a: 23, b: 45) == "(a: 23, b: 45)" ## $() == "()" - result = "(" - const isNamed = T is object or isNamedTuple(T) - var count = 0 - for name, value in fieldPairs(x): - if count > 0: result.add(", ") - when isNamed: - result.add(name) - result.add(": ") - count.inc - when compiles($value): - when value isnot string and value isnot seq and compiles(value.isNil): - if value.isNil: result.add "nil" - else: result.addQuoted(value) - else: - result.addQuoted(value) - else: - result.add("...") - when not isNamed: - if count == 1: - result.add(",") # $(1,) should print as the semantically legal (1,) - result.add(")") + ## ``` + tupleObjectDollar(result, x) +when not defined(nimPreviewSlimSystem): + import std/objectdollar + export objectdollar proc collectionToString[T](x: T, prefix, separator, suffix: string): string = result = prefix @@ -166,25 +109,25 @@ proc collectionToString[T](x: T, prefix, separator, suffix: string): string = proc `$`*[T](x: set[T]): string = ## Generic `$` operator for sets that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## ${23, 45} == "{23, 45}" + ## ``` collectionToString(x, "{", ", ", "}") proc `$`*[T](x: seq[T]): string = ## Generic `$` operator for seqs that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45]) == "@[23, 45]" + ## ``` collectionToString(x, "@[", ", ", "]") proc `$`*[T, U](x: HSlice[T, U]): string = ## Generic `$` operator for slices that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(1 .. 5) == "1 .. 5" + ## ``` result = $x.a result.add(" .. ") result.add($x.b) @@ -198,7 +141,7 @@ when not defined(nimNoArrayToString): proc `$`*[T](x: openArray[T]): string = ## Generic `$` operator for openarrays that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45].toOpenArray(0, 1)) == "[23, 45]" + ## ``` collectionToString(x, "[", ", ", "]") diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index b0f326bb5b..2162b234ff 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -47,14 +47,14 @@ proc nimLoadLibraryError(path: string) = copyMem(msg[msgIdx].addr, badExe.cstring, badExe.len) discard MessageBoxA(nil, msg[0].addr, nil, 0) cstderr.rawWrite("\n") - quit(1) + rawQuit(1) proc procAddrError(name: cstring) {.compilerproc, nonReloadable, hcrInline.} = # carefully written to avoid memory allocation: cstderr.rawWrite("could not import: ") cstderr.rawWrite(name) cstderr.rawWrite("\n") - quit(1) + rawQuit(1) # this code was inspired from Lua's source code: # Lua - An Extensible Extension Language @@ -161,38 +161,38 @@ elif defined(windows) or defined(dos): dec(m) k = k div 10 if k == 0: break - result = getProcAddress(cast[THINSTANCE](lib), addr decorated) + result = getProcAddress(cast[THINSTANCE](lib), cast[cstring](addr decorated)) if result != nil: return procAddrError(name) elif defined(genode): - proc nimUnloadLibrary(lib: LibHandle) {. - error: "nimUnloadLibrary not implemented".} + proc nimUnloadLibrary(lib: LibHandle) = + raiseAssert("nimUnloadLibrary not implemented") - proc nimLoadLibrary(path: string): LibHandle {. - error: "nimLoadLibrary not implemented".} + proc nimLoadLibrary(path: string): LibHandle = + raiseAssert("nimLoadLibrary not implemented") - proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {. - error: "nimGetProcAddr not implemented".} + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = + raiseAssert("nimGetProcAddr not implemented") -elif defined(nintendoswitch) or defined(freertos): +elif defined(nintendoswitch) or defined(freertos) or defined(zephyr) or defined(nuttx): proc nimUnloadLibrary(lib: LibHandle) = cstderr.rawWrite("nimUnLoadLibrary not implemented") cstderr.rawWrite("\n") - quit(1) + rawQuit(1) proc nimLoadLibrary(path: string): LibHandle = cstderr.rawWrite("nimLoadLibrary not implemented") cstderr.rawWrite("\n") - quit(1) + rawQuit(1) proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = cstderr.rawWrite("nimGetProAddr not implemented") cstderr.rawWrite(name) cstderr.rawWrite("\n") - quit(1) + rawQuit(1) else: {.error: "no implementation for dyncalls".} diff --git a/lib/system/embedded.nim b/lib/system/embedded.nim index 258558c3f6..ea6776f58a 100644 --- a/lib/system/embedded.nim +++ b/lib/system/embedded.nim @@ -19,8 +19,9 @@ proc nimFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} = discard proc popFrame {.compilerRtl, inl.} = discard proc setFrame(s: PFrame) {.compilerRtl, inl.} = discard -proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = discard -proc popSafePoint {.compilerRtl, inl.} = discard +when not gotoBasedExceptions: + proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = discard + proc popSafePoint {.compilerRtl, inl.} = discard proc pushCurrentException(e: ref Exception) {.compilerRtl, inl.} = discard proc popCurrentException {.compilerRtl, inl.} = discard @@ -34,6 +35,10 @@ proc quitOrDebug() {.noreturn, importc: "abort", header: "<stdlib.h>", nodecl.} proc raiseException(e: ref Exception, ename: cstring) {.compilerRtl.} = sysFatal(ReraiseDefect, "exception handling is not available") +proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring, + line: int) {.compilerRtl.} = + sysFatal(ReraiseDefect, "exception handling is not available") + proc reraiseException() {.compilerRtl.} = sysFatal(ReraiseDefect, "no exception to reraise") diff --git a/lib/system/exceptions.nim b/lib/system/exceptions.nim index b5f4fc3255..63588f8589 100644 --- a/lib/system/exceptions.nim +++ b/lib/system/exceptions.nim @@ -1,61 +1,13 @@ -const NimStackTraceMsgs = - when defined(nimHasStacktraceMsgs): compileOption("stacktraceMsgs") - else: false +## Exception and effect types used in Nim code. type - RootEffect* {.compilerproc.} = object of RootObj ## \ - ## Base effect class. - ## - ## Each effect should inherit from `RootEffect` unless you know what - ## you're doing. TimeEffect* = object of RootEffect ## Time effect. IOEffect* = object of RootEffect ## IO effect. ReadIOEffect* = object of IOEffect ## Effect describing a read IO operation. WriteIOEffect* = object of IOEffect ## Effect describing a write IO operation. ExecIOEffect* = object of IOEffect ## Effect describing an executing IO operation. - StackTraceEntry* = object ## In debug mode exceptions store the stack trace that led - ## to them. A `StackTraceEntry` is a single entry of the - ## stack trace. - procname*: cstring ## Name of the proc that is currently executing. - line*: int ## Line number of the proc that is currently executing. - filename*: cstring ## Filename of the proc that is currently executing. - when NimStackTraceMsgs: - frameMsg*: string ## When a stacktrace is generated in a given frame and - ## rendered at a later time, we should ensure the stacktrace - ## data isn't invalidated; any pointer into PFrame is - ## subject to being invalidated so shouldn't be stored. - when defined(nimStackTraceOverride): - programCounter*: uint ## Program counter - will be used to get the rest of the info, - ## when `$` is called on this type. We can't use - ## "cuintptr_t" in here. - procnameStr*, filenameStr*: string ## GC-ed objects holding the cstrings in "procname" and "filename" - - Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \ - ## Base exception class. - ## - ## Each exception has to inherit from `Exception`. See the full `exception - ## hierarchy <manual.html#exception-handling-exception-hierarchy>`_. - parent*: ref Exception ## Parent exception (can be used as a stack). - name*: cstring ## The exception's name is its Nim identifier. - ## This field is filled automatically in the - ## `raise` statement. - msg* {.exportc: "message".}: string ## The exception's message. Not - ## providing an exception message - ## is bad style. - when defined(js): - trace: string - else: - trace: seq[StackTraceEntry] - up: ref Exception # used for stacking exceptions. Not exported! - - Defect* = object of Exception ## \ - ## Abstract base class for all exceptions that Nim's runtime raises - ## but that are strictly uncatchable as they can also be mapped to - ## a `quit` / `trap` / `exit` operation. - - CatchableError* = object of Exception ## \ - ## Abstract class for all exceptions that are catchable. +type IOError* = object of CatchableError ## \ ## Raised if an IO error occurred. EOFError* = object of IOError ## \ @@ -144,25 +96,27 @@ type ## ## This is only raised if the `segfaults module <segfaults.html>`_ was imported! - ArithmeticError* {.deprecated: "See corresponding Defect".} = ArithmeticDefect - DivByZeroError* {.deprecated: "See corresponding Defect".} = DivByZeroDefect - OverflowError* {.deprecated: "See corresponding Defect".} = OverflowDefect - AccessViolationError* {.deprecated: "See corresponding Defect".} = AccessViolationDefect - AssertionError* {.deprecated: "See corresponding Defect".} = AssertionDefect - OutOfMemError* {.deprecated: "See corresponding Defect".} = OutOfMemDefect - IndexError* {.deprecated: "See corresponding Defect".} = IndexDefect +when not defined(nimPreviewSlimSystem): + type + ArithmeticError* {.deprecated: "See corresponding Defect".} = ArithmeticDefect + DivByZeroError* {.deprecated: "See corresponding Defect".} = DivByZeroDefect + OverflowError* {.deprecated: "See corresponding Defect".} = OverflowDefect + AccessViolationError* {.deprecated: "See corresponding Defect".} = AccessViolationDefect + AssertionError* {.deprecated: "See corresponding Defect".} = AssertionDefect + OutOfMemError* {.deprecated: "See corresponding Defect".} = OutOfMemDefect + IndexError* {.deprecated: "See corresponding Defect".} = IndexDefect - FieldError* {.deprecated: "See corresponding Defect".} = FieldDefect - RangeError* {.deprecated: "See corresponding Defect".} = RangeDefect - StackOverflowError* {.deprecated: "See corresponding Defect".} = StackOverflowDefect - ReraiseError* {.deprecated: "See corresponding Defect".} = ReraiseDefect - ObjectAssignmentError* {.deprecated: "See corresponding Defect".} = ObjectAssignmentDefect - ObjectConversionError* {.deprecated: "See corresponding Defect".} = ObjectConversionDefect - FloatingPointError* {.deprecated: "See corresponding Defect".} = FloatingPointDefect - FloatInvalidOpError* {.deprecated: "See corresponding Defect".} = FloatInvalidOpDefect - FloatDivByZeroError* {.deprecated: "See corresponding Defect".} = FloatDivByZeroDefect - FloatOverflowError* {.deprecated: "See corresponding Defect".} = FloatOverflowDefect - FloatUnderflowError* {.deprecated: "See corresponding Defect".} = FloatUnderflowDefect - FloatInexactError* {.deprecated: "See corresponding Defect".} = FloatInexactDefect - DeadThreadError* {.deprecated: "See corresponding Defect".} = DeadThreadDefect - NilAccessError* {.deprecated: "See corresponding Defect".} = NilAccessDefect + FieldError* {.deprecated: "See corresponding Defect".} = FieldDefect + RangeError* {.deprecated: "See corresponding Defect".} = RangeDefect + StackOverflowError* {.deprecated: "See corresponding Defect".} = StackOverflowDefect + ReraiseError* {.deprecated: "See corresponding Defect".} = ReraiseDefect + ObjectAssignmentError* {.deprecated: "See corresponding Defect".} = ObjectAssignmentDefect + ObjectConversionError* {.deprecated: "See corresponding Defect".} = ObjectConversionDefect + FloatingPointError* {.deprecated: "See corresponding Defect".} = FloatingPointDefect + FloatInvalidOpError* {.deprecated: "See corresponding Defect".} = FloatInvalidOpDefect + FloatDivByZeroError* {.deprecated: "See corresponding Defect".} = FloatDivByZeroDefect + FloatOverflowError* {.deprecated: "See corresponding Defect".} = FloatOverflowDefect + FloatUnderflowError* {.deprecated: "See corresponding Defect".} = FloatUnderflowDefect + FloatInexactError* {.deprecated: "See corresponding Defect".} = FloatInexactDefect + DeadThreadError* {.deprecated: "See corresponding Defect".} = DeadThreadDefect + NilAccessError* {.deprecated: "See corresponding Defect".} = NilAccessDefect diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 06fa450977..dae5c4a4a1 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -73,26 +73,45 @@ type when NimStackTraceMsgs: var frameMsgBuf* {.threadvar.}: string -var - framePtr {.threadvar.}: PFrame - excHandler {.threadvar.}: PSafePoint - # list of exception handlers - # a global variable for the root of all try blocks - currException {.threadvar.}: ref Exception - gcFramePtr {.threadvar.}: GcFrame -type - FrameState = tuple[gcFramePtr: GcFrame, framePtr: PFrame, - excHandler: PSafePoint, currException: ref Exception] +when not defined(nimV2): + var + framePtr {.threadvar.}: PFrame + +var + currException {.threadvar.}: ref Exception + +when not gotoBasedExceptions: + var + excHandler {.threadvar.}: PSafePoint + # list of exception handlers + # a global variable for the root of all try blocks + gcFramePtr {.threadvar.}: GcFrame + +when gotoBasedExceptions: + type + FrameState = tuple[framePtr: PFrame, + currException: ref Exception] +else: + type + FrameState = tuple[gcFramePtr: GcFrame, framePtr: PFrame, + excHandler: PSafePoint, currException: ref Exception] proc getFrameState*(): FrameState {.compilerRtl, inl.} = - return (gcFramePtr, framePtr, excHandler, currException) + when gotoBasedExceptions: + return (framePtr, currException) + else: + return (gcFramePtr, framePtr, excHandler, currException) proc setFrameState*(state: FrameState) {.compilerRtl, inl.} = - gcFramePtr = state.gcFramePtr - framePtr = state.framePtr - excHandler = state.excHandler - currException = state.currException + when gotoBasedExceptions: + framePtr = state.framePtr + currException = state.currException + else: + gcFramePtr = state.gcFramePtr + framePtr = state.framePtr + excHandler = state.excHandler + currException = state.currException proc getFrame*(): PFrame {.compilerRtl, inl.} = framePtr @@ -114,20 +133,21 @@ when false: proc setFrame*(s: PFrame) {.compilerRtl, inl.} = framePtr = s -proc getGcFrame*(): GcFrame {.compilerRtl, inl.} = gcFramePtr -proc popGcFrame*() {.compilerRtl, inl.} = gcFramePtr = gcFramePtr.prev -proc setGcFrame*(s: GcFrame) {.compilerRtl, inl.} = gcFramePtr = s -proc pushGcFrame*(s: GcFrame) {.compilerRtl, inl.} = - s.prev = gcFramePtr - zeroMem(cast[pointer](cast[int](s)+%sizeof(GcFrameHeader)), s.len*sizeof(pointer)) - gcFramePtr = s +when not gotoBasedExceptions: + proc getGcFrame*(): GcFrame {.compilerRtl, inl.} = gcFramePtr + proc popGcFrame*() {.compilerRtl, inl.} = gcFramePtr = gcFramePtr.prev + proc setGcFrame*(s: GcFrame) {.compilerRtl, inl.} = gcFramePtr = s + proc pushGcFrame*(s: GcFrame) {.compilerRtl, inl.} = + s.prev = gcFramePtr + zeroMem(cast[pointer](cast[int](s)+%sizeof(GcFrameHeader)), s.len*sizeof(pointer)) + gcFramePtr = s -proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = - s.prev = excHandler - excHandler = s + proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = + s.prev = excHandler + excHandler = s -proc popSafePoint {.compilerRtl, inl.} = - excHandler = excHandler.prev + proc popSafePoint {.compilerRtl, inl.} = + excHandler = excHandler.prev proc pushCurrentException(e: sink(ref Exception)) {.compilerRtl, inl.} = e.up = currException @@ -146,7 +166,7 @@ proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = # some platforms have native support for stack traces: const - nativeStackTraceSupported* = (defined(macosx) or defined(linux)) and + nativeStackTraceSupported = (defined(macosx) or defined(linux)) and not NimStackTrace hasSomeStackTrace = NimStackTrace or defined(nimStackTraceOverride) or (defined(nativeStackTrace) and nativeStackTraceSupported) @@ -354,7 +374,7 @@ var onUnhandledException*: (proc (errorMsg: string) {. ## The default is to write a stacktrace to `stderr` and then call `quit(1)`. ## Unstable API. -proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = +proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy, gcsafe.} = when hasSomeStackTrace: var buf = newStringOfCap(2000) if e.trace.len == 0: @@ -362,7 +382,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = else: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") @@ -373,7 +394,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = onUnhandledException(buf) else: showErrorMessage2(buf) - `=destroy`(buf) + {.gcsafe.}: + `=destroy`(buf) else: # ugly, but avoids heap allocations :-) template xadd(buf, s, slen) = @@ -387,32 +409,34 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = if e.trace.len != 0: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") xadd(buf, e.name, e.name.len) add(buf, "]\n") if onUnhandledException != nil: - onUnhandledException($buf.addr) + onUnhandledException($cast[cstring](buf.addr)) else: - showErrorMessage(buf.addr, L) + showErrorMessage(cast[cstring](buf.addr), L) -proc reportUnhandledError(e: ref Exception) {.nodestroy.} = +proc reportUnhandledError(e: ref Exception) {.nodestroy, gcsafe.} = if unhandledExceptionHook != nil: unhandledExceptionHook(e) when hostOS != "any": reportUnhandledErrorAux(e) -proc nimLeaveFinally() {.compilerRtl.} = - when defined(cpp) and not defined(noCppExceptions) and not gotoBasedExceptions: - {.emit: "throw;".} - else: - if excHandler != nil: - c_longjmp(excHandler.context, 1) +when not gotoBasedExceptions: + proc nimLeaveFinally() {.compilerRtl.} = + when defined(cpp) and not defined(noCppExceptions) and not gotoBasedExceptions: + {.emit: "throw;".} else: - reportUnhandledError(currException) - quit(1) + if excHandler != nil: + c_longjmp(excHandler.context, 1) + else: + reportUnhandledError(currException) + rawQuit(1) when gotoBasedExceptions: var nimInErrorMode {.threadvar.}: bool @@ -427,13 +451,13 @@ when gotoBasedExceptions: if nimInErrorMode and currException != nil: reportUnhandledError(currException) currException = nil - quit(1) + rawQuit(1) proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = when defined(nimPanics): if e of Defect: reportUnhandledError(e) - quit(1) + rawQuit(1) if localRaiseHook != nil: if not localRaiseHook(e): return @@ -444,11 +468,9 @@ proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = {.emit: "throw;".} else: pushCurrentException(e) - {.emit: "throw e;".} - elif defined(nimQuirky) or gotoBasedExceptions: - # XXX This check should likely also be done in the setjmp case below. - if e != currException: - pushCurrentException(e) + {.emit: "throw `e`;".} + elif quirkyExceptions or gotoBasedExceptions: + pushCurrentException(e) when gotoBasedExceptions: inc nimInErrorMode else: @@ -457,7 +479,7 @@ proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = c_longjmp(excHandler.context, 1) else: reportUnhandledError(e) - quit(1) + rawQuit(1) proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring, line: int) {.compilerRtl, nodestroy.} = @@ -500,7 +522,7 @@ proc threadTrouble() = if currException != nil: reportUnhandledError(currException) except: discard - quit 1 + rawQuit 1 proc writeStackTrace() = when hasSomeStackTrace: @@ -523,13 +545,10 @@ proc getStackTrace(e: ref Exception): string = else: result = "" -proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = +proc getStackTraceEntries*(e: ref Exception): lent seq[StackTraceEntry] = ## Returns the attached stack trace to the exception `e` as ## a `seq`. This is not yet available for the JS backend. - when not defined(nimSeqsV2): - shallowCopy(result, e.trace) - else: - result = move(e.trace) + e.trace proc getStackTraceEntries*(): seq[StackTraceEntry] = ## Returns the stack trace entries for the current stack trace. @@ -546,7 +565,7 @@ proc callDepthLimitReached() {.noinline.} = "-d:nimCallDepthLimit=<int> but really try to avoid deep " & "recursions instead.\n" showErrorMessage2(msg) - quit(1) + rawQuit(1) proc nimFrame(s: PFrame) {.compilerRtl, inl, raises: [].} = if framePtr == nil: @@ -562,7 +581,7 @@ proc nimFrame(s: PFrame) {.compilerRtl, inl, raises: [].} = when defined(cpp) and appType != "lib" and not gotoBasedExceptions and not defined(js) and not defined(nimscript) and hostOS != "standalone" and hostOS != "any" and not defined(noCppExceptions) and - not defined(nimQuirky): + not quirkyExceptions: type StdException {.importcpp: "std::exception", header: "<exception>".} = object @@ -599,9 +618,12 @@ when defined(cpp) and appType != "lib" and not gotoBasedExceptions and else: writeToStdErr msg & "\n" - quit 1 + rawQuit 1 when not defined(noSignalHandler) and not defined(useNimRtl): + type Sighandler = proc (a: cint) {.noconv, benign.} + # xxx factor with ansi_c.CSighandlerT, posix.Sighandler + proc signalHandler(sign: cint) {.exportc: "signalHandler", noconv.} = template processSignal(s, action: untyped) {.dirty.} = if s == SIGINT: action("SIGINT: Interrupted by Ctrl-C.\n") @@ -650,9 +672,13 @@ when not defined(noSignalHandler) and not defined(useNimRtl): # also return the correct exit code to the shell. discard c_raise(sign) else: - quit(1) + rawQuit(1) + + var SIG_IGN {.importc: "SIG_IGN", header: "<signal.h>".}: Sighandler proc registerSignalHandler() = + # xxx `signal` is deprecated and has many caveats, we should use `sigaction` instead, e.g. + # https://stackoverflow.com/questions/231912/what-is-the-difference-between-sigaction-and-signal c_signal(SIGINT, signalHandler) c_signal(SIGSEGV, signalHandler) c_signal(SIGABRT, signalHandler) @@ -661,14 +687,17 @@ when not defined(noSignalHandler) and not defined(useNimRtl): when declared(SIGBUS): c_signal(SIGBUS, signalHandler) when declared(SIGPIPE): - c_signal(SIGPIPE, signalHandler) + when defined(nimLegacySigpipeHandler): + c_signal(SIGPIPE, signalHandler) + else: + c_signal(SIGPIPE, SIG_IGN) registerSignalHandler() # call it in initialization section proc setControlCHook(hook: proc () {.noconv.}) = # ugly cast, but should work on all architectures: - type SignalHandler = proc (sign: cint) {.noconv, benign.} - c_signal(SIGINT, cast[SignalHandler](hook)) + when declared(Sighandler): + c_signal(SIGINT, cast[Sighandler](hook)) when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook() = diff --git a/lib/system/fatal.nim b/lib/system/fatal.nim index 64ec9cda3f..25c05e52d9 100644 --- a/lib/system/fatal.nim +++ b/lib/system/fatal.nim @@ -9,50 +9,50 @@ {.push profiler: off.} -when defined(nimHasExceptionsQuery): - const gotoBasedExceptions = compileOption("exceptions", "goto") -else: - const gotoBasedExceptions = false +const + gotoBasedExceptions = compileOption("exceptions", "goto") + quirkyExceptions = compileOption("exceptions", "quirky") when hostOS == "standalone": include "$projectpath/panicoverride" - proc sysFatal(exceptn: typedesc, message: string) {.inline.} = + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline.} = panic(message) - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline.} = + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline.} = rawoutput(message) panic(arg) -elif (defined(nimQuirky) or defined(nimPanics)) and not defined(nimscript): +elif quirkyExceptions and not defined(nimscript): import ansi_c - proc name(t: typedesc): string {.magic: "TypeTrait".} + func name(t: typedesc): string {.magic: "TypeTrait".} - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline, noreturn.} = + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline, noreturn.} = when nimvm: # TODO when doAssertRaises works in CT, add a test for it raise (ref exceptn)(msg: message & arg) else: - writeStackTrace() - var buf = newStringOfCap(200) - add(buf, "Error: unhandled exception: ") - add(buf, message) - add(buf, arg) - add(buf, " [") - add(buf, name exceptn) - add(buf, "]\n") - cstderr.rawWrite buf - quit 1 + {.noSideEffect.}: + writeStackTrace() + var buf = newStringOfCap(200) + add(buf, "Error: unhandled exception: ") + add(buf, message) + add(buf, arg) + add(buf, " [") + add(buf, name exceptn) + add(buf, "]\n") + cstderr.rawWrite buf + rawQuit 1 - proc sysFatal(exceptn: typedesc, message: string) {.inline, noreturn.} = + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline, noreturn.} = sysFatal(exceptn, message, "") else: - proc sysFatal(exceptn: typedesc, message: string) {.inline, noreturn.} = + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline, noreturn.} = raise (ref exceptn)(msg: message) - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline, noreturn.} = + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline, noreturn.} = raise (ref exceptn)(msg: message & arg) {.pop.} diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim index cb46c8c361..70dd857d5a 100644 --- a/lib/system/formatfloat.nim +++ b/lib/system/formatfloat.nim @@ -1,59 +1,6 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2019 Nim contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} - -proc writeToBuffer(buf: var array[65, char]; value: cstring) = - var i = 0 - while value[i] != '\0': - buf[i] = value[i] - inc i - -proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int = - ## This is the implementation to format floats in the Nim - ## programming language. The specific format for floating point - ## numbers is not specified in the Nim programming language and - ## might change slightly in the future, but at least wherever you - ## format a float, it should be consistent. - ## - ## returns the amount of bytes written to `buf` not counting the - ## terminating '\0' character. - ## - ## * `buf` - A buffer to write into. The buffer does not need to be - ## initialized and it will be overridden. - ## - var n: int = c_sprintf(addr buf, "%.16g", value) - var hasDot = false - for i in 0..n-1: - if buf[i] == ',': - buf[i] = '.' - hasDot = true - elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: - hasDot = true - if not hasDot: - buf[n] = '.' - buf[n+1] = '0' - buf[n+2] = '\0' - result = n + 2 - else: - result = n - # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' - # of '-1.#IND' are produced. - # We want to get rid of these here: - if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: - writeToBuffer(buf, "nan") - result = 3 - elif buf[n-1] == 'F': - if buf[0] == '-': - writeToBuffer(buf, "-inf") - result = 4 - else: - writeToBuffer(buf, "inf") - result = 3 +when not defined(nimPreviewSlimSystem): + import std/formatfloat + export formatfloat + {.deprecated: "use `std/formatfloat`".} +else: + {.error: "use `std/formatfloat`".} diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 1f71642662..9289c7f55c 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -12,6 +12,55 @@ # Refcounting + Mark&Sweep. Complex algorithms avoided. # Been there, done that, didn't work. +#[ + +A *cell* is anything that is traced by the GC +(sequences, refs, strings, closures). + +The basic algorithm is *Deferrent Reference Counting* with cycle detection. +References on the stack are not counted for better performance and easier C +code generation. + +Each cell has a header consisting of a RC and a pointer to its type +descriptor. However the program does not know about these, so they are placed at +negative offsets. In the GC code the type `PCell` denotes a pointer +decremented by the right offset, so that the header can be accessed easily. It +is extremely important that `pointer` is not confused with a `PCell`. + +In Nim the compiler cannot always know if a reference +is stored on the stack or not. This is caused by var parameters. +Consider this example: + + ```Nim + proc setRef(r: var ref TNode) = + new(r) + + proc usage = + var + r: ref TNode + setRef(r) # here we should not update the reference counts, because + # r is on the stack + setRef(r.left) # here we should update the refcounts! + ``` + +We have to decide at runtime whether the reference is on the stack or not. +The generated code looks roughly like this: + + ```C + void setref(TNode** ref) { + unsureAsgnRef(ref, newObj(TNode_TI, sizeof(TNode))) + } + void usage(void) { + setRef(&r) + setRef(&r->left) + } + ``` + +Note that for systems with a continuous stack (which most systems have) +the check whether the ref is on the stack is very cheap (only two +comparisons). +]# + {.push profiler:off.} const @@ -29,7 +78,7 @@ when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} when hasThreadSupport: - import sharedlist + import std/sharedlist const rcIncrement = 0b1000 # so that lowest 3 bits are not touched @@ -47,7 +96,7 @@ type waZctDecRef, waPush #, waDebug - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -114,7 +163,7 @@ template gcAssert(cond: bool, msg: string) = writeStackTrace() #var x: ptr int #echo x[] - quit 1 + rawQuit 1 proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = if (c.refcount and ZctFlag) == 0: @@ -123,11 +172,11 @@ proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -172,11 +221,11 @@ template gcTrace(cell, state: untyped) = when traceGC: traceCell(cell, state) # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc isOnStack(p: pointer): bool {.noinline, benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap) {.benign, raises: [].} +proc isOnStack(p: pointer): bool {.noinline, benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes proc incRef(c: PCell) {.inline.} = @@ -289,7 +338,7 @@ proc cellsetReset(s: var CellSet) = {.push stacktrace:off.} proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -309,7 +358,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -335,7 +384,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -410,7 +459,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) #gcAssert typ.kind in {tyString, tySequence} or size >= typ.base.size, "size too small" - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -435,7 +484,7 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(typ, size, gch) when defined(memProfiler): nimProfile(size) -proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = +proc newObj(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = result = rawNewObj(typ, size, gch) zeroMem(result, size) when defined(memProfiler): nimProfile(size) @@ -450,7 +499,7 @@ proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = when defined(memProfiler): nimProfile(size) {.pop.} -proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = # generates a new object and sets its reference counter to 1 incTypeSize typ, size sysAssert(allocInv(gch.region), "newObjRC1 begin") @@ -460,7 +509,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -502,9 +551,9 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len * elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res) +% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res) +% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") # This can be wrong for intermediate temps that are nevertheless on the # heap because of lambda lifting: #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") @@ -514,35 +563,8 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = gcTrace(res, csAllocated) track("growObj old", ol, 0) track("growObj new", res, newsize) - when defined(nimIncrSeqV3): - # since we steal the old seq's contents, we set the old length to 0. - cast[PGenericSeq](old).len = 0 - elif reallyDealloc: - sysAssert(allocInv(gch.region), "growObj before dealloc") - if ol.refcount shr rcShift <=% 1: - # free immediately to save space: - if (ol.refcount and ZctFlag) != 0: - var j = gch.zct.len-1 - var d = gch.zct.d - while j >= 0: - if d[j] == ol: - d[j] = res - break - dec(j) - beforeDealloc(gch, ol, "growObj stack trash") - decTypeSize(ol, ol.typ) - rawDealloc(gch.region, ol) - else: - # we split the old refcount in 2 parts. XXX This is still not entirely - # correct if the pointer that receives growObj's result is on the stack. - # A better fix would be to emit the location specific write barrier for - # 'growObj', but this is lots of more work and who knows what new problems - # this would create. - res.refcount = rcIncrement - decRef(ol) - else: - sysAssert(ol.typ != nil, "growObj: 5") - zeroMem(ol, sizeof(Cell)) + # since we steal the old seq's contents, we set the old length to 0. + cast[PGenericSeq](old).len = 0 when useCellIds: inc gch.idGenerator res.id = gch.idGenerator * 1000_000 + gch.gcThreadId @@ -589,7 +611,7 @@ proc markS(gch: var GcHeap, c: PCell) = if not containsOrIncl(gch.marked, d): forAllChildren(d, waMarkPrecise) -proc markGlobals(gch: var GcHeap) = +proc markGlobals(gch: var GcHeap) {.raises: [].} = if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() @@ -606,7 +628,7 @@ when logGC: if cycleCheckA[i] == c: return true if cycleCheckALen == len(cycleCheckA): gcAssert(false, "cycle detection overflow") - quit 1 + rawQuit 1 cycleCheckA[cycleCheckALen] = c inc cycleCheckALen @@ -644,9 +666,9 @@ proc doOperation(p: pointer, op: WalkOp) = proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = doOperation(d, WalkOp(op)) -proc collectZCT(gch: var GcHeap): bool {.benign.} +proc collectZCT(gch: var GcHeap): bool {.benign, raises: [].} -proc collectCycles(gch: var GcHeap) = +proc collectCycles(gch: var GcHeap) {.raises: [].} = when hasThreadSupport: for c in gch.toDispose: nimGCunref(c) @@ -663,16 +685,16 @@ proc collectCycles(gch: var GcHeap) = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: # mark the cell: incRef(objStart) add(gch.decStack, objStart) when false: + let cell = usrToCell(p) if isAllocatedPtr(gch.region, cell): sysAssert false, "allocated pointer but not interior?" # mark the cell: @@ -753,7 +775,7 @@ proc unmarkStackAndRegisters(gch: var GcHeap) = decRef(d[i]) gch.decStack.len = 0 -proc collectCTBody(gch: var GcHeap) = +proc collectCTBody(gch: var GcHeap) {.raises: [].} = when withRealTime: let t0 = getticks() sysAssert(allocInv(gch.region), "collectCT: begin") @@ -828,10 +850,10 @@ when withRealTime: stack.bottomSaved = stack.bottom when stackIncreases: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) + cast[int](stack.pos) - sizeof(pointer) * 6 - stackSize) else: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) + cast[int](stack.pos) + sizeof(pointer) * 6 + stackSize) GC_step(gch, us, strongAdvice) diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim deleted file mode 100644 index 45d467051e..0000000000 --- a/lib/system/gc2.nim +++ /dev/null @@ -1,749 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2017 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# xxx deadcode, consider removing unless something could be reused. - - -# Garbage Collector -# -# The basic algorithm is an incremental mark -# and sweep GC to free cycles. It is hard realtime in that if you play -# according to its rules, no deadline will ever be missed. -# Since this kind of collector is very bad at recycling dead objects -# early, Nim's codegen emits ``nimEscape`` calls at strategic -# places. For this to work even 'unsureAsgnRef' needs to mark things -# so that only return values need to be considered in ``nimEscape``. - -{.push profiler:off.} - -const - CycleIncrease = 2 # is a multiplicative increase - InitialCycleThreshold = 512*1024 # start collecting after 500KB - ZctThreshold = 500 # we collect garbage if the ZCT's size - # reaches this threshold - # this seems to be a good value - withRealTime = defined(useRealtimeGC) - -when withRealTime and not declared(getTicks): - include "system/timers" -when defined(memProfiler): - proc nimProfile(requestedSize: int) {.benign.} - -when hasThreadSupport: - include sharedlist - -type - ObjectSpaceIter = object - state: range[-1..0] - -iterToProc(allObjects, ptr ObjectSpaceIter, allObjectsAsProc) - -const - escapedBit = 0b1000 # so that lowest 3 bits are not touched - rcBlackOrig = 0b000 - rcWhiteOrig = 0b001 - rcGrey = 0b010 # traditional color for incremental mark&sweep - rcUnused = 0b011 - colorMask = 0b011 -type - WalkOp = enum - waMarkGlobal, # part of the backup mark&sweep - waMarkGrey, - waZctDecRef, - waDebug - - Phase {.pure.} = enum - None, Marking, Sweeping - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} - # A ref type can have a finalizer that is called before the object's - # storage is freed. - - GcStat = object - stackScans: int # number of performed stack scans (for statistics) - completedCollections: int # number of performed full collections - maxThreshold: int # max threshold that has been set - maxStackSize: int # max stack size - maxStackCells: int # max stack cells in ``decStack`` - cycleTableSize: int # max entries in cycle table - maxPause: int64 # max measured GC pause in nanoseconds - - GcStack {.final, pure.} = object - when nimCoroutines: - prev: ptr GcStack - next: ptr GcStack - maxStackSize: int # Used to track statistics because we can not use - # GcStat.maxStackSize when multiple stacks exist. - bottom: pointer - - when withRealTime or nimCoroutines: - pos: pointer # Used with `withRealTime` only for code clarity, see GC_Step(). - when withRealTime: - bottomSaved: pointer - - GcHeap = object # this contains the zero count and - # non-zero count table - black, red: int # either 0 or 1. - stack: GcStack - when nimCoroutines: - activeStack: ptr GcStack # current executing coroutine stack. - phase: Phase - cycleThreshold: int - when useCellIds: - idGenerator: int - greyStack: CellSeq - recGcLock: int # prevent recursion via finalizers; no thread lock - when withRealTime: - maxPause: Nanos # max allowed pause in nanoseconds; active if > 0 - region: MemRegion # garbage collected region - stat: GcStat - additionalRoots: CellSeq # explicit roots for GC_ref/unref - spaceIter: ObjectSpaceIter - pDumpHeapFile: pointer # File that is used for GC_dumpHeap - when hasThreadSupport: - toDispose: SharedList[pointer] - gcThreadId: int - -var - gch {.rtlThreadVar.}: GcHeap - -when not defined(useNimRtl): - instantiateForRegion(gch.region) - -# Which color to use for new objects is tricky: When we're marking, -# they have to be *white* so that everything is marked that is only -# reachable from them. However, when we are sweeping, they have to -# be black, so that we don't free them prematuredly. In order to save -# a comparison gch.phase == Phase.Marking, we use the pseudo-color -# 'red' for new objects. -template allocColor(): untyped = gch.red - -template gcAssert(cond: bool, msg: string) = - when defined(useGcAssert): - if not cond: - echo "[GCASSERT] ", msg - GC_disable() - writeStackTrace() - quit 1 - -proc cellToUsr(cell: PCell): pointer {.inline.} = - # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) - -proc usrToCell(usr: pointer): PCell {.inline.} = - # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) - -proc extGetCellType(c: pointer): PNimType {.compilerproc.} = - # used for code generation concerning debugging - result = usrToCell(c).typ - -proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = - result = 0 - -# this that has to equals zero, otherwise we have to round up UnitsPerPage: -when BitsPerPage mod (sizeof(int)*8) != 0: - {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} - -template color(c): untyped = c.refCount and colorMask -template setColor(c, col) = - c.refcount = c.refcount and not colorMask or col - -template markAsEscaped(c: PCell) = - c.refcount = c.refcount or escapedBit - -template didEscape(c: PCell): bool = - (c.refCount and escapedBit) != 0 - -proc writeCell(file: File; msg: cstring, c: PCell) = - var kind = -1 - if c.typ != nil: kind = ord(c.typ.kind) - let col = if c.color == rcGrey: 'g' - elif c.color == gch.black: 'b' - else: 'w' - when useCellIds: - let id = c.id - else: - let id = c - when defined(nimTypeNames): - c_fprintf(file, "%s %p %d escaped=%ld color=%c of type %s\n", - msg, id, kind, didEscape(c), col, c.typ.name) - elif leakDetector: - c_fprintf(file, "%s %p %d escaped=%ld color=%c from %s(%ld)\n", - msg, id, kind, didEscape(c), col, c.filename, c.line) - else: - c_fprintf(file, "%s %p %d escaped=%ld color=%c\n", - msg, id, kind, didEscape(c), col) - -proc writeCell(msg: cstring, c: PCell) = - stdout.writeCell(msg, c) - -proc myastToStr[T](x: T): string {.magic: "AstToStr", noSideEffect.} - -template gcTrace(cell, state: untyped) = - when traceGC: writeCell(myastToStr(state), cell) - -# forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc isOnStack(p: pointer): bool {.noinline, benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} -# we need the prototype here for debugging purposes - -proc nimGCref(p: pointer) {.compilerproc.} = - let cell = usrToCell(p) - markAsEscaped(cell) - add(gch.additionalRoots, cell) - -proc nimGCunref(p: pointer) {.compilerproc.} = - let cell = usrToCell(p) - var L = gch.additionalRoots.len-1 - var i = L - let d = gch.additionalRoots.d - while i >= 0: - if d[i] == cell: - d[i] = d[L] - dec gch.additionalRoots.len - break - dec(i) - -proc nimGCunrefNoCycle(p: pointer) {.compilerproc, inline.} = - discard "can we do some freeing here?" - -proc nimGCunrefRC1(p: pointer) {.compilerproc, inline.} = - discard "can we do some freeing here?" - -template markGrey(x: PCell) = - if x.color != 1-gch.black and gch.phase == Phase.Marking: - if not isAllocatedPtr(gch.region, x): - c_fprintf(stdout, "[GC] markGrey proc: %p\n", x) - #GC_dumpHeap() - sysAssert(false, "wtf") - x.setColor(rcGrey) - add(gch.greyStack, x) - -proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - # the code generator calls this proc! - gcAssert(not isOnStack(dest), "asgnRef") - # BUGFIX: first incRef then decRef! - if src != nil: - let s = usrToCell(src) - markAsEscaped(s) - markGrey(s) - dest[] = src - -proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, - deprecated: "old compiler compat".} = asgnRef(dest, src) - -proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc.} = - # unsureAsgnRef marks 'src' as grey only if dest is not on the - # stack. It is used by the code generator if it cannot decide whether a - # reference is in the stack or not (this can happen for var parameters). - if src != nil: - let s = usrToCell(src) - markAsEscaped(s) - if not isOnStack(dest): markGrey(s) - dest[] = src - -proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) - case n.kind - of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) - of nkList: - for i in 0..n.len-1: - forAllSlotsAux(dest, n.sons[i], op) - of nkCase: - var m = selectBranch(dest, n) - if m != nil: forAllSlotsAux(dest, m, op) - of nkNone: sysAssert(false, "forAllSlotsAux") - -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) - if dest == nil: return # nothing to do - if ntfNoRefs notin mt.flags: - case mt.kind - of tyRef, tyString, tySequence: # leaf: - doOperation(cast[PPointer](d)[], op) - of tyObject, tyTuple: - forAllSlotsAux(dest, mt.node, op) - of tyArray, tyArrayConstr, tyOpenArray: - for i in 0..(mt.size div mt.base.size)-1: - forAllChildrenAux(cast[pointer](d +% i *% mt.base.size), mt.base, op) - else: discard - -proc forAllChildren(cell: PCell, op: WalkOp) = - gcAssert(cell != nil, "forAllChildren: 1") - gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") - gcAssert(cell.typ != nil, "forAllChildren: 3") - gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 4" - let marker = cell.typ.marker - if marker != nil: - marker(cellToUsr(cell), op.int) - else: - case cell.typ.kind - of tyRef: # common case - forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) - of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) - var s = cast[PGenericSeq](d) - if s != nil: - for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% align(GenericSeqSize, cell.typ.base.align) +% i *% cell.typ.base.size), cell.typ.base, op) - else: discard - -{.push stackTrace: off, profiler:off.} -proc gcInvariant*() = - sysAssert(allocInv(gch.region), "injected") - when declared(markForDebug): - markForDebug(gch) -{.pop.} - -include gc_common - -proc initGC() = - when not defined(useNimRtl): - gch.red = (1-gch.black) - gch.cycleThreshold = InitialCycleThreshold - gch.stat.stackScans = 0 - gch.stat.completedCollections = 0 - gch.stat.maxThreshold = 0 - gch.stat.maxStackSize = 0 - gch.stat.maxStackCells = 0 - gch.stat.cycleTableSize = 0 - # init the rt - init(gch.additionalRoots) - init(gch.greyStack) - when hasThreadSupport: - init(gch.toDispose) - gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 - gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") - -proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = - # generates a new object and sets its reference counter to 0 - sysAssert(allocInv(gch.region), "rawNewObj begin") - gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) - var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") - # now it is buffered in the ZCT - res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line - # refcount is zero, color is black, but mark it to be in the ZCT - res.refcount = allocColor() - sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") - when logGC: writeCell("new cell", res) - gcTrace(res, csAllocated) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - result = cellToUsr(res) - sysAssert(allocInv(gch.region), "rawNewObj end") - -{.pop.} - -proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) - when defined(memProfiler): nimProfile(size) - -proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) - zeroMem(result, size) - when defined(memProfiler): nimProfile(size) - -proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = - # `newObj` already uses locks, so no need for them here. - let size = addInt(align(GenericSeqSize, typ.base.align), mulInt(len, typ.base.size)) - result = newObj(typ, size) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - when defined(memProfiler): nimProfile(size) - -proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = newObj(typ, size) - -proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - result = newSeq(typ, len) - -proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - collectCT(gch) - var ol = usrToCell(old) - sysAssert(ol.typ != nil, "growObj: 1") - gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") - - var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize, elemAlign = 1 - if ol.typ.kind != tyString: - elemSize = ol.typ.base.size - elemAlign = ol.typ.base.align - incTypeSize ol.typ, newsize - - var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize - copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), - newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - when false: - # this is wrong since seqs can be shared via 'shallow': - when reallyDealloc: rawDealloc(gch.region, ol) - else: - zeroMem(ol, sizeof(Cell)) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - result = cellToUsr(res) - when defined(memProfiler): nimProfile(newsize-oldsize) - -proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - result = growObj(old, newsize, gch) - -{.push profiler:off.} - - -template takeStartTime(workPackageSize) {.dirty.} = - const workPackage = workPackageSize - var debugticker = 1000 - when withRealTime: - var steps = workPackage - var t0: Ticks - if gch.maxPause > 0: t0 = getticks() - -template takeTime {.dirty.} = - when withRealTime: dec steps - dec debugticker - -template checkTime {.dirty.} = - if debugticker <= 0: - #echo "in loop" - debugticker = 1000 - when withRealTime: - if steps == 0: - steps = workPackage - if gch.maxPause > 0: - let duration = getticks() - t0 - # the GC's measuring is not accurate and needs some cleanup actions - # (stack unmarking), so subtract some short amount of time in - # order to miss deadlines less often: - if duration >= gch.maxPause - 50_000: - return false - -# ---------------- dump heap ---------------- - -template dumpHeapFile(gch: var GcHeap): File = - cast[File](gch.pDumpHeapFile) - -proc debugGraph(s: PCell) = - c_fprintf(gch.dumpHeapFile, "child %p\n", s) - -proc dumpRoot(gch: var GcHeap; s: PCell) = - if isAllocatedPtr(gch.region, s): - c_fprintf(gch.dumpHeapFile, "global_root %p\n", s) - else: - c_fprintf(gch.dumpHeapFile, "global_root_invalid %p\n", s) - -proc GC_dumpHeap*(file: File) = - ## Dumps the GCed heap's content to a file. Can be useful for - ## debugging. Produces an undocumented text file format that - ## can be translated into "dot" syntax via the "heapdump2dot" tool. - gch.pDumpHeapFile = file - var spaceIter: ObjectSpaceIter - when false: - var d = gch.decStack.d - for i in 0 .. gch.decStack.len-1: - if isAllocatedPtr(gch.region, d[i]): - c_fprintf(file, "onstack %p\n", d[i]) - else: - c_fprintf(file, "onstack_invalid %p\n", d[i]) - if gch.gcThreadId == 0: - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() - for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() - while true: - let x = allObjectsAsProc(gch.region, addr spaceIter) - if spaceIter.state < 0: break - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - writeCell(file, "cell ", c) - forAllChildren(c, waDebug) - c_fprintf(file, "end\n") - gch.pDumpHeapFile = nil - -proc GC_dumpHeap() = - var f: File - if open(f, "heap.txt", fmWrite): - GC_dumpHeap(f) - f.close() - else: - c_fprintf(stdout, "cannot write heap.txt") - -# ---------------- cycle collector ------------------------------------------- - -proc freeCyclicCell(gch: var GcHeap, c: PCell) = - gcAssert(isAllocatedPtr(gch.region, c), "freeCyclicCell: freed pointer?") - prepareDealloc(c) - gcTrace(c, csCycFreed) - when logGC: writeCell("cycle collector dealloc cell", c) - when reallyDealloc: - sysAssert(allocInv(gch.region), "free cyclic cell") - rawDealloc(gch.region, c) - else: - gcAssert(c.typ != nil, "freeCyclicCell") - zeroMem(c, sizeof(Cell)) - -proc sweep(gch: var GcHeap): bool = - takeStartTime(100) - #echo "loop start" - let white = 1-gch.black - #c_fprintf(stdout, "black is %d\n", black) - while true: - let x = allObjectsAsProc(gch.region, addr gch.spaceIter) - if gch.spaceIter.state < 0: break - takeTime() - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - gcAssert c.color != rcGrey, "cell is still grey?" - if c.color == white: freeCyclicCell(gch, c) - # Since this is incremental, we MUST not set the object to 'white' here. - # We could set all the remaining objects to white after the 'sweep' - # completed but instead we flip the meaning of black/white to save one - # traversal over the heap! - checkTime() - # prepare for next iteration: - #echo "loop end" - gch.spaceIter = ObjectSpaceIter() - result = true - -proc markRoot(gch: var GcHeap, c: PCell) {.inline.} = - if c.color == 1-gch.black: - c.setColor(rcGrey) - add(gch.greyStack, c) - -proc markIncremental(gch: var GcHeap): bool = - var L = addr(gch.greyStack.len) - takeStartTime(100) - while L[] > 0: - var c = gch.greyStack.d[0] - if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: %p\n", c) - #GC_dumpHeap() - sysAssert(false, "wtf") - - #sysAssert(isAllocatedPtr(gch.region, c), "markIncremental: isAllocatedPtr") - gch.greyStack.d[0] = gch.greyStack.d[L[] - 1] - dec(L[]) - takeTime() - if c.color == rcGrey: - c.setColor(gch.black) - forAllChildren(c, waMarkGrey) - elif c.color == (1-gch.black): - gcAssert false, "wtf why are there white objects in the greystack?" - checkTime() - gcAssert gch.greyStack.len == 0, "markIncremental: greystack not empty " - result = true - -proc markGlobals(gch: var GcHeap) = - if gch.gcThreadId == 0: - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() - for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() - -proc doOperation(p: pointer, op: WalkOp) = - if p == nil: return - var c: PCell = usrToCell(p) - gcAssert(c != nil, "doOperation: 1") - # the 'case' should be faster than function pointers because of easy - # prediction: - case op - of waZctDecRef: - #if not isAllocatedPtr(gch.region, c): - # c_fprintf(stdout, "[GC] decref bug: %p", c) - gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") - discard "use me for nimEscape?" - of waMarkGlobal: - template handleRoot = - if gch.dumpHeapFile.isNil: - markRoot(gch, c) - else: - dumpRoot(gch, c) - handleRoot() - discard allocInv(gch.region) - of waMarkGrey: - when false: - if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) - #GC_dumpHeap() - sysAssert(false, "wtf") - if c.color == 1-gch.black: - c.setColor(rcGrey) - add(gch.greyStack, c) - of waDebug: debugGraph(c) - -proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = - doOperation(d, WalkOp(op)) - -proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = - # the addresses are not as cells on the stack, so turn them to cells: - sysAssert(allocInv(gch.region), "gcMark begin") - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) - if c >% PageSize: - # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) - if objStart != nil: - # mark the cell: - markRoot(gch, objStart) - sysAssert(allocInv(gch.region), "gcMark end") - -proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = - forEachStackSlot(gch, gcMark) - -proc collectALittle(gch: var GcHeap): bool = - case gch.phase - of Phase.None: - if getOccupiedMem(gch.region) >= gch.cycleThreshold: - gch.phase = Phase.Marking - markGlobals(gch) - result = collectALittle(gch) - #when false: c_fprintf(stdout, "collectALittle: introduced bug E %ld\n", gch.phase) - #discard allocInv(gch.region) - of Phase.Marking: - when hasThreadSupport: - for c in gch.toDispose: - nimGCunref(c) - prepareForInteriorPointerChecking(gch.region) - markStackAndRegisters(gch) - inc(gch.stat.stackScans) - if markIncremental(gch): - gch.phase = Phase.Sweeping - gch.red = 1 - gch.red - of Phase.Sweeping: - gcAssert gch.greyStack.len == 0, "greystack not empty" - when hasThreadSupport: - for c in gch.toDispose: - nimGCunref(c) - if sweep(gch): - gch.phase = Phase.None - # flip black/white meanings: - gch.black = 1 - gch.black - gcAssert gch.red == 1 - gch.black, "red color is wrong" - inc(gch.stat.completedCollections) - result = true - -proc collectCTBody(gch: var GcHeap) = - when withRealTime: - let t0 = getticks() - sysAssert(allocInv(gch.region), "collectCT: begin") - - when not nimCoroutines: - gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) - #gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) - if collectALittle(gch): - gch.cycleThreshold = max(InitialCycleThreshold, getOccupiedMem() * - CycleIncrease) - gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) - sysAssert(allocInv(gch.region), "collectCT: end") - when withRealTime: - let duration = getticks() - t0 - gch.stat.maxPause = max(gch.stat.maxPause, duration) - when defined(reportMissedDeadlines): - if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration) - -when nimCoroutines: - proc currentStackSizes(): int = - for stack in items(gch.stack): - result = result + stack.stackSize() - -proc collectCT(gch: var GcHeap) = - # stackMarkCosts prevents some pathological behaviour: Stack marking - # becomes more expensive with large stacks and large stacks mean that - # cells with RC=0 are more likely to be kept alive by the stack. - when nimCoroutines: - let stackMarkCosts = max(currentStackSizes() div (16*sizeof(int)), ZctThreshold) - else: - let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold) - if (gch.greyStack.len >= stackMarkCosts or (cycleGC and - getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and - gch.recGcLock == 0: - collectCTBody(gch) - -when withRealTime: - proc toNano(x: int): Nanos {.inline.} = - result = x * 1000 - - proc GC_setMaxPause*(MaxPauseInUs: int) = - gch.maxPause = MaxPauseInUs.toNano - - proc GC_step(gch: var GcHeap, us: int, strongAdvice: bool) = - gch.maxPause = us.toNano - #if (getOccupiedMem(gch.region)>=gch.cycleThreshold) or - # alwaysGC or strongAdvice: - collectCTBody(gch) - - proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} = - if stackSize >= 0: - var stackTop {.volatile.}: pointer - gch.getActiveStack().pos = addr(stackTop) - - for stack in gch.stack.items(): - stack.bottomSaved = stack.bottom - when stackIncreases: - stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) - else: - stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) - - GC_step(gch, us, strongAdvice) - - if stackSize >= 0: - for stack in gch.stack.items(): - stack.bottom = stack.bottomSaved - -when not defined(useNimRtl): - proc GC_disable() = - inc(gch.recGcLock) - proc GC_enable() = - if gch.recGcLock > 0: - dec(gch.recGcLock) - - proc GC_setStrategy(strategy: GC_Strategy) = - discard - - proc GC_enableMarkAndSweep() = discard - proc GC_disableMarkAndSweep() = discard - - proc GC_fullCollect() = - var oldThreshold = gch.cycleThreshold - gch.cycleThreshold = 0 # forces cycle collection - collectCT(gch) - gch.cycleThreshold = oldThreshold - - proc GC_getStatistics(): string = - GC_disable() - result = "[GC] total memory: " & $(getTotalMem()) & "\n" & - "[GC] occupied memory: " & $(getOccupiedMem()) & "\n" & - "[GC] stack scans: " & $gch.stat.stackScans & "\n" & - "[GC] stack cells: " & $gch.stat.maxStackCells & "\n" & - "[GC] completed collections: " & $gch.stat.completedCollections & "\n" & - "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & - "[GC] grey stack capacity: " & $gch.greyStack.cap & "\n" & - "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n" - when nimCoroutines: - result.add "[GC] number of stacks: " & $gch.stack.len & "\n" - for stack in items(gch.stack): - result.add "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" - else: - result.add "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" - GC_enable() - -{.pop.} diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 7596de6fbe..eb08845603 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -222,9 +222,9 @@ proc stackSize(stack: ptr GcStack): int {.noinline.} = if pos != nil: when stackIncreases: - result = cast[ByteAddress](pos) -% cast[ByteAddress](stack.bottom) + result = cast[int](pos) -% cast[int](stack.bottom) else: - result = cast[ByteAddress](stack.bottom) -% cast[ByteAddress](pos) + result = cast[int](stack.bottom) -% cast[int](pos) else: result = 0 @@ -295,8 +295,8 @@ when not defined(useNimRtl): # the first init must be the one that defines the stack bottom: gch.stack.bottom = theStackBottom elif theStackBottom != gch.stack.bottom: - var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 - var b = cast[ByteAddress](gch.stack.bottom) + var a = cast[int](theStackBottom) # and not PageMask - PageSize*2 + var b = cast[int](gch.stack.bottom) #c_fprintf(stdout, "old: %p new: %p;\n",gch.stack.bottom,theStackBottom) when stackIncreases: gch.stack.bottom = cast[pointer](min(a, b)) @@ -312,11 +312,11 @@ when not defined(useNimRtl): proc isOnStack(p: pointer): bool = var stackTop {.volatile, noinit.}: pointer stackTop = addr(stackTop) - var a = cast[ByteAddress](gch.getActiveStack().bottom) - var b = cast[ByteAddress](stackTop) + var a = cast[int](gch.getActiveStack().bottom) + var b = cast[int](stackTop) when not stackIncreases: swap(a, b) - var x = cast[ByteAddress](p) + var x = cast[int](p) result = a <=% x and x <=% b when defined(sparc): # For SPARC architecture. @@ -337,7 +337,7 @@ when defined(sparc): # For SPARC architecture. # Addresses decrease as the stack grows. while sp <= max: gcMark(gch, sp[]) - sp = cast[PPointer](cast[ByteAddress](sp) +% sizeof(pointer)) + sp = cast[PPointer](cast[int](sp) +% sizeof(pointer)) elif defined(ELATE): {.error: "stack marking code is to be written for this architecture".} @@ -354,8 +354,8 @@ elif stackIncreases: template forEachStackSlotAux(gch, gcMark: untyped) {.dirty.} = for stack in gch.stack.items(): - var max = cast[ByteAddress](gch.stack.bottom) - var sp = cast[ByteAddress](addr(registers)) -% sizeof(pointer) + var max = cast[int](gch.stack.bottom) + var sp = cast[int](addr(registers)) -% sizeof(pointer) while sp >=% max: gcMark(gch, cast[PPointer](sp)[]) sp = sp -% sizeof(pointer) @@ -383,15 +383,14 @@ else: gch.getActiveStack().setPosition(addr(registers)) if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. for stack in gch.stack.items(): - var max = cast[ByteAddress](stack.bottom) - var sp = cast[ByteAddress](addr(registers)) + var max = cast[int](stack.bottom) + var sp = cast[int](addr(registers)) when defined(amd64): if stack.isActiveStack(): # words within the jmp_buf structure may not be properly aligned. let regEnd = sp +% sizeof(registers) while sp <% regEnd: gcMark(gch, cast[PPointer](sp)[]) - gcMark(gch, cast[PPointer](sp +% sizeof(pointer) div 2)[]) sp = sp +% sizeof(pointer) # Make sure sp is word-aligned sp = sp and not (sizeof(pointer) - 1) @@ -415,7 +414,7 @@ else: # end of non-portable code # ---------------------------------------------------------------------------- -proc prepareDealloc(cell: PCell) = +proc prepareDealloc(cell: PCell) {.raises: [].} = when declared(useMarkForDebug): when useMarkForDebug: gcAssert(cell notin gch.marked, "Cell still alive!") @@ -458,7 +457,7 @@ proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = initGC() type - GlobalMarkerProc = proc () {.nimcall, benign.} + GlobalMarkerProc = proc () {.nimcall, benign, raises: [].} var globalMarkersLen {.exportc.}: int globalMarkers {.exportc.}: array[0..3499, GlobalMarkerProc] @@ -472,7 +471,7 @@ proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = inc globalMarkersLen else: cstderr.rawWrite("[GC] cannot register global variable; too many global variables") - quit 1 + rawQuit 1 proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = if threadLocalMarkersLen <= high(threadLocalMarkers): @@ -480,4 +479,4 @@ proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} inc threadLocalMarkersLen else: cstderr.rawWrite("[GC] cannot register thread local variable; too many thread local variables") - quit 1 + rawQuit 1 diff --git a/lib/system/gc_hooks.nim b/lib/system/gc_hooks.nim index 70f02e6574..ace62eea0a 100644 --- a/lib/system/gc_hooks.nim +++ b/lib/system/gc_hooks.nim @@ -24,7 +24,7 @@ proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = inc globalMarkersLen else: cstderr.rawWrite("[GC] cannot register global variable; too many global variables") - quit 1 + rawQuit 1 proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = if threadLocalMarkersLen <= high(threadLocalMarkers): @@ -32,7 +32,7 @@ proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} inc threadLocalMarkersLen else: cstderr.rawWrite("[GC] cannot register thread local variable; too many thread local variables") - quit 1 + rawQuit 1 proc traverseGlobals*() = for i in 0..globalMarkersLen-1: diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 852f5d7aa4..c885a6893e 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -27,7 +27,7 @@ when defined(memProfiler): proc nimProfile(requestedSize: int) when hasThreadSupport: - import sharedlist + import std/sharedlist type WalkOp = enum @@ -36,7 +36,7 @@ type # local waMarkPrecise # fast precise marking - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -90,15 +90,15 @@ template gcAssert(cond: bool, msg: string) = if not cond: cstderr.rawWrite "[GCASSERT] " cstderr.rawWrite msg - quit 1 + rawQuit 1 proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -115,10 +115,10 @@ when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var GcHeap; size: int) {.benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap; size: int) {.benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes when defined(nimGcRefLeak): @@ -217,7 +217,7 @@ proc initGC() = gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -229,7 +229,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -255,7 +255,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: when not defined(nimSeqsV2): - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -268,7 +268,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ when leakDetector and not hasThreadSupport: @@ -336,9 +336,9 @@ when not defined(nimSeqsV2): var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res)+% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") when withBitvectors: incl(gch.allocated, res) when useCellIds: inc gch.idGenerator @@ -428,15 +428,6 @@ proc sweep(gch: var GcHeap) = if c.refcount == rcBlack: c.refcount = rcWhite else: freeCyclicCell(gch, c) -when false: - proc newGcInvariant*() = - for x in allObjects(gch.region): - if isCell(x): - var c = cast[PCell](x) - if c.typ == nil: - writeStackTrace() - quit 1 - proc markGlobals(gch: var GcHeap) = if gch.gcThreadId == 0: when defined(nimTracing): @@ -455,11 +446,10 @@ proc markGlobals(gch: var GcHeap) = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: mark(gch, objStart) diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim index 6ced04c998..d96de7eac5 100644 --- a/lib/system/gc_regions.nim +++ b/lib/system/gc_regions.nim @@ -7,6 +7,7 @@ # # "Stack GC" for embedded devices or ultra performance requirements. +import std/private/syslocks when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} diff --git a/lib/system/hti.nim b/lib/system/hti.nim index 3dbcd7615c..9acaae88bf 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -40,7 +40,7 @@ type tyPointer, tyOpenArray, tyString, - tyCString, + tyCstring, tyForward, tyInt, tyInt8, diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 0bc51d693e..3bf0b98930 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -30,12 +30,13 @@ when defined(createNimRtl): {.pragma: inl.} {.pragma: compilerRtl, compilerproc, exportc: "nimrtl_$1", dynlib.} elif defined(useNimRtl): - when defined(windows): - const nimrtl* = "nimrtl.dll" - elif defined(macosx): - const nimrtl* = "libnimrtl.dylib" - else: - const nimrtl* = "libnimrtl.so" + #[ + `{.rtl.}` should only be used for non-generic procs. + ]# + const nimrtl* = + when defined(windows): "nimrtl.dll" + elif defined(macosx): "libnimrtl.dylib" + else: "libnimrtl.so" {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl, gcsafe.} {.pragma: inl.} {.pragma: compilerRtl, compilerproc, importc: "nimrtl_$1", dynlib: nimrtl.} @@ -44,7 +45,6 @@ else: {.pragma: inl, inline.} {.pragma: compilerRtl, compilerproc.} -{.pragma: benign, gcsafe, locks: 0.} +{.pragma: benign, gcsafe.} -when defined(nimHasSinkInference): - {.push sinkInference: on.} +{.push sinkInference: on.} diff --git a/lib/system/indexerrors.nim b/lib/system/indexerrors.nim index 1b91789bd7..6a8cb8a0ae 100644 --- a/lib/system/indexerrors.nim +++ b/lib/system/indexerrors.nim @@ -1,4 +1,5 @@ # imported by other modules, unlike helpers.nim which is included +# xxx this is now included instead of imported, we should import instead template formatErrorIndexBound*[T](i, a, b: T): string = when defined(standalone): @@ -9,3 +10,6 @@ template formatErrorIndexBound*[T](i, a, b: T): string = template formatErrorIndexBound*[T](i, n: T): string = formatErrorIndexBound(i, 0, n) + +template formatFieldDefect*(f, discVal): string = + f & discVal & "'" diff --git a/lib/system/indices.nim b/lib/system/indices.nim new file mode 100644 index 0000000000..f4a1403464 --- /dev/null +++ b/lib/system/indices.nim @@ -0,0 +1,158 @@ +when not defined(nimHasSystemRaisesDefect): + {.pragma: systemRaisesDefect.} + +type + BackwardsIndex* = distinct int ## Type that is constructed by `^` for + ## reversed array accesses. + ## (See `^ template <#^.t,int>`_) + +template `^`*(x: int): BackwardsIndex = BackwardsIndex(x) + ## Builtin `roof`:idx: operator that can be used for convenient array access. + ## `a[^x]` is a shortcut for `a[a.len-x]`. + ## + ## ```nim + ## let + ## a = [1, 3, 5, 7, 9] + ## b = "abcdefgh" + ## + ## echo a[^1] # => 9 + ## echo b[^2] # => g + ## ``` + +proc `[]`*[T](s: openArray[T]; i: BackwardsIndex): T {.inline, systemRaisesDefect.} = + system.`[]`(s, s.len - int(i)) + +proc `[]`*[Idx, T](a: array[Idx, T]; i: BackwardsIndex): T {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] +proc `[]`*(s: string; i: BackwardsIndex): char {.inline, systemRaisesDefect.} = s[s.len - int(i)] + +proc `[]`*[T](s: var openArray[T]; i: BackwardsIndex): var T {.inline, systemRaisesDefect.} = + system.`[]`(s, s.len - int(i)) +proc `[]`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex): var T {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] +proc `[]`*(s: var string; i: BackwardsIndex): var char {.inline, systemRaisesDefect.} = s[s.len - int(i)] + +proc `[]=`*[T](s: var openArray[T]; i: BackwardsIndex; x: T) {.inline, systemRaisesDefect.} = + system.`[]=`(s, s.len - int(i), x) +proc `[]=`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex; x: T) {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] = x +proc `[]=`*(s: var string; i: BackwardsIndex; x: char) {.inline, systemRaisesDefect.} = + s[s.len - int(i)] = x + +template `..^`*(a, b: untyped): untyped = + ## A shortcut for `.. ^` to avoid the common gotcha that a space between + ## '..' and '^' is required. + a .. ^b + +template `..<`*(a, b: untyped): untyped = + ## A shortcut for `a .. pred(b)`. + ## ```nim + ## for i in 5 ..< 9: + ## echo i # => 5; 6; 7; 8 + ## ``` + a .. (when b is BackwardsIndex: succ(b) else: pred(b)) + +template `[]`*(s: string; i: int): char = arrGet(s, i) +template `[]=`*(s: string; i: int; val: char) = arrPut(s, i, val) + +template `^^`(s, i: untyped): untyped = + (when i is BackwardsIndex: s.len - int(i) else: int(i)) + +template spliceImpl(s, a, L, b: typed): untyped = + # make room for additional elements or cut: + var shift = b.len - max(0,L) # ignore negative slice size + var newLen = s.len + shift + if shift > 0: + # enlarge: + setLen(s, newLen) + for i in countdown(newLen-1, a+b.len): movingCopy(s[i], s[i-shift]) + else: + for i in countup(a+b.len, newLen-1): movingCopy(s[i], s[i-shift]) + # cut down: + setLen(s, newLen) + # fill the hole: + for i in 0 ..< b.len: s[a+i] = b[i] + +proc `[]`*[T, U: Ordinal](s: string, x: HSlice[T, U]): string {.inline, systemRaisesDefect.} = + ## Slice operation for strings. + ## Returns the inclusive range `[s[x.a], s[x.b]]`: + ## ```nim + ## var s = "abcdef" + ## assert s[1..3] == "bcd" + ## ``` + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + result = newString(L) + for i in 0 ..< L: result[i] = s[i + a] + +proc `[]=`*[T, U: Ordinal](s: var string, x: HSlice[T, U], b: string) {.systemRaisesDefect.} = + ## Slice assignment for strings. + ## + ## If `b.len` is not exactly the number of elements that are referred to + ## by `x`, a `splice`:idx: is performed: + ## + runnableExamples: + var s = "abcdefgh" + s[1 .. ^2] = "xyz" + assert s == "axyzh" + + var a = s ^^ x.a + var L = (s ^^ x.b) - a + 1 + if L == b.len: + for i in 0..<L: s[i+a] = b[i] + else: + spliceImpl(s, a, L, b) + +proc `[]`*[Idx, T; U, V: Ordinal](a: array[Idx, T], x: HSlice[U, V]): seq[T] {.systemRaisesDefect.} = + ## Slice operation for arrays. + ## Returns the inclusive range `[a[x.a], a[x.b]]`: + ## ```nim + ## var a = [1, 2, 3, 4] + ## assert a[0..2] == @[1, 2, 3] + ## ``` + let xa = a ^^ x.a + let L = (a ^^ x.b) - xa + 1 + result = newSeq[T](L) + for i in 0..<L: result[i] = a[Idx(i + xa)] + +proc `[]=`*[Idx, T; U, V: Ordinal](a: var array[Idx, T], x: HSlice[U, V], b: openArray[T]) {.systemRaisesDefect.} = + ## Slice assignment for arrays. + ## ```nim + ## var a = [10, 20, 30, 40, 50] + ## a[1..2] = @[99, 88] + ## assert a == [10, 99, 88, 40, 50] + ## ``` + let xa = a ^^ x.a + let L = (a ^^ x.b) - xa + 1 + if L == b.len: + for i in 0..<L: a[Idx(i + xa)] = b[i] + else: + sysFatal(RangeDefect, "different lengths for slice assignment") + +proc `[]`*[T; U, V: Ordinal](s: openArray[T], x: HSlice[U, V]): seq[T] {.systemRaisesDefect.} = + ## Slice operation for sequences. + ## Returns the inclusive range `[s[x.a], s[x.b]]`: + ## ```nim + ## var s = @[1, 2, 3, 4] + ## assert s[0..2] == @[1, 2, 3] + ## ``` + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + newSeq(result, L) + for i in 0 ..< L: result[i] = s[i + a] + +proc `[]=`*[T; U, V: Ordinal](s: var seq[T], x: HSlice[U, V], b: openArray[T]) {.systemRaisesDefect.} = + ## Slice assignment for sequences. + ## + ## If `b.len` is not exactly the number of elements that are referred to + ## by `x`, a `splice`:idx: is performed. + runnableExamples: + var s = @"abcdefgh" + s[1 .. ^2] = @"xyz" + assert s == @"axyzh" + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + if L == b.len: + for i in 0 ..< L: s[i+a] = b[i] + else: + spliceImpl(s, a, L, b) diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index d8af4eed56..125bee98ff 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -1,14 +1,24 @@ -when defined(nimHasLentIterators) and not defined(nimWorkaround14447): +## Default iterators for some Nim types. + +when defined(nimPreviewSlimSystem): + import std/assertions + +when not defined(nimNoLentIterators): template lent2(T): untyped = lent T else: template lent2(T): untyped = T +template unCheckedInc(x) = + {.push overflowChecks: off.} + inc(x) + {.pop.} + iterator items*[T: not char](a: openArray[T]): lent2 T {.inline.} = ## Iterates over each item of `a`. var i = 0 while i < len(a): yield a[i] - inc(i) + unCheckedInc(i) iterator items*[T: char](a: openArray[T]): T {.inline.} = ## Iterates over each item of `a`. @@ -18,32 +28,32 @@ iterator items*[T: char](a: openArray[T]): T {.inline.} = var i = 0 while i < len(a): yield a[i] - inc(i) + unCheckedInc(i) iterator mitems*[T](a: var openArray[T]): var T {.inline.} = ## Iterates over each item of `a` so that you can modify the yielded value. var i = 0 while i < len(a): yield a[i] - inc(i) + unCheckedInc(i) iterator items*[IX, T](a: array[IX, T]): T {.inline.} = ## Iterates over each item of `a`. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield a[i] if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator mitems*[IX, T](a: var array[IX, T]): var T {.inline.} = ## Iterates over each item of `a` so that you can modify the yielded value. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield a[i] if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator items*[T](a: set[T]): T {.inline.} = ## Iterates over each element of `a`. `items` iterates only over the @@ -51,8 +61,11 @@ iterator items*[T](a: set[T]): T {.inline.} = ## able to hold). var i = low(T).int while i <= high(T).int: - if T(i) in a: yield T(i) - inc(i) + when T is enum and not defined(js): + if cast[T](i) in a: yield cast[T](i) + else: + if T(i) in a: yield T(i) + unCheckedInc(i) iterator items*(a: cstring): char {.inline.} = ## Iterates over each item of `a`. @@ -71,7 +84,7 @@ iterator items*(a: cstring): char {.inline.} = let n = len(a) while i < n: yield a[i] - inc(i) + unCheckedInc(i) when defined(js): impl() else: when nimvm: @@ -81,7 +94,7 @@ iterator items*(a: cstring): char {.inline.} = var i = 0 while a[i] != '\0': yield a[i] - inc(i) + unCheckedInc(i) iterator mitems*(a: var cstring): var char {.inline.} = ## Iterates over each item of `a` so that you can modify the yielded value. @@ -89,6 +102,7 @@ iterator mitems*(a: var cstring): var char {.inline.} = runnableExamples: from std/sugar import collect var a = "abc\0def" + prepareMutation(a) var b = a.cstring let s = collect: for bi in mitems(b): @@ -103,7 +117,7 @@ iterator mitems*(a: var cstring): var char {.inline.} = let n = len(a) while i < n: yield a[i] - inc(i) + unCheckedInc(i) when defined(js): impl() else: when nimvm: impl() @@ -111,7 +125,7 @@ iterator mitems*(a: var cstring): var char {.inline.} = var i = 0 while a[i] != '\0': yield a[i] - inc(i) + unCheckedInc(i) iterator items*[T: enum and Ordinal](E: typedesc[T]): T = ## Iterates over the values of `E`. @@ -134,7 +148,7 @@ iterator pairs*[T](a: openArray[T]): tuple[key: int, val: T] {.inline.} = var i = 0 while i < len(a): yield (i, a[i]) - inc(i) + unCheckedInc(i) iterator mpairs*[T](a: var openArray[T]): tuple[key: int, val: var T]{.inline.} = ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. @@ -142,26 +156,26 @@ iterator mpairs*[T](a: var openArray[T]): tuple[key: int, val: var T]{.inline.} var i = 0 while i < len(a): yield (i, a[i]) - inc(i) + unCheckedInc(i) iterator pairs*[IX, T](a: array[IX, T]): tuple[key: IX, val: T] {.inline.} = ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield (i, a[i]) if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator mpairs*[IX, T](a: var array[IX, T]): tuple[key: IX, val: var T] {.inline.} = ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. ## `a[index]` can be modified. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield (i, a[i]) if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator pairs*[T](a: seq[T]): tuple[key: int, val: T] {.inline.} = ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. @@ -169,7 +183,7 @@ iterator pairs*[T](a: seq[T]): tuple[key: int, val: T] {.inline.} = let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator mpairs*[T](a: var seq[T]): tuple[key: int, val: var T] {.inline.} = @@ -179,7 +193,7 @@ iterator mpairs*[T](a: var seq[T]): tuple[key: int, val: var T] {.inline.} = let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator pairs*(a: string): tuple[key: int, val: char] {.inline.} = @@ -188,7 +202,7 @@ iterator pairs*(a: string): tuple[key: int, val: char] {.inline.} = let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") iterator mpairs*(a: var string): tuple[key: int, val: var char] {.inline.} = @@ -198,7 +212,7 @@ iterator mpairs*(a: var string): tuple[key: int, val: var char] {.inline.} = let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") iterator pairs*(a: cstring): tuple[key: int, val: char] {.inline.} = @@ -208,12 +222,12 @@ iterator pairs*(a: cstring): tuple[key: int, val: char] {.inline.} = var L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) else: var i = 0 while a[i] != '\0': yield (i, a[i]) - inc(i) + unCheckedInc(i) iterator mpairs*(a: var cstring): tuple[key: int, val: var char] {.inline.} = ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. @@ -223,12 +237,12 @@ iterator mpairs*(a: var cstring): tuple[key: int, val: var char] {.inline.} = var L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) else: var i = 0 while a[i] != '\0': yield (i, a[i]) - inc(i) + unCheckedInc(i) iterator items*[T](a: seq[T]): lent2 T {.inline.} = ## Iterates over each item of `a`. @@ -236,7 +250,7 @@ iterator items*[T](a: seq[T]): lent2 T {.inline.} = let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator mitems*[T](a: var seq[T]): var T {.inline.} = @@ -245,7 +259,7 @@ iterator mitems*[T](a: var seq[T]): var T {.inline.} = let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator items*(a: string): char {.inline.} = @@ -254,7 +268,7 @@ iterator items*(a: string): char {.inline.} = let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") iterator mitems*(a: var string): var char {.inline.} = @@ -263,7 +277,7 @@ iterator mitems*(a: var string): var char {.inline.} = let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") @@ -304,7 +318,7 @@ iterator fieldPairs*[T: tuple|object](x: T): tuple[key: string, val: RootObj] {. ## picking the appropriate code to a secondary proc which you overload for ## each field type and pass the `value` to. ## - ## .. warning::: This really transforms the 'for' and unrolls the loop. The + ## .. warning:: This really transforms the 'for' and unrolls the loop. The ## current implementation also has a bug that affects symbol binding in the ## loop body. runnableExamples: diff --git a/lib/system/iterators_1.nim b/lib/system/iterators_1.nim index be61fd62ce..d00e3f823f 100644 --- a/lib/system/iterators_1.nim +++ b/lib/system/iterators_1.nim @@ -16,7 +16,7 @@ iterator countdown*[T](a, b: T, step: Positive = 1): T {.inline.} = let x = collect(newSeq): for i in countdown(7, 3): i - + assert x == @[7, 6, 5, 4, 3] let y = collect(newseq): @@ -32,7 +32,10 @@ iterator countdown*[T](a, b: T, step: Positive = 1): T {.inline.} = elif T is IntLikeForCount and T is Ordinal: var res = int(a) while res >= int(b): - yield T(res) + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: + yield T(res) dec(res, step) else: var res = a @@ -64,7 +67,10 @@ iterator countup*[T](a, b: T, step: Positive = 1): T {.inline.} = when T is IntLikeForCount and T is Ordinal: var res = int(a) while res <= int(b): - yield T(res) + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: + yield T(res) inc(res, step) else: var res = a @@ -89,7 +95,10 @@ iterator `..`*[T](a, b: T): T {.inline.} = when T is IntLikeForCount and T is Ordinal: var res = int(a) while res <= int(b): - yield T(res) + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: + yield T(res) inc(res) else: var res = a diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index ef06437e51..5599240fdb 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -49,7 +49,7 @@ proc nimCharToStr(x: char): string {.compilerproc.} = result[0] = x proc isNimException(): bool {.asmNoStackFrame.} = - asm "return `lastJSError` && `lastJSError`.m_type;" + {.emit: "return `lastJSError` && `lastJSError`.m_type;".} proc getCurrentException*(): ref Exception {.compilerRtl, benign.} = if isNimException(): result = cast[ref Exception](lastJSError) @@ -69,6 +69,13 @@ proc getCurrentExceptionMsg*(): string = return $msg return "" +proc setCurrentException*(exc: ref Exception) = + lastJSError = cast[PJSError](exc) + +proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = + ## Used to set up exception handling for closure iterators + setCurrentException(e) + proc auxWriteStackTrace(f: PCallFrame): string = type TempFrame = tuple[procname: cstring, line: int, filename: cstring] @@ -127,7 +134,8 @@ proc unhandledException(e: ref Exception) {. when NimStackTrace: add(buf, rawWriteStackTrace()) let cbuf = cstring(buf) - framePtr = nil + when NimStackTrace: + framePtr = nil {.emit: """ if (typeof(Error) !== "undefined") { throw new Error(`cbuf`); @@ -144,7 +152,7 @@ proc raiseException(e: ref Exception, ename: cstring) {. unhandledException(e) when NimStackTrace: e.trace = rawWriteStackTrace() - asm "throw `e`;" + {.emit: "throw `e`;".} proc reraiseException() {.compilerproc, asmNoStackFrame.} = if lastJSError == nil: @@ -154,7 +162,7 @@ proc reraiseException() {.compilerproc, asmNoStackFrame.} = if isNimException(): unhandledException(cast[ref Exception](lastJSError)) - asm "throw lastJSError;" + {.emit: "throw lastJSError;".} proc raiseOverflow {.exportc: "raiseOverflow", noreturn, compilerproc.} = raise newException(OverflowDefect, "over- or underflow") @@ -168,11 +176,11 @@ proc raiseRangeError() {.compilerproc, noreturn.} = proc raiseIndexError(i, a, b: int) {.compilerproc, noreturn.} = raise newException(IndexDefect, formatErrorIndexBound(int(i), int(a), int(b))) -proc raiseFieldError(f: string) {.compilerproc, noreturn.} = - raise newException(FieldDefect, f) +proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noreturn.} = + raise newException(FieldDefect, formatFieldDefect(f, discVal)) proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = {}; for (var i = 0; i < arguments.length; ++i) { var x = arguments[i]; @@ -185,7 +193,7 @@ proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = } } return result; - """ + """.} proc makeNimstrLit(c: cstring): string {.asmNoStackFrame, compilerproc.} = {.emit: """ @@ -273,62 +281,64 @@ proc toJSStr(s: string): cstring {.compilerproc.} = result = join(res) proc mnewString(len: int): string {.asmNoStackFrame, compilerproc.} = - asm """ - return new Array(`len`); - """ + {.emit: """ + var result = new Array(`len`); + for (var i = 0; i < `len`; i++) {result[i] = 0;} + return result; + """.} proc SetCard(a: int): int {.compilerproc, asmNoStackFrame.} = # argument type is a fake - asm """ + {.emit: """ var result = 0; for (var elem in `a`) { ++result; } return result; - """ + """.} proc SetEq(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ for (var elem in `a`) { if (!`b`[elem]) return false; } for (var elem in `b`) { if (!`a`[elem]) return false; } return true; - """ + """.} proc SetLe(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ for (var elem in `a`) { if (!`b`[elem]) return false; } return true; - """ + """.} proc SetLt(a, b: int): bool {.compilerproc.} = result = SetLe(a, b) and not SetEq(a, b) proc SetMul(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { if (`b`[elem]) { result[elem] = true; } } return result; - """ + """.} proc SetPlus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { result[elem] = true; } for (var elem in `b`) { result[elem] = true; } return result; - """ + """.} proc SetMinus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { if (!`b`[elem]) { result[elem] = true; } } return result; - """ + """.} proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`a` == `b`) return 0; if (!`a`) return -1; if (!`b`) return 1; @@ -337,7 +347,7 @@ proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerproc.} = if (result != 0) return result; } return `a`.length - `b`.length; - """ + """.} proc cmp(x, y: string): int = when nimvm: @@ -348,7 +358,7 @@ proc cmp(x, y: string): int = result = cmpStrings(x, y) proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`a` == `b`) return true; if (`a` === null && `b`.length == 0) return true; if (`b` === null && `a`.length == 0) return true; @@ -358,29 +368,29 @@ proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerproc.} = for (var i = 0; i < alen; ++i) if (`a`[i] != `b`[i]) return false; return true; - """ + """.} when defined(kwin): proc rawEcho {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { buf += `toJSStr`(arguments[i]); } print(buf); - """ + """.} elif not defined(nimOldEcho): proc ewriteln(x: cstring) = log(x) proc rawEcho {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { buf += `toJSStr`(arguments[i]); } console.log(buf); - """ + """.} else: proc ewriteln(x: cstring) = @@ -408,84 +418,84 @@ else: # Arithmetic: proc checkOverflowInt(a: int) {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`a` > 2147483647 || `a` < -2147483648) `raiseOverflow`(); - """ + """.} proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` + `b`; `checkOverflowInt`(result); return result; - """ + """.} proc subInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` - `b`; `checkOverflowInt`(result); return result; - """ + """.} proc mulInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` * `b`; `checkOverflowInt`(result); return result; - """ + """.} proc divInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); return Math.trunc(`a` / `b`); - """ + """.} proc modInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); return Math.trunc(`a` % `b`); - """ + """.} -proc checkOverflowInt64(a: int) {.asmNoStackFrame, compilerproc.} = - asm """ - if (`a` > 9223372036854775807 || `a` < -9223372036854775808) `raiseOverflow`(); - """ +proc checkOverflowInt64(a: int64) {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`a` > 9223372036854775807n || `a` < -9223372036854775808n) `raiseOverflow`(); + """.} -proc addInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc addInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` + `b`; `checkOverflowInt64`(result); return result; - """ + """.} -proc subInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc subInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` - `b`; `checkOverflowInt64`(result); return result; - """ + """.} -proc mulInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc mulInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` * `b`; `checkOverflowInt64`(result); return result; - """ + """.} -proc divInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.trunc(`a` / `b`); - """ +proc divInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`b` == 0n) `raiseDivByZero`(); + if (`b` == -1n && `a` == 9223372036854775807n) `raiseOverflow`(); + return `a` / `b`; + """.} -proc modInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.trunc(`a` % `b`); - """ +proc modInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`b` == 0n) `raiseDivByZero`(); + if (`b` == -1n && `a` == 9223372036854775807n) `raiseOverflow`(); + return `a` % `b`; + """.} proc negInt(a: int): int {.compilerproc.} = result = a*(-1) @@ -493,55 +503,16 @@ proc negInt(a: int): int {.compilerproc.} = proc negInt64(a: int64): int64 {.compilerproc.} = result = a*(-1) -proc nimFloatToString(a: float): cstring {.compilerproc.} = - ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2 - # print `-0.0` properly - asm """ - function nimOnlyDigitsOrMinus(n) { - return n.toString().match(/^-?\d+$/); - } - if (Number.isSafeInteger(`a`)) - `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0" - else { - `result` = `a`+"" - if(nimOnlyDigitsOrMinus(`result`)){ - `result` = `a`+".0" - } - } - """ - proc absInt(a: int): int {.compilerproc.} = result = if a < 0: a*(-1) else: a proc absInt64(a: int64): int64 {.compilerproc.} = result = if a < 0: a*(-1) else: a -when not defined(nimNoZeroExtendMagic): - proc ze*(a: int): int {.compilerproc.} = - result = a - - proc ze64*(a: int64): int64 {.compilerproc.} = - result = a - - proc toU8*(a: int): int8 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU16*(a: int): int16 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU32*(a: int64): int32 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b -proc chckNilDisp(p: pointer) {.compilerproc.} = +proc chckNilDisp(p: JSRef) {.compilerproc.} = if p == nil: sysFatal(NilAccessDefect, "cannot dispatch; dispatcher is nil") @@ -559,22 +530,22 @@ proc nimCopyAux(dest, src: JSRef, n: ptr TNimNode) {.compilerproc.} = case n.kind of nkNone: sysAssert(false, "nimCopyAux") of nkSlot: - asm """ + {.emit: """ `dest`[`n`.offset] = nimCopy(`dest`[`n`.offset], `src`[`n`.offset], `n`.typ); - """ + """.} of nkList: - asm """ + {.emit: """ for (var i = 0; i < `n`.sons.length; i++) { nimCopyAux(`dest`, `src`, `n`.sons[i]); } - """ + """.} of nkCase: - asm """ + {.emit: """ `dest`[`n`.offset] = nimCopy(`dest`[`n`.offset], `src`[`n`.offset], `n`.typ); for (var i = 0; i < `n`.sons.length; ++i) { nimCopyAux(`dest`, `src`, `n`.sons[i][1]); } - """ + """.} proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = case ti.kind @@ -582,9 +553,9 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = if not isFatPointer(ti): result = src else: - asm "`result` = [`src`[0], `src`[1]];" + {.emit: "`result` = [`src`[0], `src`[1]];".} of tySet: - asm """ + {.emit: """ if (`dest` === null || `dest` === undefined) { `dest` = {}; } @@ -593,80 +564,72 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = } for (var key in `src`) { `dest`[key] = `src`[key]; } `result` = `dest`; - """ + """.} of tyTuple, tyObject: if ti.base != nil: result = nimCopy(dest, src, ti.base) elif ti.kind == tyObject: - asm "`result` = (`dest` === null || `dest` === undefined) ? {m_type: `ti`} : `dest`;" + {.emit: "`result` = (`dest` === null || `dest` === undefined) ? {m_type: `ti`} : `dest`;".} else: - asm "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;" + {.emit: "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;".} nimCopyAux(result, src, ti.node) - of tySequence, tyArrayConstr, tyOpenArray, tyArray: - asm """ + of tyArrayConstr, tyArray: + # In order to prevent a type change (TypedArray -> Array) and to have better copying performance, + # arrays constructors are considered separately + {.emit: """ + if(ArrayBuffer.isView(`src`)) { + if(`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new `src`.constructor(`src`); + } else { + `dest`.set(`src`, 0); + } + `result` = `dest`; + } else { + if (`src` === null) { + `result` = null; + } + else { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new Array(`src`.length); + } + `result` = `dest`; + for (var i = 0; i < `src`.length; ++i) { + `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + } + } + } + """.} + of tySequence, tyOpenArray: + {.emit: """ if (`src` === null) { `result` = null; } else { - if (`dest` === null || `dest` === undefined) { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { `dest` = new Array(`src`.length); } - else { - `dest`.length = `src`.length; - } `result` = `dest`; for (var i = 0; i < `src`.length; ++i) { `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); } } - """ + """.} of tyString: - asm """ + {.emit: """ if (`src` !== null) { `result` = `src`.slice(0); } - """ + """.} else: result = src -proc genericReset(x: JSRef, ti: PNimType): JSRef {.compilerproc.} = - asm "`result` = null;" - case ti.kind - of tyPtr, tyRef, tyVar, tyNil: - if isFatPointer(ti): - asm """ - `result` = [null, 0]; - """ - of tySet: - asm """ - `result` = {}; - """ - of tyTuple, tyObject: - if ti.kind == tyObject: - asm "`result` = {m_type: `ti`};" - else: - asm "`result` = {};" - of tySequence, tyOpenArray, tyString: - asm """ - `result` = []; - """ - of tyArrayConstr, tyArray: - asm """ - `result` = new Array(`x`.length); - for (var i = 0; i < `x`.length; ++i) { - `result`[i] = genericReset(`x`[i], `ti`.base); - } - """ - else: - discard - proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. asmNoStackFrame, compilerproc.} = # types are fake - asm """ + {.emit: """ var result = new Array(`len`); for (var i = 0; i < `len`; ++i) result[i] = nimCopy(null, `value`, `typ`); return result; - """ + """.} proc chckIndx(i, a, b: int): int {.compilerproc.} = if i >= a and i <= b: return i @@ -695,11 +658,12 @@ proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = return true proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} = - asm "`x`.push(`c`);" + {.emit: "`x`.push(`c`);".} {.pop.} proc tenToThePowerOf(b: int): BiggestFloat = + # xxx deadcode var b = b var a = 10.0 result = 1.0 @@ -713,80 +677,75 @@ proc tenToThePowerOf(b: int): BiggestFloat = const IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} -# XXX use JS's native way here -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int {. - compilerproc.} = - var - esign = 1.0 - sign = 1.0 - i = start - exponent: int - flags: int - number = 0.0 + +proc parseFloatNative(a: openarray[char]): float = + var str = "" + for x in a: + str.add x + + let cstr = cstring str + + {.emit: """ + `result` = Number(`cstr`); + """.} + +proc nimParseBiggestFloat(s: openarray[char], number: var BiggestFloat): int {.compilerproc.} = + var sign: bool + var i = 0 if s[i] == '+': inc(i) elif s[i] == '-': - sign = -1.0 + sign = true inc(i) if s[i] == 'N' or s[i] == 'n': if s[i+1] == 'A' or s[i+1] == 'a': if s[i+2] == 'N' or s[i+2] == 'n': if s[i+3] notin IdentChars: number = NaN - return i+3 - start + return i+3 return 0 if s[i] == 'I' or s[i] == 'i': if s[i+1] == 'N' or s[i+1] == 'n': if s[i+2] == 'F' or s[i+2] == 'f': if s[i+3] notin IdentChars: - number = Inf*sign - return i+3 - start + number = if sign: -Inf else: Inf + return i+3 return 0 - while s[i] in {'0'..'9'}: - # Read integer part - flags = flags or 1 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - inc(i) - while s[i] == '_': inc(i) - # Decimal? - if s[i] == '.': - var hd = 1.0 - inc(i) - while s[i] in {'0'..'9'}: - # Read fractional part - flags = flags or 2 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - hd = hd * 10.0 - inc(i) - while s[i] == '_': inc(i) - number = number / hd # this complicated way preserves precision - # Again, read integer and fractional part - if flags == 0: return 0 - # Exponent? - if s[i] in {'e', 'E'}: - inc(i) - if s[i] == '+': - inc(i) - elif s[i] == '-': - esign = -1.0 - inc(i) - if s[i] notin {'0'..'9'}: - return 0 - while s[i] in {'0'..'9'}: - exponent = exponent * 10 + ord(s[i]) - ord('0') - inc(i) - while s[i] == '_': inc(i) - # Calculate Exponent - let hd = tenToThePowerOf(exponent) - if esign > 0.0: number = number * hd - else: number = number / hd - # evaluate sign - number = number * sign - result = i - start + var buf: string + # we could also use an `array[char, N]` buffer to avoid reallocs, or + # use a 2-pass algorithm that first computes the length. + if sign: buf.add '-' + template addInc = + buf.add s[i] + inc(i) + template eatUnderscores = + while s[i] == '_': inc(i) + while s[i] in {'0'..'9'}: # Read integer part + buf.add s[i] + inc(i) + eatUnderscores() + if s[i] == '.': # Decimal? + addInc() + while s[i] in {'0'..'9'}: # Read fractional part + addInc() + eatUnderscores() + # Again, read integer and fractional part + if buf.len == ord(sign): return 0 + if s[i] in {'e', 'E'}: # Exponent? + addInc() + if s[i] == '+': inc(i) + elif s[i] == '-': addInc() + if s[i] notin {'0'..'9'}: return 0 + while s[i] in {'0'..'9'}: + addInc() + eatUnderscores() + number = parseFloatNative(buf) + result = i # Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce # 'Math.trunc' for Nim's ``div`` and ``mod`` operators: -const jsMathTrunc = """ +when defined(nimJsMathTruncPolyfill): + {.emit: """ if (!Math.trunc) { Math.trunc = function(v) { v = +v; @@ -794,5 +753,16 @@ if (!Math.trunc) { return (v - v % 1) || (v < 0 ? -0 : v === 0 ? v : 0); }; } -""" -when not defined(nodejs): {.emit: jsMathTrunc .} +""".} + +proc cmpClosures(a, b: JSRef): bool {.compilerproc, asmNoStackFrame.} = + # Both `a` and `b` need to be a closure + {.emit: """ + if (`a` !== null && `a`.ClP_0 !== undefined && + `b` !== null && `b`.ClP_0 !== undefined) { + return `a`.ClP_0 == `b`.ClP_0 && `a`.ClE_0 == `b`.ClE_0; + } else { + return `a` == `b`; + } + """ + .} diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim index fca1db91dd..a94d0995c8 100644 --- a/lib/system/memalloc.nim +++ b/lib/system/memalloc.nim @@ -1,13 +1,13 @@ when notJSnotNims: proc zeroMem*(p: pointer, size: Natural) {.inline, noSideEffect, - tags: [], locks: 0, raises: [].} + tags: [], raises: [].} ## Overwrites the contents of the memory at `p` with the value 0. ## ## Exactly `size` bytes will be overwritten. Like any procedure ## dealing with raw memory this is **unsafe**. proc copyMem*(dest, source: pointer, size: Natural) {.inline, benign, - tags: [], locks: 0, raises: [].} + tags: [], raises: [].} ## Copies the contents from the memory at `source` to the memory ## at `dest`. ## Exactly `size` bytes will be copied. The memory @@ -15,7 +15,7 @@ when notJSnotNims: ## memory this is **unsafe**. proc moveMem*(dest, source: pointer, size: Natural) {.inline, benign, - tags: [], locks: 0, raises: [].} + tags: [], raises: [].} ## Copies the contents from the memory at `source` to the memory ## at `dest`. ## @@ -25,7 +25,7 @@ when notJSnotNims: ## dealing with raw memory this is still **unsafe**, though. proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect, - tags: [], locks: 0, raises: [].} + tags: [], raises: [].} ## Compares the memory blocks `a` and `b`. `size` bytes will ## be compared. ## @@ -34,7 +34,7 @@ when notJSnotNims: ## **unsafe**. proc cmpMem*(a, b: pointer, size: Natural): int {.inline, noSideEffect, - tags: [], locks: 0, raises: [].} + tags: [], raises: [].} ## Compares the memory blocks `a` and `b`. `size` bytes will ## be compared. ## @@ -80,7 +80,7 @@ when hasAlloc and not defined(js): when defined(nimAllocStats): var stats: AllocStats - template incStat(what: untyped) = inc stats.what + template incStat(what: untyped) = atomicInc stats.what proc getAllocStats*(): AllocStats = stats else: @@ -188,8 +188,8 @@ when hasAlloc and not defined(js): cast[ptr T](realloc(p, T.sizeof * newSize)) proc dealloc*(p: pointer) {.noconv, compilerproc, rtl, benign, raises: [], tags: [].} = - ## Frees the memory allocated with `alloc`, `alloc0` or - ## `realloc`. + ## Frees the memory allocated with `alloc`, `alloc0`, + ## `realloc`, `create` or `createU`. ## ## **This procedure is dangerous!** ## If one forgets to free the memory a leak occurs; if one tries to diff --git a/lib/system/memory.nim b/lib/system/memory.nim index ebda60d8d1..156773c484 100644 --- a/lib/system/memory.nim +++ b/lib/system/memory.nim @@ -43,6 +43,7 @@ proc nimCmpMem*(a, b: pointer, size: Natural): cint {.compilerproc, nonReloadabl inc i proc nimCStrLen*(a: cstring): int {.compilerproc, nonReloadable, inline.} = + if a.isNil: return 0 when useLibC: cast[int](c_strlen(a)) else: diff --git a/lib/system/memtracker.nim b/lib/system/memtracker.nim index f0c83f1fa6..289f4e0245 100644 --- a/lib/system/memtracker.nim +++ b/lib/system/memtracker.nim @@ -35,7 +35,7 @@ type count*: int disabled: bool data*: array[400, LogEntry] - TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], locks: 0, gcsafe.} + TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], gcsafe.} var gLog*: TrackLog @@ -70,9 +70,9 @@ proc addEntry(entry: LogEntry) = if interesting: gLog.disabled = true cprintf("interesting %s:%ld %s\n", entry.file, entry.line, entry.op) - let x = cast[proc() {.nimcall, tags: [], gcsafe, locks: 0, raises: [].}](writeStackTrace) + let x = cast[proc() {.nimcall, tags: [], gcsafe, raises: [].}](writeStackTrace) x() - quit 1 + rawQuit 1 #if gLog.count > high(gLog.data): # gLogger(gLog) # gLog.count = 0 @@ -85,7 +85,7 @@ proc memTrackerWrite(address: pointer; size: int; file: cstring; line: int) {.co size: size, file: file, line: line, thread: myThreadId()) proc memTrackerOp*(op: cstring; address: pointer; size: int) {.tags: [], - locks: 0, gcsafe.} = + gcsafe.} = addEntry LogEntry(op: op, address: address, size: size, file: "", line: 0, thread: myThreadId()) diff --git a/lib/system/mm/go.nim b/lib/system/mm/go.nim index 9ec25fb656..8f3aeb964c 100644 --- a/lib/system/mm/go.nim +++ b/lib/system/mm/go.nim @@ -109,7 +109,7 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = writebarrierptr(addr(result), newSeq(typ, len)) proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = - result = newObj(typ, align(GenericSeqSize, typ.base.align) + cap * typ.base.size) + result = newObjNoInit(typ, align(GenericSeqSize, typ.base.align) + cap * typ.base.size) cast[PGenericSeq](result).len = 0 cast[PGenericSeq](result).reserved = cap cast[PGenericSeq](result).elemSize = typ.base.size diff --git a/lib/system/mm/malloc.nim b/lib/system/mm/malloc.nim index 2ba5c0fecb..47f1a95ae4 100644 --- a/lib/system/mm/malloc.nim +++ b/lib/system/mm/malloc.nim @@ -2,18 +2,27 @@ {.push stackTrace: off.} proc allocImpl(size: Natural): pointer = - c_malloc(size.csize_t) + result = c_malloc(size.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc alloc0Impl(size: Natural): pointer = - c_calloc(size.csize_t, 1) + result = c_calloc(size.csize_t, 1) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc reallocImpl(p: pointer, newSize: Natural): pointer = - c_realloc(p, newSize.csize_t) + result = c_realloc(p, newSize.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc realloc0Impl(p: pointer, oldsize, newSize: Natural): pointer = result = realloc(p, newSize.csize_t) if newSize > oldSize: - zeroMem(cast[pointer](cast[int](result) + oldSize), newSize - oldSize) + zeroMem(cast[pointer](cast[uint](result) + uint(oldSize)), newSize - oldSize) proc deallocImpl(p: pointer) = c_free(p) @@ -79,7 +88,7 @@ type proc alloc(r: var MemRegion, size: int): pointer = result = alloc(size) -proc alloc0Impl(r: var MemRegion, size: int): pointer = +proc alloc0(r: var MemRegion, size: int): pointer = result = alloc0Impl(size) proc dealloc(r: var MemRegion, p: pointer) = dealloc(p) proc deallocOsPages(r: var MemRegion) = discard diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 96dc86bfd2..26f2f0bbf0 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -46,7 +46,7 @@ else: proc raiseOutOfMem() {.noinline.} = if outOfMemHook != nil: outOfMemHook() cstderr.rawWrite("out of memory\n") - quit(1) + rawQuit(1) when defined(boehmgc): include system / mm / boehm @@ -57,6 +57,16 @@ elif defined(gogc): elif (defined(nogc) or defined(gcDestructors)) and defined(useMalloc): include system / mm / malloc + when defined(nogc): + proc GC_getStatistics(): string = "" + proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = + result = alloc0(size) + + proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = + result = newObj(typ, align(GenericSeqSize, typ.align) + len * typ.base.size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + elif defined(nogc): include system / mm / none @@ -72,8 +82,9 @@ else: # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here include "system/gc_regions" elif defined(nimV2) or usesDestructors: - var allocator {.rtlThreadVar.}: MemRegion - instantiateForRegion(allocator) + when not defined(useNimRtl): + var allocator {.rtlThreadVar.}: MemRegion + instantiateForRegion(allocator) when defined(gcHooks): include "system/gc_hooks" elif defined(gcMarkAndSweep): diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 9ece10e3ea..bdf6145a11 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -86,12 +86,12 @@ proc patchFile*(package, filename, replacement: string) = ## is interpreted to be local to the Nimscript file that contains ## the call to `patchFile`, Nim's `--path` is not used at all ## to resolve the filename! + ## The compiler also performs `path substitution <nimc.html#compiler-usage-commandminusline-switches>`_ on `replacement`. ## ## Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## patchFile("stdlib", "asyncdispatch", "patches/replacement") + ## ``` discard proc getCommand*(): string = @@ -136,17 +136,8 @@ proc dirExists*(dir: string): bool {. ## Checks if the directory `dir` exists. builtin -template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = - # xxx: warning won't be shown for nimsscript because of current logic handling - # `foreignPackageNotes` - fileExists(args) - -template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = - dirExists(args) - -proc selfExe*(): string = +proc selfExe*(): string {.deprecated: "Deprecated since v1.7; Use getCurrentCompilerExe".} = ## Returns the currently running nim or nimble executable. - # TODO: consider making this as deprecated alias of `getCurrentCompilerExe` builtin proc toExe*(filename: string): string = @@ -167,20 +158,19 @@ proc strip(s: string): string = template `--`*(key, val: untyped) = ## A shortcut for `switch <#switch,string,string>`_ ## Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## --path:somePath # same as switch("path", "somePath") ## --path:"someOtherPath" # same as switch("path", "someOtherPath") + ## --hint:"[Conf]:off" # same as switch("hint", "[Conf]:off") + ## ``` switch(strip(astToStr(key)), strip(astToStr(val))) template `--`*(key: untyped) = ## A shortcut for `switch <#switch,string,string>`_ ## Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## --listCmd # same as switch("listCmd") + ## ``` switch(strip(astToStr(key))) type @@ -261,7 +251,7 @@ proc cpDir*(`from`, to: string) {.raises: [OSError].} = checkOsError() proc exec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with ## a non-zero exit code, an OSError exception is raised. ## @@ -274,7 +264,7 @@ proc exec*(command: string) {. checkOsError() proc exec*(command: string, input: string, cache = "") {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with ## a non-zero exit code, an OSError exception is raised. log "exec: " & command: @@ -284,7 +274,7 @@ proc exec*(command: string, input: string, cache = "") {. echo output proc selfExec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external command with the current nim/nimble executable. ## `Command` must not contain the "nim " part. let c = selfExe() & " " & command @@ -349,19 +339,19 @@ template withDir*(dir: string; body: untyped): untyped = ## ## If you need a permanent change, use the `cd() <#cd,string>`_ proc. ## Usage example: - ## - ## .. code-block:: nim + ## ```nim + ## # inside /some/path/ ## withDir "foo": - ## # inside foo - ## #back to last dir - var curDir = getCurrentDir() + ## # move to /some/path/foo/ + ## # back in /some/path/ + ## ``` + let curDir = getCurrentDir() try: cd(dir) body finally: cd(curDir) - proc writeTask(name, desc: string) = if desc.len > 0: var spaces = " " @@ -399,10 +389,10 @@ when not defined(nimble): ## Defines a task. Hidden tasks are supported via an empty description. ## ## Example: - ## - ## .. code-block:: nim - ## task build, "default build is via the C backend": - ## setCommand "c" + ## ```nim + ## task build, "default build is via the C backend": + ## setCommand "c" + ## ``` ## ## For a task named `foo`, this template generates a `proc` named ## `fooTask`. This is useful if you need to call one task in @@ -410,13 +400,14 @@ when not defined(nimble): ## ## Example: ## - ## .. code-block:: nim - ## task foo, "foo": # > nim foo - ## echo "Running foo" # Running foo + ## ```nim + ## task foo, "foo": # > nim foo + ## echo "Running foo" # Running foo ## - ## task bar, "bar": # > nim bar - ## echo "Running bar" # Running bar - ## fooTask() # Running foo + ## task bar, "bar": # > nim bar + ## echo "Running bar" # Running bar + ## fooTask() # Running foo + ## ``` proc `name Task`*() = setCommand "nop" body diff --git a/lib/system/orc.nim b/lib/system/orc.nim index 8a62ef4bb0..463c40c4db 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -8,13 +8,12 @@ # # Cycle collector based on -# https://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf +# https://www.cs.purdue.edu/homes/hosking/690M/Bacon01Concurrent.pdf # And ideas from Lins' in 2008 by the notion of "critical links", see # "Cyclic reference counting" by Rafael Dueire Lins # R.D. Lins / Information Processing Letters 109 (2008) 71–78 # -type PT = Cell include cellseqs_v2 const @@ -68,10 +67,10 @@ const type GcEnv = object - traceStack: CellSeq + traceStack: CellSeq[ptr pointer] when useJumpStack: - jumpStack: CellSeq # Lins' jump stack in order to speed up traversals - toFree: CellSeq + jumpStack: CellSeq[ptr pointer] # Lins' jump stack in order to speed up traversals + toFree: CellSeq[Cell] freed, touched, edges, rcSum: int keepThreshold: bool @@ -80,10 +79,16 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = var p = s +! sizeof(RefHeader) cast[TraceProc](desc.traceImpl)(p, addr(j)) -when logOrc: +include threadids + +when logOrc or orcLeakDetector: proc writeCell(msg: cstring; s: Cell; desc: PNimTypeV2) = - cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld\n", - msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color) + when orcLeakDetector: + cfprintf(cstderr, "%s %s file: %s:%ld; color: %ld; thread: %ld\n", + msg, desc.name, s.filename, s.line, s.color, getThreadId()) + else: + cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld; thread: %ld\n", + msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color, getThreadId()) proc free(s: Cell; desc: PNimTypeV2) {.inline.} = when traceCollector: @@ -92,13 +97,13 @@ proc free(s: Cell; desc: PNimTypeV2) {.inline.} = when logOrc: writeCell("free", s, desc) - if desc.disposeImpl != nil: - cast[DisposeProc](desc.disposeImpl)(p) + if desc.destructor != nil: + cast[DestructorProc](desc.destructor)(p) when false: cstderr.rawWrite desc.name cstderr.rawWrite " " - if desc.disposeImpl == nil: + if desc.destructor == nil: cstderr.rawWrite "lacks dispose" if desc.traceImpl != nil: cstderr.rawWrite ", but has trace\n" @@ -109,26 +114,32 @@ proc free(s: Cell; desc: PNimTypeV2) {.inline.} = nimRawDispose(p, desc.align) -proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inline.} = - let p = cast[ptr pointer](q) - if p[] != nil: - var j = cast[ptr GcEnv](env) - j.traceStack.add(head p[], desc) - -proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inline.} = - let p = cast[ptr pointer](q) - if p[] != nil: - var j = cast[ptr GcEnv](env) - j.traceStack.add(head p[], cast[ptr PNimTypeV2](p[])[]) - template orcAssert(cond, msg) = when logOrc: if not cond: cfprintf(cstderr, "[Bug!] %s\n", msg) - quit 1 + rawQuit 1 + +when logOrc: + proc strstr(s, sub: cstring): cstring {.header: "<string.h>", importc.} + +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + + orcAssert strstr(desc.name, "TType") == nil, "following a TType but it's acyclic!" + + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, desc) + +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) var - roots {.threadvar.}: CellSeq + roots {.threadvar.}: CellSeq[Cell] proc unregisterCycle(s: Cell) = # swap with the last element. O(1) @@ -136,7 +147,7 @@ proc unregisterCycle(s: Cell) = when false: if idx >= roots.len or idx < 0: cprintf("[Bug!] %ld\n", idx) - quit 1 + rawQuit 1 roots.d[idx] = roots.d[roots.len-1] roots.d[idx][0].rootIdx = idx+1 dec roots.len @@ -156,7 +167,8 @@ proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) = trace(s, desc, j) when logOrc: writeCell("root still alive", s, desc) while j.traceStack.len > until: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] inc t.rc, rcIncrement if t.color != colBlack: t.setColor colBlack @@ -181,7 +193,8 @@ proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] dec t.rc, rcIncrement inc j.edges when useJumpStack: @@ -189,7 +202,7 @@ proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = t.rc = t.rc or jumpStackFlag when traceCollector: cprintf("[Now in jumpstack] %p %ld color %ld in jumpstack %ld\n", t, t.rc shr rcShift, t.color, t.rc and jumpStackFlag) - j.jumpStack.add(t, desc) + j.jumpStack.add(entry, desc) if t.color != colGray: t.setColor colGray inc j.touched @@ -219,7 +232,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = # that are still alive; we also need to mark what they # refer to as alive: while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop + let (entry, desc) = j.jumpStack.pop + let t = head entry[] # not in jump stack anymore! t.rc = t.rc and not jumpStackFlag if t.color == colGray and (t.rc shr rcShift) >= 0: @@ -233,7 +247,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = s.setColor(colWhite) trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] if t.color == colGray: if (t.rc shr rcShift) >= 0: scanBlack(t, desc, j) @@ -243,7 +258,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = # that are still alive; we also need to mark what they # refer to as alive: while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop + let (entry, desc) = j.jumpStack.pop + let t = head entry[] # not in jump stack anymore! t.rc = t.rc and not jumpStackFlag if t.color == colGray and (t.rc shr rcShift) >= 0: @@ -279,7 +295,9 @@ proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = j.toFree.add(s, desc) trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + entry[] = nil # ensure that the destructor does touch moribund objects! if t.color == col and t.rootIdx == 0: j.toFree.add(t, desc) t.setColor(colBlack) @@ -324,6 +342,8 @@ proc collectCyclesBacon(j: var GcEnv; lowMark: int) = collectColor(s, roots.d[i][1], colToCollect, j) for i in 0 ..< j.toFree.len: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1]) free(j.toFree.d[i][0], j.toFree.d[i][1]) inc j.freed, j.toFree.len @@ -336,7 +356,10 @@ const when defined(nimStressOrc): const rootsThreshold = 10 # broken with -d:nimStressOrc: 10 and for havlak iterations 1..8 else: - var rootsThreshold = defaultThreshold + var rootsThreshold {.threadvar.}: int + +when defined(nimOrcStats): + var freedCyclicObjects {.threadvar.}: int proc partialCollect(lowMark: int) = when false: @@ -351,6 +374,8 @@ proc partialCollect(lowMark: int) = roots.len - lowMark) roots.len = lowMark deinit j.traceStack + when defined(nimOrcStats): + inc freedCyclicObjects, j.freed proc collectCycles() = ## Collect cycles. @@ -378,42 +403,55 @@ proc collectCycles() = # of the cycle collector's effectiveness: # we're effective when we collected 50% or more of the nodes # we touched. If we're effective, we can reset the threshold: - if j.keepThreshold and rootsThreshold <= defaultThreshold: + if j.keepThreshold: discard elif j.freed * 2 >= j.touched: when not defined(nimFixedOrc): rootsThreshold = max(rootsThreshold div 3 * 2, 16) else: - rootsThreshold = defaultThreshold + rootsThreshold = 0 #cfprintf(cstderr, "[collectCycles] freed %ld, touched %ld new threshold %ld\n", j.freed, j.touched, rootsThreshold) elif rootsThreshold < high(int) div 4: - rootsThreshold = rootsThreshold * 3 div 2 + rootsThreshold = (if rootsThreshold <= 0: defaultThreshold else: rootsThreshold) * 3 div 2 when logOrc: cfprintf(cstderr, "[collectCycles] end; freed %ld new threshold %ld touched: %ld mem: %ld rcSum: %ld edges: %ld\n", j.freed, rootsThreshold, j.touched, getOccupiedMem(), j.rcSum, j.edges) + when defined(nimOrcStats): + inc freedCyclicObjects, j.freed + +when defined(nimOrcStats): + type + OrcStats* = object ## Statistics of the cycle collector subsystem. + freedCyclicObjects*: int ## Number of freed cyclic objects. + proc GC_orcStats*(): OrcStats = + ## Returns the statistics of the cycle collector subsystem. + result = OrcStats(freedCyclicObjects: freedCyclicObjects) proc registerCycle(s: Cell; desc: PNimTypeV2) = s.rootIdx = roots.len+1 if roots.d == nil: init(roots) add(roots, s, desc) - if roots.len >= rootsThreshold: + if roots.len - defaultThreshold >= rootsThreshold: collectCycles() when logOrc: writeCell("[added root]", s, desc) + orcAssert strstr(desc.name, "TType") == nil, "added a TType as a root!" + proc GC_runOrc* = ## Forces a cycle collection pass. collectCycles() + orcAssert roots.len == 0, "roots not empty!" proc GC_enableOrc*() = - ## Enables the cycle collector subsystem of `--gc:orc`. This is a `--gc:orc` + ## Enables the cycle collector subsystem of `--mm:orc`. This is a `--mm:orc` ## specific API. Check with `when defined(gcOrc)` for its existence. when not defined(nimStressOrc): - rootsThreshold = defaultThreshold + rootsThreshold = 0 proc GC_disableOrc*() = - ## Disables the cycle collector subsystem of `--gc:orc`. This is a `--gc:orc` + ## Disables the cycle collector subsystem of `--mm:orc`. This is a `--mm:orc` ## specific API. Check with `when defined(gcOrc)` for its existence. when not defined(nimStressOrc): rootsThreshold = high(int) @@ -424,22 +462,27 @@ proc GC_partialCollect*(limit: int) = partialCollect(limit) proc GC_fullCollect* = - ## Forces a full garbage collection pass. With `--gc:orc` triggers the cycle + ## Forces a full garbage collection pass. With `--mm:orc` triggers the cycle ## collector. This is an alias for `GC_runOrc`. collectCycles() proc GC_enableMarkAndSweep*() = - ## For `--gc:orc` an alias for `GC_enableOrc`. + ## For `--mm:orc` an alias for `GC_enableOrc`. GC_enableOrc() proc GC_disableMarkAndSweep*() = - ## For `--gc:orc` an alias for `GC_disableOrc`. + ## For `--mm:orc` an alias for `GC_disableOrc`. GC_disableOrc() +const + acyclicFlag = 1 # see also cggtypes.nim, proc genTypeInfoV2Impl + when optimizedOrc: - template markedAsCyclic(s: Cell): bool = (s.rc and maybeCycle) != 0 + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 and (s.rc and maybeCycle) != 0 else: - template markedAsCyclic(s: Cell): bool = true + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 proc rememberCycle(isDestroyAction: bool; s: Cell; desc: PNimTypeV2) {.noinline.} = if isDestroyAction: @@ -448,7 +491,7 @@ proc rememberCycle(isDestroyAction: bool; s: Cell; desc: PNimTypeV2) {.noinline. else: # do not call 'rememberCycle' again unless this cell # got an 'incRef' event: - if s.rootIdx == 0 and markedAsCyclic(s): + if s.rootIdx == 0 and markedAsCyclic(s, desc): s.setColor colBlack registerCycle(s, desc) @@ -463,6 +506,19 @@ proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = #if cell.color == colPurple: rememberCycle(result, cell, cast[ptr PNimTypeV2](p)[]) +proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + if result: + if cell.rootIdx > 0: + unregisterCycle(cell) + proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = if p != nil: var cell = head(p) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 32d3b166da..5509d0070c 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -28,7 +28,30 @@ const doNotUnmap = not (defined(amd64) or defined(i386)) or defined(windows) or defined(nimAllocNoUnmap) -when defined(emscripten) and not defined(StandaloneHeapSize): +when defined(nimAllocPagesViaMalloc): + when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + {.error: "-d:nimAllocPagesViaMalloc is only supported with --mm:arc or --mm:atomicArc or --mm:orc".} + + proc osTryAllocPages(size: int): pointer {.inline.} = + let base = c_malloc(csize_t size + PageSize - 1 + sizeof(uint32)) + if base == nil: raiseOutOfMem() + # memory layout: padding + offset (4 bytes) + user_data + # in order to deallocate: read offset at user_data - 4 bytes, + # then deallocate user_data - offset + let offset = PageSize - (cast[int](base) and (PageSize - 1)) + cast[ptr uint32](base +! (offset - sizeof(uint32)))[] = uint32(offset) + result = base +! offset + + proc osAllocPages(size: int): pointer {.inline.} = + result = osTryAllocPages(size) + if result == nil: raiseOutOfMem() + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # read offset at p - 4 bytes, then deallocate (p - offset) pointer + let offset = cast[ptr uint32](p -! sizeof(uint32))[] + c_free(p -! offset) + +elif defined(emscripten) and not defined(StandaloneHeapSize): const PROT_READ = 1 # page can be read PROT_WRITE = 2 # page can be written @@ -57,12 +80,12 @@ when defined(emscripten) and not defined(StandaloneHeapSize): let pos = cast[int](result) # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + var new_pos = cast[int](pos) +% (PageSize - (pos %% PageSize)) if (new_pos-pos) < sizeof(EmscriptenMMapBlock): new_pos = new_pos +% PageSize result = cast[pointer](new_pos) - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](result) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) mmapDescr.realSize = realSize @@ -73,135 +96,24 @@ when defined(emscripten) and not defined(StandaloneHeapSize): proc osTryAllocPages(size: int): pointer = osAllocPages(size) proc osDeallocPages(p: pointer, size: int) {.inline.} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](p) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) munmap(mmapDescr.realPointer, mmapDescr.realSize) elif defined(genode) and not defined(StandaloneHeapSize): include genode/alloc # osAllocPages, osTryAllocPages, osDeallocPages -elif defined(nintendoswitch) and not defined(StandaloneHeapSize): - - import nintendoswitch/switch_memory - - type - PSwitchBlock = ptr NSwitchBlock - ## This will hold the heap pointer data in a separate - ## block of memory that is PageSize bytes above - ## the requested memory. It's the only good way - ## to pass around data with heap allocations - NSwitchBlock {.pure, inheritable.} = object - realSize: int - heap: pointer # pointer to main heap alloc - heapMirror: pointer # pointer to virtmem mapped heap - - proc alignSize(size: int): int {.inline.} = - ## Align a size integer to be in multiples of PageSize - ## The nintendo switch will not allocate memory that is not - ## aligned to 0x1000 bytes and will just crash. - (size + (PageSize - 1)) and not (PageSize - 1) - - proc deallocate(heapMirror: pointer, heap: pointer, size: int) = - # Unmap the allocated memory - discard svcUnmapMemory(heapMirror, heap, size.uint64) - # These should be called (theoretically), but referencing them crashes the switch. - # The above call seems to free all heap memory, so these are not needed. - # virtmemFreeMap(nswitchBlock.heapMirror, nswitchBlock.realSize.csize) - # free(nswitchBlock.heap) - - proc freeMem(p: pointer) = - # Retrieve the switch block data from the pointer we set before - # The data is located just sizeof(NSwitchBlock) bytes below - # the top of the pointer to the heap - let - nswitchDescrPos = cast[ByteAddress](p) -% sizeof(NSwitchBlock) - nswitchBlock = cast[PSwitchBlock](nswitchDescrPos) - - deallocate( - nswitchBlock.heapMirror, nswitchBlock.heap, nswitchBlock.realSize - ) - - proc storeHeapData(address, heapMirror, heap: pointer, size: int) {.inline.} = - ## Store data in the heap for deallocation purposes later - - # the position of our heap pointer data. Since we allocated PageSize extra - # bytes, we should have a buffer on top of the requested size of at least - # PageSize bytes, which is much larger than sizeof(NSwitchBlock). So we - # decrement the address by sizeof(NSwitchBlock) and use that address - # to store our pointer data - let nswitchBlockPos = cast[ByteAddress](address) -% sizeof(NSwitchBlock) - - # We need to store this in a pointer obj (PSwitchBlock) so that the data sticks - # at the address we've chosen. If NSwitchBlock is used here, the data will - # be all 0 when we try to retrieve it later. - var nswitchBlock = cast[PSwitchBlock](nswitchBlockPos) - nswitchBlock.realSize = size - nswitchBlock.heap = heap - nswitchBlock.heapMirror = heapMirror - - proc getOriginalHeapPosition(address: pointer, difference: int): pointer {.inline.} = - ## This function sets the heap back to the originally requested - ## size - let - pos = cast[int](address) - newPos = cast[ByteAddress](pos) +% difference - - return cast[pointer](newPos) - - template allocPages(size: int, outOfMemoryStmt: untyped): untyped = - # This is to ensure we get a block of memory the requested - # size, as well as space to store our structure - let realSize = alignSize(size + sizeof(NSwitchBlock)) - - let heap = memalign(PageSize, realSize) - - if heap.isNil: - outOfMemoryStmt - - let heapMirror = virtmemReserveMap(realSize.csize) - result = heapMirror - - let rc = svcMapMemory(heapMirror, heap, realSize.uint64) - # Any return code not equal 0 means an error in libnx - if rc.uint32 != 0: - deallocate(heapMirror, heap, realSize) - outOfMemoryStmt - - # set result to be the original size requirement - result = getOriginalHeapPosition(result, realSize - size) - - storeHeapData(result, heapMirror, heap, realSize) - - proc osAllocPages(size: int): pointer {.inline.} = - allocPages(size): - raiseOutOfMem() - - proc osTryAllocPages(size: int): pointer = - allocPages(size): - return nil - - proc osDeallocPages(p: pointer, size: int) = - # Note that in order for the Switch not to crash, a call to - # deallocHeap(runFinalizers = true, allowGcAfterwards = false) - # must be run before gfxExit(). The Switch requires all memory - # to be deallocated before the graphics application has exited. - # - # gfxExit() can be found in <switch/gfx/gfx.h> in the github - # repo https://github.com/switchbrew/libnx - when reallyOsDealloc: - freeMem(p) - elif defined(posix) and not defined(StandaloneHeapSize): const PROT_READ = 1 # page can be read PROT_WRITE = 2 # page can be written when defined(netbsd) or defined(openbsd): - # OpenBSD security for setjmp/longjmp coroutines - var MAP_STACK {.importc: "MAP_STACK", header: "<sys/mman.h>".}: cint + # OpenBSD security for setjmp/longjmp coroutines + var MAP_STACK {.importc: "MAP_STACK", header: "<sys/mman.h>".}: cint else: const MAP_STACK = 0 # avoid sideeffects - + when defined(macosx) or defined(freebsd): const MAP_ANONYMOUS = 0x1000 const MAP_PRIVATE = 0x02 # Changes are private @@ -277,7 +189,7 @@ elif defined(windows) and not defined(StandaloneHeapSize): when reallyOsDealloc: if virtualFree(p, 0, MEM_RELEASE) == 0: cprintf "virtualFree failing!" - quit 1 + rawQuit 1 #VirtualFree(p, size, MEM_DECOMMIT) elif hostOS == "standalone" or defined(StandaloneHeapSize): diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index 59a1333f1d..0619f3fcaa 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -10,6 +10,8 @@ ## Platform detection for NimScript. This module is included by the system module! ## Do not import it directly! +# CPU architectures have alias names mapped in tools/niminst/makefile.nimf + type CpuPlatform* {.pure.} = enum ## the CPU this program will run on. none, ## unknown CPU @@ -35,13 +37,15 @@ type msp430, ## TI MSP430 microcontroller riscv32, ## RISC-V 32-bit processor riscv64, ## RISC-V 64-bit processor - wasm32 ## WASM, 32-bit + wasm32, ## WASM, 32-bit + e2k, ## MCST Elbrus 2000 + loongarch64, ## LoongArch 64-bit processor + s390x ## IBM Z OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, irix, netbsd, freebsd, openbsd, aix, palmos, qnx, amiga, - atari, netware, macos, macosx, haiku, android, js, nimVM, - standalone, nintendoswitch + atari, netware, macos, macosx, haiku, android, js, standalone, nintendoswitch const targetOS* = when defined(windows): OsPlatform.windows @@ -66,7 +70,6 @@ const elif defined(haiku): OsPlatform.haiku elif defined(android): OsPlatform.android elif defined(js): OsPlatform.js - elif defined(nimVM): OsPlatform.nimVM elif defined(standalone): OsPlatform.standalone elif defined(nintendoswitch): OsPlatform.nintendoswitch else: OsPlatform.none @@ -95,5 +98,8 @@ const elif defined(riscv32): CpuPlatform.riscv32 elif defined(riscv64): CpuPlatform.riscv64 elif defined(wasm32): CpuPlatform.wasm32 + elif defined(e2k): CpuPlatform.e2k + elif defined(loongarch64): CpuPlatform.loongarch64 + elif defined(s390x): CpuPlatform.s390x else: CpuPlatform.none ## the CPU this program will run on. diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index 0649f11767..e7eb6ac823 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -60,13 +60,13 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = b = b.prev var - profilingRequestedHook*: proc (): bool {.nimcall, locks: 0, gcsafe.} + profilingRequestedHook*: proc (): bool {.nimcall, gcsafe.} ## set this variable to provide a procedure that implements a profiler in ## user space. See the `nimprof` module for a reference implementation. when defined(memProfiler): type - MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, locks: 0, gcsafe.} + MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, gcsafe.} var profilerHook*: MemProfilerHook diff --git a/lib/system/rawquits.nim b/lib/system/rawquits.nim new file mode 100644 index 0000000000..f0ead10c6c --- /dev/null +++ b/lib/system/rawquits.nim @@ -0,0 +1,27 @@ +import system/ctypes + +when defined(nimNoQuit): + proc rawQuit(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(genode): + import genode/env + + var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr + + type GenodeEnv = GenodeEnvPtr + ## Opaque type representing Genode environment. + + proc rawQuit(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, + importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "<base/sleep.h>".} + + proc rawQuit(errorcode: int = QuitSuccess) {.inline, noreturn.} = + systemEnv.rawQuit(errorcode) + + +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc rawQuit(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc rawQuit(errorcode: cint) {. + magic: "Exit", importc: "exit", header: "<stdlib.h>", noreturn.} \ No newline at end of file diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 7424500ebd..13118e40b2 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -17,7 +17,7 @@ proc reprFloat(x: float): string {.compilerproc.} = return $x proc reprPointer(x: pointer): string {.compilerproc.} = result = newString(60) - let n = c_sprintf(addr result[0], "%p", x) + let n = c_snprintf(cast[cstring](addr result[0]), csize_t(60), "%p", x) setLen(result, n) proc reprStrAux(result: var string, s: cstring; len: int) = @@ -72,6 +72,8 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = result = $e & " (invalid data!)" +include system/repr_impl + type PByteArray = ptr UncheckedArray[byte] # array[0xffff, byte] @@ -153,7 +155,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..typ.size div bs - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), typ.base, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), typ.base, cl) add result, "]" when defined(nimSeqsV2): @@ -181,7 +183,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..cast[PGenericSeq](p).len-1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), + reprAux(result, cast[pointer](cast[int](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), typ.base, cl) add result, "]" @@ -192,14 +194,14 @@ when not defined(useNimRtl): of nkSlot: add result, $n.name add result, " = " - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) of nkList: for i in 0..n.len-1: if i > 0: add result, ",\n" reprRecordAux(result, p, n.sons[i], cl) of nkCase: var m = selectBranch(p, n) - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) if m != nil: reprRecordAux(result, p, m, cl) proc reprRecord(result: var string, p: pointer, typ: PNimType, @@ -281,7 +283,7 @@ when not defined(useNimRtl): of tyString: let sp = cast[ptr string](p) reprStrAux(result, sp[].cstring, sp[].len) - of tyCString: + of tyCstring: let cs = cast[ptr cstring](p)[] if cs.isNil: add result, "nil" else: reprStrAux(result, cs, cs.len) @@ -305,7 +307,7 @@ when not defined(useNimRtl): var bs = elemtyp.size for i in 0..length - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), elemtyp, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), elemtyp, cl) add result, "]" deinitReprClosure(cl) diff --git a/lib/system/repr_impl.nim b/lib/system/repr_impl.nim new file mode 100644 index 0000000000..b9ec1890f4 --- /dev/null +++ b/lib/system/repr_impl.nim @@ -0,0 +1,15 @@ +#[ +other APIs common to system/repr and system/reprjs could be refactored here, eg: +* reprChar +* reprBool +* reprStr + +Another possibility in future work would be to have a single include file instead +of system/repr and system/reprjs, and use `when defined(js)` inside it. +]# + +proc reprDiscriminant*(e: int, typ: PNimType): string {.compilerRtl.} = + case typ.kind + of tyEnum: reprEnum(e, typ) + of tyBool: $(e != 0) + else: $e diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index d456f4454f..d2aef536c6 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -1,39 +1,47 @@ +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/formatfloat + proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} ## imported from typetraits -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} ## imported from typetraits +proc rangeBase(T: typedesc): typedesc {.magic: "TypeTrait".} + # skip one level of range; return the base type of a range type + proc repr*(x: NimNode): string {.magic: "Repr", noSideEffect.} -proc repr*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int): string = + ## Same as $x + $x -proc repr*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int64): string = + ## Same as $x + $x proc repr*(x: uint64): string {.noSideEffect.} = - ## repr for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - $x #Calls `$` from system/strmantle.nim + ## Same as $x + $x -proc repr*(x: float): string {.magic: "FloatToStr", noSideEffect.} - ## repr for a float argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: float): string = + ## Same as $x + $x proc repr*(x: bool): string {.magic: "BoolToStr", noSideEffect.} ## repr for a boolean argument. Returns `x` ## converted to the string "false" or "true". -proc repr*(x: char): string {.noSideEffect.} = +proc repr*(x: char): string {.noSideEffect, raises: [].} = ## repr for a character argument. Returns `x` ## converted to an escaped string. ## - ## .. code-block:: Nim + ## ```Nim ## assert repr('c') == "'c'" - result.add '\'' + ## ``` + result = "'" # Elides string creations if not needed if x in {'\\', '\0'..'\31', '\127'..'\255'}: result.add '\\' @@ -43,10 +51,10 @@ proc repr*(x: char): string {.noSideEffect.} = result.add x result.add '\'' -proc repr*(x: string | cstring): string {.noSideEffect.} = +proc repr*(x: string | cstring): string {.noSideEffect, raises: [].} = ## repr for a string argument. Returns `x` ## converted to a quoted and escaped string. - result.add '\"' + result = "\"" for i in 0..<x.len: if x[i] in {'"', '\\', '\0'..'\31', '\127'..'\255'}: result.add '\\' @@ -59,13 +67,18 @@ proc repr*(x: string | cstring): string {.noSideEffect.} = result.add x[i] result.add '\"' -proc repr*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect.} +proc repr*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect, raises: [].} ## repr for an enumeration argument. This works for ## any enumeration type thanks to compiler magic. ## ## If a `repr` operator for a concrete enumeration is provided, this is ## used instead. (In other words: *Overwriting* is possible.) +proc reprDiscriminant*(e: int): string {.compilerproc.} = + # repr and reprjs can use `PNimType` to symbolize `e`; making this work here + # would require a way to pass the set of enum stringified values to cgen. + $e + proc repr*(p: pointer): string = ## repr of pointer as its hexadecimal value if p == nil: @@ -82,16 +95,21 @@ proc repr*(p: pointer): string = result[j] = HexChars[n and 0xF] n = n shr 4 -proc repr*(p: proc): string = +proc repr*(p: proc | iterator {.closure.}): string = ## repr of a proc as its address - repr(cast[pointer](p)) + repr(cast[ptr pointer](unsafeAddr p)[]) -template repr*(x: distinct): string = - repr(distinctBase(typeof(x))(x)) +template repr*[T: distinct|(range and not enum)](x: T): string = + when T is range: # add a branch to handle range + repr(rangeBase(typeof(x))(x)) + elif T is distinct: + repr(distinctBase(typeof(x))(x)) + else: + {.error: "cannot happen".} template repr*(t: typedesc): string = $t -proc reprObject[T: tuple|object](res: var string, x: T) = +proc reprObject[T: tuple|object](res: var string, x: T) {.noSideEffect, raises: [].} = res.add '(' var firstElement = true const isNamed = T is object or isNamedTuple(T) @@ -112,19 +130,19 @@ proc reprObject[T: tuple|object](res: var string, x: T) = res.add(')') -proc repr*[T: tuple|object](x: T): string = +proc repr*[T: tuple|object](x: T): string {.noSideEffect, raises: [].} = ## Generic `repr` operator for tuples that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(23, 45) == "(23, 45)" ## $(a: 23, b: 45) == "(a: 23, b: 45)" ## $() == "()" + ## ``` when T is object: result = $typeof(x) reprObject(result, x) -proc repr*[T](x: ref T | ptr T): string = +proc repr*[T](x: ref T | ptr T): string {.noSideEffect, raises: [].} = if isNil(x): return "nil" when T is object: result = $typeof(x) @@ -133,7 +151,7 @@ proc repr*[T](x: ref T | ptr T): string = result = when typeof(x) is ref: "ref " else: "ptr " result.add repr(x[]) -proc collectionToRepr[T](x: T, prefix, separator, suffix: string): string = +proc collectionToRepr[T](x: T, prefix, separator, suffix: string): string {.noSideEffect, raises: [].} = result = prefix var firstElement = true for value in items(x): @@ -147,29 +165,19 @@ proc collectionToRepr[T](x: T, prefix, separator, suffix: string): string = proc repr*[T](x: set[T]): string = ## Generic `repr` operator for sets that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## ${23, 45} == "{23, 45}" + ## ``` collectionToRepr(x, "{", ", ", "}") proc repr*[T](x: seq[T]): string = ## Generic `repr` operator for seqs that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45]) == "@[23, 45]" + ## ``` collectionToRepr(x, "@[", ", ", "]") -proc repr*[T, U](x: HSlice[T, U]): string = - ## Generic `repr` operator for slices that is lifted from the components - ## of `x`. Example: - ## - ## .. code-block:: Nim - ## $(1 .. 5) == "1 .. 5" - result = repr(x.a) - result.add(" .. ") - result.add(repr(x.b)) - proc repr*[T, IDX](x: array[IDX, T]): string = ## Generic `repr` operator for arrays that is lifted from the components. collectionToRepr(x, "[", ", ", "]") @@ -177,7 +185,10 @@ proc repr*[T, IDX](x: array[IDX, T]): string = proc repr*[T](x: openArray[T]): string = ## Generic `repr` operator for openarrays that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45].toOpenArray(0, 1)) == "[23, 45]" + ## ``` collectionToRepr(x, "[", ", ", "]") + +proc repr*[T](x: UncheckedArray[T]): string = + "[...]" diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index 36972024aa..30e84ebeeb 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -8,7 +8,12 @@ # # The generic ``repr`` procedure for the javascript backend. +when defined(nimPreviewSlimSystem): + import std/formatfloat + proc reprInt(x: int64): string {.compilerproc.} = $x +proc reprInt(x: uint64): string {.compilerproc.} = $x +proc reprInt(x: int): string {.compilerproc.} = $x proc reprFloat(x: float): string {.compilerproc.} = $x proc reprPointer(p: pointer): string {.compilerproc.} = @@ -30,6 +35,8 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = else: result = $e & " (invalid data!)" +include system/repr_impl + proc reprChar(x: char): string {.compilerRtl.} = result = "\'" case x @@ -113,7 +120,6 @@ proc reprArray(a: pointer, typ: PNimType, # We prepend @ to seq, the C backend prepends the pointer to the seq. result = if typ.kind == tySequence: "@[" else: "[" var len: int = 0 - var i: int = 0 {. emit: "`len` = `a`.length;\n" .} var dereffed: pointer = a @@ -188,8 +194,12 @@ proc reprAux(result: var string, p: pointer, typ: PNimType, return dec(cl.recDepth) case typ.kind - of tyInt..tyInt64, tyUInt..tyUInt64: + of tyInt..tyInt32, tyUInt..tyUInt32: add(result, reprInt(cast[int](p))) + of tyInt64: + add(result, reprInt(cast[int64](p))) + of tyUInt64: + add(result, reprInt(cast[uint64](p))) of tyChar: add(result, reprChar(cast[char](p))) of tyBool: @@ -200,7 +210,7 @@ proc reprAux(result: var string, p: pointer, typ: PNimType, var fp: int {. emit: "`fp` = `p`;\n" .} add(result, reprStr(cast[string](p))) - of tyCString: + of tyCstring: var fp: cstring {. emit: "`fp` = `p`;\n" .} if fp.isNil: diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index b7f24ecd55..572e77408f 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -8,10 +8,13 @@ # -# import typetraits +# import std/typetraits # strs already imported allocateds for us. -proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} + +# Some optimizations here may be not to empty-seq-initialize some symbols, then StrictNotNil complains. +{.push warning[StrictNotNil]: off.} # See https://github.com/nim-lang/Nim/issues/21401 + ## Default seq implementation used by Nim's core. type @@ -27,6 +30,10 @@ type len: int p: ptr NimSeqPayload[T] + NimRawSeq = object + len: int + p: pointer + const nimSeqVersion {.core.} = 2 # XXX make code memory safe for overflows in '*' @@ -41,6 +48,15 @@ proc newSeqPayload(cap, elemSize, elemAlign: int): pointer {.compilerRtl, raises else: result = nil +proc newSeqPayloadUninit(cap, elemSize, elemAlign: int): pointer {.compilerRtl, raises: [].} = + # Used in `newSeqOfCap()`. + if cap > 0: + var p = cast[ptr NimSeqPayloadBase](alignedAlloc(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize, elemAlign)) + p.cap = cap + result = p + else: + result = nil + template `+!`(p: pointer, s: int): pointer = cast[pointer](cast[int](p) +% s) @@ -48,7 +64,7 @@ template `-!`(p: pointer, s: int): pointer = cast[pointer](cast[int](p) -% s) proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. - noSideEffect, raises: [], compilerRtl.} = + noSideEffect, tags: [], raises: [], compilerRtl.} = {.noSideEffect.}: let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) if addlen <= 0: @@ -61,59 +77,96 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin var p = cast[ptr NimSeqPayloadBase](p) let oldCap = p.cap and not strlitFlag let newCap = max(resize(oldCap), len+addlen) + var q: ptr NimSeqPayloadBase if (p.cap and strlitFlag) == strlitFlag: - var q = cast[ptr NimSeqPayloadBase](alignedAlloc0(headerSize + elemSize * newCap, elemAlign)) + q = cast[ptr NimSeqPayloadBase](alignedAlloc(headerSize + elemSize * newCap, elemAlign)) + copyMem(q +! headerSize, p +! headerSize, len * elemSize) + else: + let oldSize = headerSize + elemSize * oldCap + let newSize = headerSize + elemSize * newCap + q = cast[ptr NimSeqPayloadBase](alignedRealloc(p, oldSize, newSize, elemAlign)) + + zeroMem(q +! headerSize +! len * elemSize, addlen * elemSize) + q.cap = newCap + result = q + +proc zeroNewElements(len: int; q: pointer; addlen, elemSize, elemAlign: int) {. + noSideEffect, tags: [], raises: [], compilerRtl.} = + {.noSideEffect.}: + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) + zeroMem(q +! headerSize +! len * elemSize, addlen * elemSize) + +proc prepareSeqAddUninit(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. + noSideEffect, tags: [], raises: [], compilerRtl.} = + {.noSideEffect.}: + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) + if addlen <= 0: + result = p + elif p == nil: + result = newSeqPayloadUninit(len+addlen, elemSize, elemAlign) + else: + # Note: this means we cannot support things that have internal pointers as + # they get reallocated here. This needs to be documented clearly. + var p = cast[ptr NimSeqPayloadBase](p) + let oldCap = p.cap and not strlitFlag + let newCap = max(resize(oldCap), len+addlen) + if (p.cap and strlitFlag) == strlitFlag: + var q = cast[ptr NimSeqPayloadBase](alignedAlloc(headerSize + elemSize * newCap, elemAlign)) copyMem(q +! headerSize, p +! headerSize, len * elemSize) q.cap = newCap result = q else: let oldSize = headerSize + elemSize * oldCap let newSize = headerSize + elemSize * newCap - var q = cast[ptr NimSeqPayloadBase](alignedRealloc0(p, oldSize, newSize, elemAlign)) + var q = cast[ptr NimSeqPayloadBase](alignedRealloc(p, oldSize, newSize, elemAlign)) q.cap = newCap result = q -proc shrink*[T](x: var seq[T]; newLen: Natural) = +proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = when nimvm: - setLen(x, newLen) + {.cast(tags: []).}: + setLen(x, newLen) else: #sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'" when not supportsCopyMem(T): for i in countdown(x.len - 1, newLen): reset x[i] # XXX This is wrong for const seqs that were moved into 'x'! - cast[ptr NimSeqV2[T]](addr x).len = newLen + {.noSideEffect.}: + cast[ptr NimSeqV2[T]](addr x).len = newLen -proc grow*[T](x: var seq[T]; newLen: Natural; value: T) = +proc grow*[T](x: var seq[T]; newLen: Natural; value: T) {.nodestroy.} = let oldLen = x.len #sysAssert newLen >= x.len, "invalid newLen parameter for 'grow'" if newLen <= oldLen: return var xu = cast[ptr NimSeqV2[T]](addr x) - if xu.p == nil or xu.p.cap < newLen: - xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, newLen - oldLen, sizeof(T), alignof(T))) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newLen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newLen - oldLen, sizeof(T), alignof(T))) xu.len = newLen for i in oldLen .. newLen-1: - xu.p.data[i] = value + wasMoved(xu.p.data[i]) + `=copy`(xu.p.data[i], value) -proc add*[T](x: var seq[T]; value: sink T) {.magic: "AppendSeqElem", noSideEffect, nodestroy.} = +proc add*[T](x: var seq[T]; y: sink T) {.magic: "AppendSeqElem", noSideEffect, nodestroy.} = ## Generic proc for adding a data item `y` to a container `x`. ## ## For containers that have an order, `add` means *append*. New generic ## containers should also call their adding proc `add` for consistency. ## Generic code becomes much easier to write if the Nim naming scheme is ## respected. - let oldLen = x.len - var xu = cast[ptr NimSeqV2[T]](addr x) - if xu.p == nil or xu.p.cap < oldLen+1: - xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, 1, sizeof(T), alignof(T))) - xu.len = oldLen+1 - # .nodestroy means `xu.p.data[oldLen] = value` is compiled into a - # copyMem(). This is fine as know by construction that - # in `xu.p.data[oldLen]` there is nothing to destroy. - # We also save the `wasMoved + destroy` pair for the sink parameter. - xu.p.data[oldLen] = value + {.cast(noSideEffect).}: + let oldLen = x.len + var xu = cast[ptr NimSeqV2[T]](addr x) + if xu.p == nil or (xu.p.cap and not strlitFlag) < oldLen+1: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, 1, sizeof(T), alignof(T))) + xu.len = oldLen+1 + # .nodestroy means `xu.p.data[oldLen] = value` is compiled into a + # copyMem(). This is fine as know by construction that + # in `xu.p.data[oldLen]` there is nothing to destroy. + # We also save the `wasMoved + destroy` pair for the sink parameter. + xu.p.data[oldLen] = y -proc setLen[T](s: var seq[T], newlen: Natural) = +proc setLen[T](s: var seq[T], newlen: Natural) {.nodestroy.} = {.noSideEffect.}: if newlen < s.len: shrink(s, newlen) @@ -121,6 +174,54 @@ proc setLen[T](s: var seq[T], newlen: Natural) = let oldLen = s.len if newlen <= oldLen: return var xu = cast[ptr NimSeqV2[T]](addr s) - if xu.p == nil or xu.p.cap < newlen: - xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newlen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) xu.len = newlen + for i in oldLen..<newlen: + xu.p.data[i] = default(T) + +proc newSeq[T](s: var seq[T], len: Natural) = + shrink(s, 0) + setLen(s, len) + +proc sameSeqPayload(x: pointer, y: pointer): bool {.compilerRtl, inl.} = + result = cast[ptr NimRawSeq](x)[].p == cast[ptr NimRawSeq](y)[].p + + +func capacity*[T](self: seq[T]): int {.inline.} = + ## Returns the current capacity of the seq. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var lst = newSeqOfCap[string](cap = 42) + lst.add "Nim" + assert lst.capacity == 42 + + let sek = cast[ptr NimSeqV2[T]](unsafeAddr self) + result = if sek.p != nil: sek.p.cap and not strlitFlag else: 0 + +func setLenUninit*[T](s: var seq[T], newlen: Natural) {.nodestroy.} = + ## Sets the length of seq `s` to `newlen`. `T` may be any sequence type. + ## New slots will not be initialized. + ## + ## If the current length is greater than the new length, + ## `s` will be truncated. + ## ```nim + ## var x = @[10, 20] + ## x.setLenUninit(5) + ## x[4] = 50 + ## assert x[4] == 50 + ## x.setLenUninit(1) + ## assert x == @[10] + ## ``` + {.noSideEffect.}: + if newlen < s.len: + shrink(s, newlen) + else: + let oldLen = s.len + if newlen <= oldLen: return + var xu = cast[ptr NimSeqV2[T]](addr s) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newlen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) + xu.len = newlen + +{.pop.} # See https://github.com/nim-lang/Nim/issues/21401 diff --git a/lib/system/setops.nim b/lib/system/setops.nim index 755eafdb81..67aa3097ab 100644 --- a/lib/system/setops.nim +++ b/lib/system/setops.nim @@ -9,7 +9,10 @@ func incl*[T](x: var set[T], y: T) {.magic: "Incl".} = a.incl(4) assert a == {1, 2, 3, 4, 5} -template incl*[T](x: var set[T], y: set[T]) = +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template incl*[T](x: var set[T], y: set[T]) {.callsite.} = ## Includes the set `y` in the set `x`. runnableExamples: var a = {1, 3, 5, 7} @@ -23,11 +26,11 @@ func excl*[T](x: var set[T], y: T) {.magic: "Excl".} = ## ## This is the same as `x = x - {y}`, but it might be more efficient. runnableExamples: - var b = {2, 3, 5, 6, 12, 545} + var b = {2, 3, 5, 6, 12, 54} b.excl(5) - assert b == {2, 3, 6, 12, 545} + assert b == {2, 3, 6, 12, 54} -template excl*[T](x: var set[T], y: set[T]) = +template excl*[T](x: var set[T], y: set[T]) {.callsite.} = ## Excludes the set `y` from the set `x`. runnableExamples: var a = {1, 3, 5, 7} diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 04e10ba04c..97431c2964 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -9,19 +9,20 @@ # set handling -type - NimSet = array[0..4*2048-1, uint8] -# bitops can't be imported here, therefore the code duplication. - -proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = +proc cardSetImpl(s: ptr UncheckedArray[uint8], len: int): int {.inline.} = var i = 0 result = 0 + var num = 0'u64 when defined(x86) or defined(amd64): while i < len - 8: - inc(result, countBits64((cast[ptr uint64](s[i].unsafeAddr))[])) + copyMem(addr num, addr s[i], 8) + inc(result, countBits64(num)) inc(i, 8) while i < len: inc(result, countBits32(uint32(s[i]))) inc(i, 1) + +proc cardSet(s: ptr UncheckedArray[uint8], len: int): int {.compilerproc, inline.} = + result = cardSetImpl(s, len) diff --git a/lib/system/stacktraces.nim b/lib/system/stacktraces.nim index 8142f8c7b8..42be9d94fb 100644 --- a/lib/system/stacktraces.nim +++ b/lib/system/stacktraces.nim @@ -22,26 +22,26 @@ when defined(nimStackTraceOverride): ## This is the same as the type `uintptr_t` in C. StackTraceOverrideGetTracebackProc* = proc (): string {. - nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} + nimcall, gcsafe, raises: [], tags: [], noinline.} StackTraceOverrideGetProgramCountersProc* = proc (maxLength: cint): seq[cuintptr_t] {. - nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} + nimcall, gcsafe, raises: [], tags: [], noinline.} StackTraceOverrideGetDebuggingInfoProc* = proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {. - nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} + nimcall, gcsafe, raises: [], tags: [], noinline.} # Default procedures (not normally used, because people opting in on this # override are supposed to register their own versions). var stackTraceOverrideGetTraceback: StackTraceOverrideGetTracebackProc = - proc (): string {.nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} = + proc (): string {.nimcall, gcsafe, raises: [], tags: [], noinline.} = discard #result = "Stack trace override procedure not registered.\n" stackTraceOverrideGetProgramCounters: StackTraceOverrideGetProgramCountersProc = - proc (maxLength: cint): seq[cuintptr_t] {.nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} = + proc (maxLength: cint): seq[cuintptr_t] {.nimcall, gcsafe, raises: [], tags: [], noinline.} = discard stackTraceOverrideGetDebuggingInfo: StackTraceOverrideGetDebuggingInfoProc = proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {. - nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} = + nimcall, gcsafe, raises: [], tags: [], noinline.} = discard # Custom procedure registration. diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 42ea9d2260..89046253bb 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -9,83 +9,28 @@ # Compilerprocs for strings that do not depend on the string implementation. -const digitsTable = "0001020304050607080910111213141516171819" & - "2021222324252627282930313233343536373839" & - "4041424344454647484950515253545556575859" & - "6061626364656667686970717273747576777879" & - "8081828384858687888990919293949596979899" - # Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c - # Generates: - # .. code-block:: nim - # var res = "" - # for i in 0 .. 99: - # if i < 10: - # res.add "0" & $i - # else: - # res.add $i - # doAssert res == digitsTable +import std/private/digitsutils -func digits10(num: uint64): int {.noinline.} = - if num < 10'u64: - result = 1 - elif num < 100'u64: - result = 2 - elif num < 1_000'u64: - result = 3 - elif num < 10_000'u64: - result = 4 - elif num < 100_000'u64: - result = 5 - elif num < 1_000_000'u64: - result = 6 - elif num < 10_000_000'u64: - result = 7 - elif num < 100_000_000'u64: - result = 8 - elif num < 1_000_000_000'u64: - result = 9 - elif num < 10_000_000_000'u64: - result = 10 - elif num < 100_000_000_000'u64: - result = 11 - elif num < 1_000_000_000_000'u64: - result = 12 - else: - result = 12 + digits10(num div 1_000_000_000_000'u64) - -template numToString(result: var string, origin: uint64, length: int) = - var num = origin - var next = length - 1 - const nbatch = 100 - - while num >= nbatch: - let originNum = num - num = num div nbatch - let index = (originNum - num * nbatch) shl 1 - result[next] = digitsTable[index + 1] - result[next - 1] = digitsTable[index] - dec(next, 2) - - # process last 1-2 digits - if num < 10: - result[next] = chr(ord('0') + num) - else: - let index = num * 2 - result[next] = digitsTable[index + 1] - result[next - 1] = digitsTable[index] - proc cmpStrings(a, b: string): int {.inline, compilerproc.} = let alen = a.len let blen = b.len let minlen = min(alen, blen) if minlen > 0: - result = c_memcmp(unsafeAddr a[0], unsafeAddr b[0], cast[csize_t](minlen)) + result = c_memcmp(unsafeAddr a[0], unsafeAddr b[0], cast[csize_t](minlen)).int if result == 0: result = alen - blen else: result = alen - blen +proc leStrings(a, b: string): bool {.inline, compilerproc.} = + # required by upcoming backends (NIR). + cmpStrings(a, b) <= 0 + +proc ltStrings(a, b: string): bool {.inline, compilerproc.} = + # required by upcoming backends (NIR). + cmpStrings(a, b) < 0 + proc eqStrings(a, b: string): bool {.inline, compilerproc.} = let alen = a.len let blen = b.len @@ -96,7 +41,7 @@ proc eqStrings(a, b: string): bool {.inline, compilerproc.} = proc hashString(s: string): int {.compilerproc.} = # the compiler needs exactly the same hash function! # this used to be used for efficient generation of string case statements - var h : uint = 0 + var h = 0'u for i in 0..len(s)-1: h = h + uint(s[i]) h = h + h shl 10 @@ -106,63 +51,28 @@ proc hashString(s: string): int {.compilerproc.} = h = h + h shl 15 result = cast[int](h) -proc addInt*(result: var string; x: int64) = - ## Converts integer to its string representation and appends it to `result`. - ## - ## .. code-block:: Nim - ## var - ## a = "123" - ## b = 45 - ## a.addInt(b) # a <- "12345" - let base = result.len - var length: int - var num: uint64 +proc eqCstrings(a, b: cstring): bool {.inline, compilerproc.} = + if pointer(a) == pointer(b): result = true + elif a.isNil or b.isNil: result = false + else: result = c_strcmp(a, b) == 0 - if x < 0: - if x == low(int64): - num = uint64(x) - else: - num = uint64(-x) - length = base + digits10(num) + 1 - setLen(result, length) - result[base] = '-' - else: - num = uint64(x) - length = base + digits10(num) - setLen(result, length) - numToString(result, num, length) - -proc nimIntToStr(x: int): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x - -proc addCstringN(result: var string, buf: cstring; buflen: int) = - # no nimvm support needed, so it doesn't need to be fast here either - let oldLen = result.len - let newLen = oldLen + buflen - result.setLen newLen - copyMem(result[oldLen].addr, buf, buflen) - -import formatfloat - -proc addFloat*(result: var string; x: float) = - ## Converts float to its string representation and appends it to `result`. - ## - ## .. code-block:: Nim - ## var - ## a = "123" - ## b = 45.67 - ## a.addFloat(b) # a <- "12345.67" - when nimvm: - result.add $x - else: - var buffer {.noinit.}: array[65, char] - let n = writeFloatToBuffer(buffer, x) - result.addCstringN(cstring(buffer[0].addr), n) - -proc nimFloatToStr(f: float): string {.compilerproc.} = - result = newStringOfCap(8) - result.addFloat f +proc hashCstring(s: cstring): int {.compilerproc.} = + # the compiler needs exactly the same hash function! + # this used to be used for efficient generation of cstring case statements + if s.isNil: return 0 + var h : uint = 0 + var i = 0 + while true: + let c = s[i] + if c == '\0': break + h = h + uint(c) + h = h + h shl 10 + h = h xor (h shr 6) + inc i + h = h + h shl 3 + h = h xor (h shr 11) + h = h + h shl 15 + result = cast[int](h) proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. importc: "strtod", header: "<stdlib.h>", noSideEffect.} @@ -173,11 +83,11 @@ const 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22] -when defined(nimHasInvariant): - {.push staticBoundChecks: off.} -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, - start = 0): int {.compilerproc.} = +{.push staticBoundChecks: off.} + +proc nimParseBiggestFloat(s: openArray[char], number: var BiggestFloat, + ): int {.compilerproc.} = # This routine attempt to parse float that can parsed quickly. # i.e. whose integer part can fit inside a 53bits integer. # their real exponent must also be <= 22. If the float doesn't follow @@ -186,7 +96,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, # This avoid the problems of decimal character portability. # see: http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ var - i = start + i = 0 sign = 1.0 kdigits, fdigits = 0 exponent = 0 @@ -209,7 +119,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, if s[i+2] == 'N' or s[i+2] == 'n': if i+3 >= s.len or s[i+3] notin IdentChars: number = NaN - return i+3 - start + return i+3 return 0 # Inf? @@ -218,7 +128,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, if s[i+2] == 'F' or s[i+2] == 'f': if i+3 >= s.len or s[i+3] notin IdentChars: number = Inf*sign - return i+3 - start + return i+3 return 0 if i < s.len and s[i] in {'0'..'9'}: @@ -252,8 +162,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, # if has no digits: return error if kdigits + fdigits <= 0 and - (i == start or # no char consumed (empty string). - (i == start + 1 and hasSign)): # or only '+' or '- + (i == 0 or # no char consumed (empty string). + (i == 1 and hasSign)): # or only '+' or '- return 0 if i+1 < s.len and s[i] in {'e', 'E'}: @@ -276,11 +186,13 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, # if exponent greater than can be represented: +/- zero or infinity if absExponent > 999: - if expNegative: + if integer == 0: + number = 0.0 + elif expNegative: number = 0.0*sign else: number = Inf*sign - return i - start + return i # if integer is representable in 53 bits: fast path # max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits) @@ -292,29 +204,30 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, number = sign * integer.float / powtens[absExponent] else: number = sign * integer.float * powtens[absExponent] - return i - start + return i # if exponent is greater try to fit extra exponent above 22 by multiplying # integer part is there is space left. let slop = 15 - kdigits - fdigits if absExponent <= 22 + slop and not expNegative: number = sign * integer.float * powtens[slop] * powtens[absExponent-slop] - return i - start + return i # if failed: slow path with strtod. var t: array[500, char] # flaviu says: 325 is the longest reasonable literal var ti = 0 let maxlen = t.high - "e+000".len # reserve enough space for exponent - result = i - start - i = start + let endPos = i + result = endPos + i = 0 # re-parse without error checking, any error should be handled by the code above. - if i < s.len and s[i] == '.': i.inc - while i < s.len and s[i] in {'0'..'9','+','-'}: + if i < endPos and s[i] == '.': i.inc + while i < endPos and s[i] in {'0'..'9','+','-'}: if ti < maxlen: t[ti] = s[i]; inc(ti) inc(i) - while i < s.len and s[i] in {'.', '_'}: # skip underscore and decimal point + while i < endPos and s[i] in {'.', '_'}: # skip underscore and decimal point inc(i) # insert exponent @@ -329,14 +242,9 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, t[ti-2] = ('0'.ord + absExponent mod 10).char absExponent = absExponent div 10 t[ti-3] = ('0'.ord + absExponent mod 10).char - number = c_strtod(addr t, nil) + number = c_strtod(cast[cstring](addr t), nil) -when defined(nimHasInvariant): - {.pop.} # staticBoundChecks - -proc nimInt64ToStr(x: int64): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x +{.pop.} # staticBoundChecks proc nimBoolToStr(x: bool): string {.compilerRtl.} = return if x: "true" else: "false" diff --git a/lib/system/strs_v2.nim b/lib/system/strs_v2.nim index 6944cdc589..dbee107774 100644 --- a/lib/system/strs_v2.nim +++ b/lib/system/strs_v2.nim @@ -34,53 +34,72 @@ template frees(s) = else: dealloc(s.p) +template allocPayload(newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](allocShared(contentSize(newLen))) + else: + cast[ptr NimStrPayload](alloc(contentSize(newLen))) + +template allocPayload0(newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](allocShared0(contentSize(newLen))) + else: + cast[ptr NimStrPayload](alloc0(contentSize(newLen))) + +template reallocPayload(p: pointer, newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](reallocShared(p, contentSize(newLen))) + else: + cast[ptr NimStrPayload](realloc(p, contentSize(newLen))) + +template reallocPayload0(p: pointer; oldLen, newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](reallocShared0(p, contentSize(oldLen), contentSize(newLen))) + else: + cast[ptr NimStrPayload](realloc0(p, contentSize(oldLen), contentSize(newLen))) + proc resize(old: int): int {.inline.} = if old <= 0: result = 4 - elif old < 65536: result = old * 2 + elif old <= high(int16): result = old * 2 else: result = old * 3 div 2 # for large arrays * 3/2 is better -proc prepareAdd(s: var NimStringV2; addlen: int) {.compilerRtl.} = - let newLen = s.len + addlen +proc prepareAdd(s: var NimStringV2; addLen: int) {.compilerRtl.} = + let newLen = s.len + addLen if isLiteral(s): let oldP = s.p # can't mutate a literal, so we need a fresh copy here: - when compileOption("threads"): - s.p = cast[ptr NimStrPayload](allocShared0(contentSize(newLen))) - else: - s.p = cast[ptr NimStrPayload](alloc0(contentSize(newLen))) + s.p = allocPayload(newLen) s.p.cap = newLen if s.len > 0: # we are about to append, so there is no need to copy the \0 terminator: copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], min(s.len, newLen)) + elif oldP == nil: + # In the case of `newString(0) & ""`, since `src.len == 0`, `appendString` + # will not set the `\0` terminator, so we set it here. + s.p.data[0] = '\0' else: let oldCap = s.p.cap and not strlitFlag if newLen > oldCap: let newCap = max(newLen, resize(oldCap)) - when compileOption("threads"): - s.p = cast[ptr NimStrPayload](reallocShared0(s.p, contentSize(oldCap), contentSize(newCap))) - else: - s.p = cast[ptr NimStrPayload](realloc0(s.p, contentSize(oldCap), contentSize(newCap))) + s.p = reallocPayload(s.p, newCap) s.p.cap = newCap + if newLen < newCap: + zeroMem(cast[pointer](addr s.p.data[newLen+1]), newCap - newLen) -proc nimAddCharV1(s: var NimStringV2; c: char) {.compilerRtl, inline.} = +proc nimAddCharV1(s: var NimStringV2; c: char) {.compilerRtl, inl.} = #if (s.p == nil) or (s.len+1 > s.p.cap and not strlitFlag): prepareAdd(s, 1) s.p.data[s.len] = c - s.p.data[s.len+1] = '\0' inc s.len + s.p.data[s.len] = '\0' proc toNimStr(str: cstring, len: int): NimStringV2 {.compilerproc.} = if len <= 0: result = NimStringV2(len: 0, p: nil) else: - when compileOption("threads"): - var p = cast[ptr NimStrPayload](allocShared0(contentSize(len))) - else: - var p = cast[ptr NimStrPayload](alloc0(contentSize(len))) + var p = allocPayload(len) p.cap = len - if len > 0: - # we are about to append, so there is no need to copy the \0 terminator: - copyMem(unsafeAddr p.data[0], str, len) + copyMem(unsafeAddr p.data[0], str, len+1) result = NimStringV2(len: len, p: p) proc cstrToNimstr(str: cstring): NimStringV2 {.compilerRtl.} = @@ -89,7 +108,7 @@ proc cstrToNimstr(str: cstring): NimStringV2 {.compilerRtl.} = proc nimToCStringConv(s: NimStringV2): cstring {.compilerproc, nonReloadable, inline.} = if s.len == 0: result = cstring"" - else: result = cstring(unsafeAddr s.p.data) + else: result = cast[cstring](unsafeAddr s.p.data) proc appendString(dest: var NimStringV2; src: NimStringV2) {.compilerproc, inline.} = if src.len > 0: @@ -99,29 +118,24 @@ proc appendString(dest: var NimStringV2; src: NimStringV2) {.compilerproc, inlin proc appendChar(dest: var NimStringV2; c: char) {.compilerproc, inline.} = dest.p.data[dest.len] = c - dest.p.data[dest.len+1] = '\0' inc dest.len + dest.p.data[dest.len] = '\0' proc rawNewString(space: int): NimStringV2 {.compilerproc.} = # this is also 'system.newStringOfCap'. if space <= 0: result = NimStringV2(len: 0, p: nil) else: - when compileOption("threads"): - var p = cast[ptr NimStrPayload](allocShared0(contentSize(space))) - else: - var p = cast[ptr NimStrPayload](alloc0(contentSize(space))) + var p = allocPayload(space) p.cap = space + p.data[0] = '\0' result = NimStringV2(len: 0, p: p) proc mnewString(len: int): NimStringV2 {.compilerproc.} = if len <= 0: result = NimStringV2(len: 0, p: nil) else: - when compileOption("threads"): - var p = cast[ptr NimStrPayload](allocShared0(contentSize(len))) - else: - var p = cast[ptr NimStrPayload](alloc0(contentSize(len))) + var p = allocPayload0(len) p.cap = len result = NimStringV2(len: len, p: p) @@ -129,8 +143,25 @@ proc setLengthStrV2(s: var NimStringV2, newLen: int) {.compilerRtl.} = if newLen == 0: discard "do not free the buffer here, pattern 's.setLen 0' is common for avoiding allocations" else: - if newLen > s.len or isLiteral(s): - prepareAdd(s, newLen - s.len) + if isLiteral(s): + let oldP = s.p + s.p = allocPayload(newLen) + s.p.cap = newLen + if s.len > 0: + copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], min(s.len, newLen)) + if newLen > s.len: + zeroMem(cast[pointer](addr s.p.data[s.len]), newLen - s.len + 1) + else: + s.p.data[newLen] = '\0' + else: + zeroMem(cast[pointer](addr s.p.data[0]), newLen + 1) + elif newLen > s.len: + let oldCap = s.p.cap and not strlitFlag + if newLen > oldCap: + let newCap = max(newLen, resize(oldCap)) + s.p = reallocPayload0(s.p, oldCap, newCap) + s.p.cap = newCap + s.p.data[newLen] = '\0' s.len = newLen @@ -147,10 +178,7 @@ proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} = # 'let y = newStringOfCap(); var x = y' # on the other hand... These get turned into moves now. frees(a) - when compileOption("threads"): - a.p = cast[ptr NimStrPayload](allocShared0(contentSize(b.len))) - else: - a.p = cast[ptr NimStrPayload](alloc0(contentSize(b.len))) + a.p = allocPayload(b.len) a.p.cap = b.len a.len = b.len copyMem(unsafeAddr a.p.data[0], unsafeAddr b.p.data[0], b.len+1) @@ -158,14 +186,11 @@ proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} = proc nimPrepareStrMutationImpl(s: var NimStringV2) = let oldP = s.p # can't mutate a literal, so we need a fresh copy here: - when compileOption("threads"): - s.p = cast[ptr NimStrPayload](allocShared0(contentSize(s.len))) - else: - s.p = cast[ptr NimStrPayload](alloc0(contentSize(s.len))) + s.p = allocPayload(s.len) s.p.cap = s.len copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], s.len+1) -proc nimPrepareStrMutationV2(s: var NimStringV2) {.compilerRtl, inline.} = +proc nimPrepareStrMutationV2(s: var NimStringV2) {.compilerRtl, inl.} = if s.p != nil and (s.p.cap and strlitFlag) == strlitFlag: nimPrepareStrMutationImpl(s) @@ -175,3 +200,25 @@ proc prepareMutation*(s: var string) {.inline.} = {.cast(noSideEffect).}: let s = unsafeAddr s nimPrepareStrMutationV2(cast[ptr NimStringV2](s)[]) + +proc nimAddStrV1(s: var NimStringV2; src: NimStringV2) {.compilerRtl, inl.} = + #if (s.p == nil) or (s.len+1 > s.p.cap and not strlitFlag): + prepareAdd(s, src.len) + appendString s, src + +proc nimDestroyStrV1(s: NimStringV2) {.compilerRtl, inl.} = + frees(s) + +proc nimStrAtLe(s: string; idx: int; ch: char): bool {.compilerRtl, inl.} = + result = idx < s.len and s[idx] <= ch + +func capacity*(self: string): int {.inline.} = + ## Returns the current capacity of the string. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var str = newStringOfCap(cap = 42) + str.add "Nim" + assert str.capacity == 42 + + let str = cast[ptr NimStringV2](unsafeAddr self) + result = if str.p != nil: str.p.cap and not strlitFlag else: 0 diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim deleted file mode 100644 index 51dbfd3a29..0000000000 --- a/lib/system/syslocks.nim +++ /dev/null @@ -1,226 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Low level system locks and condition vars. - -{.push stackTrace: off.} - -when defined(windows): - type - Handle = int - - SysLock {.importc: "CRITICAL_SECTION", - header: "<windows.h>", final, pure.} = object # CRITICAL_SECTION in WinApi - DebugInfo: pointer - LockCount: int32 - RecursionCount: int32 - OwningThread: int - LockSemaphore: int - SpinCount: int - - SysCond = Handle - - proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", - header: "<windows.h>".} - ## Initializes the lock `L`. - - proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", - header: "<windows.h>".} - ## Tries to acquire the lock `L`. - - proc tryAcquireSys(L: var SysLock): bool {.inline.} = - result = tryAcquireSysAux(L) != 0'i32 - - proc acquireSys(L: var SysLock) {.importc: "EnterCriticalSection", - header: "<windows.h>".} - ## Acquires the lock `L`. - - proc releaseSys(L: var SysLock) {.importc: "LeaveCriticalSection", - header: "<windows.h>".} - ## Releases the lock `L`. - - proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", - header: "<windows.h>".} - - proc createEvent(lpEventAttributes: pointer, - bManualReset, bInitialState: int32, - lpName: cstring): SysCond {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CreateEventA".} - - proc closeHandle(hObject: Handle) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CloseHandle".} - proc waitForSingleObject(hHandle: Handle, dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForSingleObject", noSideEffect.} - - proc signalSysCond(hEvent: SysCond) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "SetEvent".} - - proc initSysCond(cond: var SysCond) {.inline.} = - cond = createEvent(nil, 0'i32, 0'i32, nil) - proc deinitSysCond(cond: var SysCond) {.inline.} = - closeHandle(cond) - proc waitSysCond(cond: var SysCond, lock: var SysLock) = - releaseSys(lock) - discard waitForSingleObject(cond, -1'i32) - acquireSys(lock) - - proc waitSysCondWindows(cond: var SysCond) = - discard waitForSingleObject(cond, -1'i32) - -elif defined(genode): - const - Header = "genode_cpp/syslocks.h" - type - SysLock {.importcpp: "Nim::SysLock", pure, final, - header: Header.} = object - SysCond {.importcpp: "Nim::SysCond", pure, final, - header: Header.} = object - - proc initSysLock(L: var SysLock) = discard - proc deinitSys(L: var SysLock) = discard - proc acquireSys(L: var SysLock) {.noSideEffect, importcpp.} - proc tryAcquireSys(L: var SysLock): bool {.noSideEffect, importcpp.} - proc releaseSys(L: var SysLock) {.noSideEffect, importcpp.} - - proc initSysCond(L: var SysCond) = discard - proc deinitSysCond(L: var SysCond) = discard - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - noSideEffect, importcpp.} - proc signalSysCond(cond: var SysCond) {. - noSideEffect, importcpp.} - -else: - type - SysLockObj {.importc: "pthread_mutex_t", pure, final, - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[40 div sizeof(clong), clong] - - SysLockAttr {.importc: "pthread_mutexattr_t", pure, final - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[4 div sizeof(cint), cint] # actually a cint - - SysCondObj {.importc: "pthread_cond_t", pure, final, - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[48 div sizeof(clonglong), clonglong] - - SysCondAttr {.importc: "pthread_condattr_t", pure, final - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[4 div sizeof(cint), cint] # actually a cint - - SysLockType = distinct cint - - proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. - importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} - proc deinitSysAux(L: var SysLockObj) {.noSideEffect, - importc: "pthread_mutex_destroy", header: "<pthread.h>".} - - proc acquireSysAux(L: var SysLockObj) {.noSideEffect, - importc: "pthread_mutex_lock", header: "<pthread.h>".} - proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, - importc: "pthread_mutex_trylock", header: "<pthread.h>".} - - proc releaseSysAux(L: var SysLockObj) {.noSideEffect, - importc: "pthread_mutex_unlock", header: "<pthread.h>".} - - when defined(ios): - # iOS will behave badly if sync primitives are moved in memory. In order - # to prevent this once and for all, we're doing an extra malloc when - # initializing the primitive. - type - SysLock = ptr SysLockObj - SysCond = ptr SysCondObj - - when not declared(c_malloc): - proc c_malloc(size: csize_t): pointer {. - importc: "malloc", header: "<stdlib.h>".} - proc c_free(p: pointer) {. - importc: "free", header: "<stdlib.h>".} - - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) = - L = cast[SysLock](c_malloc(csize_t(sizeof(SysLockObj)))) - initSysLockAux(L[], attr) - - proc deinitSys(L: var SysLock) = - deinitSysAux(L[]) - c_free(L) - - template acquireSys(L: var SysLock) = - acquireSysAux(L[]) - template tryAcquireSys(L: var SysLock): bool = - tryAcquireSysAux(L[]) == 0'i32 - template releaseSys(L: var SysLock) = - releaseSysAux(L[]) - else: - type - SysLock = SysLockObj - SysCond = SysCondObj - - template initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) = - initSysLockAux(L, attr) - template deinitSys(L: var SysLock) = - deinitSysAux(L) - template acquireSys(L: var SysLock) = - acquireSysAux(L) - template tryAcquireSys(L: var SysLock): bool = - tryAcquireSysAux(L) == 0'i32 - template releaseSys(L: var SysLock) = - releaseSysAux(L) - - when insideRLocksModule: - let SysLockType_Reentrant{.importc: "PTHREAD_MUTEX_RECURSIVE", - header: "<pthread.h>".}: SysLockType - proc initSysLockAttr(a: var SysLockAttr) {. - importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} - proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. - importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} - - else: - proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} - proc deinitSysCondAux(cond: var SysCondObj) {.noSideEffect, - importc: "pthread_cond_destroy", header: "<pthread.h>".} - - proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj) {. - importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} - proc signalSysCondAux(cond: var SysCondObj) {. - importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - - when defined(ios): - proc initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = - cond = cast[SysCond](c_malloc(csize_t(sizeof(SysCondObj)))) - initSysCondAux(cond[], cond_attr) - - proc deinitSysCond(cond: var SysCond) = - deinitSysCondAux(cond[]) - c_free(cond) - - template waitSysCond(cond: var SysCond, lock: var SysLock) = - waitSysCondAux(cond[], lock[]) - template signalSysCond(cond: var SysCond) = - signalSysCondAux(cond[]) - else: - template initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = - initSysCondAux(cond, cond_attr) - template deinitSysCond(cond: var SysCond) = - deinitSysCondAux(cond) - - template waitSysCond(cond: var SysCond, lock: var SysLock) = - waitSysCondAux(cond, lock) - template signalSysCond(cond: var SysCond) = - signalSysCondAux(cond) - -{.pop.} diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim deleted file mode 100644 index d1f5803f49..0000000000 --- a/lib/system/sysspawn.nim +++ /dev/null @@ -1,194 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements Nim's 'spawn'. - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -{.push stackTrace:off.} - -# We declare our own condition variables here to get rid of the dummy lock -# on Windows: - -type - CondVar = object - c: SysCond - when defined(posix): - stupidLock: SysLock - counter: int - -proc createCondVar(): CondVar = - initSysCond(result.c) - when defined(posix): - initSysLock(result.stupidLock) - #acquireSys(result.stupidLock) - -proc destroyCondVar(c: var CondVar) {.inline.} = - deinitSysCond(c.c) - -proc await(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - while cv.counter <= 0: - waitSysCond(cv.c, cv.stupidLock) - dec cv.counter - releaseSys(cv.stupidLock) - else: - waitSysCondWindows(cv.c) - -proc signal(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - inc cv.counter - releaseSys(cv.stupidLock) - signalSysCond(cv.c) - -type - FastCondVar = object - event, slowPath: bool - slow: CondVar - -proc createFastCondVar(): FastCondVar = - initSysCond(result.slow.c) - when defined(posix): - initSysLock(result.slow.stupidLock) - #acquireSys(result.slow.stupidLock) - result.event = false - result.slowPath = false - -proc await(cv: var FastCondVar) = - #for i in 0 .. 50: - # if cas(addr cv.event, true, false): - # # this is a HIT: Triggers > 95% in my tests. - # return - # cpuRelax() - #cv.slowPath = true - # XXX For some reason this crashes some test programs - await(cv.slow) - cv.event = false - -proc signal(cv: var FastCondVar) = - cv.event = true - #if cas(addr cv.slowPath, true, false): - signal(cv.slow) - -type - Barrier* {.compilerproc.} = object - counter: int - cv: CondVar - -proc barrierEnter*(b: ptr Barrier) {.compilerproc.} = - atomicInc b.counter - -proc barrierLeave*(b: ptr Barrier) {.compilerproc.} = - atomicDec b.counter - if b.counter <= 0: signal(b.cv) - -proc openBarrier*(b: ptr Barrier) {.compilerproc.} = - b.counter = 0 - b.cv = createCondVar() - -proc closeBarrier*(b: ptr Barrier) {.compilerproc.} = - await(b.cv) - destroyCondVar(b.cv) - -{.pop.} - -# ---------------------------------------------------------------------------- - -type - WorkerProc = proc (thread, args: pointer) {.nimcall, gcsafe.} - Worker = object - taskArrived: CondVar - taskStarted: FastCondVar #\ - # task data: - f: WorkerProc - data: pointer - ready: bool # put it here for correct alignment! - -proc nimArgsPassingDone(p: pointer) {.compilerproc.} = - let w = cast[ptr Worker](p) - signal(w.taskStarted) - -var gSomeReady = createFastCondVar() - -proc slave(w: ptr Worker) {.thread.} = - while true: - w.ready = true # If we instead signal "workerReady" we need the scheduler - # to notice this. The scheduler could then optimize the - # layout of the worker threads (e.g. keep the list sorted) - # so that no search for a "ready" thread is necessary. - # This might be implemented later, but is more tricky than - # it looks because 'spawn' itself can run concurrently. - signal(gSomeReady) - await(w.taskArrived) - assert(not w.ready) - # shield against spurious wakeups: - if w.data != nil: - w.f(w, w.data) - w.data = nil - -const NumThreads = 4 - -var - workers: array[NumThreads, Thread[ptr Worker]] - workersData: array[NumThreads, Worker] - -proc setup() = - for i in 0 ..< NumThreads: - workersData[i].taskArrived = createCondVar() - workersData[i].taskStarted = createFastCondVar() - createThread(workers[i], slave, addr(workersData[i])) - -proc preferSpawn*(): bool = - ## Use this proc to determine quickly if a 'spawn' or a direct call is - ## preferable. If it returns 'true' a 'spawn' may make sense. In general - ## it is not necessary to call this directly; use 'spawnX' instead. - result = gSomeReady.event - -proc spawn*(call: typed) {.magic: "Spawn".} - ## always spawns a new task, so that the 'call' is never executed on - ## the calling thread. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - -template spawnX*(call: typed) = - ## spawns a new task if a CPU core is ready, otherwise executes the - ## call in the calling thread. Usually it is advised to - ## use 'spawn' in order to not block the producer for an unknown - ## amount of time. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - if preferSpawn(): spawn call - else: call - -proc nimSpawn(fn: WorkerProc; data: pointer) {.compilerproc.} = - # implementation of 'spawn' that is used by the code generator. - while true: - for i in 0.. high(workers): - let w = addr(workersData[i]) - if cas(addr w.ready, true, false): - w.data = data - w.f = fn - signal(w.taskArrived) - await(w.taskStarted) - return - await(gSomeReady) - -proc sync*() = - ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate - ## waiting, you have to use an explicit barrier. - while true: - var allReady = true - for i in 0 .. high(workers): - if not allReady: break - allReady = allReady and workersData[i].ready - if allReady: break - await(gSomeReady) - -setup() diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 49fff41e9e..e4d6479ec7 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -17,10 +17,10 @@ proc dataPointer(a: PGenericSeq, elemAlign: int): pointer = - cast[pointer](cast[ByteAddress](a) +% align(GenericSeqSize, elemAlign)) + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign)) proc dataPointer(a: PGenericSeq, elemAlign, elemSize, index: int): pointer = - cast[pointer](cast[ByteAddress](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) proc resize(old: int): int {.inline.} = if old <= 0: result = 4 @@ -47,27 +47,22 @@ else: template allocStrNoInit(size: untyped): untyped = cast[NimString](newObjNoInit(addr(strDesc), size)) -proc rawNewStringNoInit(space: int): NimString {.compilerproc.} = - var s = space - if s < 7: s = 7 +proc rawNewStringNoInit(space: int): NimString = + let s = max(space, 7) result = allocStrNoInit(sizeof(TGenericSeq) + s + 1) result.reserved = s - result.len = 0 when defined(gogc): result.elemSize = 1 proc rawNewString(space: int): NimString {.compilerproc.} = - var s = space - if s < 7: s = 7 - result = allocStr(sizeof(TGenericSeq) + s + 1) - result.reserved = s + result = rawNewStringNoInit(space) result.len = 0 - when defined(gogc): - result.elemSize = 1 + result.data[0] = '\0' proc mnewString(len: int): NimString {.compilerproc.} = - result = rawNewString(len) + result = rawNewStringNoInit(len) result.len = len + zeroMem(addr result.data[0], len + 1) proc copyStrLast(s: NimString, start, last: int): NimString {.compilerproc.} = # This is not used by most recent versions of the compiler anymore, but @@ -75,13 +70,10 @@ proc copyStrLast(s: NimString, start, last: int): NimString {.compilerproc.} = let start = max(start, 0) if s == nil: return nil let len = min(last, s.len-1) - start + 1 - if len > 0: - result = rawNewStringNoInit(len) - result.len = len - copyMem(addr(result.data), addr(s.data[start]), len) - result.data[len] = '\0' - else: - result = rawNewString(len) + result = rawNewStringNoInit(len) + result.len = len + copyMem(addr(result.data), addr(s.data[start]), len) + result.data[len] = '\0' proc copyStr(s: NimString, start: int): NimString {.compilerproc.} = # This is not used by most recent versions of the compiler anymore, but @@ -91,12 +83,13 @@ proc copyStr(s: NimString, start: int): NimString {.compilerproc.} = proc nimToCStringConv(s: NimString): cstring {.compilerproc, nonReloadable, inline.} = if s == nil or s.len == 0: result = cstring"" - else: result = cstring(addr s.data) + else: result = cast[cstring](addr s.data) proc toNimStr(str: cstring, len: int): NimString {.compilerproc.} = result = rawNewStringNoInit(len) result.len = len - copyMem(addr(result.data), str, len + 1) + copyMem(addr(result.data), str, len) + result.data[len] = '\0' proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} = if str == nil: NimString(nil) @@ -160,13 +153,9 @@ proc addChar(s: NimString, c: char): NimString = result = s if result.len >= result.space: let r = resize(result.space) - when defined(nimIncrSeqV3): - result = rawNewStringNoInit(r) - result.len = s.len - copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len+1) - else: - result = cast[NimString](growObj(result, - sizeof(TGenericSeq) + r + 1)) + result = rawNewStringNoInit(r) + result.len = s.len + copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len+1) result.reserved = r result.data[result.len] = c result.data[result.len+1] = '\0' @@ -205,17 +194,14 @@ proc addChar(s: NimString, c: char): NimString = proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.} = if dest == nil: - result = rawNewStringNoInit(addlen) + result = rawNewString(addlen) elif dest.len + addlen <= dest.space: result = dest else: # slow path: let sp = max(resize(dest.space), dest.len + addlen) - when defined(nimIncrSeqV3): - result = rawNewStringNoInit(sp) - result.len = dest.len - copyMem(addr result.data[0], unsafeAddr(dest.data[0]), dest.len+1) - else: - result = cast[NimString](growObj(dest, sizeof(TGenericSeq) + sp + 1)) + result = rawNewStringNoInit(sp) + result.len = dest.len + copyMem(addr result.data[0], unsafeAddr(dest.data[0]), dest.len+1) result.reserved = sp #result = rawNewString(sp) #copyMem(result, dest, dest.len + sizeof(TGenericSeq)) @@ -234,19 +220,16 @@ proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = proc setLengthStr(s: NimString, newLen: int): NimString {.compilerRtl.} = let n = max(newLen, 0) if s == nil: - result = mnewString(newLen) + result = mnewString(n) elif n <= s.space: result = s else: - let sp = max(resize(s.space), newLen) - when defined(nimIncrSeqV3): - result = rawNewStringNoInit(sp) - result.len = s.len - copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len+1) - zeroMem(addr result.data[s.len], newLen - s.len) - result.reserved = sp - else: - result = resizeString(s, n) + let sp = max(resize(s.space), n) + result = rawNewStringNoInit(sp) + result.len = s.len + copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len) + zeroMem(addr result.data[s.len], n - s.len) + result.reserved = sp result.len = n result.data[n] = '\0' @@ -282,15 +265,11 @@ proc incrSeqV3(s: PGenericSeq, typ: PNimType): PGenericSeq {.compilerproc.} = result = s if result.len >= result.space: let r = resize(result.space) - when defined(nimIncrSeqV3): - result = cast[PGenericSeq](newSeq(typ, r)) - result.len = s.len - copyMem(dataPointer(result, typ.base.align), dataPointer(s, typ.base.align), s.len * typ.base.size) - # since we steal the content from 's', it's crucial to set s's len to 0. - s.len = 0 - else: - result = cast[PGenericSeq](growObj(result, align(GenericSeqSize, typ.base.align) + typ.base.size * r)) - result.reserved = r + result = cast[PGenericSeq](newSeq(typ, r)) + result.len = s.len + copyMem(dataPointer(result, typ.base.align), dataPointer(s, typ.base.align), s.len * typ.base.size) + # since we steal the content from 's', it's crucial to set s's len to 0. + s.len = 0 proc setLengthSeq(seq: PGenericSeq, elemSize, elemAlign, newLen: int): PGenericSeq {. compilerRtl, inl.} = @@ -304,20 +283,10 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, elemAlign, newLen: int): PGenericS when not defined(boehmGC) and not defined(nogc) and not defined(gcMarkAndSweep) and not defined(gogc) and not defined(gcRegions): - when false: # deadcode: was used by `compileOption("gc", "v2")` + if ntfNoRefs notin extGetCellType(result).base.flags: for i in newLen..result.len-1: - let len0 = gch.tempStack.len forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), - extGetCellType(result).base, waPush) - let len1 = gch.tempStack.len - for i in len0 ..< len1: - doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic) - gch.tempStack.len = len0 - else: - if ntfNoRefs notin extGetCellType(result).base.flags: - for i in newLen..result.len-1: - forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), - extGetCellType(result).base, waZctDecRef) + extGetCellType(result).base, waZctDecRef) # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). @@ -334,36 +303,55 @@ proc setLengthSeqV2(s: PGenericSeq, typ: PNimType, newLen: int): PGenericSeq {. if s == nil: result = cast[PGenericSeq](newSeq(typ, newLen)) else: - when defined(nimIncrSeqV3): - let elemSize = typ.base.size - let elemAlign = typ.base.align - if s.space < newLen: - let r = max(resize(s.space), newLen) - result = cast[PGenericSeq](newSeq(typ, r)) - copyMem(dataPointer(result, elemAlign), dataPointer(s, elemAlign), s.len * elemSize) - # since we steal the content from 's', it's crucial to set s's len to 0. - s.len = 0 - elif newLen < s.len: - result = s - # we need to decref here, otherwise the GC leaks! - when not defined(boehmGC) and not defined(nogc) and - not defined(gcMarkAndSweep) and not defined(gogc) and - not defined(gcRegions): - if ntfNoRefs notin typ.base.flags: - for i in newLen..result.len-1: - forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), - extGetCellType(result).base, waZctDecRef) + let elemSize = typ.base.size + let elemAlign = typ.base.align + if s.space < newLen: + let r = max(resize(s.space), newLen) + result = cast[PGenericSeq](newSeq(typ, r)) + copyMem(dataPointer(result, elemAlign), dataPointer(s, elemAlign), s.len * elemSize) + # since we steal the content from 's', it's crucial to set s's len to 0. + s.len = 0 + elif newLen < s.len: + result = s + # we need to decref here, otherwise the GC leaks! + when not defined(boehmGC) and not defined(nogc) and + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcRegions): + if ntfNoRefs notin typ.base.flags: + for i in newLen..result.len-1: + forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), + extGetCellType(result).base, waZctDecRef) - # XXX: zeroing out the memory can still result in crashes if a wiped-out - # cell is aliased by another pointer (ie proc parameter or a let variable). - # This is a tough problem, because even if we don't zeroMem here, in the - # presence of user defined destructors, the user will expect the cell to be - # "destroyed" thus creating the same problem. We can destroy the cell in the - # finalizer of the sequence, but this makes destruction non-deterministic. - zeroMem(dataPointer(result, elemAlign, elemSize, newLen), (result.len-%newLen) *% elemSize) - else: - result = s - zeroMem(dataPointer(result, elemAlign, elemSize, result.len), (newLen-%result.len) *% elemSize) - result.len = newLen + # XXX: zeroing out the memory can still result in crashes if a wiped-out + # cell is aliased by another pointer (ie proc parameter or a let variable). + # This is a tough problem, because even if we don't zeroMem here, in the + # presence of user defined destructors, the user will expect the cell to be + # "destroyed" thus creating the same problem. We can destroy the cell in the + # finalizer of the sequence, but this makes destruction non-deterministic. + zeroMem(dataPointer(result, elemAlign, elemSize, newLen), (result.len-%newLen) *% elemSize) else: - result = setLengthSeq(s, typ.base.size, newLen) + result = s + zeroMem(dataPointer(result, elemAlign, elemSize, result.len), (newLen-%result.len) *% elemSize) + result.len = newLen + +func capacity*(self: string): int {.inline.} = + ## Returns the current capacity of the string. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var str = newStringOfCap(cap = 42) + str.add "Nim" + assert str.capacity == 42 + + let str = cast[NimString](self) + result = if str != nil: str.space else: 0 + +func capacity*[T](self: seq[T]): int {.inline.} = + ## Returns the current capacity of the seq. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var lst = newSeqOfCap[string](cap = 42) + lst.add "Nim" + assert lst.capacity == 42 + + let sek = cast[PGenericSeq](self) + result = if sek != nil: sek.space else: 0 diff --git a/lib/system/threadids.nim b/lib/system/threadids.nim new file mode 100644 index 0000000000..3a6eadcbbb --- /dev/null +++ b/lib/system/threadids.nim @@ -0,0 +1,103 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# we need to cache current threadId to not perform syscall all the time +var threadId {.threadvar.}: int + +when defined(windows): + proc getCurrentThreadId(): int32 {. + stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getCurrentThreadId()) + result = threadId + +elif defined(linux): + proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".} + when defined(amd64): + const NR_gettid = clong(186) + else: + var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: clong + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(NR_gettid)) + result = threadId + +elif defined(dragonfly): + proc lwp_gettid(): int32 {.importc, header: "unistd.h".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_gettid()) + result = threadId + +elif defined(openbsd): + proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getthrid()) + result = threadId + +elif defined(netbsd): + proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_self()) + result = threadId + +elif defined(freebsd): + proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} + var SYS_thr_self {.importc:"SYS_thr_self", header:"<sys/syscall.h>".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + var tid = 0.cint + if threadId == 0: + discard syscall(SYS_thr_self, addr tid) + threadId = tid + result = threadId + +elif defined(macosx): + proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} + var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"<sys/syscall.h>".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(SYS_thread_selfid)) + result = threadId + +elif defined(solaris): + type thread_t {.importc: "thread_t", header: "<thread.h>".} = distinct int + proc thr_self(): thread_t {.importc, header: "<thread.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(thr_self()) + result = threadId + +elif defined(haiku): + type thr_id {.importc: "thread_id", header: "<OS.h>".} = distinct int32 + proc find_thread(name: cstring): thr_id {.importc, header: "<OS.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(find_thread(nil)) + result = threadId diff --git a/lib/system/threadimpl.nim b/lib/system/threadimpl.nim new file mode 100644 index 0000000000..285b8f5e7f --- /dev/null +++ b/lib/system/threadimpl.nim @@ -0,0 +1,111 @@ +var + nimThreadDestructionHandlers* {.rtlThreadVar.}: seq[proc () {.closure, gcsafe, raises: [].}] +when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcRegions): + proc deallocOsPages() {.rtl, raises: [].} +proc threadTrouble() {.raises: [], gcsafe.} +# create for the main thread. Note: do not insert this data into the list +# of all threads; it's not to be stopped etc. +when not defined(useNimRtl): + #when not defined(createNimRtl): initStackBottom() + when declared(initGC): + initGC() + when not emulatedThreadVars: + type ThreadType {.pure.} = enum + None = 0, + NimThread = 1, + ForeignThread = 2 + var + threadType {.rtlThreadVar.}: ThreadType + + threadType = ThreadType.NimThread + +when defined(gcDestructors): + proc deallocThreadStorage(p: pointer) = c_free(p) +else: + template deallocThreadStorage(p: pointer) = deallocShared(p) + +template afterThreadRuns() = + for i in countdown(nimThreadDestructionHandlers.len-1, 0): + nimThreadDestructionHandlers[i]() + +proc onThreadDestruction*(handler: proc () {.closure, gcsafe, raises: [].}) = + ## Registers a *thread local* handler that is called at the thread's + ## destruction. + ## + ## A thread is destructed when the `.thread` proc returns + ## normally or when it raises an exception. Note that unhandled exceptions + ## in a thread nevertheless cause the whole process to die. + nimThreadDestructionHandlers.add handler + +when defined(boehmgc): + type GCStackBaseProc = proc(sb: pointer, t: pointer) {.noconv.} + proc boehmGC_call_with_stack_base(sbp: GCStackBaseProc, p: pointer) + {.importc: "GC_call_with_stack_base", boehmGC.} + proc boehmGC_register_my_thread(sb: pointer) + {.importc: "GC_register_my_thread", boehmGC.} + proc boehmGC_unregister_my_thread() + {.importc: "GC_unregister_my_thread", boehmGC.} + + proc threadProcWrapDispatch[TArg](sb: pointer, thrd: pointer) {.noconv, raises: [].} = + boehmGC_register_my_thread(sb) + try: + let thrd = cast[ptr Thread[TArg]](thrd) + when TArg is void: + thrd.dataFn() + else: + thrd.dataFn(thrd.data) + except: + threadTrouble() + finally: + afterThreadRuns() + boehmGC_unregister_my_thread() +else: + proc threadProcWrapDispatch[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = + try: + when TArg is void: + thrd.dataFn() + else: + when defined(nimV2): + thrd.dataFn(thrd.data) + else: + var x: TArg + deepCopy(x, thrd.data) + thrd.dataFn(x) + except: + threadTrouble() + finally: + afterThreadRuns() + when hasAllocStack: + deallocThreadStorage(thrd.rawStack) + +proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = + when defined(boehmgc): + boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) + elif not defined(nogc) and not defined(gogc) and not defined(gcRegions) and not usesDestructors: + var p {.volatile.}: pointer + # init the GC for refc/markandsweep + nimGC_setStackBottom(addr(p)) + when declared(initGC): + initGC() + when declared(threadType): + threadType = ThreadType.NimThread + threadProcWrapDispatch[TArg](thrd) + when declared(deallocOsPages): deallocOsPages() + else: + threadProcWrapDispatch(thrd) + +template nimThreadProcWrapperBody*(closure: untyped): untyped = + var thrd = cast[ptr Thread[TArg]](closure) + var core = thrd.core + when declared(globalsSlot): threadVarSetValue(globalsSlot, thrd.core) + threadProcWrapStackFrame(thrd) + # Since an unhandled exception terminates the whole process (!), there is + # no need for a ``try finally`` here, nor would it be correct: The current + # exception is tried to be re-raised by the code-gen after the ``finally``! + # However this is doomed to fail, because we already unmapped every heap + # page! + + # mark as not running anymore: + thrd.core = nil + thrd.dataFn = nil + deallocThreadStorage(cast[pointer](core)) diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim index 922150fff7..e6ad9dca58 100644 --- a/lib/system/threadlocalstorage.nim +++ b/lib/system/threadlocalstorage.nim @@ -1,82 +1,32 @@ +import std/private/threadtypes when defined(windows): - type - SysThread* = Handle - WinThreadProc = proc (x: pointer): int32 {.stdcall.} - - proc createThread(lpThreadAttributes: pointer, dwStackSize: int32, - lpStartAddress: WinThreadProc, - lpParameter: pointer, - dwCreationFlags: int32, - lpThreadId: var int32): SysThread {. - stdcall, dynlib: "kernel32", importc: "CreateThread".} - - proc winSuspendThread(hThread: SysThread): int32 {. - stdcall, dynlib: "kernel32", importc: "SuspendThread".} - - proc winResumeThread(hThread: SysThread): int32 {. - stdcall, dynlib: "kernel32", importc: "ResumeThread".} - - proc waitForMultipleObjects(nCount: int32, - lpHandles: ptr SysThread, - bWaitAll: int32, - dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} - - proc terminateThread(hThread: SysThread, dwExitCode: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "TerminateThread".} - - proc getCurrentThreadId(): int32 {. - stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} - type ThreadVarSlot = distinct int32 - when true: - proc threadVarAlloc(): ThreadVarSlot {. - importc: "TlsAlloc", stdcall, header: "<windows.h>".} - proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. - importc: "TlsSetValue", stdcall, header: "<windows.h>".} - proc tlsGetValue(dwTlsIndex: ThreadVarSlot): pointer {. - importc: "TlsGetValue", stdcall, header: "<windows.h>".} + proc threadVarAlloc(): ThreadVarSlot {. + importc: "TlsAlloc", stdcall, header: "<windows.h>".} + proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. + importc: "TlsSetValue", stdcall, header: "<windows.h>".} + proc tlsGetValue(dwTlsIndex: ThreadVarSlot): pointer {. + importc: "TlsGetValue", stdcall, header: "<windows.h>".} - proc getLastError(): uint32 {. - importc: "GetLastError", stdcall, header: "<windows.h>".} - proc setLastError(x: uint32) {. - importc: "SetLastError", stdcall, header: "<windows.h>".} + proc getLastError(): uint32 {. + importc: "GetLastError", stdcall, header: "<windows.h>".} + proc setLastError(x: uint32) {. + importc: "SetLastError", stdcall, header: "<windows.h>".} - proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer = - let realLastError = getLastError() - result = tlsGetValue(dwTlsIndex) - setLastError(realLastError) - else: - proc threadVarAlloc(): ThreadVarSlot {. - importc: "TlsAlloc", stdcall, dynlib: "kernel32".} - proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. - importc: "TlsSetValue", stdcall, dynlib: "kernel32".} - proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer {. - importc: "TlsGetValue", stdcall, dynlib: "kernel32".} - - proc setThreadAffinityMask(hThread: SysThread, dwThreadAffinityMask: uint) {. - importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} + proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer = + let realLastError = getLastError() + result = tlsGetValue(dwTlsIndex) + setLastError(realLastError) elif defined(genode): - import genode/env const GenodeHeader = "genode_cpp/threads.h" - type - SysThread* {.importcpp: "Nim::SysThread", - header: GenodeHeader, final, pure.} = object - GenodeThreadProc = proc (x: pointer) {.noconv.} - ThreadVarSlot = int - proc initThread(s: var SysThread, - env: GenodeEnv, - stackSize: culonglong, - entry: GenodeThreadProc, - arg: pointer, - affinity: cuint) {. - importcpp: "#.initThread(@)".} + type + ThreadVarSlot = int proc threadVarAlloc(): ThreadVarSlot = 0 @@ -113,60 +63,18 @@ else: when not defined(haiku): {.passc: "-pthread".} - const - schedh = "#define _GNU_SOURCE\n#include <sched.h>" - pthreadh = "#define _GNU_SOURCE\n#include <pthread.h>" - - when not declared(Time): - when defined(linux): - type Time = clong - else: - type Time = int - when (defined(linux) or defined(nintendoswitch)) and defined(amd64): type - SysThread* {.importc: "pthread_t", - header: "<sys/types.h>" .} = distinct culong - Pthread_attr {.importc: "pthread_attr_t", - header: "<sys/types.h>".} = object - abi: array[56 div sizeof(clong), clong] ThreadVarSlot {.importc: "pthread_key_t", header: "<sys/types.h>".} = distinct cuint elif defined(openbsd) and defined(amd64): type - SysThread* {.importc: "pthread_t", header: "<pthread.h>".} = object - Pthread_attr {.importc: "pthread_attr_t", - header: "<pthread.h>".} = object ThreadVarSlot {.importc: "pthread_key_t", header: "<pthread.h>".} = cint else: type - SysThread* {.importc: "pthread_t", header: "<sys/types.h>".} = object - Pthread_attr {.importc: "pthread_attr_t", - header: "<sys/types.h>".} = object ThreadVarSlot {.importc: "pthread_key_t", header: "<sys/types.h>".} = object - type - Timespec {.importc: "struct timespec", header: "<time.h>".} = object - tv_sec: Time - tv_nsec: clong - - proc pthread_attr_init(a1: var Pthread_attr): cint {. - importc, header: pthreadh.} - proc pthread_attr_setstacksize(a1: var Pthread_attr, a2: int): cint {. - importc, header: pthreadh.} - proc pthread_attr_destroy(a1: var Pthread_attr): cint {. - importc, header: pthreadh.} - - proc pthread_create(a1: var SysThread, a2: var Pthread_attr, - a3: proc (x: pointer): pointer {.noconv.}, - a4: pointer): cint {.importc: "pthread_create", - header: pthreadh.} - proc pthread_join(a1: SysThread, a2: ptr pointer): cint {. - importc, header: pthreadh.} - - proc pthread_cancel(a1: SysThread): cint {. - importc: "pthread_cancel", header: pthreadh.} proc pthread_getspecific(a1: ThreadVarSlot): pointer {. importc: "pthread_getspecific", header: pthreadh.} @@ -186,44 +94,13 @@ else: proc threadVarGetValue(s: ThreadVarSlot): pointer {.inline.} = result = pthread_getspecific(s) - type CpuSet {.importc: "cpu_set_t", header: schedh.} = object - when defined(linux) and defined(amd64): - abi: array[1024 div (8 * sizeof(culong)), culong] - - proc cpusetZero(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} - proc cpusetIncl(cpu: cint; s: var CpuSet) {. - importc: "CPU_SET", header: schedh.} - - proc setAffinity(thread: SysThread; setsize: csize_t; s: var CpuSet) {. - importc: "pthread_setaffinity_np", header: pthreadh.} - - -const - emulatedThreadVars = compileOption("tlsEmulation") when emulatedThreadVars: # the compiler generates this proc for us, so that we can get the size of # the thread local var block; we use this only for sanity checking though proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".} -# we preallocate a fixed size for thread local storage, so that no heap -# allocations are needed. Currently less than 16K are used on a 64bit machine. -# We use `float` for proper alignment: -const nimTlsSize {.intdefine.} = 16000 -type - ThreadLocalStorage = array[0..(nimTlsSize div sizeof(float)), float] - PGcThread = ptr GcThread - GcThread {.pure, inheritable.} = object - when emulatedThreadVars: - tls: ThreadLocalStorage - else: - nil - when hasSharedHeap: - next, prev: PGcThread - stackBottom, stackTop: pointer - stackSize: int - else: - nil + when emulatedThreadVars: var globalsSlot: ThreadVarSlot @@ -245,4 +122,4 @@ when not defined(useNimRtl): if nimThreadVarsSize() > sizeof(ThreadLocalStorage): c_fprintf(cstderr, """too large thread local storage size requested, use -d:\"nimTlsSize=X\" to setup even more or stop using unittest.nim""") - quit 1 + rawQuit 1 diff --git a/lib/system/threads.nim b/lib/system/threads.nim deleted file mode 100644 index def35c2388..0000000000 --- a/lib/system/threads.nim +++ /dev/null @@ -1,434 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Thread support for Nim. -## -## **Note**: This is part of the system module. Do not import it directly. -## To activate thread support you need to compile -## with the `--threads:on` command line switch. -## -## Nim's memory model for threads is quite different from other common -## programming languages (C, Pascal): Each thread has its own -## (garbage collected) heap and sharing of memory is restricted. This helps -## to prevent race conditions and improves efficiency. See `the manual for -## details of this memory model <manual.html#threads>`_. -## -## Examples -## ======== -## -## .. code-block:: Nim -## -## import std/locks -## -## var -## thr: array[0..4, Thread[tuple[a,b: int]]] -## L: Lock -## -## proc threadFunc(interval: tuple[a,b: int]) {.thread.} = -## for i in interval.a..interval.b: -## acquire(L) # lock stdout -## echo i -## release(L) -## -## initLock(L) -## -## for i in 0..high(thr): -## createThread(thr[i], threadFunc, (i*10, i*10+5)) -## joinThreads(thr) -## -## deinitLock(L) - -when not declared(ThisIsSystem): - {.error: "You must not import this module explicitly".} - -const - StackGuardSize = 4096 - ThreadStackMask = - when defined(genode): - 1024*64*sizeof(int)-1 - else: - 1024*256*sizeof(int)-1 - ThreadStackSize = ThreadStackMask+1 - StackGuardSize - -#const globalsSlot = ThreadVarSlot(0) -#sysAssert checkSlot.int == globalsSlot.int - -# create for the main thread. Note: do not insert this data into the list -# of all threads; it's not to be stopped etc. -when not defined(useNimRtl): - #when not defined(createNimRtl): initStackBottom() - when declared(initGC): - initGC() - when not emulatedThreadVars: - type ThreadType {.pure.} = enum - None = 0, - NimThread = 1, - ForeignThread = 2 - var - threadType {.rtlThreadVar.}: ThreadType - - threadType = ThreadType.NimThread - -# We jump through some hops here to ensure that Nim thread procs can have -# the Nim calling convention. This is needed because thread procs are -# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just -# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. - -type - Thread*[TArg] = object - core: PGcThread - sys: SysThread - when TArg is void: - dataFn: proc () {.nimcall, gcsafe.} - else: - dataFn: proc (m: TArg) {.nimcall, gcsafe.} - data: TArg - -var - threadDestructionHandlers {.rtlThreadVar.}: seq[proc () {.closure, gcsafe, raises: [].}] - -proc onThreadDestruction*(handler: proc () {.closure, gcsafe, raises: [].}) = - ## Registers a *thread local* handler that is called at the thread's - ## destruction. - ## - ## A thread is destructed when the `.thread` proc returns - ## normally or when it raises an exception. Note that unhandled exceptions - ## in a thread nevertheless cause the whole process to die. - threadDestructionHandlers.add handler - -template afterThreadRuns() = - for i in countdown(threadDestructionHandlers.len-1, 0): - threadDestructionHandlers[i]() - -when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcRegions): - proc deallocOsPages() {.rtl, raises: [].} - -proc threadTrouble() {.raises: [], gcsafe.} - ## defined in system/excpt.nim - -when defined(boehmgc): - type GCStackBaseProc = proc(sb: pointer, t: pointer) {.noconv.} - proc boehmGC_call_with_stack_base(sbp: GCStackBaseProc, p: pointer) - {.importc: "GC_call_with_stack_base", boehmGC.} - proc boehmGC_register_my_thread(sb: pointer) - {.importc: "GC_register_my_thread", boehmGC.} - proc boehmGC_unregister_my_thread() - {.importc: "GC_unregister_my_thread", boehmGC.} - - proc threadProcWrapDispatch[TArg](sb: pointer, thrd: pointer) {.noconv, raises: [].} = - boehmGC_register_my_thread(sb) - try: - let thrd = cast[ptr Thread[TArg]](thrd) - when TArg is void: - thrd.dataFn() - else: - thrd.dataFn(thrd.data) - except: - threadTrouble() - finally: - afterThreadRuns() - boehmGC_unregister_my_thread() -else: - proc threadProcWrapDispatch[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = - try: - when TArg is void: - thrd.dataFn() - else: - when defined(nimV2): - thrd.dataFn(thrd.data) - else: - var x: TArg - deepCopy(x, thrd.data) - thrd.dataFn(x) - except: - threadTrouble() - finally: - afterThreadRuns() - -proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = - when defined(boehmgc): - boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) - elif not defined(nogc) and not defined(gogc) and not defined(gcRegions) and not usesDestructors: - var p {.volatile.}: pointer - # init the GC for refc/markandsweep - nimGC_setStackBottom(addr(p)) - initGC() - when declared(threadType): - threadType = ThreadType.NimThread - threadProcWrapDispatch[TArg](thrd) - when declared(deallocOsPages): deallocOsPages() - else: - threadProcWrapDispatch(thrd) - -template threadProcWrapperBody(closure: untyped): untyped = - var thrd = cast[ptr Thread[TArg]](closure) - var core = thrd.core - when declared(globalsSlot): threadVarSetValue(globalsSlot, thrd.core) - threadProcWrapStackFrame(thrd) - # Since an unhandled exception terminates the whole process (!), there is - # no need for a ``try finally`` here, nor would it be correct: The current - # exception is tried to be re-raised by the code-gen after the ``finally``! - # However this is doomed to fail, because we already unmapped every heap - # page! - - # mark as not running anymore: - thrd.core = nil - thrd.dataFn = nil - deallocShared(cast[pointer](core)) - -{.push stack_trace:off.} -when defined(windows): - proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = - threadProcWrapperBody(closure) - # implicitly return 0 -elif defined(genode): - proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = - threadProcWrapperBody(closure) -else: - proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = - threadProcWrapperBody(closure) -{.pop.} - -proc running*[TArg](t: Thread[TArg]): bool {.inline.} = - ## Returns true if `t` is running. - result = t.dataFn != nil - -proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = - ## Returns the thread handle of `t`. - result = t.sys - -when hostOS == "windows": - const MAXIMUM_WAIT_OBJECTS = 64 - - proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = - ## Waits for the thread `t` to finish. - discard waitForSingleObject(t.sys, -1'i32) - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## Waits for every thread in `t` to finish. - var a: array[MAXIMUM_WAIT_OBJECTS, SysThread] - var k = 0 - while k < len(t): - var count = min(len(t) - k, MAXIMUM_WAIT_OBJECTS) - for i in 0..(count - 1): a[i] = t[i + k].sys - discard waitForMultipleObjects(int32(count), - cast[ptr SysThread](addr(a)), 1, -1) - inc(k, MAXIMUM_WAIT_OBJECTS) - -elif defined(genode): - proc joinThread*[TArg](t: Thread[TArg]) {.importcpp.} - ## Waits for the thread `t` to finish. - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## Waits for every thread in `t` to finish. - for i in 0..t.high: joinThread(t[i]) - -else: - proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = - ## Waits for the thread `t` to finish. - discard pthread_join(t.sys, nil) - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## Waits for every thread in `t` to finish. - for i in 0..t.high: joinThread(t[i]) - -when false: - # XXX a thread should really release its heap here somehow: - proc destroyThread*[TArg](t: var Thread[TArg]) = - ## Forces the thread `t` to terminate. This is potentially dangerous if - ## you don't have full control over `t` and its acquired resources. - when hostOS == "windows": - discard TerminateThread(t.sys, 1'i32) - else: - discard pthread_cancel(t.sys) - when declared(registerThread): unregisterThread(addr(t)) - t.dataFn = nil - ## if thread `t` already exited, `t.core` will be `null`. - if not isNil(t.core): - deallocShared(t.core) - t.core = nil - -when hostOS == "windows": - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread, nimcall.}, - param: TArg) = - ## Creates a new thread `t` and starts its execution. - ## - ## Entry point is the proc `tp`. - ## `param` is passed to `tp`. `TArg` can be `void` if you - ## don't need to pass any data to the thread. - t.core = cast[PGcThread](allocShared0(sizeof(GcThread))) - - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.core.stackSize = ThreadStackSize - var dummyThreadId: int32 - t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], - addr(t), 0'i32, dummyThreadId) - if t.sys <= 0: - raise newException(ResourceExhaustedError, "cannot create thread") - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - ## Pins a thread to a `CPU`:idx:. - ## - ## In other words sets a thread's `affinity`:idx:. - ## If you don't know what this means, you shouldn't use this proc. - setThreadAffinityMask(t.sys, uint(1 shl cpu)) - -elif defined(genode): - var affinityOffset: cuint = 1 - ## CPU affinity offset for next thread, safe to roll-over. - - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread, nimcall.}, - param: TArg) = - t.core = cast[PGcThread](allocShared0(sizeof(GcThread))) - - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.stackSize = ThreadStackSize - t.sys.initThread( - runtimeEnv, - ThreadStackSize.culonglong, - threadProcWrapper[TArg], addr(t), affinityOffset) - inc affinityOffset - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - {.hint: "cannot change Genode thread CPU affinity after initialization".} - discard - -else: - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread, nimcall.}, - param: TArg) = - ## Creates a new thread `t` and starts its execution. - ## - ## Entry point is the proc `tp`. `param` is passed to `tp`. - ## `TArg` can be `void` if you - ## don't need to pass any data to the thread. - t.core = cast[PGcThread](allocShared0(sizeof(GcThread))) - - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.core.stackSize = ThreadStackSize - var a {.noinit.}: Pthread_attr - doAssert pthread_attr_init(a) == 0 - let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) - when not defined(ios): - # This fails on iOS - doAssert(setstacksizeResult == 0) - if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: - raise newException(ResourceExhaustedError, "cannot create thread") - doAssert pthread_attr_destroy(a) == 0 - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - ## Pins a thread to a `CPU`:idx:. - ## - ## In other words sets a thread's `affinity`:idx:. - ## If you don't know what this means, you shouldn't use this proc. - when not defined(macosx): - var s {.noinit.}: CpuSet - cpusetZero(s) - cpusetIncl(cpu.cint, s) - setAffinity(t.sys, csize_t(sizeof(s)), s) - -proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = - createThread[void](t, tp) - -# we need to cache current threadId to not perform syscall all the time -var threadId {.threadvar.}: int - -when defined(windows): - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(getCurrentThreadId()) - result = threadId - -elif defined(linux): - proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".} - when defined(amd64): - const NR_gettid = clong(186) - else: - var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: clong - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(NR_gettid)) - result = threadId - -elif defined(dragonfly): - proc lwp_gettid(): int32 {.importc, header: "unistd.h".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_gettid()) - result = threadId - -elif defined(openbsd): - proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".} - - proc getThreadId*(): int = - ## get the ID of the currently running thread. - if threadId == 0: - threadId = int(getthrid()) - result = threadId - -elif defined(netbsd): - proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_self()) - result = threadId - -elif defined(freebsd): - proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} - var SYS_thr_self {.importc:"SYS_thr_self", header:"<sys/syscall.h>"}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - var tid = 0.cint - if threadId == 0: - discard syscall(SYS_thr_self, addr tid) - threadId = tid - result = threadId - -elif defined(macosx): - proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} - var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"<sys/syscall.h>".}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(SYS_thread_selfid)) - result = threadId - -elif defined(solaris): - type thread_t {.importc: "thread_t", header: "<thread.h>".} = distinct int - proc thr_self(): thread_t {.importc, header: "<thread.h>".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(thr_self()) - result = threadId - -elif defined(haiku): - type thr_id {.importc: "thread_id", header: "<OS.h>".} = distinct int32 - proc find_thread(name: cstring): thr_id {.importc, header: "<OS.h>".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(find_thread(nil)) - result = threadId diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim deleted file mode 100644 index 16bc3f3cae..0000000000 --- a/lib/system/widestrs.nim +++ /dev/null @@ -1,226 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Nim support for C/C++'s `wide strings`:idx:. This is part of the system -# module! Do not import it directly! - -#when not declared(ThisIsSystem): -# {.error: "You must not import this module explicitly".} - -type - Utf16Char* = distinct int16 - -when defined(nimv2): - - type - WideCString* = ptr UncheckedArray[Utf16Char] - - WideCStringObj* = object - bytes: int - data: WideCString - - proc `=destroy`(a: var WideCStringObj) = - if a.data != nil: - when compileOption("threads"): - deallocShared(a.data) - else: - dealloc(a.data) - - proc `=`(a: var WideCStringObj; b: WideCStringObj) {.error.} - - proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = - a.bytes = b.bytes - a.data = b.data - - proc createWide(a: var WideCStringObj; bytes: int) = - a.bytes = bytes - when compileOption("threads"): - a.data = cast[typeof(a.data)](allocShared0(bytes)) - else: - a.data = cast[typeof(a.data)](alloc0(bytes)) - - template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx] - template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val - - template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil) - - converter toWideCString*(x: WideCStringObj): WideCString {.inline.} = - result = x.data - -else: - template nullWide(): untyped = nil - - type - WideCString* = ref UncheckedArray[Utf16Char] - WideCStringObj* = WideCString - - template createWide(a; L) = - unsafeNew(a, L) - -proc ord(arg: Utf16Char): int = int(cast[uint16](arg)) - -proc len*(w: WideCString): int = - ## returns the length of a widestring. This traverses the whole string to - ## find the binary zero end marker! - result = 0 - while int16(w[result]) != 0'i16: inc result - -const - UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) - UNI_MAX_BMP = 0x0000FFFF - UNI_MAX_UTF16 = 0x0010FFFF - # UNI_MAX_UTF32 = 0x7FFFFFFF - # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF - - halfShift = 10 - halfBase = 0x0010000 - halfMask = 0x3FF - - UNI_SUR_HIGH_START = 0xD800 - UNI_SUR_HIGH_END = 0xDBFF - UNI_SUR_LOW_START = 0xDC00 - UNI_SUR_LOW_END = 0xDFFF - UNI_REPL = 0xFFFD - -template ones(n: untyped): untyped = ((1 shl n)-1) - -template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) = - ## Returns the unicode character `s[i]` in `result`. If `doInc == true` - ## `i` is incremented by the number of bytes that have been processed. - bind ones - - if ord(s[i]) <= 127: - result = ord(s[i]) - when doInc: inc(i) - elif ord(s[i]) shr 5 == 0b110: - #assert(ord(s[i+1]) shr 6 == 0b10) - if i <= L - 2: - result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) - when doInc: inc(i, 2) - else: - result = UNI_REPL - when doInc: inc(i) - elif ord(s[i]) shr 4 == 0b1110: - if i <= L - 3: - #assert(ord(s[i+1]) shr 6 == 0b10) - #assert(ord(s[i+2]) shr 6 == 0b10) - result = (ord(s[i]) and ones(4)) shl 12 or - (ord(s[i+1]) and ones(6)) shl 6 or - (ord(s[i+2]) and ones(6)) - when doInc: inc(i, 3) - else: - result = UNI_REPL - when doInc: inc(i) - elif ord(s[i]) shr 3 == 0b11110: - if i <= L - 4: - #assert(ord(s[i+1]) shr 6 == 0b10) - #assert(ord(s[i+2]) shr 6 == 0b10) - #assert(ord(s[i+3]) shr 6 == 0b10) - result = (ord(s[i]) and ones(3)) shl 18 or - (ord(s[i+1]) and ones(6)) shl 12 or - (ord(s[i+2]) and ones(6)) shl 6 or - (ord(s[i+3]) and ones(6)) - when doInc: inc(i, 4) - else: - result = UNI_REPL - when doInc: inc(i) - else: - result = 0xFFFD - when doInc: inc(i) - -iterator runes(s: cstring, L: int): int = - var - i = 0 - result: int - while i < L: - fastRuneAt(s, i, L, result, true) - yield result - -proc newWideCString*(size: int): WideCStringObj = - createWide(result, size * 2 + 2) - -proc newWideCString*(source: cstring, L: int): WideCStringObj = - createWide(result, L * 2 + 2) - var d = 0 - for ch in runes(source, L): - - if ch <= UNI_MAX_BMP: - if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END: - result[d] = UNI_REPLACEMENT_CHAR - else: - result[d] = cast[Utf16Char](uint16(ch)) - elif ch > UNI_MAX_UTF16: - result[d] = UNI_REPLACEMENT_CHAR - else: - let ch = ch - halfBase - result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START)) - inc d - result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START)) - inc d - result[d] = Utf16Char(0) - -proc newWideCString*(s: cstring): WideCStringObj = - if s.isNil: return nullWide - - result = newWideCString(s, s.len) - -proc newWideCString*(s: string): WideCStringObj = - result = newWideCString(s, s.len) - -proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = - result = newStringOfCap(estimate + estimate shr 2) - - var i = 0 - while w[i].int16 != 0'i16: - var ch = ord(w[i]) - inc i - if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: - # If the 16 bits following the high surrogate are in the source buffer... - let ch2 = ord(w[i]) - - # If it's a low surrogate, convert to UTF32: - if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: - ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase - inc i - else: - #invalid UTF-16 - ch = replacement - elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: - #invalid UTF-16 - ch = replacement - - if ch < 0x80: - result.add chr(ch) - elif ch < 0x800: - result.add chr((ch shr 6) or 0xc0) - result.add chr((ch and 0x3f) or 0x80) - elif ch < 0x10000: - result.add chr((ch shr 12) or 0xe0) - result.add chr(((ch shr 6) and 0x3f) or 0x80) - result.add chr((ch and 0x3f) or 0x80) - elif ch <= 0x10FFFF: - result.add chr((ch shr 18) or 0xf0) - result.add chr(((ch shr 12) and 0x3f) or 0x80) - result.add chr(((ch shr 6) and 0x3f) or 0x80) - result.add chr((ch and 0x3f) or 0x80) - else: - # replacement char(in case user give very large number): - result.add chr(0xFFFD shr 12 or 0b1110_0000) - result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) - result.add chr(0xFFFD and ones(6) or 0b10_0000_00) - -proc `$`*(s: WideCString): string = - result = s $ 80 - -when defined(nimv2): - proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string = - `$`(s.data, estimate, replacement) - - proc `$`*(s: WideCStringObj): string = - $(s.data) diff --git a/lib/system_overview.rst b/lib/system_overview.rst index d6cbe1a35f..cc0643bf16 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -1,13 +1,14 @@ .. default-role:: code +.. include:: ../doc/rstcommon.rst The System module imports several separate modules, and their documentation is in separate files: * `iterators <iterators.html>`_ +* `exceptions <exceptions.html>`_ * `assertions <assertions.html>`_ * `dollars <dollars.html>`_ -* `io <io.html>`_ -* `widestrs <widestrs.html>`_ +* `ctypes <ctypes.html>`_ Here is a short overview of the most commonly used functions from the @@ -48,27 +49,27 @@ Proc Usage Seqs ---- -============================================== ========================================== -Proc Usage -============================================== ========================================== -`newSeq<#newSeq>`_ Create a new sequence of a given length -`newSeqOfCap<#newSeqOfCap,Natural>`_ Create a new sequence with zero length - and a given capacity -`setLen<#setLen,seq[T],Natural>`_ Set the length of a sequence -`len<#len,seq[T]>`_ Return the length of a sequence -`@<#@,openArray[T]>`_ Turn an array into a sequence -`add<#add,seq[T],sinkT>`_ Add an item to the sequence -`insert<#insert,seq[T],sinkT>`_ Insert an item at a specific position -`delete<#delete,seq[T],Natural>`_ Delete an item while preserving the - order of elements (`O(n)` operation) -`del<#del,seq[T],Natural>`_ `O(1)` removal, doesn't preserve the order -`pop<#pop,seq[T]>`_ Remove and return last item of a sequence -`x & y<#&,seq[T],seq[T]>`_ Concatenate two sequences -`x[a .. b]<#[],openArray[T],HSlice[U,V]>`_ Slice of a sequence (both ends included) -`x[a .. ^b]<#[],openArray[T],HSlice[U,V]>`_ Slice of a sequence but `b` is a - reversed index (both ends included) -`x[a ..\< b]<#[],openArray[T],HSlice[U,V]>`_ Slice of a sequence (excluded upper bound) -============================================== ========================================== +============================================================= ========================================== +Proc Usage +============================================================= ========================================== +`newSeq<#newSeq>`_ Create a new sequence of a given length +`newSeqOfCap<#newSeqOfCap,Natural>`_ Create a new sequence with zero length + and a given capacity +`setLen<#setLen,seq[T],Natural>`_ Set the length of a sequence +`len<#len,seq[T]>`_ Return the length of a sequence +`@<#@,openArray[T]>`_ Turn an array into a sequence +`add<#add,seq[T],sinkT>`_ Add an item to the sequence +`insert<#insert,seq[T],sinkT>`_ Insert an item at a specific position +`delete<#delete,seq[T],Natural>`_ Delete an item while preserving the + order of elements (`O(n)` operation) +`del<#del,seq[T],Natural>`_ `O(1)` removal, doesn't preserve the order +`pop<#pop,seq[T]>`_ Remove and return last item of a sequence +`x & y<#&,seq[T],seq[T]>`_ Concatenate two sequences +`x[a .. b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence (both ends included) +`x[a .. ^b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence but `b` is a + reversed index (both ends included) +`x[a ..< b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence (excluded upper bound) +============================================================= ========================================== **See also:** * `sequtils module <sequtils.html>`_ for operations on container @@ -98,6 +99,7 @@ Proc Usage =============================== ====================================== **See also:** +* `setutils module <setutils.html>`_ for bit set convenience functions * `sets module <sets.html>`_ for hash sets * `intsets module <intsets.html>`_ for efficient int sets diff --git a/lib/windows/registry.nim b/lib/windows/registry.nim index c17f2f455c..207172f8ca 100644 --- a/lib/windows/registry.nim +++ b/lib/windows/registry.nim @@ -9,7 +9,10 @@ ## This module is experimental and its interface may change. -import winlean, os +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/widestrs type HKEY* = uint @@ -46,24 +49,26 @@ template call(f) = proc getUnicodeValue*(path, key: string; handle: HKEY): string = let hh = newWideCString path let kk = newWideCString key - var bufsize: int32 + var bufSize: int32 # try a couple of different flag settings: var flags: int32 = RRF_RT_ANY - let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufsize) + let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufSize) if err != 0: var newHandle: HKEY call regOpenKeyEx(handle, hh, 0, KEY_READ or KEY_WOW64_64KEY, newHandle) - call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufsize) - var res = newWideCString("", bufsize) - call regGetValue(newHandle, nil, kk, flags, nil, cast[pointer](res), - addr bufsize) - result = res $ bufsize + call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufSize) + if bufSize > 0: + var res = newWideCString(bufSize) + call regGetValue(newHandle, nil, kk, flags, nil, addr res[0], + addr bufSize) + result = res $ bufSize call regCloseKey(newHandle) else: - var res = newWideCString("", bufsize) - call regGetValue(handle, hh, kk, flags, nil, cast[pointer](res), - addr bufsize) - result = res $ bufsize + if bufSize > 0: + var res = newWideCString(bufSize) + call regGetValue(handle, hh, kk, flags, nil, addr res[0], + addr bufSize) + result = res $ bufSize proc regSetValue(key: HKEY, lpSubKey, lpValueName: WideCString, dwType: int32; lpData: WideCString; cbData: int32): int32 {. diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 1cdb20fe73..79681376b2 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -10,20 +10,18 @@ ## This module implements a small wrapper for some needed Win API procedures, ## so that the Nim compiler does not depend on the huge Windows module. -import dynlib +import std/dynlib when defined(nimHasStyleChecks): {.push styleChecks: off.} {.passc: "-DWIN32_LEAN_AND_MEAN".} -const - useWinUnicode* = not defined(useWinAnsi) +when defined(nimPreviewSlimSystem): + from std/syncio import FileHandle + import std/widestrs -when useWinUnicode: - type WinChar* = Utf16Char -else: - type WinChar* = char +type WinChar* = Utf16Char # See https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types type @@ -42,14 +40,14 @@ type PULONG_PTR* = ptr uint HDC* = Handle HGLRC* = Handle - BYTE* = cuchar + BYTE* = uint8 - SECURITY_ATTRIBUTES* {.final, pure.} = object + SECURITY_ATTRIBUTES* = object nLength*: int32 lpSecurityDescriptor*: pointer bInheritHandle*: WINBOOL - STARTUPINFO* {.final, pure.} = object + STARTUPINFO* = object cb*: int32 lpReserved*: cstring lpDesktop*: cstring @@ -69,17 +67,17 @@ type hStdOutput*: Handle hStdError*: Handle - PROCESS_INFORMATION* {.final, pure.} = object + PROCESS_INFORMATION* = object hProcess*: Handle hThread*: Handle dwProcessId*: int32 dwThreadId*: int32 - FILETIME* {.final, pure.} = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT + FILETIME* = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT dwLowDateTime*: DWORD dwHighDateTime*: DWORD - BY_HANDLE_FILE_INFORMATION* {.final, pure.} = object + BY_HANDLE_FILE_INFORMATION* = object dwFileAttributes*: DWORD ftCreationTime*: FILETIME ftLastAccessTime*: FILETIME @@ -91,7 +89,7 @@ type nFileIndexHigh*: DWORD nFileIndexLow*: DWORD - OSVERSIONINFO* {.final, pure.} = object + OSVERSIONINFO* = object dwOSVersionInfoSize*: DWORD dwMajorVersion*: DWORD dwMinorVersion*: DWORD @@ -180,26 +178,14 @@ proc peekNamedPipe*(hNamedPipe: Handle, lpBuffer: pointer=nil, lpBytesLeftThisMessage: ptr int32 = nil): bool {. stdcall, dynlib: "kernel32", importc: "PeekNamedPipe".} -when useWinUnicode: - proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment, lpCurrentDirectory: WideCString, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} - -else: - proc createProcessA*(lpApplicationName, lpCommandLine: cstring, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment: pointer, lpCurrentDirectory: cstring, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessA", sideEffect.} - +proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, + lpProcessAttributes: ptr SECURITY_ATTRIBUTES, + lpThreadAttributes: ptr SECURITY_ATTRIBUTES, + bInheritHandles: WINBOOL, dwCreationFlags: int32, + lpEnvironment, lpCurrentDirectory: WideCString, + lpStartupInfo: var STARTUPINFO, + lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} proc suspendThread*(hThread: Handle): int32 {.stdcall, dynlib: "kernel32", importc: "SuspendThread", sideEffect.} @@ -228,67 +214,37 @@ proc getLastError*(): int32 {.importc: "GetLastError", proc setLastError*(error: int32) {.importc: "SetLastError", stdcall, dynlib: "kernel32", sideEffect.} -when useWinUnicode: - proc formatMessageW*(dwFlags: int32, lpSource: pointer, - dwMessageId, dwLanguageId: int32, - lpBuffer: pointer, nSize: int32, - arguments: pointer): int32 {. - importc: "FormatMessageW", stdcall, dynlib: "kernel32".} -else: - proc formatMessageA*(dwFlags: int32, lpSource: pointer, +proc formatMessageW*(dwFlags: int32, lpSource: pointer, dwMessageId, dwLanguageId: int32, lpBuffer: pointer, nSize: int32, arguments: pointer): int32 {. - importc: "FormatMessageA", stdcall, dynlib: "kernel32".} + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} proc localFree*(p: pointer) {. importc: "LocalFree", stdcall, dynlib: "kernel32".} -when useWinUnicode: - proc getCurrentDirectoryW*(nBufferLength: int32, - lpBuffer: WideCString): int32 {. - importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. - importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. - importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc removeDirectoryW*(lpPathName: WideCString): int32 {. - importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} +proc getCurrentDirectoryW*(nBufferLength: int32, + lpBuffer: WideCString): int32 {. + importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. + importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. + importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc removeDirectoryW*(lpPathName: WideCString): int32 {. + importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} - proc getModuleFileNameW*(handle: Handle, buf: WideCString, - size: int32): int32 {.importc: "GetModuleFileNameW", - dynlib: "kernel32", stdcall.} -else: - proc getCurrentDirectoryA*(nBufferLength: int32, lpBuffer: cstring): int32 {. - importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc setCurrentDirectoryA*(lpPathName: cstring): int32 {. - importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {. - importc: "CreateDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc removeDirectoryA*(lpPathName: cstring): int32 {. - importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA", sideEffect.} +proc getModuleFileNameW*(handle: Handle, buf: WideCString, + size: int32): int32 {.importc: "GetModuleFileNameW", + dynlib: "kernel32", stdcall.} - proc getModuleFileNameA*(handle: Handle, buf: cstring, size: int32): int32 {. - importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} - -when useWinUnicode: - proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} - proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, - security: pointer=nil): int32 {. - importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} -else: - proc createSymbolicLinkA*(lpSymlinkFileName, lpTargetFileName: cstring, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkA", dynlib: "kernel32", stdcall, sideEffect.} - proc createHardLinkA*(lpFileName, lpExistingFileName: cstring, - security: pointer=nil): int32 {. - importc:"CreateHardLinkA", dynlib: "kernel32", stdcall, sideEffect.} +proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, + flags: DWORD): int32 {. + importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} +proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, + security: pointer=nil): int32 {. + importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} const FILE_ATTRIBUTE_READONLY* = 0x00000001'i32 @@ -339,90 +295,51 @@ type cFileName*: array[0..(MAX_PATH) - 1, WinChar] cAlternateFileName*: array[0..13, WinChar] -when useWinUnicode: - proc findFirstFileW*(lpFileName: WideCString, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} - proc findNextFileW*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} -else: - proc findFirstFileA*(lpFileName: cstring, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileA", sideEffect.} - proc findNextFileA*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileA", sideEffect.} +proc findFirstFileW*(lpFileName: WideCString, + lpFindFileData: var WIN32_FIND_DATA): Handle {. + stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} +proc findNextFileW*(hFindFile: Handle, + lpFindFileData: var WIN32_FIND_DATA): int32 {. + stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} proc findClose*(hFindFile: Handle) {.stdcall, dynlib: "kernel32", importc: "FindClose".} -when useWinUnicode: - proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, - lpBuffer: WideCString, - lpFilePart: var WideCString): int32 {. +proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, + lpBuffer: WideCString, + lpFilePart: var WideCString): int32 {. + stdcall, dynlib: "kernel32", + importc: "GetFullPathNameW", sideEffect.} +proc getFileAttributesW*(lpFileName: WideCString): int32 {. stdcall, dynlib: "kernel32", - importc: "GetFullPathNameW", sideEffect.} - proc getFileAttributesW*(lpFileName: WideCString): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesW", sideEffect.} - proc setFileAttributesW*(lpFileName: WideCString, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} + importc: "GetFileAttributesW", sideEffect.} +proc setFileAttributesW*(lpFileName: WideCString, + dwFileAttributes: int32): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} - proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: WINBOOL): WINBOOL {. - importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, + bFailIfExists: WINBOOL): WINBOOL {. + importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. - importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, - flags: DWORD): WINBOOL {. - importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. + importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, + flags: DWORD): WINBOOL {. + importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} - proc getEnvironmentStringsW*(): WideCString {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} - proc freeEnvironmentStringsW*(para1: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} +proc getEnvironmentStringsW*(): WideCString {. + stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} +proc freeEnvironmentStringsW*(para1: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} - proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", - stdcall, dynlib: "kernel32", sideEffect.} - -else: - proc getFullPathNameA*(lpFileName: cstring, nBufferLength: int32, - lpBuffer: cstring, lpFilePart: var cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFullPathNameA", sideEffect.} - proc getFileAttributesA*(lpFileName: cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesA", sideEffect.} - proc setFileAttributesA*(lpFileName: cstring, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesA", sideEffect.} - - proc copyFileA*(lpExistingFileName, lpNewFileName: cstring, - bFailIfExists: cint): cint {. - importc: "CopyFileA", stdcall, dynlib: "kernel32", sideEffect.} - - proc moveFileA*(lpExistingFileName, lpNewFileName: cstring): WINBOOL {. - importc: "MoveFileA", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileExA*(lpExistingFileName, lpNewFileName: cstring, - flags: DWORD): WINBOOL {. - importc: "MoveFileExA", stdcall, dynlib: "kernel32", sideEffect.} - - proc getEnvironmentStringsA*(): cstring {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsA", sideEffect.} - proc freeEnvironmentStringsA*(para1: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsA", sideEffect.} - - proc getCommandLineA*(): cstring {. - importc: "GetCommandLineA", stdcall, dynlib: "kernel32", sideEffect.} +proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", + stdcall, dynlib: "kernel32", sideEffect.} proc rdFileTime*(f: FILETIME): int64 = - result = ze64(f.dwLowDateTime) or (ze64(f.dwHighDateTime) shl 32) + result = int64(cast[uint32](f.dwLowDateTime)) or (int64(cast[uint32](f.dwHighDateTime)) shl 32) proc rdFileSize*(f: WIN32_FIND_DATA): int64 = - result = ze64(f.nFileSizeLow) or (ze64(f.nFileSizeHigh) shl 32) + result = int64(cast[uint32](f.nFileSizeLow)) or (int64(cast[uint32](f.nFileSizeHigh)) shl 32) proc getSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. importc: "GetSystemTimeAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} @@ -430,17 +347,10 @@ proc getSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. proc sleep*(dwMilliseconds: int32){.stdcall, dynlib: "kernel32", importc: "Sleep", sideEffect.} -when useWinUnicode: - proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: WideCString, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} - -else: - proc shellExecuteA*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: cstring, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteA", sideEffect.} +proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, + lpParameters, lpDirectory: WideCString, + nShowCmd: int32): Handle{. + stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} proc getFileInformationByHandle*(hFile: Handle, lpFileInformation: ptr BY_HANDLE_FILE_INFORMATION): WINBOOL{. @@ -478,7 +388,7 @@ type PSockAddr = ptr SockAddr - InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object + InAddr* {.importc: "IN_ADDR", header: "winsock2.h", union.} = object s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", @@ -502,9 +412,9 @@ type Sockaddr_storage* {.importc: "SOCKADDR_STORAGE", header: "winsock2.h".} = object ss_family*: uint16 - ss_pad1: array[6, byte] - ss_align: int64 - ss_pad2: array[112, byte] + ss_pad1 {.importc: "__ss_pad1".}: array[6, byte] + ss_align {.importc: "__ss_align".}: int64 + ss_pad2 {.importc: "__ss_pad2".}: array[112, byte] Servent* = object s_name*: cstring @@ -667,7 +577,7 @@ proc getaddrinfo*(nodename, servname: cstring, hints: ptr AddrInfo, res: var ptr AddrInfo): cint {. stdcall, importc: "getaddrinfo", dynlib: ws2dll.} -proc freeaddrinfo*(ai: ptr AddrInfo) {. +proc freeAddrInfo*(ai: ptr AddrInfo) {. stdcall, importc: "freeaddrinfo", dynlib: ws2dll.} proc inet_ntoa*(i: InAddr): cstring {. @@ -718,12 +628,13 @@ const # Error Constants const - ERROR_FILE_NOT_FOUND* = 2 + ERROR_FILE_NOT_FOUND* = 2 ## https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499- ERROR_PATH_NOT_FOUND* = 3 ERROR_ACCESS_DENIED* = 5 ERROR_NO_MORE_FILES* = 18 ERROR_LOCK_VIOLATION* = 33 ERROR_HANDLE_EOF* = 38 + ERROR_FILE_EXISTS* = 80 ERROR_BAD_ARGUMENTS* = 165 proc duplicateHandle*(hSourceProcessHandle: Handle, hSourceHandle: Handle, @@ -789,13 +700,6 @@ proc createFileMappingW*(hFile: Handle, lpName: pointer): Handle {. stdcall, dynlib: "kernel32", importc: "CreateFileMappingW".} -when not useWinUnicode: - proc createFileMappingA*(hFile: Handle, - lpFileMappingAttributes: pointer, - flProtect, dwMaximumSizeHigh: DWORD, - dwMaximumSizeLow: DWORD, lpName: cstring): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateFileMappingA".} - proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, dynlib: "kernel32", importc: "UnmapViewOfFile".} @@ -815,7 +719,7 @@ type POVERLAPPED_COMPLETION_ROUTINE* = proc (para1: DWORD, para2: DWORD, para3: POVERLAPPED){.stdcall.} - GUID* {.final, pure.} = object + GUID* = object D1*: int32 D2*: int16 D3*: int16 @@ -927,6 +831,9 @@ proc getProcessTimes*(hProcess: Handle; lpCreationTime, lpExitTime, lpKernelTime, lpUserTime: var FILETIME): WINBOOL {.stdcall, dynlib: "kernel32", importc: "GetProcessTimes".} +proc getSystemTimePreciseAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. + importc: "GetSystemTimePreciseAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} + type inet_ntop_proc = proc(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.gcsafe, stdcall, tags: [].} @@ -968,7 +875,7 @@ proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = var ver: OSVERSIONINFO ver.dwOSVersionInfoSize = sizeof(ver).DWORD - let res = when useWinUnicode: getVersionExW(ver.addr) else: getVersionExA(ver.addr) + let res = getVersionExW(ver.addr) if res == 0: result = nil elif ver.dwMajorVersion >= 6: @@ -1030,7 +937,7 @@ const PROCESS_QUERY_LIMITED_INFORMATION* = 0x00001000'i32 PROCESS_SET_LIMITED_INFORMATION* = 0x00002000'i32 type - WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32): void {.stdcall.} + WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32) {.stdcall.} proc postQueuedCompletionStatus*(CompletionPort: Handle, dwNumberOfBytesTransferred: DWORD, @@ -1052,16 +959,10 @@ proc openProcess*(dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, dwProcessId: DWORD): Handle {.stdcall, dynlib: "kernel32", importc: "OpenProcess".} -when defined(useWinAnsi): - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: cstring): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventA".} -else: - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: ptr Utf16Char): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} +proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: ptr Utf16Char): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} proc setEvent*(hEvent: Handle): cint {.stdcall, dynlib: "kernel32", importc: "SetEvent".} @@ -1093,7 +994,7 @@ proc wsaResetEvent*(hEvent: Handle): bool {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} type - KEY_EVENT_RECORD* {.final, pure.} = object + KEY_EVENT_RECORD* = object eventType*: int16 bKeyDown*: WINBOOL wRepeatCount*: int16 @@ -1102,17 +1003,12 @@ type uChar*: int16 dwControlKeyState*: DWORD -when defined(useWinAnsi): - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputA".} -else: - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} +proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} type - LPFIBER_START_ROUTINE* = proc (param: pointer): void {.stdcall.} + LPFIBER_START_ROUTINE* = proc (param: pointer) {.stdcall.} const FIBER_FLAG_FLOAT_SWITCH* = 0x01 @@ -1121,8 +1017,8 @@ proc CreateFiber*(stackSize: int, fn: LPFIBER_START_ROUTINE, param: pointer): po proc CreateFiberEx*(stkCommit: int, stkReserve: int, flags: int32, fn: LPFIBER_START_ROUTINE, param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} proc ConvertThreadToFiber*(param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} proc ConvertThreadToFiberEx*(param: pointer, flags: int32): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} -proc DeleteFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} -proc SwitchToFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} +proc DeleteFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} +proc SwitchToFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} proc GetCurrentFiber*(): pointer {.stdcall, importc, header: "windows.h".} proc toFILETIME*(t: int64): FILETIME = @@ -1139,7 +1035,7 @@ proc setFileTime*(hFile: Handle, lpCreationTime: LPFILETIME, type # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid_identifier_authority SID_IDENTIFIER_AUTHORITY* {.importc, header: "<windows.h>".} = object - value* {.importc: "Value"}: array[6, BYTE] + value* {.importc: "Value".}: array[6, BYTE] # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid SID* {.importc, header: "<windows.h>".} = object Revision: BYTE diff --git a/lib/wrappers/linenoise/linenoise.nim b/lib/wrappers/linenoise/linenoise.nim index 5b1fa71907..186b3b2523 100644 --- a/lib/wrappers/linenoise/linenoise.nim +++ b/lib/wrappers/linenoise/linenoise.nim @@ -32,17 +32,17 @@ proc printKeyCodes*() {.importc: "linenoisePrintKeyCodes".} proc free*(s: cstring) {.importc: "free", header: "<stdlib.h>".} -when defined nimExperimentalLinenoiseExtra: +when defined(nimExperimentalLinenoiseExtra) and not defined(windows): # C interface - type linenoiseStatus = enum + type LinenoiseStatus = enum linenoiseStatus_ctrl_unknown linenoiseStatus_ctrl_C linenoiseStatus_ctrl_D - type linenoiseData* = object - status: linenoiseStatus + type LinenoiseData* = object + status: LinenoiseStatus - proc linenoiseExtra(prompt: cstring, data: ptr linenoiseData): cstring {.importc.} + proc linenoiseExtra(prompt: cstring, data: ptr LinenoiseData): cstring {.importc.} # stable nim interface type Status* = enum @@ -65,7 +65,7 @@ when defined nimExperimentalLinenoiseExtra: if ret.line.len > 0: echo ret.line if ret.status == lnCtrlD: break echo "exiting" - var data: linenoiseData + var data: LinenoiseData let buf = linenoiseExtra(prompt, data.addr) result.line = $buf free(buf) diff --git a/lib/wrappers/mysql.nim b/lib/wrappers/mysql.nim deleted file mode 100644 index 2fefe4a8b7..0000000000 --- a/lib/wrappers/mysql.nim +++ /dev/null @@ -1,1115 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2010 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.push, callconv: cdecl.} -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -when defined(unix): - when defined(macosx): - const - lib = "(libmysqlclient|libmariadbclient)(|.20|.19|.18|.17|.16|.15).dylib" - else: - const - lib = "(libmysqlclient|libmariadbclient).so(|.20|.19|.18|.17|.16|.15)" -when defined(windows): - const - lib = "(libmysql.dll|libmariadb.dll)" -type - my_bool* = bool - Pmy_bool* = ptr my_bool - PVIO* = pointer - Pgptr* = ptr gptr - gptr* = cstring - Pmy_socket* = ptr my_socket - my_socket* = cint - PPByte* = pointer - cuint* = cint - -# ------------ Start of declaration in "mysql_com.h" --------------------- -# -# ** Common definition between mysql server & client -# -# Field/table name length - -const - NAME_LEN* = 64 - HOSTNAME_LENGTH* = 60 - USERNAME_LENGTH* = 16 - SERVER_VERSION_LENGTH* = 60 - SQLSTATE_LENGTH* = 5 - LOCAL_HOST* = "localhost" - LOCAL_HOST_NAMEDPIPE* = '.' - -const - NAMEDPIPE* = "MySQL" - SERVICENAME* = "MySQL" - -type - Enum_server_command* = enum - COM_SLEEP, COM_QUIT, COM_INIT_DB, COM_QUERY, COM_FIELD_LIST, COM_CREATE_DB, - COM_DROP_DB, COM_REFRESH, COM_SHUTDOWN, COM_STATISTICS, COM_PROCESS_INFO, - COM_CONNECT, COM_PROCESS_KILL, COM_DEBUG, COM_PING, COM_TIME, - COM_DELAYED_INSERT, COM_CHANGE_USER, COM_BINLOG_DUMP, COM_TABLE_DUMP, - COM_CONNECT_OUT, COM_REGISTER_SLAVE, COM_STMT_PREPARE, COM_STMT_EXECUTE, - COM_STMT_SEND_LONG_DATA, COM_STMT_CLOSE, COM_STMT_RESET, COM_SET_OPTION, - COM_STMT_FETCH, COM_END -{.deprecated: [Tenum_server_command: Enum_server_command].} - -const - SCRAMBLE_LENGTH* = 20 # Length of random string sent by server on handshake; - # this is also length of obfuscated password, - # received from client - SCRAMBLE_LENGTH_323* = 8 # length of password stored in the db: - # new passwords are preceded with '*' - SCRAMBLED_PASSWORD_CHAR_LENGTH* = SCRAMBLE_LENGTH * 2 + 1 - SCRAMBLED_PASSWORD_CHAR_LENGTH_323* = SCRAMBLE_LENGTH_323 * 2 - NOT_NULL_FLAG* = 1 # Field can't be NULL - PRI_KEY_FLAG* = 2 # Field is part of a primary key - UNIQUE_KEY_FLAG* = 4 # Field is part of a unique key - MULTIPLE_KEY_FLAG* = 8 # Field is part of a key - BLOB_FLAG* = 16 # Field is a blob - UNSIGNED_FLAG* = 32 # Field is unsigned - ZEROFILL_FLAG* = 64 # Field is zerofill - BINARY_FLAG* = 128 # Field is binary - # The following are only sent to new clients - ENUM_FLAG* = 256 # field is an enum - AUTO_INCREMENT_FLAG* = 512 # field is a autoincrement field - TIMESTAMP_FLAG* = 1024 # Field is a timestamp - SET_FLAG* = 2048 # field is a set - NO_DEFAULT_VALUE_FLAG* = 4096 # Field doesn't have default value - NUM_FLAG* = 32768 # Field is num (for clients) - PART_KEY_FLAG* = 16384 # Intern; Part of some key - GROUP_FLAG* = 32768 # Intern: Group field - UNIQUE_FLAG* = 65536 # Intern: Used by sql_yacc - BINCMP_FLAG* = 131072 # Intern: Used by sql_yacc - REFRESH_GRANT* = 1 # Refresh grant tables - REFRESH_LOG* = 2 # Start on new log file - REFRESH_TABLES* = 4 # close all tables - REFRESH_HOSTS* = 8 # Flush host cache - REFRESH_STATUS* = 16 # Flush status variables - REFRESH_THREADS* = 32 # Flush thread cache - REFRESH_SLAVE* = 64 # Reset master info and restart slave thread - REFRESH_MASTER* = 128 # Remove all bin logs in the index and truncate the index - # The following can't be set with mysql_refresh() - REFRESH_READ_LOCK* = 16384 # Lock tables for read - REFRESH_FAST* = 32768 # Intern flag - REFRESH_QUERY_CACHE* = 65536 # RESET (remove all queries) from query cache - REFRESH_QUERY_CACHE_FREE* = 0x00020000 # pack query cache - REFRESH_DES_KEY_FILE* = 0x00040000 - REFRESH_USER_RESOURCES* = 0x00080000 - CLIENT_LONG_PASSWORD* = 1 # new more secure passwords - CLIENT_FOUND_ROWS* = 2 # Found instead of affected rows - CLIENT_LONG_FLAG* = 4 # Get all column flags - CLIENT_CONNECT_WITH_DB* = 8 # One can specify db on connect - CLIENT_NO_SCHEMA* = 16 # Don't allow database.table.column - CLIENT_COMPRESS* = 32 # Can use compression protocol - CLIENT_ODBC* = 64 # Odbc client - CLIENT_LOCAL_FILES* = 128 # Can use LOAD DATA LOCAL - CLIENT_IGNORE_SPACE* = 256 # Ignore spaces before '(' - CLIENT_PROTOCOL_41* = 512 # New 4.1 protocol - CLIENT_INTERACTIVE* = 1024 # This is an interactive client - CLIENT_SSL* = 2048 # Switch to SSL after handshake - CLIENT_IGNORE_SIGPIPE* = 4096 # IGNORE sigpipes - CLIENT_TRANSACTIONS* = 8192 # Client knows about transactions - CLIENT_RESERVED* = 16384 # Old flag for 4.1 protocol - CLIENT_SECURE_CONNECTION* = 32768 # New 4.1 authentication - CLIENT_MULTI_STATEMENTS* = 65536 # Enable/disable multi-stmt support - CLIENT_MULTI_RESULTS* = 131072 # Enable/disable multi-results - CLIENT_REMEMBER_OPTIONS*: int = 1 shl 31 - SERVER_STATUS_IN_TRANS* = 1 # Transaction has started - SERVER_STATUS_AUTOCOMMIT* = 2 # Server in auto_commit mode - SERVER_STATUS_MORE_RESULTS* = 4 # More results on server - SERVER_MORE_RESULTS_EXISTS* = 8 # Multi query - next query exists - SERVER_QUERY_NO_GOOD_INDEX_USED* = 16 - SERVER_QUERY_NO_INDEX_USED* = 32 # The server was able to fulfill the clients request and opened a - # read-only non-scrollable cursor for a query. This flag comes - # in reply to COM_STMT_EXECUTE and COM_STMT_FETCH commands. - SERVER_STATUS_CURSOR_EXISTS* = 64 # This flag is sent when a read-only cursor is exhausted, in reply to - # COM_STMT_FETCH command. - SERVER_STATUS_LAST_ROW_SENT* = 128 - SERVER_STATUS_DB_DROPPED* = 256 # A database was dropped - SERVER_STATUS_NO_BACKSLASH_ESCAPES* = 512 - ERRMSG_SIZE* = 200 - NET_READ_TIMEOUT* = 30 # Timeout on read - NET_WRITE_TIMEOUT* = 60 # Timeout on write - NET_WAIT_TIMEOUT* = 8 * 60 * 60 # Wait for new query - ONLY_KILL_QUERY* = 1 - -const - MAX_TINYINT_WIDTH* = 3 # Max width for a TINY w.o. sign - MAX_SMALLINT_WIDTH* = 5 # Max width for a SHORT w.o. sign - MAX_MEDIUMINT_WIDTH* = 8 # Max width for a INT24 w.o. sign - MAX_INT_WIDTH* = 10 # Max width for a LONG w.o. sign - MAX_BIGINT_WIDTH* = 20 # Max width for a LONGLONG - MAX_CHAR_WIDTH* = 255 # Max length for a CHAR column - MAX_BLOB_WIDTH* = 8192 # Default width for blob - -type - Pst_net* = ptr St_net - St_net*{.final.} = object - vio*: PVIO - buff*: cstring - buff_end*: cstring - write_pos*: cstring - read_pos*: cstring - fd*: my_socket # For Perl DBI/dbd - max_packet*: int - max_packet_size*: int - pkt_nr*: cuint - compress_pkt_nr*: cuint - write_timeout*: cuint - read_timeout*: cuint - retry_count*: cuint - fcntl*: cint - compress*: my_bool # The following variable is set if we are doing several queries in one - # command ( as in LOAD TABLE ... FROM MASTER ), - # and do not want to confuse the client with OK at the wrong time - remain_in_buf*: int - len*: int - buf_length*: int - where_b*: int - return_status*: ptr cint - reading_or_writing*: char - save_char*: cchar - no_send_ok*: my_bool # For SPs and other things that do multiple stmts - no_send_eof*: my_bool # For SPs' first version read-only cursors - no_send_error*: my_bool # Set if OK packet is already sent, and - # we do not need to send error messages - # Pointer to query object in query cache, do not equal NULL (0) for - # queries in cache that have not stored its results yet - # $endif - last_error*: array[0..(ERRMSG_SIZE) - 1, char] - sqlstate*: array[0..(SQLSTATE_LENGTH + 1) - 1, char] - last_errno*: cuint - error*: char - query_cache_query*: gptr - report_error*: my_bool # We should report error (we have unreported error) - return_errno*: my_bool - - NET* = St_net - PNET* = ptr NET -{.deprecated: [Tst_net: St_net, TNET: NET].} - -const - packet_error* = - 1 - -type - Enum_field_types* = enum # For backward compatibility - TYPE_DECIMAL, TYPE_TINY, TYPE_SHORT, TYPE_LONG, TYPE_FLOAT, TYPE_DOUBLE, - TYPE_NULL, TYPE_TIMESTAMP, TYPE_LONGLONG, TYPE_INT24, TYPE_DATE, TYPE_TIME, - TYPE_DATETIME, TYPE_YEAR, TYPE_NEWDATE, TYPE_VARCHAR, TYPE_BIT, - TYPE_NEWDECIMAL = 246, TYPE_ENUM = 247, TYPE_SET = 248, - TYPE_TINY_BLOB = 249, TYPE_MEDIUM_BLOB = 250, TYPE_LONG_BLOB = 251, - TYPE_BLOB = 252, TYPE_VAR_STRING = 253, TYPE_STRING = 254, - TYPE_GEOMETRY = 255 -{.deprecated: [Tenum_field_types: Enum_field_types].} - -const - CLIENT_MULTI_QUERIES* = CLIENT_MULTI_STATEMENTS - FIELD_TYPE_DECIMAL* = TYPE_DECIMAL - FIELD_TYPE_NEWDECIMAL* = TYPE_NEWDECIMAL - FIELD_TYPE_TINY* = TYPE_TINY - FIELD_TYPE_SHORT* = TYPE_SHORT - FIELD_TYPE_LONG* = TYPE_LONG - FIELD_TYPE_FLOAT* = TYPE_FLOAT - FIELD_TYPE_DOUBLE* = TYPE_DOUBLE - FIELD_TYPE_NULL* = TYPE_NULL - FIELD_TYPE_TIMESTAMP* = TYPE_TIMESTAMP - FIELD_TYPE_LONGLONG* = TYPE_LONGLONG - FIELD_TYPE_INT24* = TYPE_INT24 - FIELD_TYPE_DATE* = TYPE_DATE - FIELD_TYPE_TIME* = TYPE_TIME - FIELD_TYPE_DATETIME* = TYPE_DATETIME - FIELD_TYPE_YEAR* = TYPE_YEAR - FIELD_TYPE_NEWDATE* = TYPE_NEWDATE - FIELD_TYPE_ENUM* = TYPE_ENUM - FIELD_TYPE_SET* = TYPE_SET - FIELD_TYPE_TINY_BLOB* = TYPE_TINY_BLOB - FIELD_TYPE_MEDIUM_BLOB* = TYPE_MEDIUM_BLOB - FIELD_TYPE_LONG_BLOB* = TYPE_LONG_BLOB - FIELD_TYPE_BLOB* = TYPE_BLOB - FIELD_TYPE_VAR_STRING* = TYPE_VAR_STRING - FIELD_TYPE_STRING* = TYPE_STRING - FIELD_TYPE_CHAR* = TYPE_TINY - FIELD_TYPE_INTERVAL* = TYPE_ENUM - FIELD_TYPE_GEOMETRY* = TYPE_GEOMETRY - FIELD_TYPE_BIT* = TYPE_BIT # Shutdown/kill enums and constants - # Bits for THD::killable. - SHUTDOWN_KILLABLE_CONNECT* = chr(1 shl 0) - SHUTDOWN_KILLABLE_TRANS* = chr(1 shl 1) - SHUTDOWN_KILLABLE_LOCK_TABLE* = chr(1 shl 2) - SHUTDOWN_KILLABLE_UPDATE* = chr(1 shl 3) - -type - Enum_shutdown_level* = enum - SHUTDOWN_DEFAULT = 0, SHUTDOWN_WAIT_CONNECTIONS = 1, - SHUTDOWN_WAIT_TRANSACTIONS = 2, SHUTDOWN_WAIT_UPDATES = 8, - SHUTDOWN_WAIT_ALL_BUFFERS = 16, SHUTDOWN_WAIT_CRITICAL_BUFFERS = 17, - KILL_QUERY = 254, KILL_CONNECTION = 255 - Enum_cursor_type* = enum # options for mysql_set_option - CURSOR_TYPE_NO_CURSOR = 0, CURSOR_TYPE_READ_ONLY = 1, - CURSOR_TYPE_FOR_UPDATE = 2, CURSOR_TYPE_SCROLLABLE = 4 - Enum_mysql_set_option* = enum - OPTION_MULTI_STATEMENTS_ON, OPTION_MULTI_STATEMENTS_OFF -{.deprecated: [Tenum_shutdown_level: Enum_shutdown_level, - Tenum_cursor_type: Enum_cursor_type, - Tenum_mysql_set_option: Enum_mysql_set_option].} - -proc my_net_init*(net: PNET, vio: PVIO): my_bool{.cdecl, dynlib: lib, - importc: "my_net_init".} -proc my_net_local_init*(net: PNET){.cdecl, dynlib: lib, - importc: "my_net_local_init".} -proc net_end*(net: PNET){.cdecl, dynlib: lib, importc: "net_end".} -proc net_clear*(net: PNET){.cdecl, dynlib: lib, importc: "net_clear".} -proc net_realloc*(net: PNET, len: int): my_bool{.cdecl, dynlib: lib, - importc: "net_realloc".} -proc net_flush*(net: PNET): my_bool{.cdecl, dynlib: lib, importc: "net_flush".} -proc my_net_write*(net: PNET, packet: cstring, length: int): my_bool{.cdecl, - dynlib: lib, importc: "my_net_write".} -proc net_write_command*(net: PNET, command: char, header: cstring, - head_len: int, packet: cstring, length: int): my_bool{. - cdecl, dynlib: lib, importc: "net_write_command".} -proc net_real_write*(net: PNET, packet: cstring, length: int): cint{.cdecl, - dynlib: lib, importc: "net_real_write".} -proc my_net_read*(net: PNET): int{.cdecl, dynlib: lib, importc: "my_net_read".} - # The following function is not meant for normal usage - # Currently it's used internally by manager.c -type - Psockaddr* = ptr Sockaddr - Sockaddr*{.final.} = object # undefined structure -{.deprecated: [Tsockaddr: Sockaddr].} - -proc my_connect*(s: my_socket, name: Psockaddr, namelen: cuint, timeout: cuint): cint{. - cdecl, dynlib: lib, importc: "my_connect".} -type - Prand_struct* = ptr Rand_struct - Rand_struct*{.final.} = object # The following is for user defined functions - seed1*: int - seed2*: int - max_value*: int - max_value_dbl*: cdouble - - Item_result* = enum - STRING_RESULT, REAL_RESULT, INT_RESULT, ROW_RESULT, DECIMAL_RESULT - PItem_result* = ptr Item_result - Pst_udf_args* = ptr St_udf_args - St_udf_args*{.final.} = object - arg_count*: cuint # Number of arguments - arg_type*: PItem_result # Pointer to item_results - args*: cstringArray # Pointer to item_results - lengths*: ptr int # Length of string arguments - maybe_null*: cstring # Length of string arguments - attributes*: cstringArray # Pointer to attribute name - attribute_lengths*: ptr int # Length of attribute arguments - - UDF_ARGS* = St_udf_args - PUDF_ARGS* = ptr UDF_ARGS # This holds information about the result - Pst_udf_init* = ptr St_udf_init - St_udf_init*{.final.} = object - maybe_null*: my_bool # 1 if function can return NULL - decimals*: cuint # for real functions - max_length*: int # For string functions - theptr*: cstring # free pointer for function data - const_item*: my_bool # free pointer for function data - - UDF_INIT* = St_udf_init - PUDF_INIT* = ptr UDF_INIT # Constants when using compression -{.deprecated: [Trand_stuct: Rand_struct, TItem_result: Item_result, - Tst_udf_args: St_udf_args, TUDF_ARGS: UDF_ARGS, - Tst_udf_init: St_udf_init, TUDF_INIT: UDF_INIT].} - -const - NET_HEADER_SIZE* = 4 # standard header size - COMP_HEADER_SIZE* = 3 # compression header extra size - # Prototypes to password functions - # These functions are used for authentication by client and server and - # implemented in sql/password.c - -proc randominit*(para1: Prand_struct, seed1: int, seed2: int){.cdecl, - dynlib: lib, importc: "randominit".} -proc my_rnd*(para1: Prand_struct): cdouble{.cdecl, dynlib: lib, - importc: "my_rnd".} -proc create_random_string*(fto: cstring, len: cuint, rand_st: Prand_struct){. - cdecl, dynlib: lib, importc: "create_random_string".} -proc hash_password*(fto: int, password: cstring, password_len: cuint){.cdecl, - dynlib: lib, importc: "hash_password".} -proc make_scrambled_password_323*(fto: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "make_scrambled_password_323".} -proc scramble_323*(fto: cstring, message: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "scramble_323".} -proc check_scramble_323*(para1: cstring, message: cstring, salt: int): my_bool{. - cdecl, dynlib: lib, importc: "check_scramble_323".} -proc get_salt_from_password_323*(res: ptr int, password: cstring){.cdecl, - dynlib: lib, importc: "get_salt_from_password_323".} -proc make_password_from_salt_323*(fto: cstring, salt: ptr int){.cdecl, - dynlib: lib, importc: "make_password_from_salt_323".} -proc octet2hex*(fto: cstring, str: cstring, length: cuint): cstring{.cdecl, - dynlib: lib, importc: "octet2hex".} -proc make_scrambled_password*(fto: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "make_scrambled_password".} -proc scramble*(fto: cstring, message: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "scramble".} -proc check_scramble*(reply: cstring, message: cstring, hash_stage2: pointer): my_bool{. - cdecl, dynlib: lib, importc: "check_scramble".} -proc get_salt_from_password*(res: pointer, password: cstring){.cdecl, - dynlib: lib, importc: "get_salt_from_password".} -proc make_password_from_salt*(fto: cstring, hash_stage2: pointer){.cdecl, - dynlib: lib, importc: "make_password_from_salt".} - # end of password.c -proc get_tty_password*(opt_message: cstring): cstring{.cdecl, dynlib: lib, - importc: "get_tty_password".} -proc errno_to_sqlstate*(errno: cuint): cstring{.cdecl, dynlib: lib, - importc: "mysql_errno_to_sqlstate".} - # Some other useful functions -proc modify_defaults_file*(file_location: cstring, option: cstring, - option_value: cstring, section_name: cstring, - remove_option: cint): cint{.cdecl, dynlib: lib, - importc: "load_defaults".} -proc load_defaults*(conf_file: cstring, groups: cstringArray, argc: ptr cint, - argv: ptr cstringArray): cint{.cdecl, dynlib: lib, - importc: "load_defaults".} -proc my_init*(): my_bool{.cdecl, dynlib: lib, importc: "my_init".} -proc my_thread_init*(): my_bool{.cdecl, dynlib: lib, importc: "my_thread_init".} -proc my_thread_end*(){.cdecl, dynlib: lib, importc: "my_thread_end".} -const - NULL_LENGTH*: int = int(not (0)) # For net_store_length - -const - STMT_HEADER* = 4 - LONG_DATA_HEADER* = 6 # ------------ Stop of declaration in "mysql_com.h" ----------------------- - # $include "mysql_time.h" - # $include "mysql_version.h" - # $include "typelib.h" - # $include "my_list.h" /* for LISTs used in 'MYSQL' and 'MYSQL_STMT' */ - # var - # mysql_port : cuint;cvar;external; - # mysql_unix_port : Pchar;cvar;external; - -const - CLIENT_NET_READ_TIMEOUT* = 365 * 24 * 3600 # Timeout on read - CLIENT_NET_WRITE_TIMEOUT* = 365 * 24 * 3600 # Timeout on write - -type - Pst_mysql_field* = ptr St_mysql_field - St_mysql_field*{.final.} = object - name*: cstring # Name of column - org_name*: cstring # Original column name, if an alias - table*: cstring # Table of column if column was a field - org_table*: cstring # Org table name, if table was an alias - db*: cstring # Database for table - catalog*: cstring # Catalog for table - def*: cstring # Default value (set by mysql_list_fields) - len*: int # Width of column (create length) - max_length*: int # Max width for selected set - name_length*: cuint - org_name_length*: cuint - table_length*: cuint - org_table_length*: cuint - db_length*: cuint - catalog_length*: cuint - def_length*: cuint - flags*: cuint # Div flags - decimals*: cuint # Number of decimals in field - charsetnr*: cuint # Character set - ftype*: Enum_field_types # Type of field. See mysql_com.h for types - extension*: pointer - - FIELD* = St_mysql_field - PFIELD* = ptr FIELD - PROW* = ptr ROW # return data as array of strings - ROW* = cstringArray - PFIELD_OFFSET* = ptr FIELD_OFFSET # offset to current field - FIELD_OFFSET* = cuint -{.deprecated: [Tst_mysql_field: St_mysql_field, TFIELD: FIELD, TROW: ROW, - TFIELD_OFFSET: FIELD_OFFSET].} - -proc IS_PRI_KEY*(n: int32): bool -proc IS_NOT_NULL*(n: int32): bool -proc IS_BLOB*(n: int32): bool -proc IS_NUM*(t: Enum_field_types): bool -proc INTERNAL_NUM_FIELD*(f: Pst_mysql_field): bool -proc IS_NUM_FIELD*(f: Pst_mysql_field): bool -type - my_ulonglong* = int64 - Pmy_ulonglong* = ptr my_ulonglong - -const - COUNT_ERROR* = not (my_ulonglong(0)) - -type - Pst_mysql_rows* = ptr St_mysql_rows - St_mysql_rows*{.final.} = object - next*: Pst_mysql_rows # list of rows - data*: ROW - len*: int - - ROWS* = St_mysql_rows - PROWS* = ptr ROWS - PROW_OFFSET* = ptr ROW_OFFSET # offset to current row - ROW_OFFSET* = ROWS -{.deprecated: [Tst_mysql_rows: St_mysql_rows, TROWS: ROWS, - TROW_OFFSET: ROW_OFFSET].} - -const - ALLOC_MAX_BLOCK_TO_DROP* = 4096 - ALLOC_MAX_BLOCK_USAGE_BEFORE_DROP* = 10 # struct for once_alloc (block) - -type - Pst_used_mem* = ptr St_used_mem - St_used_mem*{.final.} = object - next*: Pst_used_mem # Next block in use - left*: cuint # memory left in block - size*: cuint # size of block - - USED_MEM* = St_used_mem - PUSED_MEM* = ptr USED_MEM - Pst_mem_root* = ptr St_mem_root - St_mem_root*{.final.} = object - free*: PUSED_MEM # blocks with free memory in it - used*: PUSED_MEM # blocks almost without free memory - pre_alloc*: PUSED_MEM # preallocated block - min_malloc*: cuint # if block have less memory it will be put in 'used' list - block_size*: cuint # initial block size - block_num*: cuint # allocated blocks counter - # first free block in queue test counter (if it exceed - # MAX_BLOCK_USAGE_BEFORE_DROP block will be dropped in 'used' list) - first_block_usage*: cuint - error_handler*: proc (){.cdecl.} - - MEM_ROOT* = St_mem_root - PMEM_ROOT* = ptr MEM_ROOT # ------------ Stop of declaration in "my_alloc.h" ---------------------- -{.deprecated: [Tst_used_mem: St_used_mem, TUSED_MEM: USED_MEM, - Tst_mem_root: St_mem_root, TMEM_ROOT: MEM_ROOT].} - -type - Pst_mysql_data* = ptr St_mysql_data - St_mysql_data*{.final.} = object - rows*: my_ulonglong - fields*: cuint - data*: PROWS - alloc*: MEM_ROOT - prev_ptr*: ptr PROWS - - DATA* = St_mysql_data - PDATA* = ptr DATA - Option* = enum - OPT_CONNECT_TIMEOUT, OPT_COMPRESS, OPT_NAMED_PIPE, INIT_COMMAND, - READ_DEFAULT_FILE, READ_DEFAULT_GROUP, SET_CHARSET_DIR, SET_CHARSET_NAME, - OPT_LOCAL_INFILE, OPT_PROTOCOL, SHARED_MEMORY_BASE_NAME, OPT_READ_TIMEOUT, - OPT_WRITE_TIMEOUT, OPT_USE_RESULT, OPT_USE_REMOTE_CONNECTION, - OPT_USE_EMBEDDED_CONNECTION, OPT_GUESS_CONNECTION, SET_CLIENT_IP, - SECURE_AUTH, REPORT_DATA_TRUNCATION, OPT_RECONNECT -{.deprecated: [Tst_mysql_data: St_mysql_data, TDATA: DATA, Toption: Option].} - -const - MAX_MYSQL_MANAGER_ERR* = 256 - MAX_MYSQL_MANAGER_MSG* = 256 - MANAGER_OK* = 200 - MANAGER_INFO* = 250 - MANAGER_ACCESS* = 401 - MANAGER_CLIENT_ERR* = 450 - MANAGER_INTERNAL_ERR* = 500 - -type - St_dynamic_array*{.final.} = object - buffer*: cstring - elements*: cuint - max_element*: cuint - alloc_increment*: cuint - size_of_element*: cuint - - DYNAMIC_ARRAY* = St_dynamic_array - Pst_dynamic_array* = ptr St_dynamic_array - Pst_mysql_options* = ptr St_mysql_options - St_mysql_options*{.final.} = object - connect_timeout*: cuint - read_timeout*: cuint - write_timeout*: cuint - port*: cuint - protocol*: cuint - client_flag*: int - host*: cstring - user*: cstring - password*: cstring - unix_socket*: cstring - db*: cstring - init_commands*: Pst_dynamic_array - my_cnf_file*: cstring - my_cnf_group*: cstring - charset_dir*: cstring - charset_name*: cstring - ssl_key*: cstring # PEM key file - ssl_cert*: cstring # PEM cert file - ssl_ca*: cstring # PEM CA file - ssl_capath*: cstring # PEM directory of CA-s? - ssl_cipher*: cstring # cipher to use - shared_memory_base_name*: cstring - max_allowed_packet*: int - use_ssl*: my_bool # if to use SSL or not - compress*: my_bool - named_pipe*: my_bool # On connect, find out the replication role of the server, and - # establish connections to all the peers - rpl_probe*: my_bool # Each call to mysql_real_query() will parse it to tell if it is a read - # or a write, and direct it to the slave or the master - rpl_parse*: my_bool # If set, never read from a master, only from slave, when doing - # a read that is replication-aware - no_master_reads*: my_bool - separate_thread*: my_bool - methods_to_use*: Option - client_ip*: cstring - secure_auth*: my_bool # Refuse client connecting to server if it uses old (pre-4.1.1) protocol - report_data_truncation*: my_bool # 0 - never report, 1 - always report (default) - # function pointers for local infile support - local_infile_init*: proc (para1: var pointer, para2: cstring, para3: pointer): cint{. - cdecl.} - local_infile_read*: proc (para1: pointer, para2: cstring, para3: cuint): cint - local_infile_end*: proc (para1: pointer) - local_infile_error*: proc (para1: pointer, para2: cstring, para3: cuint): cint - local_infile_userdata*: pointer - - Status* = enum - STATUS_READY, STATUS_GET_RESULT, STATUS_USE_RESULT - Protocol_type* = enum # There are three types of queries - the ones that have to go to - # the master, the ones that go to a slave, and the administrative - # type which must happen on the pivot connection - PROTOCOL_DEFAULT, PROTOCOL_TCP, PROTOCOL_SOCKET, PROTOCOL_PIPE, - PROTOCOL_MEMORY - Rpl_type* = enum - RPL_MASTER, RPL_SLAVE, RPL_ADMIN - Charset_info_st*{.final.} = object - number*: cuint - primary_number*: cuint - binary_number*: cuint - state*: cuint - csname*: cstring - name*: cstring - comment*: cstring - tailoring*: cstring - ftype*: cstring - to_lower*: cstring - to_upper*: cstring - sort_order*: cstring - contractions*: ptr int16 - sort_order_big*: ptr ptr int16 - tab_to_uni*: ptr int16 - tab_from_uni*: pointer # was ^MY_UNI_IDX - state_map*: cstring - ident_map*: cstring - strxfrm_multiply*: cuint - mbminlen*: cuint - mbmaxlen*: cuint - min_sort_char*: int16 - max_sort_char*: int16 - escape_with_backslash_is_dangerous*: my_bool - cset*: pointer # was ^MY_CHARSET_HANDLER - coll*: pointer # was ^MY_COLLATION_HANDLER; - - CHARSET_INFO* = Charset_info_st - Pcharset_info_st* = ptr Charset_info_st - Pcharacter_set* = ptr Character_set - Character_set*{.final.} = object - number*: cuint - state*: cuint - csname*: cstring - name*: cstring - comment*: cstring - dir*: cstring - mbminlen*: cuint - mbmaxlen*: cuint - - MY_CHARSET_INFO* = Character_set - PMY_CHARSET_INFO* = ptr MY_CHARSET_INFO - Pst_mysql_methods* = ptr St_mysql_methods - Pst_mysql* = ptr St_mysql - St_mysql*{.final.} = object - net*: NET # Communication parameters - connector_fd*: gptr # ConnectorFd for SSL - host*: cstring - user*: cstring - passwd*: cstring - unix_socket*: cstring - server_version*: cstring - host_info*: cstring - info*: cstring - db*: cstring - charset*: Pcharset_info_st - fields*: PFIELD - field_alloc*: MEM_ROOT - affected_rows*: my_ulonglong - insert_id*: my_ulonglong # id if insert on table with NEXTNR - extra_info*: my_ulonglong # Used by mysqlshow, not used by mysql 5.0 and up - thread_id*: int # Id for connection in server - packet_length*: int - port*: cuint - client_flag*: int - server_capabilities*: int - protocol_version*: cuint - field_count*: cuint - server_status*: cuint - server_language*: cuint - warning_count*: cuint - options*: St_mysql_options - status*: Status - free_me*: my_bool # If free in mysql_close - reconnect*: my_bool # set to 1 if automatic reconnect - scramble*: array[0..(SCRAMBLE_LENGTH + 1) - 1, char] # session-wide random string - # Set if this is the original connection, not a master or a slave we have - # added though mysql_rpl_probe() or mysql_set_master()/ mysql_add_slave() - rpl_pivot*: my_bool # Pointers to the master, and the next slave connections, points to - # itself if lone connection. - master*: Pst_mysql - next_slave*: Pst_mysql - last_used_slave*: Pst_mysql # needed for round-robin slave pick - last_used_con*: Pst_mysql # needed for send/read/store/use result to work correctly with replication - stmts*: pointer # was PList, list of all statements - methods*: Pst_mysql_methods - thd*: pointer # Points to boolean flag in MYSQL_RES or MYSQL_STMT. We set this flag - # from mysql_stmt_close if close had to cancel result set of this object. - unbuffered_fetch_owner*: Pmy_bool - - MySQL* = St_mysql - PMySQL* = ptr MySQL - Pst_mysql_res* = ptr St_mysql_res - St_mysql_res*{.final.} = object - row_count*: my_ulonglong - fields*: PFIELD - data*: PDATA - data_cursor*: PROWS - lengths*: ptr int # column lengths of current row - handle*: PMySQL # for unbuffered reads - field_alloc*: MEM_ROOT - field_count*: cuint - current_field*: cuint - row*: ROW # If unbuffered read - current_row*: ROW # buffer to current row - eof*: my_bool # Used by mysql_fetch_row - unbuffered_fetch_cancelled*: my_bool # mysql_stmt_close() had to cancel this result - methods*: Pst_mysql_methods - - RES* = St_mysql_res - PRES* = ptr RES - Pst_mysql_stmt* = ptr St_mysql_stmt - PSTMT* = ptr STMT - St_mysql_methods*{.final.} = object - read_query_result*: proc (MySQL: PMySQL): my_bool{.cdecl.} - advanced_command*: proc (MySQL: PMySQL, command: Enum_server_command, header: cstring, - header_length: int, arg: cstring, arg_length: int, - skip_check: my_bool): my_bool - read_rows*: proc (MySQL: PMySQL, fields: PFIELD, fields_count: cuint): PDATA - use_result*: proc (MySQL: PMySQL): PRES - fetch_lengths*: proc (fto: ptr int, column: ROW, field_count: cuint) - flush_use_result*: proc (MySQL: PMySQL) - list_fields*: proc (MySQL: PMySQL): PFIELD - read_prepare_result*: proc (MySQL: PMySQL, stmt: PSTMT): my_bool - stmt_execute*: proc (stmt: PSTMT): cint - read_binary_rows*: proc (stmt: PSTMT): cint - unbuffered_fetch*: proc (MySQL: PMySQL, row: cstringArray): cint - free_embedded_thd*: proc (MySQL: PMySQL) - read_statistics*: proc (MySQL: PMySQL): cstring - next_result*: proc (MySQL: PMySQL): my_bool - read_change_user_result*: proc (MySQL: PMySQL, buff: cstring, passwd: cstring): cint - read_rowsfrom_cursor*: proc (stmt: PSTMT): cint - - METHODS* = St_mysql_methods - PMETHODS* = ptr METHODS - Pst_mysql_manager* = ptr St_mysql_manager - St_mysql_manager*{.final.} = object - net*: NET - host*: cstring - user*: cstring - passwd*: cstring - port*: cuint - free_me*: my_bool - eof*: my_bool - cmd_status*: cint - last_errno*: cint - net_buf*: cstring - net_buf_pos*: cstring - net_data_end*: cstring - net_buf_size*: cint - last_error*: array[0..(MAX_MYSQL_MANAGER_ERR) - 1, char] - - MANAGER* = St_mysql_manager - PMANAGER* = ptr MANAGER - Pst_mysql_parameters* = ptr St_mysql_parameters - St_mysql_parameters*{.final.} = object - p_max_allowed_packet*: ptr int - p_net_buffer_length*: ptr int - - PARAMETERS* = St_mysql_parameters - PPARAMETERS* = ptr PARAMETERS - Enum_mysql_stmt_state* = enum - STMT_INIT_DONE = 1, STMT_PREPARE_DONE, STMT_EXECUTE_DONE, STMT_FETCH_DONE - Pst_mysql_bind* = ptr St_mysql_bind - St_mysql_bind*{.final.} = object - len*: int # output length pointer - is_null*: Pmy_bool # Pointer to null indicator - buffer*: pointer # buffer to get/put data - error*: Pmy_bool # set this if you want to track data truncations happened during fetch - buffer_type*: Enum_field_types # buffer type - buffer_length*: int # buffer length, must be set for str/binary - # Following are for internal use. Set by mysql_stmt_bind_param - row_ptr*: ptr byte # for the current data position - offset*: int # offset position for char/binary fetch - length_value*: int # Used if length is 0 - param_number*: cuint # For null count and error messages - pack_length*: cuint # Internal length for packed data - error_value*: my_bool # used if error is 0 - is_unsigned*: my_bool # set if integer type is unsigned - long_data_used*: my_bool # If used with mysql_send_long_data - is_null_value*: my_bool # Used if is_null is 0 - store_param_func*: proc (net: PNET, param: Pst_mysql_bind){.cdecl.} - fetch_result*: proc (para1: Pst_mysql_bind, para2: PFIELD, row: PPByte) - skip_result*: proc (para1: Pst_mysql_bind, para2: PFIELD, row: PPByte) - - BIND* = St_mysql_bind - PBIND* = ptr BIND # statement handler - St_mysql_stmt*{.final.} = object - mem_root*: MEM_ROOT # root allocations - mysql*: PMySQL # connection handle - params*: PBIND # input parameters - `bind`*: PBIND # input parameters - fields*: PFIELD # result set metadata - result*: DATA # cached result set - data_cursor*: PROWS # current row in cached result - affected_rows*: my_ulonglong # copy of mysql->affected_rows after statement execution - insert_id*: my_ulonglong - read_row_func*: proc (stmt: Pst_mysql_stmt, row: PPByte): cint{.cdecl.} - stmt_id*: int # Id for prepared statement - flags*: int # i.e. type of cursor to open - prefetch_rows*: int # number of rows per one COM_FETCH - server_status*: cuint # Copied from mysql->server_status after execute/fetch to know - # server-side cursor status for this statement. - last_errno*: cuint # error code - param_count*: cuint # input parameter count - field_count*: cuint # number of columns in result set - state*: Enum_mysql_stmt_state # statement state - last_error*: array[0..(ERRMSG_SIZE) - 1, char] # error message - sqlstate*: array[0..(SQLSTATE_LENGTH + 1) - 1, char] - send_types_to_server*: my_bool # Types of input parameters should be sent to server - bind_param_done*: my_bool # input buffers were supplied - bind_result_done*: char # output buffers were supplied - unbuffered_fetch_cancelled*: my_bool - update_max_length*: my_bool - - STMT* = St_mysql_stmt - - Enum_stmt_attr_type* = enum - STMT_ATTR_UPDATE_MAX_LENGTH, STMT_ATTR_CURSOR_TYPE, STMT_ATTR_PREFETCH_ROWS -{.deprecated: [Tst_dynamic_array: St_dynamic_array, Tst_mysql_options: St_mysql_options, - TDYNAMIC_ARRAY: DYNAMIC_ARRAY, Tprotocol_type: Protocol_type, - Trpl_type: Rpl_type, Tcharset_info_st: Charset_info_st, - TCHARSET_INFO: CHARSET_INFO, Tcharacter_set: Character_set, - TMY_CHARSET_INFO: MY_CHARSET_INFO, Tst_mysql: St_mysql, - Tst_mysql_methods: St_mysql_methods, TMySql: MySql, - Tst_mysql_res: St_mysql_res, TMETHODS: METHODS, TRES: RES, - Tst_mysql_manager: St_mysql_manager, TMANAGER: MANAGER, - Tst_mysql_parameters: St_mysql_parameters, TPARAMETERS: PARAMETERS, - Tenum_mysql_stmt_state: Enum_mysql_stmt_state, - Tst_mysql_bind: St_mysql_bind, TBIND: BIND, Tst_mysql_stmt: St_mysql_stmt, - TSTMT: STMT, Tenum_stmt_attr_type: Enum_stmt_attr_type, - Tstatus: Status].} - -proc server_init*(argc: cint, argv: cstringArray, groups: cstringArray): cint{. - cdecl, dynlib: lib, importc: "mysql_server_init".} -proc server_end*(){.cdecl, dynlib: lib, importc: "mysql_server_end".} - # mysql_server_init/end need to be called when using libmysqld or - # libmysqlclient (exactly, mysql_server_init() is called by mysql_init() so - # you don't need to call it explicitly; but you need to call - # mysql_server_end() to free memory). The names are a bit misleading - # (mysql_SERVER* to be used when using libmysqlCLIENT). So we add more general - # names which suit well whether you're using libmysqld or libmysqlclient. We - # intend to promote these aliases over the mysql_server* ones. -proc library_init*(argc: cint, argv: cstringArray, groups: cstringArray): cint{. - cdecl, dynlib: lib, importc: "mysql_server_init".} -proc library_end*(){.cdecl, dynlib: lib, importc: "mysql_server_end".} -proc get_parameters*(): PPARAMETERS{.stdcall, dynlib: lib, - importc: "mysql_get_parameters".} - # Set up and bring down a thread; these function should be called - # for each thread in an application which opens at least one MySQL - # connection. All uses of the connection(s) should be between these - # function calls. -proc thread_init*(): my_bool{.stdcall, dynlib: lib, importc: "mysql_thread_init".} -proc thread_end*(){.stdcall, dynlib: lib, importc: "mysql_thread_end".} - # Functions to get information from the MYSQL and MYSQL_RES structures - # Should definitely be used if one uses shared libraries. -proc num_rows*(res: PRES): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_num_rows".} -proc num_fields*(res: PRES): cuint{.stdcall, dynlib: lib, - importc: "mysql_num_fields".} -proc eof*(res: PRES): my_bool{.stdcall, dynlib: lib, importc: "mysql_eof".} -proc fetch_field_direct*(res: PRES, fieldnr: cuint): PFIELD{.stdcall, - dynlib: lib, importc: "mysql_fetch_field_direct".} -proc fetch_fields*(res: PRES): PFIELD{.stdcall, dynlib: lib, - importc: "mysql_fetch_fields".} -proc row_tell*(res: PRES): ROW_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_row_tell".} -proc field_tell*(res: PRES): FIELD_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_field_tell".} -proc field_count*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_field_count".} -proc affected_rows*(MySQL: PMySQL): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_affected_rows".} -proc insert_id*(MySQL: PMySQL): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_insert_id".} -proc errno*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, importc: "mysql_errno".} -proc error*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_error".} -proc sqlstate*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_sqlstate".} -proc warning_count*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_warning_count".} -proc info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_info".} -proc thread_id*(MySQL: PMySQL): int{.stdcall, dynlib: lib, importc: "mysql_thread_id".} -proc character_set_name*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_character_set_name".} -proc set_character_set*(MySQL: PMySQL, csname: cstring): int32{.stdcall, dynlib: lib, - importc: "mysql_set_character_set".} -proc init*(MySQL: PMySQL): PMySQL{.stdcall, dynlib: lib, importc: "mysql_init".} -proc ssl_set*(MySQL: PMySQL, key: cstring, cert: cstring, ca: cstring, capath: cstring, - cipher: cstring): my_bool{.stdcall, dynlib: lib, - importc: "mysql_ssl_set".} -proc change_user*(MySQL: PMySQL, user: cstring, passwd: cstring, db: cstring): my_bool{. - stdcall, dynlib: lib, importc: "mysql_change_user".} -proc real_connect*(MySQL: PMySQL, host: cstring, user: cstring, passwd: cstring, - db: cstring, port: cuint, unix_socket: cstring, - clientflag: int): PMySQL{.stdcall, dynlib: lib, - importc: "mysql_real_connect".} -proc select_db*(MySQL: PMySQL, db: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_select_db".} -proc query*(MySQL: PMySQL, q: cstring): cint{.stdcall, dynlib: lib, importc: "mysql_query".} -proc send_query*(MySQL: PMySQL, q: cstring, len: int): cint{.stdcall, dynlib: lib, - importc: "mysql_send_query".} -proc real_query*(MySQL: PMySQL, q: cstring, len: int): cint{.stdcall, dynlib: lib, - importc: "mysql_real_query".} -proc store_result*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, - importc: "mysql_store_result".} -proc use_result*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, importc: "mysql_use_result".} - # perform query on master -proc master_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_master_query".} -proc master_send_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, - dynlib: lib, importc: "mysql_master_send_query".} - # perform query on slave -proc slave_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_slave_query".} -proc slave_send_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, - dynlib: lib, importc: "mysql_slave_send_query".} -proc get_character_set_info*(MySQL: PMySQL, charset: PMY_CHARSET_INFO){.stdcall, - dynlib: lib, importc: "mysql_get_character_set_info".} - # local infile support -const - LOCAL_INFILE_ERROR_LEN* = 512 - -# procedure mysql_set_local_infile_handler(mysql:PMYSQL; local_infile_init:function (para1:Ppointer; para2:Pchar; para3:pointer):longint; local_infile_read:function (para1:pointer; para2:Pchar; para3:dword):longint; local_infile_end:procedure (_pa -# para6:pointer);cdecl;external mysqllib name 'mysql_set_local_infile_handler'; - -proc set_local_infile_default*(MySQL: PMySQL){.cdecl, dynlib: lib, - importc: "mysql_set_local_infile_default".} - # enable/disable parsing of all queries to decide if they go on master or - # slave -proc enable_rpl_parse*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_enable_rpl_parse".} -proc disable_rpl_parse*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_disable_rpl_parse".} - # get the value of the parse flag -proc rpl_parse_enabled*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, - importc: "mysql_rpl_parse_enabled".} - # enable/disable reads from master -proc enable_reads_from_master*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_enable_reads_from_master".} -proc disable_reads_from_master*(MySQL: PMySQL){.stdcall, dynlib: lib, importc: "mysql_disable_reads_from_master".} - # get the value of the master read flag -proc reads_from_master_enabled*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_reads_from_master_enabled".} -proc rpl_query_type*(q: cstring, length: cint): Rpl_type{.stdcall, dynlib: lib, - importc: "mysql_rpl_query_type".} - # discover the master and its slaves -proc rpl_probe*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_rpl_probe".} - # set the master, close/free the old one, if it is not a pivot -proc set_master*(MySQL: PMySQL, host: cstring, port: cuint, user: cstring, passwd: cstring): cint{. - stdcall, dynlib: lib, importc: "mysql_set_master".} -proc add_slave*(MySQL: PMySQL, host: cstring, port: cuint, user: cstring, passwd: cstring): cint{. - stdcall, dynlib: lib, importc: "mysql_add_slave".} -proc shutdown*(MySQL: PMySQL, shutdown_level: Enum_shutdown_level): cint{.stdcall, - dynlib: lib, importc: "mysql_shutdown".} -proc dump_debug_info*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, - importc: "mysql_dump_debug_info".} -proc refresh*(sql: PMySQL, refresh_options: cuint): cint{.stdcall, dynlib: lib, - importc: "mysql_refresh".} -proc kill*(MySQL: PMySQL, pid: int): cint{.stdcall, dynlib: lib, importc: "mysql_kill".} -proc set_server_option*(MySQL: PMySQL, option: Enum_mysql_set_option): cint{.stdcall, - dynlib: lib, importc: "mysql_set_server_option".} -proc ping*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, importc: "mysql_ping".} -proc stat*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_stat".} -proc get_server_info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_server_info".} -proc get_client_info*(): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_client_info".} -proc get_client_version*(): int{.stdcall, dynlib: lib, - importc: "mysql_get_client_version".} -proc get_host_info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_host_info".} -proc get_server_version*(MySQL: PMySQL): int{.stdcall, dynlib: lib, - importc: "mysql_get_server_version".} -proc get_proto_info*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_get_proto_info".} -proc list_dbs*(MySQL: PMySQL, wild: cstring): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_dbs".} -proc list_tables*(MySQL: PMySQL, wild: cstring): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_tables".} -proc list_processes*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_processes".} -proc options*(MySQL: PMySQL, option: Option, arg: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_options".} -proc free_result*(result: PRES){.stdcall, dynlib: lib, - importc: "mysql_free_result".} -proc data_seek*(result: PRES, offset: my_ulonglong){.stdcall, dynlib: lib, - importc: "mysql_data_seek".} -proc row_seek*(result: PRES, offset: ROW_OFFSET): ROW_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_row_seek".} -proc field_seek*(result: PRES, offset: FIELD_OFFSET): FIELD_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_field_seek".} -proc fetch_row*(result: PRES): ROW{.stdcall, dynlib: lib, - importc: "mysql_fetch_row".} -proc fetch_lengths*(result: PRES): ptr int{.stdcall, dynlib: lib, - importc: "mysql_fetch_lengths".} -proc fetch_field*(result: PRES): PFIELD{.stdcall, dynlib: lib, - importc: "mysql_fetch_field".} -proc list_fields*(MySQL: PMySQL, table: cstring, wild: cstring): PRES{.stdcall, - dynlib: lib, importc: "mysql_list_fields".} -proc escape_string*(fto: cstring, `from`: cstring, from_length: int): int{. - stdcall, dynlib: lib, importc: "mysql_escape_string".} -proc hex_string*(fto: cstring, `from`: cstring, from_length: int): int{.stdcall, - dynlib: lib, importc: "mysql_hex_string".} -proc real_escape_string*(MySQL: PMySQL, fto: cstring, `from`: cstring, len: int): int{. - stdcall, dynlib: lib, importc: "mysql_real_escape_string".} -proc debug*(debug: cstring){.stdcall, dynlib: lib, importc: "mysql_debug".} - # function mysql_odbc_escape_string(mysql:PMYSQL; fto:Pchar; to_length:dword; from:Pchar; from_length:dword; - # param:pointer; extend_buffer:function (para1:pointer; to:Pchar; length:Pdword):Pchar):Pchar;stdcall;external mysqllib name 'mysql_odbc_escape_string'; -proc myodbc_remove_escape*(MySQL: PMySQL, name: cstring){.stdcall, dynlib: lib, - importc: "myodbc_remove_escape".} -proc thread_safe*(): cuint{.stdcall, dynlib: lib, importc: "mysql_thread_safe".} -proc embedded*(): my_bool{.stdcall, dynlib: lib, importc: "mysql_embedded".} -proc manager_init*(con: PMANAGER): PMANAGER{.stdcall, dynlib: lib, - importc: "mysql_manager_init".} -proc manager_connect*(con: PMANAGER, host: cstring, user: cstring, - passwd: cstring, port: cuint): PMANAGER{.stdcall, - dynlib: lib, importc: "mysql_manager_connect".} -proc manager_close*(con: PMANAGER){.stdcall, dynlib: lib, - importc: "mysql_manager_close".} -proc manager_command*(con: PMANAGER, cmd: cstring, cmd_len: cint): cint{. - stdcall, dynlib: lib, importc: "mysql_manager_command".} -proc manager_fetch_line*(con: PMANAGER, res_buf: cstring, res_buf_size: cint): cint{. - stdcall, dynlib: lib, importc: "mysql_manager_fetch_line".} -proc read_query_result*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_read_query_result".} -proc stmt_init*(MySQL: PMySQL): PSTMT{.stdcall, dynlib: lib, importc: "mysql_stmt_init".} -proc stmt_prepare*(stmt: PSTMT, query: cstring, len: int): cint{.stdcall, - dynlib: lib, importc: "mysql_stmt_prepare".} -proc stmt_execute*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_execute".} -proc stmt_fetch*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_fetch".} -proc stmt_fetch_column*(stmt: PSTMT, `bind`: PBIND, column: cuint, offset: int): cint{. - stdcall, dynlib: lib, importc: "mysql_stmt_fetch_column".} -proc stmt_store_result*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_store_result".} -proc stmt_param_count*(stmt: PSTMT): int{.stdcall, dynlib: lib, - importc: "mysql_stmt_param_count".} -proc stmt_attr_set*(stmt: PSTMT, attr_type: Enum_stmt_attr_type, attr: pointer): my_bool{. - stdcall, dynlib: lib, importc: "mysql_stmt_attr_set".} -proc stmt_attr_get*(stmt: PSTMT, attr_type: Enum_stmt_attr_type, attr: pointer): my_bool{. - stdcall, dynlib: lib, importc: "mysql_stmt_attr_get".} -proc stmt_bind_param*(stmt: PSTMT, bnd: PBIND): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_bind_param".} -proc stmt_bind_result*(stmt: PSTMT, bnd: PBIND): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_bind_result".} -proc stmt_close*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_close".} -proc stmt_reset*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_reset".} -proc stmt_free_result*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_free_result".} -proc stmt_send_long_data*(stmt: PSTMT, param_number: cuint, data: cstring, - len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_send_long_data".} -proc stmt_result_metadata*(stmt: PSTMT): PRES{.stdcall, dynlib: lib, - importc: "mysql_stmt_result_metadata".} -proc stmt_param_metadata*(stmt: PSTMT): PRES{.stdcall, dynlib: lib, - importc: "mysql_stmt_param_metadata".} -proc stmt_errno*(stmt: PSTMT): cuint{.stdcall, dynlib: lib, - importc: "mysql_stmt_errno".} -proc stmt_error*(stmt: PSTMT): cstring{.stdcall, dynlib: lib, - importc: "mysql_stmt_error".} -proc stmt_sqlstate*(stmt: PSTMT): cstring{.stdcall, dynlib: lib, - importc: "mysql_stmt_sqlstate".} -proc stmt_row_seek*(stmt: PSTMT, offset: ROW_OFFSET): ROW_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_stmt_row_seek".} -proc stmt_row_tell*(stmt: PSTMT): ROW_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_stmt_row_tell".} -proc stmt_data_seek*(stmt: PSTMT, offset: my_ulonglong){.stdcall, dynlib: lib, - importc: "mysql_stmt_data_seek".} -proc stmt_num_rows*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_num_rows".} -proc stmt_affected_rows*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_affected_rows".} -proc stmt_insert_id*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_insert_id".} -proc stmt_field_count*(stmt: PSTMT): cuint{.stdcall, dynlib: lib, - importc: "mysql_stmt_field_count".} -proc commit*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_commit".} -proc rollback*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_rollback".} -proc autocommit*(MySQL: PMySQL, auto_mode: my_bool): my_bool{.stdcall, dynlib: lib, - importc: "mysql_autocommit".} -proc more_results*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_more_results".} -proc next_result*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, importc: "mysql_next_result".} -proc close*(sock: PMySQL){.stdcall, dynlib: lib, importc: "mysql_close".} - # status return codes -const - NO_DATA* = 100 - DATA_TRUNCATED* = 101 - -proc reload*(x: PMySQL): cint -when defined(USE_OLD_FUNCTIONS): - proc connect*(MySQL: PMySQL, host: cstring, user: cstring, passwd: cstring): PMySQL{.stdcall, - dynlib: lib, importc: "mysql_connect".} - proc create_db*(MySQL: PMySQL, DB: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_create_db".} - proc drop_db*(MySQL: PMySQL, DB: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_drop_db".} -proc net_safe_read*(MySQL: PMySQL): cuint{.cdecl, dynlib: lib, importc: "net_safe_read".} - -proc IS_PRI_KEY(n: int32): bool = - result = (n and PRI_KEY_FLAG) != 0 - -proc IS_NOT_NULL(n: int32): bool = - result = (n and NOT_NULL_FLAG) != 0 - -proc IS_BLOB(n: int32): bool = - result = (n and BLOB_FLAG) != 0 - -proc IS_NUM_FIELD(f: Pst_mysql_field): bool = - result = (f.flags and NUM_FLAG) != 0 - -proc IS_NUM(t: Enum_field_types): bool = - result = (t <= FIELD_TYPE_INT24) or (t == FIELD_TYPE_YEAR) or - (t == FIELD_TYPE_NEWDECIMAL) - -proc INTERNAL_NUM_FIELD(f: Pst_mysql_field): bool = - result = (f.ftype <= FIELD_TYPE_INT24) and - ((f.ftype != FIELD_TYPE_TIMESTAMP) or (f.len == 14) or (f.len == 8)) or - (f.ftype == FIELD_TYPE_YEAR) - -proc reload(x: PMySQL): cint = - result = refresh(x, REFRESH_GRANT) - -{.pop.} -when defined(nimHasStyleChecks): - {.pop.} diff --git a/lib/wrappers/odbcsql.nim b/lib/wrappers/odbcsql.nim deleted file mode 100644 index 4e37e1a174..0000000000 --- a/lib/wrappers/odbcsql.nim +++ /dev/null @@ -1,841 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -when not defined(ODBCVER): - const - ODBCVER = 0x0351 ## define ODBC version 3.51 by default - -when defined(windows): - {.push callconv: stdcall.} - const odbclib = "odbc32.dll" -else: - {.push callconv: cdecl.} - const odbclib = "libodbc.so" - -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -# DATA TYPES CORRESPONDENCE -# BDE fields ODBC types -# ---------- ------------------ -# ftBlob SQL_BINARY -# ftBoolean SQL_BIT -# ftDate SQL_TYPE_DATE -# ftTime SQL_TYPE_TIME -# ftDateTime SQL_TYPE_TIMESTAMP -# ftInteger SQL_INTEGER -# ftSmallint SQL_SMALLINT -# ftFloat SQL_DOUBLE -# ftString SQL_CHAR -# ftMemo SQL_BINARY // SQL_VARCHAR -# - -type - TSqlChar* = char - TSqlSmallInt* = int16 - SqlUSmallInt* = int16 - SqlHandle* = pointer - SqlHEnv* = SqlHandle - SqlHDBC* = SqlHandle - SqlHStmt* = SqlHandle - SqlHDesc* = SqlHandle - TSqlInteger* = int32 - SqlUInteger* = int32 - TSqlLen* = int - TSqlULen* = uint - SqlPointer* = pointer - TSqlReal* = cfloat - TSqlDouble* = cdouble - TSqlFloat* = cdouble - SqlHWND* = pointer - PSQLCHAR* = cstring - PSQLINTEGER* = ptr TSqlInteger - PSQLUINTEGER* = ptr SqlUInteger - PSQLSMALLINT* = ptr TSqlSmallInt - PSQLUSMALLINT* = ptr SqlUSmallInt - PSQLREAL* = ptr TSqlReal - PSQLDOUBLE* = ptr TSqlDouble - PSQLFLOAT* = ptr TSqlFloat - PSQLHANDLE* = ptr SqlHandle - -const # SQL data type codes - SQL_UNKNOWN_TYPE* = 0 - SQL_LONGVARCHAR* = (- 1) - SQL_BINARY* = (- 2) - SQL_VARBINARY* = (- 3) - SQL_LONGVARBINARY* = (- 4) - SQL_BIGINT* = (- 5) - SQL_TINYINT* = (- 6) - SQL_BIT* = (- 7) - SQL_WCHAR* = (- 8) - SQL_WVARCHAR* = (- 9) - SQL_WLONGVARCHAR* = (- 10) - SQL_CHAR* = 1 - SQL_NUMERIC* = 2 - SQL_DECIMAL* = 3 - SQL_INTEGER* = 4 - SQL_SMALLINT* = 5 - SQL_FLOAT* = 6 - SQL_REAL* = 7 - SQL_DOUBLE* = 8 - SQL_DATETIME* = 9 - SQL_VARCHAR* = 12 - SQL_TYPE_DATE* = 91 - SQL_TYPE_TIME* = 92 - SQL_TYPE_TIMESTAMP* = 93 - SQL_DATE* = 9 - SQL_TIME* = 10 - SQL_TIMESTAMP* = 11 - SQL_INTERVAL* = 10 - SQL_GUID* = - 11 # interval codes - -when ODBCVER >= 0x0300: - const - SQL_CODE_YEAR* = 1 - SQL_CODE_MONTH* = 2 - SQL_CODE_DAY* = 3 - SQL_CODE_HOUR* = 4 - SQL_CODE_MINUTE* = 5 - SQL_CODE_SECOND* = 6 - SQL_CODE_YEAR_TO_MONTH* = 7 - SQL_CODE_DAY_TO_HOUR* = 8 - SQL_CODE_DAY_TO_MINUTE* = 9 - SQL_CODE_DAY_TO_SECOND* = 10 - SQL_CODE_HOUR_TO_MINUTE* = 11 - SQL_CODE_HOUR_TO_SECOND* = 12 - SQL_CODE_MINUTE_TO_SECOND* = 13 - SQL_INTERVAL_YEAR* = 100 + SQL_CODE_YEAR - SQL_INTERVAL_MONTH* = 100 + SQL_CODE_MONTH - SQL_INTERVAL_DAY* = 100 + SQL_CODE_DAY - SQL_INTERVAL_HOUR* = 100 + SQL_CODE_HOUR - SQL_INTERVAL_MINUTE* = 100 + SQL_CODE_MINUTE - SQL_INTERVAL_SECOND* = 100 + SQL_CODE_SECOND - SQL_INTERVAL_YEAR_TO_MONTH* = 100 + SQL_CODE_YEAR_TO_MONTH - SQL_INTERVAL_DAY_TO_HOUR* = 100 + SQL_CODE_DAY_TO_HOUR - SQL_INTERVAL_DAY_TO_MINUTE* = 100 + SQL_CODE_DAY_TO_MINUTE - SQL_INTERVAL_DAY_TO_SECOND* = 100 + SQL_CODE_DAY_TO_SECOND - SQL_INTERVAL_HOUR_TO_MINUTE* = 100 + SQL_CODE_HOUR_TO_MINUTE - SQL_INTERVAL_HOUR_TO_SECOND* = 100 + SQL_CODE_HOUR_TO_SECOND - SQL_INTERVAL_MINUTE_TO_SECOND* = 100 + SQL_CODE_MINUTE_TO_SECOND -else: - const - SQL_INTERVAL_YEAR* = - 80 - SQL_INTERVAL_MONTH* = - 81 - SQL_INTERVAL_YEAR_TO_MONTH* = - 82 - SQL_INTERVAL_DAY* = - 83 - SQL_INTERVAL_HOUR* = - 84 - SQL_INTERVAL_MINUTE* = - 85 - SQL_INTERVAL_SECOND* = - 86 - SQL_INTERVAL_DAY_TO_HOUR* = - 87 - SQL_INTERVAL_DAY_TO_MINUTE* = - 88 - SQL_INTERVAL_DAY_TO_SECOND* = - 89 - SQL_INTERVAL_HOUR_TO_MINUTE* = - 90 - SQL_INTERVAL_HOUR_TO_SECOND* = - 91 - SQL_INTERVAL_MINUTE_TO_SECOND* = - 92 - - -when ODBCVER < 0x0300: - const - SQL_UNICODE* = - 95 - SQL_UNICODE_VARCHAR* = - 96 - SQL_UNICODE_LONGVARCHAR* = - 97 - SQL_UNICODE_CHAR* = SQL_UNICODE -else: - # The previous definitions for SQL_UNICODE_ are historical and obsolete - const - SQL_UNICODE* = SQL_WCHAR - SQL_UNICODE_VARCHAR* = SQL_WVARCHAR - SQL_UNICODE_LONGVARCHAR* = SQL_WLONGVARCHAR - SQL_UNICODE_CHAR* = SQL_WCHAR -const # C datatype to SQL datatype mapping - SQL_C_CHAR* = SQL_CHAR - SQL_C_LONG* = SQL_INTEGER - SQL_C_SHORT* = SQL_SMALLINT - SQL_C_FLOAT* = SQL_REAL - SQL_C_DOUBLE* = SQL_DOUBLE - SQL_C_NUMERIC* = SQL_NUMERIC - SQL_C_DEFAULT* = 99 - SQL_SIGNED_OFFSET* = - 20 - SQL_UNSIGNED_OFFSET* = - 22 - SQL_C_DATE* = SQL_DATE - SQL_C_TIME* = SQL_TIME - SQL_C_TIMESTAMP* = SQL_TIMESTAMP - SQL_C_TYPE_DATE* = SQL_TYPE_DATE - SQL_C_TYPE_TIME* = SQL_TYPE_TIME - SQL_C_TYPE_TIMESTAMP* = SQL_TYPE_TIMESTAMP - SQL_C_INTERVAL_YEAR* = SQL_INTERVAL_YEAR - SQL_C_INTERVAL_MONTH* = SQL_INTERVAL_MONTH - SQL_C_INTERVAL_DAY* = SQL_INTERVAL_DAY - SQL_C_INTERVAL_HOUR* = SQL_INTERVAL_HOUR - SQL_C_INTERVAL_MINUTE* = SQL_INTERVAL_MINUTE - SQL_C_INTERVAL_SECOND* = SQL_INTERVAL_SECOND - SQL_C_INTERVAL_YEAR_TO_MONTH* = SQL_INTERVAL_YEAR_TO_MONTH - SQL_C_INTERVAL_DAY_TO_HOUR* = SQL_INTERVAL_DAY_TO_HOUR - SQL_C_INTERVAL_DAY_TO_MINUTE* = SQL_INTERVAL_DAY_TO_MINUTE - SQL_C_INTERVAL_DAY_TO_SECOND* = SQL_INTERVAL_DAY_TO_SECOND - SQL_C_INTERVAL_HOUR_TO_MINUTE* = SQL_INTERVAL_HOUR_TO_MINUTE - SQL_C_INTERVAL_HOUR_TO_SECOND* = SQL_INTERVAL_HOUR_TO_SECOND - SQL_C_INTERVAL_MINUTE_TO_SECOND* = SQL_INTERVAL_MINUTE_TO_SECOND - SQL_C_BINARY* = SQL_BINARY - SQL_C_BIT* = SQL_BIT - SQL_C_SBIGINT* = SQL_BIGINT + SQL_SIGNED_OFFSET # SIGNED BIGINT - SQL_C_UBIGINT* = SQL_BIGINT + SQL_UNSIGNED_OFFSET # UNSIGNED BIGINT - SQL_C_TINYINT* = SQL_TINYINT - SQL_C_SLONG* = SQL_C_LONG + SQL_SIGNED_OFFSET # SIGNED INTEGER - SQL_C_SSHORT* = SQL_C_SHORT + SQL_SIGNED_OFFSET # SIGNED SMALLINT - SQL_C_STINYINT* = SQL_TINYINT + SQL_SIGNED_OFFSET # SIGNED TINYINT - SQL_C_ULONG* = SQL_C_LONG + SQL_UNSIGNED_OFFSET # UNSIGNED INTEGER - SQL_C_USHORT* = SQL_C_SHORT + SQL_UNSIGNED_OFFSET # UNSIGNED SMALLINT - SQL_C_UTINYINT* = SQL_TINYINT + SQL_UNSIGNED_OFFSET # UNSIGNED TINYINT - SQL_C_BOOKMARK* = SQL_C_ULONG # BOOKMARK - SQL_C_GUID* = SQL_GUID - SQL_TYPE_NULL* = 0 - -when ODBCVER < 0x0300: - const - SQL_TYPE_MIN* = SQL_BIT - SQL_TYPE_MAX* = SQL_VARCHAR - -const - SQL_C_VARBOOKMARK* = SQL_C_BINARY - SQL_API_SQLDESCRIBEPARAM* = 58 - SQL_NO_TOTAL* = - 4 - -type - SQL_DATE_STRUCT* {.final, pure.} = object - Year*: TSqlSmallInt - Month*: SqlUSmallInt - Day*: SqlUSmallInt - - PSQL_DATE_STRUCT* = ptr SQL_DATE_STRUCT - SQL_TIME_STRUCT* {.final, pure.} = object - Hour*: SqlUSmallInt - Minute*: SqlUSmallInt - Second*: SqlUSmallInt - - PSQL_TIME_STRUCT* = ptr SQL_TIME_STRUCT - SQL_TIMESTAMP_STRUCT* {.final, pure.} = object - Year*: SqlUSmallInt - Month*: SqlUSmallInt - Day*: SqlUSmallInt - Hour*: SqlUSmallInt - Minute*: SqlUSmallInt - Second*: SqlUSmallInt - Fraction*: SqlUInteger - - PSQL_TIMESTAMP_STRUCT* = ptr SQL_TIMESTAMP_STRUCT - -const - SQL_NAME_LEN* = 128 - SQL_OV_ODBC3* = 3 - SQL_OV_ODBC2* = 2 - SQL_ATTR_ODBC_VERSION* = 200 # Options for SQLDriverConnect - SQL_DRIVER_NOPROMPT* = 0 - SQL_DRIVER_COMPLETE* = 1 - SQL_DRIVER_PROMPT* = 2 - SQL_DRIVER_COMPLETE_REQUIRED* = 3 - SQL_IS_POINTER* = (- 4) # whether an attribute is a pointer or not - SQL_IS_UINTEGER* = (- 5) - SQL_IS_INTEGER* = (- 6) - SQL_IS_USMALLINT* = (- 7) - SQL_IS_SMALLINT* = (- 8) # SQLExtendedFetch "fFetchType" values - SQL_FETCH_BOOKMARK* = 8 - SQL_SCROLL_OPTIONS* = 44 # SQL_USE_BOOKMARKS options - SQL_UB_OFF* = 0 - SQL_UB_ON* = 1 - SQL_UB_DEFAULT* = SQL_UB_OFF - SQL_UB_FIXED* = SQL_UB_ON - SQL_UB_VARIABLE* = 2 # SQL_SCROLL_OPTIONS masks - SQL_SO_FORWARD_ONLY* = 0x00000001 - SQL_SO_KEYSET_DRIVEN* = 0x00000002 - SQL_SO_DYNAMIC* = 0x00000004 - SQL_SO_MIXED* = 0x00000008 - SQL_SO_STATIC* = 0x00000010 - SQL_BOOKMARK_PERSISTENCE* = 82 - SQL_STATIC_SENSITIVITY* = 83 # SQL_BOOKMARK_PERSISTENCE values - SQL_BP_CLOSE* = 0x00000001 - SQL_BP_DELETE* = 0x00000002 - SQL_BP_DROP* = 0x00000004 - SQL_BP_TRANSACTION* = 0x00000008 - SQL_BP_UPDATE* = 0x00000010 - SQL_BP_OTHER_HSTMT* = 0x00000020 - SQL_BP_SCROLL* = 0x00000040 - SQL_DYNAMIC_CURSOR_ATTRIBUTES1* = 144 - SQL_DYNAMIC_CURSOR_ATTRIBUTES2* = 145 - SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES1* = 146 - SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES2* = 147 - SQL_INDEX_KEYWORDS* = 148 - SQL_INFO_SCHEMA_VIEWS* = 149 - SQL_KEYSET_CURSOR_ATTRIBUTES1* = 150 - SQL_KEYSET_CURSOR_ATTRIBUTES2* = 151 - SQL_STATIC_CURSOR_ATTRIBUTES1* = 167 - SQL_STATIC_CURSOR_ATTRIBUTES2* = 168 # supported SQLFetchScroll FetchOrientation's - SQL_CA1_NEXT* = 1 - SQL_CA1_ABSOLUTE* = 2 - SQL_CA1_RELATIVE* = 4 - SQL_CA1_BOOKMARK* = 8 # supported SQLSetPos LockType's - SQL_CA1_LOCK_NO_CHANGE* = 0x00000040 - SQL_CA1_LOCK_EXCLUSIVE* = 0x00000080 - SQL_CA1_LOCK_UNLOCK* = 0x00000100 # supported SQLSetPos Operations - SQL_CA1_POS_POSITION* = 0x00000200 - SQL_CA1_POS_UPDATE* = 0x00000400 - SQL_CA1_POS_DELETE* = 0x00000800 - SQL_CA1_POS_REFRESH* = 0x00001000 # positioned updates and deletes - SQL_CA1_POSITIONED_UPDATE* = 0x00002000 - SQL_CA1_POSITIONED_DELETE* = 0x00004000 - SQL_CA1_SELECT_FOR_UPDATE* = 0x00008000 # supported SQLBulkOperations operations - SQL_CA1_BULK_ADD* = 0x00010000 - SQL_CA1_BULK_UPDATE_BY_BOOKMARK* = 0x00020000 - SQL_CA1_BULK_DELETE_BY_BOOKMARK* = 0x00040000 - SQL_CA1_BULK_FETCH_BY_BOOKMARK* = 0x00080000 # supported values for SQL_ATTR_SCROLL_CONCURRENCY - SQL_CA2_READ_ONLY_CONCURRENCY* = 1 - SQL_CA2_LOCK_CONCURRENCY* = 2 - SQL_CA2_OPT_ROWVER_CONCURRENCY* = 4 - SQL_CA2_OPT_VALUES_CONCURRENCY* = 8 # sensitivity of the cursor to its own inserts, deletes, and updates - SQL_CA2_SENSITIVITY_ADDITIONS* = 0x00000010 - SQL_CA2_SENSITIVITY_DELETIONS* = 0x00000020 - SQL_CA2_SENSITIVITY_UPDATES* = 0x00000040 # semantics of SQL_ATTR_MAX_ROWS - SQL_CA2_MAX_ROWS_SELECT* = 0x00000080 - SQL_CA2_MAX_ROWS_INSERT* = 0x00000100 - SQL_CA2_MAX_ROWS_DELETE* = 0x00000200 - SQL_CA2_MAX_ROWS_UPDATE* = 0x00000400 - SQL_CA2_MAX_ROWS_CATALOG* = 0x00000800 - SQL_CA2_MAX_ROWS_AFFECTS_ALL* = (SQL_CA2_MAX_ROWS_SELECT or - SQL_CA2_MAX_ROWS_INSERT or SQL_CA2_MAX_ROWS_DELETE or - SQL_CA2_MAX_ROWS_UPDATE or SQL_CA2_MAX_ROWS_CATALOG) # semantics of - # SQL_DIAG_CURSOR_ROW_COUNT - SQL_CA2_CRC_EXACT* = 0x00001000 - SQL_CA2_CRC_APPROXIMATE* = 0x00002000 # the kinds of positioned statements that can be simulated - SQL_CA2_SIMULATE_NON_UNIQUE* = 0x00004000 - SQL_CA2_SIMULATE_TRY_UNIQUE* = 0x00008000 - SQL_CA2_SIMULATE_UNIQUE* = 0x00010000 # Operations in SQLBulkOperations - SQL_ADD* = 4 - SQL_SETPOS_MAX_OPTION_VALUE* = SQL_ADD - SQL_UPDATE_BY_BOOKMARK* = 5 - SQL_DELETE_BY_BOOKMARK* = 6 - SQL_FETCH_BY_BOOKMARK* = 7 # Operations in SQLSetPos - SQL_POSITION* = 0 - SQL_REFRESH* = 1 - SQL_UPDATE* = 2 - SQL_DELETE* = 3 # Lock options in SQLSetPos - SQL_LOCK_NO_CHANGE* = 0 - SQL_LOCK_EXCLUSIVE* = 1 - SQL_LOCK_UNLOCK* = 2 # SQLExtendedFetch "rgfRowStatus" element values - SQL_ROW_SUCCESS* = 0 - SQL_ROW_DELETED* = 1 - SQL_ROW_UPDATED* = 2 - SQL_ROW_NOROW* = 3 - SQL_ROW_ADDED* = 4 - SQL_ROW_ERROR* = 5 - SQL_ROW_SUCCESS_WITH_INFO* = 6 - SQL_ROW_PROCEED* = 0 - SQL_ROW_IGNORE* = 1 - SQL_MAX_DSN_LENGTH* = 32 # maximum data source name size - SQL_MAX_OPTION_STRING_LENGTH* = 256 - SQL_ODBC_CURSORS* = 110 - SQL_ATTR_ODBC_CURSORS* = SQL_ODBC_CURSORS # SQL_ODBC_CURSORS options - SQL_CUR_USE_IF_NEEDED* = 0 - SQL_CUR_USE_ODBC* = 1 - SQL_CUR_USE_DRIVER* = 2 - SQL_CUR_DEFAULT* = SQL_CUR_USE_DRIVER - SQL_PARAM_TYPE_UNKNOWN* = 0 - SQL_PARAM_INPUT* = 1 - SQL_PARAM_INPUT_OUTPUT* = 2 - SQL_RESULT_COL* = 3 - SQL_PARAM_OUTPUT* = 4 - SQL_RETURN_VALUE* = 5 # special length/indicator values - SQL_NULL_DATA* = (- 1) - SQL_DATA_AT_EXEC* = (- 2) - SQL_SUCCESS* = 0 - SQL_SUCCESS_WITH_INFO* = 1 - SQL_NO_DATA* = 100 - SQL_ERROR* = (- 1) - SQL_INVALID_HANDLE* = (- 2) - SQL_STILL_EXECUTING* = 2 - SQL_NEED_DATA* = 99 # flags for null-terminated string - SQL_NTS* = (- 3) # maximum message length - SQL_MAX_MESSAGE_LENGTH* = 512 # date/time length constants - SQL_DATE_LEN* = 10 - SQL_TIME_LEN* = 8 # add P+1 if precision is nonzero - SQL_TIMESTAMP_LEN* = 19 # add P+1 if precision is nonzero - # handle type identifiers - SQL_HANDLE_ENV* = 1 - SQL_HANDLE_DBC* = 2 - SQL_HANDLE_STMT* = 3 - SQL_HANDLE_DESC* = 4 # environment attribute - SQL_ATTR_OUTPUT_NTS* = 10001 # connection attributes - SQL_ATTR_AUTO_IPD* = 10001 - SQL_ATTR_METADATA_ID* = 10014 # statement attributes - SQL_ATTR_APP_ROW_DESC* = 10010 - SQL_ATTR_APP_PARAM_DESC* = 10011 - SQL_ATTR_IMP_ROW_DESC* = 10012 - SQL_ATTR_IMP_PARAM_DESC* = 10013 - SQL_ATTR_CURSOR_SCROLLABLE* = (- 1) - SQL_ATTR_CURSOR_SENSITIVITY* = (- 2) - SQL_QUERY_TIMEOUT* = 0 - SQL_MAX_ROWS* = 1 - SQL_NOSCAN* = 2 - SQL_MAX_LENGTH* = 3 - SQL_ASYNC_ENABLE* = 4 # same as SQL_ATTR_ASYNC_ENABLE */ - SQL_BIND_TYPE* = 5 - SQL_CURSOR_TYPE* = 6 - SQL_CONCURRENCY* = 7 - SQL_KEYSET_SIZE* = 8 - SQL_ROWSET_SIZE* = 9 - SQL_SIMULATE_CURSOR* = 10 - SQL_RETRIEVE_DATA* = 11 - SQL_USE_BOOKMARKS* = 12 - SQL_GET_BOOKMARK* = 13 # GetStmtOption Only */ - SQL_ROW_NUMBER* = 14 # GetStmtOption Only */ - SQL_ATTR_CURSOR_TYPE* = SQL_CURSOR_TYPE - SQL_ATTR_CONCURRENCY* = SQL_CONCURRENCY - SQL_ATTR_FETCH_BOOKMARK_PTR* = 16 - SQL_ATTR_ROW_STATUS_PTR* = 25 - SQL_ATTR_ROWS_FETCHED_PTR* = 26 - SQL_AUTOCOMMIT* = 102 - SQL_ATTR_AUTOCOMMIT* = SQL_AUTOCOMMIT - SQL_ATTR_ROW_NUMBER* = SQL_ROW_NUMBER - SQL_TXN_ISOLATION* = 108 - SQL_ATTR_TXN_ISOLATION* = SQL_TXN_ISOLATION - SQL_ATTR_MAX_ROWS* = SQL_MAX_ROWS - SQL_ATTR_USE_BOOKMARKS* = SQL_USE_BOOKMARKS #* connection attributes */ - SQL_ACCESS_MODE* = 101 # SQL_AUTOCOMMIT =102; - SQL_LOGIN_TIMEOUT* = 103 - SQL_OPT_TRACE* = 104 - SQL_OPT_TRACEFILE* = 105 - SQL_TRANSLATE_DLL* = 106 - SQL_TRANSLATE_OPTION* = 107 # SQL_TXN_ISOLATION =108; - SQL_CURRENT_QUALIFIER* = 109 # SQL_ODBC_CURSORS =110; - SQL_QUIET_MODE* = 111 - SQL_PACKET_SIZE* = 112 #* connection attributes with new names */ - SQL_ATTR_ACCESS_MODE* = SQL_ACCESS_MODE # SQL_ATTR_AUTOCOMMIT =SQL_AUTOCOMMIT; - SQL_ATTR_CONNECTION_DEAD* = 1209 #* GetConnectAttr only */ - SQL_ATTR_CONNECTION_TIMEOUT* = 113 - SQL_ATTR_CURRENT_CATALOG* = SQL_CURRENT_QUALIFIER - SQL_ATTR_DISCONNECT_BEHAVIOR* = 114 - SQL_ATTR_ENLIST_IN_DTC* = 1207 - SQL_ATTR_ENLIST_IN_XA* = 1208 - SQL_ATTR_LOGIN_TIMEOUT* = SQL_LOGIN_TIMEOUT # SQL_ATTR_ODBC_CURSORS =SQL_ODBC_CURSORS; - SQL_ATTR_PACKET_SIZE* = SQL_PACKET_SIZE - SQL_ATTR_QUIET_MODE* = SQL_QUIET_MODE - SQL_ATTR_TRACE* = SQL_OPT_TRACE - SQL_ATTR_TRACEFILE* = SQL_OPT_TRACEFILE - SQL_ATTR_TRANSLATE_LIB* = SQL_TRANSLATE_DLL - SQL_ATTR_TRANSLATE_OPTION* = SQL_TRANSLATE_OPTION # SQL_ATTR_TXN_ISOLATION =SQL_TXN_ISOLATION; - #* SQL_ACCESS_MODE options */ - SQL_MODE_READ_WRITE* = 0 - SQL_MODE_READ_ONLY* = 1 - SQL_MODE_DEFAULT* = SQL_MODE_READ_WRITE #* SQL_AUTOCOMMIT options */ - SQL_AUTOCOMMIT_OFF* = 0 - SQL_AUTOCOMMIT_ON* = 1 - SQL_AUTOCOMMIT_DEFAULT* = SQL_AUTOCOMMIT_ON # SQL_ATTR_CURSOR_SCROLLABLE values - SQL_NONSCROLLABLE* = 0 - SQL_SCROLLABLE* = 1 # SQL_CURSOR_TYPE options - SQL_CURSOR_FORWARD_ONLY* = 0 - SQL_CURSOR_KEYSET_DRIVEN* = 1 - SQL_CURSOR_DYNAMIC* = 2 - SQL_CURSOR_STATIC* = 3 - SQL_CURSOR_TYPE_DEFAULT* = SQL_CURSOR_FORWARD_ONLY # Default value - # SQL_CONCURRENCY options - SQL_CONCUR_READ_ONLY* = 1 - SQL_CONCUR_LOCK* = 2 - SQL_CONCUR_ROWVER* = 3 - SQL_CONCUR_VALUES* = 4 - SQL_CONCUR_DEFAULT* = SQL_CONCUR_READ_ONLY # Default value - # identifiers of fields in the SQL descriptor - SQL_DESC_COUNT* = 1001 - SQL_DESC_TYPE* = 1002 - SQL_DESC_LENGTH* = 1003 - SQL_DESC_OCTET_LENGTH_PTR* = 1004 - SQL_DESC_PRECISION* = 1005 - SQL_DESC_SCALE* = 1006 - SQL_DESC_DATETIME_INTERVAL_CODE* = 1007 - SQL_DESC_NULLABLE* = 1008 - SQL_DESC_INDICATOR_PTR* = 1009 - SQL_DESC_DATA_PTR* = 1010 - SQL_DESC_NAME* = 1011 - SQL_DESC_UNNAMED* = 1012 - SQL_DESC_OCTET_LENGTH* = 1013 - SQL_DESC_ALLOC_TYPE* = 1099 # identifiers of fields in the diagnostics area - SQL_DIAG_RETURNCODE* = 1 - SQL_DIAG_NUMBER* = 2 - SQL_DIAG_ROW_COUNT* = 3 - SQL_DIAG_SQLSTATE* = 4 - SQL_DIAG_NATIVE* = 5 - SQL_DIAG_MESSAGE_TEXT* = 6 - SQL_DIAG_DYNAMIC_FUNCTION* = 7 - SQL_DIAG_CLASS_ORIGIN* = 8 - SQL_DIAG_SUBCLASS_ORIGIN* = 9 - SQL_DIAG_CONNECTION_NAME* = 10 - SQL_DIAG_SERVER_NAME* = 11 - SQL_DIAG_DYNAMIC_FUNCTION_CODE* = 12 # dynamic function codes - SQL_DIAG_ALTER_TABLE* = 4 - SQL_DIAG_CREATE_INDEX* = (- 1) - SQL_DIAG_CREATE_TABLE* = 77 - SQL_DIAG_CREATE_VIEW* = 84 - SQL_DIAG_DELETE_WHERE* = 19 - SQL_DIAG_DROP_INDEX* = (- 2) - SQL_DIAG_DROP_TABLE* = 32 - SQL_DIAG_DROP_VIEW* = 36 - SQL_DIAG_DYNAMIC_DELETE_CURSOR* = 38 - SQL_DIAG_DYNAMIC_UPDATE_CURSOR* = 81 - SQL_DIAG_GRANT* = 48 - SQL_DIAG_INSERT* = 50 - SQL_DIAG_REVOKE* = 59 - SQL_DIAG_SELECT_CURSOR* = 85 - SQL_DIAG_UNKNOWN_STATEMENT* = 0 - SQL_DIAG_UPDATE_WHERE* = 82 # Statement attribute values for cursor sensitivity - SQL_UNSPECIFIED* = 0 - SQL_INSENSITIVE* = 1 - SQL_SENSITIVE* = 2 # GetTypeInfo() request for all data types - SQL_ALL_TYPES* = 0 # Default conversion code for SQLBindCol(), SQLBindParam() and SQLGetData() - SQL_DEFAULT* = 99 # SQLGetData() code indicating that the application row descriptor - # specifies the data type - SQL_ARD_TYPE* = (- 99) # SQL date/time type subcodes - SQL_CODE_DATE* = 1 - SQL_CODE_TIME* = 2 - SQL_CODE_TIMESTAMP* = 3 # CLI option values - SQL_FALSE* = 0 - SQL_TRUE* = 1 # values of NULLABLE field in descriptor - SQL_NO_NULLS* = 0 - SQL_NULLABLE* = 1 # Value returned by SQLGetTypeInfo() to denote that it is - # not known whether or not a data type supports null values. - SQL_NULLABLE_UNKNOWN* = 2 - SQL_CLOSE* = 0 - SQL_DROP* = 1 - SQL_UNBIND* = 2 - SQL_RESET_PARAMS* = 3 # Codes used for FetchOrientation in SQLFetchScroll(), - # and in SQLDataSources() - SQL_FETCH_NEXT* = 1 - SQL_FETCH_FIRST* = 2 - SQL_FETCH_FIRST_USER* = 31 - SQL_FETCH_FIRST_SYSTEM* = 32 # Other codes used for FetchOrientation in SQLFetchScroll() - SQL_FETCH_LAST* = 3 - SQL_FETCH_PRIOR* = 4 - SQL_FETCH_ABSOLUTE* = 5 - SQL_FETCH_RELATIVE* = 6 - SQL_NULL_HENV* = SqlHEnv(nil) - SQL_NULL_HDBC* = SqlHDBC(nil) - SQL_NULL_HSTMT* = SqlHStmt(nil) - SQL_NULL_HDESC* = SqlHDesc(nil) #* null handle used in place of parent handle when allocating HENV */ - SQL_NULL_HANDLE* = SqlHandle(nil) #* Values that may appear in the result set of SQLSpecialColumns() */ - SQL_SCOPE_CURROW* = 0 - SQL_SCOPE_TRANSACTION* = 1 - SQL_SCOPE_SESSION* = 2 #* Column types and scopes in SQLSpecialColumns. */ - SQL_BEST_ROWID* = 1 - SQL_ROWVER* = 2 - SQL_ROW_IDENTIFIER* = 1 #* Reserved values for UNIQUE argument of SQLStatistics() */ - SQL_INDEX_UNIQUE* = 0 - SQL_INDEX_ALL* = 1 #* Reserved values for RESERVED argument of SQLStatistics() */ - SQL_QUICK* = 0 - SQL_ENSURE* = 1 #* Values that may appear in the result set of SQLStatistics() */ - SQL_TABLE_STAT* = 0 - SQL_INDEX_CLUSTERED* = 1 - SQL_INDEX_HASHED* = 2 - SQL_INDEX_OTHER* = 3 - SQL_SCROLL_CONCURRENCY* = 43 - SQL_TXN_CAPABLE* = 46 - SQL_TRANSACTION_CAPABLE* = SQL_TXN_CAPABLE - SQL_USER_NAME* = 47 - SQL_TXN_ISOLATION_OPTION* = 72 - SQL_TRANSACTION_ISOLATION_OPTION* = SQL_TXN_ISOLATION_OPTION - SQL_OJ_CAPABILITIES* = 115 - SQL_OUTER_JOIN_CAPABILITIES* = SQL_OJ_CAPABILITIES - SQL_XOPEN_CLI_YEAR* = 10000 - SQL_CURSOR_SENSITIVITY* = 10001 - SQL_DESCRIBE_PARAMETER* = 10002 - SQL_CATALOG_NAME* = 10003 - SQL_COLLATION_SEQ* = 10004 - SQL_MAX_IDENTIFIER_LEN* = 10005 - SQL_MAXIMUM_IDENTIFIER_LENGTH* = SQL_MAX_IDENTIFIER_LEN - SQL_SCCO_READ_ONLY* = 1 - SQL_SCCO_LOCK* = 2 - SQL_SCCO_OPT_ROWVER* = 4 - SQL_SCCO_OPT_VALUES* = 8 #* SQL_TXN_CAPABLE values */ - SQL_TC_NONE* = 0 - SQL_TC_DML* = 1 - SQL_TC_ALL* = 2 - SQL_TC_DDL_COMMIT* = 3 - SQL_TC_DDL_IGNORE* = 4 #* SQL_TXN_ISOLATION_OPTION bitmasks */ - SQL_TXN_READ_UNCOMMITTED* = 1 - SQL_TRANSACTION_READ_UNCOMMITTED* = SQL_TXN_READ_UNCOMMITTED - SQL_TXN_READ_COMMITTED* = 2 - SQL_TRANSACTION_READ_COMMITTED* = SQL_TXN_READ_COMMITTED - SQL_TXN_REPEATABLE_READ* = 4 - SQL_TRANSACTION_REPEATABLE_READ* = SQL_TXN_REPEATABLE_READ - SQL_TXN_SERIALIZABLE* = 8 - SQL_TRANSACTION_SERIALIZABLE* = SQL_TXN_SERIALIZABLE - SQL_SS_ADDITIONS* = 1 - SQL_SS_DELETIONS* = 2 - SQL_SS_UPDATES* = 4 # SQLColAttributes defines - SQL_COLUMN_COUNT* = 0 - SQL_COLUMN_NAME* = 1 - SQL_COLUMN_TYPE* = 2 - SQL_COLUMN_LENGTH* = 3 - SQL_COLUMN_PRECISION* = 4 - SQL_COLUMN_SCALE* = 5 - SQL_COLUMN_DISPLAY_SIZE* = 6 - SQL_COLUMN_NULLABLE* = 7 - SQL_COLUMN_UNSIGNED* = 8 - SQL_COLUMN_MONEY* = 9 - SQL_COLUMN_UPDATABLE* = 10 - SQL_COLUMN_AUTO_INCREMENT* = 11 - SQL_COLUMN_CASE_SENSITIVE* = 12 - SQL_COLUMN_SEARCHABLE* = 13 - SQL_COLUMN_TYPE_NAME* = 14 - SQL_COLUMN_TABLE_NAME* = 15 - SQL_COLUMN_OWNER_NAME* = 16 - SQL_COLUMN_QUALIFIER_NAME* = 17 - SQL_COLUMN_LABEL* = 18 - SQL_COLATT_OPT_MAX* = SQL_COLUMN_LABEL - SQL_COLUMN_DRIVER_START* = 1000 - SQL_DESC_ARRAY_SIZE* = 20 - SQL_DESC_ARRAY_STATUS_PTR* = 21 - SQL_DESC_AUTO_UNIQUE_VALUE* = SQL_COLUMN_AUTO_INCREMENT - SQL_DESC_BASE_COLUMN_NAME* = 22 - SQL_DESC_BASE_TABLE_NAME* = 23 - SQL_DESC_BIND_OFFSET_PTR* = 24 - SQL_DESC_BIND_TYPE* = 25 - SQL_DESC_CASE_SENSITIVE* = SQL_COLUMN_CASE_SENSITIVE - SQL_DESC_CATALOG_NAME* = SQL_COLUMN_QUALIFIER_NAME - SQL_DESC_CONCISE_TYPE* = SQL_COLUMN_TYPE - SQL_DESC_DATETIME_INTERVAL_PRECISION* = 26 - SQL_DESC_DISPLAY_SIZE* = SQL_COLUMN_DISPLAY_SIZE - SQL_DESC_FIXED_PREC_SCALE* = SQL_COLUMN_MONEY - SQL_DESC_LABEL* = SQL_COLUMN_LABEL - SQL_DESC_LITERAL_PREFIX* = 27 - SQL_DESC_LITERAL_SUFFIX* = 28 - SQL_DESC_LOCAL_TYPE_NAME* = 29 - SQL_DESC_MAXIMUM_SCALE* = 30 - SQL_DESC_MINIMUM_SCALE* = 31 - SQL_DESC_NUM_PREC_RADIX* = 32 - SQL_DESC_PARAMETER_TYPE* = 33 - SQL_DESC_ROWS_PROCESSED_PTR* = 34 - SQL_DESC_SCHEMA_NAME* = SQL_COLUMN_OWNER_NAME - SQL_DESC_SEARCHABLE* = SQL_COLUMN_SEARCHABLE - SQL_DESC_TYPE_NAME* = SQL_COLUMN_TYPE_NAME - SQL_DESC_TABLE_NAME* = SQL_COLUMN_TABLE_NAME - SQL_DESC_UNSIGNED* = SQL_COLUMN_UNSIGNED - SQL_DESC_UPDATABLE* = SQL_COLUMN_UPDATABLE #* SQLEndTran() options */ - SQL_COMMIT* = 0 - SQL_ROLLBACK* = 1 - SQL_ATTR_ROW_ARRAY_SIZE* = 27 #* SQLConfigDataSource() options */ - ODBC_ADD_DSN* = 1 - ODBC_CONFIG_DSN* = 2 - ODBC_REMOVE_DSN* = 3 - ODBC_ADD_SYS_DSN* = 4 - ODBC_CONFIG_SYS_DSN* = 5 - ODBC_REMOVE_SYS_DSN* = 6 - - SQL_ACTIVE_CONNECTIONS* = 0 # SQLGetInfo - SQL_DATA_SOURCE_NAME* = 2 - SQL_DATA_SOURCE_READ_ONLY* = 25 - SQL_DATABASE_NAME* = 2 - SQL_DBMS_NAME* = 17 - SQL_DBMS_VERSION* = 18 - SQL_DRIVER_HDBC* = 3 - SQL_DRIVER_HENV* = 4 - SQL_DRIVER_HSTMT* = 5 - SQL_DRIVER_NAME* = 6 - SQL_DRIVER_VER* = 7 - SQL_FETCH_DIRECTION* = 8 - SQL_ODBC_VER* = 10 - SQL_DRIVER_ODBC_VER* = 77 - SQL_SERVER_NAME* = 13 - SQL_ACTIVE_ENVIRONMENTS* = 116 - SQL_ACTIVE_STATEMENTS* = 1 - SQL_SQL_CONFORMANCE* = 118 - SQL_DATETIME_LITERALS* = 119 - SQL_ASYNC_MODE* = 10021 - SQL_BATCH_ROW_COUNT* = 120 - SQL_BATCH_SUPPORT* = 121 - SQL_CATALOG_LOCATION* = 114 - #SQL_CATALOG_NAME* = 10003 - SQL_CATALOG_NAME_SEPARATOR* = 41 - SQL_CATALOG_TERM* = 42 - SQL_CATALOG_USAGE* = 92 - #SQL_COLLATION_SEQ* = 10004 - SQL_COLUMN_ALIAS* = 87 - #SQL_USER_NAME* = 47 - -proc SQLAllocHandle*(HandleType: TSqlSmallInt, InputHandle: SqlHandle, - OutputHandlePtr: var SqlHandle): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: SqlPointer, BufferLength: TSqlInteger, - StringLength: PSQLINTEGER): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLFreeHandle*(HandleType: TSqlSmallInt, Handle: SqlHandle): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetDiagRec*(HandleType: TSqlSmallInt, Handle: SqlHandle, - RecNumber: TSqlSmallInt, Sqlstate: PSQLCHAR, - NativeError: var TSqlInteger, MessageText: PSQLCHAR, - BufferLength: TSqlSmallInt, TextLength: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetDiagField*(HandleType: TSqlSmallInt, Handle: SqlHandle, - RecNumber: TSqlSmallInt, DiagIdentifier: TSqlSmallInt, - DiagInfoPtr: SqlPointer, BufferLength: TSqlSmallInt, - StringLengthPtr: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLConnect*(ConnectionHandle: SqlHDBC, ServerName: PSQLCHAR, - NameLength1: TSqlSmallInt, UserName: PSQLCHAR, - NameLength2: TSqlSmallInt, Authentication: PSQLCHAR, - NameLength3: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLDisconnect*(ConnectionHandle: SqlHDBC): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLDriverConnect*(hdbc: SqlHDBC, hwnd: SqlHWND, szCsin: cstring, - szCLen: TSqlSmallInt, szCsout: cstring, - cbCSMax: TSqlSmallInt, cbCsOut: var TSqlSmallInt, - f: SqlUSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLBrowseConnect*(hdbc: SqlHDBC, szConnStrIn: PSQLCHAR, - cbConnStrIn: TSqlSmallInt, szConnStrOut: PSQLCHAR, - cbConnStrOutMax: TSqlSmallInt, - cbConnStrOut: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLExecDirect*(StatementHandle: SqlHStmt, StatementText: PSQLCHAR, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLExecDirectW*(StatementHandle: SqlHStmt, StatementText: WideCString, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLPrepare*(StatementHandle: SqlHStmt, StatementText: PSQLCHAR, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLPrepareW*(StatementHandle: SqlHStmt, StatementText: WideCString, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLCloseCursor*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLExecute*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLFetch*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLNumResultCols*(StatementHandle: SqlHStmt, ColumnCount: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLDescribeCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - ColumnName: PSQLCHAR, BufferLength: TSqlSmallInt, - NameLength: var TSqlSmallInt, DataType: var TSqlSmallInt, - ColumnSize: var TSqlULen, - DecimalDigits: var TSqlSmallInt, Nullable: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLFetchScroll*(StatementHandle: SqlHStmt, FetchOrientation: TSqlSmallInt, - FetchOffset: TSqlLen): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLExtendedFetch*(hstmt: SqlHStmt, fFetchType: SqlUSmallInt, - irow: TSqlLen, pcrow: var TSqlULen, - rgfRowStatus: PSQLUSMALLINT): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLGetData*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - TargetType: TSqlSmallInt, TargetValue: SqlPointer, - BufferLength: TSqlLen, StrLen_or_Ind: ptr TSqlLen): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetStmtAttr*(StatementHandle: SqlHStmt, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetStmtAttr*(StatementHandle: SqlHStmt, Attribute: TSqlInteger, - Value: SqlPointer, BufferLength: TSqlInteger, - StringLength: PSQLINTEGER): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLGetInfo*(ConnectionHandle: SqlHDBC, InfoType: SqlUSmallInt, - InfoValue: SqlPointer, BufferLength: TSqlSmallInt, - StringLength: PSQLSMALLINT): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLBulkOperations*(StatementHandle: SqlHStmt, Operation: SqlUSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLPutData*(StatementHandle: SqlHStmt, Data: SqlPointer, - StrLen_or_Ind: TSQLLEN): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLBindCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - TargetType: TSqlSmallInt, TargetValue: SqlPointer, - BufferLength: TSqlLEN, StrLen_or_Ind: PSQLINTEGER): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetPos*(hstmt: SqlHStmt, irow: SqlUSmallInt, fOption: SqlUSmallInt, - fLock: SqlUSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLDataSources*(EnvironmentHandle: SqlHEnv, Direction: SqlUSmallInt, - ServerName: PSQLCHAR, BufferLength1: TSqlSmallInt, - NameLength1: PSQLSMALLINT, Description: PSQLCHAR, - BufferLength2: TSqlSmallInt, NameLength2: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLDrivers*(EnvironmentHandle: SqlHEnv, Direction: SqlUSmallInt, - DriverDescription: PSQLCHAR, BufferLength1: TSqlSmallInt, - DescriptionLength1: PSQLSMALLINT, DriverAttributes: PSQLCHAR, - BufferLength2: TSqlSmallInt, AttributesLength2: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetConnectAttr*(ConnectionHandle: SqlHDBC, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR, - BufferLength: TSqlSmallInt, NameLength: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR, - NameLength: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLRowCount*(StatementHandle: SqlHStmt, RowCount: var TSQLLEN): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLBindParameter*(hstmt: SqlHStmt, ipar: SqlUSmallInt, - fParamType: TSqlSmallInt, fCType: TSqlSmallInt, - fSqlType: TSqlSmallInt, cbColDef: TSQLULEN, - ibScale: TSqlSmallInt, rgbValue: SqlPointer, - cbValueMax: TSQLLEN, pcbValue: var TSQLLEN): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLFreeStmt*(StatementHandle: SqlHStmt, Option: SqlUSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLColAttribute*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - FieldIdentifier: SqlUSmallInt, - CharacterAttribute: PSQLCHAR, BufferLength: TSqlSmallInt, - StringLength: PSQLSMALLINT, - NumericAttribute: TSQLLEN): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLEndTran*(HandleType: TSqlSmallInt, Handle: SqlHandle, - CompletionType: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLTables*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt, szTableType: PSQLCHAR, - cbTableType: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLColumns*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt, szColumnName: PSQLCHAR, - cbColumnName: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLSpecialColumns*(StatementHandle: SqlHStmt, IdentifierType: SqlUSmallInt, - CatalogName: PSQLCHAR, NameLength1: TSqlSmallInt, - SchemaName: PSQLCHAR, NameLength2: TSqlSmallInt, - TableName: PSQLCHAR, NameLength3: TSqlSmallInt, - Scope: SqlUSmallInt, - Nullable: SqlUSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLProcedures*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLPrimaryKeys*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, TableName: PSQLCHAR, - NameLength3: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLProcedureColumns*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, ProcName: PSQLCHAR, - NameLength3: TSqlSmallInt, ColumnName: PSQLCHAR, - NameLength4: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLStatistics*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, TableName: PSQLCHAR, - NameLength3: TSqlSmallInt, Unique: SqlUSmallInt, - Reserved: SqlUSmallInt): TSqlSmallInt {. - dynlib: odbclib, importc.} -proc SQLErr*(henv: SqlHEnv, hdbc: SqlHDBC, hstmt: SqlHStmt, - szSqlState, pfNativeError, szErrorMsg: PSQLCHAR, - cbErrorMsgMax: TSqlSmallInt, - pcbErrorMsg: PSQLSMALLINT): TSqlSmallInt {. - dynlib: odbclib, importc: "SQLError".} - -{.pop.} -when defined(nimHasStyleChecks): - {.pop.} diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 313ce7d199..9921b7ffde 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -7,66 +7,88 @@ # distribution, for details about the copyright. # -## OpenSSL support +## OpenSSL wrapper. Supports OpenSSL >= 1.1.0 dynamically (as default) or statically linked +## using `--dynlibOverride:ssl`. ## -## When OpenSSL is dynamically linked, the wrapper provides partial forward and backward -## compatibility for OpenSSL versions above and below 1.1.0 +## `-d:sslVersion=1.2.3` can be used to force an SSL version. +## This version must be included in the library name. +## `-d:useOpenssl3` may be set for OpenSSL 3 instead. ## -## OpenSSL can also be statically linked using `--dynlibOverride:ssl` for OpenSSL >= 1.1.0. -## If you want to statically link against OpenSSL 1.0.x, you now have to -## define the `openssl10` symbol via `-d:openssl10`. +## There is also limited support for OpenSSL 1.0.x which may require `-d:openssl10`. ## ## Build and test examples: ## -## .. code-block:: +## ```cmd +## ./bin/nim c -d:ssl -p:. -r tests/stdlib/tssl.nim +## ./bin/nim c -d:ssl --threads:on -p:. -r tests/stdlib/thttpclient_ssl.nim ## ./bin/nim c -d:ssl -p:. -r tests/untestable/tssl.nim ## ./bin/nim c -d:ssl -p:. --dynlibOverride:ssl --passl:-lcrypto --passl:-lssl -r tests/untestable/tssl.nim +## ./bin/nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:ssl -p:testament/lib --threads:on tests/untestable/thttpclient_ssl_remotenetwork.nim +## ``` + +# https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html +# +from std/strutils import startsWith + +when defined(nimPreviewSlimSystem): + import std/syncio when defined(nimHasStyleChecks): {.push styleChecks: off.} const useWinVersion = defined(windows) or defined(nimdoc) -# To force openSSL version use -d:sslVersion=1.0.0 +# To force openSSL version use -d:sslVersion=1.2.3 # See: #10281, #10230 # General issue: # Other dynamic libraries (like libpg) load different openSSL version then what nim loads. # Having two different openSSL loaded version causes a crash. # Use this compile time define to force the openSSL version that your other dynamic libraries want. const sslVersion {.strdefine.}: string = "" +const useOpenssl3* {.booldefine.} = sslVersion.startsWith('3') when sslVersion != "": when defined(macosx): const DLLSSLName* = "libssl." & sslVersion & ".dylib" DLLUtilName* = "libcrypto." & sslVersion & ".dylib" - from posix import SocketHandle + from std/posix import SocketHandle elif defined(windows): const DLLSSLName* = "libssl-" & sslVersion & ".dll" DLLUtilName* = "libcrypto-" & sslVersion & ".dll" - from winlean import SocketHandle + from std/winlean import SocketHandle else: const DLLSSLName* = "libssl.so." & sslVersion DLLUtilName* = "libcrypto.so." & sslVersion - from posix import SocketHandle + from std/posix import SocketHandle elif useWinVersion: - when not defined(nimOldDlls) and defined(cpu64): + when defined(openssl10) or defined(nimOldDlls): + when defined(cpu64): + const + DLLSSLName* = "(ssleay32|ssleay64).dll" + DLLUtilName* = "(libeay32|libeay64).dll" + else: + const + DLLSSLName* = "ssleay32.dll" + DLLUtilName* = "libeay32.dll" + elif defined(cpu64): const DLLSSLName* = "(libssl-1_1-x64|ssleay64|libssl64).dll" DLLUtilName* = "(libcrypto-1_1-x64|libeay64).dll" else: const DLLSSLName* = "(libssl-1_1|ssleay32|libssl32).dll" - DLLUtilName* = "(libcrypto-1_1|libeay32).dll" + DLLUtilName* = "(libcrypto-1_1|libeay32).dll" - from winlean import SocketHandle + from std/winlean import SocketHandle else: + # same list of versions but ordered differently? when defined(osx): - const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.48|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + const versions = "(.3|.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.48|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" else: - const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.48|.47|.46|.45|.44|.43|.41|.39|.38|.10|)" + const versions = "(.3|.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.48|.47|.46|.45|.44|.43|.41|.39|.38|.10|)" when defined(macosx): const @@ -80,9 +102,12 @@ else: const DLLSSLName* = "libssl.so" & versions DLLUtilName* = "libcrypto.so" & versions - from posix import SocketHandle + from std/posix import SocketHandle -import dynlib +import std/dynlib + +{.pragma: lcrypto, cdecl, dynlib: DLLUtilName, importc.} +{.pragma: lssl, cdecl, dynlib: DLLSSLName, importc.} type SslStruct {.final, pure.} = object @@ -93,7 +118,6 @@ type PSTACK* = SslPtr PX509* = SslPtr PX509_NAME* = SslPtr - PEVP_MD* = SslPtr PBIO_METHOD* = SslPtr BIO* = SslPtr EVP_PKEY* = SslPtr @@ -257,7 +281,9 @@ proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} # and support SSLv3, TLSv1, TLSv1.1 and TLSv1.2 # SSLv23_method(), SSLv23_server_method(), SSLv23_client_method() are removed in 1.1.0 -when compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks): +const useStaticLink = compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks) + +when useStaticLink: # Static linking when defined(openssl10): @@ -268,6 +294,8 @@ when compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks): proc getOpenSSLVersion*(): culong = SSLeay() + + proc ERR_load_BIO_strings*() {.cdecl, dynlib: DLLUtilName, importc.} else: proc OPENSSL_init_ssl*(opts: uint64, settings: uint8): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} proc SSL_library_init*(): cint {.discardable.} = @@ -290,10 +318,13 @@ when compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks): # Static linking against OpenSSL < 1.1.0 is not supported discard + proc ERR_load_BIO_strings*() = + discard + when defined(libressl) or defined(openssl10): proc SSL_state(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc.} proc SSL_in_init*(ssl: SslPtr): cint {.inline.} = - SSl_state(ssl) and SSL_ST_INIT + SSL_state(ssl) and SSL_ST_INIT else: proc SSL_in_init*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} @@ -303,9 +334,10 @@ when compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks): proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} else: - # Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some + # Here we're trying to stay compatible between openssl versions. Some # symbols are loaded dynamically and we don't use them if not found. proc thisModule(): LibHandle {.inline.} = var thisMod {.global.}: LibHandle @@ -313,9 +345,12 @@ else: result = thisMod - proc sslModule(): LibHandle {.inline.} = + proc sslModule(): LibHandle {.inline, raises: [LibraryError], tags:[RootEffect].} = var sslMod {.global.}: LibHandle - if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) + try: + if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) + except: + raise newException(LibraryError, "Could not load SSL using " & DLLSSLName) result = sslMod @@ -340,26 +375,30 @@ else: if result.isNil and alternativeName.len > 0: result = symAddr(thisDynlib, alternativeName) - proc sslSymNullable(name: string, alternativeName = ""): pointer = + proc sslSymNullable(name: string, alternativeName = ""): pointer {.raises: [LibraryError], tags:[RootEffect].} = sslModule().symNullable(name, alternativeName) - proc sslSymThrows(name: string, alternativeName = ""): pointer = + proc sslSymThrows(name: string, alternativeName = ""): pointer {.raises: [LibraryError].} = result = sslSymNullable(name, alternativeName) if result.isNil: raiseInvalidLibrary(name) proc utilSymNullable(name: string, alternativeName = ""): pointer = utilModule().symNullable(name, alternativeName) - proc loadPSSLMethod(method1, method2: string): PSSL_METHOD = + proc loadPSSLMethod(method1, method2: string): PSSL_METHOD {.raises: [LibraryError], tags:[RootEffect].} = ## Load <method1> from OpenSSL if available, otherwise <method2> ## let methodSym = sslSymNullable(method1, method2) if methodSym.isNil: raise newException(LibraryError, "Could not load " & method1 & " nor " & method2) - let method2Proc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](methodSym) + let method2Proc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe, raises: [].}](methodSym) return method2Proc() + proc CRYPTO_set_mem_functions(a,b,c: pointer) = + let theProc = cast[proc(a,b,c: pointer) {.cdecl.}](utilModule().symNullable("CRYPTO_set_mem_functions")) + if not theProc.isNil: theProc(a, b, c) + proc SSL_library_init*(): cint {.discardable.} = ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 otherwise ## SSL_library_init @@ -376,6 +415,10 @@ else: let theProc = cast[proc() {.cdecl.}](sslSymNullable("SSL_load_error_strings")) if not theProc.isNil: theProc() + proc ERR_load_BIO_strings*() = + let theProc = cast[proc() {.cdecl.}](utilModule().symNullable("ERR_load_BIO_strings")) + if not theProc.isNil: theProc() + proc SSLv23_client_method*(): PSSL_METHOD = loadPSSLMethod("SSLv23_client_method", "TLS_client_method") @@ -425,12 +468,10 @@ else: raiseInvalidLibrary MainProc proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint = - var theProc {.global.}: proc(ctx: SslCtx, str: cstring) {.cdecl, gcsafe.} + var theProc {.global.}: proc(ctx: SslCtx, str: cstring): cint {.cdecl, gcsafe.} if theProc.isNil: theProc = cast[typeof(theProc)](sslSymThrows("SSL_CTX_set_ciphersuites")) - theProc(ctx, str) - -proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} + result = theProc(ctx, str) proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} @@ -489,11 +530,8 @@ proc BIO_do_handshake*(bio: BIO): int = proc BIO_do_connect*(bio: BIO): int = return BIO_do_handshake(bio) -when not defined(nimfix): - proc BIO_read*(b: BIO, data: cstring, length: cint): cint{.cdecl, - dynlib: DLLUtilName, importc.} - proc BIO_write*(b: BIO, data: cstring, length: cint): cint{.cdecl, - dynlib: DLLUtilName, importc.} +proc BIO_read*(b: BIO, data: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc.} +proc BIO_write*(b: BIO, data: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc.} proc BIO_free*(b: BIO): cint{.cdecl, dynlib: DLLUtilName, importc.} @@ -511,16 +549,16 @@ proc OPENSSL_sk_num*(stack: PSTACK): int {.cdecl, dynlib: DLLSSLName, importc.} proc OPENSSL_sk_value*(stack: PSTACK, index: int): pointer {.cdecl, dynlib: DLLSSLName, importc.} -proc d2i_X509*(px: ptr PX509, i: ptr ptr cuchar, len: cint): PX509 {.cdecl, +proc d2i_X509*(px: ptr PX509, i: ptr ptr uint8, len: cint): PX509 {.cdecl, dynlib: DLLUtilName, importc.} -proc i2d_X509*(cert: PX509; o: ptr ptr cuchar): cint {.cdecl, +proc i2d_X509*(cert: PX509; o: ptr ptr uint8): cint {.cdecl, dynlib: DLLUtilName, importc.} proc d2i_X509*(b: string): PX509 = ## decode DER/BER bytestring into X.509 certificate struct var bb = b.cstring - let i = cast[ptr ptr cuchar](addr bb) + let i = cast[ptr ptr uint8](addr bb) let ret = d2i_X509(addr result, i, b.len.cint) if ret.isNil: raise newException(Exception, "X.509 certificate decoding failed") @@ -530,15 +568,15 @@ proc i2d_X509*(cert: PX509): string = let encoded_length = i2d_X509(cert, nil) result = newString(encoded_length) var q = result.cstring - let o = cast[ptr ptr cuchar](addr q) + let o = cast[ptr ptr uint8](addr q) let length = i2d_X509(cert, o) if length.int <= 0: raise newException(Exception, "X.509 certificate encoding failed") -when not useWinVersion and not defined(macosx) and not defined(android) and not defined(nimNoAllocForSSL): - proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, - dynlib: DLLUtilName, importc.} +const + useNimsAlloc = not defined(nimNoAllocForSSL) and not defined(gcDestructors) +when not useWinVersion and not defined(macosx) and not defined(android) and useNimsAlloc: proc allocWrapper(size: int): pointer {.cdecl.} = allocShared(size) proc reallocWrapper(p: pointer; newSize: int): pointer {.cdecl.} = if p == nil: @@ -548,9 +586,11 @@ when not useWinVersion and not defined(macosx) and not defined(android) and not proc deallocWrapper(p: pointer) {.cdecl.} = if p != nil: deallocShared(p) -proc CRYPTO_malloc_init*() = - when not useWinVersion and not defined(macosx) and not defined(android) and not defined(nimNoAllocForSSL): - CRYPTO_set_mem_functions(allocWrapper, reallocWrapper, deallocWrapper) + proc CRYPTO_malloc_init*() = + CRYPTO_set_mem_functions(cast[pointer](allocWrapper), cast[pointer](reallocWrapper), cast[pointer](deallocWrapper)) +else: + proc CRYPTO_malloc_init*() = + discard proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cint, larg: clong, parg: pointer): clong{. cdecl, dynlib: DLLSSLName, importc.} @@ -565,10 +605,9 @@ proc SSL_ctrl*(ssl: SslPtr, cmd: cint, larg: int, parg: pointer): int{. cdecl, dynlib: DLLSSLName, importc.} proc SSL_set_tlsext_host_name*(ssl: SslPtr, name: cstring): int = - result = SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, name) ## Set the SNI server name extension to be used in a client hello. ## Returns 1 if SNI was set, 0 if current SSL configuration doesn't support SNI. - + result = SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, name) proc SSL_get_servername*(ssl: SslPtr, typ: cint = TLSEXT_NAMETYPE_host_name): cstring {.cdecl, dynlib: DLLSSLName, importc.} ## Retrieve the server name requested in the client hello. This can be used @@ -591,11 +630,11 @@ proc SSL_CTX_set_tlsext_servername_arg*(ctx: SslCtx, arg: pointer): int = type PskClientCallback* = proc (ssl: SslPtr; - hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; + hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr uint8; max_psk_len: cuint): cuint {.cdecl.} PskServerCallback* = proc (ssl: SslPtr; - identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} + identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.} proc SSL_CTX_set_psk_client_callback*(ctx: SslCtx; callback: PskClientCallback) {.cdecl, dynlib: DLLSSLName, importc.} ## Set callback called when OpenSSL needs PSK (for client). @@ -664,60 +703,60 @@ proc PEM_read_bio_RSAPublicKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: po dynlib: DLLUtilName, importc.} proc PEM_read_bio_RSAPrivateKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: pointer): PRSA {.cdecl, dynlib: DLLUtilName, importc.} -proc RSA_private_encrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl, +proc RSA_private_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, dynlib: DLLUtilName, importc.} -proc RSA_public_encrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl, +proc RSA_public_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, dynlib: DLLUtilName, importc.} -proc RSA_private_decrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl, +proc RSA_private_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, dynlib: DLLUtilName, importc.} -proc RSA_public_decrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl, +proc RSA_public_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, dynlib: DLLUtilName, importc.} proc RSA_free*(rsa: PRSA) {.cdecl, dynlib: DLLUtilName, importc.} proc RSA_size*(rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.} # sha types -proc EVP_md_null*(): EVP_MD {.cdecl, importc.} -proc EVP_md2*(): EVP_MD {.cdecl, importc.} -proc EVP_md4*(): EVP_MD {.cdecl, importc.} -proc EVP_md5*(): EVP_MD {.cdecl, importc.} -proc EVP_sha*(): EVP_MD {.cdecl, importc.} -proc EVP_sha1*(): EVP_MD {.cdecl, importc.} -proc EVP_dss*(): EVP_MD {.cdecl, importc.} -proc EVP_dss1*(): EVP_MD {.cdecl, importc.} -proc EVP_ecdsa*(): EVP_MD {.cdecl, importc.} -proc EVP_sha224*(): EVP_MD {.cdecl, importc.} -proc EVP_sha256*(): EVP_MD {.cdecl, importc.} -proc EVP_sha384*(): EVP_MD {.cdecl, importc.} -proc EVP_sha512*(): EVP_MD {.cdecl, importc.} -proc EVP_mdc2*(): EVP_MD {.cdecl, importc.} -proc EVP_ripemd160*(): EVP_MD {.cdecl, importc.} -proc EVP_whirlpool*(): EVP_MD {.cdecl, importc.} -proc EVP_MD_size*(md: EVP_MD): cint {.cdecl, importc.} +proc EVP_md_null*(): EVP_MD {.lcrypto.} +proc EVP_md2*(): EVP_MD {.lcrypto.} +proc EVP_md4*(): EVP_MD {.lcrypto.} +proc EVP_md5*(): EVP_MD {.lcrypto.} +proc EVP_sha*(): EVP_MD {.lcrypto.} +proc EVP_sha1*(): EVP_MD {.lcrypto.} +proc EVP_dss*(): EVP_MD {.lcrypto.} +proc EVP_dss1*(): EVP_MD {.lcrypto.} +proc EVP_ecdsa*(): EVP_MD {.lcrypto.} +proc EVP_sha224*(): EVP_MD {.lcrypto.} +proc EVP_sha256*(): EVP_MD {.lcrypto.} +proc EVP_sha384*(): EVP_MD {.lcrypto.} +proc EVP_sha512*(): EVP_MD {.lcrypto.} +proc EVP_mdc2*(): EVP_MD {.lcrypto.} +proc EVP_ripemd160*(): EVP_MD {.lcrypto.} +proc EVP_whirlpool*(): EVP_MD {.lcrypto.} +proc EVP_MD_size*(md: EVP_MD): cint {.lcrypto.} # hmac functions -proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.cdecl, importc.} +proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.lcrypto.} # RSA key functions -proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.cdecl, importc.} -proc EVP_PKEY_free*(p: EVP_PKEY) {.cdecl, importc.} -proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.cdecl, importc.} -proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: PEVP_MD, engine: SslPtr = nil): cint {.cdecl, importc.} -proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.cdecl, importc.} -proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.cdecl, importc.} -proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.cdecl, importc.} -proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.cdecl, importc.} -proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.cdecl, importc.} -proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.cdecl, importc.} +proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.lcrypto.} +proc EVP_PKEY_free*(p: EVP_PKEY) {.lcrypto.} +proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.lcrypto.} +proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: EVP_MD, engine: SslPtr = nil): cint {.lcrypto.} +proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.lcrypto.} +proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.lcrypto.} +proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.lcrypto.} +proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.lcrypto.} +proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.lcrypto.} +proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.lcrypto.} when defined(macosx) or defined(windows): - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc.} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc.} - proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc.} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.lcrypto.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.lcrypto.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.lcrypto.} else: # some times you will need this instead: - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new".} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free".} - proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup".} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new", dynlib: DLLUtilName.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free", dynlib: DLLUtilName.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup", dynlib: DLLUtilName.} # <openssl/md5.h> type @@ -736,11 +775,11 @@ type proc md5_Init*(c: var MD5_CTX): cint{.importc: "MD5_Init".} proc md5_Update*(c: var MD5_CTX; data: pointer; len: csize_t): cint{.importc: "MD5_Update".} proc md5_Final*(md: cstring; c: var MD5_CTX): cint{.importc: "MD5_Final".} -proc md5*(d: ptr cuchar; n: csize_t; md: ptr cuchar): ptr cuchar{.importc: "MD5".} -proc md5_Transform*(c: var MD5_CTX; b: ptr cuchar){.importc: "MD5_Transform".} +proc md5*(d: ptr uint8; n: csize_t; md: ptr uint8): ptr uint8{.importc: "MD5".} +proc md5_Transform*(c: var MD5_CTX; b: ptr uint8){.importc: "MD5_Transform".} {.pop.} -from strutils import toHex, toLowerAscii +from std/strutils import toHex, toLowerAscii proc hexStr(buf: cstring): string = # turn md5s output into a nice hex str @@ -763,10 +802,10 @@ proc md5_File*(file: string): string {.raises: [IOError,Exception].} = while (let bytes = f.readChars(buf); bytes > 0): discard md5_Update(ctx, buf[0].addr, cast[csize_t](bytes)) - discard md5_Final(buf[0].addr, ctx) + discard md5_Final(cast[cstring](buf[0].addr), ctx) f.close - result = hexStr(addr buf) + result = hexStr(cast[cstring](addr buf)) proc md5_Str*(str: string): string = ## Generate MD5 hash for a string. Result is a 32 character @@ -783,8 +822,8 @@ proc md5_Str*(str: string): string = discard md5_Update(ctx, input[i].addr, cast[csize_t](L)) i += L - discard md5_Final(addr res, ctx) - result = hexStr(addr res) + discard md5_Final(cast[cstring](addr res), ctx) + result = hexStr(cast[cstring](addr res)) when defined(nimHasStyleChecks): {.pop.} @@ -794,8 +833,21 @@ when defined(nimHasStyleChecks): # On old openSSL version some of these symbols are not available when not defined(nimDisableCertificateValidation) and not defined(windows): - proc SSL_get_peer_certificate*(ssl: SslCtx): PX509{.cdecl, dynlib: DLLSSLName, - importc.} + # SSL_get_peer_certificate removed in 3.0 + # SSL_get1_peer_certificate added in 3.0 + when useOpenssl3: + proc SSL_get1_peer_certificate*(ssl: SslCtx): PX509 {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 = + SSL_get1_peer_certificate(ssl) + elif useStaticLink: + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 {.cdecl, dynlib: DLLSSLName, importc.} + else: + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 = + let methodSym = sslSymNullable("SSL_get_peer_certificate", "SSL_get1_peer_certificate") + if methodSym.isNil: + raise newException(LibraryError, "Could not load SSL_get_peer_certificate or SSL_get1_peer_certificate") + let method2Proc = cast[proc(ssl: SslCtx): PX509 {.cdecl, gcsafe, raises: [].}](methodSym) + return method2Proc(ssl) proc X509_get_subject_name*(a: PX509): PX509_NAME{.cdecl, dynlib: DLLSSLName, importc.} @@ -809,6 +861,8 @@ when not defined(nimDisableCertificateValidation) and not defined(windows): proc X509_check_host*(cert: PX509, name: cstring, namelen: cint, flags:cuint, peername: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + proc X509_free*(cert: PX509) {.cdecl, dynlib: DLLSSLName, importc.} + # Certificates store type PX509_STORE* = SslPtr @@ -832,8 +886,24 @@ when not defined(nimDisableCertificateValidation) and not defined(windows): {.pop.} when isMainModule: + when defined(nimPreviewSlimSystem): + import std/assertions # A simple certificate test let certbytes = readFile("certificate.der") let cert = d2i_X509(certbytes) let encoded = cert.i2d_X509() assert encoded == certbytes + +# Application Layer Protocol Negociation extension (TLS-ALPN, RFC7301) +# Available in at least OpenSSL 1.1.1 and later, not sure if earlier +# --Iced Quinn + +proc SSL_CTX_set_alpn_protos*(ctx: SslCtx; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_set_alpn_protos*(ssl: SslPtr; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_alpn_select_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_alpn_selected*(ssl: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_protos_advertised_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: ptr cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_proto_select_cb*(ctx: SslCtx; cb: proc(s: SslPtr; out_proto: cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_select_next_proto*(out_proto: ptr cstring; outlen: cstring; server: cstring; server_len: cuint; client: cstring; client_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_next_proto_negotiated*(s: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} + diff --git a/lib/wrappers/postgres.nim b/lib/wrappers/postgres.nim deleted file mode 100644 index 0bde3f1e6e..0000000000 --- a/lib/wrappers/postgres.nim +++ /dev/null @@ -1,378 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This module contains the definitions for structures and externs for -# functions used by frontend postgres applications. It is based on -# Postgresql's libpq-fe.h. -# -# It is for postgreSQL version 7.4 and higher with support for the v3.0 -# connection-protocol. -# - -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -when defined(windows): - const - dllName = "libpq.dll" -elif defined(macosx): - const - dllName = "libpq.dylib" -else: - const - dllName = "libpq.so(.5|)" -type - POid* = ptr Oid - Oid* = int32 - -const - ERROR_MSG_LENGTH* = 4096 - CMDSTATUS_LEN* = 40 - -type - SockAddr* = array[1..112, int8] - PGresAttDesc*{.pure, final.} = object - name*: cstring - adtid*: Oid - adtsize*: int - - PPGresAttDesc* = ptr PGresAttDesc - PPPGresAttDesc* = ptr PPGresAttDesc - PGresAttValue*{.pure, final.} = object - length*: int32 - value*: cstring - - PPGresAttValue* = ptr PGresAttValue - PPPGresAttValue* = ptr PPGresAttValue - PExecStatusType* = ptr ExecStatusType - ExecStatusType* = enum - PGRES_EMPTY_QUERY = 0, PGRES_COMMAND_OK, PGRES_TUPLES_OK, PGRES_COPY_OUT, - PGRES_COPY_IN, PGRES_BAD_RESPONSE, PGRES_NONFATAL_ERROR, PGRES_FATAL_ERROR, - PGRES_COPY_BOTH, PGRES_SINGLE_TUPLE - PGlobjfuncs*{.pure, final.} = object - fn_lo_open*: Oid - fn_lo_close*: Oid - fn_lo_creat*: Oid - fn_lo_unlink*: Oid - fn_lo_lseek*: Oid - fn_lo_tell*: Oid - fn_lo_read*: Oid - fn_lo_write*: Oid - - PPGlobjfuncs* = ptr PGlobjfuncs - PConnStatusType* = ptr ConnStatusType - ConnStatusType* = enum - CONNECTION_OK, CONNECTION_BAD, CONNECTION_STARTED, CONNECTION_MADE, - CONNECTION_AWAITING_RESPONSE, CONNECTION_AUTH_OK, CONNECTION_SETENV, - CONNECTION_SSL_STARTUP, CONNECTION_NEEDED, CONNECTION_CHECK_WRITABLE, - CONNECTION_CONSUME, CONNECTION_GSS_STARTUP, CONNECTION_CHECK_TARGET - PGconn*{.pure, final.} = object - pghost*: cstring - pgtty*: cstring - pgport*: cstring - pgoptions*: cstring - dbName*: cstring - status*: ConnStatusType - errorMessage*: array[0..(ERROR_MSG_LENGTH) - 1, char] - Pfin*: File - Pfout*: File - Pfdebug*: File - sock*: int32 - laddr*: SockAddr - raddr*: SockAddr - salt*: array[0..(2) - 1, char] - asyncNotifyWaiting*: int32 - notifyList*: pointer - pguser*: cstring - pgpass*: cstring - lobjfuncs*: PPGlobjfuncs - - PPGconn* = ptr PGconn - PGresult*{.pure, final.} = object - ntups*: int32 - numAttributes*: int32 - attDescs*: PPGresAttDesc - tuples*: PPPGresAttValue - tupArrSize*: int32 - resultStatus*: ExecStatusType - cmdStatus*: array[0..(CMDSTATUS_LEN) - 1, char] - binary*: int32 - conn*: PPGconn - - PPGresult* = ptr PGresult - PPostgresPollingStatusType* = ptr PostgresPollingStatusType - PostgresPollingStatusType* = enum - PGRES_POLLING_FAILED = 0, PGRES_POLLING_READING, PGRES_POLLING_WRITING, - PGRES_POLLING_OK, PGRES_POLLING_ACTIVE - PPGTransactionStatusType* = ptr PGTransactionStatusType - PGTransactionStatusType* = enum - PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, - PQTRANS_UNKNOWN - PPGVerbosity* = ptr PGVerbosity - PGVerbosity* = enum - PQERRORS_TERSE, PQERRORS_DEFAULT, PQERRORS_VERBOSE, PQERRORS_SQLSTATE - PPGNotify* = ptr pgNotify - pgNotify*{.pure, final.} = object - relname*: cstring - be_pid*: int32 - extra*: cstring - - PQnoticeReceiver* = proc (arg: pointer, res: PPGresult){.cdecl.} - PQnoticeProcessor* = proc (arg: pointer, message: cstring){.cdecl.} - Ppqbool* = ptr pqbool - pqbool* = char - PPQprintOpt* = ptr PQprintOpt - PQprintOpt*{.pure, final.} = object - header*: pqbool - align*: pqbool - standard*: pqbool - html3*: pqbool - expanded*: pqbool - pager*: pqbool - fieldSep*: cstring - tableOpt*: cstring - caption*: cstring - fieldName*: ptr cstring - - PPQconninfoOption* = ptr PQconninfoOption - PQconninfoOption*{.pure, final.} = object - keyword*: cstring - envvar*: cstring - compiled*: cstring - val*: cstring - label*: cstring - dispchar*: cstring - dispsize*: int32 - - PPQArgBlock* = ptr PQArgBlock - PQArgBlock*{.pure, final.} = object - length*: int32 - isint*: int32 - p*: pointer - -proc pqinitOpenSSL*(do_ssl: int32, do_crypto: int32) {.cdecl, dynlib: dllName, - importc: "PQinitOpenSSL".} -proc pqconnectStart*(conninfo: cstring): PPGconn{.cdecl, dynlib: dllName, - importc: "PQconnectStart".} -proc pqconnectPoll*(conn: PPGconn): PostgresPollingStatusType{.cdecl, - dynlib: dllName, importc: "PQconnectPoll".} -proc pqconnectdb*(conninfo: cstring): PPGconn{.cdecl, dynlib: dllName, - importc: "PQconnectdb".} -proc pqsetdbLogin*(pghost: cstring, pgport: cstring, pgoptions: cstring, - pgtty: cstring, dbName: cstring, login: cstring, pwd: cstring): PPGconn{. - cdecl, dynlib: dllName, importc: "PQsetdbLogin".} -proc pqsetdb*(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME: cstring): PPGconn -proc pqfinish*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQfinish".} -proc pqconndefaults*(): PPQconninfoOption{.cdecl, dynlib: dllName, - importc: "PQconndefaults".} -proc pqconninfoFree*(connOptions: PPQconninfoOption){.cdecl, dynlib: dllName, - importc: "PQconninfoFree".} -proc pqresetStart*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQresetStart".} -proc pqresetPoll*(conn: PPGconn): PostgresPollingStatusType{.cdecl, - dynlib: dllName, importc: "PQresetPoll".} -proc pqreset*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQreset".} -proc pqrequestCancel*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQrequestCancel".} -proc pqdb*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQdb".} -proc pquser*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQuser".} -proc pqpass*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQpass".} -proc pqhost*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQhost".} -proc pqport*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQport".} -proc pqtty*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQtty".} -proc pqoptions*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, - importc: "PQoptions".} -proc pqstatus*(conn: PPGconn): ConnStatusType{.cdecl, dynlib: dllName, - importc: "PQstatus".} -proc pqtransactionStatus*(conn: PPGconn): PGTransactionStatusType{.cdecl, - dynlib: dllName, importc: "PQtransactionStatus".} -proc pqparameterStatus*(conn: PPGconn, paramName: cstring): cstring{.cdecl, - dynlib: dllName, importc: "PQparameterStatus".} -proc pqserverVersion*(conn: PPGconn): int32{.cdecl, - dynlib: dllName, importc: "PQserverVersion".} -proc pqprotocolVersion*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQprotocolVersion".} -proc pqerrorMessage*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, - importc: "PQerrorMessage".} -proc pqsocket*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQsocket".} -proc pqbackendPID*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQbackendPID".} -proc pqconnectionNeedsPassword*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQconnectionNeedsPassword".} -proc pqconnectionUsedPassword*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQconnectionUsedPassword".} -proc pqclientEncoding*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQclientEncoding".} -proc pqsetClientEncoding*(conn: PPGconn, encoding: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQsetClientEncoding".} -when defined(USE_SSL): - # Get the SSL structure associated with a connection - proc pqgetssl*(conn: PPGconn): PSSL{.cdecl, dynlib: dllName, - importc: "PQgetssl".} -proc pqsetErrorVerbosity*(conn: PPGconn, verbosity: PGVerbosity): PGVerbosity{. - cdecl, dynlib: dllName, importc: "PQsetErrorVerbosity".} -proc pqtrace*(conn: PPGconn, debug_port: File){.cdecl, dynlib: dllName, - importc: "PQtrace".} -proc pquntrace*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQuntrace".} -proc pqsetNoticeReceiver*(conn: PPGconn, theProc: PQnoticeReceiver, arg: pointer): PQnoticeReceiver{. - cdecl, dynlib: dllName, importc: "PQsetNoticeReceiver".} -proc pqsetNoticeProcessor*(conn: PPGconn, theProc: PQnoticeProcessor, - arg: pointer): PQnoticeProcessor{.cdecl, - dynlib: dllName, importc: "PQsetNoticeProcessor".} -proc pqexec*(conn: PPGconn, query: cstring): PPGresult{.cdecl, dynlib: dllName, - importc: "PQexec".} -proc pqexecParams*(conn: PPGconn, command: cstring, nParams: int32, - paramTypes: POid, paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQexecParams".} -proc pqprepare*(conn: PPGconn, stmtName, query: cstring, nParams: int32, - paramTypes: POid): PPGresult{.cdecl, dynlib: dllName, importc: "PQprepare".} -proc pqexecPrepared*(conn: PPGconn, stmtName: cstring, nParams: int32, - paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQexecPrepared".} -proc pqsendQuery*(conn: PPGconn, query: cstring): int32{.cdecl, dynlib: dllName, - importc: "PQsendQuery".} - ## See also https://www.postgresql.org/docs/current/libpq-async.html -proc pqsendQueryParams*(conn: PPGconn, command: cstring, nParams: int32, - paramTypes: POid, paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, - resultFormat: int32): int32{.cdecl, dynlib: dllName, - importc: "PQsendQueryParams".} -proc pqsendQueryPrepared*(conn: PPGconn, stmtName: cstring, nParams: int32, - paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, - resultFormat: int32): int32{.cdecl, dynlib: dllName, - importc: "PQsendQueryPrepared".} -proc pqSetSingleRowMode*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQsetSingleRowMode".} - ## See also https://www.postgresql.org/docs/current/libpq-single-row-mode.html -proc pqgetResult*(conn: PPGconn): PPGresult{.cdecl, dynlib: dllName, - importc: "PQgetResult".} -proc pqisBusy*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQisBusy".} -proc pqconsumeInput*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQconsumeInput".} -proc pqnotifies*(conn: PPGconn): PPGNotify{.cdecl, dynlib: dllName, - importc: "PQnotifies".} -proc pqputCopyData*(conn: PPGconn, buffer: cstring, nbytes: int32): int32{. - cdecl, dynlib: dllName, importc: "PQputCopyData".} -proc pqputCopyEnd*(conn: PPGconn, errormsg: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQputCopyEnd".} -proc pqgetCopyData*(conn: PPGconn, buffer: cstringArray, async: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetCopyData".} -proc pqgetline*(conn: PPGconn, str: cstring, len: int32): int32{.cdecl, - dynlib: dllName, importc: "PQgetline".} -proc pqputline*(conn: PPGconn, str: cstring): int32{.cdecl, dynlib: dllName, - importc: "PQputline".} -proc pqgetlineAsync*(conn: PPGconn, buffer: cstring, bufsize: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetlineAsync".} -proc pqputnbytes*(conn: PPGconn, buffer: cstring, nbytes: int32): int32{.cdecl, - dynlib: dllName, importc: "PQputnbytes".} -proc pqendcopy*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQendcopy".} -proc pqsetnonblocking*(conn: PPGconn, arg: int32): int32{.cdecl, - dynlib: dllName, importc: "PQsetnonblocking".} -proc pqisnonblocking*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQisnonblocking".} -proc pqflush*(conn: PPGconn): int32{.cdecl, dynlib: dllName, importc: "PQflush".} -proc pqfn*(conn: PPGconn, fnid: int32, result_buf, result_len: ptr int32, - result_is_int: int32, args: PPQArgBlock, nargs: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQfn".} -proc pqresultStatus*(res: PPGresult): ExecStatusType{.cdecl, dynlib: dllName, - importc: "PQresultStatus".} -proc pqresStatus*(status: ExecStatusType): cstring{.cdecl, dynlib: dllName, - importc: "PQresStatus".} -proc pqresultErrorMessage*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQresultErrorMessage".} -proc pqresultErrorField*(res: PPGresult, fieldcode: int32): cstring{.cdecl, - dynlib: dllName, importc: "PQresultErrorField".} -proc pqntuples*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQntuples".} -proc pqnfields*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQnfields".} -proc pqbinaryTuples*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQbinaryTuples".} -proc pqfname*(res: PPGresult, field_num: int32): cstring{.cdecl, - dynlib: dllName, importc: "PQfname".} -proc pqfnumber*(res: PPGresult, field_name: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQfnumber".} -proc pqftable*(res: PPGresult, field_num: int32): Oid{.cdecl, dynlib: dllName, - importc: "PQftable".} -proc pqftablecol*(res: PPGresult, field_num: int32): int32{.cdecl, - dynlib: dllName, importc: "PQftablecol".} -proc pqfformat*(res: PPGresult, field_num: int32): int32{.cdecl, - dynlib: dllName, importc: "PQfformat".} -proc pqftype*(res: PPGresult, field_num: int32): Oid{.cdecl, dynlib: dllName, - importc: "PQftype".} -proc pqfsize*(res: PPGresult, field_num: int32): int32{.cdecl, dynlib: dllName, - importc: "PQfsize".} -proc pqfmod*(res: PPGresult, field_num: int32): int32{.cdecl, dynlib: dllName, - importc: "PQfmod".} -proc pqcmdStatus*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQcmdStatus".} -proc pqoidStatus*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQoidStatus".} -proc pqoidValue*(res: PPGresult): Oid{.cdecl, dynlib: dllName, - importc: "PQoidValue".} -proc pqcmdTuples*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQcmdTuples".} -proc pqgetvalue*(res: PPGresult, tup_num: int32, field_num: int32): cstring{. - cdecl, dynlib: dllName, importc: "PQgetvalue".} -proc pqgetlength*(res: PPGresult, tup_num: int32, field_num: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetlength".} -proc pqgetisnull*(res: PPGresult, tup_num: int32, field_num: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetisnull".} -proc pqclear*(res: PPGresult){.cdecl, dynlib: dllName, importc: "PQclear".} -proc pqfreemem*(p: pointer){.cdecl, dynlib: dllName, importc: "PQfreemem".} -proc pqmakeEmptyPGresult*(conn: PPGconn, status: ExecStatusType): PPGresult{. - cdecl, dynlib: dllName, importc: "PQmakeEmptyPGresult".} -proc pqescapeString*(till, `from`: cstring, len: int): int{.cdecl, - dynlib: dllName, importc: "PQescapeString".} -proc pqescapeBytea*(bintext: cstring, binlen: int, bytealen: var int): cstring{. - cdecl, dynlib: dllName, importc: "PQescapeBytea".} -proc pqunescapeBytea*(strtext: cstring, retbuflen: var int): cstring{.cdecl, - dynlib: dllName, importc: "PQunescapeBytea".} -proc pqprint*(fout: File, res: PPGresult, ps: PPQprintOpt){.cdecl, - dynlib: dllName, importc: "PQprint".} -proc pqdisplayTuples*(res: PPGresult, fp: File, fillAlign: int32, - fieldSep: cstring, printHeader: int32, quiet: int32){. - cdecl, dynlib: dllName, importc: "PQdisplayTuples".} -proc pqprintTuples*(res: PPGresult, fout: File, printAttName: int32, - terseOutput: int32, width: int32){.cdecl, dynlib: dllName, - importc: "PQprintTuples".} -proc lo_open*(conn: PPGconn, lobjId: Oid, mode: int32): int32{.cdecl, - dynlib: dllName, importc: "lo_open".} -proc lo_close*(conn: PPGconn, fd: int32): int32{.cdecl, dynlib: dllName, - importc: "lo_close".} -proc lo_read*(conn: PPGconn, fd: int32, buf: cstring, length: int): int32{. - cdecl, dynlib: dllName, importc: "lo_read".} -proc lo_write*(conn: PPGconn, fd: int32, buf: cstring, length: int): int32{. - cdecl, dynlib: dllName, importc: "lo_write".} -proc lo_lseek*(conn: PPGconn, fd: int32, offset: int32, whence: int32): int32{. - cdecl, dynlib: dllName, importc: "lo_lseek".} -proc lo_creat*(conn: PPGconn, mode: int32): Oid{.cdecl, dynlib: dllName, - importc: "lo_creat".} -proc lo_tell*(conn: PPGconn, fd: int32): int32{.cdecl, dynlib: dllName, - importc: "lo_tell".} -proc lo_unlink*(conn: PPGconn, lobjId: Oid): int32{.cdecl, dynlib: dllName, - importc: "lo_unlink".} -proc lo_import*(conn: PPGconn, filename: cstring): Oid{.cdecl, dynlib: dllName, - importc: "lo_import".} -proc lo_export*(conn: PPGconn, lobjId: Oid, filename: cstring): int32{.cdecl, - dynlib: dllName, importc: "lo_export".} -proc pqmblen*(s: cstring, encoding: int32): int32{.cdecl, dynlib: dllName, - importc: "PQmblen".} -proc pqenv2encoding*(): int32{.cdecl, dynlib: dllName, importc: "PQenv2encoding".} -proc pqsetdb(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME: cstring): PPGconn = - result = pqsetdbLogin(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME, "", "") - -when defined(nimHasStyleChecks): - {.pop.} diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim deleted file mode 100644 index 71921c10b7..0000000000 --- a/lib/wrappers/sqlite3.nim +++ /dev/null @@ -1,392 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -when defined(windows): - when defined(nimOldDlls): - const Lib = "sqlite3.dll" - elif defined(cpu64): - const Lib = "sqlite3_64.dll" - else: - const Lib = "sqlite3_32.dll" -elif defined(macosx): - const - Lib = "libsqlite3(|.0).dylib" -else: - const - Lib = "libsqlite3.so(|.0)" - -when defined(staticSqlite): - {.pragma: mylib.} - {.compile("sqlite3.c", "-O3").} -else: - {.pragma: mylib, dynlib: Lib.} - -const - SQLITE_INTEGER* = 1 - SQLITE_FLOAT* = 2 - SQLITE_BLOB* = 4 - SQLITE_NULL* = 5 - SQLITE_TEXT* = 3 - SQLITE_UTF8* = 1 - SQLITE_UTF16LE* = 2 - SQLITE_UTF16BE* = 3 # Use native byte order - SQLITE_UTF16* = 4 # sqlite3_create_function only - SQLITE_ANY* = 5 #sqlite_exec return values - SQLITE_OK* = 0 - SQLITE_ERROR* = 1 # SQL error or missing database - SQLITE_INTERNAL* = 2 # An internal logic error in SQLite - SQLITE_PERM* = 3 # Access permission denied - SQLITE_ABORT* = 4 # Callback routine requested an abort - SQLITE_BUSY* = 5 # The database file is locked - SQLITE_LOCKED* = 6 # A table in the database is locked - SQLITE_NOMEM* = 7 # A malloc() failed - SQLITE_READONLY* = 8 # Attempt to write a readonly database - SQLITE_INTERRUPT* = 9 # Operation terminated by sqlite3_interrupt() - SQLITE_IOERR* = 10 # Some kind of disk I/O error occurred - SQLITE_CORRUPT* = 11 # The database disk image is malformed - SQLITE_NOTFOUND* = 12 # (Internal Only) Table or record not found - SQLITE_FULL* = 13 # Insertion failed because database is full - SQLITE_CANTOPEN* = 14 # Unable to open the database file - SQLITE_PROTOCOL* = 15 # Database lock protocol error - SQLITE_EMPTY* = 16 # Database is empty - SQLITE_SCHEMA* = 17 # The database schema changed - SQLITE_TOOBIG* = 18 # Too much data for one row of a table - SQLITE_CONSTRAINT* = 19 # Abort due to constraint violation - SQLITE_MISMATCH* = 20 # Data type mismatch - SQLITE_MISUSE* = 21 # Library used incorrectly - SQLITE_NOLFS* = 22 # Uses OS features not supported on host - SQLITE_AUTH* = 23 # Authorization denied - SQLITE_FORMAT* = 24 # Auxiliary database format error - SQLITE_RANGE* = 25 # 2nd parameter to sqlite3_bind out of range - SQLITE_NOTADB* = 26 # File opened that is not a database file - SQLITE_ROW* = 100 # sqlite3_step() has another row ready - SQLITE_DONE* = 101 # sqlite3_step() has finished executing - SQLITE_COPY* = 0 - SQLITE_CREATE_INDEX* = 1 - SQLITE_CREATE_TABLE* = 2 - SQLITE_CREATE_TEMP_INDEX* = 3 - SQLITE_CREATE_TEMP_TABLE* = 4 - SQLITE_CREATE_TEMP_TRIGGER* = 5 - SQLITE_CREATE_TEMP_VIEW* = 6 - SQLITE_CREATE_TRIGGER* = 7 - SQLITE_CREATE_VIEW* = 8 - SQLITE_DELETE* = 9 - SQLITE_DROP_INDEX* = 10 - SQLITE_DROP_TABLE* = 11 - SQLITE_DROP_TEMP_INDEX* = 12 - SQLITE_DROP_TEMP_TABLE* = 13 - SQLITE_DROP_TEMP_TRIGGER* = 14 - SQLITE_DROP_TEMP_VIEW* = 15 - SQLITE_DROP_TRIGGER* = 16 - SQLITE_DROP_VIEW* = 17 - SQLITE_INSERT* = 18 - SQLITE_PRAGMA* = 19 - SQLITE_READ* = 20 - SQLITE_SELECT* = 21 - SQLITE_TRANSACTION* = 22 - SQLITE_UPDATE* = 23 - SQLITE_ATTACH* = 24 - SQLITE_DETACH* = 25 - SQLITE_ALTER_TABLE* = 26 - SQLITE_REINDEX* = 27 - SQLITE_DENY* = 1 - SQLITE_IGNORE* = 2 # Original from sqlite3.h: - #define SQLITE_STATIC ((void(*)(void *))0) - #define SQLITE_TRANSIENT ((void(*)(void *))-1) - SQLITE_DETERMINISTIC* = 0x800 - -type - Sqlite3 {.pure, final.} = object - PSqlite3* = ptr Sqlite3 - PPSqlite3* = ptr PSqlite3 - Sqlite3_Backup {.pure, final.} = object - PSqlite3_Backup* = ptr Sqlite3_Backup - PPSqlite3_Backup* = ptr PSqlite3_Backup - Context{.pure, final.} = object - Pcontext* = ptr Context - TStmt{.pure, final.} = object - PStmt* = ptr TStmt - Value{.pure, final.} = object - PValue* = ptr Value - PValueArg* = array[0..127, PValue] - - Callback* = proc (para1: pointer, para2: int32, para3, - para4: cstringArray): int32{.cdecl.} - Tbind_destructor_func* = proc (para1: pointer){.cdecl, locks: 0, tags: [], gcsafe.} - Create_function_step_func* = proc (para1: Pcontext, para2: int32, - para3: PValueArg){.cdecl.} - Create_function_func_func* = proc (para1: Pcontext, para2: int32, - para3: PValueArg){.cdecl.} - Create_function_final_func* = proc (para1: Pcontext){.cdecl.} - Result_func* = proc (para1: pointer){.cdecl.} - Create_collation_func* = proc (para1: pointer, para2: int32, para3: pointer, - para4: int32, para5: pointer): int32{.cdecl.} - Collation_needed_func* = proc (para1: pointer, para2: PSqlite3, eTextRep: int32, - para4: cstring){.cdecl.} - -const - SQLITE_STATIC* = nil - SQLITE_TRANSIENT* = cast[Tbind_destructor_func](-1) - -proc close*(para1: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_close".} -proc exec*(para1: PSqlite3, sql: cstring, para3: Callback, para4: pointer, - errmsg: var cstring): int32{.cdecl, mylib, - importc: "sqlite3_exec".} -proc last_insert_rowid*(para1: PSqlite3): int64{.cdecl, mylib, - importc: "sqlite3_last_insert_rowid".} -proc changes*(para1: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_changes".} -proc total_changes*(para1: PSqlite3): int32{.cdecl, mylib, - importc: "sqlite3_total_changes".} -proc interrupt*(para1: PSqlite3){.cdecl, mylib, importc: "sqlite3_interrupt".} -proc complete*(sql: cstring): int32{.cdecl, mylib, - importc: "sqlite3_complete".} -proc complete16*(sql: pointer): int32{.cdecl, mylib, - importc: "sqlite3_complete16".} -proc busy_handler*(para1: PSqlite3, - para2: proc (para1: pointer, para2: int32): int32{.cdecl.}, - para3: pointer): int32{.cdecl, mylib, - importc: "sqlite3_busy_handler".} -proc busy_timeout*(para1: PSqlite3, ms: int32): int32{.cdecl, mylib, - importc: "sqlite3_busy_timeout".} -proc get_table*(para1: PSqlite3, sql: cstring, resultp: var cstringArray, - nrow, ncolumn: var cint, errmsg: ptr cstring): int32{.cdecl, - mylib, importc: "sqlite3_get_table".} -proc free_table*(result: cstringArray){.cdecl, mylib, - importc: "sqlite3_free_table".} - # Todo: see how translate sqlite3_mprintf, sqlite3_vmprintf, sqlite3_snprintf - # function sqlite3_mprintf(_para1:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_mprintf'; -proc mprintf*(para1: cstring): cstring{.cdecl, varargs, mylib, - importc: "sqlite3_mprintf".} - #function sqlite3_vmprintf(_para1:Pchar; _para2:va_list):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_vmprintf'; -proc free*(z: cstring){.cdecl, mylib, importc: "sqlite3_free".} - #function sqlite3_snprintf(_para1:longint; _para2:Pchar; _para3:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_snprintf'; -proc snprintf*(para1: int32, para2: cstring, para3: cstring): cstring{.cdecl, - mylib, varargs, importc: "sqlite3_snprintf".} -proc set_authorizer*(para1: PSqlite3, xAuth: proc (para1: pointer, para2: int32, - para3: cstring, para4: cstring, para5: cstring, para6: cstring): int32{. - cdecl.}, pUserData: pointer): int32{.cdecl, mylib, - importc: "sqlite3_set_authorizer".} -proc trace*(para1: PSqlite3, xTrace: proc (para1: pointer, para2: cstring){.cdecl.}, - para3: pointer): pointer{.cdecl, mylib, - importc: "sqlite3_trace".} -proc progress_handler*(para1: PSqlite3, para2: int32, - para3: proc (para1: pointer): int32{.cdecl.}, - para4: pointer){.cdecl, mylib, - importc: "sqlite3_progress_handler".} -proc commit_hook*(para1: PSqlite3, para2: proc (para1: pointer): int32{.cdecl.}, - para3: pointer): pointer{.cdecl, mylib, - importc: "sqlite3_commit_hook".} -proc open*(filename: cstring, ppDb: var PSqlite3): int32{.cdecl, mylib, - importc: "sqlite3_open".} -proc open16*(filename: pointer, ppDb: var PSqlite3): int32{.cdecl, mylib, - importc: "sqlite3_open16".} -proc errcode*(db: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_errcode".} -proc errmsg*(para1: PSqlite3): cstring{.cdecl, mylib, importc: "sqlite3_errmsg".} -proc errmsg16*(para1: PSqlite3): pointer{.cdecl, mylib, - importc: "sqlite3_errmsg16".} -proc prepare*(db: PSqlite3, zSql: cstring, nBytes: int32, ppStmt: var PStmt, - pzTail: ptr cstring): int32{.cdecl, mylib, - importc: "sqlite3_prepare".} - -proc prepare_v2*(db: PSqlite3, zSql: cstring, nByte: cint, ppStmt: var PStmt, - pzTail: ptr cstring): cint {. - importc: "sqlite3_prepare_v2", cdecl, mylib.} - -proc prepare16*(db: PSqlite3, zSql: pointer, nBytes: int32, ppStmt: var PStmt, - pzTail: var pointer): int32{.cdecl, mylib, - importc: "sqlite3_prepare16".} -proc bind_blob*(para1: PStmt, para2: int32, para3: pointer, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, mylib, - importc: "sqlite3_bind_blob".} -proc bind_double*(para1: PStmt, para2: int32, para3: float64): int32{.cdecl, - mylib, importc: "sqlite3_bind_double".} -proc bind_int*(para1: PStmt, para2: int32, para3: int32): int32{.cdecl, - mylib, importc: "sqlite3_bind_int".} -proc bind_int64*(para1: PStmt, para2: int32, para3: int64): int32{.cdecl, - mylib, importc: "sqlite3_bind_int64".} -proc bind_null*(para1: PStmt, para2: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_null".} -proc bind_text*(para1: PStmt, para2: int32, para3: cstring, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, mylib, - importc: "sqlite3_bind_text".} -proc bind_text16*(para1: PStmt, para2: int32, para3: pointer, para4: int32, - para5: Tbind_destructor_func): int32{.cdecl, mylib, - importc: "sqlite3_bind_text16".} - #function sqlite3_bind_value(_para1:Psqlite3_stmt; _para2:longint; _para3:Psqlite3_value):longint;cdecl; external Sqlite3Lib name 'sqlite3_bind_value'; - #These overloaded functions were introduced to allow the use of SQLITE_STATIC and SQLITE_TRANSIENT - #It's the c world man ;-) -proc bind_blob*(para1: PStmt, para2: int32, para3: pointer, n: int32, - para5: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_blob".} -proc bind_text*(para1: PStmt, para2: int32, para3: cstring, n: int32, - para5: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_text".} -proc bind_text16*(para1: PStmt, para2: int32, para3: pointer, para4: int32, - para5: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_text16".} -proc bind_parameter_count*(para1: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_bind_parameter_count".} -proc bind_parameter_name*(para1: PStmt, para2: int32): cstring{.cdecl, - mylib, importc: "sqlite3_bind_parameter_name".} -proc bind_parameter_index*(para1: PStmt, zName: cstring): int32{.cdecl, - mylib, importc: "sqlite3_bind_parameter_index".} -proc clear_bindings*(para1: PStmt): int32 {.cdecl, - mylib, importc: "sqlite3_clear_bindings".} -proc column_count*(PStmt: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_column_count".} -proc column_name*(para1: PStmt, para2: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_name".} -proc column_table_name*(para1: PStmt; para2: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_table_name".} -proc column_name16*(para1: PStmt, para2: int32): pointer{.cdecl, mylib, - importc: "sqlite3_column_name16".} -proc column_decltype*(para1: PStmt, i: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_decltype".} -proc column_decltype16*(para1: PStmt, para2: int32): pointer{.cdecl, - mylib, importc: "sqlite3_column_decltype16".} -proc step*(para1: PStmt): int32{.cdecl, mylib, importc: "sqlite3_step".} -proc data_count*(PStmt: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_data_count".} -proc column_blob*(para1: PStmt, iCol: int32): pointer{.cdecl, mylib, - importc: "sqlite3_column_blob".} -proc column_bytes*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_bytes".} -proc column_bytes16*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_bytes16".} -proc column_double*(para1: PStmt, iCol: int32): float64{.cdecl, mylib, - importc: "sqlite3_column_double".} -proc column_int*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_int".} -proc column_int64*(para1: PStmt, iCol: int32): int64{.cdecl, mylib, - importc: "sqlite3_column_int64".} -proc column_text*(para1: PStmt, iCol: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_text".} -proc column_text16*(para1: PStmt, iCol: int32): pointer{.cdecl, mylib, - importc: "sqlite3_column_text16".} -proc column_type*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_type".} -proc finalize*(PStmt: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_finalize".} -proc reset*(PStmt: PStmt): int32{.cdecl, mylib, importc: "sqlite3_reset".} -proc create_function*(para1: PSqlite3, zFunctionName: cstring, nArg: int32, - eTextRep: int32, para5: pointer, - xFunc: Create_function_func_func, - xStep: Create_function_step_func, - xFinal: Create_function_final_func): int32{.cdecl, - mylib, importc: "sqlite3_create_function".} -proc create_function16*(para1: PSqlite3, zFunctionName: pointer, nArg: int32, - eTextRep: int32, para5: pointer, - xFunc: Create_function_func_func, - xStep: Create_function_step_func, - xFinal: Create_function_final_func): int32{.cdecl, - mylib, importc: "sqlite3_create_function16".} -proc aggregate_count*(para1: Pcontext): int32{.cdecl, mylib, - importc: "sqlite3_aggregate_count".} -proc value_blob*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_blob".} -proc value_bytes*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_bytes".} -proc value_bytes16*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_bytes16".} -proc value_double*(para1: PValue): float64{.cdecl, mylib, - importc: "sqlite3_value_double".} -proc value_int*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_int".} -proc value_int64*(para1: PValue): int64{.cdecl, mylib, - importc: "sqlite3_value_int64".} -proc value_text*(para1: PValue): cstring{.cdecl, mylib, - importc: "sqlite3_value_text".} -proc value_text16*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_text16".} -proc value_text16le*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_text16le".} -proc value_text16be*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_text16be".} -proc value_type*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_type".} -proc aggregate_context*(para1: Pcontext, nBytes: int32): pointer{.cdecl, - mylib, importc: "sqlite3_aggregate_context".} -proc user_data*(para1: Pcontext): pointer{.cdecl, mylib, - importc: "sqlite3_user_data".} -proc get_auxdata*(para1: Pcontext, para2: int32): pointer{.cdecl, mylib, - importc: "sqlite3_get_auxdata".} -proc set_auxdata*(para1: Pcontext, para2: int32, para3: pointer, - para4: proc (para1: pointer){.cdecl.}){.cdecl, mylib, - importc: "sqlite3_set_auxdata".} -proc result_blob*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_blob".} -proc result_double*(para1: Pcontext, para2: float64){.cdecl, mylib, - importc: "sqlite3_result_double".} -proc result_error*(para1: Pcontext, para2: cstring, para3: int32){.cdecl, - mylib, importc: "sqlite3_result_error".} -proc result_error16*(para1: Pcontext, para2: pointer, para3: int32){.cdecl, - mylib, importc: "sqlite3_result_error16".} -proc result_int*(para1: Pcontext, para2: int32){.cdecl, mylib, - importc: "sqlite3_result_int".} -proc result_int64*(para1: Pcontext, para2: int64){.cdecl, mylib, - importc: "sqlite3_result_int64".} -proc result_null*(para1: Pcontext){.cdecl, mylib, - importc: "sqlite3_result_null".} -proc result_text*(para1: Pcontext, para2: cstring, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text".} -proc result_text16*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text16".} -proc result_text16le*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text16le".} -proc result_text16be*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text16be".} -proc result_value*(para1: Pcontext, para2: PValue){.cdecl, mylib, - importc: "sqlite3_result_value".} -proc create_collation*(para1: PSqlite3, zName: cstring, eTextRep: int32, - para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, mylib, importc: "sqlite3_create_collation".} -proc create_collation16*(para1: PSqlite3, zName: cstring, eTextRep: int32, - para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, mylib, importc: "sqlite3_create_collation16".} -proc collation_needed*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, mylib, importc: "sqlite3_collation_needed".} -proc collation_needed16*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, mylib, importc: "sqlite3_collation_needed16".} -proc libversion*(): cstring{.cdecl, mylib, importc: "sqlite3_libversion".} - #Alias for allowing better code portability (win32 is not working with external variables) -proc version*(): cstring{.cdecl, mylib, importc: "sqlite3_libversion".} - # Not published functions -proc libversion_number*(): int32{.cdecl, mylib, - importc: "sqlite3_libversion_number".} - -proc backup_init*(pDest: PSqlite3, zDestName: cstring, pSource: PSqlite3, zSourceName: cstring): PSqlite3_Backup {. - cdecl, mylib, importc: "sqlite3_backup_init".} - -proc backup_step*(pBackup: PSqlite3_Backup, nPage: int32): int32 {.cdecl, mylib, importc: "sqlite3_backup_step".} - -proc backup_finish*(pBackup: PSqlite3_Backup): int32 {.cdecl, mylib, importc: "sqlite3_backup_finish".} - -proc backup_pagecount*(pBackup: PSqlite3_Backup): int32 {.cdecl, mylib, importc: "sqlite3_backup_pagecount".} - -proc backup_remaining*(pBackup: PSqlite3_Backup): int32 {.cdecl, mylib, importc: "sqlite3_backup_remaining".} - -proc sqlite3_sleep*(t: int64): int64 {.cdecl, mylib, importc: "sqlite3_sleep".} - - #function sqlite3_key(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_key'; - #function sqlite3_rekey(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_rekey'; - #function sqlite3_sleep(_para1:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_sleep'; - #function sqlite3_expired(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_expired'; - #function sqlite3_global_recover:longint;cdecl; external Sqlite3Lib name 'sqlite3_global_recover'; -# implementation - -when defined(nimHasStyleChecks): - {.pop.} diff --git a/nim.nimble b/nim.nimble new file mode 100644 index 0000000000..bf195b0faf --- /dev/null +++ b/nim.nimble @@ -0,0 +1,17 @@ +include "lib/system/compilation.nim" +version = $NimMajor & "." & $NimMinor & "." & $NimPatch +author = "Andreas Rumpf" +description = "Nim package providing the compiler binary" +license = "MIT" + +bin = @["compiler/nim", "nimsuggest/nimsuggest"] +skipFiles = @["azure-pipelines.yml" , "build_all.bat" , "build_all.sh" , "build_nimble.bat" , "build_nimble.sh" , "changelog.md" , "koch.nim.cfg" , "nimblemeta.json" , "readme.md" , "security.md" ] +skipDirs = @["build" , "changelogs" , "ci" , "csources_v2" , "drnim" , "nimdoc", "testament"] + +before install: + when defined(windows): + if not "bin\nim.exe".fileExists: + exec "build_all.bat" + else: + if not "bin/nim".fileExists: + exec "./build_all.sh" diff --git a/nimdoc/extlinks/project/doc/manual.md b/nimdoc/extlinks/project/doc/manual.md new file mode 100644 index 0000000000..d44b5ca399 --- /dev/null +++ b/nimdoc/extlinks/project/doc/manual.md @@ -0,0 +1,17 @@ +=================== +Nothing User Manual +=================== + +.. importdoc:: ../main.nim, .. / sub / submodule.nim, ../../util.nim + +First section +============= + +Second *section* & +================== + +Ref. [`</a>`] or [submoduleInt] from [module nimdoc/extlinks/project/sub/submodule]. + +Ref. [proc mainfunction*(): int]. + +Ref. [utilfunction(x: int)]. diff --git a/nimdoc/extlinks/project/expected/_._/util.html b/nimdoc/extlinks/project/expected/_._/util.html new file mode 100644 index 0000000000..32dab9216a --- /dev/null +++ b/nimdoc/extlinks/project/expected/_._/util.html @@ -0,0 +1,105 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<!-- This file is generated by Nim. --> +<html xmlns="https://www.w3.org/1999/xhtml" xml:lang="en" lang="en" data-theme="auto"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="viewport" content="width=device-width, initial-scale=1.0"> +<title>nimdoc/extlinks/util + + + + + + + + + + + + + + + + +
    +
    +

    nimdoc/extlinks/util

    +
    +
    +
    + + +
    + +
    + Search: +
    +
    + Group by: + +
    + + +
    +
    + +
    + +

    +
    +

    Procs

    +
    +
    +
    + 
    proc utilfunction(x: int): int {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    + +
    + +
    +
    + +
    +
    + +
    + +
    + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
    +
    +
    + + + diff --git a/nimdoc/extlinks/project/expected/_._/util.idx b/nimdoc/extlinks/project/expected/_._/util.idx new file mode 100644 index 0000000000..d83d8c97db --- /dev/null +++ b/nimdoc/extlinks/project/expected/_._/util.idx @@ -0,0 +1,2 @@ +nimTitle util _._/util.html module nimdoc/extlinks/util 0 +nim utilfunction _._/util.html#utilfunction,int proc utilfunction(x: int): int 1 diff --git a/nimdoc/extlinks/project/expected/doc/manual.html b/nimdoc/extlinks/project/expected/doc/manual.html new file mode 100644 index 0000000000..2946f803ab --- /dev/null +++ b/nimdoc/extlinks/project/expected/doc/manual.html @@ -0,0 +1,45 @@ + + + + + + + +Nothing User Manual + + + + + + + + + + + + + + + + +
    +
    +

    Nothing User Manual

    + +

    First section

    +

    Second section &

    Ref. submodule: `</a>` or submodule: submoduleInt from module nimdoc/extlinks/project/sub/submodule.

    +

    Ref. main: proc mainfunction*(): int.

    +

    Ref. util: utilfunction(x: int).

    + + + +
    + +
    + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
    +
    +
    + + + diff --git a/nimdoc/extlinks/project/expected/doc/manual.idx b/nimdoc/extlinks/project/expected/doc/manual.idx new file mode 100644 index 0000000000..158a758f0f --- /dev/null +++ b/nimdoc/extlinks/project/expected/doc/manual.idx @@ -0,0 +1,3 @@ +markupTitle Nothing User Manual doc/manual.html Nothing User Manual 0 +heading First section doc/manual.html#first-section First section 0 +heading Second section & doc/manual.html#second-section-amp Second section & 0 diff --git a/nimdoc/extlinks/project/expected/main.html b/nimdoc/extlinks/project/expected/main.html new file mode 100644 index 0000000000..1e7c9c1269 --- /dev/null +++ b/nimdoc/extlinks/project/expected/main.html @@ -0,0 +1,142 @@ + + + + + + + +nimdoc/extlinks/project/main + + + + + + + + + + + + + + + + +
    +
    +

    nimdoc/extlinks/project/main

    +
    +
    +
    + + +
    + +
    + Search: +
    +
    + Group by: + +
    + + +
    +
    + +
    + +

    +

    my heading

    See also module nimdoc/extlinks/util or nimdoc/extlinks/project/sub/submodule module.

    +

    Ref. submodule: `</a>` proc.

    +

    Ref. Nothing User Manual: First section or Nothing User Manual: Second section & from Nothing User Manual.

    +

    + +
    +

    Types

    +
    +
    + 
    A = object
    +
    + + + +
    +
    + +
    +
    +
    +

    Procs

    +
    +
    +
    + 
    proc mainfunction(): int {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    + +
    + +
    +
    + +
    +
    + +
    + +
    + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
    +
    +
    + + + diff --git a/nimdoc/extlinks/project/expected/main.idx b/nimdoc/extlinks/project/expected/main.idx new file mode 100644 index 0000000000..d01f2e4c55 --- /dev/null +++ b/nimdoc/extlinks/project/expected/main.idx @@ -0,0 +1,4 @@ +nimTitle main main.html module nimdoc/extlinks/project/main 0 +nim A main.html#A object A 17 +nim mainfunction main.html#mainfunction proc mainfunction(): int 20 +heading my heading main.html#my-heading my heading 0 diff --git a/nimdoc/extlinks/project/expected/sub/submodule.html b/nimdoc/extlinks/project/expected/sub/submodule.html new file mode 100644 index 0000000000..408ce2060f --- /dev/null +++ b/nimdoc/extlinks/project/expected/sub/submodule.html @@ -0,0 +1,131 @@ + + + + + + + +nimdoc/extlinks/project/sub/submodule + + + + + + + + + + + + + + + + +
    +
    +

    nimdoc/extlinks/project/sub/submodule

    +
    +
    +
    + + +
    + +
    + Search: +
    +
    + Group by: + +
    + + +
    +
    + +
    + +

    Ref. main: object A from module nimdoc/extlinks/project/main.

    +

    Ref. util: utilfunction(x: int).

    +

    Ref. Nothing User Manual: Second section & from Nothing User Manual.

    +

    +
    +

    Types

    +
    +
    + 
    submoduleInt = distinct int
    +
    + + + +
    +
    + +
    +
    +
    +

    Procs

    +
    +
    +
    + 
    proc `</a>`(x, y: int): bool {....raises: [], tags: [], forbids: [].}
    +
    + + Attempt to break HTML formatting. + +
    +
    + +
    + +
    +
    + +
    +
    + +
    + +
    + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
    +
    +
    + + + diff --git a/nimdoc/extlinks/project/expected/sub/submodule.idx b/nimdoc/extlinks/project/expected/sub/submodule.idx new file mode 100644 index 0000000000..2b02c889e8 --- /dev/null +++ b/nimdoc/extlinks/project/expected/sub/submodule.idx @@ -0,0 +1,3 @@ +nimTitle submodule sub/submodule.html module nimdoc/extlinks/project/sub/submodule 0 +nim `` sub/submodule.html#,int,int proc ``(x, y: int): bool 9 +nim submoduleInt sub/submodule.html#submoduleInt type submoduleInt 13 diff --git a/nimdoc/extlinks/project/expected/theindex.html b/nimdoc/extlinks/project/expected/theindex.html new file mode 100644 index 0000000000..cf250edd16 --- /dev/null +++ b/nimdoc/extlinks/project/expected/theindex.html @@ -0,0 +1,59 @@ + + + + + + + +Index + + + + + + + + + + + + + + + + + + + + diff --git a/nimdoc/extlinks/project/main.nim b/nimdoc/extlinks/project/main.nim new file mode 100644 index 0000000000..36b778af6b --- /dev/null +++ b/nimdoc/extlinks/project/main.nim @@ -0,0 +1,23 @@ +## my heading +## ========== +## +## .. importdoc:: sub/submodule.nim, ../util.nim, doc/manual.md +## +## .. See also [Second&&&] and particularly [first section] and [Second section &]. +## +## See also [module nimdoc/extlinks/util] or [nimdoc/extlinks/project/sub/submodule module]. +## +## Ref. [`` proc]. +## +## Ref. [First section] or [Second section &] from [Nothing User Manual]. + + +import ../util, sub/submodule + +type A* = object + x: int + +proc mainfunction*(): int = + # just to suppress "not used" warnings: + if ``(1, 2): + result = utilfunction(0) diff --git a/nimdoc/extlinks/project/sub/submodule.nim b/nimdoc/extlinks/project/sub/submodule.nim new file mode 100644 index 0000000000..876e00684c --- /dev/null +++ b/nimdoc/extlinks/project/sub/submodule.nim @@ -0,0 +1,13 @@ +## .. importdoc:: ../../util.nim, ../main.nim, ../doc/manual.md +## +## Ref. [object A] from [module nimdoc/extlinks/project/main]. +## +## Ref. [utilfunction(x: int)]. +## +## Ref. [Second section &] from [Nothing User Manual]. + +proc ``*(x, y: int): bool = + ## Attempt to break HTML formatting. + result = x < y + +type submoduleInt* = distinct int diff --git a/nimdoc/extlinks/util.nim b/nimdoc/extlinks/util.nim new file mode 100644 index 0000000000..f208f98c14 --- /dev/null +++ b/nimdoc/extlinks/util.nim @@ -0,0 +1,2 @@ +proc utilfunction*(x: int): int = + x + 42 diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 23d1920097..a267041331 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -1,99 +1,57 @@ - + - + - - - - - +Not a Nim Manual + + + + -Not a Nim Manual + - - - -
    -
    -

    Not a Nim Manual

    -
    +
    +
    +

    Not a Nim Manual

    +
    -
    - -     Dark Mode -
    - -
    - Search: -
    -
    - Group by: - -
    -
    -
    -
    - -

    - + +
    + +

    Authors: Andreas Rumpf, Zahary Karadjov
    Version: |nimversion|
    +
    Authors:Andreas Rumpf, Zahary Karadjov
    Version:|nimversion|

    "Complexity" seems to be a lot like "energy": you can transfer it from the end-user to one/some of the other players, but the total amount seems to remain pretty much constant for a given task. -- Ran

    About this document

    Note: This document is a draft! Several of Nim's features may need more precise wording. This manual is constantly evolving into a proper specification.

    @@ -128,8 +86,7 @@ window.addEventListener('DOMContentLoaded', main);

    Note: Assignments, moves, and destruction are specified in the destructors document.

    The language constructs are explained using an extended BNF, in which (a)* means 0 or more a's, a+ means 1 or more a's, and (a)? means an optional a. Parentheses may be used to group elements.

    & is the lookahead operator; &a means that an a is expected but not consumed. It will be consumed in the following rule.

    -

    Non-terminals start with a lowercase letter, abstract terminal symbols are in UPPERCASE. Verbatim terminal symbols (including keywords) are quoted with '. An example:

    -ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?
    +

    Non-terminals start with a lowercase letter, abstract terminal symbols are in UPPERCASE. Verbatim terminal symbols (including keywords) are quoted with '. An example:

    ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?

    In a typical Nim program, most of the code is compiled into the executable. However, some of the code may be executed at compile-time. This can include constant expressions, macro definitions, and Nim procedures used by macro definitions. Most of the Nim language is supported at compile-time, but there are some restrictions -- see Restrictions on Compile-Time Execution for details. We use the term runtime to cover both compile-time execution and code execution in the executable.

    var a: array[0..1, char]
@@ -141,8 +98,7 @@ ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?

    Encoding

    All Nim source files are in the UTF-8 encoding (or its ASCII subset). Other encodings are not supported. Any of the standard platform line termination sequences can be used - the Unix form using ASCII LF (linefeed), the Windows form using the ASCII sequence CR LF (return followed by linefeed), or the old Macintosh form using the ASCII CR (return) character. All of these forms can be used equally, regardless of the platform.

    Indentation

    Nim's standard grammar describes an indentation sensitive language. This means that all the control structures are recognized by indentation. Indentation consists only of spaces; tabulators are not allowed.

    -

    With this notation we can now easily define the core of the grammar: A block of statements (simplified example):

    -ifStmt = 'if' expr ':' stmt
+
    With this notation we can now easily define the core of the grammar: A block of statements (simplified example):
    ifStmt = 'if' expr ':' stmt
          (IND{=} 'elif' expr ':' stmt)*
          (IND{=} 'else' ':' stmt)?
 
@@ -169,18 +125,15 @@ stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
 \x HHcharacter with hex value HH; exactly two hex digits are allowed
 \u HHHHunicode codepoint with hex value HHHH; exactly four hex digits are allowed
 \u {H+}unicode codepoint; all hex digits enclosed in {} are used for the codepoint
-
    """"long string within quotes""""
    Produces:
    -"long string within quotes"
    
+
    """"long string within quotes""""
    Produces:
    "long string within quotes"
    
 
    
 
-
    Operators
    Nim allows user defined operators. An operator is any combination of the following characters:
    -=     +     -     *     /     <     >
+
    Operators
    Nim allows user defined operators. An operator is any combination of the following characters:
    =     +     -     *     /     <     >
 @     $     ~     &     %     |
 !     ?     ^     .     :     \
    
 
    
 
    (The grammar uses the terminal OPR to refer to operator symbols as defined here.)
    
-
    The following strings denote other tokens:
    -`   (    )     {    }     [    ]    ,  ;   [.    .]  {.   .}  (.  .)  [:
    
+
    The following strings denote other tokens:
    `   (    )     {    }     [    ]    ,  ;   [.    .]  {.   .}  (.  .)  [:
    
 
    
 
    Otherwise, precedence is determined by the first character.
    
 
@@ -213,7 +166,7 @@ stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
 
    Let T's be p's return type. NRVO applies for T if sizeof(T) >= N (where N is implementation dependent), in other words, it applies for "big" structures.
    Apart from built-in operations like array indexing, memory allocation, etc. the raise statement is the only way to raise an exception.
    
-
    typedesc used as a parameter type also introduces an implicit generic. typedesc has its own set of rules:
    
+
    typedesc used as a parameter type also introduces an implicit generic. typedesc has its own set of rules:
    The !=, >, >=, in, notin, isnot operators are in fact templates:
    a > b is transformed into b < a.
    a in b is transformed into contains(b, a).
    notin and isnot have the obvious meanings.
    A template where every parameter is untyped is called an immediate template. For historical reasons templates can be explicitly annotated with an immediate pragma and then these templates do not take part in overloading resolution and the parameters' types are ignored by the compiler. Explicit immediate templates are now deprecated.
    
 
@@ -297,27 +250,25 @@ stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
 
    Introduction
    
 "Der Mensch ist doch ein Augentier -- schöne Dinge wünsch ich mir."
 
    This document is a tutorial for the programming language Nim. This tutorial assumes that you are familiar with basic programming concepts like variables, types, or statements but is kept very basic. The manual contains many more examples of the advanced language features. All code examples in this tutorial, as well as the ones found in the rest of Nim's documentation, follow the Nim style guide.
    
-
    However, this does not work. The problem is that the procedure should not only return, but return and continue after an iteration has finished. This return and continue is called a yield statement. Now the only thing left to do is to replace the proc keyword by iterator and here it is - our first iterator:
    
+
    However, this does not work. The problem is that the procedure should not only return, but return and continue after an iteration has finished. This return and continue is called a yield statement. Now the only thing left to do is to replace the proc keyword by iterator and here it is - our first iterator:
    Precedence levelOperatorsFirst characterTerminal symbol
    
 
 
 
 
 
    
-
+
    A1 headerA2 | not fooled
    
 
    C1C2 bold
    D1 code |D2
    D1 code \|D2
    
 
    E1 | text
    
 
    F2 without pipe
    
 
    not in table 
    
 
    
-  
+
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    -
    - + diff --git a/nimdoc/rst2html/source/rst_examples.rst b/nimdoc/rst2html/source/rst_examples.rst index 54f0124c8c..7fa20de6c5 100644 --- a/nimdoc/rst2html/source/rst_examples.rst +++ b/nimdoc/rst2html/source/rst_examples.rst @@ -5,6 +5,10 @@ Not a Nim Manual :Authors: Andreas Rumpf, Zahary Karadjov :Version: |nimversion| +.. role:: nim(code) + :language: nim +.. default-role:: nim + .. contents:: diff --git a/nimdoc/rsttester.nim b/nimdoc/rsttester.nim index 6d41ffb86b..be2b56c678 100644 --- a/nimdoc/rsttester.nim +++ b/nimdoc/rsttester.nim @@ -1,8 +1,14 @@ +# To run this, cd to the git repo root, and run "nim r nimdoc/rsttester.nim". +# to change expected results (after carefully verifying everything), use -d:nimTestsNimdocFixup + import os, strutils +from std/private/gitutils import diffFiles const baseDir = "nimdoc/rst2html" +const fixup = defined(nimTestsNimdocFixup) + var failures = 0 proc exec(cmd: string) = @@ -18,17 +24,19 @@ proc testRst2Html(fixup = false) = let sourceFile = expectedHtml.replace('\\', '/').replace("/expected/", "/source/").replace(".html", ".rst") exec("$1 rst2html $2" % [nimExe, sourceFile]) let producedHtml = expectedHtml.replace('\\', '/').replace("/expected/", "/source/htmldocs/") - if readFile(expectedHtml) != readFile(producedHtml): - discard execShellCmd("diff -uNdr " & expectedHtml & " " & producedHtml) + let versionCacheParam = "?v=" & $NimMajor & "." & $NimMinor & "." & $NimPatch + let producedFile = readFile(producedHtml).replace(versionCacheParam,"") #remove version cache param used for cache invalidation + if readFile(expectedHtml) != producedFile: + echo diffFiles(expectedHtml, producedHtml).output inc failures if fixup: - copyFile(producedHtml, expectedHtml) + writeFile(expectedHtml, producedFile) else: echo "SUCCESS: files identical: ", producedHtml if failures == 0: removeDir(baseDir / "source/htmldocs") -testRst2Html(defined(fixup)) +testRst2Html(fixup) # Check for failures if failures > 0: quit($failures & " failures occurred.") diff --git a/nimdoc/test_doctype/expected/test_doctype.html b/nimdoc/test_doctype/expected/test_doctype.html new file mode 100644 index 0000000000..659aec5e3c --- /dev/null +++ b/nimdoc/test_doctype/expected/test_doctype.html @@ -0,0 +1,83 @@ + + + + + + + +nimdoc/test_doctype/test_doctype + + + + + + + + + + + + + + + + +
    +
    +

    nimdoc/test_doctype/test_doctype

    +
    +
    +
    + + +
    + +
    + Search: +
    +
    + Group by: + +
    + + +
    +
    + +
    + +

    Check

    +

    +text

    + +

    Check

    +

    text

    + +
    +
    + +
    + +
    + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
    +
    +
    + + + diff --git a/nimdoc/test_doctype/test_doctype.nim b/nimdoc/test_doctype/test_doctype.nim new file mode 100644 index 0000000000..28d35fc1e8 --- /dev/null +++ b/nimdoc/test_doctype/test_doctype.nim @@ -0,0 +1,15 @@ +# Markdown (default) interpretes this as text + a Markdown code block: + +## Check +## ~~~~~ +## text +## ~~~~~ + +{.doctype: RST.} + +# Now RST interpretes this as 2 headings: + +## Check +## ~~~~~ +## text +## ~~~~~ diff --git a/nimdoc/test_out_index_dot_html/expected/foo.idx b/nimdoc/test_out_index_dot_html/expected/foo.idx index a8dabb67e0..ac76aa5324 100644 --- a/nimdoc/test_out_index_dot_html/expected/foo.idx +++ b/nimdoc/test_out_index_dot_html/expected/foo.idx @@ -1 +1,2 @@ -foo index.html#foo foo: foo() +nimTitle foo index.html module nimdoc/test_out_index_dot_html/foo 0 +nim foo index.html#foo proc foo() 1 diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 4646732f1f..4370f0df8a 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -1,143 +1,105 @@ - + - + - - - - - +nimdoc/test_out_index_dot_html/foo + + + + -foo + - - - -
    -
    -

    foo

    -
    +
    +
    +

    nimdoc/test_out_index_dot_html/foo

    +
    -
    - -     Dark Mode -
    - -
    - Search: -
    -
    - Group by: - -
    -
      -
    • - Procs -
        +
        + + +
        + +
        + Search: +
        +
        + Group by: + +
        + +
      +
    -
    -
    - -

    -
    -

    Procs

    -
    - -
    proc foo() {...}{.raises: [], tags: [].}
    -
    + +
    + +

    +
    +

    Procs

    +
    +
    +
    + 
    proc foo() {....raises: [], tags: [], forbids: [].}
    +
    + + I do foo + +
    +
    -I do foo +
    -
    - -
    + +
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    -
    - + diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index aa9ca26458..ca7c2d7af8 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -1,85 +1,43 @@ - + - + - - - - - +Index + + + + -Index + - - - -
    -
    -

    Index

    - Modules: index.

    API symbols

    +
    +
    +

    Index

    + Modules: index.

    API symbols

    foo:
    • foo: foo()
      • + data-doc-search-tag="foo: proc foo()" href="index.html#foo">foo: proc foo()
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    -
    - + diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index 5262952224..0c0be36999 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -1,19 +1,27 @@ # Small program that runs the test cases for 'nim doc'. # To run this, cd to the git repo root, and run "nim r nimdoc/tester.nim". -# to change expected results (after carefully verifying everything), use -d:fixup +# to change expected results (after carefully verifying everything), use -d:nimTestsNimdocFixup import strutils, os +from std/private/gitutils import diffFiles + +const fixup = defined(nimTestsNimdocFixup) var failures = 0 const baseDir = "nimdoc" +let + baseDirAbs = getCurrentDir() / baseDir type NimSwitches = object doc: seq[string] + docStage2: seq[string] buildIndex: seq[string] + md2html: seq[string] + md2htmlStage2: seq[string] proc exec(cmd: string) = if execShellCmd(cmd) != 0: @@ -22,6 +30,9 @@ proc exec(cmd: string) = proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) = let nimDocSwitches = switches.doc.join(" ") + nimDocStage2Switches = switches.docStage2.join(" ") + nimMd2HtmlSwitches = switches.md2html.join(" ") + nimMd2HtmlStage2Switches = switches.md2htmlStage2.join(" ") nimBuildIndexSwitches = switches.buildIndex.join(" ") putEnv("SOURCE_DATE_EPOCH", "100000") @@ -29,21 +40,37 @@ proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) = if nimDocSwitches != "": exec("$1 doc $2" % [nimExe, nimDocSwitches]) + echo("$1 doc $2" % [nimExe, nimDocSwitches]) + + if nimMd2HtmlSwitches != "": + exec("$1 md2html $2" % [nimExe, nimMd2HtmlSwitches]) + echo("$1 md2html $2" % [nimExe, nimMd2HtmlSwitches]) + + if nimDocStage2Switches != "": + exec("$1 doc $2" % [nimExe, nimDocStage2Switches]) + echo("$1 doc $2" % [nimExe, nimDocStage2Switches]) + + if nimMd2HtmlStage2Switches != "": + exec("$1 md2html $2" % [nimExe, nimMd2HtmlStage2Switches]) + echo("$1 md2html $2" % [nimExe, nimMd2HtmlStage2Switches]) if nimBuildIndexSwitches != "": exec("$1 buildIndex $2" % [nimExe, nimBuildIndexSwitches]) + echo("$1 buildIndex $2" % [nimExe, nimBuildIndexSwitches]) - for expected in walkDirRec(prjDir / "expected/"): + for expected in walkDirRec(prjDir / "expected/", checkDir=true): + let versionCacheParam = "?v=" & $NimMajor & "." & $NimMinor & "." & $NimPatch let produced = expected.replace('\\', '/').replace("/expected/", "/$1/" % [docsDir]) if not fileExists(produced): echo "FAILURE: files not found: ", produced inc failures - elif readFile(expected) != readFile(produced): + let producedFile = readFile(produced).replace(versionCacheParam,"") #remove version cache param used for cache invalidation + if readFile(expected) != producedFile: echo "FAILURE: files differ: ", produced - discard execShellCmd("diff -uNdr " & expected & " " & produced) + echo diffFiles(expected, produced).output inc failures if fixup: - copyFile(produced, expected) + writeFile(expected, producedFile) else: echo "SUCCESS: files identical: ", produced @@ -61,7 +88,7 @@ let "$1/$2.nim" % [test1Dir, test1PrjName]], buildIndex: @["--out:$1/$2/theindex.html" % [test1Dir, test1DocsDir], "$1/$2" % [test1Dir, test1DocsDir]]) -testNimDoc(test1Dir, test1DocsDir, test1Switches, defined(fixup)) +testNimDoc(test1Dir, test1DocsDir, test1Switches, fixup) # Test "nim doc --out:.. --index:on .." let @@ -74,7 +101,57 @@ let "$1/$2.nim" % [test2Dir, test2PrjName]], buildIndex: @["--out:$1/$2/theindex.html" % [test2Dir, test2DocsDir], "$1/$2" % [test2Dir, test2DocsDir]]) -testNimDoc(test2Dir, test2DocsDir, test2Switches, defined(fixup)) +testNimDoc(test2Dir, test2DocsDir, test2Switches, fixup) -# Check for failures -if failures > 0: quit($failures & " failures occurred.") +# Test `nim doc` on file with `{.doctype.}` pragma +let + test3PrjDir = "test_doctype" + test3PrjName = "test_doctype" + test3Dir = baseDir / test3PrjDir + test3DocsDir = "htmldocs" + test3Switches = NimSwitches(doc: @["$1/$2.nim" % [test3Dir, test3PrjName]]) +testNimDoc(test3Dir, test3DocsDir, test3Switches, fixup) + + +# Test concise external links (RFC#125) that work with `.idx` files. +# extlinks +# ├── project +# │   ├── main.nim +# │   ├── manual.md +# │   └── sub +# │   └── submodule.nim +# └── util.nim +# +# `main.nim` imports `submodule.nim` and `../utils.nim`. +# `main.nim`, `submodule.nim`, `manual.md` do importdoc and reference each other. +let + test4PrjName = "extlinks/project" + test4Dir = baseDir / test4PrjName + test4DirAbs = baseDirAbs / test4PrjName + test4MainModule = "main" + test4MarkupDoc = "doc" / "manual.md" + test4DocsDir = "htmldocs" + # 1st stage is with --index:only, 2nd is final + test4Switches = NimSwitches( + doc: @["--project", + "--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--index:only", + "$1/$2.nim" % [test4Dir, test4MainModule]], + md2html: + @["--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--docroot:$1" % [test4DirAbs], + "--index:only", + "$1/$2" % [test4Dir, test4MarkupDoc]], + docStage2: + @["--project", + "--outdir:$1/$2" % [test4Dir, test4DocsDir], + "$1/$2.nim" % [test4Dir, test4MainModule]], + md2htmlStage2: + @["--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--docroot:$1" % [test4DirAbs], + "$1/$2" % [test4Dir, test4MarkupDoc]], + ) +testNimDoc(test4Dir, test4DocsDir, test4Switches, fixup) + +if failures > 0: + quit "$# failures occurred; see note in nimdoc/tester.nim regarding -d:nimTestsNimdocFixup" % $failures diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index db9a7ce979..a9e4ac9c6a 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -35,7 +35,13 @@ Modified by Boyd Greenfield and narimiran --escapeSequence: #c4891b; --number: #252dbe; --literal: #a4255b; + --program: #6060c0; + --option: #508000; --raw-data: #a4255b; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal) } [data-theme="dark"] { @@ -63,68 +69,56 @@ Modified by Boyd Greenfield and narimiran --escapeSequence: #bd93f9; --number: #bd93f9; --literal: #f1fa8c; + --program: #9090c0; + --option: #90b010; --raw-data: #8be9fd; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal); } -.theme-switch-wrapper { +@media (prefers-color-scheme: dark) { + [data-theme="auto"] { + --primary-background: #171921; + --secondary-background: #1e202a; + --third-background: #2b2e3b; + --info-background: #008000; + --warning-background: #807000; + --error-background: #c03000; + --border: #0e1014; + --text: #fff; + --anchor: #8be9fd; + --anchor-focus: #8be9fd; + --input-focus: #8be9fd; + --strong: #bd93f9; + --hint: #7A7C85; + --nim-sprite-base64: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAARMAAABMCAYAAABOBlMuAAAACXBIWXMAAAsTAAALEwEAmpwYAAAFFmlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgNS42LWMxNDggNzkuMTY0MDM2LCAyMDE5LzA4LzEzLTAxOjA2OjU3ICAgICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cGhvdG9zaG9wPSJodHRwOi8vbnMuYWRvYmUuY29tL3Bob3Rvc2hvcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHhtcDpDcmVhdGVEYXRlPSIyMDE5LTEyLTAzVDAxOjE4OjIyKzAxOjAwIiB4bXA6TW9kaWZ5RGF0ZT0iMjAxOS0xMi0wM1QwMToyMDoxMCswMTowMCIgeG1wOk1ldGFkYXRhRGF0ZT0iMjAxOS0xMi0wM1QwMToyMDoxMCswMTowMCIgZGM6Zm9ybWF0PSJpbWFnZS9wbmciIHBob3Rvc2hvcDpDb2xvck1vZGU9IjMiIHBob3Rvc2hvcDpJQ0NQcm9maWxlPSJzUkdCIElFQzYxOTY2LTIuMSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDplZGViMzU3MC1iNmZjLWQyNDQtYTExZi0yMjc5YmY4NDNhYTAiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6ZWRlYjM1NzAtYjZmYy1kMjQ0LWExMWYtMjI3OWJmODQzYWEwIiB4bXBNTTpPcmlnaW5hbERvY3VtZW50SUQ9InhtcC5kaWQ6ZWRlYjM1NzAtYjZmYy1kMjQ0LWExMWYtMjI3OWJmODQzYWEwIj4gPHhtcE1NOkhpc3Rvcnk+IDxyZGY6U2VxPiA8cmRmOmxpIHN0RXZ0OmFjdGlvbj0iY3JlYXRlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDplZGViMzU3MC1iNmZjLWQyNDQtYTExZi0yMjc5YmY4NDNhYTAiIHN0RXZ0OndoZW49IjIwMTktMTItMDNUMDE6MTg6MjIrMDE6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyMS4wIChXaW5kb3dzKSIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz4JZNR8AAAfG0lEQVR4nO2deViTZ7r/7yxkJaxJ2MK+GCBAMCwS1kgUFQSKK4XWWqsz1jpjp3b0tDP1V+eqU391fqfT/mpPPd20drTFDS0KFEVWJSGAEgLIZpAICBJACIRs549Rj1WILAkBfD/XlevySp68z/0S3+/7vPdzLyidTgcLkU2bd+z39/f/q1gshsrKSoJELFCa2iaEuU9K6kb+8uXxv54/fzE8L/eswNT2zCfQpjbAGKS8lPFKSEjIXiaTCSEhIeDj4xNnapsQ5j6rktZGp6UlfxIdzQVzCplmanvmG1hTG2BIAtlc26CgoDfT0tL2e3l5AQCAjY0NkMnk/a9s2k6rrKw8UV8n1JjYTIQ5RlAw14KzmL3xze1vfJyUuMJaq9UCFovFm9qu+YbBxcSPFUYkk8l2Q0NDsvo6ocrQx5+I8Ih4bz6f/0l8fHyKlZXV4/dRKBQwmcwwMpn8A4FAoPgHhH9bV1sxa488wZxoaycnJ/a9e/duCa5fkc3WvAiTI4Ib77p+XdqHG9anbfLy8gAAgLGxMdBpF+bjvzExqJj4scKI0dHRnwQHB++orq7+AgDeMuTxJ2Jl4rqU9PT0EwEBAUQCgTDuGAaDAampqYepVKpHUHDk325Ulw0a266YuFW+Gzdu/MDPz29jfn7+XgA4aOw5ESZP6kvpCXv3vnM8NiaSamVl+fj9BepGNDoGFRN7e/slcXFxO1xcXMDJyWnH7j//H/fi4uJdgutXmgw5z5O8smn7X9euXbvf29sbMBjMhONQKBRYWVlBbGzsbjMzM3JoOG+/sKKwy1h2rd/4elpGRsYuLy+vaDweD2w2Oy1h5ZrCvEunEaeeiVnMiabyl/F2/+X9P+8JDPQHHA5napMWBAYTk6DgSNuEhIS9DAYDAP7tq1i6dOkqOp3OWbNu0wens44emeoxA9lcWwKBYEMkEm2JRKIdHo+3QKFQWJ1Op8ZgMER3d/dVq1evTnFycpr0MSkUCsTExGzH4/Gk1LTME/39/TI0Go1FoVCg1WrVY2NjipGRkcGRkRH5dPwrEZHLXMPCwjJSUlIy3dzcfB+97+rqGhYSEpIOAIiYmBguN3zL77dt3uPh4W5qUxYUBhMTb2/vjeHh4cvR6P/dILK0tITIyEg7BweHr363/Z3Ampqaf1Zcu/zMKiVsyVJvMplsRyKR7IhEor2FhYUbhUJhJCYm2pFIJB6JRAIymQx4PB7QaDRoNBowMzMDJycnwOOn7icjEokQGxu7icFgbLp///7jFY1WqwWlUgkjIyOgUCgO7Ni5Rz48PCwfHh7uGRkZeaBQKOSjo6ODCoVCXlNVKn/6uCsT13FXrVr1emho6BYKhfLMnP7+/omrU9LPX8g+UThloxEMxqJFXjxESAyPQcSEExrLWLNmzW57e/txP/fw8ABHR8cdDAaDt3xF2ru9vb03sVgs0cbGxs/FxWVZUlISj0aj+dna2oKtrS1M5PcwJCgUCry8vODRrs84vPfoH6OjoyCXy6Gvr+/R6+CWrX9s7evrk/b19bWr1Wqli4sLZ8OGDe95eXmxUSjUuAd0cHDwjoqK2sYKXFIhvnldYYTTQpgU4/8+jyASCYDGoCd+ZkYYF8OICYezl8PhuOkbQyAQIDo62s/NzS2np6cHbGxsgEajAYFAAAwGA1gsFia6CE0NgUAABwcHsLe3B61WC2q1eo9WqwWNRgNKpRLUajUQiUSgUCh6zwGHwwGTydzo5+eXBQBnZu8MEJ5keHhYPqyYWMtHR0ZBpVIhYj9FUDONgOUvT12+du3avMDAQJjssdRqNWCxCyrEZdLodDoQi8Ulx44de628NL/V1Pa8iERE8l2dHB2CJvpcq9Nqbt1qKURWj1Njxld0ZGTkAW9v70kLCQC8sEIC8O/HKx8fn2gmk8kHgCk7pRFmzrWyAikASE1tx0Jj2uH0EZHL/N7YtuvT4OBgzmz4OBYSeDweIiMjt2S++vtMP1YYEmmJsCCY8mNOIJtr6+zsHBcZGXmIw+G4mZubG8m0hU9HRwcUFxe/KxQKTyDRsQjznSmJCS9+dVRERMTfQ0NDo2xtbfUGiSFMjtHRUaitrc3Jzc09kHvxVLmp7UFAmC6oZQkvrZLL5RJhReHtiQb5scKIXC7371FRUX90dnYGIpE4JR8Jgn40Gg20t7fXFxYWfnr9+vWjz8sdYi+Osh4vzgUBwZSgtu94V+fs7Hx7YGCgra6u7khLS0u2RCwYeTQgKmYFh8fj/f/g4OAldnZ2prR1wdPd3Q1CofBQSUnJkdLi3N8E93FCY6k+Pj48FxcXjlar1ZSWlh65VvYr4kREmDNg79+/D3FxcW5OTk5uXl5evNbW1tL0jK3ZXV1d1ykUintycvInoaGhdkj+gvGxs7MDPp+/m0AgWMQvS/lyeHhYTqPRPJycnIJSU1NZ3t7eW2g0Gly/fv2oWq1Gij0hzClQ/gHhpLS0tEM8Hm/7I8Ho7++HlpYWsLa2Bg8PDxOb+OKhUCigqakJ7t+/D25ubuDu7g4oFAp0Oh08ePAAvv7666TTWUdzTG0nAsKTYMU3ryuSU18+4+bmFrZo0SIOAICVlRUsXrx4zkakLnRIJBI8CgJ8MtdJp9NBZ2enqL29XWRC8xAQxgUNAHD+3L8KGhoaCp78ABES04JCoX4jJAAAAwMDUFtbe96YpRMQEKbL41DU5ubmko6Ojj2PSgggzD36+/vrb9y4cX425zzw93/8EBjon2is44+NjSkePBjqGRwc7G5v7xBV19w8U5B/3qgrr9+/uWtXUuKKD/TZ9MXh/066/OuFmunO8dGBQ98HBbGSp/t9U6LRaDXK0dHBoeFhuVzeL22/0yFqamopufjLqRJ933ssJi0tLSXV1dWHGAzGbuObOzs8ubqa71vZKpUKOjo6blwpOF8zm/Mu5cVkLlkSaswprAHAaVihgK7O7oSGxltvfXLon3nXK4RHT2cdN4pfKDCAlZyUuMJan02nTmczAaBmunPw4qI3cbnh0/36XICq0+lgcPABp7OrK629vUP5z8++LLh2XXD05L++yxrvC4/F5EZ12WBS8saLS5Ys2U2lUufUY45SqQSlUgkqlQrUavXj19jYGGg0GtBoNKDT6UCn05VotVq1TqfToFAojFar1eh0Og0Wi8XhcDgeGo1+/PhgZmYGOBwOsFgsmJmZ/eY1F+nt7YXa2trs2Z73wdCQBgCMHp1IJpHA09MdPD3dLRIS+OtKisvWvbP7vf2lZdePVFwzbHTwyMiI3hidkZFRUKvUYzOZ48HQkBIA5nWqBAqFAktLC7C0tADmIh88Pz4uMSyUk7hn776DV4tKPn/6d/lNxp1MJqsRCASf8vn8XdMpOjRTVCoVjI2NgUqlAq1WCyMjI9DX1wf379+Hvr6+/Q8ePOgdGRmRKxSKx0WLFAqFXKlUKnQ6nUar1arHq47mxwrD4/F4Eg6HI2GxWDwej7cgkUjWFAqFam5uTjU3N6eRyeQPLSwswNraGqysrIBAIDwWFywW+zja11Qi29LSclIikeSZZPJZBovBAI8XA8HBQR9kZZ3lR8cmvFZSlGe00p8IkwONRkNERBj4+i7a4+XpHv307/IbMakWlciXJbx0nMPh7Jqo0JGh0el0MDo6Cl1dXSCVSkEmk7177969W319fe1DQ0M9KpVKoVarlWq1WjndNhUPG3ApAWDcOxLTLwSDwWAOotFoDBaLxRMIBAsrKysne3t7Xzqd7k2n0/c4OzsDlUoFHA4364IyMDAATU1NxdWikhcq6tXKyhJezljPJZKI2eERS5cZeoWCMD2srCwhPX0tVzk2djiCG//GtfLLUoBxShB0dHTU3Lx580sLC4vtJBLJKMZoNBqQSqUglUqPdnR01PT09DT19/fLHjx40DM0NNQ72933GiSVGgB4JFQK+LfoSAGgnL04yppEIh2xtLS0t7GxcaFSqR7Ozs4fMRgMcHR0nJX8pJs3b54Ui8UXjT7RHIRMIkFK8irfwcEHPwQELUmqvYHUGJkLmJubw8YNa/i9vfffY/px3myQiDTPiEl9nVDDX576jaenZ7SnpyfLUJNrNBqQyWRw+/bt4x0dHTdkMlltV1dXw/XygjkdEv4wB0YOAK0AUM70C8HQ6fSzdDrdm0qlejg6OrLc3Ny2MBiMadWjfR4PHjyAmzdvZs/1v5MxoVAokJK8iicWS95k+nH+s0EiQhqpzQGoVFtYk5a87ba0XQAA34xbpagg/5zoT7s/OGNnZ8eaaYkBuVwOnZ2d5VKpVNTS0lLS2NhYWFVZ3Dujg5qQh6uY+ocvCAiKIPn4+Jz19PSMdnV15VCpVL6Dg4NBViw6nQ5EItHRpqamqzM+2DzHzo4O69amftLQeKsAZrDLgmBY/PyYsCIhfs+SiKUFE5Y8EwqFx11cXDihoaFTjjFAoVAwPDwMHR0dourq6jNCofDHhZqUVnvjmgIAcgAgJyg40mLRokX8kJCQjT4+PussLS1n1JPl7t27UFxcfHguB6mNjY2B7G4naNRTWyygUCjAYDGAx+PB0sICSCSi3vFYLBbCwjjA8vddBQtATKb7d3saBwc7IJPJBpsHjUGDGRYLJBIJLK0sAfucmyIGg4FFi3y8AwNZtycUk5KiS02vvf7WWQaDkejg4DApQwAeh3xDaWnpPoFAcPxFqnP6sEvgGf+A8Bx3d/cvIyIiNi1evHjT8wpNj8fAwACUlZW9P9dD5+/ckcFbf9gd2dcnn9LNAovF4inmZHtXNxdOdBR3+/JlS33pdP29wolEInA4weuiYxOy5vvuTkeHDHb+8c8xvb33Z3R9/N+Df+uIjYk02DwkEsna2trS1d/fNyGeF7uTyw1/7g3R3t4O2OxA/TVghULhcQqFQk1JSfmYSNR/5wD4d6EfgUBwvLS09IhUKhW9qAV5H9YjKQwJi6uvrKw8ERoamhkSEpKp7w7yJEqlEiQSyZmysrJv53qjdaVSCZdyTk+3qFMrAJRHRPLPN95qeifj5fU7mYt8JhyMRqMhMJDFdnF25gDAvBYTpXIMWlpay2fq/8m5mDcIABYGnEcGAGI/VlhBZWX1yZdSkz55OX0dV5+7w9bGGvz8mPrFpK62QskJjf2GTqd7x8bGbpnID4BCoUAmk0lLSkqOiESik2UleS/MakQflYKrXQDQxY1a3tTe3i6KiIjY5OXlxX7e9+rr6wsuXbr0t4ffn9OgMWjghMZQRcLp+8GulRVI/QPC37Wxtnal0ajJtjY2E451ZjiBra31vE9lR2PQQKFQaAAwo98Yi8Xq9fpPd56HO6rlvKWJv/PwcK+JilyCmajWMw6HAzs7+rMFpQOCIn6zHywSFvXm5eUdFAqFZ9Rq9bgHa2trq79w4cK+zz49cAARkmcpL81v/a/Dhz49d+7c3qqqqjyVSjXuOJ1OBxKJpDw3N/fA5V+zax6978cKw/sHhM/raMrnUVdboSy4fPWQSFSjd5yFBQWIRNKEd2IEw1J4JUd88WL+R51d3XrHWVDMnxUTa2tr1zXrNiUGsrmPf7DS4tymCxcu7Kuurs55+kKQSqVN586d23vs+8NHDXUCC5Wzp3/Iy8rKeruysvLM2Nhvo7VVKhXU1tYWnj17du/T7UOdnZ2D7OzsfGGB09raVi4S1RzXl0eFw+EAj8chYjKLVFffyOrq1C8mJBLpWTFRKBRyDofzC4vFWvXk+1ev/CLOzs7eKxAIslQqFeh0Oujp6enKzs7em/XTd7OayTqfKb56sT4rK+sPAoHg5KO/o0KhAKFQmHXy5MkdF3/5+TeZmctXpIXZ29v7zqVcKWNRX1epuXu3U/y8pEw0GmndOZt0dnXVDw0P6/W5oNHoZ30mQ0NDPb29vfvj4+Pf3rR5B/7od188XnEUXr4gDgmL+0NfX5/U19d3d3l5+YGfTnyDtLmcIhXXLsu4UcvfR6PRGGtra9eysrIjYrE45+kt4Fheou/69es/unnz5vm7d+/Wmsre2WRkZGTQ1DYg/JYGiUiTm1ugBAC9IfHPiEmDpFITE7fqJI/H27lmzZpDq5LWtz55t6wUXO3ihMYerK+vz2tpaUFaM0yT8tL81ujYle+TSCTrvEunBU9/voTLd92wYcPHVCqV39XVdXCu7+oYCp1O90Kc50Jk3I5+xVcv1jc3N5d4enpSMzIyvkpK3sh78nORsKg3++yPBS/q1q+hKCm61DSekERGJ3ikp6d/ERsbm1xVVXWwtbX1hRFtFAqFPMLMUyZsDyoQCI7LZDKIiIjwzczM/GpV0vro2TTsRSUqZoX3+vXrP1u9enXi0NAQiESirIdRtggIc5oJ40zq6uryGhoa8ry8vBJCQ0O9USjU94mrN7yWc+EnvaXb5gJMvxCMp6cnl0Kh2Le1tZVXXLs8L1LXefGrWRkZGZ/x+XyeUqkEkUh0vqenZ14HZyG8OEwoJjdrygd37NxTEBkZmWBtbQ3BwcEeKBTq+/UbX3/355Pfzlmn66qk9dGbN29+k8PhbCSRSNDZ2Snb9ae/HCkpKTksEhbN2QTD5NSX+Vu3bj0cHBzsjcFg4O7du1BWVvbNwxB9BIQ5j94I2Fu3bhXW19cDl8sFLBYLHA7Hg0wmf/e77e84ffXlPz6fLSMnQ2paZkJ4eHjmtm3b+B4eHvZkMhlQKBTY29s72dvbfxgUFJT8x7ffP1NRUfHjXErnZ/qFYKKjo7dt3rz5g8DAQPtH/XHa2tpqGhsbC55/BASEuYFeMblz505NTU3NgfDw8PcwGAygUCjw9fW1IJPJn/1130Hv0tLSI4WXL4hny9inYS+Osvbz80tgMpn8jIwMPovFch2vpoiDgwM4ODhwfH19OYsWLeJv3/Hu+cbGxquzXZz5aZYlvMRJT0/fFhkZue3JZmfd3d0gEolOIr4ShPmEXjFpkFRqXlrzSnFnZ+d7Tk5OjzNfXVxcICMjY6ezszNnVdL6vU8HWhmbgKAIkrOzMyc1NTXz0YU4maAuOp0OK1as4EVFRfGEQqHg1dfePHzr1q2rs71S8WOF4f38/BLS09M/iIyM5DxdxLq5uVlcVVU1bgVwBIS5il4xAQCQyWRigUBwJikpKe3JVGQcDgdLly7l2tranti0ecf7IpEoy9hbxX6sMDydTvdevXr1ltjY2F3u7u6AxT73FJ7B3Nwc4uLiwthsdphQKCzZkL7l0/r6+oKbNeVG90+EhMXZL1++fFtycvKHrq6uz4igUqmE5ubmEiTHCWG+8dwrUXD9imz9xtd/jIuLS7N5KpsTjUZDUFCQE4PB+F4oFGYmJW888Mv5k4UTHGpGxC9LYaenp78VEhKyxdHRESgUyoyOh0KhwNraGuLi4qIDAgKi6+rqyjekb/mHMSN6N6RvSdu+ffseNpsdZm09ftuW+vp6EIvFSB9hhHnHpG7rUqm0orW1tdXS0tLj6TIEaDQaaDQaxMfH811dXTl/3Xfw+JUrVz411J01cfWG6IiIiC07d+5McHNzs7ewMGyOFw6HAwcHB6BSqVx3d/fwz7/4rkAgEBwXCoUnHpZonDGrU9J5MTEx27du3Zrm4uKC0beaqq6u/ry+vj7XEPMiIMwmkxKTimuXZe/u+fCkp6fnexPdUfF4PPj7+1szGIydLi4unF1/+kvenTt3RG1tbRXTqfma8lIG39/fP/HVV19NZrFYHpMpzjQTzMzMwNPTE+Pp6Zng6emZ4Ofnl5CesfV8bW1tznQe3/wDwvFeXl7Rvr6+Ca+88kpaUFCQh74GXzqdDrq7u6GpqankRQmdR1hYTNrhUFVVlcXj8d6ysrKy0OfstLS0hPj4eC6Xy+U2NzeDRCI5/sa2XeX37t1rGhwc7BoYGJBN1P+FFbiE5OzszGaxWImvvvrqpoCAAKfp+ERmCpPJBCaTmcnhcDJLS0u/TE59+YxUKhXoi/lg+oVgrKysGJaWlna2trYeaWlpXDabvTMgIGDSfp2KiorzbW1tL0zoPMLCYtJX6uVfs2u++PKowMPDgz+ZIslEIhECAgKAxWJlajSazJ6eHmhra4PW1tZvtmz9o6Czs7O+r6+vfWxsbFir1WosLCzsV6xYkcnj8d7z9vaelmPV0Hh5eYGnp+f2mJiY7UVFRZ/HL0v5tru7+5ZGo1FisVg8Docj4fF4CxsbG1c+nx/m7e39sYeHB7i4uIC5ufmU6r4ODQ1BZWXlifkSrYuA8DRTumIrKytPent78728vCb9HRQKBVgsFhwcHIBOpwObzd4yNja2RaVSwdDQEHR1dcHo6CjQaDRwdXWdsWPV0KBQKPDw8AA7O7udERERO2tra2FgYACoVCo4OTkBjUYDMpkMeDz+8WuqaLVaaGxsbL19+/YzSX8ICPOFqYrJidDQ0AwvLy/e80c/CwaDARKJBI86BdJoNHB3dwe1Wj0nViL6IJPJwGQywdnZGZRKJRAIBDBUx8OBgQEoLS39BtkORpjPTJg1PB61N64pmpqarvb39xvUiLkuJE9CJpPBxsbGYEICANDZ2SlHgtQQ5jtTEhMAgLq6ulyJRFJvDGNeREZGRkAikRSUFuci2cEI85opi0l+7hmBWCzOeV6dToTJcfv27cHr168jxbgR5j1TFhMAgObm5hKZDNl0MAQtLS3Xzpw6hkS8Isx7piUmUqlUIBAIJuyjgzA5Ojs7QSKRINGuCAuCaYmJsKKw68qVK59KJJIu5HFneiiVSigqKjouEolOmtoWBARDMC0xAQC4+MvPJadOnXq3ra1N8yL0dDEkOp0OSktLy/Pz8w8+3d4CAWG+Mm0xAQA4fuy/jl+8ePGju3fvGsqeBY9Wq4XKysrWU6dOvX31yi8mKyyFgGBoZiQmAAD/79D+fadPn96PCMrz0el0UFVV1frtt9+mj9fiAgFhPjNjMQEAyMvLO3Ds2LE/tLS0INmuerh27Vr9999//xoiJAgLEYOEntbVVigB4PNNm3cMpqSkfMRms50McdyFgkqlgqKiovJTp069nZ97BhEShAWJQePYj373xdF1GzbLFQrFx6Ghob766ne8KNy7dw+KiopO5ubmfmTK4tsICMbG4EkxWT99d35l4rre/v7+D0NCQvh0Ot3QU8wL1Go1SKVSTX5+/sH8/PyDSP8bhIWOUTLsLuVklQcFR65pbGzcvnLlyvfc3NwsCASCMaaac+h0OhgaGoLq6uqaCxcu/OV01tGcTw7uM7VZCAhGx2jpug/vxAd58atzoqKitq1cuXKnvb29saabE+h0Oqiurpbm5eUdrK6uPlspuDrvY0hmO4YIhUIBGq1/X2CmNqFQKL3/79HomZ/z82xEowyy9zFr80zGDqPn/hdeviBmL47ad+fOnRsRERGbQkNDo62srIw97azT2dkJxcXFx0tKSo7Mdh8hY4LD4TDPH2U4MFjMc6tLmZmZzaj+Aw6H0/t9PB4PGCxmRudNJBL0ngeZTAI0Gj3jv+1szfM88Hic8cUEAKCmqlQOAN/ELU2qkEgkySwWK3HRokVcBoMxG9MbDZ1OB83NzdDU1FRQW1t7XiAQHJ+ovu18pbr6Rg6L5ZtoM0EhcUPT0tJW8tWRb0vQqIkvgKqqmhnVfrl2TfANXo+gjKlUio4OWc1M5sjOzjnQUH8rbqLPu3t6moaGhmfc+3q25tGHUqmECoEIUKbIrVkcEkONiIh4jcvlvu7s7OxLo9GmVe7QVCgUCujq6oKGhoaCioqKo9XV1WeM3YDMVPDik1gpyas+XrVyeaKXl8czjyANjbcgI/MNmkg49Q4ECPOH3NyC4RUr+M8IcHt7B1y9WlKRl3/5kElKnD1sfXEoJCzueEBAQGJYWFgGk8nk2djYAIFAgLm4pTw6Ogqjo6Mgl8vhxo0b50tLS4/U19fnLvS2FIWXfxEDQNLmLW9ueW1TxtchHDaQyWRTm4VgYkZHR6G+vhF+/NfP+y5e+vVjiVgwZpKVydOwF0dZW1lZOTGZTD6bzU4LCAiIptPp8HTDL1MwOjoKLS0tUFdXd1IsFudIpdKKgYGB7tloJTrX4MUnsVJTEj9etzY10dHRAQAAGm81wcsZW5CVyQInL69gNCGBjwcAGBx8ANnncypOnTr3H9nn/reD55wovvrQpyIHAHFUzIocGo3mQaPRfBwdHVlubm7bXF1dgcFgABqNNvruglwuh7t374JMJoOOjo7P79y5I+ru7m7q7e1tXQi7MzOh8PIv4pCw2DdaWtte37Au7aPIyCWAxWABjUbPif9HCMbjURtKiaQBfvr5zH9evlJ0uLQ4r/nJMXNiZTIRrMAlJAcHB18HBweWo6Mjy8rKajeJRAJLS0uwtLQECwsLoFAogMfjAYvFgpmZ2XNXMyqVCoaHh2FoaAiGh4cfvwYGBqCvrw+6u7vfvnfvXlNvb29rT09Pq0QsUM7S6c4rNqS/lrZ5U+YPRBKR9M7u9xwqBUUvtNAudH766XSLE8PR49ixE78/8tVnX403Zk7fUR46NUUAIPIPCMdTKJTdNjY2QKPRgE6nA51OB1tbWyCRSIDD4YBAIAAejwcCgfDYUajVakGlUoFarQadTvfY79HX1wf9/f0gl8tBLpfDvXv3HvXw+dxQPYYXMj+d+P7Mmzv+5OHr6/OJWq1GBHeB09TcUiKuq/coKS3/eqIx/wPkiIXC3w6YjAAAAABJRU5ErkJggg=="); + + --keyword: #ff79c6; + --identifier: #f8f8f2; + --comment: #6272a4; + --operator: #ff79c6; + --punctuation: #f8f8f2; + --other: #f8f8f2; + --escapeSequence: #bd93f9; + --number: #bd93f9; + --literal: #f1fa8c; + --program: #9090c0; + --option: #90b010; + --raw-data: #8be9fd; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal); + } +} + +.theme-select-wrapper { display: flex; align-items: center; } -.theme-switch-wrapper em { - margin-left: 10px; - font-size: 1rem; -} - -.theme-switch { - display: inline-block; - height: 22px; - position: relative; - width: 50px; -} - -.theme-switch input { - display: none; -} - -.slider { - background-color: #ccc; - bottom: 0; - cursor: pointer; - left: 0; - position: absolute; - right: 0; - top: 0; - transition: .4s; -} - -.slider:before { - background-color: #fff; - bottom: 4px; - content: ""; - height: 13px; - left: 4px; - position: absolute; - transition: .4s; - width: 13px; -} - -input:checked + .slider { - background-color: #66bb6a; -} - -input:checked + .slider:before { - transform: translateX(26px); -} - -.slider.round { - border-radius: 17px; -} - -.slider.round:before { - border-radius: 50%; -} - html { font-size: 100%; -webkit-text-size-adjust: 100%; @@ -147,24 +141,39 @@ body { padding: 0; box-sizing: border-box; } -.column, -.columns { +.column, .columns { width: 100%; float: left; box-sizing: border-box; - margin-left: 1%; + margin-left: 1%; } + +@media print { + #global-links, .link-seesrc, .theme-switch-wrapper, #searchInputDiv, .search-groupby { + display:none; + } + .columns { + width:100% !important; + } } -.column:first-child, -.columns:first-child { +.column:first-child, .columns:first-child { margin-left: 0; } +.container .row { + display: flex; } + .three.columns { - width: 22%; + width: 25.0%; + height: 100vh; + position: sticky; + top: 0px; + overflow-y: auto; + padding: 2px; } .nine.columns { - width: 77.0%; } + width: 75.0%; + padding-left: 1.5em; } .twelve.columns { width: 100%; @@ -230,6 +239,12 @@ select:focus { } /* Docgen styles */ + +:target { + border: 2px solid #B5651D; + border-style: dotted; +} + /* Links */ a { color: var(--anchor); @@ -245,27 +260,32 @@ a.reference-toplevel { font-weight: bold; } +a.nimdoc { + word-spacing: 0.3em; +} + a.toc-backref { text-decoration: none; - color: var(--text); } + color: var(--text); +} a.link-seesrc { color: #607c9f; font-size: 0.9em; - font-style: italic; } + font-style: italic; +} -a:hover, -a:focus { +a:hover, a:focus { color: var(--anchor-focus); - text-decoration: underline; } + text-decoration: underline; +} a:hover span.Identifier { color: var(--anchor); } -sub, -sup { +sub, sup { position: relative; font-size: 75%; line-height: 0; @@ -292,8 +312,7 @@ img { background: transparent !important; box-shadow: none !important; } - a, - a:visited { + a, a:visited { text-decoration: underline; } a[href]:after { @@ -307,16 +326,14 @@ img { a[href^="#"]:after { content: ""; } - pre, - blockquote { + pre, blockquote { border: 1px solid #999; page-break-inside: avoid; } thead { display: table-header-group; } - tr, - img { + tr, img { page-break-inside: avoid; } img { @@ -331,22 +348,18 @@ img { h1.title { page-break-before: avoid; } - p, - h2, - h3 { + p, h2, h3 { orphans: 3; widows: 3; } - h2, - h3 { + h2, h3 { page-break-after: avoid; } } p { margin-top: 0.5em; - margin-bottom: 0.5em; -} + margin-bottom: 0.5em; } small { font-size: 85%; } @@ -354,8 +367,7 @@ small { strong { font-weight: 600; font-size: 0.95em; - color: var(--strong); -} + color: var(--strong); } em { font-style: italic; } @@ -376,8 +388,7 @@ h1.title { text-align: center; font-weight: 900; margin-top: 0.75em; - margin-bottom: 0em; -} + margin-bottom: 0em; } h2 { font-size: 1.3em; @@ -404,36 +415,29 @@ h6 { font-size: 1.1em; } -ul, -ol { +ul, ol { padding: 0; margin-top: 0.5em; margin-left: 0.75em; } -ul ul, -ul ol, -ol ol, -ol ul { +ul ul, ul ol, ol ol, ol ul { margin-bottom: 0; margin-left: 1.25em; } ul.simple > li { - list-style-type: circle; -} + list-style-type: circle; } ul.simple-boot li { - list-style-type: none; - margin-left: 0em; - margin-bottom: 0.5em; -} + list-style-type: none; + margin-left: 0em; + margin-bottom: 0.5em; } ol.simple > li, ul.simple > li { margin-bottom: 0.2em; margin-left: 0.4em } ul.simple.simple-toc > li { - margin-top: 1em; -} + margin-top: 1em; } ul.simple-toc { list-style: none; @@ -442,8 +446,7 @@ ul.simple-toc { margin-top: 1em; } ul.simple-toc > li { - list-style-type: none; -} + list-style-type: none; } ul.simple-toc-section { list-style-type: circle; @@ -453,12 +456,10 @@ ul.simple-toc-section { ul.nested-toc-section { list-style-type: circle; margin-left: -0.75em; - color: var(--text); -} + color: var(--text); } ul.nested-toc-section > li { - margin-left: 1.25em; -} + margin-left: 1.25em; } ol.arabic { @@ -505,12 +506,39 @@ hr.footnote { margin-top: 0.15em; } div.footnote-group { - margin-left: 1em; } + margin-left: 1em; +} div.footnote-label { display: inline-block; min-width: 1.7em; } +div.option-list { + border: 0.1em solid var(--border); +} +div.option-list-item { + padding-left: 12em; + padding-right: 0; + padding-bottom: 0.3em; + padding-top: 0.3em; +} +div.odd { + background-color: var(--secondary-background); +} +div.option-list-label { + margin-left: -11.5em; + margin-right: 0em; + min-width: 11.5em; + display: inline-block; + vertical-align: top; +} +div.option-list-description { + width: calc(100% - 1em); + padding-left: 1em; + padding-right: 0; + display: inline-block; +} + blockquote { font-size: 0.9em; font-style: italic; @@ -519,7 +547,12 @@ blockquote { border-left: 5px solid #bbc; } -.pre { +blockquote.markdown-quote { + font-size: 0.9rem; /* use rem to avoid recursion */ + font-style: normal; +} + +.pre, span.tok { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; font-weight: 500; font-size: 0.85em; @@ -530,6 +563,16 @@ blockquote { border-radius: 4px; } +span.tok { + border: 1px solid #808080; + padding-bottom: 0.1em; + margin-right: 0.2em; +} + +.copyToClipBoard { + position: relative; +} + pre { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; color: var(--text); @@ -548,7 +591,25 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; } + border-radius: 6px; +} + +.copyToClipBoardBtn { + visibility: hidden; + position: absolute; + width: 24px; + border-radius: 4px; + background-image: var(--clipboard-image); + right: 5px; + top: 13px; + background-color: var(--secondary-background); + padding: 11px; + border: 0; +} + +.copyToClipBoard:hover .copyToClipBoardBtn { + visibility: visible; +} .pre-scrollable { max-height: 340px; @@ -613,8 +674,8 @@ table th { font-weight: bold; } table th.docinfo-name { - background-color: transparent; - text-align: right; + background-color: transparent; + text-align: right; } table tr:hover { @@ -631,31 +692,31 @@ table.borderless td, table.borderless th { padding: 0 0.5em 0 0 !important; } .admonition { - padding: 0.3em; - background-color: var(--secondary-background); - border-left: 0.4em solid #7f7f84; - margin-bottom: 0.5em; - -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + padding: 0.3em; + background-color: var(--secondary-background); + border-left: 0.4em solid #7f7f84; + margin-bottom: 0.5em; + -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); } .admonition-info { - border-color: var(--info-background); + border-color: var(--info-background); } .admonition-info-text { - color: var(--info-background); + color: var(--info-background); } .admonition-warning { - border-color: var(--warning-background); + border-color: var(--warning-background); } .admonition-warning-text { - color: var(--warning-background); + color: var(--warning-background); } .admonition-error { - border-color: var(--error-background); + border-color: var(--error-background); } .admonition-error-text { - color: var(--error-background); + color: var(--error-background); } .first { @@ -689,8 +750,7 @@ div.footer, div.header { font-size: smaller; } div.footer { - padding-top: 5em; -} + padding-top: 5em; } div.line-block { display: block; @@ -707,19 +767,23 @@ div.topic { div.search_results { background-color: var(--third-background); - margin: 3em; + margin: 3vh 5vw; padding: 1em; border: 1px solid #4d4d4d; -} + position: fixed; + top: 10px; + isolation: isolate; + max-width: calc(100vw - 6em); + z-index: 1; + max-height: calc(100vh - 6em); + overflow-y: scroll;} div#global-links ul { margin-left: 0; - list-style-type: none; -} + list-style-type: none; } div#global-links > simple-boot { - margin-left: 3em; -} + margin-left: 3em; } hr.docutils { width: 75%; } @@ -817,9 +881,6 @@ span.classifier { span.classifier-delimiter { font-weight: bold; } -span.option { - white-space: nowrap; } - span.problematic { color: #b30000; } @@ -899,8 +960,29 @@ span.Preprocessor { span.Directive { color: #252dbe; } -span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference, -span.Other { +span.option { + font-weight: bold; + font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; + color: var(--option); } + +span.Prompt { + font-weight: bold; + color: red; } + +span.ProgramOutput { + font-weight: bold; + color: #808080; } + +span.program { + font-weight: bold; + color: var(--program); + text-decoration: underline; + text-decoration-color: var(--hint); + text-decoration-thickness: 0.05em; + text-underline-offset: 0.15em; } + +span.Command, span.Rule, span.Hyperlink, +span.Label, span.Reference, span.Other { color: var(--other); } /* Pop type, const, proc, and iterator defs in nim def blocks */ @@ -938,17 +1020,14 @@ span.pragmadots { border-radius: 4px; margin: 0 2px; cursor: pointer; - font-size: 0.8em; -} + font-size: 0.8em; } span.pragmadots:hover { - background-color: var(--hint); -} + background-color: var(--hint); } + span.pragmawrap { - display: none; -} + display: none; } span.attachedType { display: none; - visibility: hidden; -} + visibility: hidden; } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index cb80f2873c..6decf79a3c 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1,153 +1,214 @@ - + - + - - - - - +subdir/subdir_b/utils + + + + -subdir/subdir_b/utils + - - - -
    -
    -

    subdir/subdir_b/utils

    -
    +
    +
    +

    subdir/subdir_b/utils

    +
    -
    - -     Dark Mode -
    - -
    - Search: -
    -
    - Group by: - -
    -
    -
    -
    - -

    + +

    + +

    This is a description of the utils module.

    +

    Links work:

    + +

    This is now a header

    Next header

    And so on

    @@ -159,73 +220,395 @@ window.addEventListener('DOMContentLoaded', main);
    1. Other case value
    2. Second case.
    +

    Ref group fn2 or specific function like fn2() or fn2( int ) or fn2(int, float).

    +

    Ref generics like this: binarySearch or binarySearch(openArray[T], K, proc (T, K)) or proc binarySearch(openArray[T], K, proc (T, K)) or in different style: proc binarysearch(openarray[T], K, proc(T, K)). Can be combined with export symbols and type parameters: binarysearch*[T, K](openArray[T], K, proc (T, K)). With spaces binary search.

    +

    Note that proc can be used in postfix form: binarySearch proc.

    +

    Ref. type like G and type G and G[T] and type G*[T].

    +

    Group ref. with capital letters works: fN11 or fn11

    +Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string). +

    Pandoc Markdown

    Now repeat all the auto links of above in Pandoc Markdown Syntax.

    +

    Ref group fn2 or specific function like fn2() or fn2( int ) or fn2(int, float).

    +

    Ref generics like this: binarySearch or binarySearch(openArray[T], K, proc (T, K)) or proc binarySearch(openArray[T], K, proc (T, K)) or in different style: proc binarysearch(openarray[T], K, proc(T, K)). Can be combined with export symbols and type parameters: binarysearch*[T, K](openArray[T], K, proc (T, K)). With spaces binary search.

    +

    Note that proc can be used in postfix form: binarySearch proc.

    +

    Ref. type like G and type G and G[T] and type G*[T].

    +

    Group ref. with capital letters works: fN11 or fn11

    +

    Ref. `[]` is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): T Ref. `[]=` aka `[]=`(G[T], int, T). Ref. $ aka proc $ or proc `$`. Ref. $(a: ref SomeType). Ref. foo_bar aka iterator foo_bar_. Ref. fn[T; U,V: SomeFloat](). Ref. 'big or func `'big` or `'big`(string).

    + +

    Link name syntax

    Pandoc Markdown has synax for changing text of links: Ref. this proc or another symbol.

    + +

    Symbols documentation

    Let us repeat auto links from symbols section below:

    +

    There is also variant f(G[string]). See also f(G[int]).

    -
    -

    Types

    -
    - -
    SomeType = enum
+    
    
+  
    Types
    
+  
    
+    
    
+  
    G[T] = object
    
+  
    
+    
+    
+    
+  
    
+
    
+
    
+  
    SomeType = enum
   enumValueA, enumValueB, enumValueC
    
-
    
+  
    
+    
+    
+    
+  
    
+
    
 
-
-
-
-
-
    
+
    +
    -

    Procs

    -
    - -
    proc someType(): SomeType {...}{.raises: [], tags: [].}
    -
    +

    Procs

    +
    +
    +
    + 
    proc `$`[T](a: G[T]): string
    +
    + + + +
    +
    +
    + 
    proc `$`[T](a: ref SomeType): string
    +
    + + + +
    +
    -constructor. +
    +
    +
    + 
    func `'big`(a: string): SomeType {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    -
    +
    +
    +
    + 
    proc `[]`[T](x: G[T]): T
    +
    + + + +
    +
    -
    +
    +
    +
    + 
    proc `[]=`[T](a: var G[T]; index: int; value: T)
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc binarySearch[T, K](a: openArray[T]; key: K;
+                        cmp: proc (x: T; y: K): int {.closure.}): int
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc f(x: G[int]) {....raises: [], tags: [], forbids: [].}
    +
    + + There is also variant f(G[string]) + +
    +
    +
    + 
    proc f(x: G[string]) {....raises: [], tags: [], forbids: [].}
    +
    + + See also f(G[int]). + +
    +
    + +
    +
    +
    + 
    proc fn[T; U, V: SomeFloat]()
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc fn2() {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    +
    + 
    proc fn2(x: int) {....raises: [], tags: [], forbids: [].}
    +
    + + fn2 comment + +
    +
    +
    + 
    proc fn2(x: int; y: float) {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    +
    + 
    proc fn2(x: int; y: float; z: float) {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc fn3(): auto {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    proc fn4(): auto {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    proc fn5() {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    proc fn6() {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    proc fn7() {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    proc fn8(): auto {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    func fn9(a: int): int {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    func fn10(a: int): int {....raises: [], tags: [], forbids: [].}
    +
    + + comment + +
    +
    + +
    +
    +
    + 
    func fN11() {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    +
    + 
    func fN11(x: int) {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc someType(): SomeType {....raises: [], tags: [], forbids: [].}
    +
    + + constructor. + +
    +
    + +
    + + +
    +
    +

    Iterators

    +
    +
    +
    + 
    iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    + +
    + +
    +
    -

    Templates

    -
    - -
    template aEnum(): untyped
    -
    +

    Templates

    +
    +
    +
    + 
    template aEnum(): untyped
    +
    + + + +
    +
    +
    +
    +
    + 
    template bEnum(): untyped
    +
    + + + +
    +
    - -
    - -
    template bEnum(): untyped
    -
    - - - -
    - -
    template fromUtilsGen(): untyped
    -
    - -this should be shown in utils.html +
    +
    +
    + 
    template fromUtilsGen(): untyped
    +
    + + should be shown in utils.html only

    Example:

    -
    assert 3*2 == 6
    ditto +
    discard "should be in utils.html only, not in module that calls fromUtilsGen"
    ditto + +
    +
    - +
    -
    + +
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    -
    - + diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index b49a777c8e..81b27bcb9f 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,13 +1,47 @@ -This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header -Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header -And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on -More headers subdir/subdir_b/utils.html#more-headers More headers -Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 -enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA -enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB -enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC -SomeType subdir/subdir_b/utils.html#SomeType utils: SomeType -someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType -aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped -bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped -fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped +nimTitle utils subdir/subdir_b/utils.html module subdir/subdir_b/utils 0 +nim funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U proc funWithGenerics[T, U: SomeFloat](a: T; b: U) 1 +nim fn2 subdir/subdir_b/utils.html#fn2,int,float,float proc fn2(x: int; y: float; z: float) 5 +nim enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA 45 +nim enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB 45 +nim enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC 45 +nim SomeType subdir/subdir_b/utils.html#SomeType enum SomeType 45 +nim G subdir/subdir_b/utils.html#G object G 49 +nim someType subdir/subdir_b/utils.html#someType_2 proc someType(): SomeType 52 +nim fn2 subdir/subdir_b/utils.html#fn2 proc fn2() 57 +nim fn2 subdir/subdir_b/utils.html#fn2,int proc fn2(x: int) 58 +nim fn2 subdir/subdir_b/utils.html#fn2,int,float proc fn2(x: int; y: float) 61 +nim binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) proc binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int 63 +nim fn3 subdir/subdir_b/utils.html#fn3 proc fn3(): auto 66 +nim fn4 subdir/subdir_b/utils.html#fn4 proc fn4(): auto 67 +nim fn5 subdir/subdir_b/utils.html#fn5 proc fn5() 68 +nim fn6 subdir/subdir_b/utils.html#fn6 proc fn6() 70 +nim fn7 subdir/subdir_b/utils.html#fn7 proc fn7() 72 +nim fn8 subdir/subdir_b/utils.html#fn8 proc fn8(): auto 75 +nim fn9 subdir/subdir_b/utils.html#fn9,int proc fn9(a: int): int 78 +nim fn10 subdir/subdir_b/utils.html#fn10,int proc fn10(a: int): int 79 +nim fN11 subdir/subdir_b/utils.html#fN11 proc fN11() 85 +nim fN11 subdir/subdir_b/utils.html#fN11,int proc fN11(x: int) 86 +nim aEnum subdir/subdir_b/utils.html#aEnum.t template aEnum(): untyped 90 +nim bEnum subdir/subdir_b/utils.html#bEnum.t template bEnum(): untyped 95 +nim fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t template fromUtilsGen(): untyped 106 +nim f subdir/subdir_b/utils.html#f,G[int] proc f(x: G[int]) 130 +nim f subdir/subdir_b/utils.html#f,G[string] proc f(x: G[string]) 133 +nim `[]` subdir/subdir_b/utils.html#[],G[T] proc `[]`[T](x: G[T]): T 140 +nim `[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T proc `[]=`[T](a: var G[T]; index: int; value: T) 144 +nim `$` subdir/subdir_b/utils.html#$,G[T] proc `$`[T](a: G[T]): string 148 +nim `$` subdir/subdir_b/utils.html#$,ref.SomeType proc `$`[T](a: ref SomeType): string 152 +nim fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] iterator fooBar(a: seq[SomeType]): int 156 +nim fn subdir/subdir_b/utils.html#fn proc fn[T; U, V: SomeFloat]() 160 +nim `'big` subdir/subdir_b/utils.html#'big,string proc `'big`(a: string): SomeType 164 +nimgrp $ subdir/subdir_b/utils.html#$-procs-all proc 148 +nimgrp fn11 subdir/subdir_b/utils.html#fN11-procs-all proc 85 +nimgrp fn2 subdir/subdir_b/utils.html#fn2-procs-all proc 5 +nimgrp f subdir/subdir_b/utils.html#f-procs-all proc 130 +heading This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header 0 +heading Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header 0 +heading And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on 0 +heading More headers subdir/subdir_b/utils.html#more-headers More headers 0 +heading Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 0 +heading Pandoc Markdown subdir/subdir_b/utils.html#pandoc-markdown Pandoc Markdown 0 +heading Link name syntax subdir/subdir_b/utils.html#pandoc-markdown-link-name-syntax Link name syntax 0 +heading Symbols documentation subdir/subdir_b/utils.html#pandoc-markdown-symbols-documentation Symbols documentation 0 diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 48b3094e7e..43a72d99db 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -1,736 +1,538 @@ - + - + - - - - - +testproject + + + + -testproject + - - - -
    -
    -

    testproject

    -
    +
    +
    +

    testproject

    +
    -
    - -     Dark Mode -
    - -
    - Search: -
    -
    - Group by: - -
    -
    -
    -
    - -

    This is the top level module. + +

    + +

    +

    Basic usage

    +

    Encoding data

    Apart from strings you can also encode lists of integers or characters:

    + +

    Decoding data

    This is the top level module.

    Example:

    -
    import subdir / subdir_b / utils
-doAssert bar(3, 4) == 7
-foo(enumValueA, enumValueB)
-# bug #11078
-for x in "xx": discard
    top2 +
    import testproject
+import subdir / subdir_b / utils
+doAssert bar(3, 4) == 7
+foo(enumValueA, enumValueB)
+# bug #11078
+for x in "xx": discard
    top2

    Example:

    -
    discard "in top2"
    top2 after +
    import testproject
+discard "in top2"
    top2 after

    Example:

    -
    discard "in top3"
    top3 after -

    Example:

    -
    assert 3*2 == 6

    - +
    import testproject
+discard "in top3"
    top3 after

    +
    -

    Types

    -
    - -
    A {...}{.inject.} = enum
+  
    Types
    
+  
    
+    
    
+  
    A {.inject.} = enum
   aA
    
-
    
-
-The enum A.
-
-
    
-
-
    B {...}{.inject.} = enum
+  
    
+    
+    The enum A.
+    
+  
    
+
    
+
    
+  
    AnotherObject = object
+  case x*: bool
+  of true:
+    y*: proc (x: string)
+  of false:
    
+  
    
+    
+    
+    
+  
    
+
    
+
    
+  
    B {.inject.} = enum
   bB
    
-
    
-
-The enum B.
-
-
    
-
-
    Foo = enum
+  
    
+    
+    The enum B.
+    
+  
    
+
    
+
    
+  
    Foo = enum
   enumValueA2
    
-
    
+  
    
+    
+    
+    
+  
    
+
    
+
    
+  
    FooBuzz {....deprecated: "FooBuzz msg".} = int
    
+  
    
+    
    
+  Deprecated: FooBuzz msg
+
    
 
-
-
-
    
-
-
    Shapes = enum
+    
+    
+  
+
    
+
    
+  
    MyObject = object
+  someString*: string        ## This is a string
+  annotated* {.somePragma.}: string ## This is an annotated string
    
+  
    
+    
+    
+    
+  
    
+
    
+
    
+  
    Shapes = enum
   Circle,                   ## A circle
   Triangle,                 ## A three-sided shape
   Rectangle                  ## A four-sided shape
    
-
    
+  
    
+    
+    Some shapes.
+    
+  
    
+
    
+
    
+  
    T19396 = object
+  a*: int
    
+  
    
+    
+    
+    
+  
    
+
    
 
-Some shapes.
-
-
-
-
    + +
    -

    Vars

    -
    - -
    someVariable: bool
    -
    +

    Vars

    +
    +
    + 
    aVariable: array[1, int]
    +
    + + + +
    +
    +
    + 
    someVariable: bool
    +
    + + This should be visible. + +
    +
    -This should be visible. - -
    - -
    aVariable: array[1, int]
    -
    - - - -
    - -
    + +
    -

    Consts

    -
    - -
    C_A = 0x7FF0000000000000'f64
    -
    +

    Consts

    +
    +
    + 
    C_A = 0x7FF0000000000000'f64
    +
    + + + +
    +
    +
    + 
    C_B = 0o377'i8
    +
    + + + +
    +
    +
    + 
    C_C = 0o277'i8
    +
    + + + +
    +
    +
    + 
    C_D = 0o177777'i16
    +
    + + + +
    +
    - - -
    - -
    C_B = 0o377'i8
    -
    - - - -
    - -
    C_C = 0o277'i8
    -
    - - - -
    - -
    C_D = 0o177777'i16
    -
    - - - -
    - -
    + +
    -

    Procs

    -
    - -
    proc bar[T](a, b: T): T
    -
    - - - -
    - -
    proc baz[T](a, b: T): T {...}{.deprecated.}
    -
    -
    - Deprecated -
    - -This is deprecated without message. - -
    - -
    proc buzz[T](a, b: T): T {...}{.deprecated: "since v0.20".}
    -
    -
    - Deprecated: since v0.20 -
    - -This is deprecated with a message. - -
    - -
    func someFunc() {...}{.raises: [], tags: [].}
    -
    - -My someFunc. Stuff in quotes here. Some link - -
    - -
    proc fromUtils3() {...}{.raises: [], tags: [].}
    -
    - -came form utils but should be shown where fromUtilsGen is called +

    Procs

    +
    +
    +
    + 
    proc addfBug14485() {....raises: [], tags: [], forbids: [].}
    +
    + + Some proc

    Example:

    -
    discard 1
    - -
    - -
    proc isValid[T](x: T): bool
    -
    - - - -
    - -
    proc z1(): Foo {...}{.raises: [], tags: [].}
    -
    - -cz1 - -
    - -
    proc z2() {...}{.raises: [], tags: [].}
    -
    - -cz2 -

    Example:

    -
    discard "in cz2"
    - -
    - -
    proc z3() {...}{.raises: [], tags: [].}
    -
    - -cz3 - -
    - -
    proc z4() {...}{.raises: [], tags: [].}
    -
    - -cz4 - -
    - -
    proc z5(): int {...}{.raises: [], tags: [].}
    -
    - -cz5 - -
    - -
    proc z6(): int {...}{.raises: [], tags: [].}
    -
    - -cz6 - -
    - -
    proc z7(): int {...}{.raises: [], tags: [].}
    -
    - -cz7 - -
    - -
    proc z8(): int {...}{.raises: [], tags: [].}
    -
    - -cz8 - -
    - -
    proc z9() {...}{.raises: [], tags: [].}
    -
    - - -

    Example:

    -
    doAssert 1 + 1 == 2
    - -
    - -
    proc z10() {...}{.raises: [], tags: [].}
    -
    - - -

    Example: cmd: -d:foobar

    -
    discard 1
    cz10 - -
    - -
    proc z11() {...}{.raises: [], tags: [].}
    -
    - - -

    Example:

    -
    discard 1
    - -
    - -
    proc z12(): int {...}{.raises: [], tags: [].}
    -
    - - -

    Example:

    -
    discard 1
    - -
    - -
    proc z13() {...}{.raises: [], tags: [].}
    -
    - -cz13 -

    Example:

    -
    discard
    - -
    - -
    proc baz() {...}{.raises: [], tags: [].}
    -
    - - - -
    - -
    proc z17() {...}{.raises: [], tags: [].}
    -
    - -cz17 rest -

    Example:

    -
    discard 1
    rest - -
    - -
    proc p1() {...}{.raises: [], tags: [].}
    -
    - -cp1 -

    Example:

    -
    doAssert 1 == 1 # regular comments work here
    c4 -

    Example:

    -
    # c5 regular comments before 1st token work
-# regular comment
-#[
-nested regular comment
-]#
-doAssert 2 == 2 # c8
-## this is a non-nested doc comment
-
-##[
-this is a nested doc comment
-]##
-discard "c9"
-# also work after
    - -
    - -
    proc addfBug14485() {...}{.raises: [], tags: [].}
    -
    - -Some proc -

    Example:

    -
    discard "foo() = " & $[1]
-#[
-0: let's also add some broken html to make sure this won't break in future
+
    discard "foo() = " & $[1]
+#[
+0: let's also add some broken html to make sure this won't break in future
 1: </span>
 2: </span>
 3: </span
@@ -739,306 +541,720 @@ Some proc
 6: </script
 7: end of broken html
 ]#
    
+    
+
    +
    -
    - -
    proc c_printf(frmt: cstring): cint {...}{.importc: "printf", header: "<stdio.h>",
-                                     varargs, discardable.}
    -
    +
    +
    +
    + 
    proc anything() {....raises: [], tags: [], forbids: [].}
    +
    + + There is no block quote after blank lines at the beginning. + +
    +
    -the c printf. etc. +
    +
    +
    + 
    proc asyncFun1(): Future[int] {....stackTrace: false,
+                                raises: [Exception, ValueError],
+                                tags: [RootEffect], forbids: [].}
    +
    + + ok1 + +
    +
    - - -
    proc c_nonexistant(frmt: cstring): cint {...}{.importc: "nonexistant",
-    header: "<stdio.h>", varargs, discardable.}
    -
    +
    +
    +
    + 
    proc asyncFun2(): owned(Future[void]) {....stackTrace: false, raises: [Exception],
+                                        tags: [RootEffect], forbids: [].}
    +
    + + + +
    +
    +
    +
    +
    + 
    proc asyncFun3(): owned(Future[void]) {....stackTrace: false, raises: [Exception],
+                                        tags: [RootEffect], forbids: [].}
    +
    + + +

    Example:

    +
    discard
    ok1 + +
    +
    +
    +
    +
    + 
    proc bar(f: FooBuzz) {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    +
    + 
    proc bar[T](a, b: T): T
    +
    + + + +
    +
    - - -
    proc low[T: Ordinal | enum | range](x: T): T {...}{.magic: "Low", noSideEffect.}
    -
    +
    +
    +
    + 
    proc baz() {....raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    +
    + 
    proc baz[T](a, b: T): T {....deprecated.}
    +
    +
    + Deprecated +
    -

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    + This is deprecated without message. + +
    +
    + +
    +
    +
    + 
    proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
    +
    +
    + Deprecated: since v0.20 +
    + + This is deprecated with a message. + +
    +
    + +
    +
    +
    + 
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
+    header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [], forbids: [].}
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
+                                     varargs, discardable, ...raises: [], tags: [],
+                                     forbids: [].}
    +
    + + the c printf. etc. + +
    +
    + +
    +
    +
    + 
    proc fromUtils3() {....raises: [], tags: [], forbids: [].}
    +
    + + came form utils but should be shown where fromUtilsGen is called +

    Example:

    +
    discard """should be shown as examples for fromUtils3
+       in module calling fromUtilsGen"""
    + +
    +
    + +
    +
    +
    + 
    proc isValid[T](x: T): bool
    +
    + + + +
    +
    + +
    +
    +
    + 
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [], forbids: [].}
    +
    + +

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    See also:

    low(2) # => -9223372036854775808
    + +
    +
    - - -
    proc low2[T: Ordinal | enum | range](x: T): T {...}{.magic: "Low", noSideEffect.}
    -
    - -

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    +
    +
    +
    + 
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [], forbids: [].}
    +
    + +

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    See also:

    low2(2) # => -9223372036854775808

    Example:

    -
    discard "in low2"
    - -
    - -
    proc tripleStrLitTest() {...}{.raises: [], tags: [].}
    -
    - +
    discard "in low2"
    + +
    +
    +
    +
    +
    + 
    proc p1() {....raises: [], tags: [], forbids: [].}
    +
    + + cp1

    Example:

    -
    ## mullitline string litterals are tricky as their indentation can span
-## below that of the runnableExamples
-let s1a = """
+
    doAssert 1 == 1 # regular comments work here
    c4
+
    Example:
    
+
    # c5 regular comments before 1st token work
+# regular comment
+#[
+nested regular comment
+]#
+doAssert 2 == 2 # c8
+## this is a non-nested doc comment
+
+##[
+this is a nested doc comment
+]##
+discard "c9"
+# also work after
    
+    
+
    +
    + +
    +
    +
    + 
    func someFunc() {....raises: [], tags: [], forbids: [].}
    +
    + + My someFunc. Stuff in quotes here. Some link + +
    +
    + +
    +
    +
    + 
    proc tripleStrLitTest() {....raises: [], tags: [], forbids: [].}
    +
    + + +

    Example: cmd: --hint:XDeclaredButNotUsed:off

    +
    ## mullitline string litterals are tricky as their indentation can span
+## below that of the runnableExamples
+let s1a = """
 should appear at indent 0
   at indent 2
 at indent 0
-"""
-# make sure this works too
-let s1b = """start at same line
+"""
+# make sure this works too
+let s1b = """start at same line
   at indent 2
 at indent 0
-""" # comment after
-let s2 = """sandwich """
-let s3 = """"""
-when false:
-  let s5 = """
-        in s5 """
+""" # comment after
+let s2 = """sandwich """
+let s3 = """"""
+when false:
+  let s5 = """
+        in s5 """
 
-let s3b = ["""
+let s3b = ["""
 %!? #[...] # inside a multiline ...
-""", "foo"]
+""", "foo"]
 
-## make sure handles trailing spaces
-let s4 = """ 
-"""
+## make sure handles trailing spaces
+let s4 = """ 
+"""
 
-let s5 = """ x
-"""
-let s6 = """ ""
-"""
-let s7 = """"""""""
-let s8 = ["""""""""", """
-  """ ]
-discard
-# should be in
    - -
    - -
    proc asyncFun1(): Future[int] {...}{.raises: [Exception, ValueError],
-                                tags: [RootEffect].}
    -
    - -ok1 - -
    - -
    proc asyncFun2(): owned(Future[void]) {...}{.raises: [Exception], tags: [RootEffect].}
    -
    - - - -
    - -
    proc asyncFun3(): owned(Future[void]) {...}{.raises: [Exception], tags: [RootEffect].}
    -
    +let s5 = """ x +""" +let s6 = """ "" +""" +let s7 = """""""""" +let s8 = ["""""""""", """ + """ ] +discard +# should be in + +
    +
    +
    +
    +
    + 
    proc z1(): Foo {....raises: [], tags: [], forbids: [].}
    +
    + + cz1 + +
    +
    +
    +
    +
    + 
    proc z2() {....raises: [], tags: [], forbids: [].}
    +
    + + cz2

    Example:

    -
    discard
    ok1 +
    discard "in cz2"
    + +
    +
    - - -
    proc anything() {...}{.raises: [], tags: [].}
    -
    +
    +
    +
    + 
    proc z3() {....raises: [], tags: [], forbids: [].}
    +
    + + cz3 + +
    +
    -There is no block quote after blank lines at the beginning. +
    +
    +
    + 
    proc z4() {....raises: [], tags: [], forbids: [].}
    +
    + + cz4 + +
    +
    - +
    +
    +
    + 
    proc z5(): int {....raises: [], tags: [], forbids: [].}
    +
    + + cz5 + +
    +
    -
    +
    +
    +
    + 
    proc z6(): int {....raises: [], tags: [], forbids: [].}
    +
    + + cz6 + +
    +
    + +
    +
    +
    + 
    proc z7(): int {....raises: [], tags: [], forbids: [].}
    +
    + + cz7 + +
    +
    + +
    +
    +
    + 
    proc z8(): int {....raises: [], tags: [], forbids: [].}
    +
    + + cz8 + +
    +
    + +
    +
    +
    + 
    proc z9() {....raises: [], tags: [], forbids: [].}
    +
    + + +

    Example:

    +
    doAssert 1 + 1 == 2
    + +
    +
    + +
    +
    +
    + 
    proc z10() {....raises: [], tags: [], forbids: [].}
    +
    + + +

    Example: cmd: -d:foobar

    +
    discard 1
    cz10 + +
    +
    + +
    +
    +
    + 
    proc z11() {....raises: [], tags: [], forbids: [].}
    +
    + + +

    Example:

    +
    discard 1
    + +
    +
    + +
    +
    +
    + 
    proc z12(): int {....raises: [], tags: [], forbids: [].}
    +
    + + +

    Example:

    +
    discard 1
    + +
    +
    + +
    +
    +
    + 
    proc z13() {....raises: [], tags: [], forbids: [].}
    +
    + + cz13 +

    Example:

    +
    discard
    + +
    +
    + +
    +
    +
    + 
    proc z17() {....raises: [], tags: [], forbids: [].}
    +
    + + cz17 rest +

    Example:

    +
    discard 1
    rest + +
    +
    + +
    + + +
    -

    Methods

    -
    - -
    method method1(self: Moo) {...}{.raises: [], tags: [].}
    -
    +

    Methods

    +
    +
    +
    + 
    method method1(self: Moo) {.base, ...raises: [], tags: [], forbids: [].}
    +
    + + foo1 + +
    +
    -foo1 +
    +
    +
    + 
    method method2(self: Moo): int {.base, ...raises: [], tags: [], forbids: [].}
    +
    + + foo2 + +
    +
    -
    - -
    method method2(self: Moo): int {...}{.raises: [], tags: [].}
    -
    +
    +
    +
    + 
    method method3(self: Moo): int {.base, ...raises: [], tags: [], forbids: [].}
    +
    + + foo3 + +
    +
    -foo2 +
    - - -
    method method3(self: Moo): int {...}{.raises: [], tags: [].}
    -
    - -foo3 - -
    - -
    + +
    -

    Iterators

    -
    - -
    iterator fromUtils1(): int {...}{.raises: [], tags: [].}
    -
    - - +

    Iterators

    +
    +
    +
    + 
    iterator fromUtils1(): int {....raises: [], tags: [], forbids: [].}
    +
    + +

    Example:

    -
    # ok1
-assert 1 == 1
-# ok2
    +
    # ok1
+assert 1 == 1
+# ok2
    + +
    +
    -
    - -
    iterator iter1(n: int): int {...}{.raises: [], tags: [].}
    -
    +
    +
    +
    + 
    iterator iter1(n: int): int {....raises: [], tags: [], forbids: [].}
    +
    + + foo1 + +
    +
    -foo1 - - - -
    iterator iter2(n: int): int {...}{.raises: [], tags: [].}
    -
    - -foo2 +
    +
    +
    + 
    iterator iter2(n: int): int {....raises: [], tags: [], forbids: [].}
    +
    + + foo2

    Example:

    -
    discard # bar
    +
    discard # bar
    + +
    +
    - +
    -
    + +
    -

    Macros

    -
    - -
    macro bar(): untyped
    -
    - - - -
    - -
    macro z16()
    -
    - +

    Macros

    +
    +
    +
    + 
    macro bar(): untyped
    +
    + + + +
    +
    +
    +
    +
    + 
    macro z16()
    +
    + +

    Example:

    -
    discard 1
    cz16 after +
    discard 1
    cz16 after

    Example:

    -
    doAssert 2 == 1 + 1
    +
    doAssert 2 == 1 + 1
    + +
    +
    -
    - -
    macro z18(): int
    -
    +
    +
    +
    + 
    macro z18(): int
    +
    + + cz18 + +
    +
    -cz18 +
    - - -
    + +
    -

    Templates

    -
    - -
    template fromUtils2()
    -
    - -ok3 - -
    - -
    template z6t(): int
    -
    - -cz6t - -
    - -
    template foo(a, b: SomeType)
    -
    - -This does nothing - -
    - -
    template myfn()
    -
    - +

    Templates

    +
    +
    +
    + 
    template foo(a, b: SomeType)
    +
    + + This does nothing + +
    +
    +
    +
    +
    + 
    template fromUtils2()
    +
    + + ok3

    Example:

    -
    import std/strutils
-## issue #8871 preserve formatting
-## line doc comment
-# bar
-doAssert "'foo" == "'foo"
-##[
+
    discard """should be shown as examples for fromUtils2
+       in module calling fromUtilsGen"""
    
+    
+
    +
    + +
    +
    +
    + 
    template myfn()
    +
    + + +

    Example:

    +
    import std/strutils
+## issue #8871 preserve formatting
+## line doc comment
+# bar
+doAssert "'foo" == "'foo"
+##[
 foo
 bar
-]##
+]##
 
-doAssert: not "foo".startsWith "ba"
-block:
-  discard 0xff # elu par cette crapule
-# should be in
    should be still in +doAssert: not "foo".startsWith "ba" +block: + discard 0xff # elu par cette crapule +# should be inshould be still in + +
    +
    -
    - -
    template z14()
    -
    +
    +
    +
    + 
    template somePragma() {.pragma.}
    +
    + + Just some annotation + +
    +
    -cz14 +
    +
    +
    + 
    template testNimDocTrailingExample()
    +
    + + +

    Example:

    +
    discard 2
    + +
    +
    + +
    +
    +
    + 
    template z6t(): int
    +
    + + cz6t + +
    +
    + +
    +
    +
    + 
    template z14()
    +
    + + cz14

    Example:

    discard
    + +
    +
    - - -
    template z15()
    -
    - -cz15 +
    +
    +
    + 
    template z15()
    +
    + + cz15

    Example:

    discard

    Example:

    -
    discard 3
    +
    discard 3

    Example:

    -
    discard 4
    ok5 ok5b +
    discard 4
    ok5 ok5b

    Example:

    -
    assert true
    in or out? - -
    - -
    template testNimDocTrailingExample()
    -
    - - +
    assert true

    Example:

    -
    discard 2
    +
    discard 1
    in or out? + +
    +
    - +
    -
    + +
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    - - + diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index 1d8be99dac..81d65a05a2 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -1,62 +1,74 @@ -someVariable testproject.html#someVariable testproject: someVariable -C_A testproject.html#C_A testproject: C_A -C_B testproject.html#C_B testproject: C_B -C_C testproject.html#C_C testproject: C_C -C_D testproject.html#C_D testproject: C_D -bar testproject.html#bar,T,T testproject: bar[T](a, b: T): T -baz testproject.html#baz,T,T testproject: baz[T](a, b: T): T -buzz testproject.html#buzz,T,T testproject: buzz[T](a, b: T): T -aVariable testproject.html#aVariable testproject: aVariable -A testproject.html#A testproject: A -B testproject.html#B testproject: B -someFunc testproject.html#someFunc testproject: someFunc() -fromUtils1 testproject.html#fromUtils1.i testproject: fromUtils1(): int -fromUtils2 testproject.html#fromUtils2.t testproject: fromUtils2() -fromUtils3 testproject.html#fromUtils3 testproject: fromUtils3() -isValid testproject.html#isValid,T testproject: isValid[T](x: T): bool -enumValueA2 testproject.html#enumValueA2 Foo.enumValueA2 -Foo testproject.html#Foo testproject: Foo -z1 testproject.html#z1 testproject: z1(): Foo -z2 testproject.html#z2 testproject: z2() -z3 testproject.html#z3 testproject: z3() -z4 testproject.html#z4 testproject: z4() -z5 testproject.html#z5 testproject: z5(): int -z6 testproject.html#z6 testproject: z6(): int -z6t testproject.html#z6t.t testproject: z6t(): int -z7 testproject.html#z7 testproject: z7(): int -z8 testproject.html#z8 testproject: z8(): int -z9 testproject.html#z9 testproject: z9() -z10 testproject.html#z10 testproject: z10() -z11 testproject.html#z11 testproject: z11() -z12 testproject.html#z12 testproject: z12(): int -z13 testproject.html#z13 testproject: z13() -baz testproject.html#baz testproject: baz() -z17 testproject.html#z17 testproject: z17() -p1 testproject.html#p1 testproject: p1() -addfBug14485 testproject.html#addfBug14485 testproject: addfBug14485() -c_printf testproject.html#c_printf,cstring testproject: c_printf(frmt: cstring): cint -c_nonexistant testproject.html#c_nonexistant,cstring testproject: c_nonexistant(frmt: cstring): cint -low testproject.html#low,T testproject: low[T: Ordinal | enum | range](x: T): T -low2 testproject.html#low2,T testproject: low2[T: Ordinal | enum | range](x: T): T -tripleStrLitTest testproject.html#tripleStrLitTest testproject: tripleStrLitTest() -method1 testproject.html#method1.e,Moo testproject: method1(self: Moo) -method2 testproject.html#method2.e,Moo testproject: method2(self: Moo): int -method3 testproject.html#method3.e,Moo testproject: method3(self: Moo): int -iter1 testproject.html#iter1.i,int testproject: iter1(n: int): int -iter2 testproject.html#iter2.i,int testproject: iter2(n: int): int -bar testproject.html#bar.m testproject: bar(): untyped -z16 testproject.html#z16.m testproject: z16() -z18 testproject.html#z18.m testproject: z18(): int -foo testproject.html#foo.t,SomeType,SomeType testproject: foo(a, b: SomeType) -myfn testproject.html#myfn.t testproject: myfn() -z14 testproject.html#z14.t testproject: z14() -z15 testproject.html#z15.t testproject: z15() -asyncFun1 testproject.html#asyncFun1 testproject: asyncFun1(): Future[int] -asyncFun2 testproject.html#asyncFun2 testproject: asyncFun2(): owned(Future[void]) -asyncFun3 testproject.html#asyncFun3 testproject: asyncFun3(): owned(Future[void]) -testNimDocTrailingExample testproject.html#testNimDocTrailingExample.t testproject: testNimDocTrailingExample() -Circle testproject.html#Circle Shapes.Circle -Triangle testproject.html#Triangle Shapes.Triangle -Rectangle testproject.html#Rectangle Shapes.Rectangle -Shapes testproject.html#Shapes testproject: Shapes -anything testproject.html#anything testproject: anything() +nimTitle testproject testproject.html module testproject 0 +nim someVariable testproject.html#someVariable var someVariable 25 +nim C_A testproject.html#C_A const C_A 38 +nim C_B testproject.html#C_B const C_B 39 +nim C_C testproject.html#C_C const C_C 40 +nim C_D testproject.html#C_D const C_D 41 +nim bar testproject.html#bar,T,T proc bar[T](a, b: T): T 43 +nim baz testproject.html#baz,T,T proc baz[T](a, b: T): T 46 +nim buzz testproject.html#buzz,T,T proc buzz[T](a, b: T): T 50 +nim FooBuzz testproject.html#FooBuzz type FooBuzz 55 +nim bar testproject.html#bar proc bar(f: FooBuzz) 59 +nim aVariable testproject.html#aVariable var aVariable 64 +nim A testproject.html#A enum A 92 +nim B testproject.html#B enum B 97 +nim someFunc testproject.html#someFunc proc someFunc() 68 +nim fromUtils1 testproject.html#fromUtils1.i iterator fromUtils1(): int 112 +nim fromUtils2 testproject.html#fromUtils2.t template fromUtils2() 119 +nim fromUtils3 testproject.html#fromUtils3 proc fromUtils3() 69 +nim isValid testproject.html#isValid,T proc isValid[T](x: T): bool 71 +nim enumValueA2 testproject.html#enumValueA2 Foo.enumValueA2 78 +nim Foo testproject.html#Foo enum Foo 78 +nim z1 testproject.html#z1 proc z1(): Foo 81 +nim z2 testproject.html#z2 proc z2() 85 +nim z3 testproject.html#z3 proc z3() 90 +nim z4 testproject.html#z4 proc z4() 93 +nim z5 testproject.html#z5 proc z5(): int 99 +nim z6 testproject.html#z6 proc z6(): int 103 +nim z6t testproject.html#z6t.t template z6t(): int 107 +nim z7 testproject.html#z7 proc z7(): int 111 +nim z8 testproject.html#z8 proc z8(): int 115 +nim z9 testproject.html#z9 proc z9() 123 +nim z10 testproject.html#z10 proc z10() 126 +nim z11 testproject.html#z11 proc z11() 131 +nim z12 testproject.html#z12 proc z12(): int 136 +nim z13 testproject.html#z13 proc z13() 141 +nim baz testproject.html#baz proc baz() 146 +nim z17 testproject.html#z17 proc z17() 156 +nim p1 testproject.html#p1 proc p1() 168 +nim addfBug14485 testproject.html#addfBug14485 proc addfBug14485() 189 +nim c_printf testproject.html#c_printf,cstring proc c_printf(frmt: cstring): cint 205 +nim c_nonexistent testproject.html#c_nonexistent,cstring proc c_nonexistent(frmt: cstring): cint 209 +nim low testproject.html#low,T proc low[T: Ordinal | enum | range](x: T): T 212 +nim low2 testproject.html#low2,T proc low2[T: Ordinal | enum | range](x: T): T 222 +nim tripleStrLitTest testproject.html#tripleStrLitTest proc tripleStrLitTest() 235 +nim method1 testproject.html#method1.e,Moo method method1(self: Moo) 276 +nim method2 testproject.html#method2.e,Moo method method2(self: Moo): int 278 +nim method3 testproject.html#method3.e,Moo method method3(self: Moo): int 281 +nim iter1 testproject.html#iter1.i,int iterator iter1(n: int): int 286 +nim iter2 testproject.html#iter2.i,int iterator iter2(n: int): int 290 +nim bar testproject.html#bar.m macro bar(): untyped 297 +nim z16 testproject.html#z16.m macro z16() 300 +nim z18 testproject.html#z18.m macro z18(): int 309 +nim foo testproject.html#foo.t,SomeType,SomeType template foo(a, b: SomeType) 314 +nim myfn testproject.html#myfn.t template myfn() 319 +nim z14 testproject.html#z14.t template z14() 340 +nim z15 testproject.html#z15.t template z15() 345 +nim asyncFun1 testproject.html#asyncFun1 proc asyncFun1(): Future[int] 370 +nim asyncFun2 testproject.html#asyncFun2 proc asyncFun2(): owned(Future[void]) 373 +nim asyncFun3 testproject.html#asyncFun3 proc asyncFun3(): owned(Future[void]) 374 +nim testNimDocTrailingExample testproject.html#testNimDocTrailingExample.t template testNimDocTrailingExample() 383 +nim Circle testproject.html#Circle Shapes.Circle 392 +nim Triangle testproject.html#Triangle Shapes.Triangle 392 +nim Rectangle testproject.html#Rectangle Shapes.Rectangle 392 +nim Shapes testproject.html#Shapes enum Shapes 392 +nim anything testproject.html#anything proc anything() 399 +nim T19396 testproject.html#T19396 object T19396 404 +nim somePragma testproject.html#somePragma.t template somePragma() 408 +nim MyObject testproject.html#MyObject object MyObject 412 +nim AnotherObject testproject.html#AnotherObject object AnotherObject 417 +nimgrp bar testproject.html#bar-procs-all proc 43 +nimgrp baz testproject.html#baz-procs-all proc 46 +heading Basic usage testproject.html#basic-usage Basic usage 0 +heading Encoding data testproject.html#basic-usage-encoding-data Encoding data 0 +heading Decoding data testproject.html#basic-usage-decoding-data Decoding data 0 diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 81370d115c..71a487e0cf 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -1,357 +1,431 @@ - + - + - - - - - +Index + + + + -Index + - - - -
    -
    -

    Index

    - Modules: subdir/subdir_b/utils, testproject.

    API symbols

    -
    A:
      +
      +
      +

      Index

      + Modules: subdir/subdir_b/utils, testproject.

      API symbols

      +
      `$`:
      +
      `'big`:
      +
      `[]=`:
      +
      `[]`:
      +
      A:
      addfBug14485:
      • testproject: addfBug14485()
        • + data-doc-search-tag="testproject: proc addfBug14485()" href="testproject.html#addfBug14485">testproject: proc addfBug14485()
      aEnum:
      • utils: aEnum(): untyped
        • + data-doc-search-tag="utils: template aEnum(): untyped" href="subdir/subdir_b/utils.html#aEnum.t">utils: template aEnum(): untyped +
      +
      AnotherObject:
      anything:
      • testproject: anything()
        • + data-doc-search-tag="testproject: proc anything()" href="testproject.html#anything">testproject: proc anything()
      asyncFun1:
      asyncFun2:
      asyncFun3:
      aVariable:
      • testproject: aVariable
        • + data-doc-search-tag="testproject: var aVariable" href="testproject.html#aVariable">testproject: var aVariable
      B:
      • testproject: B
        • + data-doc-search-tag="testproject: enum B" href="testproject.html#B">testproject: enum B
      bar:
      baz:
      • testproject: baz()
        • + data-doc-search-tag="testproject: proc baz()" href="testproject.html#baz">testproject: proc baz()
      • testproject: baz[T](a, b: T): T
        • + data-doc-search-tag="testproject: proc baz[T](a, b: T): T" href="testproject.html#baz%2CT%2CT">testproject: proc baz[T](a, b: T): T
      bEnum:
      • utils: bEnum(): untyped
        • + data-doc-search-tag="utils: template bEnum(): untyped" href="subdir/subdir_b/utils.html#bEnum.t">utils: template bEnum(): untyped +
      +
      binarySearch:
      buzz:
      • testproject: buzz[T](a, b: T): T
        • + data-doc-search-tag="testproject: proc buzz[T](a, b: T): T" href="testproject.html#buzz%2CT%2CT">testproject: proc buzz[T](a, b: T): T
      C_A:
      • testproject: C_A
        • + data-doc-search-tag="testproject: const C_A" href="testproject.html#C_A">testproject: const C_A
      C_B:
      • testproject: C_B
        • + data-doc-search-tag="testproject: const C_B" href="testproject.html#C_B">testproject: const C_B
      C_C:
      • testproject: C_C
        • + data-doc-search-tag="testproject: const C_C" href="testproject.html#C_C">testproject: const C_C
      C_D:
      • testproject: C_D
        • + data-doc-search-tag="testproject: const C_D" href="testproject.html#C_D">testproject: const C_D
      Circle:
      • Shapes.Circle
        • + data-doc-search-tag="testproject: Shapes.Circle" href="testproject.html#Circle">testproject: Shapes.Circle
      -
      c_nonexistant:
      -
      -
      +
      Made with Nim. Generated: 1970-01-02 03:46:40 UTC
      -
      - + diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 128e8e481a..f535d7f743 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -1,5 +1,7 @@ ##[ +.. include:: ./utils_overview.rst + # This is now a header ## Next header @@ -19,18 +21,70 @@ More text. 1. Other case value 2. Second case. +Ref group fn2_ or specific function like `fn2()`_ +or `fn2( int )`_ or `fn2(int, +float)`_. + +Ref generics like this: binarySearch_ or `binarySearch(openArray[T], K, +proc (T, K))`_ or `proc binarySearch(openArray[T], K, proc (T, K))`_ or +in different style: `proc binarysearch(openarray[T], K, proc(T, K))`_. +Can be combined with export symbols and type parameters: +`binarysearch*[T, K](openArray[T], K, proc (T, K))`_. +With spaces `binary search`_. + +Note that `proc` can be used in postfix form: `binarySearch proc`_. + +Ref. type like G_ and `type G`_ and `G[T]`_ and `type G*[T]`_. + +Group ref. with capital letters works: fN11_ or fn11_ ]## +include ./utils_helpers + type SomeType* = enum enumValueA, enumValueB, enumValueC + G*[T] = object + val: T proc someType*(): SomeType = ## constructor. SomeType(2) + +proc fn2*() = discard ## comment +proc fn2*(x: int) = + ## fn2 comment + discard +proc fn2*(x: int, y: float) = + discard +proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.}): int = + discard +proc fn3*(): auto = 1 ## comment +proc fn4*(): auto = 2 * 3 + 4 ## comment +proc fn5*() ## comment +proc fn5*() = discard +proc fn6*() = + ## comment +proc fn7*() = + ## comment + discard +proc fn8*(): auto = + ## comment + 1+1 +func fn9*(a: int): int = 42 ## comment +func fn10*(a: int): int = a ## comment + +# Note capital letter N will be handled correctly in +# group references like fN11_ or fn11_ +# (or [fN11] or [fn11] in Markdown Syntax): + +func fN11*() = discard +func fN11*(x: int) = discard + # bug #9235 template aEnum*(): untyped = @@ -50,9 +104,9 @@ template bEnum*(): untyped = discard template fromUtilsGen*(): untyped = - ## this should be shown in utils.html + ## should be shown in utils.html only runnableExamples: - assert 3*2 == 6 + discard "should be in utils.html only, not in module that calls fromUtilsGen" ## ditto iterator fromUtils1*(): int = @@ -64,7 +118,96 @@ template fromUtilsGen*(): untyped = template fromUtils2*() = ## ok3 + runnableExamples: + discard """should be shown as examples for fromUtils2 + in module calling fromUtilsGen""" proc fromUtils3*() = ## came form utils but should be shown where `fromUtilsGen` is called - runnableExamples: discard 1 + runnableExamples: discard """should be shown as examples for fromUtils3 + in module calling fromUtilsGen""" + +proc f*(x: G[int]) = + ## There is also variant `f(G[string])`_ + discard +proc f*(x: G[string]) = + ## See also `f(G[int])`_. + discard + +## Ref. `[]`_ is the same as `proc \`[]\`(G[T])`_ because there are no +## overloads. The full form: `proc \`[]\`*[T](x: G[T]): T`_ + +proc `[]`*[T](x: G[T]): T = x.val + +## Ref. `[]=`_ aka `\`[]=\`(G[T], int, T)`_. + +proc `[]=`*[T](a: var G[T], index: int, value: T) = discard + +## Ref. `$`_ aka `proc $`_ or `proc \`$\``_. + +proc `$`*[T](a: G[T]): string = "" + +## Ref. `$(a: ref SomeType)`_. + +proc `$`*[T](a: ref SomeType): string = "" + +## Ref. foo_bar_ aka `iterator foo_bar_`_. + +iterator fooBar*(a: seq[SomeType]): int = discard + +## Ref. `fn[T; U,V: SomeFloat]()`_. + +proc fn*[T; U, V: SomeFloat]() = discard + +## Ref. `'big`_ or `func \`'big\``_ or `\`'big\`(string)`_. + +func `'big`*(a: string): SomeType = discard + +##[ + +Pandoc Markdown +=============== + +Now repeat all the auto links of above in Pandoc Markdown Syntax. + +Ref group [fn2] or specific function like [fn2()] +or [fn2( int )] or [fn2(int, +float)]. + +Ref generics like this: [binarySearch] or [binarySearch(openArray[T], K, +proc (T, K))] or [proc binarySearch(openArray[T], K, proc (T, K))] or +in different style: [proc binarysearch(openarray[T], K, proc(T, K))]. +Can be combined with export symbols and type parameters: +[binarysearch*[T, K](openArray[T], K, proc (T, K))]. +With spaces [binary search]. + +Note that `proc` can be used in postfix form: [binarySearch proc]. + +Ref. type like [G] and [type G] and [G[T]] and [type G*[T]]. + +Group ref. with capital letters works: [fN11] or [fn11] + +Ref. [`[]`] is the same as [proc `[]`(G[T])] because there are no +overloads. The full form: [proc `[]`*[T](x: G[T]): T] +Ref. [`[]=`] aka [`[]=`(G[T], int, T)]. +Ref. [$] aka [proc $] or [proc `$`]. +Ref. [$(a: ref SomeType)]. +Ref. [foo_bar] aka [iterator foo_bar_]. +Ref. [fn[T; U,V: SomeFloat]()]. +Ref. ['big] or [func `'big`] or [`'big`(string)]. + +Link name syntax +---------------- + +Pandoc Markdown has synax for changing text of links: +Ref. [this proc][`[]`] or [another symbol][G[T]]. + +Symbols documentation +--------------------- + +Let us repeat auto links from symbols section below: + +There is also variant [f(G[string])]. +See also [f(G[int])]. + +]## diff --git a/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim new file mode 100644 index 0000000000..d3f5edd29a --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim @@ -0,0 +1,6 @@ +proc funWithGenerics*[T, U: SomeFloat](a: T, b: U) = discard + +# We check that presence of overloaded `fn2` here does not break +# referencing in the "parent" file (the one that includes this one) +proc fn2*(x: int, y: float, z: float) = + discard diff --git a/nimdoc/testproject/subdir/subdir_b/utils_overview.rst b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst new file mode 100644 index 0000000000..58ce930bfc --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst @@ -0,0 +1,8 @@ +This is a description of the utils module. + +Links work: + +* other module: `iterators `_ (not in this dir, just an example) +* internal: `fn2(x)`_ +* internal included from another module: `funWithGenerics*[T, U: + SomeFloat](a: T, b: U)`_. diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index 69edb0d235..383c4c827d 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -1,3 +1,15 @@ +## Basic usage +## =========== +## +## Encoding data +## ------------- +## +## Apart from strings you can also encode lists of integers or characters: + +## Decoding data +## ------------- +## + import subdir / subdir_b / utils @@ -39,6 +51,14 @@ proc buzz*[T](a, b: T): T {.deprecated: "since v0.20".} = ## This is deprecated with a message. result = a + b +type + FooBuzz* {.deprecated: "FooBuzz msg".} = int + +using f: FooBuzz + +proc bar*(f) = # `f` should be expanded to `f: FooBuzz` + discard + import std/macros var aVariable*: array[1, int] @@ -131,6 +151,7 @@ when true: # BUG: this currently this won't be run since not exported # but probably should doAssert false + if false: bazNonExported() # silence XDeclaredButNotUsed proc z17*() = # BUG: a comment before 1st doc comment currently doesn't prevent @@ -185,7 +206,7 @@ when true: # procs without `=` (using comment field) ## the c printf. ## etc. - proc c_nonexistant*(frmt: cstring): cint {.importc: "nonexistant", header: "", varargs, discardable.} + proc c_nonexistent*(frmt: cstring): cint {.importc: "nonexistent", header: "", varargs, discardable.} when true: # tests RST inside comments proc low*[T: Ordinal|enum|range](x: T): T {.magic: "Low", noSideEffect.} @@ -212,7 +233,7 @@ when true: # tests RST inside comments when true: # multiline string litterals proc tripleStrLitTest*() = - runnableExamples: + runnableExamples("--hint:XDeclaredButNotUsed:off"): ## mullitline string litterals are tricky as their indentation can span ## below that of the runnableExamples let s1a = """ @@ -252,12 +273,12 @@ at indent 0 when true: # methods; issue #14691 type Moo = object - method method1*(self: Moo) = + method method1*(self: Moo) {.base.} = ## foo1 - method method2*(self: Moo): int = + method method2*(self: Moo): int {.base.} = ## foo2 result = 1 - method method3*(self: Moo): int = + method method3*(self: Moo): int {.base.} = ## foo3 1 @@ -330,20 +351,18 @@ when true: # (most) templates ## ok5 ## ok5b runnableExamples: assert true + runnableExamples: discard 1 ## in or out? - # this is an edge case; a newline separate last runnableExamples from - # next doc comment but AST isnt' aware of it; this could change in future discard 8 ## out - runnableExamples: discard 1 when true: # issue #14473 import std/[sequtils] template doit(): untyped = ## doit ## return output only - toSeq([1,2]) + toSeq(["D20210427T172228"]) # make it searcheable at least until we figure out a way to avoid echo echo doit() # using doAssert or similar to avoid echo would "hide" the original bug when true: # issue #14846 @@ -381,3 +400,23 @@ when true: # issue #15184 ## ## There is no block quote after blank lines at the beginning. discard + +type T19396* = object # bug #19396 + a*: int + b: float + +template somePragma*() {.pragma.} + ## Just some annotation + +type # bug #21483 + MyObject* = object + someString*: string ## This is a string + annotated* {.somePragma.}: string ## This is an annotated string + +type + AnotherObject* = object + case x*: bool + of true: + y*: proc (x: string) + of false: + hidden: string diff --git a/nimpretty/nimpretty.nim b/nimpretty/nimpretty.nim index a7940349d2..e5abf0d2d9 100644 --- a/nimpretty/nimpretty.nim +++ b/nimpretty/nimpretty.nim @@ -12,7 +12,7 @@ when not defined(nimpretty): {.error: "This needs to be compiled with --define:nimPretty".} -import ../compiler / [idents, msgs, syntaxes, options, pathutils, layouter] +import ../compiler / [idents, llstream, ast, msgs, syntaxes, options, pathutils, layouter] import parseopt, strutils, os, sequtils @@ -48,6 +48,42 @@ type indWidth*: Natural maxLineLen*: Positive +proc goodEnough(a, b: PNode): bool = + if a.kind == b.kind and a.safeLen == b.safeLen: + case a.kind + of nkNone, nkEmpty, nkNilLit: result = true + of nkIdent: result = a.ident.id == b.ident.id + of nkSym: result = a.sym == b.sym + of nkType: result = true + of nkCharLit, nkIntLit..nkInt64Lit, nkUIntLit..nkUInt64Lit: + result = a.intVal == b.intVal + of nkFloatLit..nkFloat128Lit: + result = a.floatVal == b.floatVal + of nkStrLit, nkRStrLit, nkTripleStrLit: + result = a.strVal == b.strVal + else: + for i in 0 ..< a.len: + if not goodEnough(a[i], b[i]): return false + return true + elif a.kind == nkStmtList and a.len == 1: + result = goodEnough(a[0], b) + elif b.kind == nkStmtList and b.len == 1: + result = goodEnough(a, b[0]) + else: + result = false + +proc finalCheck(content: string; origAst: PNode): bool {.nimcall.} = + var conf = newConfigRef() + let oldErrors = conf.errorCounter + var parser: Parser + parser.em.indWidth = 2 + let fileIdx = fileInfoIdx(conf, AbsoluteFile "nimpretty_bug.nim") + + openParser(parser, fileIdx, llStreamOpen(content), newIdentCache(), conf) + let newAst = parseAll(parser) + closeParser(parser) + result = conf.errorCounter == oldErrors # and goodEnough(newAst, origAst) + proc prettyPrint*(infile, outfile: string, opt: PrettyOptions) = var conf = newConfigRef() let fileIdx = fileInfoIdx(conf, AbsoluteFile infile) @@ -58,8 +94,10 @@ proc prettyPrint*(infile, outfile: string, opt: PrettyOptions) = parser.em.indWidth = opt.indWidth if setupParser(parser, fileIdx, newIdentCache(), conf): parser.em.maxLineLen = opt.maxLineLen - discard parseAll(parser) + let fullAst = parseAll(parser) closeParser(parser) + when defined(nimpretty): + closeEmitter(parser.em, fullAst, finalCheck) proc main = var outfile, outdir: string @@ -69,8 +107,8 @@ proc main = var backup = false # when `on`, create a backup file of input in case - # `prettyPrint` could over-write it (note that the backup may happen even - # if input is not actually over-written, when nimpretty is a noop). + # `prettyPrint` could overwrite it (note that the backup may happen even + # if input is not actually overwritten, when nimpretty is a noop). # --backup was un-documented (rely on git instead). var opt = PrettyOptions(indWidth: 0, maxLineLen: 80) @@ -78,7 +116,13 @@ proc main = for kind, key, val in getopt(): case kind of cmdArgument: - infiles.add(key.addFileExt(".nim")) + if dirExists(key): + for file in walkDirRec(key, skipSpecial = true): + if file.endsWith(".nim") or file.endsWith(".nimble"): + infiles.add(file) + else: + infiles.add(key.addFileExt(".nim")) + of cmdLongOption, cmdShortOption: case normalize(key) of "help", "h": writeHelp() diff --git a/nimpretty/tester.nim b/nimpretty/tester.nim index d646b25ced..b1f15aee61 100644 --- a/nimpretty/tester.nim +++ b/nimpretty/tester.nim @@ -1,9 +1,10 @@ # Small program that runs the test cases import strutils, os, sequtils +from std/private/gitutils import diffFiles const - dir = "nimpretty/tests/" + dir = "nimpretty/tests" outputdir = dir / "outputdir" var @@ -26,7 +27,7 @@ proc test(infile, ext: string) = let produced = dir / nimFile.changeFileExt(ext) if readFile(expected) != readFile(produced): echo "FAILURE: files differ: ", nimFile - discard execShellCmd("diff -uNdr " & expected & " " & produced) + echo diffFiles(expected, produced).output failures += 1 else: echo "SUCCESS: files identical: ", nimFile @@ -43,7 +44,7 @@ proc testTogether(infiles: seq[string]) = let produced = dir / "outputdir" / infile if readFile(expected) != readFile(produced): echo "FAILURE: files differ: ", nimFile - discard execShellCmd("diff -uNdr " & expected & " " & produced) + echo diffFiles(expected, produced).output failures += 1 else: echo "SUCCESS: files identical: ", nimFile diff --git a/nimpretty/tests/exhaustive.nim b/nimpretty/tests/exhaustive.nim index 30cfc47a98..e5a73305b8 100644 --- a/nimpretty/tests/exhaustive.nim +++ b/nimpretty/tests/exhaustive.nim @@ -267,7 +267,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) = if not em.endsInWhite: wr(" ") wr(tok.ident.s) template isUnary(tok): bool = - tok.strongSpaceB == 0 and tok.strongSpaceA > 0 + tok.spacing == {tsLeading} if not isUnary(tok) or em.lastTok in {tkOpr, tkDotDot}: wr(" ") @@ -693,11 +693,11 @@ proc newRecordGen(ctx: Context; typ: TypRef): PNode = String `interpolation`:idx: / `format`:idx: inspired by Python's ``f``-strings. -.. code-block:: nim - - import strformat - let msg = "hello" - doAssert fmt"{msg}\n" == "hello\\n" + ```nim + import strformat + let msg = "hello" + doAssert fmt"{msg}\n" == "hello\\n" + ``` Because the literal is a raw string literal, the ``\n`` is not interpreted as an escape sequence. diff --git a/nimpretty/tests/expected/exhaustive.nim b/nimpretty/tests/expected/exhaustive.nim index 82927a11cf..7f78b7e566 100644 --- a/nimpretty/tests/expected/exhaustive.nim +++ b/nimpretty/tests/expected/exhaustive.nim @@ -272,7 +272,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) = if not em.endsInWhite: wr(" ") wr(tok.ident.s) template isUnary(tok): bool = - tok.strongSpaceB == 0 and tok.strongSpaceA > 0 + tok.spacing == {tsLeading} if not isUnary(tok) or em.lastTok in {tkOpr, tkDotDot}: wr(" ") @@ -699,11 +699,11 @@ proc newRecordGen(ctx: Context; typ: TypRef): PNode = String `interpolation`:idx: / `format`:idx: inspired by Python's ``f``-strings. -.. code-block:: nim - - import strformat - let msg = "hello" - doAssert fmt"{msg}\n" == "hello\\n" + ```nim + import strformat + let msg = "hello" + doAssert fmt"{msg}\n" == "hello\\n" + ``` Because the literal is a raw string literal, the ``\n`` is not interpreted as an escape sequence. diff --git a/nimpretty/tests/expected/simple.nim b/nimpretty/tests/expected/simple.nim index d13558621a..e711eb3b6e 100644 --- a/nimpretty/tests/expected/simple.nim +++ b/nimpretty/tests/expected/simple.nim @@ -16,3 +16,14 @@ proc a() = discard ## comment 3 discard # comment 4 + + +# bug #20553 + +let `'hello` = 12 +echo `'hello` + + +proc `'u4`(n: string) = + # The leading ' is required. + discard diff --git a/nimpretty/tests/simple.nim b/nimpretty/tests/simple.nim index 435bd6bd2d..2a01a176ef 100644 --- a/nimpretty/tests/simple.nim +++ b/nimpretty/tests/simple.nim @@ -16,3 +16,14 @@ proc a() = discard## comment 3 discard # comment 4 + + +# bug #20553 + +let `'hello` = 12 +echo `'hello` + + +proc `'u4`(n: string) = + # The leading ' is required. + discard diff --git a/nimsuggest/config.nims b/nimsuggest/config.nims new file mode 100644 index 0000000000..ee19f9893c --- /dev/null +++ b/nimsuggest/config.nims @@ -0,0 +1,2 @@ +# xxx not sure why this flag isn't needed: switch("processing", "filenames") +switch("filenames", "canonical") diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index c139b8b172..67fc2f8c42 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -7,6 +7,24 @@ # distribution, for details about the copyright. # +import compiler/renderer +import compiler/types +import compiler/trees +import compiler/wordrecg +import compiler/sempass2 +import strformat +import algorithm +import tables +import times +import procmonitor + +template tryImport(module) = import module + +when compiles tryImport ../dist/checksums/src/checksums/sha1: + import ../dist/checksums/src/checksums/sha1 +else: + import checksums/sha1 + ## Nimsuggest is a tool that helps to give editors IDE like capabilities. when not defined(nimcore): @@ -16,17 +34,21 @@ import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp # Do NOT import suggest. It will lead to weird bugs with # suggestionResultHook, because suggest.nim is included by sigmatch. # So we import that one instead. -import compiler / [options, commands, modules, sem, +import compiler / [options, commands, modules, passes, passaux, msgs, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, - pathutils] + pathutils, condsyms, syntaxes, suggestsymdb] + +when defined(nimPreviewSlimSystem): + import std/typedthreads when defined(windows): import winlean else: import posix +const HighestSuggestProtocolVersion = 4 const DummyEof = "!EOF!" const Usage = """ Nimsuggest - Tool to give every editor IDE like capabilities for Nim @@ -39,15 +61,25 @@ Options: --address:HOST binds to that address, by default "" --stdin read commands from stdin and write results to stdout instead of using sockets + --clientProcessId:PID shutdown nimsuggest in case this process dies --epc use emacs epc mode --debug enable debug output --log enable verbose logging to nimsuggest.log file --v1 use version 1 of the protocol; for backwards compatibility + --v2 use version 2(default) of the protocol + --v3 use version 3 of the protocol + --v4 use version 4 of the protocol + --info:X information + --info:nimVer return the Nim compiler version that nimsuggest uses internally + --info:protocolVer return the newest protocol version that is supported + --info:capabilities return the capabilities supported by nimsuggest --refresh perform automatic refreshes to keep the analysis precise --maxresults:N limit the number of suggestions to N --tester implies --stdin and outputs a line '""" & DummyEof & """' for the tester --find attempts to find the project file of the current project + --exceptionInlayHints:on|off + globally turn exception inlay hints on|off The server then listens to the connection and takes line-based commands. @@ -66,7 +98,7 @@ type var gPort = 6000.Port - gAddress = "" + gAddress = "127.0.0.1" gMode: Mode gEmitEof: bool # whether we write '!EOF!' dummy lines gLogging = defined(logging) @@ -76,6 +108,9 @@ var requests: Channel[string] results: Channel[Suggest] +proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; tag: string, + graph: ModuleGraph); + proc writelnToChannel(line: string) = results.send(Suggest(section: ideMsg, doc: line)) @@ -96,6 +131,11 @@ const "type 'quit' to quit\n" & "type 'debug' to toggle debug mode on/off\n" & "type 'terse' to toggle terse mode on/off" + #List of currently supported capabilities. So lang servers/ides can iterate over and check for what's enabled + Capabilities = [ + "con", #current NimSuggest supports the `con` commmand + "exceptionInlayHints" + ] proc parseQuoted(cmd: string; outp: var string; start: int): int = var i = start @@ -125,6 +165,9 @@ proc sexp(s: Suggest): SexpNode = ]) if s.section == ideSug: result.add convertSexp(s.prefix) + if s.section in {ideOutline, ideExpand} and s.version == 3: + result.add convertSexp(s.endLine.int) + result.add convertSexp(s.endCol) proc sexp(s: seq[Suggest]): SexpNode = result = newSList() @@ -137,7 +180,7 @@ proc listEpc(): SexpNode = argspecs = sexp("file line column dirtyfile".split(" ").map(newSSymbol)) docstring = sexp("line starts at 1, column at 0, dirtyfile is optional") result = newSList() - for command in ["sug", "con", "def", "use", "dus", "chk", "mod"]: + for command in ["sug", "con", "def", "use", "dus", "chk", "mod", "globalSymbols", "recompile", "saved", "chkFile", "declaration", "inlayHints"]: let cmd = sexp(command) methodDesc = newSList() @@ -160,9 +203,43 @@ proc symFromInfo(graph: ModuleGraph; trackPos: TLineInfo): PSym = if m != nil and m.ast != nil: result = findNode(m.ast, trackPos) -proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; +template benchmark(benchmarkName: untyped, code: untyped) = + block: + myLog "Started [" & benchmarkName & "]..." + let t0 = epochTime() + code + let elapsed = epochTime() - t0 + let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3) + myLog "CPU Time [" & benchmarkName & "] " & elapsedStr & "s" + +proc clearInstCache(graph: ModuleGraph, projectFileIdx: FileIndex) = + if projectFileIdx == InvalidFileIdx: + graph.typeInstCache.clear() + graph.procInstCache.clear() + return + var typeIdsToDelete = newSeq[ItemId]() + for id in graph.typeInstCache.keys: + if id.module == projectFileIdx.int: + typeIdsToDelete.add id + for id in typeIdsToDelete: + graph.typeInstCache.del id + var procIdsToDelete = newSeq[ItemId]() + for id in graph.procInstCache.keys: + if id.module == projectFileIdx.int: + procIdsToDelete.add id + for id in procIdsToDelete: + graph.procInstCache.del id + +proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, tag: string, graph: ModuleGraph) = let conf = graph.config + + if conf.suggestVersion >= 3: + let command = fmt "cmd = {cmd} {file}:{line}:{col}" + benchmark command: + executeNoHooksV3(cmd, file, dirtyfile, line, col, tag, graph) + return + myLog("cmd: " & $cmd & ", file: " & file.string & ", dirtyFile: " & dirtyfile.string & "[" & $line & ":" & $col & "]") @@ -181,6 +258,7 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; if conf.suggestVersion == 1: graph.usageSym = nil if not isKnownFile: + graph.clearInstCache(dirtyIdx) graph.compileProject(dirtyIdx) if conf.suggestVersion == 0 and conf.ideCmd in {ideUse, ideDus} and dirtyfile.isEmpty: @@ -191,6 +269,7 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; graph.markClientsDirty dirtyIdx if conf.ideCmd != ideMod: if isKnownFile: + graph.clearInstCache(modIdx) graph.compileProject(modIdx) if conf.ideCmd in {ideUse, ideDus}: let u = if conf.suggestVersion != 1: graph.symFromInfo(conf.m.trackPos) else: graph.usageSym @@ -199,7 +278,10 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; else: localError(conf, conf.m.trackPos, "found no symbol at this position " & (conf $ conf.m.trackPos)) -proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; +proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, graph: ModuleGraph) = + executeNoHooks(cmd, file, dirtyfile, line, col, "", graph) + +proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; tag: string, graph: ModuleGraph) = if cmd == ideChk: graph.config.structuredErrorHook = errorHook @@ -207,7 +289,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; else: graph.config.structuredErrorHook = nil graph.config.writelnHook = myLog - executeNoHooks(cmd, file, dirtyfile, line, col, graph) + executeNoHooks(cmd, file, dirtyfile, line, col, tag, graph) proc executeEpc(cmd: IdeCmd, args: SexpNode; graph: ModuleGraph) = @@ -218,7 +300,7 @@ proc executeEpc(cmd: IdeCmd, args: SexpNode; var dirtyfile = AbsoluteFile"" if len(args) > 3: dirtyfile = AbsoluteFile args[3].getStr("") - execute(cmd, file, dirtyfile, int(line), int(column), graph) + execute(cmd, file, dirtyfile, int(line), int(column), args[3].getStr, graph) proc returnEpc(socket: Socket, uid: BiggestInt, s: SexpNode|string, returnSymbol = "return") = @@ -436,6 +518,18 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = of "terse": toggle optIdeTerse of "known": conf.ideCmd = ideKnown of "project": conf.ideCmd = ideProject + of "changed": conf.ideCmd = ideChanged + of "globalsymbols": conf.ideCmd = ideGlobalSymbols + of "declaration": conf.ideCmd = ideDeclaration + of "expand": conf.ideCmd = ideExpand + of "chkfile": conf.ideCmd = ideChkFile + of "recompile": conf.ideCmd = ideRecompile + of "type": conf.ideCmd = ideType + of "inlayhints": + if conf.suggestVersion >= 4: + conf.ideCmd = ideInlayHints + else: + err() else: err() var dirtyfile = "" var orig = "" @@ -452,6 +546,7 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = i += parseInt(cmd, line, i) i += skipWhile(cmd, seps, i) i += parseInt(cmd, col, i) + let tag = substr(cmd, i) if conf.ideCmd == ideKnown: results.send(Suggest(section: ideKnown, quality: ord(fileInfoKnown(conf, AbsoluteFile orig)))) @@ -460,23 +555,23 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = else: if conf.ideCmd == ideChk: for cm in cachedMsgs: errorHook(conf, cm.info, cm.msg, cm.sev) - execute(conf.ideCmd, AbsoluteFile orig, AbsoluteFile dirtyfile, line, col, graph) + execute(conf.ideCmd, AbsoluteFile orig, AbsoluteFile dirtyfile, line, col, tag, graph) sentinel() proc recompileFullProject(graph: ModuleGraph) = - #echo "recompiling full project" - resetSystemArtifacts(graph) - graph.vm = nil - graph.resetAllModules() - GC_fullCollect() - compileProject(graph) - #echo GC_getStatistics() + benchmark "Recompilation(clean)": + graph.resetForBackend() + graph.resetSystemArtifacts() + graph.vm = nil + graph.resetAllModules() + GC_fullCollect() + graph.compileProject() proc mainThread(graph: ModuleGraph) = let conf = graph.config - if gLogging: - for it in conf.searchPaths: - log(it.string) + myLog "searchPaths: " + for it in conf.searchPaths: + myLog(" " & it.string) proc wrHook(line: string) {.closure.} = if gMode == mepc: @@ -499,7 +594,7 @@ proc mainThread(graph: ModuleGraph) = else: os.sleep 250 idle += 1 - if idle == 20 and gRefresh: + if idle == 20 and gRefresh and conf.suggestVersion < 3: # we use some nimsuggest activity to enable a lazy recompile: conf.ideCmd = ideChk conf.writelnHook = proc (s: string) = discard @@ -518,6 +613,7 @@ proc mainCommand(graph: ModuleGraph) = registerPass graph, verbosePass registerPass graph, semPass conf.setCmd cmdIdeTools + defineSymbol(conf.symbols, $conf.backend) wantMainModule(conf) if not fileExists(conf.projectFull): @@ -527,16 +623,25 @@ proc mainCommand(graph: ModuleGraph) = conf.setErrorMaxHighMaybe # honor --errorMax even if it may not make sense here # do not print errors, but log them - conf.writelnHook = myLog - conf.structuredErrorHook = nil + conf.writelnHook = proc (msg: string) = discard + + if graph.config.suggestVersion >= 3: + graph.config.structuredErrorHook = proc (conf: ConfigRef; info: TLineInfo; msg: string; sev: Severity) = + let suggest = Suggest(section: ideChk, filePath: toFullPath(conf, info), + line: toLinenumber(info), column: toColumn(info), doc: msg, forth: $sev) + graph.suggestErrors.mgetOrPut(info.fileIndex, @[]).add suggest # compile the project before showing any input so that we already # can answer questions right away: - compileProject(graph) + benchmark "Initial compilation": + compileProject(graph) open(requests) open(results) + if graph.config.clientProcessId != 0: + hookProcMonitor(graph.config.clientProcessId) + case gMode of mstdin: createThread(inputThread, replStdin, (gPort, gAddress)) of mtcp: createThread(inputThread, replTcp, (gPort, gAddress)) @@ -584,8 +689,28 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string; conf: ConfigRef) = gMode = mepc conf.verbosity = 0 # Port number gotta be first. of "debug": incl(conf.globalOptions, optIdeDebug) - of "v2": conf.suggestVersion = 0 of "v1": conf.suggestVersion = 1 + of "v2": conf.suggestVersion = 0 + of "v3": conf.suggestVersion = 3 + of "v4": conf.suggestVersion = 4 + of "info": + case p.val.normalize + of "protocolver": + stdout.writeLine(HighestSuggestProtocolVersion) + quit 0 + of "nimver": + stdout.writeLine(system.NimVersion) + quit 0 + of "capabilities": + stdout.writeLine(Capabilities.toSeq.mapIt($it).join(" ")) + quit 0 + else: + processSwitch(pass, p, conf) + of "exceptioninlayhints": + case p.val.normalize + of "", "on": incl(conf.globalOptions, optIdeExceptionInlayHints) + of "off": excl(conf.globalOptions, optIdeExceptionInlayHints) + else: processSwitch(pass, p, conf) of "tester": gMode = mstdin gEmitEof = true @@ -600,6 +725,8 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string; conf: ConfigRef) = conf.suggestMaxResults = parseInt(p.val) of "find": findProject = true + of "clientprocessid": + conf.clientProcessId = parseInt(p.val) else: processSwitch(pass, p, conf) of cmdArgument: let a = unixToNativePath(p.key) @@ -647,6 +774,512 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if self.loadConfigsAndProcessCmdLine(cache, conf, graph): mainCommand(graph) +# v3 start + +proc recompilePartially(graph: ModuleGraph, projectFileIdx = InvalidFileIdx) = + if projectFileIdx == InvalidFileIdx: + myLog "Recompiling partially from root" + else: + myLog fmt "Recompiling partially starting from {graph.getModule(projectFileIdx)}" + + # inst caches are breaking incremental compilation when the cache caches stuff + # from dirty buffer + graph.clearInstCache(projectFileIdx) + + GC_fullCollect() + + try: + benchmark "Recompilation": + graph.compileProject(projectFileIdx) + except Exception as e: + myLog fmt "Failed to recompile partially with the following error:\n {e.msg} \n\n {e.getStackTrace()}" + try: + graph.recompileFullProject() + except Exception as e: + myLog fmt "Failed clean recompilation:\n {e.msg} \n\n {e.getStackTrace()}" + +func deduplicateSymInfoPair[SymInfoPair](xs: seq[SymInfoPair]): seq[SymInfoPair] = + # xs contains duplicate items and we want to filter them by range because the + # sym may not match. This can happen when xs contains the same definition but + # with different signature because suggestSym might be called multiple times + # for the same symbol (e. g. including/excluding the pragma) + result = newSeqOfCap[SymInfoPair](xs.len) + for itm in xs.reversed: + var found = false + for res in result: + if res.info.exactEquals(itm.info): + found = true + break + if not found: + result.add(itm) + result.reverse() + +func deduplicateSymInfoPair(xs: SuggestFileSymbolDatabase): SuggestFileSymbolDatabase = + # xs contains duplicate items and we want to filter them by range because the + # sym may not match. This can happen when xs contains the same definition but + # with different signature because suggestSym might be called multiple times + # for the same symbol (e. g. including/excluding the pragma) + result = SuggestFileSymbolDatabase( + lineInfo: newSeqOfCap[TinyLineInfo](xs.lineInfo.len), + sym: newSeqOfCap[PSym](xs.sym.len), + isDecl: newPackedBoolArray(), + caughtExceptions: newSeqOfCap[seq[PType]](xs.caughtExceptions.len), + caughtExceptionsSet: newPackedBoolArray(), + fileIndex: xs.fileIndex, + trackCaughtExceptions: xs.trackCaughtExceptions, + isSorted: false + ) + var i = xs.lineInfo.high + while i >= 0: + let itm = xs.lineInfo[i] + var found = false + for res in result.lineInfo: + if res.exactEquals(itm): + found = true + break + if not found: + result.add(xs.getSymInfoPair(i)) + dec i + result.reverse() + +proc findSymData(graph: ModuleGraph, trackPos: TLineInfo): + ref SymInfoPair = + let db = graph.fileSymbols(trackPos.fileIndex).deduplicateSymInfoPair + doAssert(db.fileIndex == trackPos.fileIndex) + for i in db.lineInfo.low..db.lineInfo.high: + if isTracked(db.lineInfo[i], TinyLineInfo(line: trackPos.line, col: trackPos.col), db.sym[i].name.s.len): + var res = db.getSymInfoPair(i) + new(result) + result[] = res + break + +func isInRange*(current, startPos, endPos: TinyLineInfo, tokenLen: int): bool = + result = + (current.line > startPos.line or (current.line == startPos.line and current.col>=startPos.col)) and + (current.line < endPos.line or (current.line == endPos.line and current.col <= endPos.col)) + +proc findSymDataInRange(graph: ModuleGraph, startPos, endPos: TLineInfo): + seq[SymInfoPair] = + result = newSeq[SymInfoPair]() + let db = graph.fileSymbols(startPos.fileIndex).deduplicateSymInfoPair + for i in db.lineInfo.low..db.lineInfo.high: + if isInRange(db.lineInfo[i], TinyLineInfo(line: startPos.line, col: startPos.col), TinyLineInfo(line: endPos.line, col: endPos.col), db.sym[i].name.s.len): + result.add(db.getSymInfoPair(i)) + +proc findSymData(graph: ModuleGraph, file: AbsoluteFile; line, col: int): + ref SymInfoPair = + let + fileIdx = fileInfoIdx(graph.config, file) + trackPos = newLineInfo(fileIdx, line, col) + result = findSymData(graph, trackPos) + +proc findSymDataInRange(graph: ModuleGraph, file: AbsoluteFile; startLine, startCol, endLine, endCol: int): + seq[SymInfoPair] = + let + fileIdx = fileInfoIdx(graph.config, file) + startPos = newLineInfo(fileIdx, startLine, startCol) + endPos = newLineInfo(fileIdx, endLine, endCol) + result = findSymDataInRange(graph, startPos, endPos) + +proc markDirtyIfNeeded(graph: ModuleGraph, file: string, originalFileIdx: FileIndex) = + let sha = $sha1.secureHashFile(file) + if graph.config.m.fileInfos[originalFileIdx.int32].hash != sha or graph.config.ideCmd in {ideSug, ideCon}: + myLog fmt "{file} changed compared to last compilation" + graph.markDirty originalFileIdx + graph.markClientsDirty originalFileIdx + else: + myLog fmt "No changes in file {file} compared to last compilation" + +proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, + defaultSection = ideNone, endLine: uint16 = 0, endCol = 0) = + let section = if defaultSection != ideNone: + defaultSection + elif sym.info.exactEquals(info): + ideDef + else: + ideUse + let suggest = symToSuggest(graph, sym, isLocal=false, section, + info, 100, PrefixMatch.None, false, 0, + endLine = endLine, endCol = endCol) + suggestResult(graph.config, suggest) + +proc suggestInlayHintResultType(graph: ModuleGraph, sym: PSym, info: TLineInfo, + defaultSection = ideNone, endLine: uint16 = 0, endCol = 0) = + let section = if defaultSection != ideNone: + defaultSection + elif sym.info.exactEquals(info): + ideDef + else: + ideUse + var suggestDef = symToSuggest(graph, sym, isLocal=false, section, + info, 100, PrefixMatch.None, false, 0, true, + endLine = endLine, endCol = endCol) + suggestDef.inlayHintInfo = suggestToSuggestInlayTypeHint(suggestDef) + suggestDef.section = ideInlayHints + if sym.kind == skForVar: + suggestDef.inlayHintInfo.allowInsert = false + suggestResult(graph.config, suggestDef) + +proc suggestInlayHintResultException(graph: ModuleGraph, sym: PSym, info: TLineInfo, + defaultSection = ideNone, caughtExceptions: seq[PType], caughtExceptionsSet: bool, endLine: uint16 = 0, endCol = 0) = + if not caughtExceptionsSet: + return + + if sym.kind == skParam and sfEffectsDelayed in sym.flags: + return + + var raisesList: seq[PType] = @[getEbase(graph, info)] + + let t = sym.typ + if not isNil(t) and not isNil(t.n) and t.n.len > 0 and t.n[0].len > exceptionEffects: + let effects = t.n[0] + if effects.kind == nkEffectList and effects.len == effectListLen: + let effs = effects[exceptionEffects] + if not isNil(effs): + raisesList = @[] + for eff in items(effs): + if not isNil(eff): + raisesList.add(eff.typ) + + var propagatedExceptionList: seq[PType] = @[] + for re in raisesList: + var exceptionIsPropagated = true + for ce in caughtExceptions: + if isNil(ce) or safeInheritanceDiff(re, ce) <= 0: + exceptionIsPropagated = false + break + if exceptionIsPropagated: + propagatedExceptionList.add(re) + + if propagatedExceptionList.len == 0: + return + + let section = if defaultSection != ideNone: + defaultSection + elif sym.info.exactEquals(info): + ideDef + else: + ideUse + var suggestDef = symToSuggest(graph, sym, isLocal=false, section, + info, 100, PrefixMatch.None, false, 0, true, + endLine = endLine, endCol = endCol) + suggestDef.inlayHintInfo = suggestToSuggestInlayExceptionHintLeft(suggestDef, propagatedExceptionList) + suggestDef.section = ideInlayHints + suggestResult(graph.config, suggestDef) + suggestDef.inlayHintInfo = suggestToSuggestInlayExceptionHintRight(suggestDef, propagatedExceptionList) + suggestResult(graph.config, suggestDef) + +const + # kinds for ideOutline and ideGlobalSymbols + searchableSymKinds = {skField, skEnumField, skIterator, skMethod, skFunc, skProc, skConverter, skTemplate} + +proc symbolEqual(left, right: PSym): bool = + # More relaxed symbol comparison + return left.info.exactEquals(right.info) and left.name == right.name + +proc findDef(n: PNode, line: uint16, col: int16): PNode = + if n.kind in {nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkMacroDef}: + if n.info.line == line: + return n + else: + for i in 0 ..< safeLen(n): + let res = findDef(n[i], line, col) + if res != nil: return res + +proc findByTLineInfo(trackPos: TLineInfo, infoPairs: SuggestFileSymbolDatabase): + ref SymInfoPair = + result = nil + if infoPairs.fileIndex == trackPos.fileIndex: + for i in infoPairs.lineInfo.low..infoPairs.lineInfo.high: + let s = infoPairs.getSymInfoPair(i) + if s.info.exactEquals trackPos: + new(result) + result[] = s + break + +proc outlineNode(graph: ModuleGraph, n: PNode, endInfo: TLineInfo, infoPairs: SuggestFileSymbolDatabase): bool = + proc checkSymbol(sym: PSym, info: TLineInfo): bool = + result = (sym.owner.kind in {skModule, skType} or sym.kind in {skProc, skMethod, skIterator, skTemplate, skType}) + + if n.kind == nkSym and n.sym.checkSymbol(n.info): + graph.suggestResult(n.sym, n.sym.info, ideOutline, endInfo.line, endInfo.col) + return true + elif n.kind == nkIdent: + let symData = findByTLineInfo(n.info, infoPairs) + if symData != nil and symData.sym.checkSymbol(symData.info): + let sym = symData.sym + graph.suggestResult(sym, sym.info, ideOutline, endInfo.line, endInfo.col) + return true + +proc handleIdentOrSym(graph: ModuleGraph, n: PNode, endInfo: TLineInfo, infoPairs: SuggestFileSymbolDatabase): bool = + for child in n: + if child.kind in {nkIdent, nkSym}: + if graph.outlineNode(child, endInfo, infoPairs): + return true + elif child.kind == nkPostfix: + if graph.handleIdentOrSym(child, endInfo, infoPairs): + return true + +proc iterateOutlineNodes(graph: ModuleGraph, n: PNode, infoPairs: SuggestFileSymbolDatabase) = + var matched = true + if n.kind == nkIdent: + let symData = findByTLineInfo(n.info, infoPairs) + if symData != nil and symData.sym.kind == skEnumField and symData.info.exactEquals(symData.sym.info): + let sym = symData.sym + graph.suggestResult(sym, sym.info, ideOutline, n.endInfo.line, n.endInfo.col) + elif (n.kind in {nkFuncDef, nkProcDef, nkTypeDef, nkMacroDef, nkTemplateDef, nkConverterDef, nkEnumFieldDef, nkConstDef}): + matched = handleIdentOrSym(graph, n, n.endInfo, infoPairs) + else: + matched = false + + if n.kind != nkFormalParams: + for child in n: + graph.iterateOutlineNodes(child, infoPairs) + +proc calculateExpandRange(n: PNode, info: TLineInfo): TLineInfo = + if ((n.kind in {nkFuncDef, nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkConverterDef} and + n.info.exactEquals(info)) or + (n.kind in {nkCall, nkCommand} and n[0].info.exactEquals(info))): + result = n.endInfo + else: + for child in n: + result = child.calculateExpandRange(info) + if result != unknownLineInfo: + return result + result = unknownLineInfo + +proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; tag: string, + graph: ModuleGraph) = + let conf = graph.config + conf.writelnHook = proc (s: string) = discard + conf.structuredErrorHook = proc (conf: ConfigRef; info: TLineInfo; + msg: string; sev: Severity) = + let suggest = Suggest(section: ideChk, filePath: toFullPath(conf, info), + line: toLinenumber(info), column: toColumn(info), doc: msg, forth: $sev) + graph.suggestErrors.mgetOrPut(info.fileIndex, @[]).add suggest + + conf.ideCmd = cmd + + myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}, tag: {tag}" + + var fileIndex: FileIndex + + if not (cmd in {ideRecompile, ideGlobalSymbols}): + if not fileInfoKnown(conf, file): + myLog fmt "{file} is unknown, returning no results" + return + + fileIndex = fileInfoIdx(conf, file) + msgs.setDirtyFile( + conf, + fileIndex, + if dirtyfile.isEmpty: AbsoluteFile"" else: dirtyfile) + + if not dirtyfile.isEmpty: + graph.markDirtyIfNeeded(dirtyFile.string, fileInfoIdx(conf, file)) + + # these commands require fully compiled project + if cmd in {ideUse, ideDus, ideGlobalSymbols, ideChk, ideInlayHints} and graph.needsCompilation(): + graph.recompilePartially() + # when doing incremental build for the project root we should make sure that + # everything is unmarked as no longer beeing dirty in case there is no + # longer reference to a particular module. E. g. A depends on B, B is marked + # as dirty and A loses B import. + graph.unmarkAllDirty() + + # these commands require partially compiled project + elif cmd in {ideSug, ideCon, ideOutline, ideHighlight, ideDef, ideChkFile, ideType, ideDeclaration, ideExpand} and + (graph.needsCompilation(fileIndex) or cmd in {ideSug, ideCon}): + # for ideSug use v2 implementation + if cmd in {ideSug, ideCon}: + conf.m.trackPos = newLineInfo(fileIndex, line, col) + conf.m.trackPosAttached = false + else: + conf.m.trackPos = default(TLineInfo) + graph.recompilePartially(fileIndex) + + case cmd + of ideDef: + let s = graph.findSymData(file, line, col) + if not s.isNil: + graph.suggestResult(s.sym, s.sym.info) + of ideType: + let s = graph.findSymData(file, line, col) + if not s.isNil: + let typeSym = s.sym.typ.sym + if typeSym != nil: + graph.suggestResult(typeSym, typeSym.info, ideType) + elif s.sym.typ.len != 0: + let genericType = s.sym.typ[0].sym + graph.suggestResult(genericType, genericType.info, ideType) + of ideUse, ideDus: + let symbol = graph.findSymData(file, line, col) + if not symbol.isNil: + var res: seq[SymInfoPair] = @[] + for s in graph.suggestSymbolsIter: + if s.sym.symbolEqual(symbol.sym): + res.add(s) + for s in res.deduplicateSymInfoPair(): + graph.suggestResult(s.sym, s.info) + of ideHighlight: + let sym = graph.findSymData(file, line, col) + if not sym.isNil: + let fs = graph.fileSymbols(fileIndex) + var usages: seq[SymInfoPair] = @[] + for i in fs.lineInfo.low..fs.lineInfo.high: + if fs.sym[i] == sym.sym: + usages.add(fs.getSymInfoPair(i)) + myLog fmt "Found {usages.len} usages in {file.string}" + for s in usages: + graph.suggestResult(s.sym, s.info) + of ideRecompile: + graph.recompileFullProject() + of ideChanged: + graph.markDirtyIfNeeded(file.string, fileIndex) + of ideSug, ideCon: + # ideSug/ideCon performs partial build of the file, thus mark it dirty for the + # future calls. + graph.markDirtyIfNeeded(file.string, fileIndex) + graph.recompilePartially(fileIndex) + let m = graph.getModule fileIndex + incl m.flags, sfDirty + of ideOutline: + let n = parseFile(fileIndex, graph.cache, graph.config) + graph.iterateOutlineNodes(n, graph.fileSymbols(fileIndex).deduplicateSymInfoPair) + of ideChk: + myLog fmt "Reporting errors for {graph.suggestErrors.len} file(s)" + for sug in graph.suggestErrorsIter: + suggestResult(graph.config, sug) + of ideChkFile: + let errors = graph.suggestErrors.getOrDefault(fileIndex, @[]) + myLog fmt "Reporting {errors.len} error(s) for {file.string}" + for error in errors: + suggestResult(graph.config, error) + of ideGlobalSymbols: + var + counter = 0 + res: seq[SymInfoPair] = @[] + + for s in graph.suggestSymbolsIter: + if (sfGlobal in s.sym.flags or s.sym.kind in searchableSymKinds) and + s.sym.info == s.info: + if contains(s.sym.name.s, file.string): + inc counter + res = res.filterIt(not it.info.exactEquals(s.info)) + res.add s + # stop after first 1000 matches... + if counter > 1000: + break + + # ... then sort them by weight ... + res.sort() do (left, right: SymInfoPair) -> int: + let + leftString = left.sym.name.s + rightString = right.sym.name.s + leftIndex = leftString.find(file.string) + rightIndex = rightString.find(file.string) + + if leftIndex == rightIndex: + result = cmp(toLowerAscii(leftString), + toLowerAscii(rightString)) + else: + result = cmp(leftIndex, rightIndex) + + # ... and send first 100 results + if res.len > 0: + for i in 0 .. min(100, res.len - 1): + let s = res[i] + graph.suggestResult(s.sym, s.info) + + of ideDeclaration: + let s = graph.findSymData(file, line, col) + if not s.isNil: + # find first mention of the symbol in the file containing the definition. + # It is either the definition or the declaration. + var first: SymInfoPair + let db = graph.fileSymbols(s.sym.info.fileIndex).deduplicateSymInfoPair + for i in db.lineInfo.low..db.lineInfo.high: + if s.sym.symbolEqual(db.sym[i]): + first = db.getSymInfoPair(i) + break + + if s.info.exactEquals(first.info): + # we are on declaration, go to definition + graph.suggestResult(first.sym, first.sym.info, ideDeclaration) + else: + # we are on definition or usage, look for declaration + graph.suggestResult(first.sym, first.info, ideDeclaration) + of ideExpand: + var level: int = high(int) + let index = skipWhitespace(tag, 0); + let trimmed = substr(tag, index) + if not (trimmed == "" or trimmed == "all"): + discard parseInt(trimmed, level, 0) + + conf.expandPosition = newLineInfo(fileIndex, line, col) + conf.expandLevels = level + conf.expandProgress = false + conf.expandNodeResult = "" + + graph.markDirty fileIndex + graph.markClientsDirty fileIndex + graph.recompilePartially() + var suggest = Suggest() + suggest.section = ideExpand + suggest.version = 3 + suggest.line = line + suggest.column = col + suggest.doc = graph.config.expandNodeResult + if suggest.doc != "": + let + n = parseFile(fileIndex, graph.cache, graph.config) + endInfo = n.calculateExpandRange(conf.expandPosition) + + suggest.endLine = endInfo.line + suggest.endCol = endInfo.col + + suggestResult(graph.config, suggest) + + graph.markDirty fileIndex + graph.markClientsDirty fileIndex + of ideInlayHints: + myLog fmt "Executing inlayHints" + var endLine = 0 + var endCol = -1 + var i = 0 + i += skipWhile(tag, seps, i) + i += parseInt(tag, endLine, i) + i += skipWhile(tag, seps, i) + i += parseInt(tag, endCol, i) + i += skipWhile(tag, seps, i) + var typeHints = true + var exceptionHints = false + while i <= tag.high: + var token: string + i += parseUntil(tag, token, seps, i) + i += skipWhile(tag, seps, i) + case token: + of "+typeHints": + typeHints = true + of "-typeHints": + typeHints = false + of "+exceptionHints": + exceptionHints = true + of "-exceptionHints": + exceptionHints = false + else: + myLog fmt "Discarding unknown inlay hint parameter {token}" + + let s = graph.findSymDataInRange(file, line, col, endLine, endCol) + for q in s: + if typeHints and q.sym.kind in {skLet, skVar, skForVar, skConst} and q.isDecl and not q.sym.hasUserSpecifiedType: + graph.suggestInlayHintResultType(q.sym, q.info, ideInlayHints) + if exceptionHints and q.sym.kind in {skProc, skFunc, skMethod, skVar, skLet, skParam} and not q.isDecl: + graph.suggestInlayHintResultException(q.sym, q.info, ideInlayHints, caughtExceptions = q.caughtExceptions, caughtExceptionsSet = q.caughtExceptionsSet) + else: + myLog fmt "Discarding {cmd}" + +# v3 end when isMainModule: handleCmdLine(newIdentCache(), newConfigRef()) else: @@ -663,10 +1296,12 @@ else: proc mockCommand(graph: ModuleGraph) = retval = graph let conf = graph.config + conf.setCmd cmdIdeTools + defineSymbol(conf.symbols, $conf.backend) clearPasses(graph) registerPass graph, verbosePass registerPass graph, semPass - conf.setCmd cmdIdeTools + wantMainModule(conf) if not fileExists(conf.projectFull): @@ -726,8 +1361,9 @@ else: if self.loadConfigsAndProcessCmdLine(cache, conf, graph): mockCommand(graph) if gLogging: + myLog("Search paths:") for it in conf.searchPaths: - log(it.string) + myLog(" " & it.string) retval.doStopCompile = proc (): bool = false return NimSuggest(graph: retval, idle: 0, cachedMsgs: @[]) diff --git a/nimsuggest/nimsuggest.nimble b/nimsuggest/nimsuggest.nimble index 53b5d1d6fc..b3790e1163 100644 --- a/nimsuggest/nimsuggest.nimble +++ b/nimsuggest/nimsuggest.nimble @@ -1,8 +1,8 @@ -version = "0.1.0" -author = "Andreas Rumpf" -description = "Tool for providing auto completion data for Nim source code." -license = "MIT" - +include "../lib/system/compilation.nim" +version = $NimMajor & "." & $NimMinor & "." & $NimPatch +author = "Andreas Rumpf" +description = "Tool for providing auto completion data for Nim source code." +license = "MIT" bin = @["nimsuggest"] -requires "nim >= 1.1.1" +requires "compiler >= 1.9.0" , "checksums" diff --git a/nimsuggest/procmonitor.nim b/nimsuggest/procmonitor.nim new file mode 100644 index 0000000000..0f1ba1e0d3 --- /dev/null +++ b/nimsuggest/procmonitor.nim @@ -0,0 +1,34 @@ +# Monitor a client process and shutdown the current process, if the client +# process is found to be dead + +import os + +when defined(posix): + import posix_utils + import posix + +when defined(windows): + import winlean + +when defined(posix): + proc monitorClientProcessIdThreadProc(pid: int) {.thread.} = + while true: + sleep(1000) + try: + sendSignal(Pid(pid), 0) + except: + discard kill(Pid(getCurrentProcessId()), cint(SIGTERM)) + +when defined(windows): + proc monitorClientProcessIdThreadProc(pid: int) {.thread.} = + var process = openProcess(SYNCHRONIZE, 0, DWORD(pid)) + if process != 0: + discard waitForSingleObject(process, INFINITE) + discard closeHandle(process) + quit(0) + +var tid: Thread[int] + +proc hookProcMonitor*(pid: int) = + when defined(posix) or defined(windows): + createThread(tid, monitorClientProcessIdThreadProc, pid) diff --git a/nimsuggest/sexp.nim b/nimsuggest/sexp.nim index cee538b6e2..03369ccb74 100644 --- a/nimsuggest/sexp.nim +++ b/nimsuggest/sexp.nim @@ -14,6 +14,9 @@ import import std/private/decode_helpers +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + type SexpEventKind* = enum ## enumeration of all events that may occur when parsing sexpError, ## an error occurred during parsing @@ -288,10 +291,6 @@ proc newSString*(s: string): SexpNode = ## Creates a new `SString SexpNode`. result = SexpNode(kind: SString, str: s) -proc newSStringMove(s: string): SexpNode = - result = SexpNode(kind: SString) - shallowCopy(result.str, s) - proc newSInt*(n: BiggestInt): SexpNode = ## Creates a new `SInt SexpNode`. result = SexpNode(kind: SInt, num: n) @@ -315,10 +314,6 @@ proc newSList*(): SexpNode = proc newSSymbol*(s: string): SexpNode = result = SexpNode(kind: SSymbol, symbol: s) -proc newSSymbolMove(s: string): SexpNode = - result = SexpNode(kind: SSymbol) - shallowCopy(result.symbol, s) - proc getStr*(n: SexpNode, default: string = ""): string = ## Retrieves the string value of a `SString SexpNode`. ## @@ -409,7 +404,7 @@ macro convertSexp*(x: untyped): untyped = ## `%` for every element. result = toSexp(x) -proc `==`* (a,b: SexpNode): bool = +func `==`* (a, b: SexpNode): bool = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -596,8 +591,7 @@ proc parseSexp(p: var SexpParser): SexpNode = case p.tok of tkString: # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: - result = newSStringMove(p.a) - p.a = "" + result = SexpNode(kind: SString, str: move p.a) discard getTok(p) of tkInt: result = newSInt(parseBiggestInt(p.a)) @@ -609,8 +603,7 @@ proc parseSexp(p: var SexpParser): SexpNode = result = newSNil() discard getTok(p) of tkSymbol: - result = newSSymbolMove(p.a) - p.a = "" + result = SexpNode(kind: SSymbol, symbol: move p.a) discard getTok(p) of tkParensLe: result = newSList() diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 425430ede8..9b9488348c 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -2,8 +2,11 @@ # Every test file can have a #[!]# comment that is deleted from the input # before 'nimsuggest' is invoked to ensure this token doesn't make a # crucial difference for Nim's parser. +# When debugging, to run a single test, use for e.g.: +# `nim r nimsuggest/tester.nim nimsuggest/tests/tsug_accquote.nim` import os, osproc, strutils, streams, re, sexp, net +from sequtils import toSeq type Test = object @@ -13,16 +16,16 @@ type disabled: bool const - curDir = when defined(windows): "" else: "" DummyEof = "!EOF!" - -template tpath(): untyped = getAppDir() / "tests" + tpath = "nimsuggest/tests" + # we could also use `stdtest/specialpaths` import std/compilesettings proc parseTest(filename: string; epcMode=false): Test = const cursorMarker = "#[!]#" - let nimsug = curDir & addFileExt("nimsuggest", ExeExt) + let nimsug = "bin" / addFileExt("nimsuggest_testing", ExeExt) + doAssert nimsug.fileExists, nimsug const libpath = querySetting(libPath) result.filename = filename result.dest = getTempDir() / extractFilename(filename) @@ -63,7 +66,7 @@ proc parseTest(filename: string; epcMode=false): Test = elif x.startsWith(">"): # since 'markers' here are not complete yet, we do the $substitutions # afterwards - result.script.add((x.substr(1).replaceWord("$path", tpath()), "")) + result.script.add((x.substr(1).replaceWord("$path", tpath).replaceWord("$file", filename), "")) elif x.len > 0: # expected output line: let x = x % ["file", filename, "lib", libpath] @@ -104,7 +107,7 @@ proc parseCmd(c: string): seq[string] = proc edit(tmpfile: string; x: seq[string]) = if x.len != 3 and x.len != 4: quit "!edit takes two or three arguments" - let f = if x.len >= 4: tpath() / x[3] else: tmpfile + let f = if x.len >= 4: tpath / x[3] else: tmpfile try: let content = readFile(f) let newcontent = content.replace(x[1], x[2]) @@ -121,12 +124,12 @@ proc exec(x: seq[string]) = proc copy(x: seq[string]) = if x.len != 3: quit "!copy takes two arguments" - let rel = tpath() + let rel = tpath copyFile(rel / x[1], rel / x[2]) proc del(x: seq[string]) = if x.len != 2: quit "!del takes one argument" - removeFile(tpath() / x[1]) + removeFile(tpath / x[1]) proc runCmd(cmd, dest: string): bool = result = cmd[0] == '!' @@ -215,7 +218,12 @@ proc sexpToAnswer(s: SexpNode): string = result.add doc result.add '\t' result.addInt a[8].getNum - if a.len >= 10: + if a.len >= 11: + result.add '\t' + result.addInt a[9].getNum + result.add '\t' + result.addInt a[10].getNum + elif a.len >= 10: result.add '\t' result.add a[9].getStr result.add '\L' @@ -226,8 +234,8 @@ proc doReport(filename, answer, resp: string; report: var string) = var hasDiff = false for i in 0..min(resp.len-1, answer.len-1): if resp[i] != answer[i]: - report.add "\n Expected: " & resp.substr(i, i+200) - report.add "\n But got: " & answer.substr(i, i+200) + report.add "\n Expected:\n" & resp + report.add "\n But got:\n" & answer hasDiff = true break if not hasDiff: @@ -249,13 +257,15 @@ proc runEpcTest(filename: string): int = for cmd in s.startup: if not runCmd(cmd, s.dest): quit "invalid command: " & cmd - let epccmd = s.cmd.replace("--tester", "--epc --v2 --log") + let epccmd = if s.cmd.contains("--v3"): + s.cmd.replace("--tester", "--epc --log") + else: + s.cmd.replace("--tester", "--epc --v2 --log") let cl = parseCmdLine(epccmd) var p = startProcess(command=cl[0], args=cl[1 .. ^1], options={poStdErrToStdOut, poUsePath, poInteractive, poDaemon}) let outp = p.outputStream - let inp = p.inputStream var report = "" var socket = newSocket() try: @@ -269,17 +279,28 @@ proc runEpcTest(filename: string): int = os.sleep(50) inc i let a = outp.readAll().strip() - let port = parseInt(a) + var port: int + try: + port = parseInt(a) + except ValueError: + echo "Error parsing port number: " & a + echo outp.readAll() + quit 1 socket.connect("localhost", Port(port)) + for req, resp in items(s.script): if not runCmd(req, s.dest): socket.sendEpcStr(req) let sx = parseSexp(socket.recvEpc()) if not req.startsWith("mod "): - let answer = sexpToAnswer(sx) + let answer = if sx[2].kind == SNil: "" else: sexpToAnswer(sx) doReport(filename, answer, resp, report) - finally: + socket.sendEpcStr "return arg" + # bugfix: this was in `finally` block, causing the original error to be + # potentially masked by another one in case `socket.sendEpcStr` raises + # (e.g. if socket couldn't connect in the 1st place) + finally: close(p) if report.len > 0: echo "==== EPC ========================================" @@ -315,8 +336,12 @@ proc runTest(filename: string): int = answer.add '\L' doReport(filename, answer, resp, report) finally: - inp.writeLine("quit") - inp.flush() + try: + inp.writeLine("quit") + inp.flush() + except IOError, OSError: + # assume it's SIGPIPE, ie, the child already died + discard close(p) if report.len > 0: echo "==== STDIN ======================================" @@ -328,11 +353,16 @@ proc main() = if os.paramCount() > 0: let x = os.paramStr(1) let xx = expandFilename x + # run only stdio when running single test failures += runTest(xx) - failures += runEpcTest(xx) else: - for x in walkFiles(tpath() / "t*.nim"): - echo "Test ", x + let files = toSeq(walkFiles(tpath / "t*.nim")) + for i, x in files: + echo "$#/$# test: $#" % [$i, $files.len, x] + when defined(i386): + if x == "nimsuggest/tests/tmacro_highlight.nim": + echo "skipping" # workaround bug #17945 + continue let xx = expandFilename x when not defined(windows): # XXX Windows IO redirection seems bonkers: diff --git a/nimsuggest/tests/module_20265.nim b/nimsuggest/tests/module_20265.nim new file mode 100644 index 0000000000..24b7d10c93 --- /dev/null +++ b/nimsuggest/tests/module_20265.nim @@ -0,0 +1,6 @@ +type A* = tuple + a: int + b: int + +var x*: A = (a: 2, b: 10) +var y* = (a: 2, b: 10) diff --git a/nimsuggest/tests/t20265_1.nim b/nimsuggest/tests/t20265_1.nim new file mode 100644 index 0000000000..553b3d5451 --- /dev/null +++ b/nimsuggest/tests/t20265_1.nim @@ -0,0 +1,8 @@ +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skField;;a;;int;;*module_20265.nim;;6;;10;;"";;100;;None +sug;;skField;;b;;int;;*module_20265.nim;;6;;16;;"";;100;;None +""" +import module_20265 +y.#[!]# diff --git a/nimsuggest/tests/t20265_2.nim b/nimsuggest/tests/t20265_2.nim new file mode 100644 index 0000000000..33edf2d9ad --- /dev/null +++ b/nimsuggest/tests/t20265_2.nim @@ -0,0 +1,8 @@ +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skField;;a;;int;;*module_20265.nim;;2;;2;;"";;100;;None +sug;;skField;;b;;int;;*module_20265.nim;;3;;2;;"";;100;;None +""" +import module_20265 +x.#[!]# diff --git a/nimsuggest/tests/t20440.nim b/nimsuggest/tests/t20440.nim new file mode 100644 index 0000000000..0456aa0743 --- /dev/null +++ b/nimsuggest/tests/t20440.nim @@ -0,0 +1,7 @@ +when not defined(js): + {.fatal: "Crash".} +echo 4 + +discard """ +$nimsuggest --v3 --tester $file +""" diff --git a/nimsuggest/tests/t20440.nims b/nimsuggest/tests/t20440.nims new file mode 100644 index 0000000000..1336be3d45 --- /dev/null +++ b/nimsuggest/tests/t20440.nims @@ -0,0 +1 @@ +switch("backend", "js") diff --git a/nimsuggest/tests/t21185.nim b/nimsuggest/tests/t21185.nim new file mode 100644 index 0000000000..bf5a0e3cc4 --- /dev/null +++ b/nimsuggest/tests/t21185.nim @@ -0,0 +1,18 @@ + +# Reduced case of 21185. Issue was first parameter being static +proc foo(x: static[int]) = discard + +type + Person = object + name: string + age: int + +let p = Person() +p.#[!]# + +discard """ +$nimsuggest --tester --v3 --maxresults:2 $file +>sug $1 +sug;;skField;;age;;int;;$file;;8;;4;;"";;100;;None +sug;;skField;;name;;string;;$file;;7;;4;;"";;100;;None +""" diff --git a/nimsuggest/tests/t22448.nim b/nimsuggest/tests/t22448.nim new file mode 100644 index 0000000000..8664bbbc3c --- /dev/null +++ b/nimsuggest/tests/t22448.nim @@ -0,0 +1,11 @@ +proc fn(a: static float) = discard +proc fn(a: int) = discard + +let x = 1 +fn(x) + +discard """ +$nimsuggest --tester --v3 $file +>chk $file +chk;;skUnknown;;;;Hint;;* +""" diff --git a/nimsuggest/tests/tchk1.nim b/nimsuggest/tests/tchk1.nim index c28b88b9b3..be6115c1ca 100644 --- a/nimsuggest/tests/tchk1.nim +++ b/nimsuggest/tests/tchk1.nim @@ -17,11 +17,11 @@ proc main = discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tchk1 [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tchk1.nim [Processing]";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"identifier expected, but got \'keyword template\'";;0 chk;;skUnknown;;;;Error;;$file;;14;;0;;"nestable statement requires indentation";;0 -chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 chk;;skUnknown;;;;Error;;$file;;17;;0;;"invalid indentation";;0 +chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 chk;;skUnknown;;;;Hint;;$file;;12;;9;;"\'foo\' is declared but not used [XDeclaredButNotUsed]";;0 chk;;skUnknown;;;;Hint;;$file;;14;;5;;"\'main\' is declared but not used [XDeclaredButNotUsed]";;0 """ diff --git a/nimsuggest/tests/tchk_compiles.nim b/nimsuggest/tests/tchk_compiles.nim index 887a947b5c..c8a3daac49 100644 --- a/nimsuggest/tests/tchk_compiles.nim +++ b/nimsuggest/tests/tchk_compiles.nim @@ -4,5 +4,5 @@ discard compiles(2 + "hello") discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tchk_compiles [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tchk_compiles.nim [Processing]";;0 """ diff --git a/nimsuggest/tests/tconcept1.nim b/nimsuggest/tests/tconcept1.nim new file mode 100644 index 0000000000..d81cd8120a --- /dev/null +++ b/nimsuggest/tests/tconcept1.nim @@ -0,0 +1,12 @@ +SomeNumber = concept + +#[!]# +discard """ +$nimsuggest --tester $file +>chk $1 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tconcept1.nim [Processing]";;0 +chk;;skUnknown;;;;Error;;$file;;1;;13;;"the \'concept\' keyword is only valid in \'type\' sections";;0 +chk;;skUnknown;;;;Error;;$file;;1;;13;;"invalid indentation";;0 +chk;;skUnknown;;;;Error;;$file;;1;;13;;"expression expected, but found \'keyword concept\'";;0 +chk;;skUnknown;;;;Error;;$file;;1;;0;;"\'SomeNumber\' cannot be assigned to";;0 +""" diff --git a/nimsuggest/tests/tconcept2.nim b/nimsuggest/tests/tconcept2.nim new file mode 100644 index 0000000000..7f7d147f5b --- /dev/null +++ b/nimsuggest/tests/tconcept2.nim @@ -0,0 +1,15 @@ + SomeNumber = concept a, type T + a.int is int + int.to(T) is type(a) + +#[!]# +discard """ +$nimsuggest --tester $file +>chk $1 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tconcept2.nim [Processing]";;0 +chk;;skUnknown;;;;Error;;$file;;1;;2;;"invalid indentation";;0 +chk;;skUnknown;;;;Error;;$file;;1;;15;;"the \'concept\' keyword is only valid in \'type\' sections";;0 +chk;;skUnknown;;;;Error;;$file;;1;;15;;"invalid indentation";;0 +chk;;skUnknown;;;;Error;;$file;;1;;15;;"expression expected, but found \'keyword concept\'";;0 +chk;;skUnknown;;;;Error;;$file;;1;;2;;"\'SomeNumber\' cannot be assigned to";;0 +""" diff --git a/nimsuggest/tests/tdef1.nim b/nimsuggest/tests/tdef1.nim index 2cd040ea14..49265bbc13 100644 --- a/nimsuggest/tests/tdef1.nim +++ b/nimsuggest/tests/tdef1.nim @@ -1,12 +1,14 @@ discard """ $nimsuggest --tester $file >def $1 -def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"Return hello";;100 ->def $1 -def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"Return hello";;100 +def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, raises: [].};;$file;;11;;5;;"Return hello";;100 +>def $2 +def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, raises: [].};;$file;;11;;5;;"Return hello";;100 +>def $2 +def;;skProc;;tdef1.hello;;proc (): string{.noSideEffect, gcsafe, raises: [].};;$file;;11;;5;;"Return hello";;100 """ -proc hello(): string = +proc hel#[!]#lo(): string = ## Return hello "Hello" diff --git a/nimsuggest/tests/tdef_let.nim b/nimsuggest/tests/tdef_let.nim new file mode 100644 index 0000000000..3e9456d2f0 --- /dev/null +++ b/nimsuggest/tests/tdef_let.nim @@ -0,0 +1,7 @@ +discard """ +$nimsuggest --tester $file +>def $1 +def;;skLet;;tdef_let.intVar;;int;;$file;;7;;4;;"";;100 +""" + +let int#[!]#Var = 10 diff --git a/nimsuggest/tests/tdot4.nim b/nimsuggest/tests/tdot4.nim index e1ff96553a..f2c6c765f5 100644 --- a/nimsuggest/tests/tdot4.nim +++ b/nimsuggest/tests/tdot4.nim @@ -15,7 +15,7 @@ discard """ $nimsuggest --tester --maxresults:2 $file >sug $1 sug;;skProc;;tdot4.main;;proc (inp: string): string;;$file;;6;;5;;"";;100;;None -sug;;skFunc;;mstrutils.replace;;proc (s: string, sub: string, by: string): string{.noSideEffect, gcsafe, locks: 0.};;*fixtures/mstrutils.nim;;9;;5;;"this is a test version of strutils.replace, it simply returns `by`";;100;;None +sug;;skFunc;;mstrutils.replace;;proc (s: string, sub: string, by: string): string{.noSideEffect, gcsafe, raises: [].};;*fixtures/mstrutils.nim;;9;;5;;"this is a test version of strutils.replace, it simply returns `by`";;100;;None """ # TODO - determine appropriate behaviour for further suggest output and test it diff --git a/nimsuggest/tests/tenum_field.nim b/nimsuggest/tests/tenum_field.nim new file mode 100644 index 0000000000..4ceb3e0216 --- /dev/null +++ b/nimsuggest/tests/tenum_field.nim @@ -0,0 +1,17 @@ +discard """ +$nimsuggest --tester $file +>sug $1 +>sug $2 +sug;;skConst;;tenum_field.BarFoo;;int literal(1);;$file;;10;;6;;"";;100;;Prefix +""" + +proc something() = discard + +const BarFoo = 1 + +type + Foo = enum + # Test that typing the name doesn't give suggestions + somethi#[!]# + # Test that the right hand side still gets suggestions + another = BarFo#[!]# diff --git a/nimsuggest/tests/tfatal1.nim b/nimsuggest/tests/tfatal1.nim new file mode 100644 index 0000000000..19778f22e2 --- /dev/null +++ b/nimsuggest/tests/tfatal1.nim @@ -0,0 +1,15 @@ +{.warning: "I'm a warning!".} +{.error: "I'm an error!".} +{.fatal: "I'm a fatal error!".} +{.error: "I'm an error after fatal error!".} + +#[!]# +discard """ +$nimsuggest --tester $file +>chk $1 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tfatal1.nim [Processing]";;0 +chk;;skUnknown;;;;Warning;;$file;;1;;9;;"I\'m a warning! [User]";;0 +chk;;skUnknown;;;;Error;;$file;;2;;7;;"I\'m an error!";;0 +chk;;skUnknown;;;;Error;;$file;;3;;7;;"fatal error: I\'m a fatal error!";;0 +chk;;skUnknown;;;;Error;;$file;;4;;7;;"I\'m an error after fatal error!";;0 +""" diff --git a/nimsuggest/tests/tgeneric_highlight.nim b/nimsuggest/tests/tgeneric_highlight.nim index 3343236132..f351ab705f 100644 --- a/nimsuggest/tests/tgeneric_highlight.nim +++ b/nimsuggest/tests/tgeneric_highlight.nim @@ -8,12 +8,10 @@ $nimsuggest --tester $file highlight;;skType;;1;;7;;3 highlight;;skProc;;1;;0;;6 highlight;;skProc;;1;;0;;6 -highlight;;skType;;1;;7;;3 highlight;;skProc;;1;;0;;6 highlight;;skType;;2;;14;;3 highlight;;skProc;;2;;7;;6 highlight;;skProc;;2;;7;;6 -highlight;;skType;;2;;14;;3 highlight;;skProc;;2;;7;;6 highlight;;skTemplate;;3;;0;;8 highlight;;skType;;3;;9;;3 diff --git a/nimsuggest/tests/tgenerics.nim b/nimsuggest/tests/tgenerics.nim new file mode 100644 index 0000000000..7f490321c0 --- /dev/null +++ b/nimsuggest/tests/tgenerics.nim @@ -0,0 +1,18 @@ +type + Hello[T] = object + value: T + +proc printHelloValue[T](hello: Hello[T]) = + echo hello.value + +proc main() = + let a = Hello[float]() + p#[!]#rintHelloValue(a) + +main() + +discard """ +$nimsuggest --tester $file +>def $1 +def;;skProc;;tgenerics.printHelloValue;;proc (hello: Hello[printHelloValue.T]);;$file;;5;;5;;"";;100 +""" diff --git a/nimsuggest/tests/tic.nim b/nimsuggest/tests/tic.nim new file mode 100644 index 0000000000..26e644f833 --- /dev/null +++ b/nimsuggest/tests/tic.nim @@ -0,0 +1,20 @@ +import std/[appdirs, assertions, cmdline, compilesettings, decls, + dirs, editdistance, effecttraits, enumerate, enumutils, envvars, + exitprocs, files, formatfloat, genasts, importutils, + isolation, jsonutils, logic, monotimes, objectdollar, + oserrors, outparams, packedsets, paths, private, setutils, sha1, + socketstreams, stackframes, staticos, strbasics, symlinks, syncio, + sysatomics, sysrand, tasks, tempfiles, time_t, typedthreads, varints, + vmutils, widestrs, with, wordwrap, wrapnils] + +proc test(a: string, b:string) = discard +proc test(a: int) = discard + +test(#[!]# + +discard """ +$nimsuggest --v3 --ic:off --tester $file +>con $1 +con;;skProc;;tic.test;;proc (a: string, b: string);;$file;;10;;5;;"";;100 +con;;skProc;;tic.test;;proc (a: int);;$file;;11;;5;;"";;100 +""" \ No newline at end of file diff --git a/nimsuggest/tests/timport_highlight.nim b/nimsuggest/tests/timport_highlight.nim new file mode 100644 index 0000000000..043f87d98f --- /dev/null +++ b/nimsuggest/tests/timport_highlight.nim @@ -0,0 +1,12 @@ +import std/paths +import json as J +import std/[os,streams]#[!]# + +discard """ +$nimsuggest --tester $file +>highlight $1 +highlight;;skModule;;1;;11;;5 +highlight;;skModule;;2;;7;;4 +highlight;;skModule;;3;;12;;2 +highlight;;skModule;;3;;15;;7 +""" diff --git a/nimsuggest/tests/tinclude.nim b/nimsuggest/tests/tinclude.nim index b67440b9e3..f5cbabf053 100644 --- a/nimsuggest/tests/tinclude.nim +++ b/nimsuggest/tests/tinclude.nim @@ -11,7 +11,7 @@ go() discard """ $nimsuggest --tester $file >def $path/tinclude.nim:7:14 -def;;skProc;;minclude_import.create;;proc (greeting: string, subject: string): Greet{.noSideEffect, gcsafe, locks: 0.};;*fixtures/minclude_include.nim;;3;;5;;"";;100 +def;;skProc;;minclude_import.create;;proc (greeting: string, subject: string): Greet{.noSideEffect, gcsafe, raises: [].};;*fixtures/minclude_include.nim;;3;;5;;"";;100 >def $path/fixtures/minclude_include.nim:3:71 def;;skType;;minclude_types.Greet;;Greet;;*fixtures/minclude_types.nim;;4;;2;;"";;100 >def $path/fixtures/minclude_include.nim:3:71 diff --git a/nimsuggest/tests/tqualified_highlight.nim b/nimsuggest/tests/tqualified_highlight.nim index 3b521ecc18..b83669e72b 100644 --- a/nimsuggest/tests/tqualified_highlight.nim +++ b/nimsuggest/tests/tqualified_highlight.nim @@ -10,5 +10,5 @@ highlight;;skProc;;1;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 -highlight;;skProc;;3;;8;;1 +highlight;;skFunc;;3;;8;;1 """ diff --git a/nimsuggest/tests/tsug_pragmas.nim b/nimsuggest/tests/tsug_pragmas.nim new file mode 100644 index 0000000000..ce9c4e8f8d --- /dev/null +++ b/nimsuggest/tests/tsug_pragmas.nim @@ -0,0 +1,40 @@ +template fooBar1() {.pragma.} +proc fooBar2() = discard +macro fooBar3(x: untyped) = discard +{.pragma: fooBar4 fooBar3.} + +proc test1() {.fooBar#[!]#.} = discard + +var test2 {.fooBar#[!]#.} = 9 + +type + Person {.fooBar#[!]#.} = object + hello {.fooBar#[!]#.}: string + Callback = proc () {.fooBar#[!]#.} + +# Check only macros/templates/pragmas are suggested +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix +sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix +sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe, raises: [].};;$file;;3;;6;;"";;50;;Prefix +>sug $2 +sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix +sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix +sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe, raises: [].};;$file;;3;;6;;"";;50;;Prefix +>sug $3 +sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix +sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix +sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe, raises: [].};;$file;;3;;6;;"";;50;;Prefix +>sug $4 +sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix +sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix +sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe, raises: [].};;$file;;3;;6;;"";;50;;Prefix +>sug $5 +sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix +sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix +sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe, raises: [].};;$file;;3;;6;;"";;50;;Prefix +""" + + diff --git a/nimsuggest/tests/tsug_recursive.nim b/nimsuggest/tests/tsug_recursive.nim new file mode 100644 index 0000000000..97ee5ca015 --- /dev/null +++ b/nimsuggest/tests/tsug_recursive.nim @@ -0,0 +1,8 @@ +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skProc;;tsug_recursive.fooBar;;proc ();;$file;;7;;5;;"";;100;;Prefix +""" + +proc fooBar() = + fooBa#[!]# diff --git a/nimsuggest/tests/tsug_template.nim b/nimsuggest/tests/tsug_template.nim index 15615f0afb..da494d279d 100644 --- a/nimsuggest/tests/tsug_template.nim +++ b/nimsuggest/tests/tsug_template.nim @@ -6,7 +6,7 @@ tmp#[!]# discard """ $nimsuggest --tester $file >sug $1 -sug;;skMacro;;tsug_template.tmpb;;macro (){.noSideEffect, gcsafe, locks: 0.};;$file;;2;;6;;"";;100;;Prefix +sug;;skMacro;;tsug_template.tmpb;;macro (){.noSideEffect, gcsafe, raises: [].};;$file;;2;;6;;"";;100;;Prefix sug;;skConverter;;tsug_template.tmpc;;converter ();;$file;;3;;10;;"";;100;;Prefix sug;;skTemplate;;tsug_template.tmpa;;template ();;$file;;1;;9;;"";;100;;Prefix """ diff --git a/nimsuggest/tests/tsug_typedecl.nim b/nimsuggest/tests/tsug_typedecl.nim index 833043d31d..2a510929db 100644 --- a/nimsuggest/tests/tsug_typedecl.nim +++ b/nimsuggest/tests/tsug_typedecl.nim @@ -21,6 +21,6 @@ $nimsuggest --tester $file >sug $1 sug;;skType;;tsug_typedecl.someType;;someType;;*nimsuggest/tests/tsug_typedecl.nim;;7;;2;;"";;100;;Prefix sug;;skType;;tsug_typedecl.super;;super;;*nimsuggest/tests/tsug_typedecl.nim;;6;;2;;"";;100;;Prefix -sug;;skType;;system.string;;string;;*lib/system.nim;;*;;*;;*;;100;;Prefix +sug;;skType;;system.string;;string;;*lib/system/basic_types.nim;;*;;*;;*;;100;;Prefix sug;;skType;;system.seq;;seq;;*lib/system.nim;;*;;*;;*;;100;;Prefix -""" \ No newline at end of file +""" diff --git a/nimsuggest/tests/ttempl_inst.nim b/nimsuggest/tests/ttempl_inst.nim index ed04a67ce1..5f5b10fe90 100644 --- a/nimsuggest/tests/ttempl_inst.nim +++ b/nimsuggest/tests/ttempl_inst.nim @@ -7,7 +7,7 @@ foo() discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"ttempl_inst [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/ttempl_inst.nim [Processing]";;0 chk;;skUnknown;;;;Hint;;$file;;4;;3;;"template/generic instantiation from here";;0 chk;;skUnknown;;;;Warning;;$file;;2;;11;;"foo [User]";;0 """ diff --git a/nimsuggest/tests/ttype_decl.nim b/nimsuggest/tests/ttype_decl.nim index 6022392d00..61d8c26cd0 100644 --- a/nimsuggest/tests/ttype_decl.nim +++ b/nimsuggest/tests/ttype_decl.nim @@ -2,8 +2,8 @@ discard """ $nimsuggest --tester --maxresults:3 $file >sug $1 sug;;skType;;ttype_decl.Other;;Other;;$file;;10;;2;;"";;100;;None -sug;;skType;;system.int;;int;;*/lib/system/basic_types.nim;;2;;2;;"";;100;;None -sug;;skType;;system.string;;string;;*/lib/system.nim;;34;;2;;"";;100;;None +sug;;skType;;system.int;;int;;*lib/system/basic_types.nim;;2;;2;;"";;100;;None +sug;;skType;;system.string;;string;;*lib/system/basic_types.nim;;23;;2;;"";;100;;None """ import strutils type diff --git a/nimsuggest/tests/tuse.nim b/nimsuggest/tests/tuse.nim index 89a9c151a9..7c1d1ad0c8 100644 --- a/nimsuggest/tests/tuse.nim +++ b/nimsuggest/tests/tuse.nim @@ -14,9 +14,9 @@ proc #[!]#someProc*() = discard """ $nimsuggest --tester $file >use $1 -def;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"";;100 -use;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, locks: 0.};;$file;;12;;0;;"";;100 +def;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, raises: [].};;$file;;9;;5;;"";;100 +use;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, raises: [].};;$file;;12;;0;;"";;100 >use $2 -def;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, locks: 0.};;$file;;9;;5;;"";;100 -use;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, locks: 0.};;$file;;12;;0;;"";;100 +def;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, raises: [].};;$file;;9;;5;;"";;100 +use;;skProc;;tuse.someProc;;proc (){.noSideEffect, gcsafe, raises: [].};;$file;;12;;0;;"";;100 """ diff --git a/nimsuggest/tests/tuse_enum.nim b/nimsuggest/tests/tuse_enum.nim new file mode 100644 index 0000000000..8a40a83489 --- /dev/null +++ b/nimsuggest/tests/tuse_enum.nim @@ -0,0 +1,15 @@ +discard """ +$nimsuggest --tester $file +>use $1 +def;;skEnumField;;tuse_enum.Colour.Red;;Colour;;$file;;10;;4;;"";;100 +use;;skEnumField;;tuse_enum.Colour.Red;;Colour;;$file;;14;;8;;"";;100 +""" + +type + Colour = enum + Red + Green + Blue + +discard #[!]#Red + diff --git a/nimsuggest/tests/tuse_structure.nim b/nimsuggest/tests/tuse_structure.nim new file mode 100644 index 0000000000..f65ab9060e --- /dev/null +++ b/nimsuggest/tests/tuse_structure.nim @@ -0,0 +1,15 @@ +# tests for use and structures + +type + Foo* = ref object of RootObj + bar*: string + +proc test(f: Foo) = + echo f.#[!]#bar + +discard """ +$nimsuggest --tester $file +>use $1 +def skField tuse_structure.Foo.bar string $file 5 4 "" 100 +use skField tuse_structure.Foo.bar string $file 8 9 "" 100 +""" diff --git a/nimsuggest/tests/tv3.nim b/nimsuggest/tests/tv3.nim new file mode 100644 index 0000000000..80e51e3644 --- /dev/null +++ b/nimsuggest/tests/tv3.nim @@ -0,0 +1,27 @@ +# tests v3 + +type + Foo* = ref object of RootObj + bar*: string + +proc test(f: Foo) = + echo f.ba#[!]#r + +#[!]# + +discard """ +$nimsuggest --v3 --tester $file +>use $1 +def skField tv3.Foo.bar string $file 5 4 "" 100 +use skField tv3.Foo.bar string $file 8 9 "" 100 +>def $1 +def skField tv3.Foo.bar string $file 5 4 "" 100 +>sug $1 +sug skField bar string $file 5 4 "" 100 Prefix +>globalSymbols test +def skProc tv3.test proc (f: Foo){.gcsafe, raises: [].} $file 7 5 "" 100 +>globalSymbols Foo +def skType tv3.Foo Foo $file 4 2 "" 100 +>def $2 +>use $2 +""" diff --git a/nimsuggest/tests/tv3_con.nim b/nimsuggest/tests/tv3_con.nim new file mode 100644 index 0000000000..4714c366ba --- /dev/null +++ b/nimsuggest/tests/tv3_con.nim @@ -0,0 +1,13 @@ +# tests v3 + +proc test(a: string, b:string) = discard +proc test(a: int) = discard + +test(#[!]# + +discard """ +$nimsuggest --v3 --tester $file +>con $1 +con;;skProc;;tv3_con.test;;proc (a: string, b: string);;$file;;3;;5;;"";;100 +con;;skProc;;tv3_con.test;;proc (a: int);;$file;;4;;5;;"";;100 +""" diff --git a/nimsuggest/tests/tv3_definition.nim b/nimsuggest/tests/tv3_definition.nim new file mode 100644 index 0000000000..03684b7cd5 --- /dev/null +++ b/nimsuggest/tests/tv3_definition.nim @@ -0,0 +1,9 @@ + +let foo = 30 +let bar = foo + fo#[!]#o + foo + +discard """ +$nimsuggest --v3 --tester $file +>def $1 +def skLet tv3_definition.foo int $file 2 4 "" 100 +""" diff --git a/nimsuggest/tests/tv3_forward_definition.nim b/nimsuggest/tests/tv3_forward_definition.nim new file mode 100644 index 0000000000..7a16ea331c --- /dev/null +++ b/nimsuggest/tests/tv3_forward_definition.nim @@ -0,0 +1,23 @@ +proc de#[!]#mo(): int + +proc de#[!]#mo(): int = 5 + +let a = de#[!]#mo() + +discard """ +$nimsuggest --v3 --tester $file +>use $1 +use skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 1 5 "" 100 +def skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 3 5 "" 100 +use skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 5 8 "" 100 +>use $2 +use skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 1 5 "" 100 +def skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 3 5 "" 100 +use skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 5 8 "" 100 +>declaration $1 +declaration skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 3 5 "" 100 +>declaration $2 +declaration skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 1 5 "" 100 +>declaration $3 +declaration skProc tv3_forward_definition.demo proc (): int{.noSideEffect, gcsafe, raises: [].} $file 1 5 "" 100 +""" diff --git a/nimsuggest/tests/tv3_generics.nim b/nimsuggest/tests/tv3_generics.nim new file mode 100644 index 0000000000..2bfb2ca1d6 --- /dev/null +++ b/nimsuggest/tests/tv3_generics.nim @@ -0,0 +1,18 @@ +type + Hello[T] = object + value: T + +proc printHelloValue[T](hello: Hello[T]) = + echo hello.value + +proc main() = + let a = Hello[float]() + p#[!]#rintHelloValue(a) + +main() + +discard """ +$nimsuggest --v3 --tester $file +>def $1 +def;;skProc;;tv3_generics.printHelloValue;;proc (hello: Hello[printHelloValue.T]);;$file;;5;;5;;"";;100 +""" diff --git a/nimsuggest/tests/tv3_globalSymbols.nim b/nimsuggest/tests/tv3_globalSymbols.nim new file mode 100644 index 0000000000..c3bb9933b6 --- /dev/null +++ b/nimsuggest/tests/tv3_globalSymbols.nim @@ -0,0 +1,14 @@ +# Tests the order of the matches +proc Btoken(): int = 5 +proc tokenA(): int = 5 +proc token(): int = 5 +proc BBtokenA(): int = 5 + +discard """ +$nimsuggest --v3 --tester $file +>globalSymbols token +def skProc tv3_globalSymbols.token proc (): int{.noSideEffect, gcsafe, raises: [].} $file 4 5 "" 100 +def skProc tv3_globalSymbols.tokenA proc (): int{.noSideEffect, gcsafe, raises: [].} $file 3 5 "" 100 +def skProc tv3_globalSymbols.Btoken proc (): int{.noSideEffect, gcsafe, raises: [].} $file 2 5 "" 100 +def skProc tv3_globalSymbols.BBtokenA proc (): int{.noSideEffect, gcsafe, raises: [].} $file 5 5 "" 100 +""" diff --git a/nimsuggest/tests/tv3_import.nim b/nimsuggest/tests/tv3_import.nim new file mode 100644 index 0000000000..3c128f85b0 --- /dev/null +++ b/nimsuggest/tests/tv3_import.nim @@ -0,0 +1,7 @@ +import tv#[!]#3 + +discard """ +$nimsuggest --v3 --tester $file +>def $1 +def skModule tv3 */tv3.nim 1 0 "" 100 +""" diff --git a/nimsuggest/tests/tv3_outline.nim b/nimsuggest/tests/tv3_outline.nim new file mode 100644 index 0000000000..518620c871 --- /dev/null +++ b/nimsuggest/tests/tv3_outline.nim @@ -0,0 +1,45 @@ +# tests v3 outline + +type + Foo* = ref object of RootObj + bar*: string + FooEnum = enum value1, value2 + FooPrivate = ref object of RootObj + barPrivate: string + +macro m(arg: untyped): untyped = discard +template t(arg: untyped): untyped = discard +proc p(): void = discard +iterator i(): int = discard +converter c(s: string): int = discard +method m(f: Foo): void = discard +func f(): void = discard + +let a = 1 +var b = 2 +const con = 2 + +proc outer(): void = + proc inner() = discard + +proc procWithLocal(): void = + let local = 10 + +discard """ +$nimsuggest --v3 --tester $file +>outline $file +outline skType tv3_outline.Foo Foo $file 4 2 "" 100 5 16 +outline skType tv3_outline.FooEnum FooEnum $file 6 2 "" 100 6 31 +outline skEnumField tv3_outline.FooEnum.value1 FooEnum $file 6 17 "" 100 6 23 +outline skEnumField tv3_outline.FooEnum.value2 FooEnum $file 6 25 "" 100 6 31 +outline skType tv3_outline.FooPrivate FooPrivate $file 7 2 "" 100 8 22 +outline skMacro tv3_outline.m macro (arg: untyped): untyped{.noSideEffect, gcsafe, raises: [].} $file 10 6 "" 100 10 40 +outline skTemplate tv3_outline.t template (arg: untyped): untyped $file 11 9 "" 100 11 43 +outline skProc tv3_outline.p proc (){.noSideEffect, gcsafe, raises: [].} $file 12 5 "" 100 12 24 +outline skConverter tv3_outline.c converter (s: string): int{.noSideEffect, gcsafe, raises: [].} $file 14 10 "" 100 14 37 +outline skFunc tv3_outline.f proc (){.noSideEffect, gcsafe, raises: [].} $file 16 5 "" 100 16 24 +outline skConst tv3_outline.con int literal(2) $file 20 6 "" 100 20 13 +outline skProc tv3_outline.outer proc (){.noSideEffect, gcsafe, raises: [].} $file 22 5 "" 100 23 24 +outline skProc tv3_outline.outer.inner proc (){.noSideEffect, gcsafe, raises: [].} $file 23 7 "" 100 23 24 +outline skProc tv3_outline.procWithLocal proc (){.noSideEffect, gcsafe, raises: [].} $file 25 5 "" 100 26 16 +""" diff --git a/nimsuggest/tests/tv3_typeDefinition.nim b/nimsuggest/tests/tv3_typeDefinition.nim new file mode 100644 index 0000000000..f86d12cc67 --- /dev/null +++ b/nimsuggest/tests/tv3_typeDefinition.nim @@ -0,0 +1,32 @@ +# tests v3 + +type + Foo* = ref object of RootObj + bar*: string + +proc test(ff: Foo) = + echo f#[!]#f.bar + +type + Fo#[!]#o2* = ref object of RootObj + +type + FooGeneric[T] = ref object of RootObj + bar*: T + +let fooGeneric = FooGeneric[string]() +echo fo#[!]#oGeneric.bar + +# bad type +echo unde#[!]#fined + +discard """ +$nimsuggest --v3 --tester $file +>type $1 +type skType tv3_typeDefinition.Foo Foo $file 4 2 "" 100 +>type $2 +type skType tv3_typeDefinition.Foo2 Foo2 $file 11 2 "" 100 +>type $3 +type skType tv3_typeDefinition.FooGeneric FooGeneric $file 14 2 "" 100 +>type $4 +""" diff --git a/nimsuggest/tests/twithin_macro.nim b/nimsuggest/tests/twithin_macro.nim index d79dae1be0..98d58381f0 100644 --- a/nimsuggest/tests/twithin_macro.nim +++ b/nimsuggest/tests/twithin_macro.nim @@ -45,7 +45,7 @@ $nimsuggest --tester --maxresults:5 $file >sug $1 sug;;skField;;age;;int;;$file;;6;;6;;"";;100;;None sug;;skField;;name;;string;;$file;;5;;6;;"";;100;;None -sug;;skMethod;;twithin_macro.age_human_yrs;;proc (self: Animal): int;;$file;;8;;9;;"";;100;;None -sug;;skMethod;;twithin_macro.vocalize;;proc (self: Animal): string;;$file;;7;;9;;"";;100;;None +sug;;skMethod;;twithin_macro.age_human_yrs;;proc (self: Animal): int{.raises: [].};;$file;;8;;9;;"";;100;;None +sug;;skMethod;;twithin_macro.vocalize;;proc (self: Animal): string{.raises: [].};;$file;;7;;9;;"";;100;;None sug;;skMethod;;twithin_macro.vocalize;;proc (self: Rabbit): string;;$file;;23;;9;;"";;100;;None """ diff --git a/nimsuggest/tests/twithin_macro_prefix.nim b/nimsuggest/tests/twithin_macro_prefix.nim index dd38108183..e89c8b9423 100644 --- a/nimsuggest/tests/twithin_macro_prefix.nim +++ b/nimsuggest/tests/twithin_macro_prefix.nim @@ -44,5 +44,5 @@ discard """ $nimsuggest --tester $file >sug $1 sug;;skField;;age;;int;;$file;;6;;6;;"";;100;;Prefix -sug;;skMethod;;twithin_macro_prefix.age_human_yrs;;proc (self: Animal): int;;$file;;8;;9;;"";;100;;Prefix +sug;;skMethod;;twithin_macro_prefix.age_human_yrs;;proc (self: Animal): int{.raises: [].};;$file;;8;;9;;"";;100;;Prefix """ diff --git a/readme.md b/readme.md index fa7a2d86d5..da54a8d0df 100644 --- a/readme.md +++ b/readme.md @@ -11,31 +11,35 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( [![Join the IRC chat][badge-nim-irc]][nim-irc] [![Join the Discord server][badge-nim-discord]][nim-discord] [![Join the Gitter chat][badge-nim-gitter]][nim-gitter] +[![Join the Matrix room](https://img.shields.io/matrix/nim-lang:matrix.org?color=blue&style=flat&logo=matrix)](https://matrix.to/#/#nim-lang:matrix.org) [![Get help][badge-nim-forum-gethelp]][nim-forum] [![View Nim posts on Stack Overflow][badge-nim-stackoverflow]][nim-stackoverflow-newest] [![Follow @nim_lang on Twitter][badge-nim-twitter]][nim-twitter] * The [forum][nim-forum] - the best place to ask questions and to discuss Nim. -* [#nim IRC Channel (Freenode)][nim-irc] - a place to discuss Nim in real-time. +* [#nim IRC Channel (Libera Chat)][nim-irc] - a place to discuss Nim in real-time. Also where most development decisions get made. * [Discord][nim-discord] - an additional place to discuss Nim in real-time. Most channels there are bridged to IRC. * [Gitter][nim-gitter] - an additional place to discuss Nim in real-time. There is a bridge between Gitter and the IRC channel. +* [Matrix][nim-matrix] - the main room to discuss Nim in real-time. [Matrix space][nim-matrix-space] contains a list of rooms, most of them are bridged to IRC. * [Telegram][nim-telegram] - an additional place to discuss Nim in real-time. There is the official Telegram channel. Not bridged to IRC. * [Stack Overflow][nim-stackoverflow] - a popular Q/A site for programming related topics that includes posts about Nim. -* [Github Wiki][nim-wiki] - Misc user-contributed content. +* [GitHub Wiki][nim-wiki] - Misc user-contributed content. ## Compiling The compiler currently officially supports the following platform and architecture combinations: - * Windows (Windows XP or greater) - x86 and x86_64 - * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l - * Mac OS X (10.04 or greater) - x86, x86_64 and ppc64 +| Operating System | Architectures Supported | +|--------------------------------|----------------------------------------| +| Windows (Windows XP or greater) | x86 and x86_64 | +| Linux (most distributions) | x86, x86_64, ppc64, and armv6l | +| Mac OS X (10.04 or greater) | x86, x86_64, ppc64, and Apple Silicon (ARM64) | More platforms are supported, however, they are not tested regularly and they may not be as stable as the above-listed platforms. @@ -45,12 +49,12 @@ Compiling the Nim compiler is quite straightforward if you follow these steps: First, the C source of an older version of the Nim compiler is needed to bootstrap the latest version because the Nim compiler itself is written in the Nim programming language. Those C sources are available within the -[``nim-lang/csources``][csources-repo] repository. +[``nim-lang/csources_v2``][csources-v2-repo] repository. Next, to build from source you will need: - * A C compiler such as ``gcc`` 3.x/later or an alternative such as ``clang``, - ``Visual C++`` or ``Intel C++``. It is recommended to use ``gcc`` 3.x or + * A C compiler such as ``gcc`` 5.x/later or an alternative such as ``clang``, + ``Visual C++`` or ``Intel C++``. It is recommended to use ``gcc`` 5.x or later. * Either ``git`` or ``wget`` to download the needed source repositories. * The ``build-essential`` package when using ``gcc`` on Ubuntu (and likely @@ -71,8 +75,7 @@ the installation instructions on the website to do so: https://nim-lang.org/inst For package maintainers: see [packaging guidelines](https://nim-lang.github.io/Nim/packaging.html). - -First, get Nim from github: +First, get Nim from GitHub: ``` git clone https://github.com/nim-lang/Nim.git @@ -84,14 +87,12 @@ Next, run the appropriate build shell script for your platform: * `build_all.sh` (Linux, Mac) * `build_all.bat` (Windows) -Windows requires a number of other dependencies that you may need to install including -PCRE and OpenSSL. Nim hosts a zip package containing known working versions of the -required DLLs [here](https://nim-lang.org/download/dlls.zip). - Finally, once you have finished the build steps (on Windows, Mac, or Linux) you should add the ``bin`` directory to your PATH. -See also [rebuilding the compiler](doc/intern.rst#rebuilding-the-compiler). +See also [bootstrapping the compiler](https://nim-lang.github.io/Nim/intern.html#bootstrapping-the-compiler). + +See also [reproducible builds](https://nim-lang.github.io/Nim/intern.html#bootstrapping-the-compiler-reproducible-builds). ## Koch @@ -105,7 +106,7 @@ can run a subset of tests by specifying a category (for example ``./koch tests cat async``). For more information on the ``koch`` build tool please see the documentation -within the [doc/koch.rst](doc/koch.rst) file. +within the [doc/koch.md](https://nim-lang.github.io/Nim/koch.html) file. ## Nimble @@ -120,7 +121,6 @@ This project exists thanks to all the people who contribute. ## Contributing [![Backers on Open Collective](https://opencollective.com/nim/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/nim/sponsors/badge.svg)](#sponsors) -[![Setup a bounty via Bountysource][badge-nim-bountysource]][nim-bountysource] [![Donate Bitcoins][badge-nim-bitcoin]][nim-bitcoin] [![Open Source Helpers](https://www.codetriage.com/nim-lang/nim/badges/users.svg)](https://www.codetriage.com/nim-lang/nim) @@ -131,8 +131,7 @@ standard library are welcomed and appreciated. Before you start contributing, you should familiarize yourself with the following repository structure: * ``bin/``, ``build/`` - these directories are empty, but are used when Nim is built. -* ``compiler/`` - the compiler source code. Also includes nimfix, and plugins within - ``compiler/nimfix`` and ``compiler/plugins`` respectively. +* ``compiler/`` - the compiler source code. Also includes plugins within ``compiler/plugins``. * ``nimsuggest`` - the nimsuggest tool that previously lived in the [``nim-lang/nimsuggest``][nimsuggest-repo] repository. * ``config/`` - the configuration for the compiler and documentation generator. * ``doc/`` - the documentation files in reStructuredText format. @@ -142,7 +141,7 @@ you should familiarize yourself with the following repository structure: dependencies written in other languages. * ``wrappers/`` - modules that wrap dependencies written in other languages. * ``tests/`` - contains categorized tests for the compiler and standard library. -* ``tools/`` - the tools including ``niminst`` and ``nimweb`` (mostly invoked via +* ``tools/`` - the tools including ``niminst`` (mostly invoked via ``koch``). * ``koch.nim`` - the tool used to bootstrap Nim, generate C sources, build the website, and generate the documentation. @@ -152,7 +151,7 @@ read [this guide][pull-request-instructions]. Ideally, you should make sure that all tests pass before submitting a pull request. However, if you are short on time, you can just run the tests specific to your -changes by only running the corresponding categories of tests. Travis CI verifies +changes by only running the corresponding categories of tests. CI verifies that all tests pass before allowing the pull request to be accepted, so only running specific tests should be harmless. Integration tests should go in ``tests/untestable``. @@ -165,7 +164,6 @@ You can also help with the development of Nim by making donations. Donations can made using: * [Open Collective](https://opencollective.com/nim) -* [Bountysource][nim-bountysource] * [Bitcoin][nim-bitcoin] If you have any questions feel free to submit a question on the @@ -204,32 +202,32 @@ Nim. You are explicitly permitted to develop commercial applications using Nim. Please read the [copying.txt](copying.txt) file for more details. -Copyright © 2006-2021 Andreas Rumpf, all rights reserved. +Copyright © 2006-2024 Andreas Rumpf, all rights reserved. [nim-site]: https://nim-lang.org [nim-forum]: https://forum.nim-lang.org [nim-issues]: https://github.com/nim-lang/Nim/issues [nim-issues-easy]: https://github.com/nim-lang/Nim/labels/Easy -[nim-irc]: https://webchat.freenode.net/?channels=nim +[nim-irc]: https://web.libera.chat/#nim [nim-twitter]: https://twitter.com/nim_lang [nim-stackoverflow]: https://stackoverflow.com/questions/tagged/nim-lang [nim-stackoverflow-newest]: https://stackoverflow.com/questions/tagged/nim-lang?sort=newest&pageSize=15 [nim-discord]: https://discord.gg/nim [nim-gitter]: https://gitter.im/nim-lang/Nim +[nim-matrix]: https://matrix.to/#/#nim-lang:matrix.org +[nim-matrix-space]: https://matrix.to/#/#nim:envs.net [nim-telegram]: https://t.me/nim_lang -[nim-bountysource]: https://www.bountysource.com/teams/nim [nim-bitcoin]: https://blockchain.info/address/1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ [nimble-repo]: https://github.com/nim-lang/nimble [nimsuggest-repo]: https://github.com/nim-lang/nimsuggest -[csources-repo]: https://github.com/nim-lang/csources -[badge-nim-travisci]: https://img.shields.io/travis/nim-lang/Nim/devel.svg?style=flat-square +[csources-repo-deprecated]: https://github.com/nim-lang/csources +[csources-v2-repo]: https://github.com/nim-lang/csources_v2 [badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square [badge-nim-discord]: https://img.shields.io/discord/371759389889003530?color=blue&label=discord&logo=discord&logoColor=gold&style=flat-square [badge-nim-gitter]: https://img.shields.io/badge/chat-on_gitter-blue.svg?style=flat-square [badge-nim-forum-gethelp]: https://img.shields.io/badge/Forum-get%20help-4eb899.svg?style=flat-square [badge-nim-twitter]: https://img.shields.io/twitter/follow/nim_lang.svg?style=social [badge-nim-stackoverflow]: https://img.shields.io/badge/stackoverflow-nim_tag-yellow.svg?style=flat-square -[badge-nim-bountysource]: https://img.shields.io/bountysource/team/nim/activity.svg?style=flat-square [badge-nim-bitcoin]: https://img.shields.io/badge/bitcoin-1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ-D69134.svg?style=flat-square [pull-request-instructions]: https://help.github.com/articles/using-pull-requests/ [nim-wiki]: https://github.com/nim-lang/Nim/wiki diff --git a/security.md b/security.md index e8d31b1b93..72a1a3e3d6 100644 --- a/security.md +++ b/security.md @@ -2,7 +2,10 @@ ## Supported Versions -Security fixes are provided in new releases and bugfix releases. +Security advisories are published at: +https://github.com/nim-lang/security/security/advisories?state=published + +Security fixes are provided in new releases and in bugfix releases. We do not backport security fixes to older releases. @@ -10,8 +13,8 @@ We do not backport security fixes to older releases. ## Reporting a Vulnerability -Please do not report vulnerabilities via GitHub issues. - -If you have discovered a vulnerability, it is best to notify us about it via +If you have discovered a vulnerability, please notify us about it via security@nim-lang.org in order to set up a meeting where we can discuss the next steps. + +Please do not report vulnerabilities via GitHub issues. diff --git a/testament/categories.nim b/testament/categories.nim index acb3ee0a87..843bef3f9a 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -13,7 +13,8 @@ # included from testament.nim import important_packages -import std/strformat +import std/[strformat, strutils] +from std/sequtils import filterIt const specialCategories = [ @@ -22,13 +23,11 @@ const "debugger", "dll", "examples", - "flags", "gc", "io", "js", "ic", "lib", - "longgc", "manyloc", "nimble-packages", "niminaction", @@ -39,7 +38,6 @@ const "coroutines", "osproc", "shouldfail", - "dir with space", "destructor" ] @@ -47,29 +45,9 @@ proc isTestFile*(file: string): bool = let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") -# --------------------- flags tests ------------------------------------------- - -proc flagTests(r: var TResults, cat: Category, options: string) = - # --genscript - const filename = testsDir/"flags"/"tgenscript" - const genopts = " --genscript" - let nimcache = nimcacheDir(filename, genopts, targetC) - testSpec r, makeTest(filename, genopts, cat) - - when defined(windows): - testExec r, makeTest(filename, " cmd /c cd " & nimcache & - " && compile_tgenscript.bat", cat) - - elif defined(posix): - testExec r, makeTest(filename, " sh -c \"cd " & nimcache & - " && sh compile_tgenscript.sh\"", cat) - - # Run - testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat) - # --------------------- DLL generation tests ---------------------------------- -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: @@ -81,10 +59,11 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat) + + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 - var test4 = makeTest("tests/dll/visibility.nim", options & " --app:lib" & rpath, cat) + var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 @@ -99,24 +78,34 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = defer: putEnv(libpathenv, libpath) testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat) - testSpec r, makeTest("tests/dll/visibility.nim", options & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: + # hcr tests + + var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + # test segfaults for now but compiles: + if isOrc: basicHcrTest.spec.action = actionCompile + testSpec r, basicHcrTest + # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", - options & " --forceBuild --hotCodeReloading:on" & rpath, cat) + options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) - hcri.args = prepareTestArgs(hcri.spec.getCmd, hcri.name, + let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) + hcri.testArgs = cmd.parseCmdLine testSpec r, hcri proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() - runBasicDLLTest c, r, cat, options - runBasicDLLTest c, r, cat, options & " -d:release" + runBasicDLLTest c, r, cat, options & " --mm:refc" + runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows): # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" @@ -126,9 +115,9 @@ proc dllTests(r: var TResults, cat: Category, options: string) = proc gcTests(r: var TResults, cat: Category, options: string) = template testWithoutMs(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options, cat) + testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release -d:useRealtimeGC", cat) + " -d:release -d:useRealtimeGC --mm:refc", cat) when filename != "gctest": testSpec r, makeTest("tests/gc" / filename, options & " --gc:orc", cat) @@ -173,20 +162,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "stackrefleak" test "cyclecollector" - test "trace_globals" - -proc longGCTests(r: var TResults, cat: Category, options: string) = - when defined(windows): - let cOptions = "-ldl -DWIN" - else: - let cOptions = "-ldl" - - var c = initResults() - # According to ioTests, this should compile the file - testSpec c, makeTest("tests/realtimeGC/shared", options, cat) - # ^- why is this not appended to r? Should this be discarded? - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat) + testWithoutBoehm "trace_globals" # ------------------------- threading tests ----------------------------------- @@ -205,6 +181,11 @@ proc ioTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) + # ^- why is this not appended to r? Should this be discarded? + # EDIT: this should be replaced by something like in D20210524T180826, + # likewise in similar instances where `testSpec c` is used, or more generally + # when a test depends on another test, as it makes tests non-independent, + # creating complications for batching and megatest logic. testSpec r, makeTest("tests/system/tio", options, cat) # ------------------------- async tests --------------------------------------- @@ -237,9 +218,9 @@ proc jsTests(r: var TResults, cat: Category, options: string) = for testfile in ["exception/texceptions", "exception/texcpt1", "exception/texcsub", "exception/tfinally", "exception/tfinally2", "exception/tfinally3", - "actiontable/tactiontable", "method/tmultimjs", + "collections/tactiontable", "method/tmultimjs", "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", - "varres/tvartup", "misc/tints", "misc/tunsignedinc", + "varres/tvartup", "int/tints", "int/tunsignedinc", "async/tjsandnativeasync"]: test "tests/" & testfile & ".nim" @@ -339,7 +320,7 @@ proc findMainFile(dir: string): string = var nimFiles = 0 for kind, file in os.walkDir(dir): if kind == pcFile: - if file.endsWith(cfgExt): return file[ .. ^(cfgExt.len+1)] & ".nim" + if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim" elif file.endsWith(".nim"): if result.len == 0: result = file inc nimFiles @@ -399,8 +380,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = testSpec r, testObj # ----------------------------- nimble ---------------------------------------- -proc listPackages(packageFilter: string): seq[NimblePackage] = - # xxx document `packageFilter`, seems like a bad API (at least should be a regex; a substring match makes no sense) +proc listPackagesAll(): seq[NimblePackage] = var nimbleDir = getEnv("NIMBLE_DIR") if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" let packageIndex = nimbleDir / "packages_official.json" @@ -409,14 +389,30 @@ proc listPackages(packageFilter: string): seq[NimblePackage] = for a in packageList: if a["name"].str == name: return a for pkg in important_packages.packages.items: - if isCurrentBatch(testamentData0, pkg.name) and packageFilter in pkg.name: - var pkg = pkg - if pkg.url.len == 0: - let pkg2 = findPackage(pkg.name) - if pkg2 == nil: - raise newException(ValueError, "Cannot find package '$#'." % pkg.name) - pkg.url = pkg2["url"].str - result.add pkg + var pkg = pkg + if pkg.url.len == 0: + let pkg2 = findPackage(pkg.name) + if pkg2 == nil: + raise newException(ValueError, "Cannot find package '$#'." % pkg.name) + pkg.url = pkg2["url"].str + result.add pkg + +proc listPackages(packageFilter: string): seq[NimblePackage] = + let pkgs = listPackagesAll() + if packageFilter.len != 0: + # xxx document `packageFilter`, seems like a bad API, + # at least should be a regex; a substring match makes no sense. + result = pkgs.filterIt(packageFilter in it.name) + else: + if testamentData0.batchArg == "allowed_failures": + result = pkgs.filterIt(it.allowFailure) + elif testamentData0.testamentNumBatch == 0: + result = pkgs + else: + let pkgs2 = pkgs.filterIt(not it.allowFailure) + for i in 0.. 0: outputExpected.add runSpec.output if not runSpec.output.endsWith "\n": @@ -631,8 +664,8 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = if buf != outputExpected: writeFile(outputExceptedFile, outputExpected) - discard execShellCmd("diff -uNdr $1 $2" % [outputExceptedFile, outputGottenFile]) - echo failString & "megatest output different!" + echo diffFiles(outputGottenFile, outputExceptedFile).output + echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile] # outputGottenFile, outputExceptedFile not removed on purpose for debugging. quit 1 else: @@ -657,13 +690,9 @@ proc processCategory(r: var TResults, cat: Category, else: jsTests(r, cat, options) of "dll": - dllTests(r, cat, options) - of "flags": - flagTests(r, cat, options) + dllTests(r, cat, options & " -d:nimDebugDlOpen") of "gc": gcTests(r, cat, options) - of "longgc": - longGCTests(r, cat, options) of "debugger": debuggerTests(r, cat, options) of "manyloc": @@ -686,7 +715,9 @@ proc processCategory(r: var TResults, cat: Category, of "niminaction": testNimInAction(r, cat, options) of "ic": - icTests(r, testsDir, cat, options) + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) + of "navigator": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) of "untestable": # These require special treatment e.g. because they depend on a third party # dependency; see `trunner_special` which runs some of those. @@ -696,7 +727,9 @@ proc processCategory(r: var TResults, cat: Category, if not handled: case cat2 of "megatest": - runJoinedTest(r, cat, testsDir) + runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") else: var testsRun = 0 var files: seq[string] diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 1dd7c69cad..ac2d133c3b 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -22,136 +22,171 @@ When this is the case, a workaround is to test this package here by adding `--pa type NimblePackage* = object name*, cmd*, url*: string useHead*: bool + allowFailure*: bool + ## When true, we still run the test but the test is allowed to fail. + ## This is useful for packages that currently fail but that we still want to + ## run in CI, e.g. so that we can monitor when they start working again and + ## are reminded about those failures without making CI fail for unrelated PRs. var packages*: seq[NimblePackage] -proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true) = - packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead) +proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailure = false) = + packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure) -# pkg "alea" +pkg "alea" pkg "argparse" -when false: - pkg "arraymancer", "nim c tests/tests_cpu.nim" -# pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim" +pkg "arraymancer", "nim c tests/tests_cpu.nim" +pkg "ast_pattern_matching", "nim c -r tests/test1.nim" +pkg "asyncftpclient", "nimble compileExample" +pkg "asyncthreadpool", "nimble test --mm:refc" pkg "awk" -pkg "bigints", url = "https://github.com/Araq/nim-bigints" +pkg "bigints" pkg "binaryheap", "nim c -r binaryheap.nim" pkg "BipBuffer" -# pkg "blscurve" # pending https://github.com/status-im/nim-blscurve/issues/39 +pkg "blscurve", allowFailure = true pkg "bncurve" pkg "brainfuck", "nim c -d:release -r tests/compile.nim" -pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump" +pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" -pkg "cello" +pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true +pkg "checksums" pkg "chroma" pkg "chronicles", "nim c -o:chr -r chronicles.nim" -# when not defined(osx): # testdatagram.nim(560, 54): Check failed -# pkg "chronos", "nim c -r -d:release tests/testall" - # pending https://github.com/nim-lang/Nim/issues/17130 +pkg "chronos", "nim c -r -d:release tests/testall" pkg "cligen", "nim c --path:. -r cligen.nim" -pkg "combparser", "nimble test --gc:orc" +pkg "combparser", "nimble test --mm:orc" pkg "compactdict" -pkg "comprehension", "nimble test", "https://github.com/alehander42/comprehension" -# pkg "criterion" # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) +pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" +pkg "cowstrings" +pkg "criterion", allowFailure = true # needs testing binary +pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" pkg "delaunay" +pkg "dnsclient" pkg "docopt" +pkg "dotenv" +# when defined(linux): pkg "drchaos" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -# pkg "fidget" # pending https://github.com/treeform/fidget/issues/133 -pkg "fragments", "nim c -r fragments/dsl.nim" +pkg "faststreams" +pkg "fidget" +pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 pkg "fusion" pkg "gara" pkg "glob" pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" -# pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup" +pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true pkg "gnuplot", "nim c gnuplot.nim" -# pkg "gram", "nim c -r --gc:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" +# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" # pending https://github.com/nim-lang/Nim/issues/16509 pkg "hts", "nim c -o:htss src/hts.nim" -# pkg "httpauth" +pkg "httpauth" +pkg "httputils" pkg "illwill", "nimble examples" pkg "inim" pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" +pkg "json_rpc" +pkg "json_serialization" pkg "jstin" pkg "karax", "nim c -r tests/tester.nim" -pkg "kdtree", "nimble test", "https://github.com/jblindsay/kdtree" +pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" pkg "loopfusion" +pkg "lockfreequeues" pkg "macroutils" pkg "manu" pkg "markdown" +pkg "measuremancer", "nimble testDeps; nimble -y test" +# when unchained is version 0.3.7 or higher, use `nimble testDeps;` pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" pkg "nake", "nim c nakefile.nim" -pkg "neo", "nim c -d:blas=openblas tests/all.nim" -# pkg "nesm", "nimble tests" # notice plural 'tests' -# pkg "nico" +pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim", url = "https://github.com/nim-lang/neo" +pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true + # inactive, tests not adapted to #23096 +pkg "netty" +pkg "nico", allowFailure = true pkg "nicy", "nim c -r src/nicy.nim" pkg "nigui", "nim c -o:niguii -r src/nigui.nim" pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround needed, see D20210308T165435 pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", "nim c src/nimes.nim" pkg "nimfp", "nim c -o:nfp -r src/fp.nim" -# pkg "nimgame2", "nim c nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. +pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim" pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" +pkg "nimib" pkg "nimlsp" pkg "nimly", "nim c -r tests/test_readme_example.nim" -# pkg "nimongo", "nimble test_ci" -# pkg "nimph", "nimble test", "https://github.com/disruptek/nimph" +pkg "nimongo", "nimble test_ci", allowFailure = true +pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true +pkg "nimPNG", useHead = true pkg "nimpy", "nim c -r tests/nimfrompy.nim" pkg "nimquery" pkg "nimsl" pkg "nimsvg" -pkg "nimterop", "nimble minitest" +pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop" pkg "nimwc", "nim c nimwc.nim" -# pkg "nimx", "nim c --threads:on test/main.nim" -# pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" -pkg "norm", "nim c -r tests/sqlite/trows.nim" +pkg "nimx", "nim c test/main.nim", allowFailure = true +pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" +pkg "norm", "testament r tests/common/tmodel.nim" +pkg "normalize" pkg "npeg", "nimble testarc" -pkg "numericalnim", "nim c -r tests/test_integrate.nim" +pkg "numericalnim", "nimble nimCI" pkg "optionsutils" pkg "ormin", "nim c -o:orminn ormin.nim" pkg "parsetoml" pkg "patty" -pkg "pixie", useHead = false +pkg "pixie" pkg "plotly", "nim c examples/all.nim" pkg "pnm" pkg "polypbren" +pkg "presto" pkg "prologue", "nimble tcompile" pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim" -pkg "pylib" pkg "rbtree" pkg "react", "nimble example" pkg "regex", "nim c src/regex" -pkg "result", "nim c -r result.nim" +pkg "results", "nim c -r results.nim" pkg "RollingHash", "nim c -r tests/test_cyclichash.nim" pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" pkg "sdl1", "nim c -r src/sdl.nim" pkg "sdl2_nim", "nim c -r sdl2/sdl.nim" -pkg "sigv4", "nim c --gc:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" +pkg "serialization" +pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" +pkg "sim" +pkg "smtp", "nimble compileExample" pkg "snip", "nimble test", "https://github.com/genotrance/snip" -pkg "stint", "nim r stint.nim" +pkg "ssostrings" +pkg "stew" +pkg "stint", "nim c stint.nim" pkg "strslice" -pkg "strunicode", "nim c -r src/strunicode.nim" +pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim" +pkg "supersnappy" pkg "synthesis" +pkg "taskpools" pkg "telebot", "nim c -o:tbot -r src/telebot.nim" pkg "tempdir" pkg "templates" -pkg "tensordsl", "nim c -r tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" +pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" pkg "terminaltables", "nim c src/terminaltables.nim" pkg "termstyle", "nim c -r termstyle.nim" +pkg "testutils" pkg "timeit" pkg "timezones" pkg "tiny_sqlite" pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" +pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union" +pkg "unittest2" pkg "unpack" +pkg "weave", "nimble test_gc_arc", useHead = true +pkg "websock" pkg "websocket", "nim c websocket.nim" -# pkg "winim" +# pkg "winim", allowFailure = true pkg "with" -pkg "ws" -pkg "yaml", "nim build" -pkg "zero_functional", "nim c -r -d:nimWorkaround14447 test.nim" +pkg "ws", allowFailure = true +pkg "yaml" +pkg "zero_functional", "nim c -r test.nim" pkg "zippy" +pkg "zxcvbn" diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim index eb913a56a9..5115390e09 100644 --- a/testament/lib/stdtest/netutils.nim +++ b/testament/lib/stdtest/netutils.nim @@ -1,6 +1,7 @@ import std/[nativesockets, asyncdispatch, os] proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port = + ## See also `asynchttpserver.getPort`. block: var name: Sockaddr_in name.sin_family = typeof(name.sin_family)(toInt(AF_INET)) @@ -8,5 +9,5 @@ proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port = name.sin_addr.s_addr = htonl(INADDR_ANY) if bindAddr(handle, cast[ptr SockAddr](addr(name)), sizeof(name).Socklen) < 0'i32: - raiseOSError(osLastError()) + raiseOSError(osLastError(), $port) result = getLocalAddr(handle, AF_INET)[1] diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim index 53b94fdbb0..e214d113df 100644 --- a/testament/lib/stdtest/specialpaths.nim +++ b/testament/lib/stdtest/specialpaths.nim @@ -13,6 +13,8 @@ import compiler/nimpaths ]# import os +when defined(nimPreviewSlimSystem): + import std/assertions # Note: all the const paths defined here are known at compile time and valid # so long Nim repo isn't relocated after compilation. @@ -46,7 +48,7 @@ proc splitTestFile*(file: string): tuple[cat: string, path: string] = else: result.path = file return result - doAssert false, "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" + raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" static: # sanity check diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 36f9512722..66afd62af5 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -1,6 +1,9 @@ import std/private/miscdollars -import std/strutils -from std/os import getEnv +when defined(nimscript): + import std/os # xxx investigate why needed +else: + from std/os import getEnv +import std/[macros, genasts] template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = ## API to deal with flaky or failing tests. This avoids disabling entire tests @@ -26,15 +29,35 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = msg2.add $expr & " " & msg echo msg2 -proc greedyOrderedSubsetLines*(lhs, rhs: string): bool = - ## returns true if each stripped line in `lhs` appears in rhs, using a greedy matching. - let rhs = rhs.strip - var currentPos = 0 - for line in lhs.strip.splitLines: - currentPos = rhs.find(line.strip, currentPos) - if currentPos < 0: - return false - return true +when not defined(js) and not defined(nimscript): + import std/strutils + + proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool = + ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching. + # xxx improve error reporting by showing the last matched pair + iterator splitLinesClosure(): string {.closure.} = + for line in splitLines(rhs.strip): + yield line + template isMatch(lhsi, rhsi): bool = + if allowPrefixMatch: + startsWith(rhsi, lhsi) + else: + lhsi == rhsi + + var rhsIter = splitLinesClosure + var currentLine = strip(rhsIter()) + + for line in lhs.strip.splitLines: + let line = line.strip + if line.len != 0: + while not isMatch(line, currentLine): + currentLine = strip(rhsIter()) + if rhsIter.finished: + return false + + if rhsIter.finished: + return false + return true template enableRemoteNetworking*: bool = ## Allows contolling whether to run some test at a statement-level granularity. @@ -66,3 +89,33 @@ template whenVMorJs*(bodyIf, bodyElse) = else: when defined(js): bodyIf else: bodyElse + +template accept*(a) = + doAssert compiles(a) + +template reject*(a) = + doAssert not compiles(a) + +template disableVm*(body) = + when nimvm: discard + else: body + +macro assertAll*(body) = + ## works in VM, unlike `check`, `require` + runnableExamples: + assertAll: + 1+1 == 2 + var a = @[1, 2] # statements work + a.len == 2 + # remove this once these support VM, pending #10129 (closed but not yet fixed) + result = newStmtList() + for a in body: + result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do: + # D20210421T014713:here + # xxx pending https://github.com/nim-lang/Nim/issues/12030, + # `typeof` should introduce its own scope, so that this + # is sufficient: `typeof(a)` instead of `typeof(block: a)` + when typeof(block: a) is void: a + else: + if not a: + raise newException(AssertionDefect, info & " " & a2) diff --git a/testament/lib/stdtest/unittest_light.nim b/testament/lib/stdtest/unittest_light.nim index 273bf72f57..4ab1d75435 100644 --- a/testament/lib/stdtest/unittest_light.nim +++ b/testament/lib/stdtest/unittest_light.nim @@ -1,3 +1,6 @@ +import std/assertions + + proc mismatch*[T](lhs: T, rhs: T): string = ## Simplified version of `unittest.require` that satisfies a common use case, ## while avoiding pulling too many dependencies. On failure, diagnostic diff --git a/testament/specs.nim b/testament/specs.nim index 6b80fe41d3..c3040c1d8f 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -9,6 +9,7 @@ import sequtils, parseutils, strutils, os, streams, parsecfg, tables, hashes, sets +import compiler/platform type TestamentData* = ref object # better to group globals under 1 object; could group the other ones here too @@ -74,13 +75,12 @@ type # xxx make sure `isJoinableSpec` takes into account each field here. action*: TTestAction file*, cmd*: string + filename*: string ## Test filename (without path). input*: string outputCheck*: TOutputCheck sortoutput*: bool output*: string line*, column*: int - tfile*: string - tline*, tcolumn*: int exitCode*: int msg*: string ccodeCheck*: seq[string] @@ -90,6 +90,7 @@ type targets*: set[TTarget] matrix*: seq[string] nimout*: string + nimoutFull*: bool # whether nimout is all compiler output or a subset parseErrors*: string # when the spec definition is invalid, this is not empty. unjoinable*: bool unbatchable*: bool @@ -100,10 +101,11 @@ type timeout*: float # in seconds, fractions possible, # but don't rely on much precision inlineErrors*: seq[InlineError] # line information to error message + debugInfo*: string # debug info to give more context proc getCmd*(s: TSpec): string = if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --clearNimblePath --nimblePath:build/deps/pkgs $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file" else: result = s.cmd @@ -126,19 +128,55 @@ when not declared(parseCfgBool): of "n", "no", "false", "0", "off": result = false else: raise newException(ValueError, "cannot interpret as a bool: " & s) +proc addLine*(self: var string; pieces: varargs[string]) = + for piece in pieces: + self.add piece + self.add "\n" + + const - inlineErrorMarker = "#[tt." + inlineErrorKindMarker = "tt." + inlineErrorMarker = "#[" & inlineErrorKindMarker proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int = + ## Extract inline error messages. + ## + ## Can parse a single message for a line: + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name] ]# + ## ``` + ## + ## Can parse multiple messages for a line when they are separated by ';': + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error + ## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error + ## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; + ## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name]; + ## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` result = i + len(inlineErrorMarker) inc col, len(inlineErrorMarker) + let msgLine = line + var msgCol = -1 + var msg = "" var kind = "" - while result < s.len and s[result] in IdentChars: - kind.add s[result] - inc result - inc col - var caret = (line, -1) + template parseKind = + while result < s.len and s[result] in IdentChars: + kind.add s[result] + inc result + inc col + if kind notin ["Hint", "Warning", "Error"]: + spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error" template skipWhitespace = while result < s.len and s[result] in Whitespace: @@ -149,37 +187,77 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T inc col inc result - skipWhitespace() - if result < s.len and s[result] == '^': - caret = (line-1, col) - inc result - inc col - skipWhitespace() + template parseCaret = + if result < s.len and s[result] == '^': + msgCol = col + inc result + inc col + skipWhitespace() + else: + spec.parseErrors.addLine "expected column marker ('^') for inline message" + + template isMsgDelimiter: bool = + s[result] == ';' and + (block: + let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1) + if s.len > nextTokenIdx + len(inlineErrorKindMarker) and + s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker: + true + else: + false) + + template trimTrailingMsgWhitespace = + while msg.len > 0 and msg[^1] in Whitespace: + setLen msg, msg.len - 1 + + template addInlineError = + doAssert msg[^1] notin Whitespace + if kind == "Error": spec.action = actionReject + spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol) + + parseKind() + skipWhitespace() + parseCaret() - var msg = "" while result < s.len-1: if s[result] == '\n': + if result > 0 and s[result - 1] == '\r': + msg[^1] = '\n' + else: + msg.add '\n' inc result inc line col = 1 - elif s[result] == ']' and s[result+1] == '#': - while msg.len > 0 and msg[^1] in Whitespace: - setLen msg, msg.len - 1 - + elif isMsgDelimiter(): + trimTrailingMsgWhitespace() inc result + skipWhitespace() + addInlineError() + inc result, len(inlineErrorKindMarker) + inc col, 1 + len(inlineErrorKindMarker) + kind.setLen 0 + msg.setLen 0 + parseKind() + skipWhitespace() + parseCaret() + elif s[result] == ']' and s[result+1] == '#': + trimTrailingMsgWhitespace() + inc result, 2 inc col, 2 - if kind == "Error": spec.action = actionReject - spec.unjoinable = true - spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: caret[0], col: caret[1]) + addInlineError() break else: msg.add s[result] inc result inc col + if spec.inlineErrors.len > 0: + spec.unjoinable = true + proc extractSpec(filename: string; spec: var TSpec): string = const tripleQuote = "\"\"\"" + specStart = "discard " & tripleQuote var s = readFile(filename) var i = 0 @@ -188,25 +266,34 @@ proc extractSpec(filename: string; spec: var TSpec): string = var line = 1 var col = 1 while i < s.len: - if s.continuesWith(tripleQuote, i): - if a < 0: a = i - elif b < 0: b = i - inc i, 2 - inc col + if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i): + # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark) + const lineMax = 10 + if a != -1: + raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line)) + elif line > lineMax: + # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file + raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)]) + i += specStart.len + a = i + elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i): + b = i + i += tripleQuote.len elif s[i] == '\n': inc line + inc i col = 1 elif s.continuesWith(inlineErrorMarker, i): i = extractErrorMsg(s, i, line, col, spec) else: inc col - inc i + inc i - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = s.substr(a+3, b-1).replace("'''", tripleQuote) + if a >= 0 and b > a: + result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"}) + elif a >= 0: + raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line)) else: - #echo "warning: file does not contain spec: " & filename result = "" proc parseTargets*(value: string): set[TTarget] = @@ -218,15 +305,6 @@ proc parseTargets*(value: string): set[TTarget] = of "js": result.incl(targetJS) else: raise newException(ValueError, "invalid target: '$#'" % v) -proc addLine*(self: var string; a: string) = - self.add a - self.add "\n" - -proc addLine*(self: var string; a, b: string) = - self.add a - self.add b - self.add "\n" - proc initSpec*(filename: string): TSpec = result.file = filename @@ -238,11 +316,13 @@ proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool = proc parseSpec*(filename: string): TSpec = result.file = filename + result.filename = extractFilename(filename) let specStr = extractSpec(filename, result) var ss = newStringStream(specStr) var p: CfgParser open(p, ss, filename, 1) var flags: HashSet[string] + var nimoutFound = false while true: var e = next(p) case e.kind @@ -277,12 +357,6 @@ proc parseSpec*(filename: string): TSpec = if result.msg.len == 0 and result.nimout.len == 0: result.parseErrors.addLine "errormsg or msg needs to be specified before column" discard parseInt(e.value, result.column) - of "tfile": - result.tfile = e.value - of "tline": - discard parseInt(e.value, result.tline) - of "tcolumn": - discard parseInt(e.value, result.tcolumn) of "output": if result.outputCheck != ocSubstr: result.outputCheck = ocEqual @@ -305,6 +379,9 @@ proc parseSpec*(filename: string): TSpec = result.action = actionReject of "nimout": result.nimout = e.value + nimoutFound = true + of "nimoutfull": + result.nimoutFull = parseCfgBool(e.value) of "batchable": result.unbatchable = not parseCfgBool(e.value) of "joinable": @@ -321,42 +398,71 @@ proc parseSpec*(filename: string): TSpec = # Valgrind only supports OSX <= 17.x result.useValgrind = disabled of "disabled": - case e.value.normalize + let value = e.value.normalize + case value of "y", "yes", "true", "1", "on": result.err = reDisabled of "n", "no", "false", "0", "off": discard - of "win", "windows": + # These values are defined in `compiler/options.isDefined` + of "win": when defined(windows): result.err = reDisabled of "linux": when defined(linux): result.err = reDisabled of "bsd": when defined(bsd): result.err = reDisabled - of "osx", "macosx": # xxx remove `macosx` alias? + of "osx": when defined(osx): result.err = reDisabled - of "unix": - when defined(unix): result.err = reDisabled - of "posix": + of "unix", "posix": when defined(posix): result.err = reDisabled - of "travis": + of "freebsd": + when defined(freebsd): result.err = reDisabled + of "littleendian": + when defined(littleendian): result.err = reDisabled + of "bigendian": + when defined(bigendian): result.err = reDisabled + of "cpu8", "8bit": + when defined(cpu8): result.err = reDisabled + of "cpu16", "16bit": + when defined(cpu16): result.err = reDisabled + of "cpu32", "32bit": + when defined(cpu32): result.err = reDisabled + of "cpu64", "64bit": + when defined(cpu64): result.err = reDisabled + # These values are for CI environments + of "travis": # deprecated if isTravis: result.err = reDisabled - of "appveyor": + of "appveyor": # deprecated if isAppVeyor: result.err = reDisabled of "azure": if isAzure: result.err = reDisabled - of "32bit": - if sizeof(int) == 4: - result.err = reDisabled - of "freebsd": - when defined(freebsd): result.err = reDisabled - of "arm64": - when defined(arm64): result.err = reDisabled - of "i386": - when defined(i386): result.err = reDisabled - of "openbsd": - when defined(openbsd): result.err = reDisabled - of "netbsd": - when defined(netbsd): result.err = reDisabled else: - result.parseErrors.addLine "cannot interpret as a bool: ", e.value + # Check whether the value exists as an OS or CPU that is + # defined in `compiler/platform`. + block checkHost: + for os in platform.OS: + # Check if the value exists as OS. + if value == os.name.normalize: + # The value exists; is it the same as the current host? + if value == hostOS.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + for cpu in platform.CPU: + # Check if the value exists as CPU. + if value == cpu.name.normalize: + # The value exists; is it the same as the current host? + if value == hostCPU.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + # The value doesn't exist as an OS, CPU, or any previous value + # defined in this case statement, so raise an error. + result.parseErrors.addLine "cannot interpret as a bool: ", e.value of "cmd": if e.value.startsWith("nim "): result.cmd = compilerPrefix & e.value[3..^1] @@ -395,6 +501,21 @@ proc parseSpec*(filename: string): TSpec = if skips.anyIt(it in result.file): result.err = reDisabled + if nimoutFound and result.nimout.len == 0 and not result.nimoutFull: + result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional" + result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable if not result.inCurrentBatch: result.err = reDisabled + + # Interpolate variables in msgs: + template varSub(msg: string): string = + try: + msg % ["/", $DirSep, "file", result.filename] + except ValueError: + result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')" + msg + result.nimout = result.nimout.varSub + result.msg = result.msg.varSub + for inlineError in result.inlineErrors.mitems: + inlineError.msg = inlineError.msg.varSub diff --git a/testament/testament.nim b/testament/testament.nim index f45d4043af..1e892e6367 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -10,17 +10,35 @@ ## This program verifies Nim against the testcases. import - strutils, pegs, os, osproc, streams, json, std/exitprocs, - backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, times, md5, azure, intsets, macros + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] + +import backend, specs, azure, htmlgen + from std/sugar import dup import compiler/nodejs import lib/stdtest/testutils from lib/stdtest/specialpaths import splitTestFile +from std/private/gitutils import diffStrings + +import ../dist/checksums/src/checksums/md5 + +proc trimUnitSep(x: var string) = + let L = x.len + if L > 0 and x[^1] == '\31': + setLen x, L-1 var useColors = true var backendLogging = true var simulate = false +var optVerbose = false +var useMegatest = true +var valgrindEnabled = true + +proc verboseCmd(cmd: string) = + if optVerbose: + echo "executing: ", cmd const failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs @@ -31,22 +49,24 @@ const Command: p|pat|pattern run all the tests matching the given pattern - all run all tests + all run all tests in category folders c|cat|category run all the tests of a certain category r|run run single test file html generate $1 from the database Arguments: arguments are passed to the compiler Options: - --print also print results to the console + --print print results to the console + --verbose print commands (compiling and running tests) --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests - --targets:"c cpp js objc" run tests for specified targets (default: all) + --targets:"c cpp js objc" run tests for specified targets (default: c) --nim:path use a particular nim executable (default: $$PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API @@ -66,13 +86,14 @@ proc isNimRepoTests(): bool = type Category = distinct string TResults = object - total, passed, skipped: int + total, passed, failedButAllowed, skipped: int + ## xxx rename passed to passedOrAllowedFailure data: string TTest = object name: string cat: Category options: string - args: seq[string] + testArgs: seq[string] spec: TSpec startTime: float debugInfo: string @@ -82,12 +103,6 @@ type let pegLineError = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" - - pegLineTemplate = - peg""" - {[^(]*} '(' {\d+} ', ' {\d+} ') ' - 'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .* - """ pegOtherError = peg"'Error:' \s* {.*}" pegOfInterest = pegLineError / pegOtherError @@ -98,7 +113,7 @@ proc isSuccess(input: string): bool = # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we # could handle colors, there would be other issues such as handling other flags - # that may appear in user config (eg: `--listFullPaths`). + # that may appear in user config (eg: `--filenames`). # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory input.startsWith("Hint: ") and input.endsWith("[SuccessX]") @@ -118,6 +133,7 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str for arg in args: result.cmdLine.add ' ' result.cmdLine.add quoteShell(arg) + verboseCmd(result.cmdLine) var p = startProcess(command, workingDir = workingDir, args = args, options = {poStdErrToStdOut, poUsePath}) var outp = outputStream(p) @@ -145,56 +161,64 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = let hashInput = options & $target result = "nimcache" / (filename & '_' & hashInput.getMD5) -proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): seq[string] = +proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): string = var options = target.defaultOptions & ' ' & options - # improve pending https://github.com/nim-lang/Nim/issues/14343 - if nimcache.len > 0: options.add ' ' & ("--nimCache:" & nimcache).quoteShell + if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) options.add ' ' & extraOptions - result = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], + # we avoid using `parseCmdLine` which is buggy, refs bug #14343 + result = cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir(), "nim", compilerPrefix]) + "filedir", filename.getFileDir(), "nim", compilerPrefix] -proc callCompiler(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target, +proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): TSpec = + result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target, extraOptions) - result.cmd = quoteShellCommand(c) - var p = startProcess(command = c[0], args = c[1 .. ^1], - options = {poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmd) + var p = startProcess(command = result.cmd, + options = {poStdErrToStdOut, poUsePath, poEvalCommand}) let outp = p.outputStream - var suc = "" + var foundSuccessMsg = false + var foundErrorMsg = false var err = "" - var tmpl = "" var x = newStringOfCap(120) result.nimout = "" while true: if outp.readLine(x): + trimUnitSep x result.nimout.add(x & '\n') if x =~ pegOfInterest: - # `err` should contain the last error/warning message + # `err` should contain the last error message err = x - elif x =~ pegLineTemplate and err == "": - # `tmpl` contains the last template expansion before the error - tmpl = x + foundErrorMsg = true elif x.isSuccess: - suc = x + foundSuccessMsg = true elif not running(p): break - close(p) result.msg = "" result.file = "" result.output = "" result.line = 0 result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 + result.err = reNimcCrash - if tmpl =~ pegLineTemplate: - result.tfile = extractFilename(matches[0]) - result.tline = parseInt(matches[1]) - result.tcolumn = parseInt(matches[2]) + result.exitCode = p.peekExitCode + close p + case result.exitCode + of 0: + if foundErrorMsg: + result.debugInfo.add " compiler exit code was 0 but some Error's were found." + else: + result.err = reSuccess + of 1: + if not foundErrorMsg: + result.debugInfo.add " compiler exit code was 1 but no Error's were found." + if foundSuccessMsg: + result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." + else: + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode + if err =~ pegLineError: result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) @@ -202,33 +226,12 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, result.msg = matches[3] elif err =~ pegOtherError: result.msg = matches[0] - elif suc.isSuccess: - result.err = reSuccess - -proc callCCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache = "", target) - var p = startProcess(command = "gcc", args = c[5 .. ^1], - options = {poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var x = newStringOfCap(120) - result.nimout = "" - result.msg = "" - result.file = "" - result.output = "" - result.line = -1 - while true: - if outp.readLine(x): - result.nimout.add(x & '\n') - elif not running(p): - break - close(p) - if p.peekExitCode == 0: - result.err = reSuccess + trimUnitSep result.msg proc initResults: TResults = result.total = 0 result.passed = 0 + result.failedButAllowed = 0 result.skipped = 0 result.data = "" @@ -256,18 +259,29 @@ template maybeStyledEcho(args: varargs[untyped]): untyped = proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3
      \n" & - "Tests skipped: $2 / $3
      \n") % - [$x.passed, $x.skipped, $x.total] + result = """ +Tests passed or allowed to fail: $2 / $1
      +Tests failed and allowed to fail: $3 / $1
      +Tests skipped: $4 / $1
      +""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] -proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum) = - # test.name is easier to find than test.name.extractFilename - # A bit hacky but simple and works with tests/testament/tshould_not_work.nim +proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string = var name = test.name.replace(DirSep, '/') name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions + name.strip() +proc addResult(r: var TResults, test: TTest, target: TTarget, + extraOptions, expected, given: string, successOrig: TResultEnum, + allowFailure = false, givenSpec: ptr TSpec = nil) = + # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need + # instead of having to pass individual fields, or abusing existing ones like expected vs given. + # test.name is easier to find than test.name.extractFilename + # A bit hacky but simple and works with tests/testament/tshould_not_work.nim + let name = testName(test, target, extraOptions, allowFailure) let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout else: successOrig @@ -282,18 +296,22 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, expected = expected, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) + template dispNonSkipped(color, outcome) = + maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" template disp(msg) = maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name if success == reSuccess: - maybeStyledEcho fgGreen, "PASS: ", fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" + dispNonSkipped(fgGreen, "PASS: ") elif success == reDisabled: if test.spec.inCurrentBatch: disp("SKIP:") else: disp("NOTINBATCH:") elif success == reJoined: disp("JOINED:") else: - maybeStyledEcho styleBright, fgRed, failString, fgCyan, name + dispNonSkipped(fgRed, failString) maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" maybeStyledEcho styleBright, fgRed, "Failure: ", $success + if givenSpec != nil and givenSpec.debugInfo.len > 0: + echo "debugInfo: " & givenSpec.debugInfo if success in {reBuildFailed, reNimcCrash, reInstallFailed}: # expected is empty, no reason to print it. echo given @@ -302,7 +320,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, maybeStyledEcho styleBright, expected, "\n" maybeStyledEcho fgYellow, "Gotten:" maybeStyledEcho styleBright, given, "\n" - + echo diffStrings(expected, given).output if backendLogging and (isAppVeyor or isAzure): let (outcome, msg) = @@ -327,71 +345,51 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, discard waitForExit(p) close(p) -proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}" - var covered = initIntSet() - for line in splitLines(given.nimout): +proc toString(inlineError: InlineError, filename: string): string = + result.add "$file($line, $col) $kind: $msg" % [ + "file", filename, + "line", $inlineError.line, + "col", $inlineError.col, + "kind", $inlineError.kind, + "msg", $inlineError.msg + ] - if line =~ pegLine: - let file = extractFilename(matches[0]) - let line = try: parseInt(matches[1]) except: -1 - let col = try: parseInt(matches[2]) except: -1 - let kind = matches[3] - let msg = matches[4] +proc inlineErrorsMsgs(expected: TSpec): string = + for inlineError in expected.inlineErrors.items: + result.addLine inlineError.toString(expected.filename) - if file == extractFilename test.name: - var i = 0 - for x in expected.inlineErrors: - if x.line == line and (x.col == col or x.col < 0) and - x.kind == kind and x.msg in msg: - covered.incl i - inc i +proc checkForInlineErrors(expected, given: TSpec): bool = + for inlineError in expected.inlineErrors: + if inlineError.toString(expected.filename) notin given.nimout: + return false + true - block coverCheck: - for j in 0..high(expected.inlineErrors): - if j notin covered: - var e = test.name - e.add '(' - e.addInt expected.inlineErrors[j].line - if expected.inlineErrors[j].col > 0: - e.add ", " - e.addInt expected.inlineErrors[j].col - e.add ") " - e.add expected.inlineErrors[j].kind - e.add ": " - e.add expected.inlineErrors[j].msg - - r.addResult(test, target, e, given.nimout, reMsgsDiffer) - break coverCheck - - r.addResult(test, target, "", given.msg, reSuccess) - inc(r.passed) - -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - if expected.inlineErrors.len > 0: - checkForInlineErrors(r, expected, given, test, target) - elif strip(expected.msg) notin strip(given.msg): - r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) +proc nimoutCheck(expected, given: TSpec): bool = + result = true + if expected.nimoutFull: + if expected.nimout != given.nimout: + result = false elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): - r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) - elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and + result = false + +proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = + if not checkForInlineErrors(expected, given) or + (not expected.nimoutFull and not nimoutCheck(expected, given)): + r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer) + elif strip(expected.msg) notin strip(given.msg): + r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer) + elif not nimoutCheck(expected, given): + r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) + elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, expected.file, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: - r.addResult(test, target, $expected.line & ':' & $expected.column, - $given.line & ':' & $given.column, - reLinesDiffer) - elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and - "internal error:" notin expected.msg: - r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer) - elif expected.tline != given.tline and expected.tline != 0 or - expected.tcolumn != given.tcolumn and expected.tcolumn != 0: - r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn, - $given.tline & ':' & $given.tcolumn, - reLinesDiffer) + r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, + $given.line & ':' & $given.column, reLinesDiffer) else: - r.addResult(test, target, expected.msg, given.msg, reSuccess) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess) inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = @@ -429,26 +427,23 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given.err = reCodeNotFound echo getCurrentExceptionMsg() -proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - if not greedyOrderedSubsetLines(expectedNimout, given.nimout): - given.err = reMsgsDiffer - -proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, - expected: TSpec; r: var TResults) = +proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string, + given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" var givenmsg: string = "" if given.err == reSuccess: if expected.needsCodegenCheck: codegenCheck(test, target, expected, expectedmsg, given) givenmsg = given.msg - if expected.nimout.len > 0: - expectedmsg = expected.nimout + if not nimoutCheck(expected, given) or + not checkForInlineErrors(expected, given): + given.err = reMsgsDiffer + expectedmsg = expected.nimout & inlineErrorsMsgs(expected) givenmsg = given.nimout.strip - nimoutCheck(test, expectedmsg, given) else: givenmsg = "$ " & given.cmd & '\n' & given.nimout if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) + r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err) proc getTestSpecTarget(): TTarget = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": @@ -456,16 +451,6 @@ proc getTestSpecTarget(): TTarget = else: result = targetC -proc checkDisabled(r: var TResults, test: TTest): bool = - if test.spec.err in {reDisabled, reJoined}: - # targetC is a lie, but parameter is required - r.addResult(test, targetC, "", "", test.spec.err) - inc(r.skipped) - inc(r.total) - result = false - else: - result = true - var count = 0 proc equalModuloLastNewline(a, b: string): bool = @@ -473,45 +458,48 @@ proc equalModuloLastNewline(a, b: string): bool = result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, - target: TTarget, nimcache: string, extraOptions = "") = + target: TTarget, extraOptions: string, nimcache: string) = test.startTime = epochTime() + if testName(test, target, extraOptions, false) in skips: + test.spec.err = reDisabled + + if test.spec.err in {reDisabled, reJoined}: + r.addResult(test, target, extraOptions, "", "", test.spec.err) + inc(r.skipped) + return + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) case expected.action of actionCompile: - var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target, - extraOptions = " --stdout --hint[Path]:off --hint[Processing]:off") - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionRun: - var given = callCompiler(expected.getCmd, test.name, test.options, - nimcache, target, extraOptions) if given.err != reSuccess: - r.addResult(test, target, "", "$ " & given.cmd & '\n' & given.nimout, given.err) + r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) else: let isJsTarget = target == targetJS var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) if not fileExists(exeFile): - r.addResult(test, target, expected.output, + r.addResult(test, target, extraOptions, expected.output, "executable not found: " & exeFile, reExeNotFound) else: let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.output, "nodejs binary not in PATH", + r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH", reExeNotFound) else: var exeCmd: string - var args = test.args + var args = test.testArgs if isJsTarget: exeCmd = nodejs # see D20210217T215950 args = @["--unhandled-rejections=strict", exeFile] & args else: exeCmd = exeFile.dup(normalizeExe) - if expected.useValgrind != disabled: + if valgrindEnabled and expected.useValgrind != disabled: var valgrindOptions = @["--error-exitcode=1"] if expected.useValgrind != leaking: valgrindOptions.add "--leak-check=yes" args = valgrindOptions & exeCmd & args exeCmd = "valgrind" - # xxx honor `testament --verbose` here var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input) # Treat all failure codes from nodejs as 1. Older versions of nodejs used # to return other codes, but for us it is sufficient to know that it's not 0. @@ -526,25 +514,45 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, else: buf if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, + given.err = reExitcodesDiffer + r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer - r.addResult(test, target, expected.output, bufB, reOutputsDiffer) - else: - compilerOutputTests(test, target, given, expected, r) + r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: - var given = callCompiler(expected.getCmd, test.name, test.options, - nimcache, target) - cmpMsgs(r, expected, given, test, target) + # Make sure its the compiler rejecting and not the system (e.g. segfault) + cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) -proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = "") = + + +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard + +proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: inc(r.total) if target notin gTargets: - r.addResult(test, target, "", "", reDisabled) + r.addResult(test, target, extraOptions, "", "", reDisabled) inc(r.skipped) elif simulate: inc count @@ -552,16 +560,16 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = else: let nimcache = nimcacheDir(test.name, test.options, target) var testClone = test - testSpecHelper(r, testClone, expected, target, nimcache, extraOptions) + let target = changeTarget(extraOptions, target) + testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = var expected = test.spec if expected.parseErrors.len > 0: # targetC is a lie, but a parameter is required - r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) + r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec) inc(r.total) return - if not checkDisabled(r, test): return expected.targets.incl targets # still no target specified at all @@ -571,46 +579,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = for m in test.spec.matrix: targetHelper(r, test, expected, m) else: - targetHelper(r, test, expected) + targetHelper(r, test, expected, "") proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = - if not checkDisabled(r, test): return for target in test.spec.targets: inc(r.total) var testClone = test - testSpecHelper(r, testClone, test.spec, target, nimcache) - -proc testC(r: var TResults, test: TTest, action: TTestAction) = - # runs C code. Doesn't support any specs, just goes by exit code. - if not checkDisabled(r, test): return - - let tname = test.name.addFileExt(".c") - inc(r.total) - maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC) - if given.err != reSuccess: - r.addResult(test, targetC, "", given.msg, given.err) - elif action == actionRun: - let exeFile = changeFileExt(test.name, ExeExt) - var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) - if exitCode != 0: given.err = reExitcodesDiffer - if given.err == reSuccess: inc(r.passed) - -proc testExec(r: var TResults, test: TTest) = - # runs executable or script, just goes by exit code - if not checkDisabled(r, test): return - - inc(r.total) - let (outp, errC) = execCmdEx(test.options.strip()) - var given: TSpec - if errC == 0: - given.err = reSuccess - else: - given.err = reExitcodesDiffer - given.msg = outp - - if given.err == reSuccess: inc(r.passed) - r.addResult(test, targetC, "", given.msg, given.err) + testSpecHelper(r, testClone, test.spec, target, "", nimcache) proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat @@ -624,14 +599,12 @@ proc makeRawTest(test, options: string, cat: Category): TTest {.used.} = result.name = test result.options = options result.spec = initSpec(addFileExt(test, ".nim")) - result.startTime = epochTime() result.spec.action = actionCompile result.spec.targets = {getTestSpecTarget()} + result.startTime = epochTime() # TODO: fix these files const disabledFilesDefault = @[ - "LockFreeHash.nim", - "sharedstrings.nim", "tableimpl.nim", "setimpl.nim", "hashcommon.nim", @@ -677,13 +650,13 @@ proc main() = var targetsStr = "" var isMainProcess = true var skipFrom = "" - var useMegatest = true var p = initOptParser() p.next() while p.kind in {cmdLongOption, cmdShortOption}: case p.key.normalize - of "print", "verbose": optPrintResults = true + of "print": optPrintResults = true + of "verbose": optVerbose = true of "failing": optFailing = true of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731 of "targets": @@ -704,7 +677,7 @@ proc main() = quit Usage of "batch": testamentData0.batchArg = p.val - if p.val != "_": + if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}: let s = p.val.split("_") doAssert s.len == 2, $(p.val, s) testamentData0.testamentBatch = s[0].parseInt @@ -721,6 +694,14 @@ proc main() = useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": case p.val: of "on": diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim index a8d216a5b2..477da1e230 100644 --- a/testament/tests/shouldfail/tccodecheck.nim +++ b/testament/tests/shouldfail/tccodecheck.nim @@ -1,5 +1,5 @@ discard """ -ccodecheck: "baz" + ccodecheck: "baz" """ proc foo(): void {.exportc: "bar".}= diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim index 89482e6739..809ddec74f 100644 --- a/testament/tests/shouldfail/tcolumn.nim +++ b/testament/tests/shouldfail/tcolumn.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "undeclared identifier: 'undeclared'" -line: 8 -column: 7 + errormsg: "undeclared identifier: 'undeclared'" + line: 9 + column: 7 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim index dbbdf5021c..6c130d107b 100644 --- a/testament/tests/shouldfail/terrormsg.nim +++ b/testament/tests/shouldfail/terrormsg.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "wrong error message" -line: 8 -column: 6 + errormsg: "wrong error message" + line: 9 + column: 6 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim index 1b38b4f2e2..605f046dba 100644 --- a/testament/tests/shouldfail/texitcode1.nim +++ b/testament/tests/shouldfail/texitcode1.nim @@ -1,3 +1,3 @@ discard """ -exitcode: 1 + exitcode: 1 """ diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim index 20d4bd1f34..b40a4f44f4 100644 --- a/testament/tests/shouldfail/tfile.nim +++ b/testament/tests/shouldfail/tfile.nim @@ -1,6 +1,6 @@ discard """ -errormsg: "undeclared identifier: 'undefined'" -file: "notthisfile.nim" + errormsg: "undeclared identifier: 'undefined'" + file: "notthisfile.nim" """ echo undefined diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim index f7a09875c5..fe782eb03d 100644 --- a/testament/tests/shouldfail/tline.nim +++ b/testament/tests/shouldfail/tline.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "undeclared identifier: 'undeclared'" -line: 9 -column: 6 + errormsg: "undeclared identifier: 'undeclared'" + line: 10 + column: 6 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim index 9879e41814..92022ee975 100644 --- a/testament/tests/shouldfail/tmaxcodesize.nim +++ b/testament/tests/shouldfail/tmaxcodesize.nim @@ -1,5 +1,5 @@ discard """ -maxcodesize: 1 + maxcodesize: 1 """ echo "Hello World" diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim index c0e3320531..0a65bfb701 100644 --- a/testament/tests/shouldfail/tnimout.nim +++ b/testament/tests/shouldfail/tnimout.nim @@ -1,6 +1,6 @@ discard """ -nimout: "Hello World!" -action: compile + nimout: "Hello World!" + action: compile """ static: diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim new file mode 100644 index 0000000000..4fc93f6d29 --- /dev/null +++ b/testament/tests/shouldfail/tnimoutfull.nim @@ -0,0 +1,14 @@ +discard """ + nimout: ''' +msg1 +msg2 +''' + action: compile + nimoutFull: true +""" + +# should fail because `msg3` is not in nimout and `nimoutFill: true` was given +static: + echo "msg1" + echo "msg2" + echo "msg3" diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim index ac0bc7a46b..eaf9e86521 100644 --- a/testament/tests/shouldfail/toutput.nim +++ b/testament/tests/shouldfail/toutput.nim @@ -1,7 +1,7 @@ discard """ -output: ''' -done -''' + output: ''' + done + ''' """ echo "broken" diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim index 7cc51ee8d6..47324ecee8 100644 --- a/testament/tests/shouldfail/toutputsub.nim +++ b/testament/tests/shouldfail/toutputsub.nim @@ -1,5 +1,5 @@ discard """ -outputsub: "something else" + outputsub: "something else" """ echo "Hello World!" diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim index aaf2b4a63c..1e7258f708 100644 --- a/testament/tests/shouldfail/treject.nim +++ b/testament/tests/shouldfail/treject.nim @@ -1,5 +1,5 @@ discard """ -action: "reject" + action: "reject" """ # Because we set action="reject", we expect this line not to compile. But the diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim index 4ce9ce26d8..69dfbc0a04 100644 --- a/testament/tests/shouldfail/tsortoutput.nim +++ b/testament/tests/shouldfail/tsortoutput.nim @@ -1,6 +1,6 @@ discard """ -sortoutput: true -output: ''' + sortoutput: true + output: ''' 2 1 ''' diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim index 4f699fd3b9..d551ff12ea 100644 --- a/testament/tests/shouldfail/tvalgrind.nim +++ b/testament/tests/shouldfail/tvalgrind.nim @@ -1,6 +1,6 @@ discard """ -valgrind: true -cmd: "nim $target --gc:arc -d:useMalloc $options $file" + valgrind: true + cmd: "nim $target --gc:arc -d:useMalloc $options $file" """ # this is the same check used by testament/specs.nim whether or not valgrind diff --git a/tests/alias/t19349.nim b/tests/alias/t19349.nim new file mode 100644 index 0000000000..1e1e582642 --- /dev/null +++ b/tests/alias/t19349.nim @@ -0,0 +1,19 @@ +discard """ + action: "compile" +""" + +type + Vec3[T: SomeNumber] = object + arr: array[3, T] + +var + cfloatArr: array[3, array[3, cfloat]] + cfloatSeq = newSeq[Vec3[cfloat]]() +for row in cfloatArr: + cfloatSeq.add(Vec3[float32](arr: [row[0], row[1], row[2]])) + +var + cuintArr: array[3, array[3, cuint]] + cuintSeq = newSeq[Vec3[cuint]]() +for row in cuintArr: + cuintSeq.add(Vec3[uint32](arr: [row[0], row[1], row[2]])) diff --git a/tests/align/talign.nim b/tests/align/talign.nim index 3b8f6b4dfb..08373ee497 100644 --- a/tests/align/talign.nim +++ b/tests/align/talign.nim @@ -51,3 +51,19 @@ type Bug[T] = object var bug: Bug[int] doAssert sizeof(bug) == 128, "Oops my size is " & $sizeof(bug) # 16 + + +block: # bug #22419 + type + ValidatorPubKey = object + blob: array[96, byte] + + proc f(): auto = + return iterator() = + var pad: int8 = 0 + var y {.align: 16.}: ValidatorPubKey + let value = cast[uint64](addr y) + doAssert value mod 16 == 0 + + f()() + diff --git a/tests/alloc/tmembug.nim b/tests/alloc/tmembug.nim new file mode 100644 index 0000000000..63b51ec5da --- /dev/null +++ b/tests/alloc/tmembug.nim @@ -0,0 +1,54 @@ +discard """ + joinable: false +""" + +import std / [atomics, strutils, sequtils] + +type + BackendMessage* = object + field*: seq[int] + +var + chan1: Channel[BackendMessage] + chan2: Channel[BackendMessage] + +chan1.open() +chan2.open() + +proc routeMessage*(msg: BackendMessage) = + discard chan2.trySend(msg) + +var + recv: Thread[void] + stopToken: Atomic[bool] + +proc recvMsg() = + while not stopToken.load(moRelaxed): + let resp = chan1.tryRecv() + if resp.dataAvailable: + routeMessage(resp.msg) + echo "child consumes ", formatSize getOccupiedMem() + +createThread[void](recv, recvMsg) + +const MESSAGE_COUNT = 100 + +proc main() = + let msg: BackendMessage = BackendMessage(field: (0..500).toSeq()) + for j in 0..0: #100: + echo "New iteration" + + for _ in 1..MESSAGE_COUNT: + chan1.send(msg) + echo "After sending" + + var counter = 0 + while counter < MESSAGE_COUNT: + let resp = recv(chan2) + counter.inc + echo "After receiving ", formatSize getOccupiedMem() + + stopToken.store true, moRelaxed + joinThreads(recv) + +main() diff --git a/tests/alloc/tmembug2.nim b/tests/alloc/tmembug2.nim new file mode 100644 index 0000000000..01bce6f141 --- /dev/null +++ b/tests/alloc/tmembug2.nim @@ -0,0 +1,58 @@ +discard """ + disabled: "true" +""" + +import std / [atomics, strutils, sequtils, isolation] + +import threading / channels + +type + BackendMessage* = object + field*: seq[int] + +const MESSAGE_COUNT = 100 + +var + chan1 = newChan[BackendMessage](MESSAGE_COUNT*2) + chan2 = newChan[BackendMessage](MESSAGE_COUNT*2) + +#chan1.open() +#chan2.open() + +proc routeMessage*(msg: BackendMessage) = + var m = isolate(msg) + discard chan2.trySend(m) + +var + thr: Thread[void] + stopToken: Atomic[bool] + +proc recvMsg() = + while not stopToken.load(moRelaxed): + var resp: BackendMessage + if chan1.tryRecv(resp): + #if resp.dataAvailable: + routeMessage(resp) + echo "child consumes ", formatSize getOccupiedMem() + +createThread[void](thr, recvMsg) + +proc main() = + let msg: BackendMessage = BackendMessage(field: (0..5).toSeq()) + for j in 0..100: + echo "New iteration" + + for _ in 1..MESSAGE_COUNT: + chan1.send(msg) + echo "After sending" + + var counter = 0 + while counter < MESSAGE_COUNT: + let resp = recv(chan2) + counter.inc + echo "After receiving ", formatSize getOccupiedMem() + + stopToken.store true, moRelaxed + joinThreads(thr) + +main() diff --git a/tests/arc/t16458.nim b/tests/arc/t16458.nim new file mode 100644 index 0000000000..6ae114287e --- /dev/null +++ b/tests/arc/t16458.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--gc:orc --d:useNimRtl" + action: "compile" +""" + +echo 134 \ No newline at end of file diff --git a/tests/arc/t16558.nim b/tests/arc/t16558.nim new file mode 100644 index 0000000000..0dbe02b331 --- /dev/null +++ b/tests/arc/t16558.nim @@ -0,0 +1,9 @@ +discard """ + matrix: "--gc:arc" + errormsg: "expression cannot be cast to 'int'" +""" + +block: # bug #16558 + var value = "hi there" + var keepInt: int + keepInt = cast[int](value) diff --git a/tests/arc/t17812.nim b/tests/arc/t17812.nim new file mode 100644 index 0000000000..dd8ac89b07 --- /dev/null +++ b/tests/arc/t17812.nim @@ -0,0 +1,41 @@ +discard """ + targets: "c js" + matrix: "--gc:refc; --gc:arc" +""" + +import std/times + +block: # bug #17812 + block: + type + Task = object + cb: proc () + + proc hello() = discard + + + let t = Task(cb: hello) + + doAssert t.repr.len > 0 + + + block: + type MyObj = object + field: DateTime + + + proc `$`(o: MyObj): string = o.repr + + doAssert ($MyObj()).len > 0 + +# bug #22175 + +type Xxx = object + value: string + +proc complete(xxx: ref Xxx, v: sink string) = + xxx.value = move(v) + +let yyy = (ref Xxx)() + +yyy.complete("test") diff --git a/tests/arc/t18645.nim b/tests/arc/t18645.nim new file mode 100644 index 0000000000..c5fddd4bbc --- /dev/null +++ b/tests/arc/t18645.nim @@ -0,0 +1,18 @@ +discard """ + matrix: "--gc:arc; --gc:refc" + output: ''' +1 +2 +3 +''' +""" + +proc bitTypeIdUnion() = + var bitId {.global.} = block: + 0 + inc bitId + echo bitId + +bitTypeIdUnion() +bitTypeIdUnion() +bitTypeIdUnion() diff --git a/tests/arc/t18971.nim b/tests/arc/t18971.nim new file mode 100644 index 0000000000..9b587d9562 --- /dev/null +++ b/tests/arc/t18971.nim @@ -0,0 +1,10 @@ +discard """ + cmd: "nim c --gc:arc $file" +""" + +type MyObj = ref object + +var o = MyObj() +proc x: var MyObj = o + +var o2 = x() diff --git a/tests/arc/t18977.nim b/tests/arc/t18977.nim new file mode 100644 index 0000000000..c775551a46 --- /dev/null +++ b/tests/arc/t18977.nim @@ -0,0 +1,26 @@ +discard """ + matrix: "--mm:arc" +""" + +type + E = enum + a, b, c, d + X = object + v: int + O = object + case kind: E + of a: + a: int + of {b, c}: + b: float + else: + d: X + +proc `=destroy`(x: var X) = + echo "x destroyed" + +var o = O(kind: d, d: X(v: 12345)) +doAssert o.d.v == 12345 + +doAssertRaises(FieldDefect): + o.kind = a diff --git a/tests/arc/t19231.nim b/tests/arc/t19231.nim new file mode 100644 index 0000000000..40fcf277c4 --- /dev/null +++ b/tests/arc/t19231.nim @@ -0,0 +1,18 @@ +discard """ + matrix: "--mm:orc" + targets: "c cpp" +""" + +type + Game* = ref object + +proc free*(game: Game) = + var mixNumOpened:cint = 0 + for i in 0.. 0: + let c = @[a[0] mod b[0]] + a = b + b = c + return a + +doAssert gcd(@[1], @[2]) == @[1] \ No newline at end of file diff --git a/tests/arc/t19862.nim b/tests/arc/t19862.nim new file mode 100644 index 0000000000..6d3f576926 --- /dev/null +++ b/tests/arc/t19862.nim @@ -0,0 +1,15 @@ +discard """ + matrix: "--gc:refc; --gc:arc" +""" + +import std/widestrs + +# bug #19862 +type NewString = object + +proc len(s: NewString): int = 10 + +converter toNewString(x: WideCStringObj): NewString = discard + +let w = newWideCString("test") +doAssert len(w) == 4 diff --git a/tests/arc/t20456.nim b/tests/arc/t20456.nim new file mode 100644 index 0000000000..ace79255a0 --- /dev/null +++ b/tests/arc/t20456.nim @@ -0,0 +1,7 @@ +discard """ + cmd: "nim check $file" + action: "compile" +""" + +when not defined(gcOrc): + {.error: "orc".} diff --git a/tests/arc/t20588.nim b/tests/arc/t20588.nim new file mode 100644 index 0000000000..008bd1dcd3 --- /dev/null +++ b/tests/arc/t20588.nim @@ -0,0 +1,25 @@ +discard """ + cmd: "nim check --warnings:off --hints:off $file" + errormsg: "" + nimout: ''' +t20588.nim(20, 12) Error: illegal type conversion to 'auto' +t20588.nim(21, 14) Error: illegal type conversion to 'typed' +t20588.nim(22, 16) Error: illegal type conversion to 'untyped' +t20588.nim(24, 7) Error: illegal type conversion to 'any' +''' +""" + + + + + + + + + +discard 0.0.auto +discard typed("abc") +discard untyped(4) +var a = newSeq[bool](1000) +if any(a): + echo "ok?" \ No newline at end of file diff --git a/tests/arc/t21184.nim b/tests/arc/t21184.nim new file mode 100644 index 0000000000..91d0c42c77 --- /dev/null +++ b/tests/arc/t21184.nim @@ -0,0 +1,77 @@ +discard """ + matrix: "--mm:orc" +""" + +import std/[with] + +type + Node* {.acyclic.} = ref object of RootObj + name: string + data: pointer + children: seq[Node] + TextNode = ref object of Node + text: string + +proc fakeEcho(s: string) = + if s.len < 0: + echo s + +proc newNode[T: Node](parent: Node): T = + new result + result.data = alloc0(250) + parent.children.add(result) + +proc newRootNode(): Node = + new result + result.data = alloc0(250) + +method printNode(node: Node) {.base.} = + fakeEcho node.name + +method printNode(node: TextNode) = + procCall printNode(Node(node)) + fakeEcho node.text + +proc printChildren(node: Node) = + for child in node.children: + child.printNode() + printChildren(child) + +proc free(node: Node) = + for child in node.children: + free(child) + dealloc(node.data) + +template node(parent: Node, body: untyped): untyped = + var node = newNode[Node](parent) + with node: + body + +proc textNode(parent: Node, text: string) = + var node = newNode[TextNode](parent) + node.text = text + +template withRootNode(body: untyped): untyped = + var root = newRootNode() + root.name = "root" + with root: + body + root.printNode() + printChildren(root) + root.free() + +proc doTest() = + withRootNode: + node: + name = "child1" + node: + name = "child2" + node: + name = "child3" + textNode "Hello, world!" + + +# bug #21171 +if isMainModule: + for i in 0..100000: + doTest() diff --git a/tests/arc/t22218.nim b/tests/arc/t22218.nim new file mode 100644 index 0000000000..7837ed1d0d --- /dev/null +++ b/tests/arc/t22218.nim @@ -0,0 +1,25 @@ +discard """ + cmd: "nim c --mm:arc $file" + errormsg: "'=copy' is not available for type ; requires a copy because it's not the last read of 'chan[]'; routine: test" +""" + +# bug #22218 +type Obj[T] = object + v: T + +proc `=copy`[T]( + dest: var Obj[T], + source: Obj[T] + ) {.error: "A channel cannot be copied".} + +from system/ansi_c import c_calloc + +proc test() = + var v: bool = true + var chan = cast[ptr Obj[int]](c_calloc(1, csize_t sizeof(Obj[int]))) + var copy = chan[] + + echo chan.v + echo v + +test() \ No newline at end of file diff --git a/tests/arc/t22237.nim b/tests/arc/t22237.nim new file mode 100644 index 0000000000..c6dbf768a1 --- /dev/null +++ b/tests/arc/t22237.nim @@ -0,0 +1,55 @@ +discard """ + matrix: "--mm:arc; --mm:orc" +""" + +import std/macros +import std/streams + +# bug #22237 + +proc iterlines_closure2(f: File | Stream): iterator (): string = + result = iterator(): string = + for line in f.lines: + if line.len == 0: + break + yield line + +proc test() = + let f = newStringStream(""" + 1 + 2 + + 3 + 4 + + 5 + 6 + 7 + + 8 +""") + while not f.atEnd(): + let iterator_inst = iterlines_closure2(f) + for item in iterator_inst(): # Fails with "SIGSEGV: Illegal storage access. (Attempt to read from nil?)" + discard + +test() + +# bug #21160 +import sequtils + +iterator allMoves(fls: seq[int]): seq[int] = + yield fls + +proc neighbors(flrs: seq[int]): iterator: seq[int] = + return iterator(): seq[int] = + for flrs2 in allMoves(flrs): + yield flrs2 + for flrs3 in allMoves(flrs2): + yield flrs3 + +let f = @[1] +for _ in neighbors(f): + discard +for _ in neighbors(f): + discard diff --git a/tests/arc/t22478.nim b/tests/arc/t22478.nim new file mode 100644 index 0000000000..5373fa1616 --- /dev/null +++ b/tests/arc/t22478.nim @@ -0,0 +1,46 @@ +discard """ + matrix: "-d:nimNoLentIterators --mm:arc" + output: '''PUSH DATA: {"test.message":{"test":{"nested":"v1"}}}''' + joinable: false +""" + +# bug #22748 +import std/[json, typetraits, times] + +# publish + +proc publish*[T](payload: T) = + discard + +type MetricsPoint* = JsonNode + +proc push*(stat: string, data: JsonNode, usec: int64 = 0) = + let payload = newJObject() + + # this results in a infinite recursion unless we deepCopy() + payload[stat] = data #.deepCopy + + echo "PUSH DATA: ", payload + + publish[MetricsPoint](payload) + +var scopes {.threadvar.}: seq[JsonNode] + +type WithTimeCallback*[T] = proc(data: var JsonNode): T + +proc pushScoped*[T](metric: string, blk: WithTimeCallback[T]): T {.gcsafe.} = + scopes.add newJObject() + defer: discard scopes.pop() + + let stc = (cpuTime() * 1000_000).int64 + result = blk(scopes[^1]) + let dfc = (cpuTime() * 1000_000).int64 - stc + + push(metric, scopes[^1], dfc) + +# demo code + +discard pushScoped[int]("test.message") do (data: var JsonNode) -> int: + data["test"] = %*{ + "nested": "v1" + } \ No newline at end of file diff --git a/tests/arc/t22787.nim b/tests/arc/t22787.nim new file mode 100644 index 0000000000..5840a984b9 --- /dev/null +++ b/tests/arc/t22787.nim @@ -0,0 +1,37 @@ +discard """ + joinable: false +""" + +import std/assertions + +proc foo = + var s:seq[string] + var res = "" + + for i in 0..3: + s.add ("test" & $i) + s.add ("test" & $i) + + var lastname:string + + for i in s: + var name = i[0..4] + + if name != lastname: + res.add "NEW:" & name & "\n" + else: + res.add name & ">" & lastname & "\n" + + lastname = name + + doAssert res == """ +NEW:test0 +test0>test0 +NEW:test1 +test1>test1 +NEW:test2 +test2>test2 +NEW:test3 +test3>test3 +""" +foo() \ No newline at end of file diff --git a/tests/arc/t23247.nim b/tests/arc/t23247.nim new file mode 100644 index 0000000000..0fadc50cd9 --- /dev/null +++ b/tests/arc/t23247.nim @@ -0,0 +1,52 @@ +discard """ + matrix: ";-d:useMalloc" +""" + +# bug #23247 +import std/hashes + +func baseAddr[T](x: openArray[T]): ptr T = + # Return the address of the zero:th element of x or `nil` if x is empty + if x.len == 0: nil else: cast[ptr T](x) + +func makeUncheckedArray[T](p: ptr T): ptr UncheckedArray[T] = + cast[ptr UncheckedArray[T]](p) + +type + LabelKey = object + data: seq[string] + refs: ptr UncheckedArray[string] + refslen: int + + Gauge = ref object + metrics: seq[seq[seq[string]]] + +template values(key: LabelKey): openArray[string] = + if key.refslen > 0: + key.refs.toOpenArray(0, key.refslen - 1) + else: + key.data + +proc hash(key: LabelKey): Hash = + hash(key.values) + +proc view(T: type LabelKey, values: openArray[string]): T = + # TODO some day, we might get view types - until then.. + LabelKey(refs: baseAddr(values).makeUncheckedArray(), refslen: values.len()) + +template withValue2(k: untyped) = + discard hash(k) + +proc setGauge( + collector: Gauge, + labelValues: openArray[string], +) = + let v = LabelKey.view(labelValues) + withValue2(v) + collector.metrics.add @[@labelValues, @labelValues] + discard @labelValues + +var nim_gc_mem_bytes = Gauge() +let threadID = $getThreadId() +setGauge(nim_gc_mem_bytes, @[threadID]) +setGauge(nim_gc_mem_bytes, @[threadID]) \ No newline at end of file diff --git a/tests/arc/t9650.nim b/tests/arc/t9650.nim new file mode 100644 index 0000000000..a8182db68c --- /dev/null +++ b/tests/arc/t9650.nim @@ -0,0 +1,87 @@ +discard """ + matrix: "--gc:arc" +""" + +import typetraits + +# bug #9650 +type + SharedPtr*[T] = object + val: ptr tuple[atomicCounter: int, value: T] + + Node*[T] = object + value: T + next: SharedPtr[Node[T]] + + ForwardList*[T] = object + first: SharedPtr[Node[T]] + +proc `=destroy`*[T](p: var SharedPtr[T]) = + if p.val != nil: + let c = atomicDec(p.val[].atomicCounter) + if c == 0: + when not supportsCopyMem(T): + `=destroy`(p.val[]) + dealloc(p.val) + p.val = nil + +proc `=`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) = + if dest.val != src.val: + if dest.val != nil: + `=destroy`(dest) + if src.val != nil: + discard atomicInc(src.val[].atomicCounter) + dest.val = src.val + +proc `=sink`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) = + if dest.val != nil and dest.val != src.val: + `=destroy`(dest) + dest.val = src.val + + +proc newSharedPtr*[T](val: sink T): SharedPtr[T] = + result.val = cast[type(result.val)](alloc(sizeof(result.val[]))) + reset(result.val[]) + result.val.atomicCounter = 1 + result.val.value = val + +proc isNil*[T](p: SharedPtr[T]): bool = + p.val == nil + +template `->`*[T](p: SharedPtr[T], name: untyped): untyped = + p.val.value.name + +proc createNode[T](val: T): SharedPtr[ Node[T] ]= + result = newSharedPtr(Node[T](value: val)) + +proc push_front*[T](list: var ForwardList[T], val: T) = + var newElem = createNode(val) + newElem->next = list.first + list.first = newElem + +proc pop_front*[T](list: var ForwardList[T]) = + let head = list.first + list.first = head->next + +proc toString*[T](list: ForwardList[T]): string = + result = "[" + var head = list.first + while not head.isNil: + result &= $(head->value) & ", " + head = head->next + result &= ']' + +block: + var x: ForwardList[int] + x.push_front(1) + x.push_front(2) + x.push_front(3) + + doAssert toString(x) == "[3, 2, 1, ]" + + x.pop_front() + x.pop_front() + doAssert toString(x) == "[1, ]" + + x.pop_front() + doAssert toString(x) == "[]" diff --git a/tests/arc/taliased_reassign.nim b/tests/arc/taliased_reassign.nim new file mode 100644 index 0000000000..5563fae8c4 --- /dev/null +++ b/tests/arc/taliased_reassign.nim @@ -0,0 +1,41 @@ +discard """ + matrix: "--mm:orc" +""" + +# bug #20993 + +type + Dual[int] = object # must be generic (even if fully specified) + p: int +proc D(p: int): Dual[int] = Dual[int](p: p) +proc `+`(x: Dual[int], y: Dual[int]): Dual[int] = D(x.p + y.p) + +type + Tensor[T] = object + buf: seq[T] +proc newTensor*[T](s: int): Tensor[T] = Tensor[T](buf: newSeq[T](s)) +proc `[]`*[T](t: Tensor[T], idx: int): T = t.buf[idx] +proc `[]=`*[T](t: var Tensor[T], idx: int, val: T) = t.buf[idx] = val + +proc `+.`[T](t1, t2: Tensor[T]): Tensor[T] = + let n = t1.buf.len + result = newTensor[T](n) + for i in 0 ..< n: + result[i] = t1[i] + t2[i] + +proc toTensor*[T](a: sink seq[T]): Tensor[T] = + ## This breaks it: Using `T` instead makes it work + type U = typeof(a[0]) + var t: Tensor[U] # Tensor[T] works + t.buf = a + result = t + +proc loss() = + var B = toTensor(@[D(123)]) + let a = toTensor(@[D(-10)]) + B = B +. a + doAssert B[0].p == 113, "I want to be 113, but I am " & $B[0].p + +loss() + + diff --git a/tests/arc/tarc_macro.nim b/tests/arc/tarc_macro.nim new file mode 100644 index 0000000000..33ade1da4a --- /dev/null +++ b/tests/arc/tarc_macro.nim @@ -0,0 +1,57 @@ +import macros + +var destroyCalled = false +macro bar() = + let s = newTree(nnkAccQuoted, ident"=destroy") + # let s = ident"`=destroy`" # this would not work + result = quote do: + type Foo = object + # proc `=destroy`(a: var Foo) = destroyCalled = true # this would not work + proc `s`(a: var Foo) = destroyCalled = true + block: + let a = Foo() +bar() +doAssert destroyCalled + +# custom `op` +var destroyCalled2 = false +macro bar(ident) = + var x = 1.5 + result = quote("@") do: + type Foo = object + let `@ident` = 0 # custom op interpolated symbols need quoted (``) + proc `=destroy`(a: var Foo) = + doAssert @x == 1.5 + doAssert compiles(@x == 1.5) + let b1 = @[1,2] + let b2 = @@[1,2] + doAssert $b1 == "[1, 2]" + doAssert $b2 == "@[1, 2]" + destroyCalled2 = true + block: + let a = Foo() +bar(someident) +doAssert destroyCalled2 + +proc `&%`(x: int): int = 1 +proc `&%`(x, y: int): int = 2 + +macro bar2() = + var x = 3 + result = quote("&%") do: + var y = &%x # quoting operator + doAssert &%&%y == 1 # unary operator => need to escape + doAssert y &% y == 2 # binary operator => no need to escape + doAssert y == 3 +bar2() + +block: + macro foo(a: openArray[string] = []): string = + echo a # Segfault doesn't happen if this is removed + newLit "" + + proc bar(a: static[openArray[string]] = []) = + const tmp = foo(a) + + # bug #22909 + doAssert not compiles(bar()) diff --git a/tests/arc/tarc_orc.nim b/tests/arc/tarc_orc.nim new file mode 100644 index 0000000000..674ba0dbbe --- /dev/null +++ b/tests/arc/tarc_orc.nim @@ -0,0 +1,162 @@ +discard """ + targets: "c cpp" + matrix: "--mm:arc; --mm:orc" +""" + +block: + type + PublicKey = array[32, uint8] + PrivateKey = array[64, uint8] + + proc ed25519_create_keypair(publicKey: ptr PublicKey; privateKey: ptr PrivateKey) = + publicKey[][0] = uint8(88) + + type + KeyPair = object + public: PublicKey + private: PrivateKey + + proc initKeyPair(): KeyPair = + ed25519_create_keypair(result.public.addr, result.private.addr) + + let keys = initKeyPair() + doAssert keys.public[0] == 88 + + +template minIndexByIt: untyped = + var other = 3 + other + +proc bug20303() = + var hlibs = @["hello", "world", "how", "are", "you"] + let res = hlibs[minIndexByIt()] + doAssert res == "are" + +bug20303() + +proc main() = # todo bug with templates + block: # bug #11267 + var a: seq[char] = block: @[] + doAssert a == @[] + # 2 + proc b: seq[string] = + discard + @[] + doAssert b() == @[] +static: main() +main() + + +type Obj = tuple + value: int + arr: seq[int] + +proc bug(): seq[Obj] = + result.add (value: 0, arr: @[]) + result[^1].value = 1 + result[^1].arr.add 1 + +# bug #19990 +let s = bug() +doAssert s[0] == (value: 1, arr: @[1]) + +block: # bug #21974 + type Test[T] = ref object + values : seq[T] + counter: int + + proc newTest[T](): Test[T] = + result = new(Test[T]) + result.values = newSeq[T](16) + result.counter = 0 + + proc push[T](self: Test[T], value: T) = + self.counter += 1 + if self.counter >= self.values.len: + self.values.setLen(self.values.len * 2) + self.values[self.counter - 1] = value + + proc pop[T](self: Test[T]): T = + result = self.values[0] + self.values[0] = self.values[self.counter - 1] # <--- This line + self.counter -= 1 + + + type X = tuple + priority: int + value : string + + var a = newTest[X]() + a.push((1, "One")) + doAssert a.pop.value == "One" + +# bug #21987 + +type + EmbeddedImage* = distinct Image + Image = object + len: int + +proc imageCopy*(image: Image): Image {.nodestroy.} + +proc `=destroy`*(x: var Image) = + discard +proc `=sink`*(dest: var Image; source: Image) = + `=destroy`(dest) + wasMoved(dest) + +proc `=dup`*(source: Image): Image {.nodestroy.} = + result = imageCopy(source) + +proc `=copy`*(dest: var Image; source: Image) = + dest = imageCopy(source) # calls =sink implicitly + +proc `=destroy`*(x: var EmbeddedImage) = discard + +proc `=dup`*(source: EmbeddedImage): EmbeddedImage {.nodestroy.} = source + +proc `=copy`*(dest: var EmbeddedImage; source: EmbeddedImage) {.nodestroy.} = + dest = source + +proc imageCopy*(image: Image): Image = + result = image + +proc main2 = + block: + var a = Image(len: 2).EmbeddedImage + var b = Image(len: 1).EmbeddedImage + b = a + doAssert Image(a).len == 2 + doAssert Image(b).len == 2 + + block: + var a = Image(len: 2) + var b = Image(len: 1) + b = a + doAssert a.len == 2 + doAssert b.len == 0 + +main2() + +block: + type + TestObj = object of RootObj + name: string + + TestSubObj = object of TestObj + objname: string + + proc `=destroy`(x: TestObj) = + `=destroy`(x.name) + + proc `=destroy`(x: TestSubObj) = + `=destroy`(x.objname) + `=destroy`(TestObj(x)) + + proc testCase() = + let t1 {.used.} = TestSubObj(objname: "tso1", name: "to1") + + proc main() = + testCase() + + main() diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 55803085f2..88582303da 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -1,5 +1,7 @@ discard """ output: ''' +Destructor for TestTestObj +=destroy called 123xyzabc destroyed: false destroyed: false @@ -26,14 +28,89 @@ new line after - @['a'] finalizer aaaaa hello +true +copying +123 +42 ok -closed destroying variable: 20 destroying variable: 10 +closed ''' - cmd: "nim c --gc:arc --deepcopy:on $file" + cmd: "nim c --mm:arc --deepcopy:on -d:nimAllocPagesViaMalloc $file" """ +block: # bug #23627 + type + TestObj = object of RootObj + + Test2 = object of RootObj + foo: TestObj + + TestTestObj = object of RootObj + shit: TestObj + + proc `=destroy`(x: TestTestObj) = + echo "Destructor for TestTestObj" + let test = Test2(foo: TestObj()) + + proc testCaseT() = + let tt1 {.used.} = TestTestObj(shit: TestObj()) + + + proc main() = + testCaseT() + + main() + + +# bug #9401 + +type + MyObj = object + len: int + data: ptr UncheckedArray[float] + +proc `=destroy`*(m: MyObj) = + + echo "=destroy called" + + if m.data != nil: + deallocShared(m.data) + +type + MyObjDistinct = distinct MyObj + +proc `=copy`*(m: var MyObj, m2: MyObj) = + if m.data == m2.data: return + if m.data != nil: + `=destroy`(m) + m.len = m2.len + if m.len > 0: + m.data = cast[ptr UncheckedArray[float]](allocShared(sizeof(float) * m.len)) + copyMem(m.data, m2.data, sizeof(float) * m.len) + + +proc `=sink`*(m: var MyObj, m2: MyObj) = + if m.data != m2.data: + if m.data != nil: + `=destroy`(m) + m.len = m2.len + m.data = m2.data + +proc newMyObj(len: int): MyObj = + result.len = len + result.data = cast[ptr UncheckedArray[float]](allocShared(sizeof(float) * len)) + +proc newMyObjDistinct(len: int): MyObjDistinct = + MyObjDistinct(newMyObj(len)) + +proc fooDistinct = + doAssert newMyObjDistinct(2).MyObj.len == 2 + +fooDistinct() + + proc takeSink(x: sink string): bool = true proc b(x: sink string): string = @@ -51,7 +128,7 @@ bbb("123") type Variable = ref object value: int -proc `=destroy`(self: var typeof(Variable()[])) = +proc `=destroy`(self: typeof(Variable()[])) = echo "destroying variable: ",self.value proc newVariable(value: int): Variable = @@ -71,7 +148,7 @@ proc test(count: int) = test(3) proc test2(count: int) = - #block: #XXX: Fails with block currently + block: #XXX: Fails with block currently var v {.global.} = newVariable(20) var count = count - 1 @@ -105,7 +182,7 @@ type B = ref object of A x: int -proc `=destroy`(x: var AObj) = +proc `=destroy`(x: AObj) = close(x.io) echo "closed" @@ -390,3 +467,302 @@ proc newPixelBuffer(): PixelBuffer = discard newPixelBuffer() + +# bug #17199 + +proc passSeq(data: seq[string]) = + # used the system.& proc initially + let wat = data & "hello" + +proc test2 = + let name = @["hello", "world"] + passSeq(name) + doAssert name == @["hello", "world"] + +static: test2() # was buggy +test2() + +proc merge(x: sink seq[string], y: sink string): seq[string] = + newSeq(result, x.len + 1) + for i in 0..x.len-1: + result[i] = move(x[i]) + result[x.len] = move(y) + +proc passSeq2(data: seq[string]) = + # used the system.& proc initially + let wat = merge(data, "hello") + +proc test3 = + let name = @["hello", "world"] + passSeq2(name) + doAssert name == @["hello", "world"] + +static: test3() # was buggy +test3() + +# bug #17712 +proc t17712 = + var ppv = new int + discard @[ppv] + var el: ref int + el = [ppv][0] + echo el != nil + +t17712() + +# bug #18030 + +type + Foo = object + n: int + +proc `=copy`(dst: var Foo, src: Foo) = + echo "copying" + dst.n = src.n + +proc `=sink`(dst: var Foo, src: Foo) = + echo "sinking" + dst.n = src.n + +var a: Foo + +proc putValue[T](n: T) + +proc useForward = + putValue(123) + +proc putValue[T](n: T) = + var b = Foo(n:n) + a = b + echo b.n + +useForward() + + +# bug #17319 +type + BrokenObject = ref object + brokenType: seq[int] + +proc use(obj: BrokenObject) = + discard + +method testMethod(self: BrokenObject) {.base.} = + iterator testMethodIter() {.closure.} = + use(self) + + var nameIterVar = testMethodIter + nameIterVar() + +let mikasa = BrokenObject() +mikasa.testMethod() + +# bug #19205 +type + InputSectionBase* = object of RootObj + relocations*: seq[int] # traced reference. string has a similar SIGSEGV. + InputSection* = object of InputSectionBase + +proc fooz(sec: var InputSectionBase) = + if sec of InputSection: # this line SIGSEGV. + echo 42 + +var sec = create(InputSection) +sec[] = InputSection(relocations: newSeq[int]()) +fooz sec[] + +block: + type + Data = ref object + id: int + proc main = + var x = Data(id: 99) + var y = x + x[] = Data(id: 778)[] + doAssert y.id == 778 + doAssert x[].id == 778 + main() + +block: # bug #19857 + type + ValueKind = enum VNull, VFloat, VObject # need 3 elements. Cannot remove VNull or VObject + + Value = object + case kind: ValueKind + of VFloat: fnum: float + of VObject: tab: Table[int, int] # OrderedTable[T, U] also makes it fail. + # "simpler" types also work though + else: discard # VNull can be like this, but VObject must be filled + + # required. Pure proc works + FormulaNode = proc(c: OrderedTable[string, int]): Value + + proc toF(v: Value): float = + doAssert v.kind == VFloat + case v.kind + of VFloat: result = v.fnum + else: discard + + + proc foo() = + let fuck = initOrderedTable[string, int]() + proc cb(fuck: OrderedTable[string, int]): Value = + # works: + #result = Value(kind: VFloat, fnum: fuck["field_that_does_not_exist"].float) + # broken: + discard "actuall runs!" + let t = fuck["field_that_does_not_exist"] + echo "never runs, but we crash after! ", t + + doAssertRaises(KeyError): + let fn = FormulaNode(cb) + let v = fn(fuck) + #echo v + let res = v.toF() + + foo() + +import std/options + +# bug #21592 +type Event* = object + code*: string + +type App* = ref object of RootObj + id*: string + +method process*(self: App): Option[Event] {.base.} = + raise Exception.new_exception("not impl") + +# bug #21617 +type Test2 = ref object of RootObj + +method bug(t: Test2): seq[float] {.base.} = discard + +block: # bug #22664 + type + ElementKind = enum String, Number + Element = object + case kind: ElementKind + of String: + str: string + of Number: + num: float + Calc = ref object + stack: seq[Element] + + var calc = new Calc + + calc.stack.add Element(kind: Number, num: 200.0) + doAssert $calc.stack == "@[(kind: Number, num: 200.0)]" + let calc2 = calc + calc2.stack = calc.stack # This nulls out the object in the stack + doAssert $calc.stack == "@[(kind: Number, num: 200.0)]" + doAssert $calc2.stack == "@[(kind: Number, num: 200.0)]" + +block: # bug #19250 + type + Bar[T] = object + err: proc(): string + + Foo[T] = object + run: proc(): Bar[T] + + proc bar[T](err: proc(): string): Bar[T] = + assert not err.isNil + Bar[T](err: err) + + proc foo(): Foo[char] = + result.run = proc(): Bar[char] = + # works + # result = Bar[char](err: proc(): string = "x") + # not work + result = bar[char](proc(): string = "x") + + proc bug[T](fs: Foo[T]): Foo[T] = + result.run = proc(): Bar[T] = + let res = fs.run() + + # works + # var errors = @[res.err] + + # not work + var errors: seq[proc(): string] + errors.add res.err + + return bar[T] do () -> string: + for err in errors: + result.add res.err() + + doAssert bug(foo()).run().err() == "x" + +block: # bug #22259 + type + ProcWrapper = tuple + p: proc() {.closure.} + + + proc f(wrapper: ProcWrapper) = + let s = @[wrapper.p] + let a = [wrapper.p] + + proc main = + # let wrapper: ProcWrapper = ProcWrapper(p: proc {.closure.} = echo 10) + let wrapper: ProcWrapper = (p: proc {.closure.} = echo 10) + f(wrapper) + + main() + +block: + block: # bug #22923 + block: + let + a: int = 100 + b: int32 = 200'i32 + + let + x = arrayWith(a, 8) # compiles + y = arrayWith(b, 8) # internal error + z = arrayWith(14, 8) # integer literal also results in a crash + + doAssert x == [100, 100, 100, 100, 100, 100, 100, 100] + doAssert $y == "[200, 200, 200, 200, 200, 200, 200, 200]" + doAssert z == [14, 14, 14, 14, 14, 14, 14, 14] + + block: + let a: string = "nim" + doAssert arrayWith(a, 3) == ["nim", "nim", "nim"] + + let b: char = 'c' + doAssert arrayWith(b, 3) == ['c', 'c', 'c'] + + let c: uint = 300'u + doAssert $arrayWith(c, 3) == "[300, 300, 300]" + +block: # bug #23505 + type + K = object + C = object + value: ptr K + + proc init(T: type C): C = + let tmp = new K + C(value: addr tmp[]) + + discard init(C) + +block: # bug #23524 + type MyType = object + a: int + + proc `=destroy`(typ: MyType) = discard + + var t1 = MyType(a: 100) + var t2 = t1 # Should be a copy? + + proc main() = + t2 = t1 + doAssert t1.a == 100 + doAssert t2.a == 100 + + main() diff --git a/tests/arc/tasyncawait.nim b/tests/arc/tasyncawait.nim index 75d9bc9b58..91a7453cd3 100644 --- a/tests/arc/tasyncawait.nim +++ b/tests/arc/tasyncawait.nim @@ -40,7 +40,7 @@ proc readMessages(client: AsyncFD) {.async.} = clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tests/arc/tasyncleak.nim b/tests/arc/tasyncleak.nim index eb0c452131..8e3a7b3e7b 100644 --- a/tests/arc/tasyncleak.nim +++ b/tests/arc/tasyncleak.nim @@ -1,5 +1,5 @@ discard """ - outputsub: "(allocCount: 4302, deallocCount: 4300)" + outputsub: "(allocCount: 4050, deallocCount: 4048)" cmd: "nim c --gc:orc -d:nimAllocStats $file" """ diff --git a/tests/arc/tcaseobj.nim b/tests/arc/tcaseobj.nim index a967a509bc..be1d722edc 100644 --- a/tests/arc/tcaseobj.nim +++ b/tests/arc/tcaseobj.nim @@ -10,6 +10,7 @@ begin end prevented (ok: true, value: "ok") +@[(kind: P, pChildren: @[])] myobj destroyed ''' """ @@ -59,7 +60,7 @@ proc `=destroy`(o: var TMyObj) = o.p = nil echo "myobj destroyed" -proc `=`(dst: var TMyObj, src: TMyObj) = +proc `=copy`(dst: var TMyObj, src: TMyObj) = `=destroy`(dst) dst.p = alloc(src.len) dst.len = src.len @@ -169,18 +170,24 @@ proc test_myobject = x.x1 = "x1" x.x2 = "x2" x.y1 = "ljhkjhkjh" - x.kind1 = true + {.cast(uncheckedAssign).}: + x.kind1 = true x.y2 = @["1", "2"] - x.kind2 = true + {.cast(uncheckedAssign).}: + x.kind2 = true x.z1 = "yes" - x.kind2 = false + {.cast(uncheckedAssign).}: + x.kind2 = false x.z2 = @["1", "2"] - x.kind2 = true + {.cast(uncheckedAssign).}: + x.kind2 = true x.z1 = "yes" x.kind2 = true # should be no effect doAssert(x.z1 == "yes") - x.kind2 = false - x.kind1 = x.kind2 # support self assignment with effect + {.cast(uncheckedAssign).}: + x.kind2 = false + {.cast(uncheckedAssign).}: + x.kind1 = x.kind2 # support self assignment with effect try: x.kind1 = x.flag # flag is not accesible @@ -206,8 +213,9 @@ type error*: string proc init(): RocksDBResult[string] = - result.ok = true - result.value = "ok" + {.cast(uncheckedAssign).}: + result.ok = true + result.value = "ok" echo init() @@ -221,7 +229,8 @@ type MyObj = object of true: x1: string var a = MyObj(kind: false, x0: 1234) -a.kind = true +{.cast(uncheckedAssign).}: + a.kind = true doAssert(a.x1 == "") block: @@ -244,3 +253,88 @@ block: doAssert j1.x1 == 2 doAssert j0.x0 == 2 + +# ------------------------------------ +# bug #20305 + +type + ContentNodeKind = enum + P, Br, Text + ContentNode = object + case kind: ContentNodeKind + of P: pChildren: seq[ContentNode] + of Br: discard + of Text: textStr: string + +proc bug20305 = + var x = ContentNode(kind: P, pChildren: @[ + ContentNode(kind: P, pChildren: @[ContentNode(kind: Text, textStr: "brrr")]) + ]) + x.pChildren.add ContentNode(kind: Br) + x.pChildren.del(0) + {.cast(uncheckedAssign).}: + x.pChildren[0].kind = P + echo x.pChildren + +bug20305() + +# bug #21023 +block: + block: + type + MGErrorKind = enum + mgeUnexpected, mgeNotFound + + type Foo = object + kind: MGErrorKind + ex: Exception + + type Boo = object + a: seq[int] + + type + Result2 = object + case o: bool + of false: + e: Foo + of true: + v: Boo + + proc startSessionSync(): Result2 = + return Result2(o: true) + + proc mainSync = + let ff = startSessionSync() + doAssert ff.o == true + + mainSync() + + block: + type + MGErrorKind = enum + mgeUnexpected, mgeNotFound + + type Foo = object + kind: MGErrorKind + ex: Exception + + type Boo = object + a: seq[int] + + type + Result2 = object + case o: bool + of false: + e: Foo + of true: + v: Boo + s: int + + proc startSessionSync(): Result2 = + return Result2(o: true, s: 12) + + proc mainSync = + let ff = startSessionSync() + doAssert ff.s == 12 + + mainSync() diff --git a/tests/arc/tcaseobjcopy.nim b/tests/arc/tcaseobjcopy.nim index ed07b404e9..fb26a49736 100644 --- a/tests/arc/tcaseobjcopy.nim +++ b/tests/arc/tcaseobjcopy.nim @@ -169,18 +169,23 @@ proc test_myobject = x.x1 = "x1" x.x2 = "x2" x.y1 = "ljhkjhkjh" - x.kind1 = true + {.cast(uncheckedAssign).}: + x.kind1 = true x.y2 = @["1", "2"] - x.kind2 = true + {.cast(uncheckedAssign).}: + x.kind2 = true x.z1 = "yes" - x.kind2 = false + {.cast(uncheckedAssign).}: + x.kind2 = false x.z2 = @["1", "2"] - x.kind2 = true + {.cast(uncheckedAssign).}: + x.kind2 = true x.z1 = "yes" x.kind2 = true # should be no effect doAssert(x.z1 == "yes") - x.kind2 = false - x.kind1 = x.kind2 # support self assignment with effect + {.cast(uncheckedAssign).}: + x.kind2 = false + x.kind1 = x.kind2 # support self assignment with effect try: x.kind1 = x.flag # flag is not accesible @@ -206,7 +211,8 @@ type error*: string proc init(): RocksDBResult[string] = - result.ok = true + {.cast(uncheckedAssign).}: + result.ok = true result.value = "ok" echo init() @@ -221,7 +227,8 @@ type MyObj = object of true: x1: string var a = MyObj(kind: false, x0: 1234) -a.kind = true +{.cast(uncheckedAssign).}: + a.kind = true doAssert(a.x1 == "") block: diff --git a/tests/arc/tcomputedgoto.nim b/tests/arc/tcomputedgoto.nim index 541a748c62..07487684a6 100644 --- a/tests/arc/tcomputedgoto.nim +++ b/tests/arc/tcomputedgoto.nim @@ -1,16 +1,19 @@ discard """ - cmd: '''nim c --gc:arc $file''' - output: '''2 -2''' + cmd: '''nim c --mm:arc $file''' + output: ''' +2 +2 +destroyed +''' """ type ObjWithDestructor = object a: int -proc `=destroy`(self: var ObjWithDestructor) = +proc `=destroy`(self: ObjWithDestructor) = echo "destroyed" -proc `=`(self: var ObjWithDestructor, other: ObjWithDestructor) = +proc `=copy`(self: var ObjWithDestructor, other: ObjWithDestructor) = echo "copied" proc test(a: range[0..1], arg: ObjWithDestructor) = @@ -38,4 +41,4 @@ proc test(a: range[0..1], arg: ObjWithDestructor) = if iteration == 2: break -test(1, ObjWithDestructor()) \ No newline at end of file +test(1, ObjWithDestructor()) diff --git a/tests/arc/tcomputedgotocopy.nim b/tests/arc/tcomputedgotocopy.nim index 8337123ba0..07487684a6 100644 --- a/tests/arc/tcomputedgotocopy.nim +++ b/tests/arc/tcomputedgotocopy.nim @@ -1,13 +1,16 @@ discard """ - cmd: '''nim c --gc:arc $file''' - output: '''2 -2''' + cmd: '''nim c --mm:arc $file''' + output: ''' +2 +2 +destroyed +''' """ type ObjWithDestructor = object a: int -proc `=destroy`(self: var ObjWithDestructor) = +proc `=destroy`(self: ObjWithDestructor) = echo "destroyed" proc `=copy`(self: var ObjWithDestructor, other: ObjWithDestructor) = diff --git a/tests/arc/tconst_to_sink.nim b/tests/arc/tconst_to_sink.nim index ddcc46e67e..25f6593419 100644 --- a/tests/arc/tconst_to_sink.nim +++ b/tests/arc/tconst_to_sink.nim @@ -1,6 +1,7 @@ discard """ output: '''@[(s1: "333", s2: ""), (s1: "abc", s2: "def"), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "lastone", s2: "")]''' - cmd: "nim c --gc:arc $file" + matrix: "--gc:arc" + targets: "c cpp" """ # bug #13240 diff --git a/tests/arc/tcontrolflow.nim b/tests/arc/tcontrolflow.nim index 80cc2b187e..dbc1159034 100644 --- a/tests/arc/tcontrolflow.nim +++ b/tests/arc/tcontrolflow.nim @@ -76,7 +76,7 @@ var c = Control(x: 7) run(c) -proc sysFatal(exceptn: typedesc, message: string) {.inline, noreturn.} = +proc sysFatal(exceptn: typedesc, message: string) {.inline.} = var buf = newStringOfCap(200) add(buf, "##") add(buf, message) diff --git a/tests/arc/tcursor_field_obj_constr.nim b/tests/arc/tcursor_field_obj_constr.nim new file mode 100644 index 0000000000..b87f734bd5 --- /dev/null +++ b/tests/arc/tcursor_field_obj_constr.nim @@ -0,0 +1,44 @@ +discard """ + output: '''a +b +c''' + cmd: "nim c --gc:arc $file" +""" + +# bug #18469 + +type + Edge = object + neighbor {.cursor.}: Node + + NodeObj = object + neighbors: seq[Edge] + label: string + visited: bool + Node = ref NodeObj + + Graph = object + nodes: seq[Node] + +proc `=destroy`(x: var NodeObj) = + echo x.label + `=destroy`(x.neighbors) + `=destroy`(x.label) + +proc addNode(self: var Graph; label: string): Node = + self.nodes.add(Node(label: label)) + result = self.nodes[^1] + +proc addEdge(self: Graph; source, neighbor: Node) = + source.neighbors.add(Edge(neighbor: neighbor)) + +proc main = + var graph: Graph + let nodeA = graph.addNode("a") + let nodeB = graph.addNode("b") + let nodeC = graph.addNode("c") + + graph.addEdge(nodeA, neighbor = nodeB) + graph.addEdge(nodeA, neighbor = nodeC) + +main() diff --git a/tests/arc/tcursorloop.nim b/tests/arc/tcursorloop.nim new file mode 100644 index 0000000000..a37a6a0366 --- /dev/null +++ b/tests/arc/tcursorloop.nim @@ -0,0 +1,45 @@ +discard """ + cmd: '''nim c --gc:arc --expandArc:traverse --hint:Performance:off $file''' + nimout: ''' +--expandArc: traverse + +var + it + jt_cursor +try: + `=copy`(it, root) + block :tmp: + while ( + not (it == nil)): + if true: + echo [it.s] + `=copy`(it, it.ri) + jt_cursor = root + if ( + not (jt_cursor == nil)): + echo [jt_cursor.s] + jt_cursor = jt_cursor.ri +finally: + `=destroy`(it) +-- end of expandArc ------------------------ +''' +""" + +type + Node = ref object + le, ri: Node + s: string + +proc traverse(root: Node) = + var it = root + while it != nil: + if true: + echo it.s + it = it.ri + + var jt = root + if jt != nil: + echo jt.s + jt = jt.ri + +traverse(nil) diff --git a/tests/arc/tcustomtrace.nim b/tests/arc/tcustomtrace.nim new file mode 100644 index 0000000000..5e0ecfb248 --- /dev/null +++ b/tests/arc/tcustomtrace.nim @@ -0,0 +1,240 @@ +discard """ + outputsub: '''1 +2 +3 +4 +5 +6 +89 +90 +90 +0 0 1 +0 1 2 +0 2 3 +1 0 4 +1 1 5 +1 2 6 +1 3 7 +after 6 6 +MEM 0''' +joinable: false + cmd: "nim c --gc:orc -d:useMalloc $file" + valgrind: "true" +""" + +import typetraits + +type + myseq*[T] = object + len, cap: int + data: ptr UncheckedArray[T] + +# XXX make code memory safe for overflows in '*' +var + allocCount, deallocCount: int + +proc `=destroy`*[T](x: var myseq[T]) = + if x.data != nil: + when not supportsCopyMem(T): + for i in 0..= x.cap: resize(x) + result = addr(x.data[x.len]) + inc x.len + +template add*[T](x: var myseq[T]; y: T) = + reserveSlot(x)[] = y + +proc shrink*[T](x: var myseq[T]; newLen: int) = + assert newLen <= x.len + assert newLen >= 0 + when not supportsCopyMem(T): + for i in countdown(x.len - 1, newLen - 1): + `=destroy`(x.data[i]) + x.len = newLen + +proc grow*[T](x: var myseq[T]; newLen: int; value: T) = + if newLen <= x.len: return + assert newLen >= 0 + let oldCap = x.cap + if oldCap == 0: x.cap = newLen + else: x.cap = max(newLen, (oldCap * 3) shr 1) + if x.data == nil: inc allocCount + x.data = cast[type(x.data)](realloc0(x.data, oldCap * sizeof(T), x.cap * sizeof(T))) + for i in x.len.. 0'u64: + bi.digits.add cry.uint32 + +proc `$`(x: BigInt): string = + if x.digits.len == 0 or (x.digits.len == 1 and x.digits[0] == 0'u32): + return "0" + result = ""; var n = x; var msd = n.digits.high + while msd >= 0: + if n.digits[msd] == 0'u32: msd.dec; continue + var brw = 0.uint64 + for i in countdown(msd, 0): + let dvdnd = n.digits[i].uint64 + (brw shl 32) + let q = dvdnd div 10'u64; brw = dvdnd - q * 10'u64 + n.digits[i] = q.uint32 + result &= $brw + for i in 0 .. result.high shr 1: # reverse result string in place + let tmp = result[^(i + 1)] + result[^(i + 1)] = result[i] + result[i] = tmp + +type TriVal = (uint32, uint32, uint32) +type LogRep = (float64, TriVal) +type LogRepf = proc(x: LogRep): LogRep +const one: LogRep = (0.0'f64, (0'u32, 0'u32, 0'u32)) +proc `<`(me: LogRep, othr: LogRep): bool = me[0] < othr[0] + +proc convertTriVal2BigInt(tpl: TriVal): BigInt = + result = oneBigInt + let (x2, x3, x5) = tpl + for _ in 1 .. x2: result.shladd 1, zeroBigInt + for _ in 1 .. x3: result.shladd 1, result + for _ in 1 .. x5: result.shladd 2, result + +const lb2 = 1.0'f64 +const lb3 = 3.0'f64.log2 +const lb5 = 5.0'f64.log2 + +proc mul2(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb2, (x2 + 1, x3, x5)) + +proc mul3(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb3, (x2, x3 + 1, x5)) + +proc mul5(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb5, (x2, x3, x5 + 1)) + +type + LazyListObj = object + hd: LogRep + tlf: proc(): LazyList {.closure.} + tl: LazyList + LazyList = ref LazyListObj + +var destroyed = 0 + +proc `=destroy`(ll: var LazyListObj) = + destroyed += 1 + if ll.tlf == nil and ll.tl == nil: return + + when defined(trace20): + echo "destroying: ", (destroyed, ll.hd[1].convertTriVal2BigInt) + if ll.tlf != nil: ll.tlf.`=destroy` + if ll.tl != nil: ll.tl.`=destroy` + #wasMoved(ll) + +proc rest(ll: LazyList): LazyList = # not thread-safe; needs lock on thunk + if ll.tlf != nil: ll.tl = ll.tlf(); ll.tlf = nil + ll.tl + +var created = 0 +iterator hammings(until: int): TriVal = + proc merge(x, y: LazyList): LazyList = + let xh = x.hd; let yh = y.hd; created += 1 + when defined(trace20): + echo "merge create: ", (created - 1, (if xh < yh: xh else: yh)[1].convertTriVal2BigInt) + if xh < yh: LazyList(hd: xh, tlf: proc(): auto = merge x.rest, y) + else: LazyList(hd: yh, tlf: proc(): auto = merge x, y.rest) + proc smult(mltf: LogRepf; s: LazyList): LazyList = + proc smults(ss: LazyList): LazyList = + when defined(trace20): + echo "mult create: ", (created, ss.hd.mltf[1].convertTriVal2BigInt) + created += 1; LazyList(hd: ss.hd.mltf, tlf: proc(): auto = ss.rest.smults) + s.smults + proc unnsm(s: LazyList, mltf: LogRepf): LazyList = + var r: LazyList = nil + when defined(trace20): + echo "first create: ", (created, one[1].convertTriVal2BigInt) + let frst = LazyList(hd: one, tlf: proc(): LazyList = r); created += 1 + r = if s == nil: smult(mltf, frst) else: s.merge smult(mltf, frst) + r + yield one[1] + var hmpll: LazyList = ((nil.unnsm mul5).unnsm mul3).unnsm mul2 + for _ in 2 .. until: + yield hmpll.hd[1]; hmpll = hmpll.rest # almost forever + +proc main = + var s = "" + for h in hammings(20): s &= $h.convertTrival2BigInt & " " + doAssert s == "1 2 3 4 5 6 8 9 10 12 15 16 18 20 24 25 27 30 32 36 ", + "Algorithmic error finding first 20 Hamming numbers!!!" + + when not defined(trace20): + var lsth: TriVal + for h in hammings(200): lsth = h + doAssert $lsth.convertTriVal2BigInt == "16200", + "Algorithmic error finding 200th Hamming number!!!" + +let mem = getOccupiedMem() +main() +GC_FullCollect() +let mb = getOccupiedMem() - mem +doAssert mb == 0, "Found memory leak of " & $mb & " bytes!!!" + +echo getAllocStats() +echo "created ", created, " destroyed ", destroyed diff --git a/tests/arc/thard_alignment.nim b/tests/arc/thard_alignment.nim index e644572f07..30cfddb055 100644 --- a/tests/arc/thard_alignment.nim +++ b/tests/arc/thard_alignment.nim @@ -1,9 +1,11 @@ discard """ disabled: "arm64" -cmd: "nim c --gc:arc $file" +cmd: "nim c --mm:arc -u:nimPreviewNonVarDestructor $file" output: "y" """ +# TODO: fixme: investigate why it failed with non-var destructors + {.passC: "-march=native".} proc isAlignedCheck(p: pointer, alignment: int) = @@ -18,7 +20,7 @@ type proc set1(x: float): m256d {.importc: "_mm256_set1_pd", header: "immintrin.h".} func `+`(a,b: m256d): m256d {.importc: "_mm256_add_pd", header: "immintrin.h".} proc `$`(a: m256d): string = - result = $(cast[ptr float](a.unsafeAddr)[]) + result = $(cast[ptr float](a.addr)[]) var res: seq[seq[m256d]] diff --git a/tests/arc/titeration_doesnt_copy.nim b/tests/arc/titeration_doesnt_copy.nim new file mode 100644 index 0000000000..e1cdb6166c --- /dev/null +++ b/tests/arc/titeration_doesnt_copy.nim @@ -0,0 +1,56 @@ +discard """ + output: "true" +""" + +type + Idx = object + i: int + Node = object + n: int + next: seq[Idx] + FooBar = object + s: seq[Node] + +proc `=copy`(dest: var Idx; source: Idx) {.error.} +proc `=copy`(dest: var Node; source: Node) {.error.} +proc `=copy`(dest: var FooBar; source: FooBar) {.error.} + +proc doSomething(ss: var seq[int], s: FooBar) = + for i in 0 .. s.s.len-1: + for elm in items s.s[i].next: + ss.add s.s[elm.i].n + +when isMainModule: + const foo = FooBar(s: @[Node(n: 1, next: @[Idx(i: 0)])]) + var ss: seq[int] + doSomething(ss, foo) + echo ss == @[1] + +from sequtils import mapIt +from strutils import join + +proc toBinSeq*(b: uint8): seq[uint8] = + ## Return binary sequence from each bits of uint8. + runnableExamples: + from sequtils import repeat + doAssert 0'u8.toBinSeq == 0'u8.repeat(8) + doAssert 0b1010_1010.toBinSeq == @[1'u8, 0, 1, 0, 1, 0, 1, 0] + result = @[] + var c = b + for i in 1..8: + result.add (uint8(c and 0b1000_0000) shr 7) + c = c shl 1 + +proc toBinString*(data: openArray[uint8], col: int): string = + ## Return binary string from each bits of uint8. + runnableExamples: + doAssert @[0b0000_1111'u8, 0b1010_1010].toBinString(8) == "0000111110101010" + doAssert @[0b1000_0000'u8, 0b0000_0000].toBinString(1) == "10" + result = "" + for b in items data.mapIt(it.toBinSeq.mapIt(it.`$`[0].char)): + for i, c in b: + if i < col: + result.add c + +doAssert @[0b0000_1111'u8, 0b1010_1010].toBinString(8) == "0000111110101010" +doAssert @[0b1000_0000'u8, 0b0000_0000].toBinString(1) == "10" diff --git a/tests/arc/tkeys_lent.nim b/tests/arc/tkeys_lent.nim new file mode 100644 index 0000000000..2c92350b11 --- /dev/null +++ b/tests/arc/tkeys_lent.nim @@ -0,0 +1,17 @@ +discard """ + output: '''{"string": 2}''' + cmd: "nim c --gc:orc $file" +""" + +import tables + +proc use(x: int) = echo x + +proc main = + var tab = {"string": 1}.toTable + for keyAAA in tab.keys(): + template alias(): untyped = tab[keyAAA] + alias() = 2 + echo tab + +main() diff --git a/tests/arc/tmalloc.nim b/tests/arc/tmalloc.nim new file mode 100644 index 0000000000..1d72646c85 --- /dev/null +++ b/tests/arc/tmalloc.nim @@ -0,0 +1,16 @@ +discard """ + matrix: "--mm:arc -d:useMalloc; --mm:arc" +""" + +block: # bug #22058 + template foo(): auto = + {.noSideEffect.}: + newSeq[byte](1) + + type V = object + v: seq[byte] + + proc bar(): V = + V(v: foo()) + + doAssert bar().v == @[byte(0)] diff --git a/tests/arc/tmove_regression.nim b/tests/arc/tmove_regression.nim new file mode 100644 index 0000000000..7d9a867c39 --- /dev/null +++ b/tests/arc/tmove_regression.nim @@ -0,0 +1,22 @@ +discard """ + output: '''/1/2 +/1 +/ +''' +"""" + +# bug #22001 + +import std / [os, strutils] + +proc finOp2(path, name: string): (string, File) = # Find & open FIRST `name` + var current = path + while true: + if current.isRootDir: break # <- current=="" => current.isRootDir + current = current.parentDir + let dir = current + echo dir.replace('\\', '/') # Commenting out try/except below hides bug + try: result[0] = dir/name; result[1] = open(result[0]); return + except CatchableError: discard + +discard finOp2("/1/2/3", "4") # All same if this->inside a proc diff --git a/tests/arc/tmovebug.nim b/tests/arc/tmovebug.nim index 888027186b..8ad7c955ce 100644 --- a/tests/arc/tmovebug.nim +++ b/tests/arc/tmovebug.nim @@ -93,6 +93,7 @@ destroy destroy destroy sink +sink destroy copy (f: 1) @@ -107,6 +108,8 @@ sink destroy copy destroy +(f: 1) +destroy ''' """ @@ -290,7 +293,7 @@ when false: # bug #13456 -iterator combinations[T](s: openarray[T], k: int): seq[T] = +iterator combinations[T](s: openArray[T], k: int): seq[T] = let n = len(s) assert k >= 0 and k <= n var pos = newSeq[int](k) @@ -453,7 +456,7 @@ initFoo7(2) # bug #14902 -iterator zip[T](s: openarray[T]): (T, T) = +iterator zip[T](s: openArray[T]): (T, T) = var i = 0 while i < 10: yield (s[i mod 2], s[i mod 2 + 1]) @@ -765,8 +768,74 @@ proc initC(): C = C(o: initO()) proc pair(): tuple[a: C, b: C] = - result.a = initC() # <- when firstWrite tries to find this node to start its analysis it fails, because injectdestructors uses copyTree/shallowCopy - result.b = initC() + result = (a: initC(), b: initC())# <- when firstWrite tries to find this node to start its analysis it fails, because injectdestructors uses copyTree/shallowCopy discard pair() + +# bug #17450 +proc noConsume(x: OO) {.nosinks.} = echo x + +proc main3 = + var i = 1 + noConsume: + block: + OO(f: i) + +main3() + +# misc +proc smoltest(x: bool): bool = + while true: + if true: return x + +discard smoltest(true) + +# bug #18002 +type + TTypeAttachedOp = enum + attachedAsgn + attachedSink + attachedTrace + + PNode = ref object + discard + +proc genAddrOf(n: PNode) = + assert n != nil, "moved?!" + +proc atomicClosureOp = + let x = PNode() + + genAddrOf: + block: + x + + case attachedTrace + of attachedSink: discard + of attachedAsgn: discard + of attachedTrace: genAddrOf(x) + +atomicClosureOp() + + +template assertEq(a, b: untyped): untyped = + block: + let + aval = a + bval = b + + if aval != bval: + quit "bug!" + +proc convoluted = + let _ = (; + var val1: string; + if true: val1 = "22" + true + ) + + assertEq val1, "22" + assertEq val1, "22" + +convoluted() diff --git a/tests/arc/tmovebugcopy.nim b/tests/arc/tmovebugcopy.nim index 7c52281477..ec43157777 100644 --- a/tests/arc/tmovebugcopy.nim +++ b/tests/arc/tmovebugcopy.nim @@ -253,7 +253,7 @@ when false: # bug #13456 -iterator combinations[T](s: openarray[T], k: int): seq[T] = +iterator combinations[T](s: openArray[T], k: int): seq[T] = let n = len(s) assert k >= 0 and k <= n var pos = newSeq[int](k) @@ -416,7 +416,7 @@ initFoo7(2) # bug #14902 -iterator zip[T](s: openarray[T]): (T, T) = +iterator zip[T](s: openArray[T]): (T, T) = var i = 0 while i < 10: yield (s[i mod 2], s[i mod 2 + 1]) diff --git a/tests/arc/tnewseq_legacy.nim b/tests/arc/tnewseq_legacy.nim new file mode 100644 index 0000000000..4730d2c2ba --- /dev/null +++ b/tests/arc/tnewseq_legacy.nim @@ -0,0 +1,13 @@ +discard """ + output: "(allocCount: 201, deallocCount: 201)" + cmd: "nim c --gc:orc -d:nimAllocStats $file" +""" + +proc main(prefix: string) = + var c: seq[string] + for i in 0..<100: + newSeq(c, 100) + c[i] = prefix & $i + +main("abc") +echo getAllocStats() diff --git a/tests/arc/topenarray.nim b/tests/arc/topenarray.nim new file mode 100644 index 0000000000..67c512e4f8 --- /dev/null +++ b/tests/arc/topenarray.nim @@ -0,0 +1,70 @@ +discard """ + input: "hi" + output: ''' +hi +Nim +''' + matrix: "--gc:arc -d:useMalloc; --gc:arc" +""" +{.experimental: "views".} + +block: # bug 18627 + proc setPosition(params: openArray[string]) = + for i in params.toOpenArray(0, params.len - 1): + echo i + + proc uciLoop() = + let params = @[readLine(stdin)] + setPosition(params) + + uciLoop() + + proc uciLoop2() = + let params = @["Nim"] + for i in params.toOpenArray(0, params.len - 1): + echo i + uciLoop2() + +when defined(nimPreviewSlimSystem): + import std/assertions + +block: # bug #20954 + block: + doAssertRaises(IndexDefect): + var v: array[10, int] + + echo len(toOpenArray(v, 20, 30)) + + block: + doAssertRaises(IndexDefect): + var v: seq[int] + + echo len(toOpenArray(v, 20, 30)) + +# bug #20422 + +proc f(a: var string) = + var v = a.toOpenArray(1, 3) + v[0] = 'a' + +var a = "Hello" +f(a) +doAssert a == "Hallo" + +# bug #22132 +block: + func foo[T](arr: openArray[T], idx: int = arr.low): string = + doAssert idx == 0 + return $arr + + let bug = ["0", "c", "a"] + + let str = foo(bug) + + const expected = """["0", "c", "a"]""" + doAssert str == expected + + const noBugConst = ["0", "c", "a"] + doAssert foo(noBugConst) == expected + let noBugSeq = @["0", "c", "a"] + doAssert foo(noBugSeq) == expected diff --git a/tests/arc/topt_cursor.nim b/tests/arc/topt_cursor.nim index 5c35b454fb..7941329219 100644 --- a/tests/arc/topt_cursor.nim +++ b/tests/arc/topt_cursor.nim @@ -12,7 +12,7 @@ try: x_cursor = ("different", 54) else: x_cursor = ("string here", 80) echo [ - :tmpD = `$`(x_cursor) + :tmpD = `$$`(x_cursor) :tmpD] finally: `=destroy`(:tmpD) diff --git a/tests/arc/topt_no_cursor.nim b/tests/arc/topt_no_cursor.nim index f1eb8575a5..0a4984a690 100644 --- a/tests/arc/topt_no_cursor.nim +++ b/tests/arc/topt_no_cursor.nim @@ -1,34 +1,24 @@ discard """ - output: '''(repo: "", package: "meo", ext: "") -doing shady stuff... -3 -6 -(@[1], @[2]) -192.168.0.1 -192.168.0.1 -192.168.0.1 -192.168.0.1''' - cmd: '''nim c --gc:arc --expandArc:newTarget --expandArc:delete --expandArc:p1 --expandArc:tt --hint:Performance:off --assertions:off --expandArc:extractConfig $file''' - nimout: '''--expandArc: newTarget + nimoutFull: true + cmd: '''nim c -r --warnings:off --hints:off --mm:arc --expandArc:newTarget --expandArc:delete --expandArc:p1 --expandArc:tt --hint:Performance:off --assertions:off --expandArc:extractConfig --expandArc:mergeShadowScope --expandArc:check $file''' + nimout: ''' +--expandArc: newTarget var splat :tmp :tmp_1 - :tmp_2 -splat = splitFile(path) -:tmp = splat.dir -wasMoved(splat.dir) -:tmp_1 = splat.name -wasMoved(splat.name) -:tmp_2 = splat.ext -wasMoved(splat.ext) +splat = splitDrive do: + let blitTmp = path + blitTmp +:tmp = splat.drive +`=wasMoved`(splat.drive) +:tmp_1 = splat.path_1 +`=wasMoved`(splat.path_1) result = ( - let blitTmp = :tmp - blitTmp, - let blitTmp_1 = :tmp_1 + let blitTmp_1 = :tmp blitTmp_1, - let blitTmp_2 = :tmp_2 + let blitTmp_2 = :tmp_1 blitTmp_2) `=destroy`(splat) -- end of expandArc ------------------------ @@ -49,14 +39,14 @@ var lresult lvalue lnext - _ + tmpTupleAsgn lresult = @[123] -_ = ( +tmpTupleAsgn = ( let blitTmp = lresult blitTmp, ";") -lvalue = _[0] -lnext = _[1] -result.value = move lvalue +lvalue = tmpTupleAsgn[0] +lnext = tmpTupleAsgn[1] +`=sink`(result.value, move lvalue) `=destroy`(lnext) `=destroy_1`(lvalue) -- end of expandArc ------------------------ @@ -71,14 +61,12 @@ var try: it_cursor = x a = ( - wasMoved(:tmpD) - `=copy`(:tmpD, it_cursor.key) + :tmpD = `=dup`(it_cursor.key) :tmpD, - wasMoved(:tmpD_1) - `=copy`(:tmpD_1, it_cursor.val) + :tmpD_1 = `=dup`(it_cursor.val) :tmpD_1) echo [ - :tmpD_2 = `$`(a) + :tmpD_2 = `$$`(a) :tmpD_2] finally: `=destroy`(:tmpD_2) @@ -103,21 +91,85 @@ try: `=copy`(lan_ip, splitted[1]) echo [lan_ip] echo [splitted[1]] + {.push, overflowChecks: false.} inc(i, 1) + {.pop.} finally: `=destroy`(splitted) finally: `=destroy_1`(lan_ip) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +--expandArc: mergeShadowScope + +var shadowScope +`=copy`(shadowScope, c.currentScope) +rawCloseScope(c) +block :tmp: + var sym + var i = 0 + let L = len(shadowScope.symbols) + block :tmp_1: + while i < L: + var :tmpD + sym = shadowScope.symbols[i] + addInterfaceDecl(c): + :tmpD = `=dup`(sym) + :tmpD + {.push, overflowChecks: false.} + inc(i, 1) + {.pop.} +`=destroy`(shadowScope) +-- end of expandArc ------------------------ +--expandArc: check + +var par +this.isValid = fileExists(this.value) +if dirExists(this.value): + var :tmpD + par = (dir: + :tmpD = `=dup`(this.value) + :tmpD, front: "") else: + var + :tmpD_1 + :tmpD_2 + :tmpD_3 + par = (dir_1: parentDir(this.value), front_1: + :tmpD_1 = `=dup`( + :tmpD_3 = splitDrive do: + :tmpD_2 = `=dup`(this.value) + :tmpD_2 + :tmpD_3.path) + :tmpD_1) + `=destroy`(:tmpD_3) +if dirExists(par.dir): + `=sink`(this.matchDirs, getSubDirs(par.dir, par.front)) +else: + `=sink`(this.matchDirs, []) +`=destroy`(par) +-- end of expandArc ------------------------ +--expandArc: check + +check(this) +-- end of expandArc ------------------------ +(package: "", ext: "meo") +doing shady stuff... +3 +6 +(@[1], @[2]) +192.168.0.1 +192.168.0.1 +192.168.0.1 +192.168.0.1 +''' """ -import os +import os, std/private/ntpath -type Target = tuple[repo, package, ext: string] +type Target = tuple[package, ext: string] proc newTarget*(path: string): Target = - let splat = path.splitFile - result = (repo: splat.dir, package: splat.name, ext: splat.ext) + let splat = path.splitDrive + result = (package: splat.drive, ext: splat.path) echo newTarget("meo") @@ -277,3 +329,48 @@ proc extractConfig() = echo splitted[1] # Without this line everything works extractConfig() + + +type + Symbol = ref object + name: string + + Scope = ref object + parent: Scope + symbols: seq[Symbol] + + PContext = ref object + currentScope: Scope + +proc rawCloseScope(c: PContext) = + c.currentScope = c.currentScope.parent + +proc addInterfaceDecl(c: PContext; s: Symbol) = + c.currentScope.symbols.add s + +proc mergeShadowScope*(c: PContext) = + let shadowScope = c.currentScope + c.rawCloseScope + for sym in shadowScope.symbols: + c.addInterfaceDecl(sym) + +mergeShadowScope(PContext(currentScope: Scope(parent: Scope()))) + +type + Foo = ref object + isValid*: bool + value*: string + matchDirs*: seq[string] + +proc getSubDirs(parent, front: string): seq[string] = @[] + +method check(this: Foo) {.base.} = + this.isValid = fileExists(this.value) + let par = if dirExists(this.value): (dir: this.value, front: "") + else: (dir: parentDir(this.value), front: splitDrive(this.value).path) + if dirExists(par.dir): + this.matchDirs = getSubDirs(par.dir, par.front) + else: + this.matchDirs = @[] + +check(Foo()) diff --git a/tests/arc/topt_refcursors.nim b/tests/arc/topt_refcursors.nim index f097150a34..8c638a4a1e 100644 --- a/tests/arc/topt_refcursors.nim +++ b/tests/arc/topt_refcursors.nim @@ -1,26 +1,35 @@ discard """ output: '''''' cmd: '''nim c --gc:arc --expandArc:traverse --hint:Performance:off $file''' - nimout: '''--expandArc: traverse + nimout: ''' +--expandArc: traverse var it_cursor - jt_cursor -it_cursor = root -block :tmp: - while ( - not (it_cursor == nil)): - echo [it_cursor.s] - it_cursor = it_cursor.ri -jt_cursor = root -block :tmp_1: - while ( - not (jt_cursor == nil)): - var ri_1_cursor - ri_1_cursor = jt_cursor.ri - echo [jt_cursor.s] - jt_cursor = ri_1_cursor --- end of expandArc ------------------------''' + jt +try: + it_cursor = root + block :tmp: + while ( + not (it_cursor == nil)): + echo [it_cursor.s] + it_cursor = it_cursor.ri + `=copy`(jt, root) + block :tmp_1: + while ( + not (jt == nil)): + var ri_1 + try: + `=copy`(ri_1, jt.ri) + echo [jt.s] + `=sink`(jt, ri_1) + `=wasMoved`(ri_1) + finally: + `=destroy`(ri_1) +finally: + `=destroy`(jt) +-- end of expandArc ------------------------ +''' """ type diff --git a/tests/arc/topt_wasmoved_destroy_pairs.nim b/tests/arc/topt_wasmoved_destroy_pairs.nim index 2f971f1122..c96340a30d 100644 --- a/tests/arc/topt_wasmoved_destroy_pairs.nim +++ b/tests/arc/topt_wasmoved_destroy_pairs.nim @@ -1,7 +1,8 @@ discard """ output: '''''' cmd: '''nim c --gc:arc --expandArc:main --expandArc:tfor --hint:Performance:off $file''' - nimout: '''--expandArc: main + nimout: ''' +--expandArc: main var a @@ -29,6 +30,7 @@ try: x = f() block :tmp: var i_cursor + mixin inc var i_1 = 0 block :tmp_1: while i_1 < 4: @@ -37,25 +39,25 @@ try: if i_cursor == 2: return add(a): - wasMoved(:tmpD) - `=copy`(:tmpD, x) + :tmpD = `=dup`(x) :tmpD inc i_1, 1 if cond: add(a): let blitTmp = x - wasMoved(x) + `=wasMoved`(x) blitTmp else: add(b): let blitTmp_1 = x - wasMoved(x) + `=wasMoved`(x) blitTmp_1 finally: `=destroy`(x) `=destroy_1`(b) `=destroy_1`(a) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +''' """ proc f(): seq[int] = diff --git a/tests/arc/torcmisc.nim b/tests/arc/torcmisc.nim index 20dd18fb31..e41ad7c773 100644 --- a/tests/arc/torcmisc.nim +++ b/tests/arc/torcmisc.nim @@ -32,3 +32,35 @@ when true: waitFor hello(Flags()) echo "success" +# bug #18240 +import tables + +type + TopicHandler* = proc(topic: string, + data: seq[byte]): Future[void] {.gcsafe, raises: [Defect].} + + PeerID* = object + data*: seq[byte] + + PeerInfo* = ref object of RootObj + peerId*: PeerID + + Connection* = ref object of RootObj + peerInfo*: PeerInfo + + PubSubPeer* = ref object of RootObj + codec*: string + + PubSub* = ref object of RootObj + topics*: Table[string, seq[TopicHandler]] + peers*: Table[PeerID, PubSubPeer] + +proc getOrCreatePeer*(myParam: PubSub, peerId: PeerID, protos: seq[string]): PubSubPeer = + myParam.peers.withValue(peerId, peer): + return peer[] + +method handleConn*(myParam: PubSub, + conn: Connection, + proto: string) {.base, async.} = + myParam.peers.withValue(conn.peerInfo.peerId, peer): + let peerB = peer[] diff --git a/tests/arc/tref_cast_error.nim b/tests/arc/tref_cast_error.nim index b0d2faf771..20139c1be3 100644 --- a/tests/arc/tref_cast_error.nim +++ b/tests/arc/tref_cast_error.nim @@ -1,6 +1,6 @@ discard """ cmd: "nim c --gc:arc $file" - errormsg: "expression cannot be cast to ref RootObj" + errormsg: "expression cannot be cast to 'ref RootObj'" joinable: false """ diff --git a/tests/arc/trepr.nim b/tests/arc/trepr.nim index 50d433208b..abf28330a3 100644 --- a/tests/arc/trepr.nim +++ b/tests/arc/trepr.nim @@ -9,6 +9,7 @@ nil 2 Obj(member: ref @["hello"]) ref (member: ref @["hello"]) +ObjUa(v: 0, a: [...]) ''' """ @@ -87,3 +88,10 @@ macro extract(): untyped = test(parseExpr("discard")) extract() + +type + ObjUa = ref object + v: int + a: UncheckedArray[char] + +echo ObjUa().repr diff --git a/tests/arc/tshared_ptr_crash.nim b/tests/arc/tshared_ptr_crash.nim new file mode 100644 index 0000000000..1794834dbb --- /dev/null +++ b/tests/arc/tshared_ptr_crash.nim @@ -0,0 +1,67 @@ +discard """ + cmd: "nim c --threads:on --gc:arc $file" + action: compile +""" + +# bug #17893 + +type + SharedPtr*[T] = object + val: ptr tuple[value: T, atomicCounter: int] + +proc `=destroy`*[T](p: var SharedPtr[T]) = + mixin `=destroy` + if p.val != nil: + if atomicLoadN(addr p.val[].atomicCounter, AtomicConsume) == 0: + `=destroy`(p.val[]) + deallocShared(p.val) + else: + discard atomicDec(p.val[].atomicCounter) + +proc `=copy`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) = + if src.val != nil: + discard atomicInc(src.val[].atomicCounter) + if dest.val != nil: + `=destroy`(dest) + dest.val = src.val + +proc newSharedPtr*[T](val: sink T): SharedPtr[T] {.nodestroy.} = + result.val = cast[typeof(result.val)](allocShared(sizeof(result.val[]))) + result.val.atomicCounter = 0 + result.val.value = val + +proc isNil*[T](p: SharedPtr[T]): bool {.inline.} = + p.val == nil + +proc `[]`*[T](p: SharedPtr[T]): var T {.inline.} = + when compileOption("boundChecks"): + doAssert(p.val != nil, "deferencing nil shared pointer") + result = p.val.value + +type + Sender*[T] = object + queue: SharedPtr[seq[T]] + +proc newSender*[T](queue: sink SharedPtr[seq[T]]): Sender[T] = + result = Sender[T](queue: queue) + +proc send*[T](self: Sender[T]; t: sink T) = + self.queue[].add t + +proc newChannel*(): Sender[int] = + let queue = newSharedPtr(newSeq[int]()) + result = newSender(queue) + + +var + p: Thread[Sender[int]] + +proc threadFn(tx: Sender[int]) = + send tx, 0 + +proc multiThreadedChannel = + let tx = newChannel() + createThread(p, threadFn, tx) + joinThread(p) + +multiThreadedChannel() diff --git a/tests/arc/tunref_cycle.nim b/tests/arc/tunref_cycle.nim new file mode 100644 index 0000000000..82551b7f7d --- /dev/null +++ b/tests/arc/tunref_cycle.nim @@ -0,0 +1,26 @@ +discard """ + outputsub: '''inside closure +hello world''' + cmd: "nim c --gc:orc -d:useMalloc $file" + valgrind: true +""" + +# bug #18579 + +var fp: proc (env: pointer) {.cdecl.} +var env: pointer + +proc store(f: proc (){.closure.}) = + proc closeOver() = + echo "inside closure" + f() + (fp,env) = (cast[proc(env: pointer){.cdecl.}](rawProc closeOver), rawEnv closeOver) + GC_ref(cast[RootRef](env)) + +proc run() = + fp(env) + GC_unref(cast[RootRef](env)) + +store(proc() = echo "hello world") +run() +GC_fullCollect() diff --git a/tests/arc/tweave.nim b/tests/arc/tweave.nim index 220d65f97d..1c60ac4030 100644 --- a/tests/arc/tweave.nim +++ b/tests/arc/tweave.nim @@ -8,6 +8,9 @@ discard """ import std/atomics +when defined(nimPreviewSlimSystem): + import std/[assertions, typedthreads] + const MemBlockSize = 256 type diff --git a/tests/arc/twrong_sinkinference.nim b/tests/arc/twrong_sinkinference.nim index 59930a9fad..ecf09d28a7 100644 --- a/tests/arc/twrong_sinkinference.nim +++ b/tests/arc/twrong_sinkinference.nim @@ -1,6 +1,6 @@ discard """ cmd: "nim c --gc:arc $file" - errormsg: "type mismatch: got " + errormsg: "type mismatch: got " line: 18 """ diff --git a/tests/array/t15117.nim b/tests/array/t15117.nim new file mode 100644 index 0000000000..157b04beec --- /dev/null +++ b/tests/array/t15117.nim @@ -0,0 +1,27 @@ +discard """ + matrix: "--cc:vcc" + disabled: "linux" + disabled: "bsd" + disabled: "osx" + disabled: "unix" + disabled: "posix" +""" +{.experimental: "views".} + +let a: array[0, byte] = [] +discard a + +type B = object + a:int +let b: array[0, B] = [] +let c: array[0, ptr B] = [] +let d: array[0, ref B] = [] +discard b +discard c +discard d + +discard default(array[0, B]) + +type + View1 = openArray[byte] +discard default(View1) diff --git a/tests/array/t20248.nim b/tests/array/t20248.nim new file mode 100644 index 0000000000..66142548b9 --- /dev/null +++ b/tests/array/t20248.nim @@ -0,0 +1,14 @@ +discard """ +cmd: "nim check --hints:off $file" +errormsg: "ordinal type expected; given: Error Type" +nimout: ''' +t20248.nim(10, 36) Error: ordinal type expected; given: Error Type +t20248.nim(14, 20) Error: ordinal type expected; given: Error Type +''' +""" + +type Vec[N: static[int]] = array[0 ..< N, float] + +var v: Vec[32] + +var stuff: array[0 ..< 16, int] diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index 81a43f203e..e9f330e3be 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -44,7 +44,7 @@ block tarray: arr: TMyarray - proc sum(a: openarray[int]): int = + proc sum(a: openArray[int]): int = result = 0 var i = 0 while i < len(a): @@ -587,3 +587,21 @@ block t12466: a[0'u16 + i] = i for i in 0'u16 ..< 8'u16: a[0'u16 + i] = i + +block t17705: + # https://github.com/nim-lang/Nim/pull/17705 + var a = array[0, int].low + a = int(a) + var b = array[0, int].high + b = int(b) + +block t18643: + # https://github.com/nim-lang/Nim/issues/18643 + let a: array[0, int] = [] + var caught = false + let b = 9999999 + try: + echo a[b] + except IndexDefect: + caught = true + doAssert caught, "IndexDefect not caught!" diff --git a/tests/misc/tinvalidarrayaccess.nim b/tests/array/tinvalidarrayaccess.nim similarity index 100% rename from tests/misc/tinvalidarrayaccess.nim rename to tests/array/tinvalidarrayaccess.nim diff --git a/tests/misc/tinvalidarrayaccess2.nim b/tests/array/tinvalidarrayaccess2.nim similarity index 100% rename from tests/misc/tinvalidarrayaccess2.nim rename to tests/array/tinvalidarrayaccess2.nim diff --git a/tests/array/tlargeindex.nim b/tests/array/tlargeindex.nim new file mode 100644 index 0000000000..61bcbd61d5 --- /dev/null +++ b/tests/array/tlargeindex.nim @@ -0,0 +1,18 @@ +discard """ + cmd: "nim check --hints:off $file" +""" + +# issue #17163 +var e: array[int32, byte] #[tt.Error + ^ index type 'int32' for array is too large]# +var f: array[uint32, byte] #[tt.Error + ^ index type 'uint32' for array is too large]# +var g: array[int64, byte] #[tt.Error + ^ index type 'int64' for array is too large]# +var h: array[uint64, byte] #[tt.Error + ^ index type 'uint64' for array is too large]# + +# crash in issue #23204 +proc y[N](): array[N, int] = default(array[N, int]) #[tt.Error + ^ index type 'int' for array is too large]# +discard y[int]() diff --git a/tests/assert/panicoverride.nim b/tests/assert/panicoverride.nim new file mode 100644 index 0000000000..53ad64215b --- /dev/null +++ b/tests/assert/panicoverride.nim @@ -0,0 +1,15 @@ +# panicoverride.nim + +proc printf(fmt: cstring) {.varargs, importc, header:"stdio.h".} +proc exit(code: cint) {.importc, header:"stdlib.h".} + +{.push stack_trace: off, profiler:off.} + +proc rawoutput(s: cstring) = + printf("RAW: %s\n", s) + +proc panic(s: cstring) {.noreturn.} = + printf("PANIC: %s\n", s) + exit(0) + +{.pop.} \ No newline at end of file diff --git a/tests/assert/t21195.nim b/tests/assert/t21195.nim new file mode 100644 index 0000000000..b690d22a0f --- /dev/null +++ b/tests/assert/t21195.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--verbosity:0 --os:standalone --mm:none" +""" +# bug #21195 +var n = 11 +assert(n == 12) diff --git a/tests/assert/tassert2.nim b/tests/assert/tassert2.nim index 5d849aaad7..e32ab72e1e 100644 --- a/tests/assert/tassert2.nim +++ b/tests/assert/tassert2.nim @@ -24,7 +24,7 @@ except AssertionDefect as e: try: assert false # assert test with no msg except AssertionDefect as e: - assert e.msg.endsWith "tassert2.nim(25, 10) `false` " + assert e.msg.endsWith "tassert2.nim(25, 3) `false` " try: let a = 1 @@ -100,7 +100,7 @@ block: ## checks for issue https://github.com/nim-lang/Nim/issues/9301 doAssert 1 + 1 == 3 except AssertionDefect as e: # used to const fold as false - assert e.msg.endsWith "tassert2.nim(100, 14) `1 + 1 == 3` " + assert e.msg.endsWith "tassert2.nim(100, 5) `1 + 1 == 3` " block: ## checks AST isn't transformed as it used to let a = 1 @@ -108,4 +108,4 @@ block: ## checks AST isn't transformed as it used to doAssert a > 1 except AssertionDefect as e: # used to rewrite as `1 < a` - assert e.msg.endsWith "tassert2.nim(108, 14) `a > 1` " + assert e.msg.endsWith "tassert2.nim(108, 5) `a > 1` " diff --git a/tests/assert/tassert_c.nim b/tests/assert/tassert_c.nim index 5c8f529adf..e3e3b81479 100644 --- a/tests/assert/tassert_c.nim +++ b/tests/assert/tassert_c.nim @@ -1,14 +1,14 @@ discard """ - cmd: "nim $target $options --excessiveStackTrace:off $file" + matrix: "-d:nimPreviewSlimSystem --stackTrace:on --excessiveStackTrace:off" output: '''true''' """ - +import std/assertions const expected = """ tassert_c.nim(35) tassert_c tassert_c.nim(34) foo assertions.nim(*) failedAssertImpl assertions.nim(*) raiseAssert -fatal.nim(*) sysFatal""" +""" proc tmatch(x, p: string): bool = var i = 0 diff --git a/tests/assign/moverload_asgn2.nim b/tests/assign/moverload_asgn2.nim index 6620adbeb9..cfea48cd1c 100644 --- a/tests/assign/moverload_asgn2.nim +++ b/tests/assign/moverload_asgn2.nim @@ -1,3 +1,7 @@ +discard """ + matrix: "--mm:refc" +""" + type Concrete* = object a*, b*: string diff --git a/tests/assign/tassign.nim b/tests/assign/tassign.nim index da097ca83f..fdec04d226 100644 --- a/tests/assign/tassign.nim +++ b/tests/assign/tassign.nim @@ -80,7 +80,7 @@ block tcopy: block tgenericassign: type - TAny = object {.pure.} + TAny {.pure.} = object value: pointer rawType: pointer diff --git a/tests/assign/tvariantasgn.nim b/tests/assign/tvariantasgn.nim index 1c3deeae15..4c3c38ca58 100644 --- a/tests/assign/tvariantasgn.nim +++ b/tests/assign/tvariantasgn.nim @@ -24,3 +24,16 @@ s = TAny(kind: nkInt, intVal: 78) # s = nr # works nr = s # fails! echo "came here" + +block: # bug #12464 + type + Foo = object + case isFunc: bool + of false: nil + of true: + fun: proc(): int + + const i = Foo(isFunc: false) + + let j = i + doAssert not j.isFunc diff --git a/tests/ast_pattern_matching.nim b/tests/ast_pattern_matching.nim index c08234b9e3..0ba0aa57a8 100644 --- a/tests/ast_pattern_matching.nim +++ b/tests/ast_pattern_matching.nim @@ -133,8 +133,8 @@ proc matchLengthKind*(arg: NimNode; kind: NimNodeKind; length: int): MatchingErr matchLengthKind(arg, {kind}, length) proc matchValue(arg: NimNode; kind: set[NimNodeKind]; value: SomeInteger): MatchingError {.compileTime.} = - let kindFail = not(kind.card == 0 or arg.kind in kind) - let valueFail = arg.intVal != int(value) + template kindFail: bool = not(kind.card == 0 or arg.kind in kind) + template valueFail: bool = arg.intVal != int(value) if kindFail or valueFail: result.node = arg result.kind = WrongKindValue @@ -561,7 +561,7 @@ when isMainModule: echo "got the ident m" testRecCase: - type Obj[T] = object {.inheritable.} + type Obj[T] {.inheritable.} = object name: string case isFat: bool of true: diff --git a/tests/astspec/tastspec.nim b/tests/astspec/tastspec.nim index 3413d32f33..c99d8ec792 100644 --- a/tests/astspec/tastspec.nim +++ b/tests/astspec/tastspec.nim @@ -327,19 +327,43 @@ static: testArrayAccessOperator(x[y]) - - ## Parentheses scope: - let ast = myquote: - (1, 2, (3)) + (a + b) * c ast.matchAst: - of nnkPar(nnkIntLit(intVal = 1), nnkIntLit(intVal = 2), nnkPar(nnkIntLit(intVal = 3))): - echo "ok" + of nnkInfix(ident"*", nnkPar(nnkInfix(ident"+", ident"a", ident"b")), ident"c"): + echo "parentheses ok" + ## Tuple Constructors + + scope: + let ast = myquote: + (1, 2, 3) + (a: 1, b: 2, c: 3) + (1,) + (a: 1) + () + + for it in ast: + echo it.lispRepr + it.matchAst: + of nnkTupleConstr(nnkIntLit(intVal = 1), nnkIntLit(intVal = 2), nnkIntLit(intVal = 3)): + echo "simple tuple ok" + of nnkTupleConstr( + nnkExprColonExpr(ident"a", nnkIntLit(intVal = 1)), + nnkExprColonExpr(ident"b", nnkIntLit(intVal = 2)), + nnkExprColonExpr(ident"c", nnkIntLit(intVal = 3)) + ): + echo "named tuple ok" + of nnkTupleConstr(nnkIntLit(intVal = 1)): + echo "one tuple ok" + of nnkTupleConstr(nnkExprColonExpr(ident"a", nnkIntLit(intVal = 1))): + echo "named one tuple ok" + of nnkTupleConstr(): + echo "empty tuple ok" ## Curly braces @@ -847,11 +871,20 @@ static: scope: macro testRecCase(ast: untyped): untyped = - ast.peelOff({nnkStmtList, nnkTypeSection})[2].matchAst: - of nnkObjectTy( - nnkPragma( - ident"inheritable" + ast.peelOff({nnkStmtList, nnkTypeSection}).matchAst: + of nnkTypeDef( + nnkPragmaExpr( + ident"Obj", + nnkPragma(ident"inheritable") ), + nnkGenericParams( + nnkIdentDefs( + ident"T", + nnkEmpty(), + nnkEmpty()) + ), + nnkObjectTy( + nnkEmpty(), nnkEmpty(), nnkRecList( # list of object parameters nnkIdentDefs( @@ -890,6 +923,7 @@ static: ident"T" ), nnkEmpty() + ) ) ) ) @@ -898,10 +932,8 @@ static: ): echo "ok" - - testRecCase: - type Obj[T] = object {.inheritable.} + type Obj[T] {.inheritable.} = object name: string case isFat: bool of true: @@ -929,7 +961,7 @@ static: (x & y & z) is string ast.peelOff({nnkStmtList, nnkTypeSection}).matchAst: - of nnkTypeDef(_, _, nnkTypeClassTy(nnkArglist, _, _, nnkStmtList)): + of nnkTypeDef(_, _, nnkTypeClassTy(nnkArgList, _, _, nnkStmtList)): # note this isn't nnkConceptTy! echo "ok" diff --git a/tests/async/nim.cfg b/tests/async/nim.cfg new file mode 100644 index 0000000000..be50f572c3 --- /dev/null +++ b/tests/async/nim.cfg @@ -0,0 +1 @@ +--experimental:strictEffects diff --git a/tests/async/t11558.nim b/tests/async/t11558.nim new file mode 100644 index 0000000000..99961e7b6d --- /dev/null +++ b/tests/async/t11558.nim @@ -0,0 +1,13 @@ +discard """ +output: "Hello\n9" +""" +import std/asyncdispatch + +proc foo(): Future[string] {.async.} = + "Hello" + +proc bar(): Future[int] {.async.} = + result = 9 + +echo waitFor foo() +echo waitFor bar() diff --git a/tests/async/t17045.nim b/tests/async/t17045.nim new file mode 100644 index 0000000000..2b5acf48a4 --- /dev/null +++ b/tests/async/t17045.nim @@ -0,0 +1,28 @@ +discard """ + targets: "c cpp" + matrix: "--mm:refc; --mm:arc" +""" + +type Future = ref object + +iterator paths: string = + # without "when nimvm" everything works + when nimvm: + yield "test.md" + else: + yield "test.md" + +template await(f: Future): string = + # need this yield, also the template has to return something + yield f + "hello world" + +proc generatePostContextsAsync() = + iterator generatePostContextsAsyncIter(): Future {.closure.} = + for filePath in paths(): + var temp = await Future() + + # need this line + var nameIterVar = generatePostContextsAsyncIter + +generatePostContextsAsync() \ No newline at end of file diff --git a/tests/async/t20111.nim b/tests/async/t20111.nim new file mode 100644 index 0000000000..0aaa7d8864 --- /dev/null +++ b/tests/async/t20111.nim @@ -0,0 +1,19 @@ +discard """ + action: "run" +""" +import asyncdispatch +type + Sync = object + Async = object + SyncRes = (Sync, string) + AsyncRes = (Async, string) + +proc foo(val: Sync | Async): Future[(Async, string) | (Sync, string)] {.multisync.} = + return (val, "hello") + +let + myAsync = Async() + mySync = Sync() + +doAssert typeof(waitFor foo(myAsync)) is AsyncRes +doAssert typeof(foo(mySync)) is SyncRes diff --git a/tests/async/t21447.nim b/tests/async/t21447.nim new file mode 100644 index 0000000000..e4f7ae31f6 --- /dev/null +++ b/tests/async/t21447.nim @@ -0,0 +1,6 @@ +discard """ + action: "compile" + cmd: "nim c -d:release -d:futureLogging $file" +""" + +import std/asyncdispatch diff --git a/tests/async/t21893.nim b/tests/async/t21893.nim new file mode 100644 index 0000000000..658cb02ebc --- /dev/null +++ b/tests/async/t21893.nim @@ -0,0 +1,13 @@ +discard """ +output: "@[97]\ntrue" +""" + +import asyncdispatch + +proc test(): Future[bool] {.async.} = + const S4 = @[byte('a')] + echo S4 + return true + +echo waitFor test() + diff --git a/tests/async/t22210.nim b/tests/async/t22210.nim new file mode 100644 index 0000000000..fcf9394725 --- /dev/null +++ b/tests/async/t22210.nim @@ -0,0 +1,41 @@ +discard """ +output: ''' +stage 1 +stage 2 +stage 3 +(status: 200, data: "SOMEDATA") +''' +""" + +import std/asyncdispatch + + +# bug #22210 +type + ClientResponse = object + status*: int + data*: string + +proc subFoo1(): Future[int] {.async.} = + await sleepAsync(100) + return 200 + +proc subFoo2(): Future[string] {.async.} = + await sleepAsync(100) + return "SOMEDATA" + +proc testFoo(): Future[ClientResponse] {.async.} = + try: + let status = await subFoo1() + doAssert(status == 200) + let data = await subFoo2() + return ClientResponse(status: status, data: data) + finally: + echo "stage 1" + await sleepAsync(100) + echo "stage 2" + await sleepAsync(200) + echo "stage 3" + +when isMainModule: + echo waitFor testFoo() \ No newline at end of file diff --git a/tests/async/t22210_2.nim b/tests/async/t22210_2.nim new file mode 100644 index 0000000000..9db664a32d --- /dev/null +++ b/tests/async/t22210_2.nim @@ -0,0 +1,73 @@ +import std/asyncdispatch + + +# bug #22210 +type + ClientResponse = object + status*: int + data*: string + +proc subFoo1(): Future[int] {.async.} = + await sleepAsync(100) + return 200 + +proc subFoo2(): Future[string] {.async.} = + await sleepAsync(100) + return "SOMEDATA" + + +proc testFoo2(): Future[ClientResponse] {.async.} = + var flag = 0 + try: + let status = await subFoo1() + doAssert(status == 200) + let data = await subFoo2() + result = ClientResponse(status: status, data: data) + finally: + inc flag + await sleepAsync(100) + inc flag + await sleepAsync(200) + inc flag + doAssert flag == 3 + +discard waitFor testFoo2() + +proc testFoo3(): Future[ClientResponse] {.async.} = + var flag = 0 + try: + let status = await subFoo1() + doAssert(status == 200) + let data = await subFoo2() + if false: + return ClientResponse(status: status, data: data) + finally: + inc flag + await sleepAsync(100) + inc flag + await sleepAsync(200) + inc flag + doAssert flag == 3 + +discard waitFor testFoo3() + + +proc testFoo4(): Future[ClientResponse] {.async.} = + var flag = 0 + try: + let status = await subFoo1() + doAssert(status == 200) + let data = await subFoo2() + if status == 200: + return ClientResponse(status: status, data: data) + else: + return ClientResponse() + finally: + inc flag + await sleepAsync(100) + inc flag + await sleepAsync(200) + inc flag + doAssert flag == 3 + +discard waitFor testFoo4() diff --git a/tests/async/tasync_gcunsafe.nim b/tests/async/tasync_gcunsafe.nim index 00c92b109d..f3e6bc6918 100644 --- a/tests/async/tasync_gcunsafe.nim +++ b/tests/async/tasync_gcunsafe.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "'anotherGCSafeAsyncProcIter' is not GC-safe as it calls 'asyncGCUnsafeProc'" + errormsg: "'anotherGCSafeAsyncProc (Async)' is not GC-safe as it calls 'asyncGCUnsafeProc'" cmd: "nim c --threads:on $file" file: "asyncmacro.nim" """ diff --git a/tests/async/tasync_noasync.nim b/tests/async/tasync_noasync.nim index 54f4f597ff..0927148bf0 100644 --- a/tests/async/tasync_noasync.nim +++ b/tests/async/tasync_noasync.nim @@ -1,13 +1,42 @@ discard """ - errormsg: "'yield' only allowed in an iterator" - cmd: "nim c $file" - file: "asyncmacro.nim" + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout: ''' +tasync_noasync.nim(21, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(25, 12) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(28, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(31, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(35, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(38, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(40, 8) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +''' """ import async proc a {.async.} = discard +# Bad await usage +proc nonAsyncProc = + await a() + +proc nestedNonAsyncProc {.async.} = + proc nested = + await a() + +iterator customIterator: int = + await a() + +macro awaitInMacro = + await a() + +type DummyRef = ref object of RootObj +method awaitInMethod(_: DummyRef) {.base.} = + await a() + +proc improperMultisync {.multisync.} = + await a() + await a() # if we overload a fallback handler to get diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index 9f787929b3..98f71b1924 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -1,6 +1,5 @@ discard """ exitcode: 0 - disabled: "windows" output: "Matched" """ import asyncdispatch, strutils @@ -68,53 +67,24 @@ import re const expected = """ b failure Async traceback: - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncmacro\.nim\(\d+?\)\s+?a - asyncmacro\.nim\(\d+?\)\s+?aNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?aIter - asyncmacro\.nim\(\d+?\)\s+?b - asyncmacro\.nim\(\d+?\)\s+?bNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?bIter - #\[ - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncmacro\.nim\(\d+?\)\s+?a - asyncmacro\.nim\(\d+?\)\s+?aNimAsyncContinue - ## Resumes an async procedure - asyncmacro\.nim\(\d+?\)\s+?aIter - asyncfutures\.nim\(\d+?\)\s+?read - \]# + tasync_traceback\.nim\(\d+?\) tasync_traceback + tasync_traceback\.nim\(\d+?\) a \(Async\) + tasync_traceback\.nim\(\d+?\) b \(Async\) Exception message: b failure -Exception type: + bar failure Async traceback: - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncdispatch\.nim\(\d+?\)\s+?waitFor - asyncdispatch\.nim\(\d+?\)\s+?poll + tasync_traceback\.nim\(\d+?\) tasync_traceback + asyncdispatch\.nim\(\d+?\) waitFor + asyncdispatch\.nim\(\d+?\) poll ## Processes asynchronous completion events - asyncdispatch\.nim\(\d+?\)\s+?runOnce - asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks + asyncdispatch\.nim\(\d+?\) runOnce + asyncdispatch\.nim\(\d+?\) processPendingCallbacks ## Executes pending callbacks - asyncmacro\.nim\(\d+?\)\s+?barNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?barIter - #\[ - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncdispatch\.nim\(\d+?\)\s+?waitFor - asyncdispatch\.nim\(\d+?\)\s+?poll - ## Processes asynchronous completion events - asyncdispatch\.nim\(\d+?\)\s+?runOnce - asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks - ## Executes pending callbacks - asyncmacro\.nim\(\d+?\)\s+?fooNimAsyncContinue - ## Resumes an async procedure - asyncmacro\.nim\(\d+?\)\s+?fooIter - asyncfutures\.nim\(\d+?\)\s+?read - \]# + tasync_traceback\.nim\(\d+?\) bar \(Async\) Exception message: bar failure -Exception type: + """ # TODO: is asyncmacro good enough location for fooIter traceback/debugging? just put the callsite info for all? @@ -122,28 +92,31 @@ Exception type: let resLines = splitLines(result.strip) let expLines = splitLines(expected.strip) -if resLines.len != expLines.len: - echo("Not matched! Wrong number of lines!") - echo expLines.len - echo resLines.len - echo("Expected: -----------") - echo expected - echo("Gotten: -------------") - echo result - echo("---------------------") - quit(QuitFailure) +when not defined(cpp): # todo fixme + if resLines.len != expLines.len: + echo("Not matched! Wrong number of lines!") + echo expLines.len + echo resLines.len + echo("Expected: -----------") + echo expected + echo("Gotten: -------------") + echo result + echo("---------------------") + quit(QuitFailure) -var ok = true -for i in 0 ..< resLines.len: - if not resLines[i].match(re(expLines[i])): - echo "Not matched! Line ", i + 1 - echo "Expected:" - echo expLines[i] - echo "Actual:" - echo resLines[i] - ok = false + var ok = true + for i in 0 ..< resLines.len: + if not resLines[i].match(re(expLines[i])): + echo "Not matched! Line ", i + 1 + echo "Expected:" + echo expLines[i] + echo "Actual:" + echo resLines[i] + ok = false -if ok: - echo("Matched") + if ok: + echo("Matched") + else: + quit(QuitFailure) else: - quit(QuitFailure) + echo("Matched") diff --git a/tests/async/tasyncawait.nim b/tests/async/tasyncawait.nim index f658a15ed1..e86542b2d3 100644 --- a/tests/async/tasyncawait.nim +++ b/tests/async/tasyncawait.nim @@ -34,7 +34,7 @@ proc readMessages(client: AsyncFD) {.async.} = clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tests/async/tasyncdial.nim b/tests/async/tasyncdial.nim index a755de74e6..f0377dfd53 100644 --- a/tests/async/tasyncdial.nim +++ b/tests/async/tasyncdial.nim @@ -3,7 +3,6 @@ discard """ OK AF_INET OK AF_INET6 ''' - disabled: "travis" """ import diff --git a/tests/async/tasyncintemplate.nim b/tests/async/tasyncintemplate.nim new file mode 100644 index 0000000000..4bddb1d185 --- /dev/null +++ b/tests/async/tasyncintemplate.nim @@ -0,0 +1,62 @@ +discard """ + output: ''' +42 +43 +43 +1 +2 +3 +4 +''' +""" + +# xxx move to tests/async/tasyncintemplate.nim +import asyncdispatch + +block: # bug #16159 + template foo() = + proc temp(): Future[int] {.async.} = return 42 + proc tempVoid(): Future[void] {.async.} = echo await temp() + foo() + waitFor tempVoid() + +block: # aliasing `void` + template foo() = + type Foo = void + proc temp(): Future[int] {.async.} = return 43 + proc tempVoid(): Future[Foo] {.async.} = echo await temp() + proc tempVoid2() {.async.} = echo await temp() + foo() + waitFor tempVoid() + waitFor tempVoid2() + +block: # sanity check + template foo() = + proc bad(): int {.async.} = discard + doAssert not compiles(bad()) + +block: # bug #16786 + block: + proc main(a: int|string)= + proc bar(b: int|string) = echo b + bar(a) + main(1) + + block: + proc main(a: int) : Future[void] {.async.} = + proc bar(b: int): Future[void] {.async.} = echo b + await bar(a) + waitFor main(2) + + block: + proc main(a: int) : Future[void] {.async.} = + proc bar(b: int | string): Future[void] {.async.} = echo b + await bar(a) + waitFor main(3) + + block: + # bug + proc main(a: int|string) = + proc bar(b: int): Future[void] {.async.} = echo b + waitFor bar(a) + main(4) diff --git a/tests/async/tasyncnetudp.nim b/tests/async/tasyncnetudp.nim index ef6dfc5e19..dade96fb20 100644 --- a/tests/async/tasyncnetudp.nim +++ b/tests/async/tasyncnetudp.nim @@ -59,7 +59,7 @@ proc readMessages(server: AsyncSocket) {.async.} = while i < maxResponses: let (data, fromIp, fromPort) = await recvFrom(server, 16384) - if data.startswith("Message ") and fromIp == "127.0.0.1": + if data.startsWith("Message ") and fromIp == "127.0.0.1": await sendTo(server, fromIp, fromPort, data) inc(msgCount) diff --git a/tests/async/tasyncssl.nim b/tests/async/tasyncssl.nim index a582818eb3..57de3271d5 100644 --- a/tests/async/tasyncssl.nim +++ b/tests/async/tasyncssl.nim @@ -1,5 +1,6 @@ discard """ cmd: "nim $target --hints:on --define:ssl $options $file" + disabled: osx """ import asyncdispatch, asyncnet, net, strutils @@ -36,7 +37,7 @@ when defined(ssl): inc(clientCount) break else: - if line.startswith("Message "): + if line.startsWith("Message "): inc(msgCount) else: doAssert false diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index f7487525af..77d03f8f61 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -234,7 +234,7 @@ when not defined(windows): if fd == -1: raiseOsError(osLastError()) let length = len(data).cint - if posix.write(fd, cast[pointer](unsafeAddr data[0]), + if posix.write(fd, cast[pointer](addr data[0]), len(data).cint) != length: raiseOsError(osLastError()) if posix.close(fd) == -1: @@ -288,8 +288,8 @@ when not defined(windows): events: set[Event] proc vnode_test(): bool = - proc validate(test: openarray[ReadyKey], - check: openarray[valType]): bool = + proc validate(test: openArray[ReadyKey], + check: openArray[valType]): bool = result = false if len(test) == len(check): for checkItem in check: diff --git a/tests/async/tnewasyncudp.nim b/tests/async/tnewasyncudp.nim index 76462c21de..68de796a02 100644 --- a/tests/async/tnewasyncudp.nim +++ b/tests/async/tnewasyncudp.nim @@ -58,7 +58,7 @@ proc launchSwarm(name: ptr SockAddr) {.async.} = 16384, cast[ptr SockAddr](addr saddr), addr slen) size = 0 - var grammString = $cstring(addr buffer) + var grammString = $cast[cstring](addr buffer) if grammString == message: saveSendingPort(sockport) inc(recvCount) @@ -80,8 +80,8 @@ proc readMessages(server: AsyncFD) {.async.} = 16384, cast[ptr SockAddr](addr(saddr)), addr(slen)) size = 0 - var grammString = $cstring(addr buffer) - if grammString.startswith("Message ") and + var grammString = $cast[cstring](addr buffer) + if grammString.startsWith("Message ") and saddr.sin_addr.s_addr == nativesockets.ntohl(INADDR_LOOPBACK.uint32): await sendTo(server, addr grammString[0], len(grammString), cast[ptr SockAddr](addr saddr), slen) diff --git a/tests/async/twinasyncrw.nim b/tests/async/twinasyncrw.nim index 69b092607e..f0a8f6a62d 100644 --- a/tests/async/twinasyncrw.nim +++ b/tests/async/twinasyncrw.nim @@ -19,7 +19,7 @@ when defined(windows): retFuture.complete() return true else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true var aiList = getAddrInfo(address, port, domain) @@ -45,7 +45,7 @@ when defined(windows): freeAddrInfo(aiList) if not success: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) return retFuture proc winRecv*(socket: AsyncFD, size: int, @@ -63,7 +63,7 @@ when defined(windows): if flags.isDisconnectionError(lastError): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) elif res == 0: # Disconnected retFuture.complete("") @@ -88,7 +88,7 @@ when defined(windows): if flags.isDisconnectionError(lastError): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: retFuture.complete(res) # TODO: The following causes a massive slowdown. @@ -112,7 +112,7 @@ when defined(windows): if flags.isDisconnectionError(lastError): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: written.inc(res) if res != netSize: @@ -136,7 +136,7 @@ when defined(windows): var client = nativesockets.accept(sock.SocketHandle, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) if client == osInvalidSocket: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: retFuture.complete((getAddrString(cast[ptr SockAddr](addr sockAddress)), client.AsyncFD)) @@ -220,7 +220,7 @@ when defined(windows): clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tests/avr/nim.cfg b/tests/avr/nim.cfg new file mode 100644 index 0000000000..d6eba8eda9 --- /dev/null +++ b/tests/avr/nim.cfg @@ -0,0 +1,12 @@ +avr.standalone.gcc.path = "/usr/bin" +avr.standalone.gcc.exe = "avr-gcc" +avr.standalone.gcc.linkerexe = "avr-gcc" +passC = "-Os" +passC = "-DF_CPU=16000000UL" +passC = "-mmcu=atmega328p" +passL = "-mmcu=atmega328p" +passC = "-flto" +passL = "-flto" +cpu = "avr" +deadCodeElim = "on" +gc = "arc" diff --git a/tests/avr/panicoverride.nim b/tests/avr/panicoverride.nim new file mode 100644 index 0000000000..770933ddd4 --- /dev/null +++ b/tests/avr/panicoverride.nim @@ -0,0 +1,13 @@ +proc printf(frmt: cstring) {.varargs, importc, header: "", cdecl.} +proc exit(code: int) {.importc, header: "", cdecl.} + +{.push stack_trace: off, profiler:off.} + +proc rawoutput(s: string) = + printf("%s\n", s) + +proc panic(s: string) = + rawoutput(s) + exit(1) + +{.pop.} diff --git a/tests/avr/thello.nim b/tests/avr/thello.nim new file mode 100644 index 0000000000..7ebaeae5fd --- /dev/null +++ b/tests/avr/thello.nim @@ -0,0 +1,6 @@ +discard """ + cmd: "nim c --compileOnly --os:standalone --exceptions:quirky -d:noSignalHandler -d:danger --threads:off $file" + action: "compile" +""" + +echo "hi" diff --git a/tests/bind/tdatabind.nim b/tests/bind/tdatabind.nim deleted file mode 100644 index f6455749c9..0000000000 --- a/tests/bind/tdatabind.nim +++ /dev/null @@ -1,95 +0,0 @@ -discard """ - disabled: true -""" - -import events -type - TProperty*[T] = object of TObject - getProc: proc(property: TProperty[T]): T {.nimcall.} - setProc: proc(property: var TProperty[T], value: T) {.nimcall.} - value: T - ValueChanged*: TEventHandler - Binders: seq[TProperty[T]] - EEmitter: TEventEmitter - # Not a descriptive name but it was that or TPropertyValueChangeEventArgs. - TValueEventArgs[T] = object of TEventArgs - Property*: TProperty[T] - - -proc newProperty*[T](value: T): TProperty[T] = - var prop: TProperty[T] - - prop.EEmitter = initEventEmitter() - prop.Binders = @[] - prop.ValueChanged = initEventHandler("ValueChanged") - prop.value = value - - proc getter(property: TProperty[T]): T = - return property.value - - prop.getProc = getter - - proc setter(property: var TProperty[T], v: T) = - property.value = v - - # fire event here - var args: TValueEventArgs[T] - args.Property = property - property.EEmitter.emit(property.ValueChanged, args) - - prop.setProc = setter - - return prop - -proc `prop`[T] (p: TProperty[T]): T = - # I'm assuming this is trying to get a value from the property. - # i.e. myVar = myProperty - return p.getProc(p) - -proc `~=`[T] (p: var TProperty[T], v: T) = - # Assuming this is setting the value. - p.setProc(p, v) - -proc `$`[T] (p: TProperty[T]): string = - var value = p.getProc(p) - return $value - -proc propertyBind*[T](p1: var TProperty[T], p2: var TProperty[T]) = - p1.Binders.add(p2) - - # make handler -> handler[T] so trigger even more generics bugs ... - proc handler(e: TEventArgs) = - type TEA = TValueEventArgs[T] - var args = TEA(e) - var val = args.Property.getProc(p1) - for i in countup(0, len(e.Property.ValueChanged.Binders) -1): - var binded = e.Property.ValueChanged.Binders[i] - binded.setProc(binded, val) - - echo("Property 1 has changed to " & $val) - - if p1.ValueChanged.containsHandler(handler) == false: - addHandler(p1.ValueChanged, handler) - -proc `->`[T](p1: var TProperty[T], p2: var TProperty[T]) = - propertyBind(p2,p1) - -when true: - # Initial value testing - var myProp = newProperty(5) - - echo(myProp) - - myProp ~= 7 # Temp operator until overloading of '=' is implemented. - echo(myProp) - - # Binding testing - - var prop1 = newProperty(9) - var prop2: TProperty[int] - - prop2 -> prop1 # Binds prop2 to prop1 - - prop1 ~= 7 - echo(prop2) # Output: 7 - diff --git a/tests/borrow/tborrow.nim b/tests/borrow/tborrow.nim deleted file mode 100644 index 9c403021ed..0000000000 --- a/tests/borrow/tborrow.nim +++ /dev/null @@ -1,35 +0,0 @@ -discard """ - output: '''4887 true -0.5''' -""" - -# test the new borrow feature that works with generics: - -proc `++`*[T: int | float](a, b: T): T = - result = a + b - -type - DI = distinct int - DF = distinct float - DS = distinct string - -proc `++`(x, y: DI): DI {.borrow.} -proc `++`(x, y: DF): DF {.borrow.} - -proc `$`(x: DI): string {.borrow.} -proc `$`(x: DF): string {.borrow.} - -echo 4544.DI ++ 343.DI, " ", (4.5.DF ++ 0.5.DF).float == 5.0 - -# issue #14440 - -type Radians = distinct float64 - -func `-=`(a: var Radians, b: Radians) {.borrow.} - -var a = Radians(1.5) -let b = Radians(1.0) - -a -= b - -echo a.float64 diff --git a/tests/borrow/tinvalidborrow.nim b/tests/borrow/tinvalidborrow.nim deleted file mode 100644 index 89aa4e2e86..0000000000 --- a/tests/borrow/tinvalidborrow.nim +++ /dev/null @@ -1,16 +0,0 @@ -discard """ - errormsg: "no symbol to borrow from found" - line: 11 -""" - -# bug #516 - -type - TAtom = culong - -proc `==`*(a, b: TAtom): bool {.borrow.} - -var - d, e: TAtom - -echo( $(d == e) ) diff --git a/tests/compilepragma/test.c b/tests/c/tcompile.c similarity index 100% rename from tests/compilepragma/test.c rename to tests/c/tcompile.c diff --git a/tests/compilepragma/test.nim b/tests/c/tcompile.nim similarity index 83% rename from tests/compilepragma/test.nim rename to tests/c/tcompile.nim index 56087fa577..cf99fd7edb 100644 --- a/tests/compilepragma/test.nim +++ b/tests/c/tcompile.nim @@ -3,7 +3,7 @@ discard """ joinable: "false" """ -{.compile: "test.c".} +{.compile: "tcompile.c".} proc foo(a, b: cint): cint {.importc: "foo", cdecl.} diff --git a/tests/misc/temit.nim b/tests/c/temit.nim similarity index 92% rename from tests/misc/temit.nim rename to tests/c/temit.nim index ee7455d4cb..1943c94ea6 100644 --- a/tests/misc/temit.nim +++ b/tests/c/temit.nim @@ -4,6 +4,7 @@ discard """ # Test the new ``emit`` pragma: {.emit: """ +#include static int cvariable = 420; """.} diff --git a/tests/misc/treservedcidentsasfields.nim b/tests/c/treservedcidentsasfields.nim similarity index 96% rename from tests/misc/treservedcidentsasfields.nim rename to tests/c/treservedcidentsasfields.nim index a9a9546517..6cdf9e8552 100644 --- a/tests/misc/treservedcidentsasfields.nim +++ b/tests/c/treservedcidentsasfields.nim @@ -1,39 +1,39 @@ -discard """ - targets: "c cpp" -""" - -import macros - -macro make_test_type(idents: varargs[untyped]): untyped = - result = nnkStmtList.newTree() - - var ident_defs: seq[NimNode] = @[] - for i in idents: - ident_defs.add newIdentDefs(i, ident("int")) - - result.add newTree(nnkTypeSection, - newTree(nnkTypeDef, - ident("TestType"), - newEmptyNode(), - newTree(nnkObjectTy, - newEmptyNode(), - newEmptyNode(), - newTree(nnkRecList, - ident_defs - ) - ) - ) - ) - -make_test_type( - auto, bool, catch, char, class, compl, const_cast, default, delete, double, - dynamic_cast, explicit, extern, false, float, friend, goto, int, long, - mutable, namespace, new, operator, private, protected, public, register, - reinterpret_cast, restrict, short, signed, sizeof, static_cast, struct, switch, - this, throw, true, typedef, typeid, typeof, typename, union, packed, unsigned, - virtual, void, volatile, wchar_t, alignas, alignof, constexpr, decltype, nullptr, - noexcept, thread_local, static_assert, char16_t, char32_t -) - -# Make sure the type makes it to codegen. -var test_instance: TestType +discard """ + targets: "c cpp" +""" + +import macros + +macro make_test_type(idents: varargs[untyped]): untyped = + result = nnkStmtList.newTree() + + var ident_defs: seq[NimNode] = @[] + for i in idents: + ident_defs.add newIdentDefs(i, ident("int")) + + result.add newTree(nnkTypeSection, + newTree(nnkTypeDef, + ident("TestType"), + newEmptyNode(), + newTree(nnkObjectTy, + newEmptyNode(), + newEmptyNode(), + newTree(nnkRecList, + ident_defs + ) + ) + ) + ) + +make_test_type( + auto, bool, catch, char, class, compl, const_cast, default, delete, double, + dynamic_cast, explicit, extern, false, float, friend, goto, int, long, + mutable, namespace, new, operator, private, protected, public, register, + reinterpret_cast, restrict, short, signed, sizeof, static_cast, struct, switch, + this, throw, true, typedef, typeid, typeof, typename, union, packed, unsigned, + virtual, void, volatile, wchar_t, alignas, alignof, constexpr, decltype, nullptr, + noexcept, thread_local, static_assert, char16_t, char32_t +) + +# Make sure the type makes it to codegen. +var test_instance: TestType diff --git a/tests/casestmt/t18964.nim b/tests/casestmt/t18964.nim new file mode 100644 index 0000000000..1d2de2bbc7 --- /dev/null +++ b/tests/casestmt/t18964.nim @@ -0,0 +1,12 @@ +discard """ +errormsg: "invalid order of case branches" +""" + +import macros + +macro genCase(val: string): untyped = + result = nnkCaseStmt.newTree(val, + nnkElse.newTree(quote do: echo "else"), + nnkOfBranch.newTree(newLit("miauz"), quote do: echo "first branch")) + +genCase("miauz") diff --git a/tests/casestmt/tcasestmt.nim b/tests/casestmt/tcasestmt.nim index 53cccdb641..aea0c96a42 100644 --- a/tests/casestmt/tcasestmt.nim +++ b/tests/casestmt/tcasestmt.nim @@ -287,3 +287,28 @@ doAssert(foo2("Y", "a2") == 0) doAssert(foo2("Y", "2a") == 2) doAssert(foo2("N", "a3") == 3) doAssert(foo2("z", "2") == 0) + + +# bug #20031 +proc main(a: uint64) = + case a + else: + discard + +static: + main(10) +main(10) + +block: + # Just needs to compile + proc bar(): int {.discardable.} = discard + + proc foo() {.noreturn.} = discard + + case "*" + of "*": + bar() + else: + # Make sure this noreturn doesn't + # cause the discardable to not discard + foo() diff --git a/tests/casestmt/tcstring.nim b/tests/casestmt/tcstring.nim new file mode 100644 index 0000000000..288373402d --- /dev/null +++ b/tests/casestmt/tcstring.nim @@ -0,0 +1,52 @@ +discard """ + targets: "c cpp js" +""" + +type Result = enum none, a, b, c, d, e, f + +proc foo1(x: cstring): Result = + const y = cstring"hash" + const arr = [cstring"it", cstring"finally"] + result = none + case x + of "Andreas", "Rumpf": result = a + of cstring"aa", "bb": result = b + of "cc", y, "when": result = c + of "will", arr, "be", "generated": result = d + of nil: result = f + +var results = [ + foo1("Rumpf"), foo1("Andreas"), + foo1("aa"), foo1(cstring"bb"), + foo1("cc"), foo1("hash"), + foo1("finally"), foo1("generated"), + foo1("no"), foo1("another no"), + foo1(nil)] +doAssert results == [a, a, b, b, c, c, d, d, none, none, f], $results + +proc foo2(x: cstring): Result = + const y = cstring"hash" + const arr = [cstring"it", cstring"finally"] + doAssert not (compiles do: + result = case x + of "Andreas", "Rumpf": a + of cstring"aa", "bb": b + of "cc", y, "when": c + of "will", arr, "be", "generated": d) + case x + of "Andreas", "Rumpf": a + of cstring"aa", "bb": b + of "cc", y, "when": c + of "will", arr, "be", "generated": d + of nil: f + else: e + +results = [ + foo2("Rumpf"), foo2("Andreas"), + foo2("aa"), foo2(cstring"bb"), + foo2("cc"), foo2("hash"), + foo2("finally"), foo2("generated"), + foo2("no"), foo2("another no"), + foo2(nil)] + +doAssert results == [a, a, b, b, c, c, d, d, e, e, f], $results diff --git a/tests/casestmt/tincompletecaseobject.nim b/tests/casestmt/tincompletecaseobject.nim index 909ee4e1c3..aa5deda7a2 100644 --- a/tests/casestmt/tincompletecaseobject.nim +++ b/tests/casestmt/tincompletecaseobject.nim @@ -1,6 +1,6 @@ discard """ errormsg: ''' -not all cases are covered; missing: {nnkComesFrom, nnkDotCall, nnkHiddenCallConv, nnkVarTuple, nnkCurlyExpr, nnkRange, nnkCheckedFieldExpr, nnkDerefExpr, nnkElifExpr, nnkElseExpr, nnkLambda, nnkDo, nnkBind, nnkClosedSymChoice, nnkHiddenSubConv, nnkConv, nnkStaticExpr, nnkAddr, nnkHiddenAddr, nnkHiddenDeref, nnkObjDownConv, nnkObjUpConv, nnkChckRangeF, nnkChckRange64, nnkChckRange, nnkStringToCString, nnkCStringToString, nnkFastAsgn, nnkGenericParams, nnkFormalParams, nnkOfInherit, nnkImportAs, nnkConverterDef, nnkMacroDef, nnkTemplateDef, nnkIteratorDef, nnkOfBranch, nnkElifBranch, nnkExceptBranch, nnkElse, nnkAsmStmt, nnkTypeDef, nnkFinally, nnkContinueStmt, nnkImportStmt, nnkImportExceptStmt, nnkExportStmt, nnkExportExceptStmt, nnkFromStmt, nnkIncludeStmt, nnkUsingStmt, nnkBlockExpr, nnkStmtListType, nnkBlockType, nnkWith, nnkWithout, nnkTypeOfExpr, nnkObjectTy, nnkTupleTy, nnkTupleClassTy, nnkTypeClassTy, nnkStaticTy, nnkRecList, nnkRecCase, nnkRecWhen, nnkVarTy, nnkConstTy, nnkMutableTy, nnkDistinctTy, nnkProcTy, nnkIteratorTy, nnkSharedTy, nnkEnumTy, nnkEnumFieldDef, nnkArglist, nnkPattern, nnkReturnToken, nnkClosure, nnkGotoState, nnkState, nnkBreakState, nnkFuncDef, nnkTupleConstr} +not all cases are covered; missing: {nnkComesFrom, nnkDotCall, nnkHiddenCallConv, nnkVarTuple, nnkCurlyExpr, nnkRange, nnkCheckedFieldExpr, nnkDerefExpr, nnkElifExpr, nnkElseExpr, nnkLambda, nnkDo, nnkBind, nnkClosedSymChoice, nnkHiddenSubConv, nnkConv, nnkStaticExpr, nnkAddr, nnkHiddenAddr, nnkHiddenDeref, nnkObjDownConv, nnkObjUpConv, nnkChckRangeF, nnkChckRange64, nnkChckRange, nnkStringToCString, nnkCStringToString, nnkFastAsgn, nnkGenericParams, nnkFormalParams, nnkOfInherit, nnkImportAs, nnkConverterDef, nnkMacroDef, nnkTemplateDef, nnkIteratorDef, nnkOfBranch, nnkElifBranch, nnkExceptBranch, nnkElse, nnkAsmStmt, nnkTypeDef, nnkFinally, nnkContinueStmt, nnkImportStmt, nnkImportExceptStmt, nnkExportStmt, nnkExportExceptStmt, nnkFromStmt, nnkIncludeStmt, nnkUsingStmt, nnkBlockExpr, nnkStmtListType, nnkBlockType, nnkWith, nnkWithout, nnkTypeOfExpr, nnkObjectTy, nnkTupleTy, nnkTupleClassTy, nnkTypeClassTy, nnkStaticTy, nnkRecList, nnkRecCase, nnkRecWhen, nnkVarTy, nnkConstTy, nnkMutableTy, nnkDistinctTy, nnkProcTy, nnkIteratorTy, nnkSharedTy, nnkEnumTy, nnkEnumFieldDef, nnkArgList, nnkPattern, nnkReturnToken, nnkClosure, nnkGotoState, nnkState, nnkBreakState, nnkFuncDef, nnkTupleConstr} ''' """ @@ -62,7 +62,7 @@ type nnkSharedTy, # 'shared T' nnkEnumTy, nnkEnumFieldDef, - nnkArglist, nnkPattern + nnkArgList, nnkPattern nnkReturnToken, nnkClosure, nnkGotoState, diff --git a/tests/casestmt/tincompletecaseobject2.nim b/tests/casestmt/tincompletecaseobject2.nim index c080cfeb19..bbeae19097 100644 --- a/tests/casestmt/tincompletecaseobject2.nim +++ b/tests/casestmt/tincompletecaseobject2.nim @@ -1,12 +1,5 @@ discard """ cmd: "nim check $file" -errormsg: "not all cases are covered; missing: {A, B}" -nimout: ''' -tincompletecaseobject2.nim(18, 1) Error: not all cases are covered; missing: {' ', '!', '\"', '#', '$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'} -tincompletecaseobject2.nim(22, 1) Error: not all cases are covered; missing: {B, C, D} -tincompletecaseobject2.nim(25, 1) Error: not all cases are covered; missing: {A, C} -tincompletecaseobject2.nim(28, 1) Error: not all cases are covered; missing: {A, B} -''' """ type ABCD = enum A, B, C, D @@ -15,15 +8,19 @@ type AliasRangeABC = RangeABC PrintableChars = range[' ' .. '~'] -case PrintableChars 'x': +case PrintableChars 'x': #[tt.Error +^ not all cases are covered; missing: {' ', '!', '\"', '#', '$$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'}]# of '0'..'9', 'A'..'Z', 'a'..'z': discard of '(', ')': discard -case AliasABCD A: +case AliasABCD A: #[tt.Error +^ not all cases are covered; missing: {B, C, D}]# of A: discard -case RangeABC A: +case RangeABC A: #[tt.Error +^ not all cases are covered; missing: {A, C}]# of B: discard -case AliasRangeABC A: +case AliasRangeABC A: #[tt.Error +^ not all cases are covered; missing: {A, B}]# of C: discard diff --git a/tests/cast/tcast.nim b/tests/cast/tcast.nim new file mode 100644 index 0000000000..205444ea3a --- /dev/null +++ b/tests/cast/tcast.nim @@ -0,0 +1,21 @@ +discard """ + targets: "c cpp js" +""" + +proc main() = + block: # bug #16806 + let + a = 42u16 + b = cast[int16](a) + doAssert a.int16 == 42 + doAssert b in int16.low..int16.high + + block: # bug #16808 + doAssert cast[int8](cast[uint8](int8(-12))) == int8(-12) + doAssert cast[int16](cast[uint16](int16(-12))) == int16(-12) + doAssert cast[int32](cast[uint32](int32(-12))) == int32(-12) + + doAssert cast[int8](int16.high) == -1 + +static: main() +main() diff --git a/tests/ccgbugs/m1/defs.nim b/tests/ccgbugs/m1/defs.nim new file mode 100644 index 0000000000..ed78d8b72f --- /dev/null +++ b/tests/ccgbugs/m1/defs.nim @@ -0,0 +1,4 @@ +type MyObj* = object + field1*: int + s*: string + ch*: char diff --git a/tests/ccgbugs/m19445.c b/tests/ccgbugs/m19445.c new file mode 100644 index 0000000000..74c23d4b45 --- /dev/null +++ b/tests/ccgbugs/m19445.c @@ -0,0 +1,3 @@ +#include "m19445.h" + +const Foo f = {10, 20, 30, 40}; \ No newline at end of file diff --git a/tests/ccgbugs/m2/defs.nim b/tests/ccgbugs/m2/defs.nim new file mode 100644 index 0000000000..798d1fea81 --- /dev/null +++ b/tests/ccgbugs/m2/defs.nim @@ -0,0 +1,4 @@ +type MyObj* = object + s*: string + field1*: int + ch*: char diff --git a/tests/ccgbugs/t10128.nim b/tests/ccgbugs/t10128.nim new file mode 100644 index 0000000000..48970916f2 --- /dev/null +++ b/tests/ccgbugs/t10128.nim @@ -0,0 +1,18 @@ +# bug #10128 +let data = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" +var seq2 = newSeq[char](data.len) +for i in 0.. 7: + raise newException(IOError, "H2") + + +proc foo() = + var i = 0 + while true: + try: + bar(i) + echo i + + except ValueError: + debugEcho("ValueError") + + except IOError: + raise newException(IOError, "Err2") + +when isMainModule: + foo() \ No newline at end of file diff --git a/tests/ccgbugs/t21995.nim b/tests/ccgbugs/t21995.nim new file mode 100644 index 0000000000..0ec88aa59a --- /dev/null +++ b/tests/ccgbugs/t21995.nim @@ -0,0 +1,9 @@ +discard """ + targets: "c cpp" + output: "Hi!" +""" + +try: + raise +except: + echo "Hi!" \ No newline at end of file diff --git a/tests/ccgbugs/t22462.nim b/tests/ccgbugs/t22462.nim new file mode 100644 index 0000000000..9adfbb19ba --- /dev/null +++ b/tests/ccgbugs/t22462.nim @@ -0,0 +1,20 @@ +discard """ + action: "run" + output: ''' +1 +1 +1 +''' + matrix: "--mm:refc" + targets: "c cpp" +""" + +type Object = object + someComplexType: seq[int] + index: Natural + +func newObject(): Object = result.index.inc + +for i in 1..3: + let o = newObject() + echo o.index diff --git a/tests/ccgbugs/t5296.nim b/tests/ccgbugs/t5296.nim index 990b4bee29..8fbed35c4f 100644 --- a/tests/ccgbugs/t5296.nim +++ b/tests/ccgbugs/t5296.nim @@ -1,6 +1,7 @@ discard """ cmd: "nim c -d:release $file" -output: 1 +output: '''1 +-1''' """ proc bug() : void = @@ -12,3 +13,9 @@ proc bug() : void = echo x bug() + +# bug #19051 +type GInt[T] = int + +var a = 1 +echo -a diff --git a/tests/ccgbugs/t8967.nim b/tests/ccgbugs/t8967.nim index e342b7eae2..0301a2e4f0 100644 --- a/tests/ccgbugs/t8967.nim +++ b/tests/ccgbugs/t8967.nim @@ -4,7 +4,11 @@ discard """ import marshal -let orig: set[char] = {'A'..'Z'} -let m = $$orig -let old = to[set[char]](m) -doAssert orig - old == {} +template main() = + let orig: set[char] = {'A'..'Z'} + let m = $$orig + let old = to[set[char]](m) + doAssert orig - old == {} + +static: main() +main() diff --git a/tests/ccgbugs/targ_lefttoright.nim b/tests/ccgbugs/targ_lefttoright.nim index a49b877391..a0adce1572 100644 --- a/tests/ccgbugs/targ_lefttoright.nim +++ b/tests/ccgbugs/targ_lefttoright.nim @@ -29,7 +29,7 @@ template test = var b = 1 say (b += 1; b), (b += 1; b) #2,3 - type C = object {.byRef.} + type C {.byRef.} = object i: int proc say(a, b: C) = diff --git a/tests/ccgbugs/tassign_nil_strings.nim b/tests/ccgbugs/tassign_nil_strings.nim index f6fab7baac..e32bfcade6 100644 --- a/tests/ccgbugs/tassign_nil_strings.nim +++ b/tests/ccgbugs/tassign_nil_strings.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim $target $options $file" + matrix: "--mm:refc" output: "Hello" ccodecheck: "\\i@'a = ((NimStringDesc*) NIM_NIL)'" """ diff --git a/tests/ccgbugs/tbug21505.nim b/tests/ccgbugs/tbug21505.nim new file mode 100644 index 0000000000..0c0811ec5b --- /dev/null +++ b/tests/ccgbugs/tbug21505.nim @@ -0,0 +1,39 @@ +discard """ + action: "compile" + targets: "cpp" + cmd: "nim cpp $file" +""" + +# see #21505: ensure compilation of imported C++ objects with explicit constructors while retaining default initialization through codegen changes due to #21279 + +{.emit:"""/*TYPESECTION*/ + +struct ExplObj +{ + explicit ExplObj(int bar = 0) {} +}; + +struct BareObj +{ + BareObj() {} +}; + +""".} + +type + ExplObj {.importcpp.} = object + BareObj {.importcpp.} = object + +type + Composer = object + explObj: ExplObj + bareObj: BareObj + +proc foo = + var composer1 {.used.}: Composer + let composer2 {.used.} = Composer() + +var composer1 {.used.}: Composer +let composer2 {.used.} = Composer() + +foo() \ No newline at end of file diff --git a/tests/ccgbugs/tccgen1.nim b/tests/ccgbugs/tccgen1.nim index 4917c98483..be571de088 100644 --- a/tests/ccgbugs/tccgen1.nim +++ b/tests/ccgbugs/tccgen1.nim @@ -7,7 +7,7 @@ type Features: seq[Feature] # Read-Only PNode* = ref Node - Node = object {.inheritable.} + Node {.inheritable.} = object attributes*: seq[PAttr] childNodes*: seq[PNode] FLocalName: string # Read-only diff --git a/tests/ccgbugs/tcgbug.nim b/tests/ccgbugs/tcgbug.nim index db9c116be8..871d6b59c6 100644 --- a/tests/ccgbugs/tcgbug.nim +++ b/tests/ccgbugs/tcgbug.nim @@ -24,6 +24,7 @@ q(a) # bug #914 when defined(windows): + import std/widestrs var x = newWideCString("Hello") echo "success" @@ -91,3 +92,44 @@ proc test(c: Helper): string = c.formatted echo test(Helper(isKind: true, formatted: "ok")) + + +# bug #19613 + +type + Eth2Digest = object + data: array[42, byte] + + BlockId* = object + root*: Eth2Digest + + BlockSlotId* = object + bid*: BlockId + slot*: uint64 + +func init*(T: type BlockSlotId, bid: BlockId, slot: uint64): T = + #debugecho "init ", bid, " ", slot + BlockSlotId(bid: bid, slot: slot) + +proc bug19613 = + var x: BlockSlotId + x.bid.root.data[0] = 42 + + x = + if x.slot > 0: + BlockSlotId.init(x.bid, x.slot) + else: + BlockSlotId.init(x.bid, x.slot) + doAssert x.bid.root.data[0] == 42 + +bug19613() + +proc foo = # bug #23280 + let foo = @[1,2,3,4,5,6] + doAssert toOpenArray(foo, 0, 5).len == 6 + doAssert toOpenArray(foo, 0, 5).len mod 6 == 0 # this should output 0 + doAssert toOpenArray(foo, 0, 5).max mod 6 == 0 + let L = toOpenArray(foo, 0, 5).len + doAssert L mod 6 == 0 + +foo() diff --git a/tests/ccgbugs/tcodegenbug1.nim b/tests/ccgbugs/tcodegenbug1.nim index c62bae1efc..d2ab97ede0 100644 --- a/tests/ccgbugs/tcodegenbug1.nim +++ b/tests/ccgbugs/tcodegenbug1.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:refc" output: '''obj = (inner: (kind: Just, id: 7)) obj.inner.id = 7 id = 7 diff --git a/tests/ccgbugs/tctypes.nim b/tests/ccgbugs/tctypes.nim new file mode 100644 index 0000000000..be6009115a --- /dev/null +++ b/tests/ccgbugs/tctypes.nim @@ -0,0 +1,43 @@ +discard """ + targets: "c cpp" + matrix: "--gc:refc; --gc:arc" +""" + +# bug #7308 +proc foo(x: seq[int32]) = + var y = newSeq[cint](1) + +proc bar = + var t = newSeq[int32](1) + foo(t) + +bar() + + +# bug #16246 + +proc testWeirdTypeAliases() = + var values = newSeq[cuint](8) + # var values: seq[cuint] does not produce codegen error + var drawCb = proc(): seq[uint32] = + result = newSeq[uint32](10) + +testWeirdTypeAliases() + +block: # bug #11797 + block: + type cdouble2 = cdouble + type Foo1 = seq[cdouble] + type Foo2 = seq[cdouble2] + static: doAssert Foo1 is Foo2 + var a1: Foo1 + var a2: Foo2 + doAssert a1 == @[] + doAssert a2 == @[] + + block: + proc foo[T: int|cint](fun: proc(): T) = discard + proc foo1(): cint = 1 + proc foo3(): int32 = 2 + foo(proc(): cint = foo1()) + foo(proc(): int32 = foo3()) diff --git a/tests/ccgbugs/tdeepcopy_addr_rval.nim b/tests/ccgbugs/tdeepcopy_addr_rval.nim index 07fb8f8ef7..4a0b0deaac 100644 --- a/tests/ccgbugs/tdeepcopy_addr_rval.nim +++ b/tests/ccgbugs/tdeepcopy_addr_rval.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:refc; --mm:orc --deepcopy:on" output: "3" """ diff --git a/tests/ccgbugs/tderefblock.nim b/tests/ccgbugs/tderefblock.nim new file mode 100644 index 0000000000..d3ba076679 --- /dev/null +++ b/tests/ccgbugs/tderefblock.nim @@ -0,0 +1,76 @@ +discard """ + matrix: "--mm:refc -d:release -d:danger;--mm:orc -d:useMalloc -d:release -d:danger" + output: "42" +""" + +# bug #20107 + +type Foo = object + a, b, c, d: uint64 + +proc c(i: uint64): Foo = + Foo(a: i, b: i, c: i, d: i) + +func x(f: Foo): lent Foo {.inline.} = + f + +proc m() = + let f = block: + let i = c(42) + x(i) + + echo $f.a + +m() + +block: # bug #21540 + type + Option = object + val: string + has: bool + + proc some(val: string): Option = + result.has = true + result.val = val + + # Remove lent and it works + proc get(self: Option): lent string = + result = self.val + + type + StringStream = ref object + data: string + pos: int + + proc readAll(s: StringStream): string = + result = newString(s.data.len) + copyMem(addr(result[0]), addr(s.data[0]), s.data.len) + + proc newStringStream(s: string = ""): StringStream = + new(result) + result.data = s + + proc parseJson(s: string): string = + let stream = newStringStream(s) + result = stream.readAll() + + proc main = + let initialFEN = block: + let initialFEN = some parseJson("startpos") + initialFEN.get + + doAssert initialFEN == "startpos" + + main() + +import std/[ + json, + options +] + +block: # bug #21540 + let cheek = block: + let initialFEN = some("""{"initialFen": "startpos"}""".parseJson{"initialFen"}.getStr) + initialFEN.get + + doAssert cheek == "startpos" diff --git a/tests/ccgbugs/tforward_decl_only.nim b/tests/ccgbugs/tforward_decl_only.nim index 74fbae303d..b115dcbe70 100644 --- a/tests/ccgbugs/tforward_decl_only.nim +++ b/tests/ccgbugs/tforward_decl_only.nim @@ -1,7 +1,5 @@ discard """ -ccodecheck: "\\i !@('struct tyObject_MyRefObject'[0-z]+' {')" -ccodecheck: "\\i !@('mymoduleInit')" -ccodecheck: "\\i @('mymoduleDatInit')" +ccodecheck: "\\i !@('struct tyObject_MyRefObject'[0-z]+' _')" output: "hello" """ diff --git a/tests/ccgbugs/thtiobj.nim b/tests/ccgbugs/thtiobj.nim index 7a656905fb..6db24dad0d 100644 --- a/tests/ccgbugs/thtiobj.nim +++ b/tests/ccgbugs/thtiobj.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:refc" targets: "c cpp" """ diff --git a/tests/ccgbugs/tmangle.nim b/tests/ccgbugs/tmangle.nim new file mode 100644 index 0000000000..0050cef921 --- /dev/null +++ b/tests/ccgbugs/tmangle.nim @@ -0,0 +1,16 @@ +block: + proc hello() = + let NAN_INFINITY = 12 + doAssert NAN_INFINITY == 12 + let INF = "2.0" + doAssert INF == "2.0" + let NAN = 2.3 + doAssert NAN == 2.3 + + hello() + +block: + proc hello(NAN: float) = + doAssert NAN == 2.0 + + hello(2.0) diff --git a/tests/ccgbugs/tmissingbracket.nim b/tests/ccgbugs/tmissingbracket.nim index 468e13366e..2919efe0e4 100644 --- a/tests/ccgbugs/tmissingbracket.nim +++ b/tests/ccgbugs/tmissingbracket.nim @@ -11,7 +11,7 @@ type className* : string TClassOfTobj = object of TClassOfTCustomObject nil - TCustomObject = ref object {.inheritable.} + TCustomObject {.inheritable.} = ref object class* : ptr TClassOfTCustomObject TObj = ref object of TCustomObject data: int diff --git a/tests/ccgbugs/tmissinginit.nim b/tests/ccgbugs/tmissinginit.nim index 8806a2f219..9eb58221c5 100644 --- a/tests/ccgbugs/tmissinginit.nim +++ b/tests/ccgbugs/tmissinginit.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:refc" output: '''0 0 0 diff --git a/tests/ccgbugs/tmissingvolatile.nim b/tests/ccgbugs/tmissingvolatile.nim index 1eccdc6b1b..b877eff71c 100644 --- a/tests/ccgbugs/tmissingvolatile.nim +++ b/tests/ccgbugs/tmissingvolatile.nim @@ -1,6 +1,6 @@ discard """ output: "1" - cmd: r"nim c --hints:on $options -d:release $file" + cmd: r"nim c --hints:on $options --mm:refc -d:release $file" ccodecheck: "'NI volatile state;'" targets: "c" """ diff --git a/tests/ccgbugs/tnoalias.nim b/tests/ccgbugs/tnoalias.nim index 0c6e84e44b..2c3c2f0f40 100644 --- a/tests/ccgbugs/tnoalias.nim +++ b/tests/ccgbugs/tnoalias.nim @@ -1,5 +1,5 @@ discard """ - ccodecheck: "\\i@'NI* NIM_NOALIAS field;' @'NIM_CHAR* NIM_NOALIAS x,' @'void* NIM_NOALIAS q'" + ccodecheck: "\\i@'NI* NIM_NOALIAS field;' @'NIM_CHAR* NIM_NOALIAS x_p0,' @'void* NIM_NOALIAS q'" """ type @@ -7,7 +7,7 @@ type field {.noalias.}: ptr UncheckedArray[int] proc p(x {.noalias.}: openArray[char]) = - var q {.noalias.}: pointer = unsafeAddr(x[0]) + var q {.noalias.}: pointer = addr(x[0]) var bn: BigNum p "abc" diff --git a/tests/ccgbugs/tsamename3.nim b/tests/ccgbugs/tsamename3.nim new file mode 100644 index 0000000000..a69391e5c7 --- /dev/null +++ b/tests/ccgbugs/tsamename3.nim @@ -0,0 +1,111 @@ +block: # bug #15526 + block: + type Foo = ref object + x1: int + let f1 = Foo(x1: 1) + block: + type Foo = ref object + x2: int + let f2 = Foo(x2: 2) + +block: # ditto + template fn() = + block: + type Foo = ref object + x1: int + let f1 = Foo(x1: 1) + doAssert f1.x1 == 1 + block: + type Foo = ref object + x2: int + let f2 = Foo(x2: 2) + doAssert f2.x2 == 2 + static: fn() + fn() + +block: # bug #17162 + template fn = + var ret: string + block: + type A = enum a0, a1, a2 + for ai in A: + ret.add $ai + block: + type A = enum b0, b1, b2, b3 + for ai in A: + ret.add $ai + doAssert ret == "a0a1a2b0b1b2b3" + + static: fn() # ok + fn() # was bug + +block: # ditto + proc fn = + var ret: string + block: + type A = enum a0, a1, a2 + for ai in A: + ret.add $ai + block: + type A = enum b0, b1, b2, b3 + for ai in A: + ret.add $ai + doAssert ret == "a0a1a2b0b1b2b3" + + static: fn() # ok + fn() # was bug + +block: # bug #5170 + block: + type Foo = object + x1: int + let f1 = Foo(x1: 1) + block: + type Foo = object + x2: int + let f2 = Foo(x2: 2) + +block: # ditto + block: + type Foo = object + bar: bool + var f1: Foo + + block: + type Foo = object + baz: int + var f2: Foo + doAssert f2.baz == 0 + + block: + template fn() = + block: + type Foo = object + x1: int + let f1 = Foo(x1: 1) + doAssert f1.x1 == 1 + block: + type Foo = object + x2: int + let f2 = Foo(x2: 2) + doAssert f2.x2 == 2 + static: fn() + fn() + +when true: # ditto, refs https://github.com/nim-lang/Nim/issues/5170#issuecomment-582712132 + type Foo1 = object # at top level + bar: bool + var f1: Foo1 + + block: + type Foo1 = object + baz: int + var f2: Foo1 + doAssert f2.baz == 0 + +block: # make sure `hashType` doesn't recurse infinitely + type + PFoo = ref object + a, b: PFoo + c: int + var a: PFoo diff --git a/tests/ccgbugs/twrong_setconstr.nim b/tests/ccgbugs/twrong_setconstr.nim deleted file mode 100644 index 8be0b82b52..0000000000 --- a/tests/ccgbugs/twrong_setconstr.nim +++ /dev/null @@ -1,146 +0,0 @@ -discard """ - output: "" -""" - -# bug #2880 - -type - TMsgKind* = enum - errUnknown, errIllFormedAstX, errInternal, errCannotOpenFile, errGenerated, - errXCompilerDoesNotSupportCpp, errStringLiteralExpected, - errIntLiteralExpected, errInvalidCharacterConstant, - errClosingTripleQuoteExpected, errClosingQuoteExpected, - errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong, - errInvalidNumber, errInvalidNumberOctalCode, errNumberOutOfRange, - errNnotAllowedInCharacter, errClosingBracketExpected, errMissingFinalQuote, - errIdentifierExpected, errNewlineExpected, errInvalidModuleName, - errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected, - errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected, - errInvalidPragma, errUnknownPragma, errInvalidDirectiveX, - errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, - errExceptionExpected, errExceptionAlreadyHandled, - errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, - errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine, - errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, - errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, - errInvalidVarSubstitution, errUnknownVar, errUnknownCcompiler, - errOnOrOffExpectedButXFound, errOnOffOrListExpectedButXFound, - errNoneBoehmRefcExpectedButXFound, - errNoneSpeedOrSizeExpectedButXFound, errGuiConsoleOrLibExpectedButXFound, - errUnknownOS, errUnknownCPU, errGenOutExpectedButXFound, - errArgsNeedRunOption, errInvalidMultipleAsgn, errColonOrEqualsExpected, - errExprExpected, errUndeclaredIdentifier, errUseQualifier, errTypeExpected, - errSystemNeeds, errExecutionOfProgramFailed, errNotOverloadable, - errInvalidArgForX, errStmtHasNoEffect, errXExpectsTypeOrValue, - errXExpectsArrayType, errIteratorCannotBeInstantiated, errExprXAmbiguous, - errConstantDivisionByZero, errOrdinalTypeExpected, - errOrdinalOrFloatTypeExpected, errOverOrUnderflow, - errCannotEvalXBecauseIncompletelyDefined, errChrExpectsRange0_255, - errDynlibRequiresExportc, errUndeclaredFieldX, errNilAccess, - errIndexOutOfBounds, errIndexTypesDoNotMatch, errBracketsInvalidForType, - errValueOutOfSetBounds, errFieldInitTwice, errFieldNotInit, - errExprXCannotBeCalled, errExprHasNoType, errExprXHasNoType, - errCastNotInSafeMode, errExprCannotBeCastedToX, errCommaOrParRiExpected, - errCurlyLeOrParLeExpected, errSectionExpected, errRangeExpected, - errMagicOnlyInSystem, errPowerOfTwoExpected, - errStringMayNotBeEmpty, errCallConvExpected, errProcOnlyOneCallConv, - errSymbolMustBeImported, errExprMustBeBool, errConstExprExpected, - errDuplicateCaseLabel, errRangeIsEmpty, errSelectorMustBeOfCertainTypes, - errSelectorMustBeOrdinal, errOrdXMustNotBeNegative, errLenXinvalid, - errWrongNumberOfVariables, errExprCannotBeRaised, errBreakOnlyInLoop, - errTypeXhasUnknownSize, errConstNeedsConstExpr, errConstNeedsValue, - errResultCannotBeOpenArray, errSizeTooBig, errSetTooBig, - errBaseTypeMustBeOrdinal, errInheritanceOnlyWithNonFinalObjects, - errInheritanceOnlyWithEnums, errIllegalRecursionInTypeX, - errCannotInstantiateX, errExprHasNoAddress, errXStackEscape, - errVarForOutParamNeeded, - errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX, - errAmbiguousCallXYZ, errWrongNumberOfArguments, - errXCannotBePassedToProcVar, - errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProc, errImplOfXNotAllowed, - errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValueX, - errInvalidDiscard, errIllegalConvFromXtoY, errCannotBindXTwice, - errInvalidOrderInArrayConstructor, - errInvalidOrderInEnumX, errEnumXHasHoles, errExceptExpected, errInvalidTry, - errOptionExpected, errXisNoLabel, errNotAllCasesCovered, - errUnknownSubstitionVar, errComplexStmtRequiresInd, errXisNotCallable, - errNoPragmasAllowedForX, errNoGenericParamsAllowedForX, - errInvalidParamKindX, errDefaultArgumentInvalid, errNamedParamHasToBeIdent, - errNoReturnTypeForX, errConvNeedsOneArg, errInvalidPragmaX, - errXNotAllowedHere, errInvalidControlFlowX, - errXisNoType, errCircumNeedsPointer, errInvalidExpression, - errInvalidExpressionX, errEnumHasNoValueX, errNamedExprExpected, - errNamedExprNotAllowed, errXExpectsOneTypeParam, - errArrayExpectsTwoTypeParams, errInvalidVisibilityX, errInitHereNotAllowed, - errXCannotBeAssignedTo, errIteratorNotAllowed, errXNeedsReturnType, - errNoReturnTypeDeclared, - errInvalidCommandX, errXOnlyAtModuleScope, - errXNeedsParamObjectType, - errTemplateInstantiationTooNested, errInstantiationFrom, - errInvalidIndexValueForTuple, errCommandExpectsFilename, - errMainModuleMustBeSpecified, - errXExpected, - errTIsNotAConcreteType, - errInvalidSectionStart, errGridTableNotImplemented, errGeneralParseError, - errNewSectionExpected, errWhitespaceExpected, errXisNoValidIndexFile, - errCannotRenderX, errVarVarTypeNotAllowed, errInstantiateXExplicitly, - errOnlyACallOpCanBeDelegator, errUsingNoSymbol, - errMacroBodyDependsOnGenericTypes, - errDestructorNotGenericEnough, - errInlineIteratorsAsProcParams, - errXExpectsTwoArguments, - errXExpectsObjectTypes, errXcanNeverBeOfThisSubtype, errTooManyIterations, - errCannotInterpretNodeX, errFieldXNotFound, errInvalidConversionFromTypeX, - errAssertionFailed, errCannotGenerateCodeForX, errXRequiresOneArgument, - errUnhandledExceptionX, errCyclicTree, errXisNoMacroOrTemplate, - errXhasSideEffects, errIteratorExpected, errLetNeedsInit, - errThreadvarCannotInit, errWrongSymbolX, errIllegalCaptureX, - errXCannotBeClosure, errXMustBeCompileTime, - errCannotInferTypeOfTheLiteral, - errCannotInferReturnType, - errGenericLambdaNotAllowed, - errCompilerDoesntSupportTarget, - errUser, - warnCannotOpenFile, - warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit, - warnDeprecated, warnConfigDeprecated, - warnSmallLshouldNotBeUsed, warnUnknownMagic, warnRedefinitionOfLabel, - warnUnknownSubstitutionX, warnLanguageXNotSupported, - warnFieldXNotSupported, warnCommentXIgnored, - warnNilStatement, warnTypelessParam, - warnDifferentHeaps, warnWriteToForeignHeap, warnUnsafeCode, - warnEachIdentIsTuple - warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2, - warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed, - warnUser, - hintSuccess, hintSuccessX, - hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded, - hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled, - hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath, - hintConditionAlwaysTrue, hintName, hintPattern, - hintExecuting, hintLinking, hintDependency, - hintSource, hintStackTrace, hintGCStats, - hintUser - -const - warnMin = warnCannotOpenFile - hintMax = high(TMsgKind) - -type - TNoteKind = range[warnMin..hintMax] # "notes" are warnings or hints - TNoteKinds = set[TNoteKind] - -const - NotesVerbosityConst: array[0..0, TNoteKinds] = [ - {low(TNoteKind)..high(TNoteKind)} - {hintGCStats}] - fuckyou = NotesVerbosityConst[0] - -var - gNotesFromConst: TNoteKinds = NotesVerbosityConst[0] - gNotesFromConst2: TNoteKinds = fuckyou - -if hintGCStats in gNotesFromConst: - echo "hintGCStats in gNotesFromConst A" - -if hintGCStats in gNotesFromConst2: - echo "hintGCStats in gNotesFromConst B" diff --git a/tests/ccgbugs/twrong_tupleconv.nim b/tests/ccgbugs/twrong_tupleconv.nim index 7a887d1835..031712dac9 100644 --- a/tests/ccgbugs/twrong_tupleconv.nim +++ b/tests/ccgbugs/twrong_tupleconv.nim @@ -1,3 +1,8 @@ +discard """ + targets: "c cpp" + matrix: "--gc:refc; --gc:arc" +""" + # bug #1833 iterator myitems*[T](a: var seq[T]): var T {.inline.} = ## iterates over each item of `a` so that you can modify the yielded value. @@ -18,3 +23,13 @@ var ys = @[(1,"a"),(2,"b"),(3,"c")] for y in myitems(ys): inc y[0] +# bug #16331 +type T1 = tuple[a, b: int] + +proc p(b: bool): string = + var x: T1 = (10, 20) + x = if b: (x.b, x.a) else: (-x.b, -x.a) + $x + +assert p(false) == "(a: -20, b: -10)" +assert p(true) == "(a: 20, b: 10)" diff --git a/tests/ccgbugs2/tcodegen.nim b/tests/ccgbugs2/tcodegen.nim new file mode 100644 index 0000000000..aac1ecaf39 --- /dev/null +++ b/tests/ccgbugs2/tcodegen.nim @@ -0,0 +1,47 @@ +discard """ + targets: "c cpp" +""" + +# bug #19094 +type + X = object + filler: array[2048, int] + innerAddress: uint + +proc initX(): X = + result.innerAddress = cast[uint](result.addr) + +proc initXInPlace(x: var X) = + x.innerAddress = cast[uint](x.addr) + +block: # NRVO1 + var x = initX() + let innerAddress = x.innerAddress + let outerAddress = cast[uint](x.addr) + doAssert(innerAddress == outerAddress) # [OK] + +block: # NRVO2 + var x: X + initXInPlace(x) + let innerAddress = x.innerAddress + let outerAddress = cast[uint](x.addr) + doAssert(innerAddress == outerAddress) # [OK] + +block: # bug #22354 + type Object = object + foo: int + + proc takeFoo(self: var Object): int = + result = self.foo + self.foo = 999 + + proc doSomething(self: var Object; foo: int = self.takeFoo()) = + discard + + proc main() = + var obj = Object(foo: 2) + obj.doSomething() + doAssert obj.foo == 999 + + + main() diff --git a/tests/closure/t11042.nim b/tests/closure/t11042.nim new file mode 100644 index 0000000000..6a39283160 --- /dev/null +++ b/tests/closure/t11042.nim @@ -0,0 +1,55 @@ +discard """ + output:''' +foo: 1 +foo: 2 +bar: 1 +bar: 2 +foo: 1 +foo: 2 +bar: 1 +bar: 2 +bar: 3 +bar: 4 +bar: 5 +bar: 6 +bar: 7 +bar: 8 +bar: 9 +''' +""" + +# bug #11042 +block: + iterator foo: int = + for x in 1..2: + echo "foo: ", x + for y in 1..2: + discard + + for x in foo(): discard + + let bar = iterator: int = + for x in 1..2: + echo "bar: ", x + for y in 1..2: + discard + + for x in bar(): discard + + +block: + iterator foo: int = + for x in 1..2: + echo "foo: ", x + for y in 1..2: + discard + + for x in foo(): discard + + let bar = iterator: int = + for x in 1..9: + echo "bar: ", x + for y in 1..2: + discard + + for x in bar(): discard \ No newline at end of file diff --git a/tests/closure/t19095.nim b/tests/closure/t19095.nim new file mode 100644 index 0000000000..880456e029 --- /dev/null +++ b/tests/closure/t19095.nim @@ -0,0 +1,35 @@ +discard """ + action: compile +""" + +block: + func inCheck() = + discard + + iterator iter(): int = + yield 0 + yield 0 + + func search() = + let inCheck = 0 + + for i in iter(): + + proc hello() = + inCheck() + + search() +block: + iterator iter(): int = + yield 0 + yield 0 + + func search() = + let lmrMoveCounter = 0 + + for i in iter(): + + proc hello() = + discard lmrMoveCounter + + search() diff --git a/tests/closure/t20152.nim b/tests/closure/t20152.nim new file mode 100644 index 0000000000..484ea07416 --- /dev/null +++ b/tests/closure/t20152.nim @@ -0,0 +1,20 @@ +discard """ + action: compile +""" + +proc foo() = + iterator it():int {.closure.} = + yield 1 + proc useIter() {.nimcall.} = + var iii = it # <-- illegal capture + doAssert iii() == 1 + useIter() +foo() + +proc foo2() = + proc bar() = # Local function, but not a closure, because no captures + echo "hi" + proc baz() {.nimcall.} = # Calls local function + bar() + baz() +foo2() diff --git a/tests/closure/t9334.nim b/tests/closure/t9334.nim new file mode 100644 index 0000000000..36a9a7d77c --- /dev/null +++ b/tests/closure/t9334.nim @@ -0,0 +1,19 @@ +discard """ + cmd: "nim $target --hints:off $options -r $file" + nimout: '''@[1] +@[1, 1] +''' + nimoutFull: true +""" +proc p(s: var seq[int]): auto = + let sptr = addr s + return proc() = sptr[].add 1 + +proc f = + var data = @[1] + p(data)() + echo repr data + +static: + f() # prints [1] +f() # prints [1, 1] diff --git a/tests/closure/tclosure.nim b/tests/closure/tclosure.nim index 546d7026dd..401a71d400 100644 --- a/tests/closure/tclosure.nim +++ b/tests/closure/tclosure.nim @@ -38,10 +38,10 @@ joinable: false block tclosure: - proc map(n: var openarray[int], fn: proc (x: int): int {.closure}) = + proc map(n: var openArray[int], fn: proc (x: int): int {.closure}) = for i in 0..n.len-1: n[i] = fn(n[i]) - proc each(n: openarray[int], fn: proc(x: int) {.closure.}) = + proc each(n: openArray[int], fn: proc(x: int) {.closure.}) = for i in 0..n.len-1: fn(n[i]) @@ -65,7 +65,7 @@ block tclosure: # bug #5015 - type Mutator = proc(matched: string): string {.noSideEffect, gcsafe, locks: 0.} + type Mutator = proc(matched: string): string {.noSideEffect, gcsafe.} proc putMutated( MutatorCount: static[int], @@ -239,19 +239,19 @@ block doNotation: b.onClick do (e: Event): echo "click at ", e.x, ",", e.y - b.onFocusLost: + b.onFocusLost do (): echo "lost focus 1" - b.onFocusLost do: + b.onFocusLost do (): echo "lost focus 2" - b.onUserEvent("UserEvent 1") do: + b.onUserEvent("UserEvent 1") do (): discard - b.onUserEvent "UserEvent 2": + onUserEvent(b, "UserEvent 2") do (): discard - b.onUserEvent("UserEvent 3"): + b.onUserEvent("UserEvent 3") do (): discard b.onUserEvent("UserEvent 4", () => echo "event 4") @@ -491,3 +491,14 @@ block tnoclosure: row = zip(row & @[0], @[0] & row).mapIt(it[0] + it[1]) echo row pascal(10) + +block: # bug #22297 + iterator f: int {.closure.} = + try: + yield 12 + finally: + return 14 + + let s = f + doAssert s() == 12 + doAssert s() == 14 diff --git a/tests/closure/tclosure_issues.nim b/tests/closure/tclosure_issues.nim index 4688834de9..b1a2d7c6b6 100644 --- a/tests/closure/tclosure_issues.nim +++ b/tests/closure/tclosure_issues.nim @@ -71,12 +71,12 @@ block tissue7104: proc sp(cb: proc())= cb() - sp: + sp do (): var i = 0 echo "ok ", i - sp(): + sp do (): inc i echo "ok ", i - sp do: + sp do (): inc i echo "ok ", i diff --git a/tests/closure/tinvalidclosure.nim b/tests/closure/tinvalidclosure.nim index 47f3f105f7..37d0f68a2f 100644 --- a/tests/closure/tinvalidclosure.nim +++ b/tests/closure/tinvalidclosure.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "type mismatch: got " + errormsg: "type mismatch: got " line: 12 """ diff --git a/tests/closure/tinvalidclosure5.nim b/tests/closure/tinvalidclosure5.nim index d03d93867f..3b5f46a40c 100644 --- a/tests/closure/tinvalidclosure5.nim +++ b/tests/closure/tinvalidclosure5.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "type mismatch: got but expected 'A = proc (){.nimcall.}'" + errormsg: "type mismatch: got but expected 'A = proc (){.nimcall.}'" line: 9 """ diff --git a/tests/closure/tnested.nim b/tests/closure/tnested.nim index ca8d0e08b4..31963ea863 100644 --- a/tests/closure/tnested.nim +++ b/tests/closure/tnested.nim @@ -33,7 +33,7 @@ py py px 6 -proc (){.closure, gcsafe, locks: 0.} +proc (){.closure, noSideEffect, gcsafe.} ''' """ diff --git a/tests/closure/tstmtlist.nim b/tests/closure/tstmtlist.nim new file mode 100644 index 0000000000..6a1390617a --- /dev/null +++ b/tests/closure/tstmtlist.nim @@ -0,0 +1,9 @@ +discard """ + action: compile +""" + +proc foo(x: proc()) = x() +foo: echo "a" #[tt.Warning + ^ statement list expression assumed to be anonymous proc; this is deprecated, use `do (): ...` or `proc () = ...` instead [StmtListLambda]]# +foo do: echo "b" #[tt.Warning + ^ statement list expression assumed to be anonymous proc; this is deprecated, use `do (): ...` or `proc () = ...` instead [StmtListLambda]]# diff --git a/tests/codegen/titaniummangle.nim b/tests/codegen/titaniummangle.nim new file mode 100644 index 0000000000..4b45e59aea --- /dev/null +++ b/tests/codegen/titaniummangle.nim @@ -0,0 +1,193 @@ +discard """ + targets: "c" + matrix: "--debugger:native" + ccodecheck: "'_ZN14titaniummangle8testFuncE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE6stringN14titaniummangle3FooE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3int7varargsI6stringE'" + ccodecheck: "'_ZN14titaniummangle8testFuncEN14titaniummangle3BooE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE8typeDescIN14titaniummangle17EnumAnotherSampleEE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3ptrI14uncheckedArrayI3intEE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3setIN14titaniummangle10EnumSampleEE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE4procI6string6stringE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3intN10Comparable10ComparableE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3int3int'" + ccodecheck: "'_ZN14titaniummangle8testFuncEN14titaniummangle10EnumSampleE'" + ccodecheck: "'_ZN14titaniummangle8testFuncEN14titaniummangle17EnumAnotherSampleE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3int3int'" + ccodecheck: "'_ZN14titaniummangle8testFuncEN14titaniummangle10EnumSampleE'" + ccodecheck: "'_ZN14titaniummangle8testFuncEN14titaniummangle17EnumAnotherSampleE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE5tupleI3int3intE7cstring'" + ccodecheck: "'_ZN14titaniummangle8testFuncE5tupleI5float5floatE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3ptrI3intE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3ptrIN14titaniummangle3FooEE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3ptrI3ptrI3intEE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3refIN14titaniummangle3FooEE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3varIN14titaniummangle3FooEE5int325int323refIN14titaniummangle3FooEE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE3varI3intE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE9openArrayI6stringE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE5arrayI7range013intE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE9ContainerI3intE'" + ccodecheck: "'_ZN14titaniummangle8testFuncE10Container2I5int325int32E'" + ccodecheck: "'_ZN14titaniummangle8testFuncE9ContainerI10Container2I5int325int32EE'" +""" + +#When debugging this notice that if one check fails, it can be due to any of the above. + +type + Comparable = concept x, y + (x < y) is bool + + Foo = object + a: int32 + b: int32 + + FooTuple = tuple + a: int + b: int + + Container[T] = object + data: T + + Container2[T, T2] = object + data: T + data2: T2 + + Boo = distinct Foo + + Coo = Foo + + Doo = Boo | Foo + + TestProc = proc(a:string): string + +type EnumSample = enum + a, b, c + +type EnumAnotherSample = enum + a, b, c + +proc testFunc(a: set[EnumSample]) = + echo $a + +proc testFunc(a: typedesc) = + echo $a + +proc testFunc(a: ptr Foo) = + echo repr a + +proc testFunc(s: string, a: Coo) = + echo repr a + +proc testFunc(s: int, a: Comparable) = + echo repr a + +proc testFunc(a: TestProc) = + let b = "" + echo repr a("") + +proc testFunc(a: ref Foo) = + echo repr a + +proc testFunc(b: Boo) = + echo repr b + +proc testFunc(a: ptr UncheckedArray[int]) = + echo repr a + +proc testFunc(a: ptr int) = + echo repr a + +proc testFunc(a: ptr ptr int) = + echo repr a + +proc testFunc(e: FooTuple, str: cstring) = + echo e + +proc testFunc(e: (float, float)) = + echo e + +proc testFunc(e: EnumSample) = + echo e + +proc testFunc(e: var int) = + echo e + +proc testFunc(e: var Foo, a, b: int32, refFoo: ref Foo) = + echo e + +proc testFunc(xs: Container[int]) = + let a = 2 + echo xs + +proc testFunc(xs: Container2[int32, int32]) = + let a = 2 + echo xs + +proc testFunc(xs: Container[Container2[int32, int32]]) = + let a = 2 + echo xs + +proc testFunc(xs: seq[int]) = + let a = 2 + echo xs + +proc testFunc(xs: openArray[string]) = + let a = 2 + echo xs + +proc testFunc(xs: array[2, int]) = + let a = 2 + echo xs + +proc testFunc(e: EnumAnotherSample) = + echo e + +proc testFunc(a, b: int) = + echo "hola" + discard + +proc testFunc(a: int, xs: varargs[string]) = + let a = 10 + for x in xs: + echo x + +proc testFunc() = + var a = 2 + var aPtr = a.addr + var foo = Foo() + let refFoo : ref Foo = new(Foo) + let b = Foo().Boo() + let d: Doo = Foo() + testFunc("", Coo()) + testFunc(1, ) + testFunc(b) + testFunc(EnumAnotherSample) + var t = [1, 2] + let uArr = cast[ptr UncheckedArray[int]](t.addr) + testFunc(uArr) + testFunc({}) + testFunc(proc(s:string): string = "test") + testFunc(20, a.int32) + testFunc(20, 2) + testFunc(EnumSample.c) + testFunc(EnumAnotherSample.c) + testFunc((2, 1), "adios") + testFunc((22.1, 1.2)) + testFunc(a.addr) + testFunc(foo.addr) + testFunc(aPtr.addr) + testFunc(refFoo) + testFunc(foo, 2, 1, refFoo) + testFunc(a) + testFunc(@[2, 1, 2]) + testFunc(@["hola"]) + testFunc(2, "hola", "adios") + let arr: array[2, int] = [2, 1] + testFunc(arr) + testFunc(Container[int](data: 10)) + let c2 = Container2[int32, int32](data: 10, data2: 20) + testFunc(c2) + testFunc(Container[Container2[int32, int32]](data: c2)) + + +testFunc() \ No newline at end of file diff --git a/tests/actiontable/tactiontable.nim b/tests/collections/tactiontable.nim similarity index 100% rename from tests/actiontable/tactiontable.nim rename to tests/collections/tactiontable.nim diff --git a/tests/collections/tseq.nim b/tests/collections/tseq.nim index a7a0c724ec..0f8084c787 100644 --- a/tests/collections/tseq.nim +++ b/tests/collections/tseq.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:refc; --mm:orc" output: ''' Hithere, what's your name?Hathere, what's your name? fA13msg1falsefB14msg2truefC15msg3false @@ -11,8 +12,7 @@ FilterIt: [1, 3, 7] Concat: [1, 3, 5, 7, 2, 4, 6] Deduplicate: [1, 2, 3, 4, 5, 7] @[()] -@[1, 42, 3] -@[1, 42, 3] +Minmax: (1, 7) 2345623456 ''' """ @@ -157,42 +157,51 @@ block tsequtils: let someObjSeq = aSeq.mapIt(it.field) echo someObjSeq + block minmax: + doAssert minmax(@[0]) == (0, 0) + doAssert minmax(@[0, 1]) == (0, 1) + doAssert minmax(@[1, 0]) == (0, 1) + doAssert minmax(@[8,2,1,7,3,9,4,0,5]) == (0, 9) + echo "Minmax: ", $(minmax(concat(seq1, seq2))) -block tshallowseq: - proc xxx() = - var x: seq[int] = @[1, 2, 3] - var y: seq[int] - system.shallowCopy(y, x) - y[1] = 42 - echo y - echo x - xxx() +when not defined(nimseqsv2): + block tshallowseq: + proc xxx() = + var x: seq[int] = @[1, 2, 3] + var y: seq[int] + system.shallowCopy(y, x) + y[1] = 42 + doAssert y == @[1, 42, 3] + doAssert x == @[1, 42, 3] + xxx() -block tshallowemptyseq: - proc test() = - var nilSeq: seq[int] = @[] - var emptySeq: seq[int] = newSeq[int]() - block: - var t = @[1,2,3] - shallow(nilSeq) - t = nilSeq - doAssert t == @[] - block: - var t = @[1,2,3] - shallow(emptySeq) - t = emptySeq - doAssert t == @[] - block: - var t = @[1,2,3] - shallowCopy(t, nilSeq) - doAssert t == @[] - block: - var t = @[1,2,3] - shallowCopy(t, emptySeq) - doAssert t == @[] - test() + block tshallowemptyseq: + proc test() = + var nilSeq: seq[int] = @[] + var emptySeq: seq[int] = newSeq[int]() + block: + var t = @[1,2,3] + when defined(gcRefc): + shallow(nilSeq) + t = nilSeq + doAssert t == @[] + block: + var t = @[1,2,3] + when defined(gcRefc): + shallow(emptySeq) + t = emptySeq + doAssert t == @[] + block: + var t = @[1,2,3] + shallowCopy(t, nilSeq) + doAssert t == @[] + block: + var t = @[1,2,3] + shallowCopy(t, emptySeq) + doAssert t == @[] + test() import strutils @@ -214,3 +223,20 @@ for i in 0..100: var test = newSeqOfCap[uint32](1) test.setLen(1) doAssert test[0] == 0, $(test[0], i) + + +# bug #22560 +doAssert len(newSeqOfCap[int](42)) == 0 + +block: # bug #17197 + type Matrix = seq[seq[int]] + + proc needlemanWunsch(sequence1: string, sequence2: string, gap_penal: int8, match: int8, indel_penal: int8): bool = + let seq2_len = sequence2.len + + var grid: Matrix + for i in sequence1: + grid.add(newSeqOfCap[seq[int]](seq2_len)) + result = true + + doAssert needlemanWunsch("ABC", "DEFG", 1, 2, 3) diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index 61197e9f0e..95f9418a00 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -7,7 +7,11 @@ And we get here 3 ''' joinable: false +targets: "c cpp js" """ + +# xxx wrap in a template to test in VM, see https://github.com/timotheecour/Nim/issues/534#issuecomment-769565033 + import hashes, sequtils, tables, algorithm proc sortedPairs[T](t: T): auto = toSeq(t.pairs).sorted @@ -167,22 +171,20 @@ block tableconstr: # NEW: doAssert 56 in 50..100 - doAssert 56 in ..60 + doAssert 56 in 0..60 block ttables2: proc TestHashIntInt() = var tab = initTable[int,int]() - let n = 100_000 + let n = 10 for i in 1..n: tab[i] = i for i in 1..n: var x = tab[i] if x != i : echo "not found ", i - proc run1() = - for i in 1 .. 50: - TestHashIntInt() + TestHashIntInt() # bug #2107 @@ -194,7 +196,6 @@ block ttables2: delTab[5] = 5 - run1() echo "2" block tablesref: @@ -444,3 +445,13 @@ block emptyOrdered: var t2: OrderedTable[int, string] doAssert t1 == t2 +block: # Table[ref, int] + type A = ref object + x: int + var t: OrderedTable[A, int] + let a1 = A(x: 3) + let a2 = A(x: 3) + t[a1] = 10 + t[a2] = 11 + doAssert t[a1] == 10 + doAssert t[a2] == 11 diff --git a/tests/compiler/samplelib.nim b/tests/compiler/samplelib.nim new file mode 100644 index 0000000000..c709717220 --- /dev/null +++ b/tests/compiler/samplelib.nim @@ -0,0 +1,2 @@ +# Sample library used by tcmdlineclib.nim +proc test(): int {.cdecl, exportc, dynlib.} = 123 diff --git a/tests/compiler/t12684.nim b/tests/compiler/t12684.nim new file mode 100644 index 0000000000..a5f33d2c90 --- /dev/null +++ b/tests/compiler/t12684.nim @@ -0,0 +1,7 @@ +discard """ + cmd: "nim check --hints:off --warnings:off $file" + errormsg: "undeclared identifier: 'Undeclared'" +""" + +var x: Undeclared +import compiler/nimeval diff --git a/tests/compiler/tasm.nim b/tests/compiler/tasm.nim new file mode 100644 index 0000000000..9f60231e0b --- /dev/null +++ b/tests/compiler/tasm.nim @@ -0,0 +1,15 @@ +proc testAsm() = + let src = 41 + var dst = 0 + + asm """ + mov %1, %0\n\t + add $1, %0 + : "=r" (`dst`) + : "r" (`src`)""" + + doAssert dst == 42 + +when defined(gcc) or defined(clang) and not defined(cpp): + {.passc: "-std=c99".} + testAsm() \ No newline at end of file diff --git a/tests/compiler/tbrees.nim b/tests/compiler/tbtrees.nim similarity index 99% rename from tests/compiler/tbrees.nim rename to tests/compiler/tbtrees.nim index 5f6482ed94..973c264202 100644 --- a/tests/compiler/tbrees.nim +++ b/tests/compiler/tbtrees.nim @@ -68,7 +68,7 @@ proc main = when true: var b2 = initBTree[int, string]() var t2 = initTable[int, string]() - const iters = 100_000 + const iters = 100 for i in 1..iters: let x = rand(high(int)) if not t2.hasKey(x): diff --git a/tests/compiler/tcmdlineclib.nim b/tests/compiler/tcmdlineclib.nim new file mode 100644 index 0000000000..c06d6f1034 --- /dev/null +++ b/tests/compiler/tcmdlineclib.nim @@ -0,0 +1,3 @@ +proc test(): int {.importc, cdecl.} + +doAssert test() == 123 diff --git a/tests/compiler/tcmdlineclib.nims b/tests/compiler/tcmdlineclib.nims new file mode 100644 index 0000000000..f7899d92e7 --- /dev/null +++ b/tests/compiler/tcmdlineclib.nims @@ -0,0 +1,10 @@ +import os + +selfExec "c --app:lib " & (projectDir() / "samplelib.nim") +switch("clibdir", projectDir()) +--clib:samplelib + +# Make test executable can load sample shared library. +# `-rpath` option doesn't work and ignored on Windows. +# But the dll file in same directory as executable file is loaded. +switch("passL", "-Wl,-rpath," & projectDir()) diff --git a/tests/compiler/tdebugutils.nim b/tests/compiler/tdebugutils.nim new file mode 100644 index 0000000000..50b15cb782 --- /dev/null +++ b/tests/compiler/tdebugutils.nim @@ -0,0 +1,38 @@ +discard """ + joinable: false + disabled: true +""" + +#[ +This test illustrates some features of `debugutils` to debug the compiler. + +## example +this shows how to enable compiler logging just for a section of user code, +without generating tons of unrelated log messages for code you're not interested +in debugging. + +```sh +# enable some debugging code, e.g. the `when false:` block in `semExpr` +nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim +bin/nim_temp c tests/compiler/tdebugutils.nim +``` + +(use --filenames:abs for abs files) + +## result +("<", "tdebugutils.nim(16, 3)", {.define(nimCompilerDebug).}, nil) +(">", "tdebugutils.nim(17, 3)", let a = 2.5 * 3, {}, nkLetSection) +(">", "tdebugutils.nim(17, 15)", 2.5 * 3, {efAllowDestructor, efWantValue}, nkInfix) +(">", "tdebugutils.nim(17, 11)", 2.5, {efAllowStmt, efDetermineType, efOperand}, nkFloatLit) +("<", "tdebugutils.nim(17, 11)", 2.5, float64) +(">", "tdebugutils.nim(17, 17)", 3, {efAllowStmt, efDetermineType, efOperand}, nkIntLit) +("<", "tdebugutils.nim(17, 17)", 3, int literal(3)) +("<", "tdebugutils.nim(17, 15)", 2.5 * 3, float) +("<", "tdebugutils.nim(17, 3)", let a = 2.5 * 3, nil) +(">", "tdebugutils.nim(18, 3)", {.undef(nimCompilerDebug).}, {}, nkPragma) +]# + +proc main = + {.define(nimCompilerDebug).} + let a = 2.5 * 3 + {.undef(nimCompilerDebug).} diff --git a/tests/compiler/tgrammar.nim b/tests/compiler/tgrammar.nim new file mode 100644 index 0000000000..772d0f0ccb --- /dev/null +++ b/tests/compiler/tgrammar.nim @@ -0,0 +1,7 @@ +discard """ + matrix: "-d:nimTestGrammar" +""" + +import compiler/parser + +checkSameGrammar() diff --git a/tests/compiler/tnimblecmd.nim b/tests/compiler/tnimblecmd.nim index d39179be18..53bce46259 100644 --- a/tests/compiler/tnimblecmd.nim +++ b/tests/compiler/tnimblecmd.nim @@ -1,26 +1,75 @@ -include compiler/[nimblecmd] +include compiler/[nimblecmd], sets proc v(s: string): Version = s.newVersion -# #head is special in the sense that it's assumed to always be newest. -doAssert v"1.0" < v"#head" -doAssert v"1.0" < v"1.1" -doAssert v"1.0.1" < v"1.1" -doAssert v"1" < v"1.1" -doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer. -doAssert v"#a111" < v"#head" -let conf = newConfigRef() -var rr = newStringTable() -addPackage conf, rr, "irc-#a111", unknownLineInfo -addPackage conf, rr, "irc-#head", unknownLineInfo -addPackage conf, rr, "irc-0.1.0", unknownLineInfo -#addPackage conf, rr, "irc", unknownLineInfo -#addPackage conf, rr, "another", unknownLineInfo -addPackage conf, rr, "another-0.1", unknownLineInfo +proc testVersionsComparison = + # #head is special in the sense that it's assumed to always be newest. + doAssert v"1.0" < v"#head" + doAssert v"1.0" < v"1.1" + doAssert v"1.0.1" < v"1.1" + doAssert v"1" < v"1.1" + doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer. + doAssert v"#a111" < v"#head" -addPackage conf, rr, "ab-0.1.3", unknownLineInfo -addPackage conf, rr, "ab-0.1", unknownLineInfo -addPackage conf, rr, "justone-1.0", unknownLineInfo +proc testAddPackageWithoutChecksum = + ## For backward compatibility it is not required all packages to have a + ## sha1 checksum at the end of the name of the Nimble cache directory. + ## This way a new compiler will be able to work with an older Nimble. -doAssert toSeq(rr.chosen) == - @["irc-#head", "ab-0.1.3", "justone-1.0", "another-0.1"] + let conf = newConfigRef() + var rr: PackageInfo + + addPackage conf, rr, "irc-#a111", unknownLineInfo + addPackage conf, rr, "irc-#head", unknownLineInfo + addPackage conf, rr, "irc-0.1.0", unknownLineInfo + + addPackage conf, rr, "another-0.1", unknownLineInfo + + addPackage conf, rr, "ab-0.1.3", unknownLineInfo + addPackage conf, rr, "ab-0.1", unknownLineInfo + addPackage conf, rr, "justone-1.0", unknownLineInfo + + doAssert toSeq(rr.chosen).toHashSet == + ["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"].toHashSet + +proc testAddPackageWithChecksum = + let conf = newConfigRef() + var rr: PackageInfo + + # in the case of packages with the same version, but different checksums for + # now the first one will be chosen + + addPackage conf, rr, "irc-#a111-DBC1F902CB79946E990E38AF51F0BAD36ACFABD9", + unknownLineInfo + addPackage conf, rr, "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD1", + unknownLineInfo + addPackage conf, rr, "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD2", + unknownLineInfo + addPackage conf, rr, "irc-0.1.0-6EE6DE936B32E82C7DBE526DA3463574F6568FAF", + unknownLineInfo + + addPackage conf, rr, "another-0.1", unknownLineInfo + addPackage conf, rr, "another-0.1-F07EE6040579F0590608A8FD34F5F2D91D859340", + unknownLineInfo + + addPackage conf, rr, "ab-0.1.3-34BC3B72CE46CF5A496D1121CFEA7369385E9EA2", + unknownLineInfo + addPackage conf, rr, "ab-0.1.3-24BC3B72CE46CF5A496D1121CFEA7369385E9EA2", + unknownLineInfo + addPackage conf, rr, "ab-0.1-A3CFFABDC4759F7779D541F5E031AED17169390A", + unknownLineInfo + + # lower case hex digits is also a valid sha1 checksum + addPackage conf, rr, "justone-1.0-f07ee6040579f0590608a8fd34f5f2d91d859340", + unknownLineInfo + + doAssert toSeq(rr.chosen).toHashSet == [ + "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD1", + "another-0.1", + "ab-0.1.3-34BC3B72CE46CF5A496D1121CFEA7369385E9EA2", + "justone-1.0-f07ee6040579f0590608a8fd34f5f2d91d859340" + ].toHashSet + +testVersionsComparison() +testAddPackageWithoutChecksum() +testAddPackageWithChecksum() diff --git a/tests/compiler/tprefixmatches.nim b/tests/compiler/tprefixmatches.nim index a89a6f613a..6a31867298 100644 --- a/tests/compiler/tprefixmatches.nim +++ b/tests/compiler/tprefixmatches.nim @@ -5,7 +5,7 @@ macro check(val, body: untyped): untyped = result = newStmtList() expectKind body, nnkStmtList for b in body: - expectKind b, nnkPar + expectKind b, nnkTupleConstr expectLen b, 2 let p = b[0] let s = b[1] diff --git a/tests/compilerapi/tcompilerapi.nim b/tests/compilerapi/tcompilerapi.nim index ab995cb6db..2d6cfa0ca3 100644 --- a/tests/compilerapi/tcompilerapi.nim +++ b/tests/compilerapi/tcompilerapi.nim @@ -23,7 +23,7 @@ proc initInterpreter(script: string): Interpreter = proc main() = let i = initInterpreter("myscript.nim") - i.implementRoutine("*", "exposed", "addFloats", proc (a: VmArgs) = + i.implementRoutine("nim", "exposed", "addFloats", proc (a: VmArgs) = setResult(a, getFloat(a, 0) + getFloat(a, 1) + getFloat(a, 2)) ) i.evalScript() @@ -69,3 +69,30 @@ block error_hook: doAssertRaises(VMQuit): i.evalScript() + +block resetmacrocache: + let std = findNimStdLibCompileTime() + let intr = createInterpreter("script.nim", [std, std / "pure", std / "core"]) + proc evalString(intr: Interpreter; code: string) = + let stream = llStreamOpen(code) + intr.evalScript(stream) + llStreamClose(stream) + let code = """ +import std/[macrocache, macros] +static: + let counter = CacheCounter"valTest" + inc counter + assert counter.value == 1 + + const mySeq = CacheSeq"addTest" + mySeq.add(newLit(5)) + mySeq.add(newLit("hello ic")) + assert mySeq.len == 2 + + const mcTable = CacheTable"subTest" + mcTable["toAdd"] = newStmtList() #would crash if not empty + assert mcTable.len == 1 +""" + intr.evalString(code) + intr.evalString(code) + destroyInterpreter(intr) \ No newline at end of file diff --git a/tests/compiles/mstaticlib.nim b/tests/compiles/mstaticlib.nim new file mode 100644 index 0000000000..6ed593691a --- /dev/null +++ b/tests/compiles/mstaticlib.nim @@ -0,0 +1 @@ +echo 1234 \ No newline at end of file diff --git a/tests/compiles/t8630.nim b/tests/compiles/t8630.nim index aa2be11cd5..8b447d0610 100644 --- a/tests/compiles/t8630.nim +++ b/tests/compiles/t8630.nim @@ -7,7 +7,7 @@ bar proc test(strings: seq[string]) = for s in strings: - var p3 = unsafeAddr(s) + var p3 = addr(s) echo p3[] test(@["foo", "bar"]) diff --git a/tests/compiles/trecursive_generic_in_compiles.nim b/tests/compiles/trecursive_generic_in_compiles.nim index 88d3faeedf..ab31ad0f5e 100644 --- a/tests/compiles/trecursive_generic_in_compiles.nim +++ b/tests/compiles/trecursive_generic_in_compiles.nim @@ -50,7 +50,7 @@ proc `==`*[T](xs, ys: List[T]): bool = proc asList*[T](xs: varargs[T]): List[T] = ## Creates list from varargs - proc initListImpl(i: int, xs: openarray[T]): List[T] = + proc initListImpl(i: int, xs: openArray[T]): List[T] = if i > high(xs): Nil[T]() else: diff --git a/tests/compiles/tstaticlib.nim b/tests/compiles/tstaticlib.nim new file mode 100644 index 0000000000..a18b59204c --- /dev/null +++ b/tests/compiles/tstaticlib.nim @@ -0,0 +1,22 @@ +import std/[os, osproc, strformat] + + +const dir = "tests/compiles" +const fileName = dir / "mstaticlib.nim" +const nim = getCurrentCompilerExe() + +block: # bug #18578 + const libName = dir / "tstaticlib1.a" + let (_, status) = execCmdEx(fmt"{nim} c -o:{libName} --app:staticlib {fileName}") + doAssert status == 0 + doAssert fileExists(libName) + removeFile(libName) + +block: # bug #16947 + const libName = dir / "tstaticlib2.a" + writeFile(libName, "echo 124") + doAssert fileExists(libName) + let (_, status) = execCmdEx(fmt"{nim} c -o:{libName} --app:staticlib {fileName}") + doAssert status == 0 + doAssert fileExists(libName) + removeFile(libName) diff --git a/tests/concepts/t18409.nim b/tests/concepts/t18409.nim new file mode 100644 index 0000000000..0edba2d31e --- /dev/null +++ b/tests/concepts/t18409.nim @@ -0,0 +1,37 @@ +discard """ + action: "compile" +""" + +# A vector space over a field F concept. +type VectorSpace*[F] = concept x, y, type V + vector_add(x, y) is V + scalar_mul(x, F) is V + dimension(V) is Natural + +# Real numbers (here floats) form a vector space. +func vector_add*(v: float, w: float): float = v + w +func scalar_mul*(v: float, s: float): float = v * s +func dimension*(x: typedesc[float]): Natural = 1 + +# 2-tuples of real numbers form a vector space. +func vector_add*(v, w: (float, float)): (float, float) = + (vector_add(v[0], w[0]), vector_add(v[1], w[1])) + +func scalar_mul*(v: (float, float), s: float): (float, float) = + (scalar_mul(v[0], s), scalar_mul(v[1], s)) + +func dimension*(x: typedesc[(float, float)]): Natural = 2 + +# Check concept requirements. +assert float is VectorSpace +assert (float, float) is VectorSpace + +# Commutivity axiom for vector spaces over the same field. +func axiom_commutivity*[F](u, v: VectorSpace[F]): bool = + vector_add(u, v) == vector_add(v, u) + +# This is okay. +assert axiom_commutivity(2.2, 3.3) + +# This is not. +assert axiom_commutivity((2.2, 3.3), (4.4, 5.5)) diff --git a/tests/concepts/t19730.nim b/tests/concepts/t19730.nim new file mode 100644 index 0000000000..575d45dda6 --- /dev/null +++ b/tests/concepts/t19730.nim @@ -0,0 +1,20 @@ +discard """ + output: '''1.01.01.01.0 +1.01.01.01.0 +''' +""" + +type + Color = concept c + c.r is SomeFloat + c.g is SomeFloat + c.b is SomeFloat + c.a is SomeFloat + +proc useColor(color: Color) = + echo(color.r, color.g, color.b, color.a) + +let color = (r: 1.0, g: 1.0, b: 1.0, a: 1.0) +useColor(color) + +useColor((r: 1.0, g: 1.0, b: 1.0, a: 1.0)) diff --git a/tests/concepts/t20237.nim b/tests/concepts/t20237.nim new file mode 100644 index 0000000000..175c7a9d1d --- /dev/null +++ b/tests/concepts/t20237.nim @@ -0,0 +1,3 @@ +type Foo* = concept + ## doc comment + proc foo(x: Self) diff --git a/tests/concepts/t3330.nim b/tests/concepts/t3330.nim index b92af54853..901f8d2f40 100644 --- a/tests/concepts/t3330.nim +++ b/tests/concepts/t3330.nim @@ -1,4 +1,5 @@ discard """ +matrix: "--mm:refc" errormsg: "type mismatch: got " nimout: ''' t3330.nim(70, 4) Error: type mismatch: got @@ -48,7 +49,6 @@ expression: test(bar)''' - ## line 60 type Foo[T] = concept k diff --git a/tests/concepts/t976.nim b/tests/concepts/t976.nim new file mode 100644 index 0000000000..776d538271 --- /dev/null +++ b/tests/concepts/t976.nim @@ -0,0 +1,57 @@ +discard """ + output: ''' +Printable +''' +joinable: false +""" + +#[ + The converter is a proper example of a confounding variable + Moved to an isolated file +]# + +type + Obj1[T] = object + v: T +converter toObj1[T](t: T): Obj1[T] = + return Obj1[T](v: t) +block t976: + type + int1 = distinct int + int2 = distinct int + int1g = concept x + x is int1 + int2g = concept x + x is int2 + + proc take[T: int1g](value: int1) = + when T is int2: + static: error("killed in take(int1)") + + proc take[T: int2g](vale: int2) = + when T is int1: + static: error("killed in take(int2)") + + var i1: int1 = 1.int1 + var i2: int2 = 2.int2 + + take[int1](i1) + take[int2](i2) + + template reject(e) = + static: assert(not compiles(e)) + + reject take[string](i2) + reject take[int1](i2) + + # bug #6249 + type + Obj2 = ref object + PrintAble = concept x + $x is string + + proc `$`[T](nt: Obj1[T]): string = + when T is PrintAble: result = "Printable" + else: result = "Non Printable" + + echo Obj2() \ No newline at end of file diff --git a/tests/concepts/tconcepts.nim b/tests/concepts/tconcepts.nim index acdff6f246..ea3ddc4017 100644 --- a/tests/concepts/tconcepts.nim +++ b/tests/concepts/tconcepts.nim @@ -31,6 +31,7 @@ e 20 10 5 +9 ''' """ @@ -438,3 +439,13 @@ import mvarconcept block tvar: # bug #2346, bug #2404 echo randomInt(5) + +block tcomment: + type + Foo = concept + ## Some comment + proc bar(x: Self) + + proc bar(x: int) = echo x + proc foo(x: Foo) = x.bar + foo(9) diff --git a/tests/concepts/tconcepts_issues.nim b/tests/concepts/tconcepts_issues.nim index c76049bdde..c6d0267c5c 100644 --- a/tests/concepts/tconcepts_issues.nim +++ b/tests/concepts/tconcepts_issues.nim @@ -1,7 +1,6 @@ discard """ output: ''' 20.0 USD -Printable true true true @@ -27,6 +26,8 @@ false true -1 Meow +10 0.0 +1 2.0 ''' joinable: false """ @@ -76,55 +77,6 @@ block t3414: let s2 = s1.find(10) - -type - Obj1[T] = object - v: T -converter toObj1[T](t: T): Obj1[T] = - return Obj1[T](v: t) -block t976: - type - int1 = distinct int - int2 = distinct int - int1g = concept x - x is int1 - int2g = concept x - x is int2 - - proc take[T: int1g](value: int1) = - when T is int2: - static: error("killed in take(int1)") - - proc take[T: int2g](vale: int2) = - when T is int1: - static: error("killed in take(int2)") - - var i1: int1 = 1.int1 - var i2: int2 = 2.int2 - - take[int1](i1) - take[int2](i2) - - template reject(e) = - static: assert(not compiles(e)) - - reject take[string](i2) - reject take[int1](i2) - - # bug #6249 - type - Obj2 = ref object - PrintAble = concept x - $x is string - - proc `$`[T](nt: Obj1[T]): string = - when T is PrintAble: result = "Printable" - else: result = "Non Printable" - - echo Obj2() - - - block t1128: type TFooContainer[T] = object @@ -324,15 +276,15 @@ block t6691: block t6782: type Reader = concept c - c.read(openarray[byte], int, int) is int + c.read(openArray[byte], int, int) is int Rdr = concept c - c.rd(openarray[byte], int, int) is int + c.rd(openArray[byte], int, int) is int type TestFile = object - proc read(r: TestFile, dest: openarray[byte], offset: int, limit: int): int = + proc read(r: TestFile, dest: openArray[byte], offset: int, limit: int): int = result = 0 - proc rd(r: TestFile, dest: openarray[byte], offset: int, limit: int): int = + proc rd(r: TestFile, dest: openArray[byte], offset: int, limit: int): int = result = 0 doAssert TestFile is Reader @@ -484,10 +436,10 @@ type var address = pointer(nil) proc prod(r: var QuadraticExt, b: QuadraticExt) = if address == nil: - address = unsafeAddr b + address = addr b prod(r, b) else: - assert address == unsafeAddr b + assert address == addr b type Fp2[N: static int, T] {.byref.} = object @@ -500,3 +452,105 @@ var r, b: Fp2[6, uint64] prod(r, b) + +block: # bug #21263 + type + DateDayFraction = concept # no T, an atom + proc date(a: Self): int + proc fraction(b: Self): float + Date = distinct int + DateDayFractionImpl = object + date : int + fraction : float + + proc date(a: Date): int = a.int + proc fraction(a:Date): float = 0.0 + + proc date(a: DateDayFractionImpl): int = a.date + proc fraction(b: DateDayFractionImpl): float = b.fraction + + + proc print(a: DateDayFraction) = + echo a.date, " ", a.fraction + + print(10.Date) # ok + print(DateDayFractionImpl(date: 1, fraction: 2)) # error + +import sets +import deques + +type AnyTree[V] = concept t, type T + for v in t.leaves(V): + v is V + +type BreadthOrder[V] = ref object + frontier: Deque[V] + visited: HashSet[V] + +proc expand[V, T](order: ref BreadthOrder[T], tree: AnyTree[V], node: V, transform: (V) -> (T)) = + for leaf in tree.leaves(node): + if not order.visited.containsOrIncl(transform(leaf)): + order.frontier.addLast(transform(leaf)) + +proc hasNext[V](order: ref BreadthOrder[V]): bool = + order.frontier.len > 0 + +proc init[V](_: typedesc[BreadthOrder]): ref BreadthOrder[V] = + result.new() + result[] = BreadthOrder[V](frontier: initDeque[V](), visited: initHashSet[V]()) + +proc popNext[V](order: ref BreadthOrder[V]): V = + order.frontier.popFirst() + +type LevelNode[V] = tuple + depth: uint + node: V + +proc depthOf*[V](orderType: typedesc[BreadthOrder], tree: AnyTree[V], root, goal: V): uint = + if root == goal: + return 0 + var order = init[LevelNode[V]](orderType) + order.expand(tree, root, (leaf) => (1.uint, leaf)) + while order.hasNext(): + let depthNode: LevelNode[V] = order.popNext() + if depthNode.node == goal: + return depthNode.depth + order.expand(tree, depthNode.node, (leaf) => (depthNode.depth + 1, leaf)) + +type CappedStringTree = ref object + symbols: string + cap: Natural + +iterator leaves*(t: CappedStringTree, s: string): string = + if s.len < t.cap: + for c in t.symbols: + yield s & c + +block: # bug #12852 + var tree = CappedStringTree(symbols: "^v><", cap: 5) + + doAssert BreadthOrder.depthOf(tree, "", ">>>") == 3 + +block: #bug #22723 + type + Node = concept n, type T + for i in n.children: + i is T + n.parent is T + + Nd = ref object + parent: Nd + children: seq[Nd] + + proc addChild(parent, child: Node) = + parent.children.add(child) + child.parent = parent + + proc foo = + var + a = Nd() + b = Nd() + a.addChild(b) + doAssert a.children.len == 1 + + foo() diff --git a/tests/concepts/texplain.nim b/tests/concepts/texplain.nim index de3d540cdd..8cf04ae826 100644 --- a/tests/concepts/texplain.nim +++ b/tests/concepts/texplain.nim @@ -1,45 +1,33 @@ discard """ cmd: "nim c --verbosity:0 --colors:off $file" nimout: ''' -texplain.nim(164, 10) Hint: Non-matching candidates for e(y) +texplain.nim(162, 10) Hint: Non-matching candidates for e(y) proc e(i: int): int first type mismatch at position: 1 required type for i: int but expression 'y' is of type: MatchingType -texplain.nim(167, 7) Hint: Non-matching candidates for e(10) +texplain.nim(165, 7) Hint: Non-matching candidates for e(10) proc e(o: ExplainedConcept): int first type mismatch at position: 1 required type for o: ExplainedConcept but expression '10' is of type: int literal(10) -texplain.nim(130, 6) ExplainedConcept: undeclared field: 'foo' -texplain.nim(130, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(130, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(130, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed -texplain.nim(131, 6) ExplainedConcept: undeclared field: 'bar' -texplain.nim(131, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(131, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(131, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed +texplain.nim(128, 6) ExplainedConcept: undeclared field: 'foo' +texplain.nim(128, 5) ExplainedConcept: concept predicate failed +texplain.nim(129, 6) ExplainedConcept: undeclared field: 'bar' +texplain.nim(128, 5) ExplainedConcept: concept predicate failed -texplain.nim(170, 10) Hint: Non-matching candidates for e(10) +texplain.nim(168, 10) Hint: Non-matching candidates for e(10) proc e(o: ExplainedConcept): int first type mismatch at position: 1 required type for o: ExplainedConcept but expression '10' is of type: int literal(10) -texplain.nim(130, 6) ExplainedConcept: undeclared field: 'foo' -texplain.nim(130, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(130, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(130, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed -texplain.nim(131, 6) ExplainedConcept: undeclared field: 'bar' -texplain.nim(131, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(131, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(131, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed +texplain.nim(128, 6) ExplainedConcept: undeclared field: 'foo' +texplain.nim(128, 5) ExplainedConcept: concept predicate failed +texplain.nim(129, 6) ExplainedConcept: undeclared field: 'bar' +texplain.nim(128, 5) ExplainedConcept: concept predicate failed -texplain.nim(174, 20) Error: type mismatch: got +texplain.nim(172, 20) Error: type mismatch: got but expected one of: proc e(i: int): int first type mismatch at position: 1 @@ -49,11 +37,11 @@ proc e(o: ExplainedConcept): int first type mismatch at position: 1 required type for o: ExplainedConcept but expression 'n' is of type: NonMatchingType -texplain.nim(174, 9) template/generic instantiation of `assert` from here -texplain.nim(130, 5) ExplainedConcept: concept predicate failed +texplain.nim(172, 9) template/generic instantiation of `assert` from here +texplain.nim(128, 5) ExplainedConcept: concept predicate failed expression: e(n) -texplain.nim(175, 20) Error: type mismatch: got +texplain.nim(173, 20) Error: type mismatch: got but expected one of: proc r(i: string): int first type mismatch at position: 1 @@ -63,15 +51,15 @@ proc r(o: RegularConcept): int first type mismatch at position: 1 required type for o: RegularConcept but expression 'n' is of type: NonMatchingType -texplain.nim(175, 9) template/generic instantiation of `assert` from here -texplain.nim(134, 5) RegularConcept: concept predicate failed +texplain.nim(173, 9) template/generic instantiation of `assert` from here +texplain.nim(132, 5) RegularConcept: concept predicate failed proc r[T](a: SomeNumber; b: T; c: auto) first type mismatch at position: 1 required type for a: SomeNumber but expression 'n' is of type: NonMatchingType expression: r(n) -texplain.nim(176, 20) Hint: Non-matching candidates for r(y) +texplain.nim(174, 20) Hint: Non-matching candidates for r(y) proc r(i: string): int first type mismatch at position: 1 required type for i: string @@ -81,28 +69,20 @@ proc r[T](a: SomeNumber; b: T; c: auto) required type for a: SomeNumber but expression 'y' is of type: MatchingType -texplain.nim(184, 2) Error: type mismatch: got +texplain.nim(182, 2) Error: type mismatch: got but expected one of: proc f(o: NestedConcept) first type mismatch at position: 1 required type for o: NestedConcept but expression 'y' is of type: MatchingType -texplain.nim(134, 6) RegularConcept: undeclared field: 'foo' -texplain.nim(134, 6) RegularConcept: undeclared field: '.' -texplain.nim(134, 6) RegularConcept: expression '.' cannot be called -texplain.nim(134, 6) RegularConcept: expression '' has no type (or is ambiguous) -texplain.nim(134, 5) RegularConcept: concept predicate failed -texplain.nim(135, 6) RegularConcept: undeclared field: 'bar' -texplain.nim(135, 6) RegularConcept: undeclared field: '.' -texplain.nim(135, 6) RegularConcept: expression '.' cannot be called -texplain.nim(135, 6) RegularConcept: expression '' has no type (or is ambiguous) -texplain.nim(134, 5) RegularConcept: concept predicate failed -texplain.nim(138, 5) NestedConcept: concept predicate failed +texplain.nim(132, 6) RegularConcept: undeclared field: 'foo' +texplain.nim(132, 5) RegularConcept: concept predicate failed +texplain.nim(133, 6) RegularConcept: undeclared field: 'bar' +texplain.nim(132, 5) RegularConcept: concept predicate failed +texplain.nim(136, 5) NestedConcept: concept predicate failed expression: f(y)''' errormsg: "type mismatch: got " - line: 184 - """ @@ -123,7 +103,25 @@ expression: f(y)''' -# line 120 HERE + + + + + + + + + + + + + + + + + + +# line 124 HERE type ExplainedConcept {.explain.} = concept o diff --git a/tests/concepts/tspec.nim b/tests/concepts/tspec.nim index 1bca3c11b8..52f13a40a8 100644 --- a/tests/concepts/tspec.nim +++ b/tests/concepts/tspec.nim @@ -7,8 +7,7 @@ discard """ 2 3 yes int -string int -true''' +string int''' joinable: false """ @@ -102,5 +101,5 @@ type Monoid = concept proc z(x: typedesc[int]): int = 0 -echo(int is Monoid) +doAssert int is Monoid diff --git a/tests/concepts/tusertypeclasses.nim b/tests/concepts/tusertypeclasses.nim index c7104f2a6c..83e2b176eb 100644 --- a/tests/concepts/tusertypeclasses.nim +++ b/tests/concepts/tusertypeclasses.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:refc" output: '''Sortable Sortable Container @@ -9,6 +10,8 @@ int ''' """ +# todo wait for https://github.com/nim-lang/Nim/pull/20380 + import typetraits template reject(expr) = assert(not compiles(x)) diff --git a/tests/concepts/tusertypeclasses2.nim b/tests/concepts/tusertypeclasses2.nim index c9978f6ef7..6132bc2d86 100644 --- a/tests/concepts/tusertypeclasses2.nim +++ b/tests/concepts/tusertypeclasses2.nim @@ -1,3 +1,7 @@ +discard """ + targets: "c js" +""" + block: type hasFieldX = concept z @@ -42,3 +46,18 @@ block: foo2(x) foo3(x) foo4(x) + +block: # bug #9550 + block: + type Foo = concept c + for v in c: (v is char) + + func foo(c: Foo) = (for v in c: discard) + foo @['a', 'b' ,'c'] + + block: + type Foo = concept c + for v in c: (v is char) + + func foo(c: Foo) = (for v in c: discard) + foo ['a', 'b' ,'c'] diff --git a/tests/concepts/twrapconcept.nim b/tests/concepts/twrapconcept.nim index 377b63afec..c3dea2ff90 100644 --- a/tests/concepts/twrapconcept.nim +++ b/tests/concepts/twrapconcept.nim @@ -1,7 +1,6 @@ discard """ errormsg: "type mismatch: got " - line: 21 - nimout: "twrapconcept.nim(11, 5) Foo: concept predicate failed" + nimout: "twrapconcept.nim(10, 5) Foo: concept predicate failed" """ # https://github.com/nim-lang/Nim/issues/5127 diff --git a/tests/config.nims b/tests/config.nims index 41edf00053..690123c4a3 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -6,9 +6,13 @@ switch("path", "$lib/../testament/lib") ## prevent common user config settings to interfere with testament expectations ## Indifidual tests can override this if needed to test for these options. switch("colors", "off") -switch("listFullPaths", "off") + switch("excessiveStackTrace", "off") -switch("spellSuggest", "0") + +when (NimMajor, NimMinor, NimPatch) >= (1,5,1): + # to make it easier to test against older nim versions, (best effort only) + switch("filenames", "legacyRelProj") + switch("spellSuggest", "0") # for std/unittest switch("define", "nimUnittestOutputLevel:PRINT_FAILURES") @@ -24,6 +28,20 @@ hint("Processing", off) # switch("define", "nimTestsEnableFlaky") # switch("hint", "ConvFromXtoItselfNotNeeded") +# switch("warningAsError", "InheritFromException") # would require fixing a few tests -# experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 -switch("define", "nimExperimentalAsyncjsThen") +# experimental APIs are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 +# sync with `kochdocs.docDefines` or refactor. +switch("define", "nimExperimentalLinenoiseExtra") + +# preview APIs are expected to be the new default in upcoming versions +switch("define", "nimPreviewFloatRoundtrip") +#switch("define", "nimPreviewDotLikeOps") # deprecated? +switch("define", "nimPreviewJsonutilsHoleyEnum") +switch("define", "nimPreviewHashRef") +switch("define", "nimPreviewRangeDefault") +switch("define", "nimPreviewNonVarDestructor") + +switch("warningAserror", "UnnamedBreak") +switch("legacy", "verboseTypeMismatch") +switch("experimental", "vtables") diff --git a/tests/constr/tglobal.nim b/tests/constr/tglobal.nim deleted file mode 100644 index 056ac9f81d..0000000000 --- a/tests/constr/tglobal.nim +++ /dev/null @@ -1,7 +0,0 @@ -discard """ -output: "0" -""" - -# b.nim -import a_module -echo foo() \ No newline at end of file diff --git a/tests/constructors/a.nim b/tests/constructors/a.nim new file mode 100644 index 0000000000..03788fc57a --- /dev/null +++ b/tests/constructors/a.nim @@ -0,0 +1,2 @@ +type A* = object + a: uint8 \ No newline at end of file diff --git a/tests/constructors/b.nim b/tests/constructors/b.nim new file mode 100644 index 0000000000..437dd05504 --- /dev/null +++ b/tests/constructors/b.nim @@ -0,0 +1,2 @@ +type B* = object +proc A*(a, b: float): B = discard \ No newline at end of file diff --git a/tests/constructors/t18990.nim b/tests/constructors/t18990.nim new file mode 100644 index 0000000000..2f60f3c2c9 --- /dev/null +++ b/tests/constructors/t18990.nim @@ -0,0 +1,3 @@ +import a, b +discard A(1f, 1f) # works +proc x(b = A(1f, 1f)) = discard # doesn't work \ No newline at end of file diff --git a/tests/constr/tconstr1.nim b/tests/constructors/tconstr1.nim similarity index 100% rename from tests/constr/tconstr1.nim rename to tests/constructors/tconstr1.nim diff --git a/tests/constr/tconstr2.nim b/tests/constructors/tconstr2.nim similarity index 100% rename from tests/constr/tconstr2.nim rename to tests/constructors/tconstr2.nim diff --git a/tests/controlflow/tcontrolflow.nim b/tests/controlflow/tcontrolflow.nim index 258f3f50d2..dd21a2bb67 100644 --- a/tests/controlflow/tcontrolflow.nim +++ b/tests/controlflow/tcontrolflow.nim @@ -19,7 +19,7 @@ block tbreak: run = false block myblock: if true: - break + break myblock echo "leaving myblock" x = true doAssert(x) @@ -95,3 +95,22 @@ block tnestif: else: writeLine(stdout, "looks like Python") #OUT i == 2 + +# bug https://github.com/nim-lang/RFCs/issues/451 +for i in 1..2: # works + break + +block: # works + for i in 1..2: + break + +block: # works + block: + discard 12 + 3 + for i in 1..2: + break + +block named: # works + if true: + break named + doAssert false, "not reached" diff --git a/tests/controlflow/tunamedbreak.nim b/tests/controlflow/tunamedbreak.nim new file mode 100644 index 0000000000..46113cabcf --- /dev/null +++ b/tests/controlflow/tunamedbreak.nim @@ -0,0 +1,15 @@ + +discard """ + cmd: "nim check $file" + action: "reject" + nimout: ''' +tunamedbreak.nim(12, 5) Error: Using an unnamed break in a block is deprecated; Use a named block with a named break instead [UnnamedBreak] +tunamedbreak.nim(15, 3) Error: Using an unnamed break in a block is deprecated; Use a named block with a named break instead [UnnamedBreak] + ''' +""" +for i in 1..2: # errors + block: + break + +block: # errors + break diff --git a/tests/controlflow/tunreachable.nim b/tests/controlflow/tunreachable.nim new file mode 100644 index 0000000000..06321ce8a9 --- /dev/null +++ b/tests/controlflow/tunreachable.nim @@ -0,0 +1,79 @@ +discard """ + cmd: "nim check --warningAsError:UnreachableCode $file" + action: "reject" + nimout: ''' +tunreachable.nim(26, 3) Error: unreachable code after 'return' statement or '{.noReturn.}' proc [UnreachableCode] +tunreachable.nim(33, 3) Error: unreachable code after 'return' statement or '{.noReturn.}' proc [UnreachableCode] +tunreachable.nim(42, 3) Error: unreachable code after 'return' statement or '{.noReturn.}' proc [UnreachableCode] +tunreachable.nim(65, 5) Error: unreachable code after 'return' statement or '{.noReturn.}' proc [UnreachableCode] +tunreachable.nim(77, 5) Error: unreachable code after 'return' statement or '{.noReturn.}' proc [UnreachableCode] +''' +""" + +# bug #9839 +template myquit1():untyped= + ## foo + quit(1) +template myquit2():untyped= + echo 123 + myquit1() + +proc main1()= + + # BUG: uncommenting this doesn't give `Error: unreachable statement` + myquit2() + + echo "after" + +main1() + +proc main2() = + myquit1() + + echo "after" + +main2() + +proc main3() = + if true: + return + else: + return + echo "after" + +main3() + + +block: + # Cases like strings are not checked for exhaustiveness unless they have an else + proc main4(x: string) = + case x + of "a": + return + # reachable + echo "after" + + main4("a") + + proc main5(x: string) = + case x + of "a": + return + else: + return + # unreachable + echo "after" + + main5("a") + +block: + # In this case no else is needed because it's exhaustive + proc exhaustive(x: bool) = + case x + of true: + return + of false: + return + echo "after" + + exhaustive(true) diff --git a/tests/converter/m18986.nim b/tests/converter/m18986.nim new file mode 100644 index 0000000000..0ebf343aed --- /dev/null +++ b/tests/converter/m18986.nim @@ -0,0 +1,3 @@ +import std/macros + +converter Lit*(x: uint): NimNode = newLit(x) diff --git a/tests/converter/mdontleak.nim b/tests/converter/mdontleak.nim new file mode 100644 index 0000000000..e55c3f87c2 --- /dev/null +++ b/tests/converter/mdontleak.nim @@ -0,0 +1,3 @@ + +converter toBool(x: uint32): bool = x != 0 +# Note: This convertes is not exported! diff --git a/tests/converter/t18986.nim b/tests/converter/t18986.nim new file mode 100644 index 0000000000..ef300fa49c --- /dev/null +++ b/tests/converter/t18986.nim @@ -0,0 +1,10 @@ +discard """ + output: "Found a 0" +""" + +import m18986 except Lit +import std/macros + +# bug #18986 +var x = 0.uint +echo "Found a ", x diff --git a/tests/converter/t21531.nim b/tests/converter/t21531.nim new file mode 100644 index 0000000000..b0198684d7 --- /dev/null +++ b/tests/converter/t21531.nim @@ -0,0 +1,10 @@ +import std/typetraits + +type Foo* = distinct string + +converter toBase*(headers: var Foo): var string = + headers.distinctBase + +proc bar*(headers: var Foo) = + for x in headers: discard + diff --git a/tests/converter/t7097.nim b/tests/converter/t7097.nim index fdb5735886..d8e953080b 100644 --- a/tests/converter/t7097.nim +++ b/tests/converter/t7097.nim @@ -10,8 +10,8 @@ proc initBytesRange*(s: var Bytes, ibegin = 0, iend = -1): BytesRange = let e = if iend < 0: s.len + iend + 1 else: iend assert ibegin > 0 and e <= s.len - - shallow(s) + when defined(gcRefc): + shallow(s) result.bytes = s result.ibegin = ibegin result.iend = e diff --git a/tests/converter/t7098.nim b/tests/converter/t7098.nim index 8e7634882a..30c9c1e254 100644 --- a/tests/converter/t7098.nim +++ b/tests/converter/t7098.nim @@ -14,8 +14,8 @@ proc initBytesRange*(s: var Bytes, ibegin = 0, iend = -1): BytesRange = let e = if iend < 0: s.len + iend + 1 else: iend assert ibegin >= 0 and e <= s.len - - shallow(s) + when defined(gcRefc): + shallow(s) result.bytes = s result.ibegin = ibegin result.iend = e diff --git a/tests/converter/tconverter_unique_ptr.nim b/tests/converter/tconverter_unique_ptr.nim index 23c1a3d96d..6902f9e9e0 100644 --- a/tests/converter/tconverter_unique_ptr.nim +++ b/tests/converter/tconverter_unique_ptr.nim @@ -22,12 +22,11 @@ proc `$`(x: MyLen): string {.borrow.} proc `==`(x1, x2: MyLen): bool {.borrow.} -proc `=destroy`*(m: var MySeq) {.inline.} = +proc `=destroy`*(m: MySeq) {.inline.} = if m.data != nil: deallocShared(m.data) - m.data = nil -proc `=`*(m: var MySeq, m2: MySeq) = +proc `=copy`*(m: var MySeq, m2: MySeq) = if m.data == m2.data: return if m.data != nil: `=destroy`(m) @@ -77,13 +76,12 @@ converter literalToLen*(x: int{lit}): MyLen = # Unique pointer implementation #------------------------------------------------------------- -proc `=destroy`*[T](p: var UniquePtr[T]) = +proc `=destroy`*[T](p: UniquePtr[T]) = if p.val != nil: `=destroy`(p.val[]) dealloc(p.val) - p.val = nil -proc `=`*[T](dest: var UniquePtr[T], src: UniquePtr[T]) {.error.} +proc `=copy`*[T](dest: var UniquePtr[T], src: UniquePtr[T]) {.error.} proc `=sink`*[T](dest: var UniquePtr[T], src: UniquePtr[T]) {.inline.} = if dest.val != nil and dest.val != src.val: @@ -118,13 +116,12 @@ type ## as it returns only `lent T` val: ptr T -proc `=destroy`*[T](p: var ConstPtr[T]) = +proc `=destroy`*[T](p: ConstPtr[T]) = if p.val != nil: `=destroy`(p.val[]) dealloc(p.val) - p.val = nil -proc `=`*[T](dest: var ConstPtr[T], src: ConstPtr[T]) {.error.} +proc `=copy`*[T](dest: var ConstPtr[T], src: ConstPtr[T]) {.error.} proc `=sink`*[T](dest: var ConstPtr[T], src: ConstPtr[T]) {.inline.} = if dest.val != nil and dest.val != src.val: diff --git a/tests/converter/tconverter_with_varargs.nim b/tests/converter/tconverter_with_varargs.nim index 6d7e31e858..fae83221b2 100644 --- a/tests/converter/tconverter_with_varargs.nim +++ b/tests/converter/tconverter_with_varargs.nim @@ -8,7 +8,7 @@ type converter to_py*(i: int) : PPyRef = nil when false: - proc to_tuple*(vals: openarray[PPyRef]): PPyRef = + proc to_tuple*(vals: openArray[PPyRef]): PPyRef = discard proc abc(args: varargs[PPyRef]) = diff --git a/tests/converter/tdontleak.nim b/tests/converter/tdontleak.nim new file mode 100644 index 0000000000..4965fa90ad --- /dev/null +++ b/tests/converter/tdontleak.nim @@ -0,0 +1,10 @@ +discard """ + output: '''5''' +joinable: false +""" + +import mdontleak +# bug #19213 + +let a = 5'u32 +echo a diff --git a/tests/converter/texplicit_conversion.nim b/tests/converter/texplicit_conversion.nim index 6b2e96faf1..e36d78ad5c 100644 --- a/tests/converter/texplicit_conversion.nim +++ b/tests/converter/texplicit_conversion.nim @@ -11,3 +11,9 @@ converter toInt(s: string): int = let x = (int)"234" echo x + +block: # Test for nkconv + proc foo(o: var int) = + assert o == 0 + var a = 0 + foo(int(a)) \ No newline at end of file diff --git a/tests/coroutines/tgc.nim b/tests/coroutines/tgc.nim index e2f8b64697..770d413f5c 100644 --- a/tests/coroutines/tgc.nim +++ b/tests/coroutines/tgc.nim @@ -1,6 +1,6 @@ discard """ matrix: "--gc:refc; --gc:arc; --gc:orc" - target: "c" + targets: "c" """ when compileOption("gc", "refc") or not defined(openbsd): diff --git a/tests/coroutines/twait.nim b/tests/coroutines/twait.nim index 71782ece16..2edfcf675b 100644 --- a/tests/coroutines/twait.nim +++ b/tests/coroutines/twait.nim @@ -1,7 +1,7 @@ discard """ output: "Exit 1\nExit 2" matrix: "--gc:refc; --gc:arc; --gc:orc" - target: "c" + targets: "c" """ when compileOption("gc", "refc") or not defined(openbsd): diff --git a/tests/cpp/t12946.nim b/tests/cpp/t12946.nim new file mode 100644 index 0000000000..79cd562511 --- /dev/null +++ b/tests/cpp/t12946.nim @@ -0,0 +1,8 @@ +discard """ + targets: "c cpp" +""" + +import std/atomics +type Futex = distinct Atomic[int32] + +var x: Futex diff --git a/tests/cpp/t22679.nim b/tests/cpp/t22679.nim new file mode 100644 index 0000000000..81defcb58e --- /dev/null +++ b/tests/cpp/t22679.nim @@ -0,0 +1,50 @@ +discard """ + cmd: "nim cpp $file" + output:''' +cppNZ.x = 123 +cppNZInit.x = 123 +hascpp.cppnz.x = 123 +hasCppInit.cppnz.x = 123 +hasCppCtor.cppnz.x = 123 +''' +""" +{.emit:"""/*TYPESECTION*/ +struct CppNonZero { + int x = 123; +}; +""".} + +import sugar +type + CppNonZero {.importcpp, inheritable.} = object + x: cint + + HasCpp = object + cppnz: CppNonZero + +proc initCppNonZero: CppNonZero = + CppNonZero() + +proc initHasCpp: HasCpp = + HasCpp() + +proc ctorHasCpp: HasCpp {.constructor.} = + discard + +proc main = + var cppNZ: CppNonZero + dump cppNZ.x + + var cppNZInit = initCppNonZero() + dump cppNZInit.x + + var hascpp: HasCpp + dump hascpp.cppnz.x + + var hasCppInit = initHasCpp() + dump hasCppInit.cppnz.x + + var hasCppCtor = ctorHasCpp() + dump hasCppCtor.cppnz.x + +main() \ No newline at end of file diff --git a/tests/cpp/t22680.nim b/tests/cpp/t22680.nim new file mode 100644 index 0000000000..80f1a8319b --- /dev/null +++ b/tests/cpp/t22680.nim @@ -0,0 +1,50 @@ +discard """ + cmd: "nim cpp $file" + output:''' +cppNZ.x = 123 +cppNZInit.x = 123 +inheritCpp.x = 123 +inheritCppInit.x = 123 +inheritCppCtor.x = 123 +''' +""" +import std/sugar + +{.emit:"""/*TYPESECTION*/ +struct CppNonZero { + int x = 123; +}; +""".} + +type + CppNonZero {.importcpp, inheritable.} = object + x: cint + + InheritCpp = object of CppNonZero + +proc initCppNonZero: CppNonZero = + CppNonZero() + +proc initInheritCpp: InheritCpp = + InheritCpp() + +proc ctorInheritCpp: InheritCpp {.constructor.} = + discard + +proc main = + var cppNZ: CppNonZero + dump cppNZ.x + + var cppNZInit = initCppNonZero() + dump cppNZInit.x + + var inheritCpp: InheritCpp + dump inheritCpp.x + + var inheritCppInit = initInheritCpp() + dump inheritCppInit.x + + var inheritCppCtor = ctorInheritCpp() + dump inheritCppCtor.x + +main() \ No newline at end of file diff --git a/tests/cpp/t22712.nim b/tests/cpp/t22712.nim new file mode 100644 index 0000000000..34ef67ac85 --- /dev/null +++ b/tests/cpp/t22712.nim @@ -0,0 +1,15 @@ +discard """ +targets: "cpp" +errormsg: "constructor in an imported type needs importcpp pragma" +line: 14 +""" +{.emit: """/*TYPESECTION*/ +struct CppStruct { + CppStruct(); +}; +""".} + +type CppStruct {.importcpp.} = object + +proc makeCppStruct(): CppStruct {.constructor.} = + discard \ No newline at end of file diff --git a/tests/cpp/t23306.nim b/tests/cpp/t23306.nim new file mode 100644 index 0000000000..ebb4edb8d9 --- /dev/null +++ b/tests/cpp/t23306.nim @@ -0,0 +1,12 @@ +discard """ +targets: "cpp" +""" + +type K = object + h: iterator(f: K): K + +iterator d(g: K): K {.closure.} = + defer: + discard + +discard K(h: d) \ No newline at end of file diff --git a/tests/cpp/t23434.nim b/tests/cpp/t23434.nim new file mode 100644 index 0000000000..04a83227e4 --- /dev/null +++ b/tests/cpp/t23434.nim @@ -0,0 +1,17 @@ +discard """ +cmd:"nim cpp $file" +errormsg: "type mismatch: got " +line: 17 +""" +type SomeObject = object + value: int + +proc printValue(self: SomeObject) {.virtual.} = + echo "The value is ", self.value + +proc callAProc(p: proc(self: SomeObject){.noconv.}) = + let someObj = SomeObject(value: 4) + echo "calling param proc" + p(someObj) + +callAProc(printValue) \ No newline at end of file diff --git a/tests/cpp/t23657.nim b/tests/cpp/t23657.nim new file mode 100644 index 0000000000..63deb7fb0d --- /dev/null +++ b/tests/cpp/t23657.nim @@ -0,0 +1,54 @@ +discard """ + targets: "cpp" + cmd: "nim cpp -r $file" + output: ''' +1.0 +1.0 +''' + +""" +{.emit:"""/*TYPESECTION*/ +struct Point { + float x, y, z; + Point(float x, float y, float z): x(x), y(y), z(z) {} + Point() = default; +}; +struct Direction { + float x, y, z; + Direction(float x, float y, float z): x(x), y(y), z(z) {} + Direction() = default; +}; +struct Axis { + Point origin; + Direction direction; + Axis(Point origin, Direction direction): origin(origin), direction(direction) {} + Axis() = default; +}; + +""".} + +type + Point {.importcpp.} = object + x, y, z: float + + Direction {.importcpp.} = object + x, y, z: float + + Axis {.importcpp.} = object + origin: Point + direction: Direction + +proc makeAxis(origin: Point, direction: Direction): Axis {. constructor, importcpp:"Axis(@)".} +proc makePoint(x, y, z: float): Point {. constructor, importcpp:"Point(@)".} +proc makeDirection(x, y, z: float): Direction {. constructor, importcpp:"Direction(@)".} + +var axis1 = makeAxis(Point(x: 1.0, y: 2.0, z: 3.0), Direction(x: 4.0, y: 5.0, z: 6.0)) #Triggers the error (T1) +var axis2Ctor = makeAxis(makePoint(1.0, 2.0, 3.0), makeDirection(4.0, 5.0, 6.0)) #Do not triggers + +proc main() = #Do not triggers as Tx are inside the body + let test = makeAxis(Point(x: 1.0, y: 2.0, z: 3.0), Direction(x: 4.0, y: 5.0, z: 6.0)) + echo test.origin.x + +main() + +echo $axis1.origin.x #Make sures it's init \ No newline at end of file diff --git a/tests/cpp/t6986.nim b/tests/cpp/t6986.nim index ffd277adb0..16e455c3ba 100644 --- a/tests/cpp/t6986.nim +++ b/tests/cpp/t6986.nim @@ -5,6 +5,9 @@ discard """ import sequtils, strutils +when defined(nimPreviewSlimSystem): + import std/syncio + let rules = toSeq(lines("input")) .mapIt(it.split(" => ").mapIt(it.replace("/", ""))) diff --git a/tests/cpp/tasync_cpp.nim b/tests/cpp/tasync_cpp.nim index 696274ec47..3f4ec62084 100644 --- a/tests/cpp/tasync_cpp.nim +++ b/tests/cpp/tasync_cpp.nim @@ -1,7 +1,7 @@ discard """ targets: "cpp" output: "hello" - cmd: "nim cpp --clearNimblePath --nimblePath:build/deps/pkgs $file" + cmd: "nim cpp --clearNimblePath --nimblePath:build/deps/pkgs2 $file" """ # bug #3299 diff --git a/tests/cpp/tcodegendecl.nim b/tests/cpp/tcodegendecl.nim new file mode 100644 index 0000000000..e128c5eb7d --- /dev/null +++ b/tests/cpp/tcodegendecl.nim @@ -0,0 +1,17 @@ +discard """ + targets: "cpp" + cmd: "nim cpp $file" + output: "3" +""" + +{.emit:"""/*TYPESECTION*/ + int operate(int x, int y, int (*func)(const int&, const int&)){ + return func(x, y); + }; +""".} + +proc operate(x, y: int32, fn: proc(x, y: int32 ): int32 {.cdecl.}): int32 {.importcpp:"$1(@)".} + +proc add(a {.codegenDecl:"const $#& $#".}, b {.codegenDecl:"const $# $#", byref.}: int32): int32 {.cdecl.} = a + b + +echo operate(1, 2, add) \ No newline at end of file diff --git a/tests/cpp/tconstructor.nim b/tests/cpp/tconstructor.nim new file mode 100644 index 0000000000..922ee54fd4 --- /dev/null +++ b/tests/cpp/tconstructor.nim @@ -0,0 +1,131 @@ +discard """ + targets: "cpp" + cmd: "nim cpp $file" + output: ''' +1 +0 +123 +0 +123 +___ +0 +777 +10 +123 +0 +777 +10 +123 +() +''' +""" + +{.emit:"""/*TYPESECTION*/ +struct CppClass { + int x; + int y; + CppClass(int inX, int inY) { + this->x = inX; + this->y = inY; + } + //CppClass() = default; +}; +""".} + +type CppClass* {.importcpp, inheritable.} = object + x: int32 + y: int32 + +proc makeCppClass(x, y: int32): CppClass {.importcpp: "CppClass(@)", constructor.} +#test globals are init with the constructor call +var shouldCompile {.used.} = makeCppClass(1, 2) + +proc newCpp*[T](): ptr T {.importcpp:"new '*0()".} + +#creation +type NimClassNoNarent* = object + x: int32 + +proc makeNimClassNoParent(x:int32): NimClassNoNarent {. constructor.} = + result.x = x + discard + +let nimClassNoParent = makeNimClassNoParent(1) +echo nimClassNoParent.x #acess to this just fine. Notice the field will appear last because we are dealing with constructor calls here + +var nimClassNoParentDef {.used.}: NimClassNoNarent #test has a default constructor. + +#inheritance +type NimClass* = object of CppClass + +proc makeNimClass(x:int32): NimClass {. constructor:"NimClass('1 #1) : CppClass(0, #1) ".} = + result.x = x + +#optinially define the default constructor so we get rid of the cpp warn and we can declare the obj (note: default constructor of 'tyObject_NimClass__apRyyO8cfRsZtsldq1rjKA' is implicitly deleted because base class 'CppClass' has no default constructor) +proc makeCppClass(): NimClass {. constructor: "NimClass() : CppClass(0, 0) ".} = + result.x = 1 + +let nimClass = makeNimClass(1) +var nimClassDef {.used.}: NimClass #since we explictly defined the default constructor we can declare the obj + +#bug: 22662 +type + BugClass* = object + x: int # Not initialized + +proc makeBugClass(): BugClass {.constructor.} = + discard + +proc main = + for i in 0 .. 1: + var n = makeBugClass() + echo n.x + n.x = 123 + echo n.x + +main() +#bug: +echo "___" +type + NimClassWithDefault = object + x: int + y = 777 + case kind: bool = true + of true: + z: int = 10 + else: discard + +proc makeNimClassWithDefault(): NimClassWithDefault {.constructor.} = + result = NimClassWithDefault() + +proc init = + for i in 0 .. 1: + var n = makeNimClassWithDefault() + echo n.x + echo n.y + echo n.z + n.x = 123 + echo n.x + +init() + +#tests that the ctor is not declared with nodecl. +#nodelc also prevents the creation of a default one when another is created. +type Foo {.exportc.} = object + +proc makeFoo(): Foo {.used, constructor, nodecl.} = discard + +echo $Foo() + +type Boo = object +proc `=copy`(dest: var Boo; src: Boo) = discard + +proc makeBoo(): Boo {.constructor.} = Boo() +proc makeBoo2(): Boo = Boo() + +block: + proc main = + var b = makeBoo() + var b2 = makeBoo2() + + main() \ No newline at end of file diff --git a/tests/cpp/tcovariancerules.nim b/tests/cpp/tcovariancerules.nim index 49fe8015b1..9d49f2cbde 100644 --- a/tests/cpp/tcovariancerules.nim +++ b/tests/cpp/tcovariancerules.nim @@ -31,7 +31,7 @@ import macros macro skipElse(n: untyped): untyped = n[0] template acceptWithCovariance(x, otherwise): untyped = - when nimEnableCovariance: + when defined nimEnableCovariance: x else: reject(x) @@ -79,16 +79,16 @@ proc wantsCovariantSeq2(s: seq[AnimalRef]) = proc wantsCovariantSeq3(s: seq[RefAlias[Animal]]) = for a in s: echo a.x -proc wantsCovariantOpenArray(s: openarray[ref Animal]) = +proc wantsCovariantOpenArray(s: openArray[ref Animal]) = for a in s: echo a.x -proc modifiesCovariantOpenArray(s: var openarray[ref Animal]) = +proc modifiesCovariantOpenArray(s: var openArray[ref Animal]) = for a in s: echo a.x -proc modifiesDerivedOpenArray(s: var openarray[ref Dog]) = +proc modifiesDerivedOpenArray(s: var openArray[ref Dog]) = for a in s: echo a.x -proc wantsNonCovariantOpenArray(s: openarray[Animal]) = +proc wantsNonCovariantOpenArray(s: openArray[Animal]) = for a in s: echo a.x proc wantsCovariantArray(s: array[2, ref Animal]) = diff --git a/tests/cpp/tdont_init_instantiation.nim b/tests/cpp/tdont_init_instantiation.nim index fe487fba08..a13a3f6b4e 100644 --- a/tests/cpp/tdont_init_instantiation.nim +++ b/tests/cpp/tdont_init_instantiation.nim @@ -20,7 +20,7 @@ template class C { }; """.} -type C{.importcpp, header: "", nodecl.} [X] = object +type C[X] {.importcpp, header: "", nodecl.} = object proc mkC[X]: C[X] {.importcpp: "C<'*0>()", constructor, nodecl.} proc foo(): C[int] = diff --git a/tests/cpp/temitlist.nim b/tests/cpp/temitlist.nim index 852537518a..9170be0796 100644 --- a/tests/cpp/temitlist.nim +++ b/tests/cpp/temitlist.nim @@ -1,12 +1,14 @@ discard """ targets: "cpp" - output: '''6.0 + output: ''' +6.0 0''' +disabled: "windows" # pending bug #18011 """ # bug #4730 -type Vector* {.importcpp: "std::vector", header: "".}[T] = object +type Vector*[T] {.importcpp: "std::vector", header: "".} = object template `[]=`*[T](v: var Vector[T], key: int, val: T) = {.emit: [v, "[", key, "] = ", val, ";"].} diff --git a/tests/cpp/tempty_generic_obj.nim b/tests/cpp/tempty_generic_obj.nim index 913c1ec3c4..6125190b4f 100644 --- a/tests/cpp/tempty_generic_obj.nim +++ b/tests/cpp/tempty_generic_obj.nim @@ -1,7 +1,9 @@ discard """ targets: "cpp" - output: '''int + output: ''' +int float''' +disabled: "windows" # pending bug #18011 """ import typetraits diff --git a/tests/cpp/tinitializers.nim b/tests/cpp/tinitializers.nim new file mode 100644 index 0000000000..0199fb96bc --- /dev/null +++ b/tests/cpp/tinitializers.nim @@ -0,0 +1,60 @@ +discard """ + cmd: "nim cpp $file" +""" + +{.emit:"""/*TYPESECTION*/ +struct CppStruct { + CppStruct(int x, char* y): x(x), y(y){} + void doSomething() {} + int x; + char* y; +}; +""".} +type + CppStruct {.importcpp, inheritable.} = object + ChildStruct = object of CppStruct + HasCppStruct = object + cppstruct: CppStruct + +proc constructCppStruct(a:cint = 5, b:cstring = "hello"): CppStruct {.importcpp: "CppStruct(@)", constructor.} +proc doSomething(this: CppStruct) {.importcpp.} +proc returnCppStruct(): CppStruct = discard +proc initChildStruct: ChildStruct = ChildStruct() +proc makeChildStruct(): ChildStruct {.constructor:"""ChildStruct(): CppStruct(5, "10")""".} = discard +proc initHasCppStruct(x: cint): HasCppStruct = + HasCppStruct(cppstruct: constructCppStruct(x)) + +proc main = + var hasCppStruct = initHasCppStruct(2) #generates cppstruct = { 10 } inside the struct + hasCppStruct.cppstruct.doSomething() + discard returnCppStruct() #generates result = { 10 } + discard initChildStruct() #generates ChildStruct temp ({}) bypassed with makeChildStruct + (proc (s:CppStruct) = discard)(CppStruct()) #CppStruct temp ({10}) +main() + + +#Should handle ObjectCalls +{.emit:"""/*TYPESECTION*/ +struct Foo { +}; +struct Boo { + Boo(int x, char* y, Foo f): x(x), y(y), foo(f){} + int x; + char* y; + Foo foo; +}; +""".} +type + Foo {.importcpp, inheritable, bycopy.} = object + Boo {.importcpp, inheritable.} = object + x: int32 + y: cstring + foo: Foo + +proc makeBoo(a:cint = 10, b:cstring = "hello", foo: Foo = Foo()): Boo {.importcpp, constructor.} + +proc main2() = + let cppStruct = makeBoo() + (proc (s:Boo) = discard)(Boo()) + +main2() \ No newline at end of file diff --git a/tests/cpp/tmember.nim b/tests/cpp/tmember.nim new file mode 100644 index 0000000000..1a5b6fd975 --- /dev/null +++ b/tests/cpp/tmember.nim @@ -0,0 +1,75 @@ +discard """ + targets: "cpp" + cmd: "nim cpp $file" + output: ''' +2 +false +hello foo +hello boo +hello boo +FunctorSupport! +static +static +destructing +destructing +''' +""" +proc print(s: cstring) {.importcpp:"printf(@)", header:"".} + +type + Doo {.exportc.} = object + test: int + +proc memberProc(f: Doo) {.exportc, member.} = + echo $f.test + +proc destructor(f: Doo) {.member: "~'1()", used.} = + print "destructing\n" + +proc `==`(self, other: Doo): bool {.member:"operator==('2 const & #2) const -> '0"} = + self.test == other.test + +let doo = Doo(test: 2) +doo.memberProc() +echo doo == Doo(test: 1) + +#virtual +proc newCpp*[T](): ptr T {.importcpp:"new '*0()".} +type + Foo {.exportc.} = object of RootObj + FooPtr = ptr Foo + Boo = object of Foo + BooPtr = ptr Boo + +proc salute(self: FooPtr) {.member: "virtual $1()".} = + echo "hello foo" + +proc salute(self: BooPtr) {.member: "virtual $1()".} = + echo "hello boo" + +let foo = newCpp[Foo]() +let boo = newCpp[Boo]() +let booAsFoo = cast[FooPtr](newCpp[Boo]()) + +foo.salute() +boo.salute() +booAsFoo.salute() + +type + NimFunctor = object + discard +proc invoke(f: NimFunctor, n:int) {.member:"operator ()('2 #2)" .} = + echo "FunctorSupport!" + +{.experimental: "callOperator".} +proc `()`(f: NimFunctor, n:int) {.importcpp:"#(@)" .} +NimFunctor()(1) + +#static +proc staticProc(self: FooPtr) {.member: "static $1()".} = + echo "static" + +proc importedStaticProc() {.importcpp:"Foo::staticProc()".} + +foo.staticProc() +importedStaticProc() diff --git a/tests/cpp/tmember_forward_declaration.nim b/tests/cpp/tmember_forward_declaration.nim new file mode 100644 index 0000000000..2f4a79daa3 --- /dev/null +++ b/tests/cpp/tmember_forward_declaration.nim @@ -0,0 +1,27 @@ +discard """ + targets: "cpp" + cmd: "nim cpp $file" + output: ''' +abc called +def called +abc called +''' +""" + +type Foo = object + +proc abc(this: Foo, x: int): void {.member: "$1('2 #2)".} +proc def(this: Foo, y: int): void {.virtual: "$1('2 #2)".} + +proc abc(this: Foo, x: int): void = + echo "abc called" + if x > 0: + this.def(x - 1) + +proc def(this: Foo, y: int): void = + echo "def called" + this.abc(y) + +var x = Foo() +x.abc(1) + diff --git a/tests/cpp/tnoinitfield.nim b/tests/cpp/tnoinitfield.nim new file mode 100644 index 0000000000..4deffece82 --- /dev/null +++ b/tests/cpp/tnoinitfield.nim @@ -0,0 +1,30 @@ +discard """ + targets: "cpp" + cmd: "nim cpp $file" + output: ''' +''' +""" +{.emit: """/*TYPESECTION*/ + struct Foo { + Foo(int a){}; + }; + struct Boo { + Boo(int a){}; + }; + + """.} + +type + Foo {.importcpp.} = object + Boo {.importcpp, noInit.} = object + Test {.exportc.} = object + foo {.noInit.}: Foo + boo: Boo + +proc makeTest(): Test {.constructor: "Test() : foo(10), boo(1)".} = + discard + +proc main() = + var t = makeTest() + +main() \ No newline at end of file diff --git a/tests/cpp/torc.nim b/tests/cpp/torc.nim new file mode 100644 index 0000000000..9f1a41a21b --- /dev/null +++ b/tests/cpp/torc.nim @@ -0,0 +1,75 @@ +discard """ + targets: "cpp" + matrix: "--gc:orc" +""" + +import std/options + +# bug #18410 +type + O = object of RootObj + val: pointer + +proc p(): Option[O] = none(O) + +doAssert $p() == "none(O)" + +# bug #17351 +type + Foo = object of RootObj + Foo2 = object of Foo + Bar = object + x: Foo2 + +var b = Bar() +discard b + +# bug #4678 +{.emit: """/*TYPESECTION*/ +enum class SomeEnum {A,B,C}; +""".} +type + EnumVector[T: enum] {.importcpp: "std::vector", header: "vector".} = object + SomeEnum {.importcpp, nodecl.} = enum + A,B,C + +proc asVector*[T](t: T): EnumVector[T] = + discard +# Nim generates this signature here: +# N_NIMCALL(std::vector<> , asvector_106028_3197418230)(SomeEnum t0) + +discard asVector(SomeEnum.A) + + +block: # bug #10219 + type + Vector[T] {.importcpp: "std::vector", header: "vector".} = object + + proc initVector[T](n: csize_t): Vector[T] + {.importcpp: "std::vector<'*0>(@)", header: "vector".} + + proc unsafeIndex[T](this: var Vector[T], i: csize_t): var T + {.importcpp: "#[#]", header: "vector".} + + proc `[]`[T](this: var Vector[T], i: Natural): var T {.inline, noinit.} = + when compileOption("boundChecks"): + # this.checkIndex i + discard + result = this.unsafeIndex(csize_t(i)) + + var v1 = initVector[int](10) + doAssert v1[0] == 0 + +block: # bug #12703 bug #19588 + type + cstringConstImpl {.importc:"const char*".} = cstring + constChar = distinct cstringConstImpl + + {.emit: """ + const char* foo() { + return "hello"; + } + """.} + proc foo(): constChar {.importcpp.} # change to importcpp for C++ backend + doAssert $(foo().cstring) == "hello" + diff --git a/tests/cpp/tpassbypragmas.nim b/tests/cpp/tpassbypragmas.nim new file mode 100644 index 0000000000..f4301656af --- /dev/null +++ b/tests/cpp/tpassbypragmas.nim @@ -0,0 +1,27 @@ +discard """ + targets: "cpp" + cmd: "nim cpp $file" +""" +{.emit:"""/*TYPESECTION*/ + + template + struct Box { + T first; + }; + struct Foo { + void test(void (*func)(Box& another)){ + + }; + }; +""".} + +type + Foo {.importcpp.} = object + Box[T] {.importcpp:"Box<'0>".} = object + first: T + +proc test(self: Foo, fn: proc(another {.byref.}: Box[Foo]) {.cdecl.}) {.importcpp.} + +proc fn(another {.byref.} : Box[Foo]) {.cdecl.} = discard + +Foo().test(fn) \ No newline at end of file diff --git a/tests/cpp/tretvar.nim b/tests/cpp/tretvar.nim index 83c37721ec..0c37653461 100644 --- a/tests/cpp/tretvar.nim +++ b/tests/cpp/tretvar.nim @@ -16,9 +16,9 @@ type proc c_str(a: stdString): cstring {.importcpp: "(char *)(#.c_str())", header: "".} -proc len(a: stdString): csize {.importcpp: "(#.length())", header: "".} +proc len(a: stdString): csize_t {.importcpp: "(#.length())", header: "".} -proc setChar(a: var stdString, i: csize, c: char) {.importcpp: "(#[#] = #)", header: "".} +proc setChar(a: var stdString, i: csize_t, c: char) {.importcpp: "(#[#] = #)", header: "".} proc `*`*[T](this: stdUniquePtr[T]): var T {.noSideEffect, importcpp: "(* #)", header: "".} diff --git a/tests/cpp/tvector_iterator.nim b/tests/cpp/tvector_iterator.nim index 4d686955fe..c3886c5476 100644 --- a/tests/cpp/tvector_iterator.nim +++ b/tests/cpp/tvector_iterator.nim @@ -12,8 +12,8 @@ struct Vector { """.} type - Vector {.importcpp: "Vector".} [T] = object - VectorIterator {.importcpp: "Vector<'0>::Iterator".} [T] = object + Vector[T] {.importcpp: "Vector".} = object + VectorIterator[T] {.importcpp: "Vector<'0>::Iterator".} = object var x: VectorIterator[void] diff --git a/tests/cpp/tvirtual.nim b/tests/cpp/tvirtual.nim new file mode 100644 index 0000000000..385d052b8f --- /dev/null +++ b/tests/cpp/tvirtual.nim @@ -0,0 +1,126 @@ +discard """ + targets: "cpp" + cmd: "nim cpp $file" + output: ''' +hello foo +hello boo +hello boo +Const Message: hello world +NimPrinter: hello world +NimPrinterConstRef: hello world +NimPrinterConstRefByRef: hello world +''' +""" + +{.emit:"""/*TYPESECTION*/ +#include + class CppPrinter { + public: + + virtual void printConst(char* message) const { + std::cout << "Const Message: " << message << std::endl; + } + virtual void printConstRef(char* message, const int& flag) const { + std::cout << "Const Ref Message: " << message << std::endl; + } + virtual void printConstRef2(char* message, const int& flag) const { + std::cout << "Const Ref2 Message: " << message << std::endl; + } + +}; +""".} + +proc newCpp*[T](): ptr T {.importcpp:"new '*0()".} +type + Foo = object of RootObj + FooPtr = ptr Foo + Boo = object of Foo + BooPtr = ptr Boo + CppPrinter {.importcpp, inheritable.} = object + NimPrinter {.exportc.} = object of CppPrinter + +proc salute(self: FooPtr) {.virtual.} = + echo "hello foo" + +proc salute(self: BooPtr) {.virtual.} = + echo "hello boo" + +let foo = newCpp[Foo]() +let boo = newCpp[Boo]() +let booAsFoo = cast[FooPtr](newCpp[Boo]()) + +#polymorphism works +foo.salute() +boo.salute() +booAsFoo.salute() +let message = "hello world".cstring + +proc printConst(self: CppPrinter, message: cstring) {.importcpp.} +CppPrinter().printConst(message) + +#notice override is optional. +#Will make the cpp compiler to fail if not virtual function with the same signature if found in the base type +proc printConst(self: NimPrinter, message: cstring) {.virtual:"$1('2 #2) const override".} = + echo "NimPrinter: " & $message + +proc printConstRef(self: NimPrinter, message: cstring, flag: int32) {.virtual:"$1('2 #2, const '3& #3 ) const override".} = + echo "NimPrinterConstRef: " & $message + +proc printConstRef2(self: NimPrinter, message: cstring, flag {.byref.}: int32) {.virtual:"$1('2 #2, const '3 #3 ) const override".} = + echo "NimPrinterConstRefByRef: " & $message + +NimPrinter().printConst(message) +var val : int32 = 10 +NimPrinter().printConstRef(message, val) +NimPrinter().printConstRef2(message, val) + +#bug 22269 +type Doo = object +proc naiveMember(x: Doo): int {. virtual .} = 2 +discard naiveMember(Doo()) + +#asmnostackframe works with virtual +{.emit:"""/*TYPESECTION*/ + template + struct Box { + T* first; + + Box(int x){ + first = new T(x); + }; + }; + struct Inner { + int val; + //Coo() = default; + Inner(int x){ + val = x; + }; + }; + struct Base { + virtual Box test() = 0; + }; +""".} + +type + Inner {.importcpp.} = object + Base {.importcpp, inheritable.} = object + Child = object of Base + Box[T] {.importcpp, inheritable.} = object + first: T + +proc makeBox[T](x:int32): Box[T] {.importcpp:"Box<'0>(@)", constructor.} + +proc test(self: Child): Box[Inner] {.virtual, asmnostackframe.} = + let res {.exportc.} = makeBox[Inner](100) + {.emit:"return res;".} + + +discard Child().test() + +import virtualptr + +#We dont want to pull Loo directly by using it as we are testing that the pointer pulls it. +proc makeMoo(): Moo {.importcpp:"{ new Loo() }".} + +makeMoo().loo.salute() + diff --git a/tests/cpp/virtualptr.nim b/tests/cpp/virtualptr.nim new file mode 100644 index 0000000000..f96264081c --- /dev/null +++ b/tests/cpp/virtualptr.nim @@ -0,0 +1,9 @@ +type + Loo* {.exportc.} = object + LooPtr* = ptr Loo + Moo* {.exportc.} = object + loo*: LooPtr + + +proc salute*(foo: LooPtr) {.virtual.} = + discard diff --git a/tests/defer/t22309.nim b/tests/defer/t22309.nim new file mode 100644 index 0000000000..34ca4843b2 --- /dev/null +++ b/tests/defer/t22309.nim @@ -0,0 +1,11 @@ +block: + defer: + let a = 42 + doAssert not declared(a) + +proc lol() = + defer: + let a = 42 + doAssert not declared(a) + +lol() diff --git a/tests/deprecated/tannot.nim b/tests/deprecated/tannot.nim deleted file mode 100644 index d14f6cc235..0000000000 --- a/tests/deprecated/tannot.nim +++ /dev/null @@ -1,9 +0,0 @@ -discard """ - nimout: '''tannot.nim(9, 1) Warning: efgh; foo1 is deprecated [Deprecated] -tannot.nim(9, 8) Warning: abcd; foo is deprecated [Deprecated] -''' -""" - -let foo* {.deprecated: "abcd".} = 42 -var foo1* {.deprecated: "efgh".} = 42 -foo1 = foo diff --git a/tests/deprecated/tmessages.nim b/tests/deprecated/tmessages.nim deleted file mode 100644 index 5884e396d5..0000000000 --- a/tests/deprecated/tmessages.nim +++ /dev/null @@ -1,10 +0,0 @@ -discard """ - nimout:'''tmessages.nim(10, 1) Warning: Deprecated since v1.2.0, use 'HelloZ'; hello is deprecated [Deprecated] -''' -""" - -proc hello[T](a: T) {.deprecated: "Deprecated since v1.2.0, use 'HelloZ'".} = - discard - - -hello[int](12) diff --git a/tests/deprecated/tmodule1.nim b/tests/deprecated/tmodule1.nim deleted file mode 100644 index 9548368890..0000000000 --- a/tests/deprecated/tmodule1.nim +++ /dev/null @@ -1,23 +0,0 @@ -discard """ - nimout: '''tmodule1.nim(11, 8) Warning: goodbye; importme is deprecated [Deprecated] -tmodule1.nim(14, 10) Warning: Ty is deprecated [Deprecated] -tmodule1.nim(17, 10) Warning: hello; Ty1 is deprecated [Deprecated] -tmodule1.nim(20, 8) Warning: aVar is deprecated [Deprecated] -tmodule1.nim(22, 3) Warning: aProc is deprecated [Deprecated] -tmodule1.nim(23, 3) Warning: hello; aProc1 is deprecated [Deprecated] -''' -""" - -import importme - -block: - var z: Ty - z = 0 -block: - var z: Ty1 - z = 0 -block: - echo aVar -block: - aProc() - aProc1() diff --git a/tests/destructor/const_smart_ptr.nim b/tests/destructor/const_smart_ptr.nim index 4d8c7c9a37..25dd46500d 100644 --- a/tests/destructor/const_smart_ptr.nim +++ b/tests/destructor/const_smart_ptr.nim @@ -2,13 +2,12 @@ type ConstPtr*[T] = object val: ptr T -proc `=destroy`*[T](p: var ConstPtr[T]) = +proc `=destroy`*[T](p: ConstPtr[T]) = if p.val != nil: `=destroy`(p.val[]) dealloc(p.val) - p.val = nil -proc `=`*[T](dest: var ConstPtr[T], src: ConstPtr[T]) {.error.} +proc `=copy`*[T](dest: var ConstPtr[T], src: ConstPtr[T]) {.error.} proc `=sink`*[T](dest: var ConstPtr[T], src: ConstPtr[T]) {.inline.} = if dest.val != nil and dest.val != src.val: @@ -31,12 +30,11 @@ type len: int data: ptr UncheckedArray[float] -proc `=destroy`*(m: var MySeqNonCopyable) {.inline.} = +proc `=destroy`*(m: MySeqNonCopyable) {.inline.} = if m.data != nil: deallocShared(m.data) - m.data = nil -proc `=`*(m: var MySeqNonCopyable, m2: MySeqNonCopyable) {.error.} +proc `=copy`*(m: var MySeqNonCopyable, m2: MySeqNonCopyable) {.error.} proc `=sink`*(m: var MySeqNonCopyable, m2: MySeqNonCopyable) {.inline.} = if m.data != m2.data: diff --git a/tests/destructor/t12037.nim b/tests/destructor/t12037.nim index c2c41dfb54..30266690ff 100644 --- a/tests/destructor/t12037.nim +++ b/tests/destructor/t12037.nim @@ -9,11 +9,11 @@ copy length and contents 1 @[42] proc test() = var sq1 = @[42] echo "showing original type, length, and contents ", sq1.typeof, " ", sq1.len, " ", sq1 - doAssert cast[int](sq1[0].unsafeAddr) != 0 + doAssert cast[int](sq1[0].addr) != 0 var sq2 = sq1 # copy of original echo "copy length and contents ", sq2.len, " ", sq2 - doAssert cast[int](sq2[0].unsafeAddr) != 0 - doAssert cast[int](sq1[0].unsafeAddr) != 0 + doAssert cast[int](sq2[0].addr) != 0 + doAssert cast[int](sq1[0].addr) != 0 test() diff --git a/tests/destructor/t16607.nim b/tests/destructor/t16607.nim index 5cc9d4a088..f98a6d5174 100644 --- a/tests/destructor/t16607.nim +++ b/tests/destructor/t16607.nim @@ -15,8 +15,7 @@ proc initO(): O = O(initialized: true) proc pair(): tuple[a, b: O] = - result.a = initO() - result.b = initO() + result = (a: initO(), b: initO()) proc main() = discard pair() diff --git a/tests/destructor/t5342.nim b/tests/destructor/t5342.nim index 19354ea64d..0acd5ef9d5 100644 --- a/tests/destructor/t5342.nim +++ b/tests/destructor/t5342.nim @@ -1,5 +1,6 @@ discard """ - matrix: "--gc:refc; --gc:arc" + matrix: "--mm:refc; --mm:arc" + targets: "c js" output: ''' 1 2 diff --git a/tests/destructor/tarray_indexing.nim b/tests/destructor/tarray_indexing.nim index 657101c4d2..a9dfdf4ed2 100644 --- a/tests/destructor/tarray_indexing.nim +++ b/tests/destructor/tarray_indexing.nim @@ -63,7 +63,7 @@ proc fillPages*(mem: UserProcessMemory, start: uint32, data: seq[byte]) = #echo cast[uint64](addr mem.pageAccess[i]) let page = mem.pageAccess[i] assert page != nil - #copyMem(page, unsafeAddr data[i * 0x1000 - start], 0x1000) + #copyMem(page, addr data[i * 0x1000 - start], 0x1000) const base = 0x00100000 diff --git a/tests/destructor/tatomicptrs.nim b/tests/destructor/tatomicptrs.nim index 7313afbf58..7bd5482b23 100644 --- a/tests/destructor/tatomicptrs.nim +++ b/tests/destructor/tatomicptrs.nim @@ -27,7 +27,7 @@ template decRef(x): untyped = atomicDec(x.refcount) proc makeShared*[T](x: sink T): SharedPtr[T] = # XXX could benefit from a macro that generates it. - result = cast[SharedPtr[T]](allocShared(sizeof(x))) + result = cast[SharedPtr[T]](allocShared0(sizeof(x))) result.x[] = x echo "allocating" @@ -39,7 +39,7 @@ proc `=destroy`*[T](dest: var SharedPtr[T]) = echo "deallocating" dest.x = nil -proc `=`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) = +proc `=copy`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) = var s = src.x if s != nil: incRef(s) #atomicSwap(dest, s) @@ -50,6 +50,9 @@ proc `=`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) = deallocShared(s) echo "deallocating" +proc `=dup`*[T](src: SharedPtr[T]): SharedPtr[T] = + `=copy`(result, src) + proc `=sink`*[T](dest: var SharedPtr[T]; src: SharedPtr[T]) = ## XXX make this an atomic store: if dest.x != src.x: @@ -72,6 +75,7 @@ template `.=`*[T](s: SharedPtr[T]; field, value: untyped) = from macros import unpackVarargs template `.()`*[T](s: SharedPtr[T]; field: untyped, args: varargs[untyped]): untyped = + # xxx this isn't used, the test should be improved unpackVarargs(s.x.field, args) @@ -119,7 +123,7 @@ proc `=destroy`*[T](m: var MySeq[T]) {.inline.} = deallocShared(m.data) m.data = nil -proc `=`*[T](m: var MySeq[T], m2: MySeq[T]) = +proc `=copy`*[T](m: var MySeq[T], m2: MySeq[T]) = if m.data == m2.data: return if m.data != nil: `=destroy`(m) @@ -130,6 +134,9 @@ proc `=`*[T](m: var MySeq[T], m2: MySeq[T]) = m.data = cast[ptr UncheckedArray[T]](allocShared(bytes)) copyMem(m.data, m2.data, bytes) +proc `=dup`*[T](m: MySeq[T]): MySeq[T] = + `=copy`[T](result, m) + proc `=sink`*[T](m: var MySeq[T], m2: MySeq[T]) {.inline.} = if m.data != m2.data: if m.data != nil: diff --git a/tests/destructor/tbintree2.nim b/tests/destructor/tbintree2.nim index 0bc52457c9..d56c2850bf 100644 --- a/tests/destructor/tbintree2.nim +++ b/tests/destructor/tbintree2.nim @@ -21,21 +21,21 @@ proc merge(lower, greater: owned Node): owned Node = elif greater.isNil: result = lower elif lower.y < greater.y: - lower.right = merge(lower.right, greater) + lower.right = merge(move lower.right, greater) result = lower else: - greater.left = merge(lower, greater.left) + greater.left = merge(lower, move greater.left) result = greater proc splitBinary(orig: owned Node, value: int32): (owned Node, owned Node) = if orig.isNil: result = (nil, nil) elif orig.x < value: - let splitPair = splitBinary(orig.right, value) + let splitPair = splitBinary(move orig.right, value) orig.right = splitPair[0] result = (orig, splitPair[1]) else: - let splitPair = splitBinary(orig.left, value) + let splitPair = splitBinary(move orig.left, value) orig.left = splitPair[1] result = (splitPair[0], orig) diff --git a/tests/destructor/tcustomseqs.nim b/tests/destructor/tcustomseqs.nim index 4087dc4a83..17a19f871f 100644 --- a/tests/destructor/tcustomseqs.nim +++ b/tests/destructor/tcustomseqs.nim @@ -121,7 +121,7 @@ proc createSeq*[T](elems: varargs[T]): myseq[T] = result.data = cast[type(result.data)](alloc(result.cap * sizeof(T))) inc allocCount when supportsCopyMem(T): - copyMem(result.data, unsafeAddr(elems[0]), result.cap * sizeof(T)) + copyMem(result.data, addr(elems[0]), result.cap * sizeof(T)) else: for i in 0..= self.cap: resize(self) @@ -60,22 +58,22 @@ proc add*(self: var mystring; c: char) = inc self.len proc ensure(self: var mystring; newLen: int) = - if newLen >= cap: - cap = max((cap * 3) shr 1, newLen) - if cap > 0: - if data == nil: inc allocCount - data = cast[type(data)](realloc(data, cap + 1)) + if newLen >= self.cap: + self.cap = max((self.cap * 3) shr 1, newLen) + if self.cap > 0: + if self.data == nil: inc allocCount + self.data = cast[type(self.data)](realloc(self.data, self.cap + 1)) proc add*(self: var mystring; y: mystring) = - let newLen = len + y.len + let newLen = self.len + y.len ensure(self, newLen) - copyMem(addr data[len], y.data, y.data.len + 1) - len = newLen + copyMem(addr self.data[self.len], y.data, y.data.len + 1) + self.len = newLen proc create*(lit: string): mystring = let newLen = lit.len ensure(result, newLen) - copyMem(addr result.data[result.len], unsafeAddr lit[0], newLen + 1) + copyMem(addr result.data[result.len], addr lit[0], newLen + 1) result.len = newLen proc `&`*(a, b: mystring): mystring = diff --git a/tests/destructor/tdistinctseq.nim b/tests/destructor/tdistinctseq.nim new file mode 100644 index 0000000000..5a2ac5eadd --- /dev/null +++ b/tests/destructor/tdistinctseq.nim @@ -0,0 +1,8 @@ +discard """ + matrix: "-u:nimPreviewNonVarDestructor;" +""" +type DistinctSeq* = distinct seq[int] + +# `=destroy`(cast[ptr DistinctSeq](0)[]) +var x = @[].DistinctSeq +`=destroy`(x) diff --git a/tests/destructor/tdont_return_unowned_from_owned.nim b/tests/destructor/tdont_return_unowned_from_owned.nim index a726960c61..ffe87cd76d 100644 --- a/tests/destructor/tdont_return_unowned_from_owned.nim +++ b/tests/destructor/tdont_return_unowned_from_owned.nim @@ -1,8 +1,12 @@ discard """ cmd: "nim check --newruntime --hints:off $file" - nimout: '''tdont_return_unowned_from_owned.nim(36, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type -tdont_return_unowned_from_owned.nim(39, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type -tdont_return_unowned_from_owned.nim(42, 6) Error: type mismatch: got + nimout: ''' +tdont_return_unowned_from_owned.nim(26, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref +tdont_return_unowned_from_owned.nim(27, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref +tdont_return_unowned_from_owned.nim(31, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type +tdont_return_unowned_from_owned.nim(43, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type +tdont_return_unowned_from_owned.nim(46, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type +tdont_return_unowned_from_owned.nim(49, 6) Error: type mismatch: got but expected one of: proc new[T](a: var ref T; finalizer: proc (x: ref T) {.nimcall.}) first type mismatch at position: 2 @@ -10,18 +14,21 @@ proc new[T](a: var ref T; finalizer: proc (x: ref T) {.nimcall.}) 2 other mismatching symbols have been suppressed; compile with --showAllMismatches:on to see them expression: new(result) -tdont_return_unowned_from_owned.nim(42, 6) Error: illformed AST: -tdont_return_unowned_from_owned.nim(50, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref -tdont_return_unowned_from_owned.nim(51, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref -tdont_return_unowned_from_owned.nim(55, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type +tdont_return_unowned_from_owned.nim(49, 6) Error: illformed AST: ''' - errormsg: "cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type" - line: 55 + errormsg: "illformed AST:" """ +proc testA(result: var (RootRef, RootRef)) = + let r: owned RootRef = RootRef() + result[0] = r + result[1] = RootRef() +proc testB(): RootRef = + let r: owned RootRef = RootRef() + result = r @@ -39,17 +46,11 @@ proc newObjB(): Obj = result = Obj() proc newObjC(): Obj = - new(result) + new(result) # illFormedAst raises GlobalError, + # without pipeline parsing, it needs to placed at the end + # in case that it disturbs other errors let a = newObjA() let b = newObjB() let c = newObjC() -proc testA(result: var (RootRef, RootRef)) = - let r: owned RootRef = RootRef() - result[0] = r - result[1] = RootRef() - -proc testB(): RootRef = - let r: owned RootRef = RootRef() - result = r diff --git a/tests/destructor/tgotoexceptions7.nim b/tests/destructor/tgotoexceptions7.nim index 6e564a0443..c04bd6ba0b 100644 --- a/tests/destructor/tgotoexceptions7.nim +++ b/tests/destructor/tgotoexceptions7.nim @@ -25,7 +25,8 @@ proc helper = doAssert(false) proc main(i: int) = var obj = Obj(kind: kindA, s: "abc") - obj.kind = kindB + {.cast(uncheckedAssign).}: + obj.kind = kindB obj.i = 2 try: var objA = ObjA() diff --git a/tests/destructor/tmatrix.nim b/tests/destructor/tmatrix.nim index 98ca95c949..2fd5af7895 100644 --- a/tests/destructor/tmatrix.nim +++ b/tests/destructor/tmatrix.nim @@ -31,7 +31,7 @@ proc `=sink`*(a: var Matrix; b: Matrix) = a.m = b.m a.n = b.n -proc `=`*(a: var Matrix; b: Matrix) = +proc `=copy`*(a: var Matrix; b: Matrix) = if a.data != nil and a.data != b.data: dealloc(a.data) deallocCount.inc @@ -43,6 +43,9 @@ proc `=`*(a: var Matrix; b: Matrix) = allocCount.inc copyMem(a.data, b.data, b.m * b.n * sizeof(float)) +proc `=dup`*(a: Matrix): Matrix = + `=copy`(result, a) + proc matrix*(m, n: int, s: float): Matrix = ## Construct an m-by-n constant matrix. result.m = m diff --git a/tests/destructor/tmove.nim b/tests/destructor/tmove.nim new file mode 100644 index 0000000000..2762aff900 --- /dev/null +++ b/tests/destructor/tmove.nim @@ -0,0 +1,18 @@ +discard """ + targets: "c cpp" +""" + +block: + var called = 0 + + proc bar(a: var int): var int = + inc called + result = a + + proc foo = + var a = 2 + var s = move bar(a) + doAssert called == 1 + doAssert s == 2 + + foo() diff --git a/tests/destructor/tmove_objconstr.nim b/tests/destructor/tmove_objconstr.nim index 5faaabb8b0..cdc1eb1c0d 100644 --- a/tests/destructor/tmove_objconstr.nim +++ b/tests/destructor/tmove_objconstr.nim @@ -50,7 +50,7 @@ proc `=destroy`(o: var Pony) = echo "Pony is dying!" proc getPony: Pony = - result.name = "Sparkles" + result = Pony(name: "Sparkles") iterator items(p: Pony): int = for i in 1..4: @@ -137,28 +137,29 @@ doAssert seq3[0] == 1.0 var seq4, seq5: MySeqNonCopyable (seq4, i, seq5) = myfunc2(2, 3) -seq4 = block: - var tmp = newMySeq(4, 1.0) - tmp[0] = 3.0 - tmp +proc foo = + seq4 = block: + var tmp = newMySeq(4, 1.0) + tmp[0] = 3.0 + tmp -doAssert seq4[0] == 3.0 + doAssert seq4[0] == 3.0 -import macros -seq4 = - if i > 0: newMySeq(2, 5.0) - elif i < -100: raise newException(ValueError, "Parse Error") - else: newMySeq(2, 3.0) + seq4 = + if i > 0: newMySeq(2, 5.0) + elif i < -100: raise newException(ValueError, "Parse Error") + else: newMySeq(2, 3.0) -seq4 = - case (char) i: - of 'A', {'W'..'Z'}: newMySeq(2, 5.0) - of 'B': quit(-1) - else: - let (x1, x2, x3) = myfunc2(2, 3) - x3 + seq4 = + case (char) i: + of 'A', {'W'..'Z'}: newMySeq(2, 5.0) + of 'B': quit(-1) + else: + let (x1, x2, x3) = myfunc2(2, 3) + x3 +foo() #------------------------------------------------------------ #-- Move into array constructor @@ -178,7 +179,7 @@ proc myfuncLoop(x: int): MySeqNonCopyable = discard myfuncLoop(3) #------------------------------------------------------------ -# Move into table via openarray +# Move into table via openArray #------------------------------------------------------------ type diff --git a/tests/destructor/tnewruntime_misc.nim b/tests/destructor/tnewruntime_misc.nim index 1a5b0e3b0f..21c70557db 100644 --- a/tests/destructor/tnewruntime_misc.nim +++ b/tests/destructor/tnewruntime_misc.nim @@ -7,7 +7,7 @@ axc ... destroying GenericObj[T] GenericObj[system.int] test -(allocCount: 13, deallocCount: 11) +(allocCount: 12, deallocCount: 10) 3''' """ @@ -25,8 +25,13 @@ putEnv("HEAPTRASHING", "Indeed") let s1 = getAllocStats() + +proc newTableOwned[A, B](initialSize = defaultInitialSize): owned(TableRef[A, B]) = + new(result) + result[] = initTable[A, B](initialSize) + proc main = - var w = newTable[string, owned Node]() + var w = newTableOwned[string, owned Node]() w["key"] = Node(field: "value") echo w["key"][] echo getEnv("HEAPTRASHING") @@ -132,7 +137,16 @@ proc xx(xml: string): MyObject = discard xx("test") -echo getAllocStats() - s1 + +# Windows has 1 extra allocation in `getEnv` - there it allocates parameter to +# `_wgetenv` (WideCString). Therefore subtract by 1 to match other OSes' +# allocation. +when defined(windows): + import std/importutils + privateAccess(AllocStats) + echo getAllocStats() - s1 - AllocStats(allocCount: 1, deallocCount: 1) +else: + echo getAllocStats() - s1 # bug #13457 var s = "abcde" diff --git a/tests/destructor/tnewruntime_strutils.nim b/tests/destructor/tnewruntime_strutils.nim index 8e5378f77d..9c8d419734 100644 --- a/tests/destructor/tnewruntime_strutils.nim +++ b/tests/destructor/tnewruntime_strutils.nim @@ -5,7 +5,8 @@ discard """ @[(input: @["KXSC", "BGMC"]), (input: @["PXFX"]), (input: @["WXRQ", "ZSCZD"])] 14 First tasks completed. -Second tasks completed.''' +Second tasks completed. +test1''' """ import strutils, os, std / wordwrap @@ -241,3 +242,13 @@ when true: test_string_b.setLen new_len_b echo "Second tasks completed." + +# bug #17450 +proc main = + var i = 1 + echo: + block: + "test" & $i + +main() + diff --git a/tests/destructor/tnonvardestructor.nim b/tests/destructor/tnonvardestructor.nim new file mode 100644 index 0000000000..1b44137906 --- /dev/null +++ b/tests/destructor/tnonvardestructor.nim @@ -0,0 +1,247 @@ +discard """ + targets: "c cpp" + matrix: "--mm:arc; --mm:orc" +""" + +block: + type + PublicKey = array[32, uint8] + PrivateKey = array[64, uint8] + + proc ed25519_create_keypair(publicKey: ptr PublicKey; privateKey: ptr PrivateKey) = + publicKey[][0] = uint8(88) + + type + KeyPair = object + public: PublicKey + private: PrivateKey + + proc initKeyPair(): KeyPair = + ed25519_create_keypair(result.public.addr, result.private.addr) + + let keys = initKeyPair() + doAssert keys.public[0] == 88 + + +template minIndexByIt: untyped = + var other = 3 + other + +proc bug20303() = + var hlibs = @["hello", "world", "how", "are", "you"] + let res = hlibs[minIndexByIt()] + doAssert res == "are" + +bug20303() + +proc main() = # todo bug with templates + block: # bug #11267 + var a: seq[char] = block: @[] + doAssert a == @[] + # 2 + proc b: seq[string] = + discard + @[] + doAssert b() == @[] +static: main() +main() + + +type Obj = tuple + value: int + arr: seq[int] + +proc bug(): seq[Obj] = + result.add (value: 0, arr: @[]) + result[^1].value = 1 + result[^1].arr.add 1 + +# bug #19990 +let s = bug() +doAssert s[0] == (value: 1, arr: @[1]) + +block: # bug #21974 + type Test[T] = ref object + values : seq[T] + counter: int + + proc newTest[T](): Test[T] = + result = new(Test[T]) + result.values = newSeq[T](16) + result.counter = 0 + + proc push[T](self: Test[T], value: T) = + self.counter += 1 + if self.counter >= self.values.len: + self.values.setLen(self.values.len * 2) + self.values[self.counter - 1] = value + + proc pop[T](self: Test[T]): T = + result = self.values[0] + self.values[0] = self.values[self.counter - 1] # <--- This line + self.counter -= 1 + + + type X = tuple + priority: int + value : string + + var a = newTest[X]() + a.push((1, "One")) + doAssert a.pop.value == "One" + +# bug #21987 + +type + EmbeddedImage* = distinct Image + Image = object + len: int + +proc imageCopy*(image: Image): Image {.nodestroy.} + +proc `=destroy`*(x: Image) = + discard + +proc `=sink`*(dest: var Image; source: Image) = + `=destroy`(dest) + wasMoved(dest) + +proc `=dup`*(source: Image): Image {.nodestroy.} = + result = imageCopy(source) + +proc `=copy`*(dest: var Image; source: Image) = + dest = imageCopy(source) # calls =sink implicitly + +proc `=destroy`*(x: EmbeddedImage) = discard + +proc `=dup`*(source: EmbeddedImage): EmbeddedImage {.nodestroy.} = source + +proc `=copy`*(dest: var EmbeddedImage; source: EmbeddedImage) {.nodestroy.} = + dest = source + +proc imageCopy*(image: Image): Image = + result = image + +proc main2 = + block: + var a = Image(len: 2).EmbeddedImage + var b = Image(len: 1).EmbeddedImage + b = a + doAssert Image(a).len == 2 + doAssert Image(b).len == 2 + + block: + var a = Image(len: 2) + var b = Image(len: 1) + b = a + doAssert a.len == 2 + doAssert b.len == 0 + +main2() + +type + Edge = object + neighbor {.cursor.}: Node + + NodeObj = object + neighbors: seq[Edge] + label: string + visited: bool + Node = ref NodeObj + + Graph = object + nodes: seq[Node] + +proc `=destroy`(x: NodeObj) = + `=destroy`(x.neighbors) + `=destroy`(x.label) + +proc addNode(self: var Graph; label: string): Node = + self.nodes.add(Node(label: label)) + result = self.nodes[^1] + +proc addEdge(self: Graph; source, neighbor: Node) = + source.neighbors.add(Edge(neighbor: neighbor)) + +block: + proc main = + var graph: Graph + let nodeA = graph.addNode("a") + let nodeB = graph.addNode("b") + let nodeC = graph.addNode("c") + + graph.addEdge(nodeA, neighbor = nodeB) + graph.addEdge(nodeA, neighbor = nodeC) + + main() + +block: + type RefObj = ref object + + proc `[]`(val: static[int]) = # works with different name/overload or without static arg + discard + + template noRef(T: typedesc): typedesc = # works without template indirection + typeof(default(T)[]) + + proc `=destroy`(x: noRef(RefObj)) = + discard + + proc foo = + var x = new RefObj + doAssert $(x[]) == "()" + + # bug #11705 + foo() + +block: # bug #22197 + type + H5IdObj = object + H5Id = ref H5IdObj + + FileID = distinct H5Id + + H5GroupObj = object + file_id: FileID + H5Group = ref H5GroupObj + + ## This would make it work! + #proc `=destroy`*(x: FileID) = `=destroy`(cast[H5Id](x)) + ## If this does not exist, it also works! + proc newFileID(): FileID = FileID(H5Id()) + + proc `=destroy`(grp: H5GroupObj) = + ## Closes the group and resets all references to nil. + if cast[pointer](grp.fileId) != nil: + `=destroy`(grp.file_id) + + var grp = H5Group() + reset(grp.file_id) + reset(grp) + +import std/tables + +block: # bug #22286 + type + A = object + B = object + a: A + C = object + b: B + + proc `=destroy`(self: A) = + echo "destroyed" + + proc `=destroy`(self: C) = + `=destroy`(self.b) + + var c = C() + +block: # https://forum.nim-lang.org/t/10642 + type AObj = object + name: string + tableField: Table[string, string] + + proc `=destroy`(x: AObj) = + `=destroy`(x.name) + `=destroy`(x.tableField) diff --git a/tests/destructor/topttree.nim b/tests/destructor/topttree.nim index fa54956895..8cf757e8bf 100644 --- a/tests/destructor/topttree.nim +++ b/tests/destructor/topttree.nim @@ -1,4 +1,5 @@ discard """ + disabled: i386 output: '''10.0 60.0 90.0 diff --git a/tests/destructor/tprevent_assign2.nim b/tests/destructor/tprevent_assign2.nim index ef20672d59..eb5588b1a2 100644 --- a/tests/destructor/tprevent_assign2.nim +++ b/tests/destructor/tprevent_assign2.nim @@ -1,7 +1,7 @@ discard """ - errormsg: "'=copy' is not available for type ; requires a copy because it's not the last read of 'otherTree'" + errormsg: "'=dup' is not available for type , which is inferred from unavailable '=copy'; requires a copy because it's not the last read of 'otherTree'; another read is done here: tprevent_assign2.nim(51, 31); routine: preventThis" file: "tprevent_assign2.nim" - line: 48 + line: 49 """ type @@ -9,7 +9,8 @@ type x: int proc `=destroy`(f: var Foo) = f.x = 0 -proc `=`(a: var Foo; b: Foo) {.error.} # = a.x = b.x +proc `=copy`(a: var Foo; b: Foo) {.error.} # = a.x = b.x + proc `=sink`(a: var Foo; b: Foo) = a.x = b.x proc createTree(x: int): Foo = @@ -18,7 +19,7 @@ proc createTree(x: int): Foo = proc take2(a, b: sink Foo) = echo a.x, " ", b.x -proc allowThis() = +when false: var otherTree: Foo try: for i in 0..3: @@ -51,5 +52,5 @@ proc preventThis() = else: discard -allowThis() +#allowThis() preventThis() diff --git a/tests/destructor/tprevent_assign3.nim b/tests/destructor/tprevent_assign3.nim index 0577aa5ff0..aa834a66ce 100644 --- a/tests/destructor/tprevent_assign3.nim +++ b/tests/destructor/tprevent_assign3.nim @@ -1,7 +1,7 @@ discard """ - errormsg: "'=copy' is not available for type ; requires a copy because it's not the last read of 'otherTree'" + errormsg: "'=dup' is not available for type ; requires a copy because it's not the last read of 'otherTree'" file: "tprevent_assign3.nim" - line: 46 + line: 47 """ type @@ -9,7 +9,8 @@ type x: int proc `=destroy`(f: var Foo) = f.x = 0 -proc `=`(a: var Foo; b: Foo) {.error.} # = a.x = b.x +proc `=copy`(a: var Foo; b: Foo) {.error.} # = a.x = b.x +proc `=dup`(a: Foo): Foo {.error.} proc `=sink`(a: var Foo; b: Foo) = a.x = b.x proc createTree(x: int): Foo = @@ -18,7 +19,7 @@ proc createTree(x: int): Foo = proc take2(a, b: sink Foo) = echo a.x, " ", b.x -proc allowThis() = +when false: var otherTree: Foo try: for i in 0..3: @@ -47,7 +48,7 @@ proc preventThis2() = finally: echo otherTree -allowThis() +#allowThis() preventThis2() diff --git a/tests/destructor/trecursive.nim b/tests/destructor/trecursive.nim index 17a40e5a99..e7afa6ba97 100644 --- a/tests/destructor/trecursive.nim +++ b/tests/destructor/trecursive.nim @@ -47,7 +47,7 @@ proc `=destroy`(x: var MyObject) = proc `=`(x: var MyObject, y: MyObject) {.error.} proc newMyObject(i: int): MyObject = - result.p = create(int) + result.p = createShared(int) result.p[] = i proc test: seq[MyObject] = diff --git a/tests/destructor/tsink.nim b/tests/destructor/tsink.nim new file mode 100644 index 0000000000..82cbdfbe5d --- /dev/null +++ b/tests/destructor/tsink.nim @@ -0,0 +1,16 @@ +discard """ + matrix: "--mm:arc" +""" + +type AnObject = object of RootObj + value*: int + +proc mutate(shit: sink AnObject) = + shit.value = 1 + +proc foo = # bug #23359 + var bar = AnObject(value: 42) + mutate(bar) + doAssert bar.value == 42 + +foo() diff --git a/tests/destructor/tuse_ownedref_after_move.nim b/tests/destructor/tuse_ownedref_after_move.nim index ce96b741ed..69348d5303 100644 --- a/tests/destructor/tuse_ownedref_after_move.nim +++ b/tests/destructor/tuse_ownedref_after_move.nim @@ -1,6 +1,6 @@ discard """ cmd: '''nim c --newruntime $file''' - errormsg: "'=copy' is not available for type ; requires a copy because it's not the last read of ':envAlt.b1'; another read is done here: tuse_ownedref_after_move.nim(52, 4)" + errormsg: "'=copy' is not available for type ; requires a copy because it's not the last read of ':envAlt.b1'; routine: main" line: 48 """ diff --git a/tests/destructor/tuse_result_prevents_sinks.nim b/tests/destructor/tuse_result_prevents_sinks.nim index d2777bd976..e74c16da36 100644 --- a/tests/destructor/tuse_result_prevents_sinks.nim +++ b/tests/destructor/tuse_result_prevents_sinks.nim @@ -18,7 +18,7 @@ proc `=sink`(self: var Foo; other: Foo) = proc `=destroy`(self: var Foo) = discard template preventCursorInference(x) = - let p = unsafeAddr(x) + let p = addr(x) proc test(): Foo = result = Foo() diff --git a/tests/destructor/tv2_cast.nim b/tests/destructor/tv2_cast.nim index ef0b3a9362..4ff2dc9a07 100644 --- a/tests/destructor/tv2_cast.nim +++ b/tests/destructor/tv2_cast.nim @@ -1,10 +1,109 @@ discard """ - cmd: '''nim c --gc:arc $file''' output: '''@[1] @[116, 101, 115, 116] -@[1953719668, 875770417]''' +@[1953719668, 875770417] +destroying O1''' + cmd: '''nim c --mm:arc --expandArc:main --expandArc:main1 --expandArc:main2 --expandArc:main3 --hints:off --assertions:off $file''' + nimout: ''' +--expandArc: main + +var + data + :tmpD + :tmpD_1 + :tmpD_2 +data = + :tmpD = `=dup`(cast[string]( + :tmpD_2 = encode(cast[seq[byte]]( + :tmpD_1 = newString(100) + :tmpD_1)) + :tmpD_2)) + :tmpD +`=destroy`(:tmpD_2) +`=destroy_1`(:tmpD_1) +`=destroy_1`(data) +-- end of expandArc ------------------------ +--expandArc: main1 + +var + s + data + :tmpD + :tmpD_1 +s = newString(100) +data = + :tmpD = `=dup`(cast[string]( + :tmpD_1 = encode(toOpenArrayByte(s, 0, len(s) - 1)) + :tmpD_1)) + :tmpD +`=destroy`(:tmpD_1) +`=destroy_1`(data) +`=destroy_1`(s) +-- end of expandArc ------------------------ +--expandArc: main2 + +var + s + data + :tmpD + :tmpD_1 +s = newSeq(100) +data = + :tmpD = `=dup`(cast[string]( + :tmpD_1 = encode(s) + :tmpD_1)) + :tmpD +`=destroy`(:tmpD_1) +`=destroy_1`(data) +`=destroy`(s) +-- end of expandArc ------------------------ +--expandArc: main3 + +var + data + :tmpD + :tmpD_1 + :tmpD_2 +data = + :tmpD = `=dup`(cast[string]( + :tmpD_2 = encode do: + :tmpD_1 = newSeq(100) + :tmpD_1 + :tmpD_2)) + :tmpD +`=destroy`(:tmpD_2) +`=destroy`(:tmpD_1) +`=destroy_1`(data) +-- end of expandArc ------------------------ +''' """ +func encode*(src: openArray[byte]): seq[byte] = + result = newSeq[byte](src.len) + +template compress*(src: string): string = + cast[string](encode(cast[seq[byte]](src))) + +proc main = + let data = compress(newString(100)) +main() + +proc main1 = + var + s = newString(100) + let data = cast[string](encode(s.toOpenArrayByte(0, s.len-1))) +main1() + +proc main2 = + var + s = newSeq[byte](100) + let data = cast[string](encode(s)) +main2() + +proc main3 = + let data = cast[string](encode(newSeq[byte](100))) +main3() + # bug #11018 discard cast[seq[uint8]](@[1]) discard cast[seq[uint8]]("test") @@ -20,4 +119,26 @@ echo a #issue 11204 var ac {.compileTime.} = @["a", "b"] -const bc = ac.len \ No newline at end of file +const bc = ac.len + + +type + O = object of RootRef + i: int + + O1 = object of O + O2 = object of O + +proc `=destroy`(o: var O) = + echo "destroying O" + +proc `=destroy`(o: var O1) = + echo "destroying O1" + +proc `=destroy`(o: var O2) = + echo "destroying O2" + +proc test = + let o3 = cast[ref O2]((ref O1)()) + +test() diff --git a/tests/destructor/twasmoved.nim b/tests/destructor/twasmoved.nim new file mode 100644 index 0000000000..5663227022 --- /dev/null +++ b/tests/destructor/twasmoved.nim @@ -0,0 +1,14 @@ +type + Foo = object + id: int + +proc `=wasMoved`(x: var Foo) = + x.id = -1 + +proc foo = + var s = Foo(id: 999) + var m = move s + doAssert s.id == -1 + doAssert m.id == 999 + +foo() diff --git a/tests/destructor/twasmoved_error.nim b/tests/destructor/twasmoved_error.nim new file mode 100644 index 0000000000..1cd57e3df1 --- /dev/null +++ b/tests/destructor/twasmoved_error.nim @@ -0,0 +1,37 @@ +discard """ + cmd: '''nim c --mm:arc $file''' + errormsg: "'=wasMoved' is not available for type ; routine: main" +""" + +# bug #19291 + +const + screenWidth = 800 + screenHeight = 450 + +var + ready = false +type + Game = object + +proc `=destroy`(x: var Game) = + assert ready, "Window is already opened" + ready = false + +proc `=sink`(x: var Game; y: Game) {.error.} +proc `=copy`(x: var Game; y: Game) {.error.} +proc `=wasMoved`(x: var Game) {.error.} + +proc initGame(width, height: int32, title: string): Game = + assert not ready, "Window is already closed" + ready = true + +proc update(x: Game) = discard + +proc main = + var g = initGame(screenWidth, screenHeight, "Tetris raylib") + g.update() + var g2 = g + echo "hello" + +main() diff --git a/tests/discard/t23677.nim b/tests/discard/t23677.nim new file mode 100644 index 0000000000..1ed7386bde --- /dev/null +++ b/tests/discard/t23677.nim @@ -0,0 +1,12 @@ +discard """ + errormsg: "expression '0' is of type 'int literal(0)' and has to be used (or discarded); start of expression here: t23677.nim(1, 1)" + line: 10 + column: 3 +""" + +# issue #23677 + +if true: + 0 +else: + raise newException(ValueError, "err") diff --git a/tests/discard/tdiscardable.nim b/tests/discard/tdiscardable.nim index 0320501397..5988f59497 100644 --- a/tests/discard/tdiscardable.nim +++ b/tests/discard/tdiscardable.nim @@ -5,6 +5,7 @@ tdiscardable 1 something defered something defered +hi ''' """ @@ -65,3 +66,77 @@ proc main2() = main1() main2() + +block: # bug #13583 + block: + proc hello(): int {.discardable.} = 12 + + iterator test(): int {.closure.} = + while true: + hello() + + let t = test + + block: + proc hello(): int {.discardable.} = 12 + + iterator test(): int {.closure.} = + while true: + block: + yield 12 + hello() + + let t = test + doAssert t() == 12 + + block: + proc hello(): int {.discardable.} = 12 + + iterator test(): int {.closure.} = + while true: + yield 12 + hello() + + let t = test + doAssert t() == 12 + +block: + proc bar(): string {.discardable.} = + "15" + + proc foo(): int = + while true: + raise newException(ValueError, "check") + 12 + + doAssertRaises(ValueError): + doAssert foo() == 12 + +block: # issue #10440 + proc x(): int {.discardable.} = discard + try: + x() + finally: + echo "hi" + +import macros + +block: # issue #14665 + macro test(): untyped = + let b = @[1, 2, 3, 4] + + result = nnkStmtList.newTree() + var i = 0 + while i < b.len: + if false: + # this quote do is mandatory, removing it fixes the problem + result.add quote do: + let testtest = 5 + else: + result.add quote do: + let test = 6 + inc i + # removing this continue fixes the problem too + continue + inc i + test() diff --git a/tests/discard/tfinallyerrmsg.nim b/tests/discard/tfinallyerrmsg.nim new file mode 100644 index 0000000000..fbc8140aaf --- /dev/null +++ b/tests/discard/tfinallyerrmsg.nim @@ -0,0 +1,19 @@ +discard """ + cmd: "nim check $file" +""" + +block: # issue #19672 + try: + 10 #[tt.Error + ^ expression '10' is of type 'int literal(10)' and has to be used (or discarded); start of expression here: tfinallyerrmsg.nim(5, 1)]# + finally: + echo "Finally block" + +block: # issue #13871 + template t(body: int) = + try: + body + finally: + echo "expression" + t: 2 #[tt.Error + ^ expression '2' is of type 'int literal(2)' and has to be used (or discarded)]# diff --git a/tests/discard/tillegaldiscardtypes.nim b/tests/discard/tillegaldiscardtypes.nim new file mode 100644 index 0000000000..b7877bcd2c --- /dev/null +++ b/tests/discard/tillegaldiscardtypes.nim @@ -0,0 +1,14 @@ +discard """ + cmd: "nim check $file" + errormsg: "statement returns no value that can be discarded" + nimout: ''' +tillegaldiscardtypes.nim(11, 3) Error: statement returns no value that can be discarded +tillegaldiscardtypes.nim(12, 3) Error: statement returns no value that can be discarded +''' +""" + +proc b(v: int) = # bug #21360 + discard @[] + discard [] + +b(0) \ No newline at end of file diff --git a/tests/distinct/tborrow.nim b/tests/distinct/tborrow.nim new file mode 100644 index 0000000000..e34248de5b --- /dev/null +++ b/tests/distinct/tborrow.nim @@ -0,0 +1,132 @@ +discard """ + output: '''4887 true +0.5''' +""" + +# test the new borrow feature that works with generics: + +proc `++`*[T: int | float](a, b: T): T = + result = a + b + +type + DI = distinct int + DF = distinct float + DS = distinct string + +proc `++`(x, y: DI): DI {.borrow.} +proc `++`(x, y: DF): DF {.borrow.} + +proc `$`(x: DI): string {.borrow.} +proc `$`(x: DF): string {.borrow.} + +echo 4544.DI ++ 343.DI, " ", (4.5.DF ++ 0.5.DF).float == 5.0 + +# issue #14440 + +type Radians = distinct float64 + +func `-=`(a: var Radians, b: Radians) {.borrow.} + +var a = Radians(1.5) +let b = Radians(1.0) + +a -= b + +echo a.float64 + +block: #14449 + type + Foo[T] = object + foo: T + + Bar[T] {.borrow:`.`.} = distinct Foo[T] + SomeThing {.borrow:`.`.} = distinct Foo[float] + OtherThing {.borrow:`.`.} = distinct SomeThing + + var + a: Bar[int] + b: SomeThing + c: OtherThing + a.foo = 300 + b.foo = 400 + c.foo = 42 + assert a.foo == 300 + assert b.foo == 400d + assert c.foo == 42d + +block: # Borrow from muliple aliasses #16666 + type + AImpl = object + i: int + + A = AImpl + + B {.borrow: `.`.} = distinct A + C = B + D {.borrow: `.`.} = distinct C + E {.borrow: `.`.} = distinct D + + let + b = default(B) + d = default(D) + e = default(E) + + assert b.i == 0 + assert d.i == 0 + assert e.i == 0 + +block: # Borrow from generic alias + type + AImpl[T] = object + i: T + B[T] = AImpl[T] + C {.borrow: `.`.} = distinct B[int] + D = B[float] + E {.borrow: `.`.} = distinct D + + let + c = default(C) + e = default(E) + assert c.i == int(0) + assert e.i == 0d + +block: # issue #22069 + type + Vehicle[C: static[int]] = object + color: array[C, int] + Car[C: static[int]] {.borrow: `.`.} = distinct Vehicle[C] + MuscleCar = Car[128] + var x: MuscleCar + doAssert x.color is array[128, int] + +block: # issue #22646 + type + Vec[N : static[int], T: SomeNumber] = object + arr: array[N, T] + Vec3[T: SomeNumber] = Vec[3, T] + + proc `[]=`[N,T](v: var Vec[N,T]; ix:int; c:T): void {.inline.} = v.arr[ix] = c + proc `[]`[N,T](v: Vec[N,T]; ix: int): T {.inline.} = v.arr[ix] + + proc dot[N,T](u,v: Vec[N,T]): T {. inline .} = discard + proc length[N,T](v: Vec[N,T]): T = discard + proc cross[T](v1,v2:Vec[3,T]): Vec[3,T] = discard + proc normalizeWorks[T](v: Vec[3,T]): Vec[3,T] = discard ## <- Explicit size makes it work! + proc foo[N,T](u, v: Vec[N,T]): Vec[N,T] = discard ## <- broken + proc normalize[N,T](v: Vec[N,T]): Vec[N,T] = discard ## <- broken + + type Color = distinct Vec3[float] + + template borrowOps(typ: typedesc): untyped = + proc `[]=`(v: var typ; ix: int; c: float): void {.borrow.} + proc `[]`(v: typ; ix: int): float {.borrow.} + proc dot(v, u: typ): float {.borrow.} + proc cross(v, u: typ): typ {.borrow.} + proc length(v: typ): float {.borrow.} + proc normalizeWorks(v: typ): typ {.borrow.} ## Up to here everything works + proc foo(u, v: typ): typ {.borrow.} ## Broken + proc normalize(v: typ): typ {.borrow.} ## Broken + borrowOps(Color) + var x: Vec[3, float] + let y = Color(x) + doAssert Vec3[float](y) == x diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim index d64f33443f..b6ba7aa993 100644 --- a/tests/distinct/tdistinct.nim +++ b/tests/distinct/tdistinct.nim @@ -1,4 +1,5 @@ discard """ + targets: "c js" output: ''' tdistinct 25 @@ -7,6 +8,7 @@ false false false Foo +foo ''' """ @@ -133,8 +135,93 @@ block tRequiresInit: reject: var s: DistinctString s = "test" - doAssert s == "test" + doAssert string(s) == "test" accept: - let s = "test" - doAssert s == "test" + let s = DistinctString("test") + doAssert string(s) == "test" + +block: #17322 + type + A[T] = distinct string + + proc foo(a: var A) = + a.string.add "foo" + + type + B = distinct A[int] + + var b: B + foo(A[int](b)) + echo A[int](b).string + b.string.add "bar" + assert b.string == "foobar" + +type Foo = distinct string + +proc main() = # proc instead of template because of MCS/UFCS. + # xxx put everything here to test under RT + VM + block: # bug #12282 + block: + proc test() = + var s: Foo + s.string.add('c') + doAssert s.string == "c" # was failing + test() + + block: + proc add(a: var Foo, b: char) {.borrow.} + proc test() = + var s: Foo + s.add('c') + doAssert s.string == "c" # was ok + test() + + block: + proc add(a: var Foo, b: char) {.borrow.} + proc test() = + var s: string + s.Foo.add('c') + doAssert s.string == "c" # was failing + test() + block: #18061 + type + A = distinct (0..100) + B = A(0) .. A(10) + proc test(b: B) = discard + let + a = A(10) + b = B(a) + test(b) + + proc test(a: A) = discard + discard cast[B](A(1)) + var c: B + + + block: # bug #9423 + block: + type Foo = seq[int] + type Foo2 = distinct Foo + template fn() = + var a = Foo2(@[1]) + a.Foo.add 2 + doAssert a.Foo == @[1, 2] + fn() + + block: + type Stack[T] = distinct seq[T] + proc newStack[T](): Stack[T] = + Stack[T](newSeq[T]()) + proc push[T](stack: var Stack[T], elem: T) = + seq[T](stack).add(elem) + proc len[T](stack: Stack[T]): int = + seq[T](stack).len + proc fn() = + var stack = newStack[int]() + stack.push(5) + doAssert stack.len == 1 + fn() + +static: main() +main() diff --git a/tests/distinct/tinvalidborrow.nim b/tests/distinct/tinvalidborrow.nim new file mode 100644 index 0000000000..d4b19fa8de --- /dev/null +++ b/tests/distinct/tinvalidborrow.nim @@ -0,0 +1,42 @@ +discard """ + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout:''' +tinvalidborrow.nim(25, 3) Error: only a 'distinct' type can borrow `.` +tinvalidborrow.nim(26, 3) Error: only a 'distinct' type can borrow `.` +tinvalidborrow.nim(27, 1) Error: borrow proc without distinct type parameter is meaningless +tinvalidborrow.nim(36, 1) Error: borrow with generic parameter is not supported +tinvalidborrow.nim(41, 1) Error: borrow from proc return type mismatch: 'T' +tinvalidborrow.nim(42, 1) Error: borrow from '[]=' is not supported +''' +""" + + + + + +# bug #516 + +type + TAtom = culong + Test {.borrow:`.`.} = distinct int + Foo[T] = object + a: int + Bar[T] {.borrow:`.`.} = Foo[T] + OtherFoo {.borrow:`.`.} = Foo[int] +proc `==`*(a, b: TAtom): bool {.borrow.} + +var + d, e: TAtom + +discard( $(d == e) ) + +# issue #4121 +type HeapQueue[T] = distinct seq[T] +proc len*[T](h: HeapQueue[T]): int {.borrow.} + +# issue #3564 +type vec4[T] = distinct array[4, float32] + +proc `[]`(v: vec4, i: int): float32 {.borrow.} +proc `[]=`(v: vec4, i: int, va: float32) {.borrow.} diff --git a/tests/distinct/tnil.nim b/tests/distinct/tnil.nim index 5bdb97f375..b54c07432d 100644 --- a/tests/distinct/tnil.nim +++ b/tests/distinct/tnil.nim @@ -1,29 +1,21 @@ -discard """ -output: ''' -1 -0 -0 -''' -""" {.experimental: "notnil".} type MyPointer = distinct pointer MyString = distinct string MyInt = distinct int -proc foo(a: MyPointer) = +proc foo(a: MyPointer): int = # workaround a Windows 'repr' difference: - echo cast[int](a) + cast[int](a) -foo(cast[MyPointer](1)) -foo(cast[MyPointer](nil)) -foo(nil) +doAssert foo(cast[MyPointer](1)) == 1 +doAssert foo(cast[MyPointer](nil)) == 0 +doAssert foo(MyPointer(nil)) == 0 var p: MyPointer p = cast[MyPointer](1) p = cast[MyPointer](nil) p = nil.MyPointer -p = nil var i: MyInt i = 1.MyInt diff --git a/tests/distinct/typeclassborrow.nim b/tests/distinct/typeclassborrow.nim new file mode 100644 index 0000000000..5e0c639534 --- /dev/null +++ b/tests/distinct/typeclassborrow.nim @@ -0,0 +1,56 @@ +import std/tables + +type + Foo = distinct seq[int] + Bar[N: static[int]] = distinct seq[int] + Baz = distinct Bar[10] + +proc newSeq(s: var Foo, n: Natural) {.borrow.} +proc newSeq(s: var Bar, n: Natural) {.borrow.} +proc newSeq(s: var Baz, n: Natural) {.borrow.} + + +proc `$`(s: Foo): string {.borrow.} +proc `$`(s: Bar): string {.borrow.} +proc `$`(s: Baz): string {.borrow.} + +proc doThing(b: Bar) = discard +proc doThing(b: Baz) {.borrow.} + +var + foo: Foo + bar: Bar[10] + baz: Baz + +newSeq(foo, 100) +newSeq(bar, bar.N) +newSeq(baz, 10) + +bar.doThing() +baz.doThing() + +assert $seq[int](foo) == $foo +assert $seq[int](bar) == $bar +assert $seq[int](baz) == $baz + +type + Fine* = distinct string + +proc `==`*(x, y: Fine): bool {.borrow.} = + ## Here is the documentation + runnableExamples: + var x = Fine("1234") + var y = Fine("1234") + doAssert x == y + doAssert false + + +var x = Fine("1234") +var y = Fine("1234") +doAssert x == y + +block: # bug #22902 + type + DistinctTable = distinct Table[int, int] + + proc `[]`(t: DistinctTable; key: int): lent int {.borrow.} diff --git a/tests/dll/client.nim b/tests/dll/client.nim index 150af3a174..62697569f0 100644 --- a/tests/dll/client.nim +++ b/tests/dll/client.nim @@ -1,5 +1,4 @@ discard """ - output: "Done" cmd: "nim $target --debuginfo --hints:on --define:useNimRtl $options $file" """ @@ -37,5 +36,8 @@ proc eval(n: PNode): int = for i in 0..100_000: discard eval(buildTree(2)) -echo "Done" - +# bug https://forum.nim-lang.org/t/8176; Error: ambiguous identifier: 'nimrtl' +import std/strutils +doAssert join(@[1, 2]) == "12" +doAssert join(@[1.5, 2.5]) == "1.52.5" +doAssert join(@["a", "bc"]) == "abc" diff --git a/tests/dll/nimhcr_basic.nim b/tests/dll/nimhcr_basic.nim new file mode 100644 index 0000000000..2e1f39ae06 --- /dev/null +++ b/tests/dll/nimhcr_basic.nim @@ -0,0 +1,8 @@ +discard """ + output: ''' +Hello world +''' +""" +# for now orc only tests successful compilation + +echo "Hello world" diff --git a/tests/dll/nimhcr_unit.nim b/tests/dll/nimhcr_unit.nim index 0b924bdf75..249f3f9f17 100644 --- a/tests/dll/nimhcr_unit.nim +++ b/tests/dll/nimhcr_unit.nim @@ -106,14 +106,14 @@ macro carryOutTests(callingConv: untyped): untyped = echo `procName`, " implementation #1 ", x return x + 1 - let fp1 = cast[F](hcrRegisterProc("dummy_module", `procName`, `p1`)) + let fp1 = cast[F](hcrRegisterProc("dummy_module", `procName`, cast[pointer](`p1`))) echo fp1(10) proc `p2`(x: int): int {.placeholder.} = echo `procName`, " implementation #2 ", x return x + 2 - let fp2 = cast[F](hcrRegisterProc("dummy_module", `procName`, `p2`)) + let fp2 = cast[F](hcrRegisterProc("dummy_module", `procName`, cast[pointer](`p2`))) echo fp1(20) echo fp2(20) @@ -121,7 +121,7 @@ macro carryOutTests(callingConv: untyped): untyped = echo `procName`, " implementation #3 ", x return x + 3 - let fp3 = cast[F](hcrRegisterProc("dummy_module", `procName`, `p3`)) + let fp3 = cast[F](hcrRegisterProc("dummy_module", `procName`, cast[pointer](`p3`))) echo fp1(30) echo fp2(30) echo fp3(30) diff --git a/tests/effects/tdiagnostic_messages.nim b/tests/effects/tdiagnostic_messages.nim new file mode 100644 index 0000000000..b1acf8c5cf --- /dev/null +++ b/tests/effects/tdiagnostic_messages.nim @@ -0,0 +1,37 @@ +discard """ + nimoutFull: true + action: "reject" + cmd: "nim r --hint:Conf:off $file" + nimout: ''' +tdiagnostic_messages.nim(36, 6) Error: 'a' can have side effects +> tdiagnostic_messages.nim(37, 30) Hint: 'a' calls `.sideEffect` 'callWithSideEffects' +>> tdiagnostic_messages.nim(29, 6) Hint: 'callWithSideEffects' called by 'a' +>>> tdiagnostic_messages.nim(31, 34) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaVarParam' +>>>> tdiagnostic_messages.nim(25, 6) Hint: 'indirectCallViaVarParam' called by 'callWithSideEffects' +>>>>> tdiagnostic_messages.nim(26, 7) Hint: 'indirectCallViaVarParam' calls routine via hidden pointer indirection +>>> tdiagnostic_messages.nim(32, 33) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaPointer' +>>>> tdiagnostic_messages.nim(27, 6) Hint: 'indirectCallViaPointer' called by 'callWithSideEffects' +>>>>> tdiagnostic_messages.nim(28, 32) Hint: 'indirectCallViaPointer' calls routine via pointer indirection +>>> tdiagnostic_messages.nim(33, 3) Hint: 'callWithSideEffects' calls `.sideEffect` 'myEcho' +>>>> tdiagnostic_messages.nim(24, 6) Hint: 'myEcho' called by 'callWithSideEffects' +>>> tdiagnostic_messages.nim(34, 3) Hint: 'callWithSideEffects' accesses global state 'globalVar' +>>>> tdiagnostic_messages.nim(23, 5) Hint: 'globalVar' accessed by 'callWithSideEffects' + +''' +""" + +var globalVar = 0 +proc myEcho(a: string) {.sideEffect.} = discard +proc indirectCallViaVarParam(call: var proc(): int {.nimcall.}): int = + call() +proc indirectCallViaPointer(call: pointer): int = + cast[ptr proc(): int](call)[]() +proc callWithSideEffects(): int = + var p = proc (): int {.nimcall.} = 0 + discard indirectCallViaVarParam(p) + discard indirectCallViaPointer(addr p) + myEcho "" + globalVar + +func a: int = + discard callWithSideEffects() diff --git a/tests/effects/teffects1.nim b/tests/effects/teffects1.nim index 66d6a35189..1d267b5faf 100644 --- a/tests/effects/teffects1.nim +++ b/tests/effects/teffects1.nim @@ -1,12 +1,10 @@ discard """ - cmd: "nim check $file" - nimout: '''teffects1.nim(22, 28) template/generic instantiation from here -teffects1.nim(23, 13) Error: can raise an unlisted exception: ref IOError -teffects1.nim(22, 29) Hint: 'lier' cannot raise 'IO2Error' [XCannotRaiseY] -teffects1.nim(38, 21) Error: type mismatch: got but expected 'MyProcType = proc (x: int): string{.closure.}' -.raise effects differ''' + cmd: "nim check --hint:Conf:off --hint:XDeclaredButNotUsed:off $file" + nimout: ''' +teffects1.nim(17, 28) template/generic instantiation from here +''' """ - +{.push warningAsError[Effect]: on.} type TObj {.pure, inheritable.} = object TObjB = object of TObj @@ -16,28 +14,34 @@ type proc forw: int {. .} -proc lier(): int {.raises: [IO2Error].} = - #[tt.Hint ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]# +proc lier(): int {.raises: [IO2Error].} = #[tt.Hint + ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]# writeLine stdout, "arg" #[tt.Error - ^ can raise an unlisted exception: ref IOError - ]# + ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]# proc forw: int = raise newException(IOError, "arg") +block: + proc someProc(t: string) {.raises: [Defect].} = + discard + let vh: proc(topic: string) {.raises: [].} = someProc + {.push raises: [Defect].} type MyProcType* = proc(x: int): string #{.raises: [ValueError, Defect].} -proc foo(x: int): string {.raises: [ValueError].} = +proc foo(x: int): string {.nimcall, raises: [ValueError].} = if x > 9: raise newException(ValueError, "Use single digit") $x var p: MyProcType = foo #[tt.Error ^ -type mismatch: got but expected 'MyProcType = proc (x: int): string{.closure.}' - +type mismatch: got but expected 'MyProcType = proc (x: int): string{.closure.}' + Calling convention mismatch: got '{.nimcall.}', but expected '{.closure.}'. +.raise effects differ ]# {.pop.} +{.pop.} diff --git a/tests/effects/teffects11.nim b/tests/effects/teffects11.nim new file mode 100644 index 0000000000..20b7026ab4 --- /dev/null +++ b/tests/effects/teffects11.nim @@ -0,0 +1,21 @@ +discard """ +action: compile +errormsg: "type mismatch: got " +line: 21 +""" + +type + Effect1 = object + Effect2 = object + Effect3 = object + +proc test(fnc: proc(x: int): void {.forbids: [Effect2].}) {.tags: [Effect1, Effect3, RootEffect].} = + fnc(1) + +proc t1(x: int): void = echo $x +proc t2(x: int): void {.tags: [Effect2].} = echo $x +proc t3(x: int): void {.tags: [Effect3].} = echo $x + +test(t1) +test(t3) +test(t2) diff --git a/tests/effects/teffects12.nim b/tests/effects/teffects12.nim new file mode 100644 index 0000000000..5f5499c385 --- /dev/null +++ b/tests/effects/teffects12.nim @@ -0,0 +1,52 @@ +discard """ +action: compile +""" + +import std/locks + +type + Test2Effect* = object + Test2* = object + value2*: int + Test1Effect* = object + Test1* = object + value1*: int + Main* = object + test1Lock: Lock + test1: Test1 + test2Lock: Lock + test2: Test2 + +proc `=copy`(obj1: var Test2, obj2: Test2) {.error.} +proc `=copy`(obj1: var Test1, obj2: Test1) {.error.} +proc `=copy`(obj1: var Main, obj2: Main) {.error.} + +proc withTest1(main: var Main, + fn: proc(test1: var Test1) {.gcsafe, forbids: [Test1Effect].}) {.gcsafe, tags: [Test1Effect, RootEffect].} = + withLock(main.test1Lock): + fn(main.test1) + +proc withTest2(main: var Main, + fn: proc(test1: var Test2) {.gcsafe, forbids: [Test2Effect].}) {.gcsafe, tags: [Test2Effect, RootEffect].} = + withLock(main.test2Lock): + fn(main.test2) + +proc newMain(): Main = + var test1lock: Lock + initLock(test1Lock) + var test2lock: Lock + initLock(test2Lock) + var main = Main(test1Lock: move(test1Lock), test1: Test1(value1: 1), + test2Lock: move(test2Lock), test2: Test2(value2: 2)) + main.withTest1(proc(test1: var Test1) = test1.value1 += 1) + main.withTest2(proc(test2: var Test2) = test2.value2 += 1) + move main + +var main = newMain() +main.withTest1(proc(test1: var Test1) = + test1.value1 += 1 + main.withTest2(proc(test2: var Test2) = test2.value2 += 1) +) + +main.withTest1(proc(test1: var Test1) {.tags: [].} = echo $test1.value1) +main.withTest2(proc(test2: var Test2) {.tags: [].} = echo $test2.value2) diff --git a/tests/effects/teffects13.nim b/tests/effects/teffects13.nim new file mode 100644 index 0000000000..73082f9978 --- /dev/null +++ b/tests/effects/teffects13.nim @@ -0,0 +1,19 @@ +discard """ +action: compile +errormsg: "writeSomething() has an illegal effect: WriteIO" +line: 19 +""" + +type + IO = object of RootEffect ## input/output effect + ReadIO = object of IO ## input effect + WriteIO = object of IO ## output effect + +proc readSomething(): string {.tags: [ReadIO].} = "" +proc writeSomething(): void {.tags: [WriteIO].} = echo "..." + +proc noWritesPlease() {.forbids: [WriteIO].} = + # this is OK: + echo readSomething() + # the compiler prevents this: + writeSomething() diff --git a/tests/effects/teffects14.nim b/tests/effects/teffects14.nim new file mode 100644 index 0000000000..6291d95696 --- /dev/null +++ b/tests/effects/teffects14.nim @@ -0,0 +1,15 @@ +discard """ +action: compile +errormsg: "func1() has an illegal effect: IO" +line: 15 +""" + +type IO = object ## input/output effect +proc func1(): string {.tags: [IO].} = discard +proc func2(): string = discard + +proc no_IO_please() {.forbids: [IO].} = + # this is OK because it didn't define any tag: + discard func2() + # the compiler prevents this: + let y = func1() diff --git a/tests/effects/teffects15.nim b/tests/effects/teffects15.nim new file mode 100644 index 0000000000..c3079cdbc1 --- /dev/null +++ b/tests/effects/teffects15.nim @@ -0,0 +1,18 @@ +discard """ +action: compile +errormsg: "method1(c) has an illegal effect: IO" +line: 18 +""" + +type + IO = object ## input/output effect + CustomObject* = object of RootObj + text: string + +method method1(obj: var CustomObject): string {.tags: [IO].} = obj.text & "." +method method2(obj: var CustomObject): string = obj.text & ":" + +proc noIO() {.forbids: [IO].} = + var c = CustomObject(text: "a") + echo c.method2() + echo c.method1() diff --git a/tests/effects/teffects16.nim b/tests/effects/teffects16.nim new file mode 100644 index 0000000000..ee8f782a3a --- /dev/null +++ b/tests/effects/teffects16.nim @@ -0,0 +1,20 @@ +discard """ +action: compile +errormsg: "writeSomething(\"a\") can have an unlisted effect: WriteIO" +line: 20 +""" + +type + IO = object of RootEffect ## input/output effect + ReadIO = object of IO ## input effect + WriteIO = object of IO ## output effect + LogIO = object of IO ## another output effect + +proc readSomething(): string {.tags: [ReadIO].} = "" +proc writeSomething(msg: string): void {.tags: [WriteIO].} = echo msg +proc logSomething(msg: string): void {.tags: [LogIo].} = echo msg + +proc noWritesPlease() {.forbids: [WriteIO], tags: [LogIO, ReadIO].} = + echo readSomething() + logSomething("a") + writeSomething("a") diff --git a/tests/effects/teffects17.nim b/tests/effects/teffects17.nim new file mode 100644 index 0000000000..5e6b838962 --- /dev/null +++ b/tests/effects/teffects17.nim @@ -0,0 +1,17 @@ +discard """ +action: compile +errormsg: "writeSomething(\"a\") has an illegal effect: WriteIO" +line: 17 +""" + +type + IO = object of RootEffect ## input/output effect + ReadIO = object of IO ## input effect + WriteIO = object of IO ## output effect + +proc readSomething(): string {.tags: [ReadIO].} = "" +proc writeSomething(msg: string): void {.tags: [WriteIO].} = echo msg + +proc illegalEffectNegation() {.forbids: [WriteIO], tags: [ReadIO, WriteIO].} = + echo readSomething() + writeSomething("a") diff --git a/tests/effects/teffects18.nim b/tests/effects/teffects18.nim new file mode 100644 index 0000000000..576e766358 --- /dev/null +++ b/tests/effects/teffects18.nim @@ -0,0 +1,19 @@ +discard """ +action: compile +errormsg: "type mismatch: got " +line: 19 +""" + +type MyEffect = object +type ProcType1 = proc (i: int): void {.forbids: [MyEffect].} +type ProcType2 = proc (i: int): void + +proc testFunc(p: ProcType1): void = p(1) + +proc toBeCalled(i: int): void {.tags: [MyEffect].} = echo $i + +let emptyTags = proc(i: int): void {.tags: [].} = echo $i +let noTags: ProcType2 = proc(i: int): void = toBeCalled(i) + +testFunc(emptyTags) +testFunc(noTags) diff --git a/tests/effects/teffects19.nim b/tests/effects/teffects19.nim new file mode 100644 index 0000000000..6b4ab08194 --- /dev/null +++ b/tests/effects/teffects19.nim @@ -0,0 +1,23 @@ +discard """ +action: compile +errormsg: "type mismatch: got " +line: 23 +""" + +type MyEffect = object +type ProcType1 = proc (i: int): void {.forbids: [MyEffect].} +type ProcType2 = proc (i: int): void + +proc caller1(p: ProcType1): void = p(1) +proc caller2(p: ProcType2): void = p(1) + +proc effectful(i: int): void {.tags: [MyEffect].} = echo $i +proc effectless(i: int): void {.forbids: [MyEffect].} = echo $i + +proc toBeCalled1(i: int): void = effectful(i) +proc toBeCalled2(i: int): void = effectless(i) + +caller1(toBeCalled2) +caller2(toBeCalled1) +caller2(toBeCalled2) +caller1(toBeCalled1) diff --git a/tests/effects/teffects2.nim b/tests/effects/teffects2.nim index e4b50aba50..777a4cebc5 100644 --- a/tests/effects/teffects2.nim +++ b/tests/effects/teffects2.nim @@ -2,9 +2,9 @@ discard """ errormsg: "can raise an unlisted exception: ref IOError" line: 19 """ - +{.push warningAsError[Effect]: on.} type - TObj = object {.pure, inheritable.} + TObj {.pure, inheritable.} = object TObjB = object of TObj a, b, c: string @@ -17,3 +17,4 @@ proc lier(): int {.raises: [IOError].} = proc forw: int = raise newException(IOError, "arg") +{.pop.} diff --git a/tests/effects/teffects3.nim b/tests/effects/teffects3.nim index ee5470c47c..4c050510a0 100644 --- a/tests/effects/teffects3.nim +++ b/tests/effects/teffects3.nim @@ -4,7 +4,7 @@ discard """ """ type - TObj = object {.pure, inheritable.} + TObj {.pure, inheritable.} = object TObjB = object of TObj a, b, c: string fn: proc (): int {.tags: [].} diff --git a/tests/effects/teffects4.nim b/tests/effects/teffects4.nim index 88cc0efa93..b875754b6b 100644 --- a/tests/effects/teffects4.nim +++ b/tests/effects/teffects4.nim @@ -4,7 +4,7 @@ discard """ """ type - TObj = object {.pure, inheritable.} + TObj {.pure, inheritable.} = object TObjB = object of TObj a, b, c: string fn: proc (): int {.tags: [ReadIOEffect].} diff --git a/tests/effects/teffects6.nim b/tests/effects/teffects6.nim index 6a4eea1553..d3af224348 100644 --- a/tests/effects/teffects6.nim +++ b/tests/effects/teffects6.nim @@ -21,7 +21,7 @@ createMenuItem(s, "Go to definition...", ) -proc noRaise(x: proc()) {.raises: [].} = +proc noRaise(x: proc()) {.raises: [], effectsOf: x.} = # unknown call that might raise anything, but valid: x() @@ -34,3 +34,23 @@ proc use*() = use() + +# bug #12642 +import os + +proc raises() {.raises: Exception.} = discard +proc harmless() {.raises: [].} = discard + +let x = if paramStr(1) == "true": harmless else: raises + +let + choice = 0 + +proc withoutSideEffects(): int = 0 +proc withSideEffects(): int = echo "foo" # the echo causes the side effect + +let procPtr = case choice + of 0: withoutSideEffects + else: withSideEffects + +echo procPtr.repr diff --git a/tests/effects/teffects7.nim b/tests/effects/teffects7.nim index 73865b18db..9b7fbf5f01 100644 --- a/tests/effects/teffects7.nim +++ b/tests/effects/teffects7.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: ref ValueError" line: 10 """ - +{.push warningAsError[Effect]: on.} proc foo() {.raises: [].} = try: discard @@ -12,3 +12,5 @@ proc foo() {.raises: [].} = discard foo() + +{.pop.} diff --git a/tests/effects/teffects8.nim b/tests/effects/teffects8.nim index fb3c088d6c..359b3a1df6 100644 --- a/tests/effects/teffects8.nim +++ b/tests/effects/teffects8.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: Exception" line: 10 """ - +{.push warningAsError[Effect]: on.} proc foo() {.raises: [].} = try: discard @@ -10,3 +10,4 @@ proc foo() {.raises: [].} = raise foo() +{.pop.} diff --git a/tests/effects/teffectsmisc.nim b/tests/effects/teffectsmisc.nim new file mode 100644 index 0000000000..8fb95b2759 --- /dev/null +++ b/tests/effects/teffectsmisc.nim @@ -0,0 +1,39 @@ +discard """ + output: ''' +printing from adder +''' +""" + +import std/sugar + +block: + proc makeAdder(a: int): (int) -> void = + proc discard_adder(x: int) {.closure.} = + discard a + x + + proc echo_adder(x: int) {.closure.} = + echo("printing from adder") + + if a > 0: + discard_adder + else: + echo_adder + + let newAdder = makeAdder(0) + newAdder(5) + +block: + proc makeAdder(a: int): (int) -> void = + proc discard_adder(x: int) {.closure.} = + discard a + x + + proc echo_adder(x: int) {.closure.} = + echo("printing from adder") + + if a > 0: + echo_adder + else: + discard_adder + + let newAdder = makeAdder(0) + newAdder(5) diff --git a/tests/effects/tfuncs_cannot_mutate.nim b/tests/effects/tfuncs_cannot_mutate.nim index 4768af2b3c..9934d27a7d 100644 --- a/tests/effects/tfuncs_cannot_mutate.nim +++ b/tests/effects/tfuncs_cannot_mutate.nim @@ -1,9 +1,6 @@ discard """ - errormsg: "'mutate' can have side effects" - nimout: '''an object reachable from 'n' is potentially mutated -tfuncs_cannot_mutate.nim(39, 15) the mutation is here -tfuncs_cannot_mutate.nim(37, 7) is the statement that connected the mutation to the parameter''' - line: 33 + errormsg: "cannot mutate location select(x, z).data within a strict func" + line: 35 """ {.experimental: "strictFuncs".} @@ -25,8 +22,7 @@ func len(n: Node): int = it = it.ri func doNotDistract(n: Node) = - var m = Node() - m.data = "abc" + var m = Node(data: "abc") func select(a, b: Node): Node = b diff --git a/tests/effects/tfuncs_cannot_mutate2.nim b/tests/effects/tfuncs_cannot_mutate2.nim new file mode 100644 index 0000000000..86f8110171 --- /dev/null +++ b/tests/effects/tfuncs_cannot_mutate2.nim @@ -0,0 +1,24 @@ +discard """ + errormsg: "cannot mutate location x[0].a within a strict func" + line: 12 +""" + +{.experimental: "strictFuncs".} + +func copy[T](x: var openArray[T]; y: openArray[T]) = + for i in 0..high(x): + x[i] = y[i] + + x[0].a = nil + +type + R = ref object + a, b: R + data: string + +proc main = + var a, b: array[3, R] + b = [R(data: "a"), R(data: "b"), R(data: "c")] + copy a, b + +main() diff --git a/tests/effects/tfuncs_cannot_mutate3.nim b/tests/effects/tfuncs_cannot_mutate3.nim new file mode 100644 index 0000000000..029152029d --- /dev/null +++ b/tests/effects/tfuncs_cannot_mutate3.nim @@ -0,0 +1,35 @@ +discard """ + errormsg: "cannot mutate location kid.parent within a strict func" + line: 16 +""" + +{.experimental: "strictFuncs".} + +type + Node = ref object + name: string + kids: seq[Node] + parent: Node + +func initParents(tree: Node) = + for kid in tree.kids: + kid.parent = tree + initParents(kid) + +proc process(intro: Node): Node = + var tree = Node(name: "root", kids: @[ + intro, + Node(name: "one", kids: @[ + Node(name: "two"), + Node(name: "three"), + ]), + Node(name: "four"), + ]) + initParents(tree) + +proc main() = + var intro = Node(name: "intro") + var tree = process(intro) + echo intro.parent.name + +main() diff --git a/tests/effects/tfuncs_cannot_mutate_simple.nim b/tests/effects/tfuncs_cannot_mutate_simple.nim index 9de20d1ec1..0ae4a0db91 100644 --- a/tests/effects/tfuncs_cannot_mutate_simple.nim +++ b/tests/effects/tfuncs_cannot_mutate_simple.nim @@ -1,8 +1,6 @@ discard """ - errormsg: "'edit' can have side effects" - nimout: '''an object reachable from 'x' is potentially mutated -tfuncs_cannot_mutate_simple.nim(17, 4) the mutation is here''' - line: 16 + errormsg: '''cannot mutate location x.data within a strict func''' + line: 15 """ {.experimental: "strictFuncs".} diff --git a/tests/effects/tgcsafe.nim b/tests/effects/tgcsafe.nim index 363624f195..cfac3ddd81 100644 --- a/tests/effects/tgcsafe.nim +++ b/tests/effects/tgcsafe.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "'mainUnsafe' is not GC-safe" + errormsg: "'mainUnsafe' is not GC-safe as it performs an indirect call here" line: 26 cmd: "nim $target --hints:on --threads:on $options $file" """ @@ -13,7 +13,7 @@ proc myproc(i: int) {.gcsafe.} = if isNil(global_proc): return -proc mymap(x: proc ()) = +proc mymap(x: proc ()) {.effectsOf: x.} = x() var diff --git a/tests/effects/tgcsafe2.nim b/tests/effects/tgcsafe2.nim index 07da4e3f84..6268592a95 100644 --- a/tests/effects/tgcsafe2.nim +++ b/tests/effects/tgcsafe2.nim @@ -1,5 +1,5 @@ discard """ - errormsg: '''type mismatch: got ''' + errormsg: '''type mismatch: got ''' line: 11 """ #5620 diff --git a/tests/effects/tgcsafe3.nim b/tests/effects/tgcsafe3.nim index 5137efe4c5..36ea5112cd 100644 --- a/tests/effects/tgcsafe3.nim +++ b/tests/effects/tgcsafe3.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "'myproc' is not GC-safe as it accesses 'global_proc' which is a global using GC'ed memory" + errormsg: "'myproc' is not GC-safe as it calls 'global_proc'" line: 12 cmd: "nim $target --hints:on --threads:on $options $file" """ diff --git a/tests/effects/thooks.nim b/tests/effects/thooks.nim new file mode 100644 index 0000000000..23cc005cd1 --- /dev/null +++ b/tests/effects/thooks.nim @@ -0,0 +1,16 @@ +discard """ + matrix: "--warningAsError:Effect" +""" + +import std/isolation + +# bug #23129 +type + Thing = object + x: string + +proc send(x: string) = + let wrapper = Thing(x: x) + discard isolate(wrapper) + +send("la") \ No newline at end of file diff --git a/tests/effects/tlaxeffects.nim b/tests/effects/tlaxeffects.nim new file mode 100644 index 0000000000..7eedc372a6 --- /dev/null +++ b/tests/effects/tlaxeffects.nim @@ -0,0 +1,11 @@ +discard """ + cmd: "nim $target $options --legacy:laxEffects $file" +""" + + +type + Foo = object + bar: seq[Foo] + +proc `==`(a, b: Foo): bool = + a.bar == b.bar diff --git a/tests/effects/tnestedprocs.nim b/tests/effects/tnestedprocs.nim new file mode 100644 index 0000000000..125896d442 --- /dev/null +++ b/tests/effects/tnestedprocs.nim @@ -0,0 +1,63 @@ +discard """ + cmd: "nim check --hints:off $file" + nimout: '''tnestedprocs.nim(27, 8) Error: 'inner' can have side effects +> tnestedprocs.nim(29, 13) Hint: 'inner' calls `.sideEffect` 'outer2' +>> tnestedprocs.nim(26, 6) Hint: 'outer2' called by 'inner' + +tnestedprocs.nim(45, 8) Error: 'inner' can have side effects +> tnestedprocs.nim(47, 13) Hint: 'inner' calls `.sideEffect` 'outer6' +>> tnestedprocs.nim(44, 6) Hint: 'outer6' called by 'inner' + +tnestedprocs.nim(58, 41) Error: type mismatch: got but expected 'proc (){.closure, noSideEffect.}' + Pragma mismatch: got '{..}', but expected '{.noSideEffect.}'. +''' + errormsg: "type mismatch: got but expected 'proc (){.closure, noSideEffect.}'" +""" +{.experimental: "strictEffects".} +proc outer {.noSideEffect.} = + proc inner(p: int) = + if p == 0: + outer() + + inner(4) + +outer() + +proc outer2 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + outer2() + + inner(4) + +outer2() + +proc outer3(p: int) {.noSideEffect.} = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + p.outer3() + + inner(4) + +outer3(5) + +proc outer6 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + outer6() + + inner(4) + echo "bad" + +outer6() + + +proc outer4 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + let x: proc () {.noSideEffect.} = outer4 + x() + + inner(4) + +outer4() diff --git a/tests/effects/tnosideeffect.nim b/tests/effects/tnosideeffect.nim index 9cabb35a2c..9fc2f74d48 100644 --- a/tests/effects/tnosideeffect.nim +++ b/tests/effects/tnosideeffect.nim @@ -1,5 +1,5 @@ block: # `.noSideEffect` - func foo(bar: proc(): int): int = bar() + func foo(bar: proc(): int): int {.effectsOf: bar.} = bar() var count = 0 proc fn1(): int = 1 proc fn2(): int = (count.inc; count) diff --git a/tests/effects/toutparam.nim b/tests/effects/toutparam.nim deleted file mode 100644 index 00572aa6b1..0000000000 --- a/tests/effects/toutparam.nim +++ /dev/null @@ -1,28 +0,0 @@ -discard """ - cmd: '''nim c --warningAsError[Uninit]:on --skipCfg --skipParentCfg $file''' - errormsg: "use explicit initialization of 'x' for clarity [Uninit]" - line: 24 - disabled: "true" -""" - -proc gah[T](x: out T) = - x = 3 - -proc main = - var a: array[2, int] - var x: int - gah(x) - a[0] = 3 - a[x] = 3 - echo x - -main() - -proc mainB = - var a: array[2, int] - var x: int - a[0] = 3 - a[x] = 3 - echo x - -mainB() diff --git a/tests/effects/tstrict_caseobjects.nim b/tests/effects/tstrict_caseobjects.nim new file mode 100644 index 0000000000..20bc810e0b --- /dev/null +++ b/tests/effects/tstrict_caseobjects.nim @@ -0,0 +1,47 @@ +discard """ + errormsg: "field access outside of valid case branch: x.x" + line: 45 +""" + +{.experimental: "strictCaseObjects".} + +type + NodeKind = enum + nkParent, + nkChild + + Node {.acyclic.} = ref object + case kind: NodeKind + of nkParent: + children: seq[Node] + of nkChild: + name: string + +let list = @[Node(kind: nkParent, children: @[]), Node(kind: nkChild, name: "hello")] +for node in list: + case node.kind + of nkChild: + echo $node.name # here this time there is a warning + else: discard + + +type + Foo = object + case b: bool + of false: + s: string + of true: + x: int + +var x = Foo(b: true, x: 4) +case x.b +of true: + echo x.x +of false: + echo "no" + +case x.b +of false: + echo x.x +of true: + echo "no" diff --git a/tests/effects/tstrict_effects.nim b/tests/effects/tstrict_effects.nim new file mode 100644 index 0000000000..eee8fb71ac --- /dev/null +++ b/tests/effects/tstrict_effects.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "s1 can raise an unlisted exception: CatchableError" + line: 27 +""" + +{.push warningAsError[Effect]: on.} +{.experimental: "strictEffects".} + +# bug #18376 + +{.push raises: [Defect].} +type Call = proc (x: int): int {.gcsafe, raises: [Defect, CatchableError].} + +type Bar* = object + foo*: Call + +proc passOn*(x: Call) = discard + +proc barCal(b: var Bar, s: string, s1: Call) = + #compiler complains that his line can throw CatchableError + passOn s1 + + +proc passOnB*(x: Call) {.effectsOf: x.} = discard + +proc barCal2(b: var Bar, s: string, s1: Call) = + passOnB s1 diff --git a/tests/effects/tstrict_effects2.nim b/tests/effects/tstrict_effects2.nim new file mode 100644 index 0000000000..acc0a0540a --- /dev/null +++ b/tests/effects/tstrict_effects2.nim @@ -0,0 +1,28 @@ +discard """ + errormsg: "can raise an unlisted exception: Exception" + line: 23 +""" + +{.push warningAsError[Effect]: on.} +{.experimental: "strictEffects".} + +# bug #13905 + +proc atoi(v: cstring): cint {.importc: "atoi", cdecl, raises: [].} + +type Conv = proc(v: cstring): cint {.cdecl, raises: [].} + +var x: Conv = atoi + +# bug #17475 + +type + Callback = proc() + +proc f(callback: Callback) {.raises: [].} = + callback() + +proc main = + f(proc () = raise newException(IOError, "IO")) + +main() diff --git a/tests/effects/tstrict_effects3.nim b/tests/effects/tstrict_effects3.nim new file mode 100644 index 0000000000..0d98a0343d --- /dev/null +++ b/tests/effects/tstrict_effects3.nim @@ -0,0 +1,57 @@ +discard """ + action: compile +""" + +{.push warningAsError[Effect]: on.} + +{.experimental: "strictEffects".} + +proc fn(a: int, p1, p2: proc()) {.effectsOf: p1.} = + if a == 7: + p1() + if a<0: + raise newException(ValueError, $a) + +proc main() {.raises: [ValueError].} = + fn(1, proc()=discard, proc() = raise newException(IOError, "foo")) +main() + +# bug #19159 + +import macros + +func mkEnter() = + template helper = + discard + when defined pass: + helper() + else: + let ast = getAst(helper()) + + +# bug #6559 +type + SafeFn = proc (): void {. raises: [] } + +proc ok() {. raises: [] .} = discard +proc fail() {. raises: [] .} + +let f1 : SafeFn = ok +let f2 : SafeFn = fail + + +proc fail() = discard +f1() +f2() + +import std/json + +# bug #22254 +proc senri(a, b: seq[JsonNode]) {.raises: [].} = discard a == b + +# bug #22253 +proc serika() {.raises: [].} = discard default(JsonNode) == nil + +senri(@[newJBool(true)], @[newJBool(false)]) +serika() + diff --git a/tests/effects/tstrict_effects_sort.nim b/tests/effects/tstrict_effects_sort.nim new file mode 100644 index 0000000000..8928ed0d39 --- /dev/null +++ b/tests/effects/tstrict_effects_sort.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "cmpE can raise an unlisted exception: Exception" + line: 27 +""" + +{.push warningAsError[Effect]: on.} + +{.experimental: "strictEffects".} + +import algorithm + +type + MyInt = distinct int + +var toSort = @[MyInt 1, MyInt 2, MyInt 3] + +proc cmpN(a, b: MyInt): int = + cmp(a.int, b.int) + +proc harmless {.raises: [].} = + toSort.sort cmpN + +proc cmpE(a, b: MyInt): int {.raises: [Exception].} = + cmp(a.int, b.int) + +proc harmfull {.raises: [].} = + toSort.sort cmpE diff --git a/tests/effects/tstrict_funcs.nim b/tests/effects/tstrict_funcs.nim index 044bc7ee15..9d20f5d7ee 100644 --- a/tests/effects/tstrict_funcs.nim +++ b/tests/effects/tstrict_funcs.nim @@ -27,3 +27,20 @@ block: var x = @[0, 1] let z = x &&& 2 + + +func copy[T](x: var openArray[T]; y: openArray[T]) = + for i in 0..high(x): + x[i] = y[i] + +type + R = ref object + a, b: R + data: string + +proc main = + var a, b: array[3, R] + b = [R(data: "a"), R(data: "b"), R(data: "c")] + copy a, b + +main() diff --git a/tests/effects/tstrict_funcs_imports.nim b/tests/effects/tstrict_funcs_imports.nim index 4e9b9fe66d..bf68b61b2e 100644 --- a/tests/effects/tstrict_funcs_imports.nim +++ b/tests/effects/tstrict_funcs_imports.nim @@ -7,11 +7,19 @@ discard """ when defined(linux): import linenoise +when defined(nimPreviewSlimSystem): + import std/[ + assertions, + formatfloat, + objectdollar, + syncio, + widestrs, + ] + import algorithm, asyncdispatch, asyncfile, - asyncftpclient, asyncfutures, asynchttpserver, asyncmacro, @@ -31,11 +39,6 @@ import cpuload, critbits, cstrutils, - db_common, - db_mysql, - db_odbc, - db_postgres, - db_sqlite, deques, distros, dynlib, @@ -63,7 +66,6 @@ import macros, marshal, math, - md5, memfiles, mersenne, mimetypes, @@ -84,10 +86,9 @@ import parseutils, parsexml, pathnorm, - # pegs, + pegs, posix_utils, prelude, - punycode, random, rationals, rdstdin, @@ -102,7 +103,6 @@ import sets, sharedlist, sharedtables, - smtp, ssl_certs, ssl_config, stats, @@ -154,7 +154,6 @@ import std/[ monotimes, packedsets, setutils, - sha1, socketstreams, stackframes, sums, diff --git a/tests/effects/tstrict_funcs_imports_js.nim b/tests/effects/tstrict_funcs_imports_js.nim index b7fcd343ab..667887ff0c 100644 --- a/tests/effects/tstrict_funcs_imports_js.nim +++ b/tests/effects/tstrict_funcs_imports_js.nim @@ -7,7 +7,6 @@ discard """ import asyncjs, dom, - dom_extensions, jsconsole, jsffi, jsre diff --git a/tests/effects/tstrictfuncs_misc.nim b/tests/effects/tstrictfuncs_misc.nim new file mode 100644 index 0000000000..8c573bb3a6 --- /dev/null +++ b/tests/effects/tstrictfuncs_misc.nim @@ -0,0 +1,65 @@ +discard """ + action: compile +""" + +{.experimental: "strictFuncs".} + +func sortedFake1[T](a: openArray[T]): seq[T] = + for i in 0 .. a.high: result.add a[i] +func sortedFake2[T](a: openArray[T]): seq[T] = + result = newSeq[T](a.len) + for i in 0 .. a.high: result[i] = a[i] +type Foo1 = object +type Foo2 = ref object +block: + let a1 = sortedFake1([Foo1()]) # ok + let a2 = sortedFake1([Foo2()]) # ok +block: + let a1 = sortedFake2([Foo1()]) # ok + let a2 = sortedFake2([Foo2()]) # error: Error: 'sortedFake2' can have side effects + + +import std/sequtils +type Foob = ref object + x: int +let a1 = zip(@[1,2], @[1,2]) # ok +let a2 = zip(@[Foob(x: 1)], @[Foob(x: 2)]) # error in 1.6.0 RC2, but not 1.4.x + + +# bug #20863 +type + Fooc = ref object + +func twice(foo: Fooc) = + var a = newSeq[Fooc](2) + a[0] = foo # No error. + a[1] = foo # Error: 'twice' can have side effects. + +let foo = Fooc() +twice(foo) + +# bug #17387 +import json + +func parseColumn(columnNode: JsonNode) = + let columnName = columnNode["name"].str + +parseColumn(%*{"a": "b"}) + +type + MyTable = object + data: seq[int] + + JsonNode3 = ref object + fields: MyTable + +proc `[]`(t: MyTable, key: string): int = + result = t.data[0] + +proc `[]`(x: JsonNode3, key: string): int = + result = x.fields[key] + +func parseColumn(columnNode: JsonNode3) = + var columnName = columnNode["test"] + +parseColumn(JsonNode3()) diff --git a/tests/enum/m16462_1.nim b/tests/enum/m16462_1.nim new file mode 100644 index 0000000000..631c63256d --- /dev/null +++ b/tests/enum/m16462_1.nim @@ -0,0 +1,3 @@ +type + Scancode* {.pure.} = enum + SCANCODE_LEFT = 80 \ No newline at end of file diff --git a/tests/enum/m16462_2.nim b/tests/enum/m16462_2.nim new file mode 100644 index 0000000000..631c63256d --- /dev/null +++ b/tests/enum/m16462_2.nim @@ -0,0 +1,3 @@ +type + Scancode* {.pure.} = enum + SCANCODE_LEFT = 80 \ No newline at end of file diff --git a/tests/enum/mcrossmodule.nim b/tests/enum/mcrossmodule.nim new file mode 100644 index 0000000000..e534a202c2 --- /dev/null +++ b/tests/enum/mcrossmodule.nim @@ -0,0 +1,6 @@ + +type + OtherEnum* = enum + Success, Failed, More + +proc some*(x: OtherEnum): bool = x == Success diff --git a/tests/enum/t16462.nim b/tests/enum/t16462.nim new file mode 100644 index 0000000000..9f38286bb5 --- /dev/null +++ b/tests/enum/t16462.nim @@ -0,0 +1,5 @@ +import m16462_1 except Scancode +import m16462_2 + +# bug #16462 +let a = SCANCODE_LEFT \ No newline at end of file diff --git a/tests/enum/t21863.nim b/tests/enum/t21863.nim new file mode 100644 index 0000000000..d0d8b1fcd8 --- /dev/null +++ b/tests/enum/t21863.nim @@ -0,0 +1,28 @@ +discard """ +cmd: "nim check --hints:off $file" +action: reject +nimout: ''' +t21863.nim(28, 16) Error: undeclared field: 'A' + found 'A' [enumField declared in t21863.nim(24, 18)] + found 'A' [enumField declared in t21863.nim(25, 18)] +t21863.nim(28, 16) Error: undeclared field: '.' +t21863.nim(28, 16) Error: undeclared field: '.' +t21863.nim(28, 16) Error: expression '' has no type (or is ambiguous) +''' +""" + + + + + + + + + +block: + type + EnumA = enum A, B + EnumB = enum A + EnumC = enum C + + discard EnumC.A diff --git a/tests/enum/tambiguousoverloads.nim b/tests/enum/tambiguousoverloads.nim new file mode 100644 index 0000000000..12c78c848c --- /dev/null +++ b/tests/enum/tambiguousoverloads.nim @@ -0,0 +1,26 @@ +discard """ +cmd: "nim check --hints:off $file" +""" + +block: # bug #21887 + type + EnumA = enum A = 300, B + EnumB = enum A = 10 + EnumC = enum C + + doAssert typeof(EnumC(A)) is EnumC #[tt.Error + ^ ambiguous identifier: 'A' -- use one of the following: + EnumA.A: EnumA + EnumB.A: EnumB]# + +block: # issue #22598 + type + A = enum + red + B = enum + red + + let a = red #[tt.Error + ^ ambiguous identifier: 'red' -- use one of the following: + A.red: A + B.red: B]# diff --git a/tests/enum/tcrossmodule.nim b/tests/enum/tcrossmodule.nim new file mode 100644 index 0000000000..c21072198e --- /dev/null +++ b/tests/enum/tcrossmodule.nim @@ -0,0 +1,15 @@ +import mcrossmodule + +type + MyEnum = enum + Success + +template t = + doAssert some(Success) + +t() + +block: # legacy support for behavior before overloadableEnums + # warning: ambiguous enum field 'Success' assumed to be of type MyEnum + let x = {Success} + doAssert x is set[MyEnum] diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index aa338ee2c0..a03019c5d7 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -6,7 +6,7 @@ ABCDC foo first0second32third64 my value A1my value Bconc2valueCabc4abc -my value A0my value Bconc1valueCabc3valueC +my value A0my value Bconc1valueCabc3abc ''' """ @@ -124,25 +124,25 @@ block tnamedfields: # trick the optimizer with a variable: var x = valueD echo valueA, ord(valueA), valueB, ord(valueB), valueC, valueD, ord(valueD), x + doAssert $x == $valueD, $x + doAssert $x == "abc", $x - -block toptions: +block tfakeOptions: type - # please make sure we have under 32 options (improves code efficiency!) - TOption = enum - optNone, optForceFullMake, optBoehmGC, optRefcGC, optRangeCheck, - optBoundsCheck, optOverflowCheck, optNilCheck, optAssert, optLineDir, - optWarns, optHints, optListCmd, optCompileOnly, - optSafeCode, # only allow safe code - optStyleCheck, optOptimizeSpeed, optOptimizeSize, optGenDynLib, - optGenGuiApp, optStackTrace + TFakeOption = enum + fakeNone, fakeForceFullMake, fakeBoehmGC, fakeRefcGC, fakeRangeCheck, + fakeBoundsCheck, fakeOverflowCheck, fakeNilCheck, fakeAssert, fakeLineDir, + fakeWarns, fakeHints, fakeListCmd, fakeCompileOnly, + fakeSafeCode, # only allow safe code + fakeStyleCheck, fakeOptimizeSpeed, fakeOptimizeSize, fakeGenDynLib, + fakeGenGuiApp, fakeStackTrace - TOptionset = set[TOption] + TFakeOptionset = set[TFakeOption] var - gOptions: TOptionset = {optRefcGC, optRangeCheck, optBoundsCheck, - optOverflowCheck, optAssert, optWarns, optHints, optLineDir, optStackTrace} + gFakeOptions: TFakeOptionset = {fakeRefcGC, fakeRangeCheck, fakeBoundsCheck, + fakeOverflowCheck, fakeAssert, fakeWarns, fakeHints, fakeLineDir, fakeStackTrace} compilerArgs: int gExitcode: int8 @@ -155,11 +155,113 @@ block nonzero: # bug #6959 C let slice = SomeEnum.low..SomeEnum.high -block size_one_byte: #issue 15752 +block size_one_byte: # bug #15752 type Flag = enum Disabled = 0x00 Enabled = 0xFF static: - assert 1 == sizeof(Flag) \ No newline at end of file + assert 1 == sizeof(Flag) + +block: # bug #12589 + when not defined(i386): + type + OGRwkbGeometryType {.size: sizeof(cuint).} = enum + wkbPoint25D = 0x80000001.cuint, wkbLineString25D = 0x80000002, + wkbPolygon25D = 0x80000003 + + proc typ(): OGRwkbGeometryType = + return wkbPoint25D + + when not defined(gcRefc): + doAssert $typ() == "wkbPoint25D" + + block: # bug #21280 + type + Test = enum + B = 19 + A = int64.high() + + doAssert ord(A) == int64.high() + +import std/enumutils +from std/sequtils import toSeq +import std/macros + +block: # unordered enum + block: + type + unordered_enum = enum + a = 1 + b = 0 + + doAssert (ord(a), ord(b)) == (1, 0) + doAssert unordered_enum.toSeq == @[a, b] + + block: + type + unordered_enum = enum + a = 1 + b = 0 + c + + doAssert (ord(a), ord(b), ord(c)) == (1, 0, 2) + + block: + type + unordered_enum = enum + a = 100 + b + c = 50 + d + + doAssert (ord(a), ord(b), ord(c), ord(d)) == (100, 101, 50, 51) + + block: + type + unordered_enum = enum + a = 7 + b = 6 + c = 5 + d + + doAssert (ord(a), ord(b), ord(c), ord(d)) == (7, 6, 5, 8) + doAssert unordered_enum.toSeq == @[a, b, c, d] + + block: + type + unordered_enum = enum + a = 100 + b + c = 500 + d + e + f = 50 + g + h + + doAssert (ord(a), ord(b), ord(c), ord(d), ord(e), ord(f), ord(g), ord(h)) == + (100, 101, 500, 501, 502, 50, 51, 52) + + block: + type + unordered_enum = enum + A + B + C = -1 + D + E + G = -999 + + doAssert (ord(A), ord(B), ord(C), ord(D), ord(E), ord(G)) == + (0, 1, -1, 2, 3, -999) + + block: + type + SomeEnum = enum + seA = 3 + seB = 2 + seC = "foo" + + doAssert (ord(seA), ord(seB), ord(seC)) == (3, 2, 4) diff --git a/tests/enum/tenum_duplicate.nim b/tests/enum/tenum_duplicate.nim new file mode 100644 index 0000000000..4bcad7f6f2 --- /dev/null +++ b/tests/enum/tenum_duplicate.nim @@ -0,0 +1,10 @@ +discard """ + errormsg: "duplicate value in enum 'd'" +""" + +type + unordered_enum = enum + a = 1 + b = 0 + c + d = 2 diff --git a/tests/enum/tenum_invalid.nim b/tests/enum/tenum_invalid.nim new file mode 100644 index 0000000000..8ae0a10574 --- /dev/null +++ b/tests/enum/tenum_invalid.nim @@ -0,0 +1,8 @@ +discard """ +cmd: "nim check $file" +""" + +type + Test = enum + A = 9.0 #[tt.Error + ^ ordinal type expected; given: float]# diff --git a/tests/enum/toverloadable_enums.nim b/tests/enum/toverloadable_enums.nim new file mode 100644 index 0000000000..9bb5514674 --- /dev/null +++ b/tests/enum/toverloadable_enums.nim @@ -0,0 +1,120 @@ +discard """ + output: '''B +0 +E2-B''' +joinable: false +""" + +{.experimental: "overloadableEnums".} + +type + E1 = enum + value1, + value2 + E2 = enum + value1, + value2 = 4 + +const + Lookuptable = [ + E1.value1: "1", + value2: "2" + ] + +when false: + const + Lookuptable: array[E1, string] = [ + value1: "1", + value2: "2" + ] + + +proc p(e: E1): int = + # test that the 'case' statement is smart enough: + case e + of value1: echo "A" + of value2: echo "B" + + +let v = p value2 # ERROR: ambiguous! +# (value2|value2) nkClosedSymChoice -> nkSym + +proc x(p: int) = discard +proc x(p: string) = discard + +proc takeCallback(param: proc(p: int)) = discard + +takeCallback x + +echo ord v + +block: # https://github.com/nim-lang/RFCs/issues/8 + type + Enum1 = enum + A, B, C + Enum2 = enum + A, Z + + proc f(e: Enum1): int = ord(e) + proc g(e: Enum2): int = ord(e) + + proc h(e: Enum1): int = ord(e) + proc h(e: Enum2): int = ord(e) + + let fA = f(A) # Type of A is well defined + let gA = g(A) # Same as above + + let hA1 = h(Enum1.A) # A requires disambiguation + let hA2 = h(Enum2.A) # Similarly + let hA3 = h(B) + let hA4 = h(B) + let x = ord(Enum1.A) # Also + doAssert fA == 0 + doAssert gA == 0 + doAssert hA1 == 0 + doAssert hA2 == 0 + doAssert x == 0 + doAssert hA3 == 1 + doAssert hA4 == 1 + +# bug #18769 +proc g3[T](x: T, e: E2): int = + case e + of value1: echo "E2-A" # Error: type mismatch: got 'E1' for 'value1' but expected 'E2 = enum' + of value2: echo "E2-B" + +let v5 = g3(99, E2.value2) + +block: # only allow enums to overload enums + # mirrors behavior without overloadableEnums + proc foo() = discard + block: + type Foo = enum foo + doAssert foo is Foo + foo() + +import macros +block: # test with macros/templates + type + Enum1 = enum + value01, value02 + Enum2 = enum + value01, value10 + + macro isOneM(a: untyped): bool = + result = newCall(bindSym"==", a, ident"value01") + + macro isOneMS(a: untyped): bool = + result = newCall(bindSym"==", a, bindSym"value01") + + template isOneT(a: untyped): bool = + a == value01 + + let e1 = Enum1.value01 + let e2 = Enum2.value01 + doAssert isOneM(e1) + doAssert isOneM(e2) + doAssert isOneMS(e1) + doAssert isOneMS(e2) + doAssert isOneT(e1) + doAssert isOneT(e2) diff --git a/tests/enum/tpure_enums_conflict.nim b/tests/enum/tpure_enums_conflict.nim index 3c7528a725..4411fd2a61 100644 --- a/tests/enum/tpure_enums_conflict.nim +++ b/tests/enum/tpure_enums_conflict.nim @@ -1,4 +1,5 @@ discard """ + disabled: true # pure enums behave like overloaded enums on ambiguity now which gives a different error message errormsg: "ambiguous identifier: 'amb'" line: 19 """ diff --git a/tests/enum/tredefinition.nim b/tests/enum/tredefinition.nim new file mode 100644 index 0000000000..615ca6b1c4 --- /dev/null +++ b/tests/enum/tredefinition.nim @@ -0,0 +1,9 @@ +discard """ + cmd: '''nim check --hints:off $file''' + action: reject +nimout: ''' +tredefinition.nim(9, 25) Error: redefinition of 'Key_a'; previous declaration here: tredefinition.nim(9, 18) +''' +""" + +type Key* = enum Key_A, Key_a \ No newline at end of file diff --git a/tests/clearmsg/mb.nim b/tests/errmsgs/mb.nim similarity index 100% rename from tests/clearmsg/mb.nim rename to tests/errmsgs/mb.nim diff --git a/tests/clearmsg/mc.nim b/tests/errmsgs/mc.nim similarity index 100% rename from tests/clearmsg/mc.nim rename to tests/errmsgs/mc.nim diff --git a/tests/errmsgs/t10251.nim b/tests/errmsgs/t10251.nim new file mode 100644 index 0000000000..19adf02ebc --- /dev/null +++ b/tests/errmsgs/t10251.nim @@ -0,0 +1,19 @@ +discard """ + action:reject + cmd: "nim check $options $file" + nimout: ''' +t10251.nim(19, 23) Error: redefinition of 'goo1'; previous declaration here: t10251.nim(19, 11) +''' +""" + +# line 10 +type + Enum1 = enum + foo, bar, baz + Enum2 = enum + foo, bar, baz + + +type + Enum3 {.pure.} = enum # fixed (by accident?) in https://github.com/nim-lang/Nim/pull/18263 + goo0, goo1, goo2, goo1 diff --git a/tests/errmsgs/t10376.nim b/tests/errmsgs/t10376.nim index 2ce16d6a22..814c860dc8 100644 --- a/tests/errmsgs/t10376.nim +++ b/tests/errmsgs/t10376.nim @@ -1,6 +1,7 @@ discard """ + matrix: "--mm:refc" errormsg: "finalizer must be a direct reference to a proc" - line: 29 + line: 30 """ type diff --git a/tests/errmsgs/t10489_a.nim b/tests/errmsgs/t10489_a.nim index 71a6cc3c44..c762ce8769 100644 --- a/tests/errmsgs/t10489_a.nim +++ b/tests/errmsgs/t10489_a.nim @@ -1,5 +1,5 @@ discard """ -errormsg: "invalid type: 'macro (body: untyped): untyped{.noSideEffect, gcsafe, locks: 0.}' for let. Did you mean to call the macro with '()'?" +errormsg: "invalid type: 'macro (body: untyped): untyped{.noSideEffect, gcsafe.}' for let. Did you mean to call the macro with '()'?" line: 9 """ diff --git a/tests/errmsgs/t10489_b.nim b/tests/errmsgs/t10489_b.nim index 4b0b876e53..df05f0e6e9 100644 --- a/tests/errmsgs/t10489_b.nim +++ b/tests/errmsgs/t10489_b.nim @@ -1,5 +1,5 @@ discard """ -errormsg: "invalid type: 'macro (body: untyped): untyped{.noSideEffect, gcsafe, locks: 0.}' for const. Did you mean to call the macro with '()'?" +errormsg: "invalid type: 'macro (body: untyped): untyped{.noSideEffect, gcsafe.}' for const. Did you mean to call the macro with '()'?" line: 9 """ diff --git a/tests/errmsgs/t10542.nim b/tests/errmsgs/t10542.nim new file mode 100644 index 0000000000..b57dbf18ba --- /dev/null +++ b/tests/errmsgs/t10542.nim @@ -0,0 +1,24 @@ +discard """ + matrix: "--hintaserror:ConvFromXtoItselfNotNeeded" +""" + +# bug #10542 + +proc f(args: varargs[string, string], length: int) = + doAssert args.len == length + +# main use case that requires type conversion (no warning here) +f("a", "b", 2) +f("a", 1) + + +proc m(args: varargs[cstring, cstring]) = + doAssert args.len == 2 + +# main use case that requires type conversion (no warning here) +m("a", "b") + +# if an argument already is cstring there's a warning +let x: cstring = "x" +m("a", x) +m(x, "a") \ No newline at end of file diff --git a/tests/errmsgs/t10594.nim b/tests/errmsgs/t10594.nim index c9506c5425..d27dd64332 100644 --- a/tests/errmsgs/t10594.nim +++ b/tests/errmsgs/t10594.nim @@ -3,5 +3,5 @@ discard """ line: 7 """ -template foo(v: varargs[int]) = unsafeAddr v +template foo(v: varargs[int]) = addr v foo(1, 2) diff --git a/tests/errmsgs/t10735.nim b/tests/errmsgs/t10735.nim index 307acac2d4..f480d35ac8 100644 --- a/tests/errmsgs/t10735.nim +++ b/tests/errmsgs/t10735.nim @@ -1,10 +1,12 @@ discard """ cmd: "nim check $file" - errormsg: "selector must be of an ordinal type, float or string" + errormsg: "illformed AST: case buf[pos]" nimout: ''' -t10735.nim(38, 5) Error: 'let' symbol requires an initialization -t10735.nim(39, 10) Error: undeclared identifier: 'pos' -t10735.nim(39, 9) Error: type mismatch: got +t10735.nim(43, 5) Error: 'let' symbol requires an initialization +t10735.nim(44, 10) Error: undeclared identifier: 'pos' +t10735.nim(44, 10) Error: expression 'pos' has no type (or is ambiguous) +t10735.nim(44, 10) Error: expression 'pos' has no type (or is ambiguous) +t10735.nim(44, 9) Error: type mismatch: got but expected one of: proc `[]`(s: string; i: BackwardsIndex): char first type mismatch at position: 0 @@ -26,11 +28,14 @@ proc `[]`[T](s: openArray[T]; i: BackwardsIndex): T first type mismatch at position: 0 proc `[]`[T](s: var openArray[T]; i: BackwardsIndex): var T first type mismatch at position: 0 +template `[]`(a: WideCStringObj; idx: int): Utf16Char + first type mismatch at position: 0 template `[]`(s: string; i: int): char first type mismatch at position: 0 expression: `[]`(buf, pos) -t10735.nim(39, 9) Error: selector must be of an ordinal type, float or string +t10735.nim(44, 9) Error: expression '' has no type (or is ambiguous) +t10735.nim(46, 3) Error: illformed AST: case buf[pos] ''' joinable: false """ diff --git a/tests/errmsgs/t14444.nim b/tests/errmsgs/t14444.nim new file mode 100644 index 0000000000..27365236ed --- /dev/null +++ b/tests/errmsgs/t14444.nim @@ -0,0 +1,14 @@ +discard """ + matrix: "--hints:off" + exitcode: "1" + output: ''' +t14444.nim(13) t14444 +fatal.nim(53) sysFatal +Error: unhandled exception: index out of bounds, the container is empty [IndexDefect] +''' +""" + +when true: # bug #14444 + var i: string + i[10] = 'j' + echo i \ No newline at end of file diff --git a/tests/errmsgs/t16654.nim b/tests/errmsgs/t16654.nim new file mode 100644 index 0000000000..b2b57619bd --- /dev/null +++ b/tests/errmsgs/t16654.nim @@ -0,0 +1,12 @@ +discard """ + cmd: "nim check $options $file" + errormsg: "type mismatch: got " +""" + +when true: # bug #16654 + func fn[T](a: T, op: proc(a: T): float) = discard + proc main() = + let v = 1 + proc bar(r: auto): auto = v + fn(1, bar) + main() diff --git a/tests/errmsgs/t17460.nim b/tests/errmsgs/t17460.nim new file mode 100644 index 0000000000..bb8e21198f --- /dev/null +++ b/tests/errmsgs/t17460.nim @@ -0,0 +1,19 @@ +discard """ + cmd: "nim check $options $file" + errormsg: "tuple expected for tuple unpacking, but got 'array[0..2, int]'" +""" + +iterator xclusters*[T](a: openArray[T]; s: static[int]): array[s, T] {.inline.} = + var result: array[s, T] # iterators have no default result variable + var i = 0 + while i < len(a): + for j, x in mpairs(result): + x = a[(i + j) mod len(a)] + yield result + inc(i) + +proc m = + for (i, j, k) in xclusters([1, 2, 3, 4, 5], 3): + echo i, j, k + +m() diff --git a/tests/errmsgs/t18327.nim b/tests/errmsgs/t18327.nim new file mode 100644 index 0000000000..686a1bd0cd --- /dev/null +++ b/tests/errmsgs/t18327.nim @@ -0,0 +1,5 @@ +discard """ + errormsg: "parameter 'n' requires a type" +""" + +proc fn3(n) = discard \ No newline at end of file diff --git a/tests/errmsgs/t18886.nim b/tests/errmsgs/t18886.nim new file mode 100644 index 0000000000..8ed160c645 --- /dev/null +++ b/tests/errmsgs/t18886.nim @@ -0,0 +1,18 @@ +discard """ + cmd: "nim check --hints:off $file" + errormsg: "" + nimout: ''' +t18886.nim(18, 24) Error: ambiguous identifier: 'bar' -- you need a helper proc to disambiguate the following: + t18886.bar: proc (i: ptr int){.noSideEffect, gcsafe.} + t18886.bar: proc (i: ptr char){.noSideEffect, gcsafe.} +''' +""" + +type Foo = (proc(_: pointer): void) + + +proc bar(i: ptr[int]) = discard +proc bar(i: ptr[char]) = discard + + +let fooBar = cast[Foo](bar) \ No newline at end of file diff --git a/tests/errmsgs/t18983.nim b/tests/errmsgs/t18983.nim new file mode 100644 index 0000000000..3451875cb4 --- /dev/null +++ b/tests/errmsgs/t18983.nim @@ -0,0 +1,7 @@ +discard """ + errormsg: "illegal recursion in type 'A'" +""" + +type + A* = A + B = (A,) diff --git a/tests/errmsgs/t19224.nim b/tests/errmsgs/t19224.nim new file mode 100644 index 0000000000..7a9ecb2e70 --- /dev/null +++ b/tests/errmsgs/t19224.nim @@ -0,0 +1,12 @@ +discard """ +cmd: "nim check --hints:off $file" +errormsg: "" +nimout: ''' +t19224.nim(10, 10) Error: cannot infer element type of items([]) +t19224.nim(12, 10) Error: cannot infer element type of items(@[]) +''' +""" + +for _ in []: discard + +for _ in @[]: discard diff --git a/tests/errmsgs/t19882.nim b/tests/errmsgs/t19882.nim new file mode 100644 index 0000000000..1f2f95ab73 --- /dev/null +++ b/tests/errmsgs/t19882.nim @@ -0,0 +1,10 @@ + +discard """ + errormsg: "cannot instantiate 'A[T, P]' inside of type definition: 'init'; Maybe generic arguments are missing?" +""" +type A[T,P] = object + b:T + c:P +proc init(): ref A = + new(result) +var a = init() diff --git a/tests/errmsgs/t19882_2.nim b/tests/errmsgs/t19882_2.nim new file mode 100644 index 0000000000..7f3055a5da --- /dev/null +++ b/tests/errmsgs/t19882_2.nim @@ -0,0 +1,5 @@ +discard """ + errormsg: "cannot instantiate: 'A[T]'; the object's generic parameters cannot be inferred and must be explicitly given" +""" +type A[T] = object +var a = A() \ No newline at end of file diff --git a/tests/errmsgs/t21257.nim b/tests/errmsgs/t21257.nim new file mode 100644 index 0000000000..eecdb1dfe6 --- /dev/null +++ b/tests/errmsgs/t21257.nim @@ -0,0 +1,20 @@ +discard """ + action: compile + cmd: "nim check $file" +""" + +type AC_WINCTRL_Fields* = distinct uint8 + +type AC_STATUSA_WSTATE0* {.pure.} = enum + ABOVE = 0x0, + INSIDE = 0x1, + BELOW = 0x2, + +type AC_WINCTRL_WINTSEL0* {.pure.} = enum + ABOVE = 0x0, + INSIDE = 0x1, + BELOW = 0x2, + OUTSIDE = 0x3, + +proc write*(WINTSEL0: AC_WINCTRL_WINTSEL0 = ABOVE) = + discard diff --git a/tests/errmsgs/t22097.nim b/tests/errmsgs/t22097.nim new file mode 100644 index 0000000000..b50db08a39 --- /dev/null +++ b/tests/errmsgs/t22097.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "for a 'var' type a variable needs to be passed; but 'uint16(x)' is immutable" +""" + +proc toUInt16(x: var uint16) = + discard + +var x = uint8(1) +toUInt16 x \ No newline at end of file diff --git a/tests/errmsgs/t22284.nim b/tests/errmsgs/t22284.nim new file mode 100644 index 0000000000..827155e6ba --- /dev/null +++ b/tests/errmsgs/t22284.nim @@ -0,0 +1,25 @@ +discard """ + errormsg: "j(uRef, proc (config: F; sources: auto) {.raises: [].} = discard ) can raise an unlisted exception: Exception" +""" + +import std/macros + +macro h(): untyped = + result = newTree(nnkStmtList) + result.add quote do: + new int + +type F = object + +proc j[SecondarySources]( + uRef: ref SecondarySources, + u: proc (config: F, sources: ref SecondarySources)): F = + u(result, uRef) + +template programMain(body: untyped) = + proc main {.raises: [].} = body # doesn't SIGSEGV without this {.raises: [].} + main() + +programMain: + var uRef = h() + discard j(uRef, u = proc(config: F, sources: auto) {.raises: [].} = discard) \ No newline at end of file diff --git a/tests/errmsgs/t22753.nim b/tests/errmsgs/t22753.nim new file mode 100644 index 0000000000..af6a871f13 --- /dev/null +++ b/tests/errmsgs/t22753.nim @@ -0,0 +1,35 @@ +discard """ +cmd: "nim check --hints:off $file" +errormsg: "type mismatch" +nimoutFull: true +nimout: ''' +t22753.nim(34, 13) Error: array expects two type parameters +t22753.nim(35, 1) Error: expression 'x' has no type (or is ambiguous) +t22753.nim(35, 1) Error: expression 'x' has no type (or is ambiguous) +t22753.nim(35, 2) Error: type mismatch: got <> +but expected one of: +proc `[]=`(s: var string; i: BackwardsIndex; x: char) + first type mismatch at position: 0 +proc `[]=`[I: Ordinal; T, S](a: T; i: I; x: sink S) + first type mismatch at position: 0 +proc `[]=`[Idx, T; U, V: Ordinal](a: var array[Idx, T]; x: HSlice[U, V]; + b: openArray[T]) + first type mismatch at position: 0 +proc `[]=`[Idx, T](a: var array[Idx, T]; i: BackwardsIndex; x: T) + first type mismatch at position: 0 +proc `[]=`[T, U: Ordinal](s: var string; x: HSlice[T, U]; b: string) + first type mismatch at position: 0 +proc `[]=`[T; U, V: Ordinal](s: var seq[T]; x: HSlice[U, V]; b: openArray[T]) + first type mismatch at position: 0 +proc `[]=`[T](s: var openArray[T]; i: BackwardsIndex; x: T) + first type mismatch at position: 0 +template `[]=`(a: WideCStringObj; idx: int; val: Utf16Char) + first type mismatch at position: 0 +template `[]=`(s: string; i: int; val: char) + first type mismatch at position: 0 + +expression: `[]=`(x, 0, 9) +''' +""" +var x: array[3] # bug #22753 +x[0] = 9 \ No newline at end of file diff --git a/tests/errmsgs/t22852.nim b/tests/errmsgs/t22852.nim new file mode 100644 index 0000000000..7c352a49c9 --- /dev/null +++ b/tests/errmsgs/t22852.nim @@ -0,0 +1,9 @@ +discard """ + exitcode: 1 + outputsub: ''' +Error: unhandled exception: value out of range: -2 notin 0 .. 9223372036854775807 [RangeDefect] +''' +""" + +# bug #22852 +echo [0][2..^2] diff --git a/tests/errmsgs/t22996.nim b/tests/errmsgs/t22996.nim new file mode 100644 index 0000000000..3a51fd8b0f --- /dev/null +++ b/tests/errmsgs/t22996.nim @@ -0,0 +1,7 @@ +discard """ + errormsg: "invalid type: 'typedesc[string]' for const" +""" + +# bug #22996 +type MyObject = ref object + _ = string diff --git a/tests/errmsgs/t23060.nim b/tests/errmsgs/t23060.nim new file mode 100644 index 0000000000..abb79bcc32 --- /dev/null +++ b/tests/errmsgs/t23060.nim @@ -0,0 +1,5 @@ +discard """ + errormsg: "undeclared identifier: '♔♕♖♗♘♙'" +""" + +♔♕♖♗♘♙ = 1 \ No newline at end of file diff --git a/tests/errmsgs/t23419.nim b/tests/errmsgs/t23419.nim new file mode 100644 index 0000000000..59a72f081c --- /dev/null +++ b/tests/errmsgs/t23419.nim @@ -0,0 +1,5 @@ +discard """ + errormsg: "invalid type: 'void' in this context: '(array[0..-1, void],)' for var" +""" + +var a: (array[0, void], ) diff --git a/tests/errmsgs/t23435.nim b/tests/errmsgs/t23435.nim new file mode 100644 index 0000000000..5e2e4c82a7 --- /dev/null +++ b/tests/errmsgs/t23435.nim @@ -0,0 +1,12 @@ +discard """ + outputsub: "Error: unhandled exception: value out of range: -15 notin 0 .. 9223372036854775807 [RangeDefect]" + exitcode: "1" +""" + +# bug #23435 +proc foo() = + for _ in @[1, 3, 5]: + discard "abcde"[25..<10] + break + +foo() diff --git a/tests/errmsgs/t23536.nim b/tests/errmsgs/t23536.nim new file mode 100644 index 0000000000..610a85babd --- /dev/null +++ b/tests/errmsgs/t23536.nim @@ -0,0 +1,26 @@ +discard """ + matrix: "--stackTrace:on --excessiveStackTrace:off" +""" + +const expected = """ +wrong trace: +t23536.nim(22) t23536 +t23536.nim(17) foo +assertions.nim(41) failedAssertImpl +assertions.nim(36) raiseAssert +fatal.nim(53) sysFatal +""" + + +try: + proc foo = # bug #23536 + doAssert false + + for i in 0 .. 1: + + + foo() +except AssertionDefect: + let e = getCurrentException() + let trace = e.getStackTrace + doAssert "wrong trace:\n" & trace == expected diff --git a/tests/errmsgs/t2614.nim b/tests/errmsgs/t2614.nim new file mode 100644 index 0000000000..031ecb9d1f --- /dev/null +++ b/tests/errmsgs/t2614.nim @@ -0,0 +1,21 @@ +discard """ + cmd: "nim check $options --hints:off $file" + errormsg: "" + nimout: ''' +t2614.nim(19, 27) Error: type mismatch: got but expected 'array[0..1, proc (){.closure.}]' + Calling convention mismatch: got '{.nimcall.}', but expected '{.closure.}'. +t2614.nim(21, 22) Error: type mismatch: got but expected 'seq[proc (){.closure.}]' + Calling convention mismatch: got '{.nimcall.}', but expected '{.closure.}'. +''' +""" + +proc g +proc f = + if false: g() +proc g = + if false: f() + +var a = [f, g] # This works +var b: array[2, proc()] = [f, g] # Error + +var c: seq[proc()] = @[f, g] \ No newline at end of file diff --git a/tests/errmsgs/t5167_4.nim b/tests/errmsgs/t5167_4.nim index 7a263622bc..dafd7754d9 100644 --- a/tests/errmsgs/t5167_4.nim +++ b/tests/errmsgs/t5167_4.nim @@ -1,5 +1,5 @@ discard """ -errormsg: "type mismatch: got " +errormsg: "type mismatch: got " line: 19 """ diff --git a/tests/errmsgs/t5167_5.nim b/tests/errmsgs/t5167_5.nim index a9e2608451..dea7e40b30 100644 --- a/tests/errmsgs/t5167_5.nim +++ b/tests/errmsgs/t5167_5.nim @@ -1,13 +1,9 @@ discard """ -cmd: "nim check $file" -errormsg: "'t' has unspecified generic parameters" -nimout: ''' -t5167_5.nim(10, 16) Error: expression 'system' has no type (or is ambiguous) -t5167_5.nim(21, 9) Error: 't' has unspecified generic parameters -''' +cmd: "nim check --mm:refc $file" """ # issue #11942 -discard newSeq[system]() +discard newSeq[system]() #[tt.Error + ^ expression 'system' has no type (or is ambiguous)]# # issue #5167 template t[B]() = @@ -18,8 +14,12 @@ macro m[T]: untyped = nil proc bar(x: proc (x: int)) = echo "bar" -let x = t -bar t +let x = t #[tt.Error + ^ 't' has unspecified generic parameters]# +bar t #[tt.Error +^ type mismatch: got