mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-02-12 06:18:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Atlas: URL rewrite rules; --autoinit flag (#21963)

Browse Source
This commit is contained in:
Andreas Rumpf
2023-05-30 14:00:09 +02:00
committed by GitHub
parent 546af8c571
commit 4d20227438
3 changed files with 340 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
doc/atlas.md
View File

@@ -23,10 +23,12 @@ to create a workspace out of the current working directory.
Projects plus their dependencies are stored in a workspace:
```
$workspace / main project
$workspace / other project
$workspace / _deps / dependency A
$workspace / _deps / dependency B
```
The deps directory can be set via `--deps:DIR` during `atlas init`.
@@ -134,3 +136,38 @@ if there are no uncommitted changes.
### Others
Run `atlas --help` for more features.
### Overrides
You can override how Atlas resolves a package name or a URL. The overrides use
a simple pattern matching language and are flexible enough to integrate private
gitlab repositories.
To setup an override file, edit the `$workspace/atlas.workspace` file to contain
a line like `overrides="urls.rules"`. Then create a file `urls.rules` that can
contain lines like:
```
customProject -> https://gitlab.company.com/customProject
https://github.com/araq/ormin -> https://github.com/useMyForkInstead/ormin
```
The `$` has a special meaning in a pattern:
================= ========================================================
``$$`` Matches a single dollar sign.
``$*`` Matches until the token following the ``$*`` was found.
The match is allowed to be of 0 length.
``$+`` Matches until the token following the ``$+`` was found.
The match must consist of at least one char.
``$s`` Skips optional whitespace.
================= ========================================================
For example, here is how to override any github link:
```
https://github.com/$+ -> https://utopia.forall/$#
```
You can use `$1` or `$#` to refer to captures.

67
tools/atlas/atlas.nim
View File

@@ -11,7 +11,7 @@
import std / [parseopt, strutils, os, osproc, tables, sets, json, jsonutils,
parsecfg, streams, terminal]
import parse_requires, osutils, packagesjson
import parse_requires, osutils, packagesjson, compiledpatterns
from unicode import nil
@@ -57,6 +57,7 @@ Options:
--workspace=DIR use DIR as workspace
--genlock generate a lock file (use with `clone` and `update`)
--uselock use the lock file for the build
--autoinit auto initialize a workspace
--colors=on|off turn on|off colored output
--version show the version
--help show this help
@@ -105,8 +106,10 @@ type
hasPackageList: bool
keepCommits: bool
cfgHere: bool
usesOverrides: bool
p: Table[string, string] # name -> url mapping
errors, warnings: int
overrides: Patterns
lockOption: LockOption
lockFileToWrite: seq[LockFileEntry]
lockFileToUse: Table[string, LockFileEntry]
@@ -382,14 +385,27 @@ proc fillPackageLookupTable(c: var AtlasContext) =
proc toUrl(c: var AtlasContext; p: string): string =
if p.isUrl:
if c.usesOverrides:
result = c.overrides.substitute(p)
if result.len > 0: return result
result = p
else:
# either the project name or the URL can be overwritten!
if c.usesOverrides:
result = c.overrides.substitute(p)
if result.len > 0: return result
fillPackageLookupTable(c)
result = c.p.getOrDefault(unicode.toLower p)
if result.len == 0:
result = getUrlFromGithub(p)
if result.len == 0:
inc c.errors
result = getUrlFromGithub(p)
if result.len == 0:
inc c.errors
if c.usesOverrides:
let newUrl = c.overrides.substitute(result)
if newUrl.len > 0: return newUrl
proc toName(p: string): PackageName =
if p.isUrl:
@@ -760,17 +776,42 @@ proc absoluteDepsDir(workspace, value: string): string =
else:
result = workspace / value
when MockupRun:
proc autoWorkspace(): string =
result = getCurrentDir()
while result.len > 0 and dirExists(result / ".git"):
result = result.parentDir()
proc autoWorkspace(): string =
result = getCurrentDir()
while result.len > 0 and dirExists(result / ".git"):
result = result.parentDir()
proc createWorkspaceIn(workspace, depsDir: string) =
if not fileExists(workspace / AtlasWorkspace):
writeFile workspace / AtlasWorkspace, "deps=\"$#\"" % escape(depsDir, "", "")
createDir absoluteDepsDir(workspace, depsDir)
proc parseOverridesFile(c: var AtlasContext; filename: string) =
const Separator = " -> "
let path = c.workspace / filename
var f: File
if open(f, path):
c.usesOverrides = true
try:
var lineCount = 1
for line in lines(path):
let splitPos = line.find(Separator)
if splitPos >= 0 and line[0] != '#':
let key = line.substr(0, splitPos-1)
let val = line.substr(splitPos+len(Separator))
if key.len == 0 or val.len == 0:
error c, toName(path), "key/value must not be empty"
let err = c.overrides.addPattern(key, val)
if err.len > 0:
error c, toName(path), "(" & $lineCount & "): " & err
else:
discard "ignore the line"
inc lineCount
finally:
close f
else:
error c, toName(path), "cannot open: " & path
proc readConfig(c: var AtlasContext) =
let configFile = c.workspace / AtlasWorkspace
var f = newFileStream(configFile, fmRead)
@@ -789,6 +830,8 @@ proc readConfig(c: var AtlasContext) =
case e.key.normalize
of "deps":
c.depsDir = absoluteDepsDir(c.workspace, e.value)
of "overrides":
parseOverridesFile(c, e.value)
else:
warn c, toName(configFile), "ignored unknown setting: " & e.key
of cfgOption:
@@ -813,7 +856,7 @@ proc main =
fatal action & " command must be executed in a project, not in the workspace"
var c = AtlasContext(projectDir: getCurrentDir(), workspace: "")
var autoinit = false
for kind, key, val in getopt():
case kind
of cmdArgument:
@@ -842,6 +885,7 @@ proc main =
else:
writeHelp()
of "cfghere": c.cfgHere = true
of "autoinit": autoinit = true
of "genlock":
if c.lockOption != useLock:
c.lockOption = genLock
@@ -870,6 +914,9 @@ proc main =
if c.workspace.len > 0:
readConfig c
info c, toName(c.workspace), "is the current workspace"
elif autoinit:
c.workspace = autoWorkspace()
createWorkspaceIn c.workspace, c.depsDir
elif action notin ["search", "list"]:
fatal "No workspace found. Run `atlas init` if you want this current directory to be your workspace."

246
tools/atlas/compiledpatterns.nim Normal file
View File

@@ -0,0 +1,246 @@
#
# Atlas Package Cloner
# (c) Copyright 2021 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
##[
Syntax taken from strscans.nim:
================= ========================================================
``$$`` Matches a single dollar sign.
``$*`` Matches until the token following the ``$*`` was found.
The match is allowed to be of 0 length.
``$+`` Matches until the token following the ``$+`` was found.
The match must consist of at least one char.
``$s`` Skips optional whitespace.
================= ========================================================
]##
import tables
from strutils import continuesWith, Whitespace
type
Opcode = enum
MatchVerbatim # needs verbatim match
Capture0Until
Capture1Until
Capture0UntilEnd
Capture1UntilEnd
SkipWhitespace
Instr = object
opc: Opcode
arg1: uint8
arg2: uint16
Pattern* = object
code: seq[Instr]
usedMatches: int
error: string
# A rewrite rule looks like:
#
# foo$*bar -> https://gitlab.cross.de/$1
proc compile*(pattern: string; strings: var seq[string]): Pattern =
proc parseSuffix(s: string; start: int): int =
result = start
while result < s.len and s[result] != '$':
inc result
result = Pattern(code: @[], usedMatches: 0, error: "")
var p = 0
while p < pattern.len:
if pattern[p] == '$' and p+1 < pattern.len:
case pattern[p+1]
of '$':
if result.code.len > 0 and result.code[^1].opc in {
MatchVerbatim, Capture0Until, Capture1Until, Capture0UntilEnd, Capture1UntilEnd}:
# merge with previous opcode
let key = strings[result.code[^1].arg2] & "$"
var idx = find(strings, key)
if idx < 0:
idx = strings.len
strings.add key
result.code[^1].arg2 = uint16(idx)
else:
var idx = find(strings, "$")
if idx < 0:
idx = strings.len
strings.add "$"
result.code.add Instr(opc: MatchVerbatim,
arg1: uint8(0), arg2: uint16(idx))
inc p, 2
of '+', '*':
let isPlus = pattern[p+1] == '+'
let pEnd = parseSuffix(pattern, p+2)
let suffix = pattern.substr(p+2, pEnd-1)
p = pEnd
if suffix.len == 0:
result.code.add Instr(opc: if isPlus: Capture1UntilEnd else: Capture0UntilEnd,
arg1: uint8(result.usedMatches), arg2: uint16(0))
else:
var idx = find(strings, suffix)
if idx < 0:
idx = strings.len
strings.add suffix
result.code.add Instr(opc: if isPlus: Capture1Until else: Capture0Until,
arg1: uint8(result.usedMatches), arg2: uint16(idx))
inc result.usedMatches
of 's':
result.code.add Instr(opc: SkipWhitespace)
inc p, 2
else:
result.error = "unknown syntax '$" & pattern[p+1] & "'"
break
elif pattern[p] == '$':
result.error = "unescaped '$'"
break
else:
let pEnd = parseSuffix(pattern, p)
let suffix = pattern.substr(p, pEnd-1)
var idx = find(strings, suffix)
if idx < 0:
idx = strings.len
strings.add suffix
result.code.add Instr(opc: MatchVerbatim,
arg1: uint8(0), arg2: uint16(idx))
p = pEnd
type
MatchObj = object
m: int
a: array[20, (int, int)]
proc matches(s: Pattern; strings: seq[string]; input: string): MatchObj =
template failed =
result.m = -1
return result
var i = 0
for instr in s.code:
case instr.opc
of MatchVerbatim:
if continuesWith(input, strings[instr.arg2], i):
inc i, strings[instr.arg2].len
else:
failed()
of Capture0Until, Capture1Until:
block searchLoop:
let start = i
while i < input.len:
if continuesWith(input, strings[instr.arg2], i):
if instr.opc == Capture1Until and i == start:
failed()
result.a[result.m] = (start, i-1)
inc result.m
inc i, strings[instr.arg2].len
break searchLoop
inc i
failed()
of Capture0UntilEnd, Capture1UntilEnd:
if instr.opc == Capture1UntilEnd and i >= input.len:
failed()
result.a[result.m] = (i, input.len-1)
inc result.m
i = input.len
of SkipWhitespace:
while i < input.len and input[i] in Whitespace: inc i
if i < input.len:
# still unmatched stuff was left:
failed()
proc translate(m: MatchObj; outputPattern, input: string): string =
result = newStringOfCap(outputPattern.len)
var i = 0
var patternCount = 0
while i < outputPattern.len:
if i+1 < outputPattern.len and outputPattern[i] == '$':
if outputPattern[i+1] == '#':
inc i, 2
if patternCount < m.a.len:
let (a, b) = m.a[patternCount]
for j in a..b: result.add input[j]
inc patternCount
elif outputPattern[i+1] in {'1'..'9'}:
var n = ord(outputPattern[i+1]) - ord('0')
inc i, 2
while i < outputPattern.len and outputPattern[i] in {'0'..'9'}:
n = n * 10 + (ord(outputPattern[i]) - ord('0'))
inc i
patternCount = n
if n-1 < m.a.len:
let (a, b) = m.a[n-1]
for j in a..b: result.add input[j]
else:
# just ignore the wrong pattern:
inc i
else:
result.add outputPattern[i]
inc i
proc replace*(s: Pattern; outputPattern, input: string): string =
var strings: seq[string] = @[]
let m = s.matches(strings, input)
if m.m < 0:
result = ""
else:
result = translate(m, outputPattern, input)
type
Patterns* = object
s: seq[(Pattern, string)]
t: Table[string, string]
strings: seq[string]
proc initPatterns*(): Patterns =
Patterns(s: @[], t: initTable[string, string](), strings: @[])
proc addPattern*(p: var Patterns; inputPattern, outputPattern: string): string =
if '$' notin inputPattern and '$' notin outputPattern:
p.t[inputPattern] = outputPattern
result = ""
else:
let code = compile(inputPattern, p.strings)
if code.error.len > 0:
result = code.error
else:
p.s.add (code, outputPattern)
result = ""
proc substitute*(p: Patterns; input: string): string =
result = p.t.getOrDefault(input)
if result.len == 0:
for i in 0..<p.s.len:
let m = p.s[i][0].matches(p.strings, input)
if m.m >= 0:
return translate(m, p.s[i][1], input)
proc replacePattern*(inputPattern, outputPattern, input: string): string =
var strings: seq[string] = @[]
let code = compile(inputPattern, strings)
result = replace(code, outputPattern, input)
when isMainModule:
# foo$*bar -> https://gitlab.cross.de/$1
const realInput = "$fooXXbar$z00end"
var strings: seq[string] = @[]
let code = compile("$$foo$*bar$$$*z00$*", strings)
echo code
let m = code.matches(strings, realInput)
echo m.m
echo translate(m, "$1--$#-$#-", realInput)
echo translate(m, "https://gitlab.cross.de/$1", realInput)