diff --git a/doc/atlas.md b/doc/atlas.md index 2a4171a8be..c6b8c7a9a4 100644 --- a/doc/atlas.md +++ b/doc/atlas.md @@ -23,10 +23,12 @@ to create a workspace out of the current working directory. Projects plus their dependencies are stored in a workspace: +``` $workspace / main project $workspace / other project $workspace / _deps / dependency A $workspace / _deps / dependency B +``` The deps directory can be set via `--deps:DIR` during `atlas init`. @@ -134,3 +136,38 @@ if there are no uncommitted changes. ### Others Run `atlas --help` for more features. + + +### Overrides + +You can override how Atlas resolves a package name or a URL. The overrides use +a simple pattern matching language and are flexible enough to integrate private +gitlab repositories. + +To setup an override file, edit the `$workspace/atlas.workspace` file to contain +a line like `overrides="urls.rules"`. Then create a file `urls.rules` that can +contain lines like: + +``` +customProject -> https://gitlab.company.com/customProject +https://github.com/araq/ormin -> https://github.com/useMyForkInstead/ormin +``` + +The `$` has a special meaning in a pattern: + +================= ======================================================== +``$$`` Matches a single dollar sign. +``$*`` Matches until the token following the ``$*`` was found. + The match is allowed to be of 0 length. +``$+`` Matches until the token following the ``$+`` was found. + The match must consist of at least one char. +``$s`` Skips optional whitespace. +================= ======================================================== + +For example, here is how to override any github link: + +``` +https://github.com/$+ -> https://utopia.forall/$# +``` + +You can use `$1` or `$#` to refer to captures. diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim index a8b7371030..3b84f4ce5a 100644 --- a/tools/atlas/atlas.nim +++ b/tools/atlas/atlas.nim @@ -11,7 +11,7 @@ import std / [parseopt, strutils, os, osproc, tables, sets, json, jsonutils, parsecfg, streams, terminal] -import parse_requires, osutils, packagesjson +import parse_requires, osutils, packagesjson, compiledpatterns from unicode import nil @@ -57,6 +57,7 @@ Options: --workspace=DIR use DIR as workspace --genlock generate a lock file (use with `clone` and `update`) --uselock use the lock file for the build + --autoinit auto initialize a workspace --colors=on|off turn on|off colored output --version show the version --help show this help @@ -105,8 +106,10 @@ type hasPackageList: bool keepCommits: bool cfgHere: bool + usesOverrides: bool p: Table[string, string] # name -> url mapping errors, warnings: int + overrides: Patterns lockOption: LockOption lockFileToWrite: seq[LockFileEntry] lockFileToUse: Table[string, LockFileEntry] @@ -382,14 +385,27 @@ proc fillPackageLookupTable(c: var AtlasContext) = proc toUrl(c: var AtlasContext; p: string): string = if p.isUrl: + if c.usesOverrides: + result = c.overrides.substitute(p) + if result.len > 0: return result result = p else: + # either the project name or the URL can be overwritten! + if c.usesOverrides: + result = c.overrides.substitute(p) + if result.len > 0: return result + fillPackageLookupTable(c) result = c.p.getOrDefault(unicode.toLower p) - if result.len == 0: - result = getUrlFromGithub(p) + if result.len == 0: - inc c.errors + result = getUrlFromGithub(p) + if result.len == 0: + inc c.errors + + if c.usesOverrides: + let newUrl = c.overrides.substitute(result) + if newUrl.len > 0: return newUrl proc toName(p: string): PackageName = if p.isUrl: @@ -760,17 +776,42 @@ proc absoluteDepsDir(workspace, value: string): string = else: result = workspace / value -when MockupRun: - proc autoWorkspace(): string = - result = getCurrentDir() - while result.len > 0 and dirExists(result / ".git"): - result = result.parentDir() +proc autoWorkspace(): string = + result = getCurrentDir() + while result.len > 0 and dirExists(result / ".git"): + result = result.parentDir() proc createWorkspaceIn(workspace, depsDir: string) = if not fileExists(workspace / AtlasWorkspace): writeFile workspace / AtlasWorkspace, "deps=\"$#\"" % escape(depsDir, "", "") createDir absoluteDepsDir(workspace, depsDir) +proc parseOverridesFile(c: var AtlasContext; filename: string) = + const Separator = " -> " + let path = c.workspace / filename + var f: File + if open(f, path): + c.usesOverrides = true + try: + var lineCount = 1 + for line in lines(path): + let splitPos = line.find(Separator) + if splitPos >= 0 and line[0] != '#': + let key = line.substr(0, splitPos-1) + let val = line.substr(splitPos+len(Separator)) + if key.len == 0 or val.len == 0: + error c, toName(path), "key/value must not be empty" + let err = c.overrides.addPattern(key, val) + if err.len > 0: + error c, toName(path), "(" & $lineCount & "): " & err + else: + discard "ignore the line" + inc lineCount + finally: + close f + else: + error c, toName(path), "cannot open: " & path + proc readConfig(c: var AtlasContext) = let configFile = c.workspace / AtlasWorkspace var f = newFileStream(configFile, fmRead) @@ -789,6 +830,8 @@ proc readConfig(c: var AtlasContext) = case e.key.normalize of "deps": c.depsDir = absoluteDepsDir(c.workspace, e.value) + of "overrides": + parseOverridesFile(c, e.value) else: warn c, toName(configFile), "ignored unknown setting: " & e.key of cfgOption: @@ -813,7 +856,7 @@ proc main = fatal action & " command must be executed in a project, not in the workspace" var c = AtlasContext(projectDir: getCurrentDir(), workspace: "") - + var autoinit = false for kind, key, val in getopt(): case kind of cmdArgument: @@ -842,6 +885,7 @@ proc main = else: writeHelp() of "cfghere": c.cfgHere = true + of "autoinit": autoinit = true of "genlock": if c.lockOption != useLock: c.lockOption = genLock @@ -870,6 +914,9 @@ proc main = if c.workspace.len > 0: readConfig c info c, toName(c.workspace), "is the current workspace" + elif autoinit: + c.workspace = autoWorkspace() + createWorkspaceIn c.workspace, c.depsDir elif action notin ["search", "list"]: fatal "No workspace found. Run `atlas init` if you want this current directory to be your workspace." diff --git a/tools/atlas/compiledpatterns.nim b/tools/atlas/compiledpatterns.nim new file mode 100644 index 0000000000..69751d82bf --- /dev/null +++ b/tools/atlas/compiledpatterns.nim @@ -0,0 +1,246 @@ +# +# Atlas Package Cloner +# (c) Copyright 2021 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ + +Syntax taken from strscans.nim: + +================= ======================================================== +``$$`` Matches a single dollar sign. +``$*`` Matches until the token following the ``$*`` was found. + The match is allowed to be of 0 length. +``$+`` Matches until the token following the ``$+`` was found. + The match must consist of at least one char. +``$s`` Skips optional whitespace. +================= ======================================================== + +]## + +import tables +from strutils import continuesWith, Whitespace + +type + Opcode = enum + MatchVerbatim # needs verbatim match + Capture0Until + Capture1Until + Capture0UntilEnd + Capture1UntilEnd + SkipWhitespace + + Instr = object + opc: Opcode + arg1: uint8 + arg2: uint16 + + Pattern* = object + code: seq[Instr] + usedMatches: int + error: string + +# A rewrite rule looks like: +# +# foo$*bar -> https://gitlab.cross.de/$1 + +proc compile*(pattern: string; strings: var seq[string]): Pattern = + proc parseSuffix(s: string; start: int): int = + result = start + while result < s.len and s[result] != '$': + inc result + + result = Pattern(code: @[], usedMatches: 0, error: "") + var p = 0 + while p < pattern.len: + if pattern[p] == '$' and p+1 < pattern.len: + case pattern[p+1] + of '$': + if result.code.len > 0 and result.code[^1].opc in { + MatchVerbatim, Capture0Until, Capture1Until, Capture0UntilEnd, Capture1UntilEnd}: + # merge with previous opcode + let key = strings[result.code[^1].arg2] & "$" + var idx = find(strings, key) + if idx < 0: + idx = strings.len + strings.add key + result.code[^1].arg2 = uint16(idx) + else: + var idx = find(strings, "$") + if idx < 0: + idx = strings.len + strings.add "$" + result.code.add Instr(opc: MatchVerbatim, + arg1: uint8(0), arg2: uint16(idx)) + inc p, 2 + of '+', '*': + let isPlus = pattern[p+1] == '+' + + let pEnd = parseSuffix(pattern, p+2) + let suffix = pattern.substr(p+2, pEnd-1) + p = pEnd + if suffix.len == 0: + result.code.add Instr(opc: if isPlus: Capture1UntilEnd else: Capture0UntilEnd, + arg1: uint8(result.usedMatches), arg2: uint16(0)) + else: + var idx = find(strings, suffix) + if idx < 0: + idx = strings.len + strings.add suffix + result.code.add Instr(opc: if isPlus: Capture1Until else: Capture0Until, + arg1: uint8(result.usedMatches), arg2: uint16(idx)) + inc result.usedMatches + + of 's': + result.code.add Instr(opc: SkipWhitespace) + inc p, 2 + else: + result.error = "unknown syntax '$" & pattern[p+1] & "'" + break + elif pattern[p] == '$': + result.error = "unescaped '$'" + break + else: + let pEnd = parseSuffix(pattern, p) + let suffix = pattern.substr(p, pEnd-1) + var idx = find(strings, suffix) + if idx < 0: + idx = strings.len + strings.add suffix + result.code.add Instr(opc: MatchVerbatim, + arg1: uint8(0), arg2: uint16(idx)) + p = pEnd + +type + MatchObj = object + m: int + a: array[20, (int, int)] + +proc matches(s: Pattern; strings: seq[string]; input: string): MatchObj = + template failed = + result.m = -1 + return result + + var i = 0 + for instr in s.code: + case instr.opc + of MatchVerbatim: + if continuesWith(input, strings[instr.arg2], i): + inc i, strings[instr.arg2].len + else: + failed() + of Capture0Until, Capture1Until: + block searchLoop: + let start = i + while i < input.len: + if continuesWith(input, strings[instr.arg2], i): + if instr.opc == Capture1Until and i == start: + failed() + result.a[result.m] = (start, i-1) + inc result.m + inc i, strings[instr.arg2].len + break searchLoop + inc i + failed() + + of Capture0UntilEnd, Capture1UntilEnd: + if instr.opc == Capture1UntilEnd and i >= input.len: + failed() + result.a[result.m] = (i, input.len-1) + inc result.m + i = input.len + of SkipWhitespace: + while i < input.len and input[i] in Whitespace: inc i + if i < input.len: + # still unmatched stuff was left: + failed() + +proc translate(m: MatchObj; outputPattern, input: string): string = + result = newStringOfCap(outputPattern.len) + var i = 0 + var patternCount = 0 + while i < outputPattern.len: + if i+1 < outputPattern.len and outputPattern[i] == '$': + if outputPattern[i+1] == '#': + inc i, 2 + if patternCount < m.a.len: + let (a, b) = m.a[patternCount] + for j in a..b: result.add input[j] + inc patternCount + elif outputPattern[i+1] in {'1'..'9'}: + var n = ord(outputPattern[i+1]) - ord('0') + inc i, 2 + while i < outputPattern.len and outputPattern[i] in {'0'..'9'}: + n = n * 10 + (ord(outputPattern[i]) - ord('0')) + inc i + patternCount = n + if n-1 < m.a.len: + let (a, b) = m.a[n-1] + for j in a..b: result.add input[j] + else: + # just ignore the wrong pattern: + inc i + else: + result.add outputPattern[i] + inc i + +proc replace*(s: Pattern; outputPattern, input: string): string = + var strings: seq[string] = @[] + let m = s.matches(strings, input) + if m.m < 0: + result = "" + else: + result = translate(m, outputPattern, input) + + +type + Patterns* = object + s: seq[(Pattern, string)] + t: Table[string, string] + strings: seq[string] + +proc initPatterns*(): Patterns = + Patterns(s: @[], t: initTable[string, string](), strings: @[]) + +proc addPattern*(p: var Patterns; inputPattern, outputPattern: string): string = + if '$' notin inputPattern and '$' notin outputPattern: + p.t[inputPattern] = outputPattern + result = "" + else: + let code = compile(inputPattern, p.strings) + if code.error.len > 0: + result = code.error + else: + p.s.add (code, outputPattern) + result = "" + +proc substitute*(p: Patterns; input: string): string = + result = p.t.getOrDefault(input) + if result.len == 0: + for i in 0..= 0: + return translate(m, p.s[i][1], input) + +proc replacePattern*(inputPattern, outputPattern, input: string): string = + var strings: seq[string] = @[] + let code = compile(inputPattern, strings) + result = replace(code, outputPattern, input) + +when isMainModule: + # foo$*bar -> https://gitlab.cross.de/$1 + const realInput = "$fooXXbar$z00end" + var strings: seq[string] = @[] + let code = compile("$$foo$*bar$$$*z00$*", strings) + echo code + + let m = code.matches(strings, realInput) + echo m.m + + echo translate(m, "$1--$#-$#-", realInput) + + echo translate(m, "https://gitlab.cross.de/$1", realInput) +