From 269a1c1feccb0788cd0cc80cc7d266da3b5906e5 Mon Sep 17 00:00:00 2001 From: Zoom Date: Thu, 5 Mar 2026 20:54:19 +0400 Subject: [PATCH] nimdoc: fix char literal tokenization (#25576) This fixes highlighter's tokenization of char literals inside parentheses and brackets. The Nim syntax highlighter in `docutils/highlite.nim` incorrectly tokenizes character literals that appear after punctuation characters, such as all kinds of brackets. For `echo('v', "hello")`, the tokenizer treated the first `'` as punctuation because the preceding token was punctuation `(`. As a result, the second `'` (after `v`) was interpreted as the start of a character literal and the literal incorrectly extended to the end of the line. See other examples in the screenshot: Screenshot 2026-03-04 at 16-09-06 _y_test This regression originates from a condition added in PR #23015 that prevented opening a `gtCharLit` token when the previous token kind was punctuation. Nim syntax allows character literals after punctuation such as `(`, `[`, `{`, `:`, `;`, or `,`, of course. The only case mentioned in the manual explicitly that actually requires special handling is stroped proc declaration for literals (see the [last paragraph here](https://nim-lang.github.io/Nim/manual.html#lexical-analysis-character-literals)): ```nim proc `'customLiteral`(s: string) ``` This PR narrows the conditional to not entering charlit only after backticks. --- lib/packages/docutils/highlite.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 7542b88018..40228287a7 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -326,7 +326,8 @@ proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) = pos = nimNumber(g, pos) of '\'': inc(pos) - if g.kind != gtPunctuation: + let followsBacktick = pos >= 2 and g.buf[pos - 2] == '`' + if not followsBacktick: g.kind = gtCharLit while true: case g.buf[pos] @@ -338,6 +339,8 @@ proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) = of '\\': inc(pos, 2) else: inc(pos) + else: + g.kind = gtPunctuation of '\"': inc(pos) if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'):