mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-06-07 02:54:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for backslash \ to consume a newline

Browse Source
This commit is contained in:
gingerBill
2021-03-14 12:53:57 +00:00
parent 8f6439fa6b
commit 8cc4cba06c
2 changed files with 41 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/tokenizer.cpp
View File

@@ -224,6 +224,13 @@ bool operator<=(TokenPos const &a, TokenPos const &b) { return token_pos_cmp(a,
bool operator> (TokenPos const &a, TokenPos const &b) { return token_pos_cmp(a, b) > 0; }
bool operator>=(TokenPos const &a, TokenPos const &b) { return token_pos_cmp(a, b) >= 0; }
TokenPos token_pos_add_column(TokenPos pos) {
pos.column += 1;
pos.offset += 1;
return pos;
}
struct Token {
TokenKind kind;
String string;
@@ -242,6 +249,10 @@ Token make_token_ident(char const *s) {
return t;
}
bool token_is_newline(Token const &tok) {
return tok.kind == Token_Semicolon && tok.string == "\n";
}
struct ErrorCollector {
TokenPos prev;
@@ -653,6 +664,22 @@ void tokenizer_err(Tokenizer *t, char const *msg, ...) {
t->error_count++;
}
void tokenizer_err(Tokenizer *t, TokenPos const &pos, char const *msg, ...) {
va_list va;
isize column = t->read_curr - t->line+1;
if (column < 1) {
column = 1;
}
Token token = {};
token.pos = pos;
va_start(va, msg);
syntax_error_va(token, msg, va);
va_end(va);
t->error_count++;
}
void advance_to_next_rune(Tokenizer *t) {
if (t->read_curr < t->end) {
Rune rune;
@@ -958,7 +985,7 @@ bool scan_escape(Tokenizer *t) {
}
void tokenizer_get_token(Tokenizer *t, Token *token) {
void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
// Skip whitespace
for (;;) {
switch (t->curr_rune) {
@@ -984,6 +1011,8 @@ void tokenizer_get_token(Tokenizer *t, Token *token) {
token->pos.offset = cast(i32)(t->curr - t->start);
token->pos.column = cast(i32)(t->curr - t->line + 1);
TokenPos current_pos = token->pos;
bool insert_semicolon = false;
Rune curr_rune = t->curr_rune;
@@ -1050,6 +1079,17 @@ void tokenizer_get_token(Tokenizer *t, Token *token) {
token->kind = Token_Semicolon;
return;
case '\\':
if (t->flags & TokenizerFlag_InsertSemicolon) {
t->insert_semicolon = false;
}
tokenizer_get_token(t, token);
if (token->pos.line == current_pos.line) {
tokenizer_err(t, token_pos_add_column(current_pos), "Expected a newline after \\");
}
// NOTE(bill): tokenizer_get_token has been called already, return early
return;
case '\'': // Rune Literal
{
insert_semicolon = true;
@@ -1197,7 +1237,6 @@ void tokenizer_get_token(Tokenizer *t, Token *token) {
insert_semicolon = true;
token->kind = Token_CloseBrace;
break;
case '\\': token->kind = Token_BackSlash; break;
case '%':
token->kind = Token_Mod;