mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-04-18 20:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace line with column_minus_one in Tokenizer; Remove dead code

Browse Source
This commit is contained in:
gingerBill
2021-08-02 00:26:54 +01:00
parent be76da2c90
commit 97be36d18a
1 changed files with 26 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/tokenizer.cpp
View File

@@ -699,15 +699,6 @@ enum TokenizerInitError {
};
struct TokenizerState {
Rune curr_rune; // current character
u8 * curr; // character pos
u8 * read_curr; // pos from start
u8 * line; // current line pos
i32 line_count;
bool insert_semicolon;
};
enum TokenizerFlags {
TokenizerFlag_None = 0,
TokenizerFlag_InsertSemicolon = 1<<0,
@@ -722,7 +713,7 @@ struct Tokenizer {
Rune curr_rune; // current character
u8 * curr; // character pos
u8 * read_curr; // pos from start
u8 * line; // current line pos
i32 column_minus_one;
i32 line_count;
i32 error_count;
@@ -733,30 +724,9 @@ struct Tokenizer {
};
TokenizerState save_tokenizer_state(Tokenizer *t) {
TokenizerState state = {};
state.curr_rune = t->curr_rune;
state.curr = t->curr;
state.read_curr = t->read_curr;
state.line = t->line;
state.line_count = t->line_count;
state.insert_semicolon = t->insert_semicolon;
return state;
}
void restore_tokenizer_state(Tokenizer *t, TokenizerState *state) {
t->curr_rune = state->curr_rune;
t->curr = state->curr;
t->read_curr = state->read_curr;
t->line = state->line;
t->line_count = state->line_count;
t->insert_semicolon = state->insert_semicolon;
}
void tokenizer_err(Tokenizer *t, char const *msg, ...) {
va_list va;
isize column = t->read_curr - t->line+1;
i32 column = t->column_minus_one+1;
if (column < 1) {
column = 1;
}
@@ -775,7 +745,7 @@ void tokenizer_err(Tokenizer *t, char const *msg, ...) {
void tokenizer_err(Tokenizer *t, TokenPos const &pos, char const *msg, ...) {
va_list va;
isize column = t->read_curr - t->line+1;
i32 column = t->column_minus_one+1;
if (column < 1) {
column = 1;
}
@@ -788,13 +758,19 @@ void tokenizer_err(Tokenizer *t, TokenPos const &pos, char const *msg, ...) {
}
void advance_to_next_rune(Tokenizer *t) {
#if 1
if (t->curr_rune == '\n') {
t->column_minus_one = 0;
t->line_count++;
}
#else
// NOTE(bill, 2021-08-02): This is branchless but it is slower in practice
i32 is_newline = t->curr_rune == '\n';
t->column_minus_one *= 1-is_newline;
t->line_count += is_newline;
#endif
if (t->read_curr < t->end) {
t->curr = t->read_curr;
if (t->curr_rune == '\n') {
t->line = t->curr;
t->line_count++;
}
Rune rune = *t->read_curr;
if (rune == 0) {
tokenizer_err(t, "Illegal character NUL");
@@ -813,10 +789,6 @@ void advance_to_next_rune(Tokenizer *t) {
t->curr_rune = rune;
} else {
t->curr = t->end;
if (t->curr_rune == '\n') {
t->line = t->curr;
t->line_count++;
}
t->curr_rune = GB_RUNE_EOF;
}
}
@@ -827,7 +799,7 @@ void init_tokenizer_with_file_contents(Tokenizer *t, String const &fullpath, gbF
t->line_count = 1;
t->start = cast(u8 *)fc->data;
t->line = t->read_curr = t->curr = t->start;
t->read_curr = t->curr = t->start;
t->end = t->start + fc->size;
advance_to_next_rune(t);
@@ -921,7 +893,7 @@ void scan_number_to_token(Tokenizer *t, Token *token, bool seen_decimal_point) {
token->string = {t->curr, 1};
token->pos.file_id = t->curr_file_id;
token->pos.line = t->line_count;
token->pos.column = cast(i32)(t->curr-t->line+1);
token->pos.column = t->column_minus_one+1;
if (seen_decimal_point) {
token->string.text -= 1;
@@ -1113,6 +1085,15 @@ gb_inline void tokenizer_skip_line(Tokenizer *t) {
while (t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) {
advance_to_next_rune(t);
}
#elif 0
while (t->read_curr != t->end && t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) {
t->curr = t->read_curr;
t->curr_rune = *t->read_curr;
if (t->curr_rune == 0) {
tokenizer_err(t, "Illegal character NUL");
}
t->read_curr++;
}
#else
while (t->read_curr != t->end && t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) {
t->curr = t->read_curr;
@@ -1161,7 +1142,7 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
token->pos.file_id = t->curr_file_id;
token->pos.line = t->line_count;
token->pos.offset = cast(i32)(t->curr - t->start);
token->pos.column = cast(i32)(t->curr - t->line + 1);
token->pos.column = t->column_minus_one+1;
TokenPos current_pos = token->pos;