mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-01-09 06:23:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Prepare tokenizer for optimizations

Browse Source
This commit is contained in:
gingerBill
2021-08-02 16:47:32 +01:00
parent 7f3d4cb504
commit b0e64ca7e8
2 changed files with 37 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/tokenizer.cpp
View File

@@ -1070,38 +1070,51 @@ bool scan_escape(Tokenizer *t) {
gb_inline void tokenizer_skip_line(Tokenizer *t) {
#if 0
while (t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) {
advance_to_next_rune(t);
}
#elif 0
#if 1
while (t->read_curr != t->end && t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) {
t->curr = t->read_curr;
t->curr_rune = *t->read_curr;
t->column_minus_one++;
t->curr = t->read_curr++;
t->curr_rune = *t->curr;
if (t->curr_rune == 0) {
tokenizer_err(t, "Illegal character NUL");
}
t->read_curr++;
}
#else
while (t->read_curr != t->end && t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) {
t->curr = t->read_curr;
t->curr_rune = *t->read_curr;
if (t->curr_rune == 0) {
tokenizer_err(t, "Illegal character NUL");
loop_start:;
u8 *p = t->read_curr;
for (; p < t->end; p++) {
switch (*p) {
case '\n':
case 0xff:
goto loop_end;
case 0:
goto illegal_nul;
}
t->read_curr++;
}
loop_end:
p = gb_min(p, t->end);
t->column_minus_one += cast(i32)(p - t->read_curr);
t->read_curr = p;
t->curr = p-1;
t->curr_rune = *t->curr;
return;
illegal_nul:;
p = gb_min(p, t->end);
t->column_minus_one += cast(i32)(p - t->read_curr);
t->read_curr = p;
t->curr = p-1;
t->curr_rune = *t->curr;
tokenizer_err(t, "Illegal character NUL");
goto loop_start;
#endif
}
void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
// Skip whitespace
if (t->flags & TokenizerFlag_InsertSemicolon && t->insert_semicolon) {
gb_inline void tokenizer_skip_whitespace(Tokenizer *t, bool on_newline) {
if (on_newline) {
for (;;) {
switch (t->curr_rune) {
case '\n':
break;
case ' ':
case '\t':
case '\r':
@@ -1113,7 +1126,7 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
} else {
for (;;) {
switch (t->curr_rune) {
case '\n': // only on t->insert_semicolon
case '\n':
case ' ':
case '\t':
case '\r':
@@ -1123,7 +1136,10 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
break;
}
}
}
void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
tokenizer_skip_whitespace(t, t->insert_semicolon);
token->kind = Token_Invalid;
token->string.text = t->curr;