mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2025-12-29 01:14:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve tokenization for false integer literals but not effect possible float literals

Browse Source
This commit is contained in:
gingerBill
2024-06-28 09:24:34 +01:00
parent dde7cb6e7f
commit 35651cfc17
1 changed files with 15 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/tokenizer.cpp
View File

@@ -432,8 +432,10 @@ gb_internal gb_inline i32 digit_value(Rune r) {
return 16; // NOTE(bill): Larger than highest possible
}
gb_internal gb_inline void scan_mantissa(Tokenizer *t, i32 base) {
base = 16; // always check for any possible letter
gb_internal gb_inline void scan_mantissa(Tokenizer *t, i32 base, bool force_base) {
if (!force_base) {
base = 16; // always check for any possible letter
}
while (digit_value(t->curr_rune) < base || t->curr_rune == '_') {
advance_to_next_rune(t);
}
@@ -458,7 +460,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
token->string.len += 1;
token->pos.column -= 1;
token->kind = Token_Float;
scan_mantissa(t, 10);
scan_mantissa(t, 10, true);
goto exponent;
}
@@ -468,7 +470,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
switch (t->curr_rune) {
case 'b': // Binary
advance_to_next_rune(t);
scan_mantissa(t, 2);
scan_mantissa(t, 2, false);
if (t->curr - prev <= 2) {
tokenizer_err(t, "Invalid binary integer");
token->kind = Token_Invalid;
@@ -476,7 +478,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
goto end;
case 'o': // Octal
advance_to_next_rune(t);
scan_mantissa(t, 8);
scan_mantissa(t, 8, false);
if (t->curr - prev <= 2) {
tokenizer_err(t, "Invalid octal integer");
token->kind = Token_Invalid;
@@ -484,7 +486,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
goto end;
case 'd': // Decimal
advance_to_next_rune(t);
scan_mantissa(t, 10);
scan_mantissa(t, 10, false);
if (t->curr - prev <= 2) {
tokenizer_err(t, "Invalid explicitly decimal integer");
token->kind = Token_Invalid;
@@ -492,7 +494,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
goto end;
case 'z': // Dozenal
advance_to_next_rune(t);
scan_mantissa(t, 12);
scan_mantissa(t, 12, false);
if (t->curr - prev <= 2) {
tokenizer_err(t, "Invalid dozenal integer");
token->kind = Token_Invalid;
@@ -500,7 +502,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
goto end;
case 'x': // Hexadecimal
advance_to_next_rune(t);
scan_mantissa(t, 16);
scan_mantissa(t, 16, false);
if (t->curr - prev <= 2) {
tokenizer_err(t, "Invalid hexadecimal integer");
token->kind = Token_Invalid;
@@ -509,7 +511,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
case 'h': // Hexadecimal Float
token->kind = Token_Float;
advance_to_next_rune(t);
scan_mantissa(t, 16);
scan_mantissa(t, 16, false);
if (t->curr - prev <= 2) {
tokenizer_err(t, "Invalid hexadecimal float");
token->kind = Token_Invalid;
@@ -534,12 +536,12 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci
}
goto end;
default:
scan_mantissa(t, 10);
scan_mantissa(t, 10, true);
goto fraction;
}
}
scan_mantissa(t, 10);
scan_mantissa(t, 10, true);
fraction:
@@ -551,7 +553,7 @@ fraction:
advance_to_next_rune(t);
token->kind = Token_Float;
scan_mantissa(t, 10);
scan_mantissa(t, 10, true);
}
exponent:
@@ -561,7 +563,7 @@ exponent:
if (t->curr_rune == '-' || t->curr_rune == '+') {
advance_to_next_rune(t);
}
scan_mantissa(t, 10);
scan_mantissa(t, 10, false);
}
switch (t->curr_rune) {