diff --git a/src/checker.cpp b/src/checker.cpp index f81017c38..d4094d6ed 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -2521,7 +2521,7 @@ DECL_ATTRIBUTE_PROC(foreign_block_decl_attribute) { if (ev.kind == ExactValue_String) { auto cc = string_to_calling_convention(ev.value_string); if (cc == ProcCC_Invalid) { - error(elem, "Unknown procedure calling convention: '%.*s'\n", LIT(ev.value_string)); + error(elem, "Unknown procedure calling convention: '%.*s'", LIT(ev.value_string)); } else { c->foreign_context.default_cc = cc; } @@ -2533,7 +2533,7 @@ DECL_ATTRIBUTE_PROC(foreign_block_decl_attribute) { if (ev.kind == ExactValue_String) { String link_prefix = ev.value_string; if (!is_foreign_name_valid(link_prefix)) { - error(elem, "Invalid link prefix: '%.*s'\n", LIT(link_prefix)); + error(elem, "Invalid link prefix: '%.*s'", LIT(link_prefix)); } else { c->foreign_context.link_prefix = link_prefix; } @@ -3138,7 +3138,10 @@ void check_collect_value_decl(CheckerContext *c, Ast *decl) { bool success = false; if (value != nullptr) { if (value->kind == Ast_BasicLit && value->BasicLit.token.kind == Token_String) { - String v = value->BasicLit.token.string; + String v = {}; + if (value->tav.value.kind == ExactValue_String) { + v = value->tav.value.value_string; + } if (v == "file") { kind = EntityVisiblity_PrivateToFile; success = true; diff --git a/src/exact_value.cpp b/src/exact_value.cpp index c8e1ae99f..98a0b95d3 100644 --- a/src/exact_value.cpp +++ b/src/exact_value.cpp @@ -297,13 +297,13 @@ ExactValue exact_value_float_from_string(String string) { } -ExactValue exact_value_from_basic_literal(Token token) { - switch (token.kind) { - case Token_String: return exact_value_string(token.string); - case Token_Integer: return exact_value_integer_from_string(token.string); - case Token_Float: return exact_value_float_from_string(token.string); +ExactValue exact_value_from_basic_literal(TokenKind kind, String const &string) { + switch (kind) { + case Token_String: return exact_value_string(string); + case Token_Integer: return exact_value_integer_from_string(string); + case Token_Float: return exact_value_float_from_string(string); case Token_Imag: { - String str = token.string; + String str = string; Rune last_rune = cast(Rune)str[str.len-1]; str.len--; // Ignore the 'i|j|k' f64 imag = float_from_string(str); @@ -317,7 +317,7 @@ ExactValue exact_value_from_basic_literal(Token token) { } case Token_Rune: { Rune r = GB_RUNE_INVALID; - utf8_decode(token.string.text, token.string.len, &r); + utf8_decode(string.text, string.len, &r); return exact_value_i64(r); } default: diff --git a/src/parser.cpp b/src/parser.cpp index 911082b01..694dce735 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -583,12 +583,38 @@ Ast *ast_undef(AstFile *f, Token token) { return result; } +ExactValue exact_value_from_token(AstFile *f, Token const &token) { + String s = token.string; + switch (token.kind) { + case Token_Rune: + if (!unquote_string(ast_allocator(f), &s, 0)) { + syntax_error(token, "Invalid rune literal"); + } + break; + case Token_String: + if (!unquote_string(ast_allocator(f), &s, 0, s.text[0] == '`')) { + syntax_error(token, "Invalid string literal"); + } + break; + } + return exact_value_from_basic_literal(token.kind, s); +} + +String string_value_from_token(AstFile *f, Token const &token) { + ExactValue value = exact_value_from_token(f, token); + String str = {}; + if (value.kind == ExactValue_String) { + str = value.value_string; + } + return str; +} + Ast *ast_basic_lit(AstFile *f, Token basic_lit) { Ast *result = alloc_ast_node(f, Ast_BasicLit); result->BasicLit.token = basic_lit; result->tav.mode = Addressing_Constant; - result->tav.value = exact_value_from_basic_literal(basic_lit); + result->tav.value = exact_value_from_token(f, basic_lit); return result; } @@ -3227,9 +3253,9 @@ Ast *parse_proc_type(AstFile *f, Token proc_token) { ProcCallingConvention cc = ProcCC_Invalid; if (f->curr_token.kind == Token_String) { Token token = expect_token(f, Token_String); - auto c = string_to_calling_convention(token.string); + auto c = string_to_calling_convention(string_value_from_token(f, token)); if (c == ProcCC_Invalid) { - syntax_error(token, "Unknown procedure calling convention: '%.*s'\n", LIT(token.string)); + syntax_error(token, "Unknown procedure calling convention: '%.*s'", LIT(token.string)); } else { cc = c; } @@ -5114,7 +5140,7 @@ void parse_setup_file_decls(Parser *p, AstFile *f, String base_dir, Slice } else if (node->kind == Ast_ImportDecl) { ast_node(id, ImportDecl, node); - String original_string = string_trim_whitespace(id->relpath.string); + String original_string = string_trim_whitespace(string_value_from_token(f, id->relpath)); String import_path = {}; bool ok = determine_path_from_string(&p->file_decl_mutex, node, base_dir, original_string, &import_path); if (!ok) { @@ -5134,7 +5160,7 @@ void parse_setup_file_decls(Parser *p, AstFile *f, String base_dir, Slice auto fullpaths = array_make(permanent_allocator(), 0, fl->filepaths.count); for_array(fp_idx, fl->filepaths) { - String file_str = fl->filepaths[fp_idx].string; + String file_str = string_trim_whitespace(string_value_from_token(f, fl->filepaths[fp_idx])); String fullpath = file_str; if (allow_check_foreign_filepath()) { String foreign_path = {}; diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 71e11b301..42044bae1 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -717,7 +717,6 @@ struct Tokenizer { i32 line_count; i32 error_count; - Array allocated_strings; TokenizerFlags flags; bool insert_semicolon; @@ -806,12 +805,6 @@ void init_tokenizer_with_file_contents(Tokenizer *t, String const &fullpath, gbF if (t->curr_rune == GB_RUNE_BOM) { advance_to_next_rune(t); // Ignore BOM at file beginning } - - if (t->allocated_strings.count != 0) { - array_clear(&t->allocated_strings); - } else { - array_init(&t->allocated_strings, heap_allocator()); - } } TokenizerInitError init_tokenizer(Tokenizer *t, String const &fullpath, TokenizerFlags flags = TokenizerFlag_None) { @@ -857,10 +850,6 @@ gb_inline void destroy_tokenizer(Tokenizer *t) { if (t->start != nullptr) { gb_free(heap_allocator(), t->start); } - for_array(i, t->allocated_strings) { - gb_free(heap_allocator(), t->allocated_strings[i].text); - } - array_free(&t->allocated_strings); } gb_inline i32 digit_value(Rune r) { @@ -1237,15 +1226,6 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { tokenizer_err(t, "Invalid rune literal"); } token->string.len = t->curr - token->string.text; - success = unquote_string(heap_allocator(), &token->string, 0); - if (success > 0) { - if (success == 2) { - array_add(&t->allocated_strings, token->string); - } - } else { - tokenizer_err(t, "Invalid rune literal"); - } - goto semicolon_check; } break; @@ -1288,15 +1268,6 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { } } token->string.len = t->curr - token->string.text; - success = unquote_string(heap_allocator(), &token->string, 0, has_carriage_return); - if (success > 0) { - if (success == 2) { - array_add(&t->allocated_strings, token->string); - } - } else { - tokenizer_err(t, "Invalid string literal"); - } - goto semicolon_check; } break;