From 1fff96e088eda39dd7c759b7d059ac11f3d120be Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 31 Aug 2021 21:13:53 +0100 Subject: [PATCH] Make `-insert-semicolon` the default now --- src/build_settings.cpp | 6 ++- src/main.cpp | 60 ++++++++++++++++++++-- src/parser.cpp | 114 ++++++++--------------------------------- src/tokenizer.cpp | 105 +++++++++++++++++-------------------- 4 files changed, 129 insertions(+), 156 deletions(-) diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 5d93ab1ca..c614a6dc8 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -120,8 +120,10 @@ enum CommandKind : u32 { Command_doc = 1<<5, Command_version = 1<<6, Command_test = 1<<7, + + Command_strip_semicolon = 1<<8, - Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test, + Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test|Command_strip_semicolon, Command__does_build = Command_run|Command_build|Command_test, Command_all = ~(u32)0, }; @@ -134,6 +136,7 @@ char const *odin_command_strings[32] = { "doc", "version", "test", + "strip-semicolon", }; @@ -201,7 +204,6 @@ struct BuildContext { bool different_os; bool keep_object_files; bool disallow_do; - bool insert_semicolon; bool ignore_warnings; diff --git a/src/main.cpp b/src/main.cpp index 4edbc0420..23c770e37 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1307,7 +1307,7 @@ bool parse_build_flags(Array args) { break; case BuildFlag_InsertSemicolon: - build_context.insert_semicolon = true; + gb_printf_err("-insert-semicolon flag is not required any more\n"); break; case BuildFlag_StrictStyle: @@ -1644,7 +1644,7 @@ void print_show_help(String const arg0, String const &command) { } else if (command == "run") { print_usage_line(1, "run same as 'build', but also then runs the newly compiled executable."); } else if (command == "check") { - print_usage_line(1, "check parse and type check .odin file"); + print_usage_line(1, "check parse and type check .odin file(s)"); } else if (command == "test") { print_usage_line(1, "test build ands runs procedures with the attribute @(test) in the initial package"); } else if (command == "query") { @@ -1656,14 +1656,18 @@ void print_show_help(String const arg0, String const &command) { print_usage_line(3, "odin doc core/path core/path/filepath"); } else if (command == "version") { print_usage_line(1, "version print version"); - } + } else if (command == "strip-semicolon") { + print_usage_line(1, "strip-semicolon"); + print_usage_line(2, "parse and type check .odin file(s) and then remove unneeded semicolons from the entire project"); + } bool doc = command == "doc"; bool build = command == "build"; bool run_or_build = command == "run" || command == "build" || command == "test"; bool test_only = command == "test"; - bool check_only = command == "check"; - bool check = run_or_build || command == "check"; + bool strip_semicolon = command == "strip-semicolon"; + bool check_only = command == "check" || strip_semicolon; + bool check = run_or_build || check_only; print_usage_line(0, ""); print_usage_line(1, "Flags"); @@ -1730,6 +1734,10 @@ void print_show_help(String const arg0, String const &command) { print_usage_line(1, "-keep-temp-files"); print_usage_line(2, "Keeps the temporary files generated during compilation"); print_usage_line(0, ""); + } else if (strip_semicolon) { + print_usage_line(1, "-keep-temp-files"); + print_usage_line(2, "Keeps the temporary files generated during stripping the unneeded semicolons from files"); + print_usage_line(0, ""); } if (check) { @@ -2002,6 +2010,36 @@ bool check_env(void) { return true; } +struct StripSemicolonFile { + String old_fullpath; + String new_fullpath; + AstFile *file; +}; + +int strip_semicolons(Parser *parser) { + #if 0 + isize file_count = 0; + for_array(i, parser->packages) { + AstPackage *pkg = parser->packages[i]; + file_count += pkg->files.count; + } + gb_printf_err("File count: %td\n", file_count); + + auto generated_files = array_make(permanent_allocator(), 0, file_count); + + for_array(i, parser->packages) { + AstPackage *pkg = parser->packages[i]; + for_array(j, pkg->files) { + AstFile *file = pkg->files[j]; + gb_printf_err("%.*s\n", LIT(file->fullpath)); + } + } + + #endif + return 0; +} + + int main(int arg_count, char const **arg_ptr) { #define TIME_SECTION(str) do { debugf("[Section] %s\n", str); timings_start_section(&global_timings, str_lit(str)); } while (0) @@ -2090,6 +2128,14 @@ int main(int arg_count, char const **arg_ptr) { build_context.command_kind = Command_check; build_context.no_output_files = true; init_filename = args[2]; + } else if (command == "strip-semicolon") { + if (args.count < 3) { + usage(args[0]); + return 1; + } + build_context.command_kind = Command_strip_semicolon; + build_context.no_output_files = true; + init_filename = args[2]; } else if (command == "query") { if (args.count < 3) { usage(args[0]); @@ -2209,6 +2255,10 @@ int main(int arg_count, char const **arg_ptr) { if (any_errors()) { return 1; } + + if (build_context.command_kind == Command_strip_semicolon) { + return strip_semicolons(parser); + } if (build_context.generate_docs) { if (global_error_collector.count != 0) { diff --git a/src/parser.cpp b/src/parser.cpp index 98064d63b..ec1a5776b 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1281,9 +1281,6 @@ Token peek_token(AstFile *f) { } bool skip_possible_newline(AstFile *f) { - if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) { - return false; - } if (token_is_newline(f->curr_token)) { advance_token(f); return true; @@ -1292,9 +1289,6 @@ bool skip_possible_newline(AstFile *f) { } bool skip_possible_newline_for_literal(AstFile *f) { - if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) { - return false; - } Token curr = f->curr_token; if (token_is_newline(curr)) { Token next = peek_token(f); @@ -1500,62 +1494,11 @@ bool is_semicolon_optional_for_node(AstFile *f, Ast *s) { if (s == nullptr) { return false; } - - if (build_context.insert_semicolon) { - return true; - } - - switch (s->kind) { - case Ast_EmptyStmt: - case Ast_BlockStmt: - return true; - - case Ast_IfStmt: - case Ast_WhenStmt: - case Ast_ForStmt: - case Ast_RangeStmt: - case Ast_SwitchStmt: - case Ast_TypeSwitchStmt: - return true; - - case Ast_HelperType: - return is_semicolon_optional_for_node(f, s->HelperType.type); - case Ast_DistinctType: - return is_semicolon_optional_for_node(f, s->DistinctType.type); - - case Ast_PointerType: - return is_semicolon_optional_for_node(f, s->PointerType.type); - - case Ast_StructType: - case Ast_UnionType: - case Ast_EnumType: - // Require semicolon within a procedure body - return f->curr_proc == nullptr; - case Ast_ProcLit: - return true; - - case Ast_PackageDecl: - case Ast_ImportDecl: - case Ast_ForeignImportDecl: - return true; - - case Ast_ValueDecl: - if (s->ValueDecl.is_mutable) { - return false; - } - if (s->ValueDecl.values.count > 0) { - return is_semicolon_optional_for_node(f, s->ValueDecl.values[s->ValueDecl.values.count-1]); - } - break; - - case Ast_ForeignBlockDecl: - return is_semicolon_optional_for_node(f, s->ForeignBlockDecl.body); - } - - return false; + return true; } void expect_semicolon_newline_error(AstFile *f, Token const &token, Ast *s) { + #if 0 if (!build_context.insert_semicolon && token.string == "\n") { switch (token.kind) { case Token_CloseBrace: @@ -1571,6 +1514,14 @@ void expect_semicolon_newline_error(AstFile *f, Token const &token, Ast *s) { tok.pos.column -= 1; syntax_error(tok, "Expected ';', got newline"); } + #endif +} + +void assign_removal_flag_to_semicolon(Token *token) { + // NOTE(bill): this is used for rewriting files to strip unneeded semicolons + if (token->kind == Token_Semicolon && token->string == ";") { + token->flags |= TokenFlag_Remove; + } } void expect_semicolon(AstFile *f, Ast *s) { @@ -1578,6 +1529,7 @@ void expect_semicolon(AstFile *f, Ast *s) { if (allow_token(f, Token_Semicolon)) { expect_semicolon_newline_error(f, f->prev_token, s); + assign_removal_flag_to_semicolon(&f->prev_token); return; } switch (f->curr_token.kind) { @@ -1592,6 +1544,7 @@ void expect_semicolon(AstFile *f, Ast *s) { prev_token = f->prev_token; if (prev_token.kind == Token_Semicolon) { expect_semicolon_newline_error(f, f->prev_token, s); + assign_removal_flag_to_semicolon(&f->prev_token); return; } @@ -1599,40 +1552,19 @@ void expect_semicolon(AstFile *f, Ast *s) { return; } - - if (s != nullptr) { - bool insert_semi = (f->tokenizer.flags & TokenizerFlag_InsertSemicolon) != 0; - if (insert_semi) { - switch (f->curr_token.kind) { - case Token_CloseBrace: - case Token_CloseParen: - case Token_else: - case Token_EOF: - return; + switch (f->curr_token.kind) { + case Token_CloseBrace: + case Token_CloseParen: + case Token_else: + case Token_EOF: + return; - default: - if (is_semicolon_optional_for_node(f, s)) { - return; - } - break; - } - } else if (prev_token.pos.line != f->curr_token.pos.line) { + default: if (is_semicolon_optional_for_node(f, s)) { return; } - } else { - switch (f->curr_token.kind) { - case Token_CloseBrace: - case Token_CloseParen: - case Token_else: - return; - case Token_EOF: - if (is_semicolon_optional_for_node(f, s)) { - return; - } - break; - } + break; } String node_string = ast_strings[s->kind]; String p = token_to_string(f->curr_token); @@ -4144,7 +4076,7 @@ Ast *parse_for_stmt(AstFile *f) { if (f->curr_token.string != ";") { syntax_error(f->curr_token, "Expected ';', got %.*s", LIT(token_to_string(f->curr_token))); } else { - expect_semicolon(f, nullptr); + expect_token(f, Token_Semicolon); } if (f->curr_token.kind != Token_OpenBrace && @@ -4717,12 +4649,10 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) { if (!string_ends_with(f->fullpath, str_lit(".odin"))) { return ParseFile_WrongExtension; } - TokenizerFlags tokenizer_flags = TokenizerFlag_InsertSemicolon; - zero_item(&f->tokenizer); f->tokenizer.curr_file_id = f->id; - TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath, tokenizer_flags); + TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath); if (err != TokenizerInit_None) { switch (err) { case TokenizerInit_Empty: diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 1e2a3caf3..3a3c40a7d 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -120,7 +120,7 @@ TOKEN_KIND(Token__KeywordBegin, ""), \ TOKEN_KIND(Token__KeywordEnd, ""), \ TOKEN_KIND(Token_Count, "") -enum TokenKind { +enum TokenKind : u8 { #define TOKEN_KIND(e, s) e TOKEN_KINDS #undef TOKEN_KIND @@ -235,21 +235,26 @@ TokenPos token_pos_add_column(TokenPos pos) { return pos; } +enum TokenFlag : u8 { + TokenFlag_Remove = 1<<1, +}; + struct Token { TokenKind kind; + u8 flags; String string; TokenPos pos; }; Token empty_token = {Token_Invalid}; -Token blank_token = {Token_Ident, {cast(u8 *)"_", 1}}; +Token blank_token = {Token_Ident, 0, {cast(u8 *)"_", 1}}; Token make_token_ident(String s) { - Token t = {Token_Ident, s}; + Token t = {Token_Ident, 0, s}; return t; } Token make_token_ident(char const *s) { - Token t = {Token_Ident, make_string_c(s)}; + Token t = {Token_Ident, 0, make_string_c(s)}; return t; } @@ -701,12 +706,6 @@ enum TokenizerInitError { TokenizerInit_Count, }; - -enum TokenizerFlags { - TokenizerFlag_None = 0, - TokenizerFlag_InsertSemicolon = 1<<0, -}; - struct Tokenizer { i32 curr_file_id; String fullpath; @@ -721,7 +720,6 @@ struct Tokenizer { i32 error_count; - TokenizerFlags flags; bool insert_semicolon; }; @@ -789,8 +787,7 @@ void advance_to_next_rune(Tokenizer *t) { } } -void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size, TokenizerFlags flags) { - t->flags = flags; +void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size) { t->fullpath = fullpath; t->line_count = 1; @@ -804,7 +801,7 @@ void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, } } -TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath, TokenizerFlags flags = TokenizerFlag_None) { +TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath) { TokenizerInitError err = TokenizerInit_None; char *c_str = alloc_cstring(temporary_allocator(), fullpath); @@ -813,15 +810,13 @@ TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &full gbFileContents fc = gb_file_read_contents(heap_allocator(), true, c_str); if (fc.size > I32_MAX) { - t->flags = flags; t->fullpath = fullpath; t->line_count = 1; err = TokenizerInit_FileTooLarge; gb_file_free_contents(&fc); } else if (fc.data != nullptr) { - init_tokenizer_with_data(t, fullpath, fc.data, fc.size, flags); + init_tokenizer_with_data(t, fullpath, fc.data, fc.size); } else { - t->flags = flags; t->fullpath = fullpath; t->line_count = 1; gbFile f = {}; @@ -1163,9 +1158,7 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { return; case '\\': - if (t->flags & TokenizerFlag_InsertSemicolon) { - t->insert_semicolon = false; - } + t->insert_semicolon = false; tokenizer_get_token(t, token); if (token->pos.line == current_pos.line) { tokenizer_err(t, token_pos_add_column(current_pos), "Expected a newline after \\"); @@ -1493,43 +1486,41 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { token->string.len = t->curr - token->string.text; semicolon_check:; - if (t->flags & TokenizerFlag_InsertSemicolon) { - switch (token->kind) { - case Token_Invalid: - case Token_Comment: - // Preserve insert_semicolon info - break; - case Token_Ident: - case Token_context: - case Token_typeid: - case Token_break: - case Token_continue: - case Token_fallthrough: - case Token_return: - case Token_or_return: - /*fallthrough*/ - case Token_Integer: - case Token_Float: - case Token_Imag: - case Token_Rune: - case Token_String: - case Token_Undef: - /*fallthrough*/ - case Token_Question: - case Token_Pointer: - case Token_CloseParen: - case Token_CloseBracket: - case Token_CloseBrace: - /*fallthrough*/ - case Token_Increment: - case Token_Decrement: - /*fallthrough*/ - t->insert_semicolon = true; - break; - default: - t->insert_semicolon = false; - break; - } + switch (token->kind) { + case Token_Invalid: + case Token_Comment: + // Preserve insert_semicolon info + break; + case Token_Ident: + case Token_context: + case Token_typeid: + case Token_break: + case Token_continue: + case Token_fallthrough: + case Token_return: + case Token_or_return: + /*fallthrough*/ + case Token_Integer: + case Token_Float: + case Token_Imag: + case Token_Rune: + case Token_String: + case Token_Undef: + /*fallthrough*/ + case Token_Question: + case Token_Pointer: + case Token_CloseParen: + case Token_CloseBracket: + case Token_CloseBrace: + /*fallthrough*/ + case Token_Increment: + case Token_Decrement: + /*fallthrough*/ + t->insert_semicolon = true; + break; + default: + t->insert_semicolon = false; + break; } return;