mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-04-18 20:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make -insert-semicolon the default now

Browse Source
This commit is contained in:
gingerBill
2021-08-31 21:13:53 +01:00
parent d399d2256b
commit 1fff96e088
4 changed files with 129 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/build_settings.cpp
View File

@@ -120,8 +120,10 @@ enum CommandKind : u32 {
Command_doc = 1<<5,
Command_version = 1<<6,
Command_test = 1<<7,
Command_strip_semicolon = 1<<8,
Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test,
Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test|Command_strip_semicolon,
Command__does_build = Command_run|Command_build|Command_test,
Command_all = ~(u32)0,
};
@@ -134,6 +136,7 @@ char const *odin_command_strings[32] = {
"doc",
"version",
"test",
"strip-semicolon",
};
@@ -201,7 +204,6 @@ struct BuildContext {
bool different_os;
bool keep_object_files;
bool disallow_do;
bool insert_semicolon;
bool ignore_warnings;

60
src/main.cpp
View File

@@ -1307,7 +1307,7 @@ bool parse_build_flags(Array<String> args) {
break;
case BuildFlag_InsertSemicolon:
build_context.insert_semicolon = true;
gb_printf_err("-insert-semicolon flag is not required any more\n");
break;
case BuildFlag_StrictStyle:
@@ -1644,7 +1644,7 @@ void print_show_help(String const arg0, String const &command) {
} else if (command == "run") {
print_usage_line(1, "run same as 'build', but also then runs the newly compiled executable.");
} else if (command == "check") {
print_usage_line(1, "check parse and type check .odin file");
print_usage_line(1, "check parse and type check .odin file(s)");
} else if (command == "test") {
print_usage_line(1, "test build ands runs procedures with the attribute @(test) in the initial package");
} else if (command == "query") {
@@ -1656,14 +1656,18 @@ void print_show_help(String const arg0, String const &command) {
print_usage_line(3, "odin doc core/path core/path/filepath");
} else if (command == "version") {
print_usage_line(1, "version print version");
}
} else if (command == "strip-semicolon") {
print_usage_line(1, "strip-semicolon");
print_usage_line(2, "parse and type check .odin file(s) and then remove unneeded semicolons from the entire project");
}
bool doc = command == "doc";
bool build = command == "build";
bool run_or_build = command == "run" || command == "build" || command == "test";
bool test_only = command == "test";
bool check_only = command == "check";
bool check = run_or_build || command == "check";
bool strip_semicolon = command == "strip-semicolon";
bool check_only = command == "check" || strip_semicolon;
bool check = run_or_build || check_only;
print_usage_line(0, "");
print_usage_line(1, "Flags");
@@ -1730,6 +1734,10 @@ void print_show_help(String const arg0, String const &command) {
print_usage_line(1, "-keep-temp-files");
print_usage_line(2, "Keeps the temporary files generated during compilation");
print_usage_line(0, "");
} else if (strip_semicolon) {
print_usage_line(1, "-keep-temp-files");
print_usage_line(2, "Keeps the temporary files generated during stripping the unneeded semicolons from files");
print_usage_line(0, "");
}
if (check) {
@@ -2002,6 +2010,36 @@ bool check_env(void) {
return true;
}
struct StripSemicolonFile {
String old_fullpath;
String new_fullpath;
AstFile *file;
};
int strip_semicolons(Parser *parser) {
#if 0
isize file_count = 0;
for_array(i, parser->packages) {
AstPackage *pkg = parser->packages[i];
file_count += pkg->files.count;
}
gb_printf_err("File count: %td\n", file_count);
auto generated_files = array_make<StripSemicolonFile>(permanent_allocator(), 0, file_count);
for_array(i, parser->packages) {
AstPackage *pkg = parser->packages[i];
for_array(j, pkg->files) {
AstFile *file = pkg->files[j];
gb_printf_err("%.*s\n", LIT(file->fullpath));
}
}
#endif
return 0;
}
int main(int arg_count, char const **arg_ptr) {
#define TIME_SECTION(str) do { debugf("[Section] %s\n", str); timings_start_section(&global_timings, str_lit(str)); } while (0)
@@ -2090,6 +2128,14 @@ int main(int arg_count, char const **arg_ptr) {
build_context.command_kind = Command_check;
build_context.no_output_files = true;
init_filename = args[2];
} else if (command == "strip-semicolon") {
if (args.count < 3) {
usage(args[0]);
return 1;
}
build_context.command_kind = Command_strip_semicolon;
build_context.no_output_files = true;
init_filename = args[2];
} else if (command == "query") {
if (args.count < 3) {
usage(args[0]);
@@ -2209,6 +2255,10 @@ int main(int arg_count, char const **arg_ptr) {
if (any_errors()) {
return 1;
}
if (build_context.command_kind == Command_strip_semicolon) {
return strip_semicolons(parser);
}
if (build_context.generate_docs) {
if (global_error_collector.count != 0) {

114
src/parser.cpp
View File

@@ -1281,9 +1281,6 @@ Token peek_token(AstFile *f) {
}
bool skip_possible_newline(AstFile *f) {
if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) {
return false;
}
if (token_is_newline(f->curr_token)) {
advance_token(f);
return true;
@@ -1292,9 +1289,6 @@ bool skip_possible_newline(AstFile *f) {
}
bool skip_possible_newline_for_literal(AstFile *f) {
if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) {
return false;
}
Token curr = f->curr_token;
if (token_is_newline(curr)) {
Token next = peek_token(f);
@@ -1500,62 +1494,11 @@ bool is_semicolon_optional_for_node(AstFile *f, Ast *s) {
if (s == nullptr) {
return false;
}
if (build_context.insert_semicolon) {
return true;
}
switch (s->kind) {
case Ast_EmptyStmt:
case Ast_BlockStmt:
return true;
case Ast_IfStmt:
case Ast_WhenStmt:
case Ast_ForStmt:
case Ast_RangeStmt:
case Ast_SwitchStmt:
case Ast_TypeSwitchStmt:
return true;
case Ast_HelperType:
return is_semicolon_optional_for_node(f, s->HelperType.type);
case Ast_DistinctType:
return is_semicolon_optional_for_node(f, s->DistinctType.type);
case Ast_PointerType:
return is_semicolon_optional_for_node(f, s->PointerType.type);
case Ast_StructType:
case Ast_UnionType:
case Ast_EnumType:
// Require semicolon within a procedure body
return f->curr_proc == nullptr;
case Ast_ProcLit:
return true;
case Ast_PackageDecl:
case Ast_ImportDecl:
case Ast_ForeignImportDecl:
return true;
case Ast_ValueDecl:
if (s->ValueDecl.is_mutable) {
return false;
}
if (s->ValueDecl.values.count > 0) {
return is_semicolon_optional_for_node(f, s->ValueDecl.values[s->ValueDecl.values.count-1]);
}
break;
case Ast_ForeignBlockDecl:
return is_semicolon_optional_for_node(f, s->ForeignBlockDecl.body);
}
return false;
return true;
}
void expect_semicolon_newline_error(AstFile *f, Token const &token, Ast *s) {
#if 0
if (!build_context.insert_semicolon && token.string == "\n") {
switch (token.kind) {
case Token_CloseBrace:
@@ -1571,6 +1514,14 @@ void expect_semicolon_newline_error(AstFile *f, Token const &token, Ast *s) {
tok.pos.column -= 1;
syntax_error(tok, "Expected ';', got newline");
}
#endif
}
void assign_removal_flag_to_semicolon(Token *token) {
// NOTE(bill): this is used for rewriting files to strip unneeded semicolons
if (token->kind == Token_Semicolon && token->string == ";") {
token->flags |= TokenFlag_Remove;
}
}
void expect_semicolon(AstFile *f, Ast *s) {
@@ -1578,6 +1529,7 @@ void expect_semicolon(AstFile *f, Ast *s) {
if (allow_token(f, Token_Semicolon)) {
expect_semicolon_newline_error(f, f->prev_token, s);
assign_removal_flag_to_semicolon(&f->prev_token);
return;
}
switch (f->curr_token.kind) {
@@ -1592,6 +1544,7 @@ void expect_semicolon(AstFile *f, Ast *s) {
prev_token = f->prev_token;
if (prev_token.kind == Token_Semicolon) {
expect_semicolon_newline_error(f, f->prev_token, s);
assign_removal_flag_to_semicolon(&f->prev_token);
return;
}
@@ -1599,40 +1552,19 @@ void expect_semicolon(AstFile *f, Ast *s) {
return;
}
if (s != nullptr) {
bool insert_semi = (f->tokenizer.flags & TokenizerFlag_InsertSemicolon) != 0;
if (insert_semi) {
switch (f->curr_token.kind) {
case Token_CloseBrace:
case Token_CloseParen:
case Token_else:
case Token_EOF:
return;
switch (f->curr_token.kind) {
case Token_CloseBrace:
case Token_CloseParen:
case Token_else:
case Token_EOF:
return;
default:
if (is_semicolon_optional_for_node(f, s)) {
return;
}
break;
}
} else if (prev_token.pos.line != f->curr_token.pos.line) {
default:
if (is_semicolon_optional_for_node(f, s)) {
return;
}
} else {
switch (f->curr_token.kind) {
case Token_CloseBrace:
case Token_CloseParen:
case Token_else:
return;
case Token_EOF:
if (is_semicolon_optional_for_node(f, s)) {
return;
}
break;
}
break;
}
String node_string = ast_strings[s->kind];
String p = token_to_string(f->curr_token);
@@ -4144,7 +4076,7 @@ Ast *parse_for_stmt(AstFile *f) {
if (f->curr_token.string != ";") {
syntax_error(f->curr_token, "Expected ';', got %.*s", LIT(token_to_string(f->curr_token)));
} else {
expect_semicolon(f, nullptr);
expect_token(f, Token_Semicolon);
}
if (f->curr_token.kind != Token_OpenBrace &&
@@ -4717,12 +4649,10 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
if (!string_ends_with(f->fullpath, str_lit(".odin"))) {
return ParseFile_WrongExtension;
}
TokenizerFlags tokenizer_flags = TokenizerFlag_InsertSemicolon;
zero_item(&f->tokenizer);
f->tokenizer.curr_file_id = f->id;
TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath, tokenizer_flags);
TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath);
if (err != TokenizerInit_None) {
switch (err) {
case TokenizerInit_Empty:

105
src/tokenizer.cpp
View File

@@ -120,7 +120,7 @@ TOKEN_KIND(Token__KeywordBegin, ""), \
TOKEN_KIND(Token__KeywordEnd, ""), \
TOKEN_KIND(Token_Count, "")
enum TokenKind {
enum TokenKind : u8 {
#define TOKEN_KIND(e, s) e
TOKEN_KINDS
#undef TOKEN_KIND
@@ -235,21 +235,26 @@ TokenPos token_pos_add_column(TokenPos pos) {
return pos;
}
enum TokenFlag : u8 {
TokenFlag_Remove = 1<<1,
};
struct Token {
TokenKind kind;
u8 flags;
String string;
TokenPos pos;
};
Token empty_token = {Token_Invalid};
Token blank_token = {Token_Ident, {cast(u8 *)"_", 1}};
Token blank_token = {Token_Ident, 0, {cast(u8 *)"_", 1}};
Token make_token_ident(String s) {
Token t = {Token_Ident, s};
Token t = {Token_Ident, 0, s};
return t;
}
Token make_token_ident(char const *s) {
Token t = {Token_Ident, make_string_c(s)};
Token t = {Token_Ident, 0, make_string_c(s)};
return t;
}
@@ -701,12 +706,6 @@ enum TokenizerInitError {
TokenizerInit_Count,
};
enum TokenizerFlags {
TokenizerFlag_None = 0,
TokenizerFlag_InsertSemicolon = 1<<0,
};
struct Tokenizer {
i32 curr_file_id;
String fullpath;
@@ -721,7 +720,6 @@ struct Tokenizer {
i32 error_count;
TokenizerFlags flags;
bool insert_semicolon;
};
@@ -789,8 +787,7 @@ void advance_to_next_rune(Tokenizer *t) {
}
}
void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size, TokenizerFlags flags) {
t->flags = flags;
void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size) {
t->fullpath = fullpath;
t->line_count = 1;
@@ -804,7 +801,7 @@ void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data,
}
}
TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath, TokenizerFlags flags = TokenizerFlag_None) {
TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath) {
TokenizerInitError err = TokenizerInit_None;
char *c_str = alloc_cstring(temporary_allocator(), fullpath);
@@ -813,15 +810,13 @@ TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &full
gbFileContents fc = gb_file_read_contents(heap_allocator(), true, c_str);
if (fc.size > I32_MAX) {
t->flags = flags;
t->fullpath = fullpath;
t->line_count = 1;
err = TokenizerInit_FileTooLarge;
gb_file_free_contents(&fc);
} else if (fc.data != nullptr) {
init_tokenizer_with_data(t, fullpath, fc.data, fc.size, flags);
init_tokenizer_with_data(t, fullpath, fc.data, fc.size);
} else {
t->flags = flags;
t->fullpath = fullpath;
t->line_count = 1;
gbFile f = {};
@@ -1163,9 +1158,7 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
return;
case '\\':
if (t->flags & TokenizerFlag_InsertSemicolon) {
t->insert_semicolon = false;
}
t->insert_semicolon = false;
tokenizer_get_token(t, token);
if (token->pos.line == current_pos.line) {
tokenizer_err(t, token_pos_add_column(current_pos), "Expected a newline after \\");
@@ -1493,43 +1486,41 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
token->string.len = t->curr - token->string.text;
semicolon_check:;
if (t->flags & TokenizerFlag_InsertSemicolon) {
switch (token->kind) {
case Token_Invalid:
case Token_Comment:
// Preserve insert_semicolon info
break;
case Token_Ident:
case Token_context:
case Token_typeid:
case Token_break:
case Token_continue:
case Token_fallthrough:
case Token_return:
case Token_or_return:
/*fallthrough*/
case Token_Integer:
case Token_Float:
case Token_Imag:
case Token_Rune:
case Token_String:
case Token_Undef:
/*fallthrough*/
case Token_Question:
case Token_Pointer:
case Token_CloseParen:
case Token_CloseBracket:
case Token_CloseBrace:
/*fallthrough*/
case Token_Increment:
case Token_Decrement:
/*fallthrough*/
t->insert_semicolon = true;
break;
default:
t->insert_semicolon = false;
break;
}
switch (token->kind) {
case Token_Invalid:
case Token_Comment:
// Preserve insert_semicolon info
break;
case Token_Ident:
case Token_context:
case Token_typeid:
case Token_break:
case Token_continue:
case Token_fallthrough:
case Token_return:
case Token_or_return:
/*fallthrough*/
case Token_Integer:
case Token_Float:
case Token_Imag:
case Token_Rune:
case Token_String:
case Token_Undef:
/*fallthrough*/
case Token_Question:
case Token_Pointer:
case Token_CloseParen:
case Token_CloseBracket:
case Token_CloseBrace:
/*fallthrough*/
case Token_Increment:
case Token_Decrement:
/*fallthrough*/
t->insert_semicolon = true;
break;
default:
t->insert_semicolon = false;
break;
}
return;