mirrors/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2026-04-25 08:44:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #12321 from vigoux/treesitter-runtime

Browse Source 
treesitter: update runtime
This commit is contained in:
Matthieu Coudron
2020-06-06 15:37:51 +02:00
committed by GitHub
parent 39f802bef4 6b949211a0
commit dbc8ec9446
13 changed files with 792 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
runtime/lua/vim/treesitter.lua
View File

@@ -162,16 +162,17 @@ function Query:match_preds(match, pattern, bufnr)
-- Here we only want to return if a predicate DOES NOT match, and
-- continue on the other case. This way unknown predicates will not be considered,
-- which allows some testing and easier user extensibility (#12173).
-- Also, tree-sitter strips the leading # from predicates for us.
if pred[1] == "eq?" then
local node = match[pred[2]]
local node_text = get_node_text(node, bufnr)
local str
if type(pred[3]) == "string" then
-- (eq? @aa "foo")
-- (#eq? @aa "foo")
str = pred[3]
else
-- (eq? @aa @bb)
-- (#eq? @aa @bb)
str = get_node_text(match[pred[3]], bufnr)
end

39
scripts/update-ts-runtime.sh Executable file
View File

@@ -0,0 +1,39 @@
#!/bin/sh
#
# This script will update the treesitter runtime to the provided commit.
# Usage :
# $0 <tree-sitter commit sha>
set -e
ts_source_dir="/tmp/tree-sitter"
ts_url="https://github.com/tree-sitter/tree-sitter.git"
base_dir="$(cd "$(dirname $(dirname $0))" && pwd)"
ts_dest_dir="$base_dir/src/tree_sitter/"
ts_current_commit="$ts_dest_dir/treesitter_commit_hash.txt"
echo "Updating treesitter runtime from $(cat "$ts_current_commit") to $1..."
if [ ! -d "$ts_source_dir" ]; then
echo "Cloning treesitter..."
git clone "$ts_url" "$ts_source_dir"
else
echo "Found a non-empty $ts_source_dir directory..."
git -C "$ts_source_dir" fetch
fi
echo "Checking out $1..."
git -C "$ts_source_dir" checkout $1
echo "Removing old files..."
find "$ts_dest_dir" -not -name "LICENSE" -not -name "README.md" -not -type d -delete
echo "Copying files..."
cp -t "$ts_dest_dir" -r "$ts_source_dir/lib/src"/*
cp -t "$ts_dest_dir" "$ts_source_dir/lib/include/tree_sitter"/*
echo "$1" > "$ts_current_commit"
make
TEST_FILE="$base_dir/test/functional/lua/treesitter_spec.lua" make test

16
src/tree_sitter/README.md Normal file
View File

@@ -0,0 +1,16 @@
Tree-sitter vendor runtime
==========================
This is the vendor runtime code for treesitter.
The original code can be found [here](https://github.com/tree-sitter/tree-sitter).
As this code is not ours, if you find any bugs, feel free to open an issue, so that we can
investigate and determine if this should go upstream.
# Updating
To update the treesitter runtime, use the `update-ts-runtime.sh` script in the `scripts` directory:
```sh
./scripts/update-ts-runtime.sh <commit you want to update to>
```

1
src/tree_sitter/alloc.h
View File

@@ -51,6 +51,7 @@ static inline void ts_free(void *buffer) {
#include <stdlib.h>
static inline bool ts_toggle_allocation_recording(bool value) {
(void)value;
return false;
}

4
src/tree_sitter/language.c
View File

@@ -33,8 +33,8 @@ void ts_language_table_entry(
assert(symbol < self->token_count);
uint32_t action_index = ts_language_lookup(self, state, symbol);
const TSParseActionEntry *entry = &self->parse_actions[action_index];
result->action_count = entry->count;
result->is_reusable = entry->reusable;
result->action_count = entry->entry.count;
result->is_reusable = entry->entry.reusable;
result->actions = (const TSParseAction *)(entry + 1);
}
}

2
src/tree_sitter/language.h
View File

@@ -93,7 +93,7 @@ static inline TSStateId ts_language_next_state(const TSLanguage *self,
if (count > 0) {
TSParseAction action = actions[count - 1];
if (action.type == TSParseActionTypeShift) {
return action.params.extra ? state : action.params.state;
return action.params.shift.extra ? state : action.params.shift.state;
}
}
return 0;

32
src/tree_sitter/parser.c
View File

@@ -101,9 +101,10 @@ typedef struct {
static const char *ts_string_input_read(
void *_self,
uint32_t byte,
TSPoint _,
TSPoint pt,
uint32_t *length
) {
(void)pt;
TSStringInput *self = (TSStringInput *)_self;
if (byte >= self->length) {
*length = 0;
@@ -210,6 +211,7 @@ static ErrorComparison ts_parser__compare_versions(
ErrorStatus a,
ErrorStatus b
) {
(void)self;
if (!a.is_in_error && b.is_in_error) {
if (a.cost < b.cost) {
return ErrorComparisonTakeLeft;
@@ -951,15 +953,15 @@ static bool ts_parser__do_all_potential_reductions(
switch (action.type) {
case TSParseActionTypeShift:
case TSParseActionTypeRecover:
if (!action.params.extra && !action.params.repetition) has_shift_action = true;
if (!action.params.shift.extra && !action.params.shift.repetition) has_shift_action = true;
break;
case TSParseActionTypeReduce:
if (action.params.child_count > 0)
if (action.params.reduce.child_count > 0)
ts_reduce_action_set_add(&self->reduce_actions, (ReduceAction){
.symbol = action.params.symbol,
.count = action.params.child_count,
.dynamic_precedence = action.params.dynamic_precedence,
.production_id = action.params.production_id,
.symbol = action.params.reduce.symbol,
.count = action.params.reduce.child_count,
.dynamic_precedence = action.params.reduce.dynamic_precedence,
.production_id = action.params.reduce.production_id,
});
default:
break;
@@ -1250,7 +1252,7 @@ static void ts_parser__recover(
// be counted in error cost calculations.
unsigned n;
const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) {
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.shift.extra) {
MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_extra(&mutable_lookahead);
lookahead = ts_subtree_from_mut(mutable_lookahead);
@@ -1379,9 +1381,9 @@ static bool ts_parser__advance(
switch (action.type) {
case TSParseActionTypeShift: {
if (action.params.repetition) break;
if (action.params.shift.repetition) break;
TSStateId next_state;
if (action.params.extra) {
if (action.params.shift.extra) {
// TODO: remove when TREE_SITTER_LANGUAGE_VERSION 9 is out.
if (state == ERROR_STATE) continue;
@@ -1389,7 +1391,7 @@ static bool ts_parser__advance(
next_state = state;
LOG("shift_extra");
} else {
next_state = action.params.state;
next_state = action.params.shift.state;
LOG("shift state:%u", next_state);
}
@@ -1398,7 +1400,7 @@ static bool ts_parser__advance(
next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead));
}
ts_parser__shift(self, version, next_state, lookahead, action.params.extra);
ts_parser__shift(self, version, next_state, lookahead, action.params.shift.extra);
if (did_reuse) reusable_node_advance(&self->reusable_node);
return true;
}
@@ -1406,10 +1408,10 @@ static bool ts_parser__advance(
case TSParseActionTypeReduce: {
bool is_fragile = table_entry.action_count > 1;
bool is_extra = lookahead.ptr == NULL;
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.symbol), action.params.child_count);
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.reduce.symbol), action.params.reduce.child_count);
StackVersion reduction_version = ts_parser__reduce(
self, version, action.params.symbol, action.params.child_count,
action.params.dynamic_precedence, action.params.production_id,
self, version, action.params.reduce.symbol, action.params.reduce.child_count,
action.params.reduce.dynamic_precedence, action.params.reduce.production_id,
is_fragile, is_extra
);
if (reduction_version != STACK_VERSION_NONE) {

50
src/tree_sitter/parser.h
View File

@@ -62,13 +62,13 @@ typedef struct {
TSStateId state;
bool extra : 1;
bool repetition : 1;
};
} shift;
struct {
TSSymbol symbol;
int16_t dynamic_precedence;
uint8_t child_count;
uint8_t production_id;
};
} reduce;
} params;
TSParseActionType type : 4;
} TSParseAction;
@@ -83,7 +83,7 @@ typedef union {
struct {
uint8_t count;
bool reusable : 1;
};
} entry;
} TSParseActionEntry;
struct TSLanguage {
@@ -167,22 +167,28 @@ struct TSLanguage {
#define ACTIONS(id) id
#define SHIFT(state_value) \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = {.state = state_value}, \
} \
#define SHIFT(state_value) \
{ \
{ \
.params = { \
.shift = { \
.state = state_value \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
#define SHIFT_REPEAT(state_value) \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = { \
.state = state_value, \
.repetition = true \
.shift = { \
.state = state_value, \
.repetition = true \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
@@ -194,20 +200,26 @@ struct TSLanguage {
#define SHIFT_EXTRA() \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = {.extra = true} \
.params = { \
.shift = { \
.extra = true \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
#define REDUCE(symbol_val, child_count_val, ...) \
{ \
{ \
.type = TSParseActionTypeReduce, \
.params = { \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
} \
.reduce = { \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
}, \
.type = TSParseActionTypeReduce \
} \
}

1021
src/tree_sitter/query.c
View File

File diff suppressed because it is too large Load Diff

2
src/tree_sitter/stack.c
View File

@@ -480,6 +480,7 @@ StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t c
}
inline StackAction pop_pending_callback(void *payload, const StackIterator *iterator) {
(void)payload;
if (iterator->subtree_count >= 1) {
if (iterator->is_pending) {
return StackActionPop | StackActionStop;
@@ -532,6 +533,7 @@ SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) {
}
inline StackAction pop_all_callback(void *payload, const StackIterator *iterator) {
(void)payload;
return iterator->node->link_count == 0 ? StackActionPop : StackActionNone;
}

12
src/tree_sitter/subtree.c
View File

@@ -21,7 +21,7 @@ typedef struct {
#define TS_MAX_INLINE_TREE_LENGTH UINT8_MAX
#define TS_MAX_TREE_POOL_SIZE 32
static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}};
static const ExternalScannerState empty_state = {{.short_data = {0}}, .length = 0};
// ExternalScannerState
@@ -208,7 +208,7 @@ Subtree ts_subtree_new_leaf(
.has_external_tokens = has_external_tokens,
.is_missing = false,
.is_keyword = is_keyword,
.first_leaf = {.symbol = 0, .parse_state = 0},
{{.first_leaf = {.symbol = 0, .parse_state = 0}}}
};
return (Subtree) {.ptr = data};
}
@@ -464,15 +464,17 @@ MutableSubtree ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol,
*data = (SubtreeHeapData) {
.ref_count = 1,
.symbol = symbol,
.production_id = production_id,
.visible = metadata.visible,
.named = metadata.named,
.has_changes = false,
.fragile_left = fragile,
.fragile_right = fragile,
.is_keyword = false,
.node_count = 0,
.first_leaf = {.symbol = 0, .parse_state = 0},
{{
.node_count = 0,
.production_id = production_id,
.first_leaf = {.symbol = 0, .parse_state = 0},
}}
};
MutableSubtree result = {.ptr = data};
ts_subtree_set_children(result, children->contents, children->size, language);

1
src/tree_sitter/treesitter_commit_hash.txt Normal file
View File

@@ -0,0 +1 @@
81d533d2d1b580fdb507accabc91ceddffb5b6f0

8
test/functional/lua/treesitter_spec.lua
View File

@@ -240,17 +240,17 @@ static int nlua_schedule(lua_State *const lstate)
; TODO(bfredl): overlapping matches are unreliable,
; we need a proper priority mechanism
;(type_identifier) @type
((type_identifier) @Special (eq? @Special "LuaRef"))
((type_identifier) @Special (#eq? @Special "LuaRef"))
(primitive_type) @type
(sized_type_specifier) @type
; defaults to very magic syntax, for best compatibility
((identifier) @Identifier (match? @Identifier "^l(u)a_"))
((identifier) @Identifier (#match? @Identifier "^l(u)a_"))
; still support \M etc prefixes
((identifier) @Constant (match? @Constant "\M^\[A-Z_]\+$"))
((identifier) @Constant (#match? @Constant "\M^\[A-Z_]\+$"))
((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right) (eq? @WarningMsg.left @WarningMsg.right))
((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right) (#eq? @WarningMsg.left @WarningMsg.right))
(comment) @comment
]]