mirrors/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2025-11-03 17:24:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

eval: Remove most of msgpack* functions limitations

Browse Source
This commit is contained in:
ZyX
2015-07-17 00:42:53 +03:00
parent 5a7135fa1c
commit 7fbefd585e
5 changed files with 1048 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
runtime/doc/eval.txt
View File

@@ -1517,6 +1517,12 @@ v:mouse_col Column number for a mouse click obtained with |getchar()|.
This is the screen column number, like with |virtcol()|. The
value is zero when there was no mouse button click.
*v:msgpack_types* *msgpack_types-variable*
v:msgpack_types Dictionary containing msgpack types used by |msgpackparse()|
and |msgpackdump()|. All types inside dictionary are fixed
(not editable) empty lists. To check whether some list is one
of msgpack types, use |is| operator.
*v:oldfiles* *oldfiles-variable*
v:oldfiles List of file names that is loaded from the |viminfo| file on
startup. These are the files that Vim remembers marks for.
@@ -4624,11 +4630,11 @@ msgpackdump({list}) *msgpackdump()*
messagepack).
Limitations:
1. |Funcref|s cannot be dumped as funcrefs, they are dumped as
NIL objects instead.
2. NIL and BOOL objects are never dumped, as well as objects
from EXT family.
3. Strings are always dumped as BIN strings.
1. |Funcref|s cannot be dumped.
2. Containers that reference themselves cannot be dumped.
3. Dictionary keys are always dumped as STR strings.
4. Other strings are always dumped as BIN strings.
5. Points 3. and 4. do not apply to |msgpack-special-dict|s.
msgpackparse({list}) *msgpackparse()*
Convert a |readfile()|-style list to a list of VimL objects.
@@ -4639,15 +4645,66 @@ msgpackparse({list}) *msgpackparse()*
< This will read |shada-file| to `shada_objects` list.
Limitations:
1. Strings that contain one of EXT format family objects
cannot be parsed by msgpackparse().
2. It may appear that parsed integers do not fit in |Number|
range. Even if your NeoVim installation uses 64-bit
Numbers, it may appear that string contain 64-bit unsigned
number above INT64_MAX.
3. NIL objects are parsed as zeroes. BOOL objects are parsed
as either 1 (true) or 0 (false).
4. BIN and STR strings cannot be distinguished after parsing.
1. Mapping ordering is not preserved unless messagepack
mapping is dumped using generic mapping
(|msgpack-special-map|).
2. Since the parser aims to preserve all data untouched
(except for 1.) some strings are parsed to
|msgpack-special-dict| format which is not convenient to
use.
*msgpack-special-dict*
Some messagepack strings may be parsed to special
dictionaries. Special dictionaries are dictionaries which
1. Contain exactly two keys: `_TYPE` and `_VALUE`.
2. `_TYPE` key is one of the types found in |v:msgpack_types|
variable.
3. Value for `_VALUE` has the following format (Key column
contains name of the key from |v:msgpack_types|):
Key Value ~
nil Zero, ignored when dumping.
boolean One or zero. When dumping it is only checked that
value is a |Number|.
integer |List| with four numbers: sign (-1 or 1), highest two
bits, number with bits from 62nd to 31st, lowest 31
bits. I.e. to get actual number one will need to use
code like >
_VALUE[0] * ((_VALUE[1] << 62)
& (_VALUE[2] << 31)
& _VALUE[3])
< Special dictionary with this type will appear in
|msgpackparse()| output under one of the following
circumstances:
1. |Number| is 32-bit and value is either above
INT32_MAX or below INT32_MIN.
2. |Number| is 64-bit and value is above INT64_MAX. It
cannot possibly be below INT64_MIN because msgpack
C parser does not support such values.
float |Float|. This value cannot possibly appear in
|msgpackparse()| output.
string |readfile()|-style list of strings. This value will
appear in |msgpackparse()| output if string contains
zero byte or if string is a mapping key and mapping is
being represented as special dictionary for other
reasons.
binary |readfile()|-style list of strings. This value will
appear in |msgpackparse()| output if binary string
contains zero byte.
array |List|. This value cannot appear in |msgpackparse()|
output.
*msgpack-special-map*
map |List| of |List|s with two items (key and value) each.
This value will appear in |msgpackparse()| output if
parsed mapping contains one of the following keys:
1. Any key that is not a string (including keys which
are binary strings).
2. String with NUL byte inside.
3. Duplicate key.
4. Empty key.
ext |List| with two values: first is a signed integer
representing extension type. Second is
|readfile()|-style list of strings.
nextnonblank({lnum}) *nextnonblank()*
Return the line number of the first line at or below {lnum}

736
src/nvim/eval.c
View File

@@ -97,6 +97,7 @@
#include "nvim/os/dl.h"
#include "nvim/os/input.h"
#include "nvim/event/loop.h"
#include "nvim/lib/kvec.h"
#define DICT_MAXNEST 100 /* maximum nesting of lists and dicts */
@@ -439,6 +440,7 @@ static struct vimvar {
{VV_NAME("progpath", VAR_STRING), VV_RO},
{VV_NAME("command_output", VAR_STRING), 0},
{VV_NAME("completed_item", VAR_DICT), VV_RO},
{VV_NAME("msgpack_types", VAR_DICT), VV_RO},
};
/* shorthand */
@@ -469,6 +471,29 @@ typedef struct {
uint64_t id;
} TerminalJobData;
/// Structure representing current VimL to messagepack conversion state
typedef struct {
enum {
kMPConvDict, ///< Convert dict_T *dictionary.
kMPConvList, ///< Convert list_T *list.
kMPConvPairs, ///< Convert mapping represented as a list_T* of pairs.
} type;
union {
struct {
const dict_T *dict; ///< Currently converted dictionary.
const hashitem_T *hi; ///< Currently converted dictionary item.
size_t todo; ///< Amount of items left to process.
} d; ///< State of dictionary conversion.
struct {
const list_T *list; ///< Currently converted list.
const listitem_T *li; ///< Currently converted list item.
} l; ///< State of list or generic mapping conversion.
} data; ///< Data to convert.
} MPConvStackVal;
/// Stack used to convert VimL values to messagepack.
typedef kvec_t(MPConvStackVal) MPConvStack;
#ifdef INCLUDE_GENERATED_DECLARATIONS
# include "eval.c.generated.h"
@@ -489,6 +514,40 @@ static int disable_job_defer = 0;
static uint64_t current_job_id = 1;
static PMap(uint64_t) *jobs = NULL;
typedef enum {
kMPNil,
kMPBoolean,
kMPInteger,
kMPFloat,
kMPString,
kMPBinary,
kMPArray,
kMPMap,
kMPExt,
} MessagePackType;
static const char *const msgpack_type_names[] = {
[kMPNil] = "nil",
[kMPBoolean] = "boolean",
[kMPInteger] = "integer",
[kMPFloat] = "float",
[kMPString] = "string",
[kMPBinary] = "binary",
[kMPArray] = "array",
[kMPMap] = "map",
[kMPExt] = "ext",
};
static const list_T *msgpack_type_lists[] = {
[kMPNil] = NULL,
[kMPBoolean] = NULL,
[kMPInteger] = NULL,
[kMPFloat] = NULL,
[kMPString] = NULL,
[kMPBinary] = NULL,
[kMPArray] = NULL,
[kMPMap] = NULL,
[kMPExt] = NULL,
};
/*
* Initialize the global and v: variables.
*/
@@ -521,6 +580,26 @@ void eval_init(void)
/* add to compat scope dict */
hash_add(&compat_hashtab, p->vv_di.di_key);
}
dict_T *const msgpack_types_dict = dict_alloc();
for (size_t i = 0; i < ARRAY_SIZE(msgpack_type_names); i++) {
list_T *const type_list = list_alloc();
type_list->lv_lock = VAR_FIXED;
dictitem_T *const di = dictitem_alloc((char_u *) msgpack_type_names[i]);
di->di_flags = DI_FLAGS_RO | DI_FLAGS_FIX;
di->di_tv = (typval_T) {
.v_type = VAR_LIST,
.vval = { .v_list = type_list, },
};
msgpack_type_lists[i] = type_list;
if (dict_add(msgpack_types_dict, di) == FAIL) {
// There must not be duplicate items in this dictionary by definition.
assert(false);
}
}
msgpack_types_dict->dv_lock = VAR_FIXED;
set_vim_var_dict(VV_MSGPACK_TYPES, msgpack_types_dict);
set_vim_var_dict(VV_COMPLETED_ITEM, dict_alloc());
set_vim_var_nr(VV_SEARCHFORWARD, 1L);
set_vim_var_nr(VV_HLSEARCH, 1L);
@@ -11921,12 +12000,389 @@ static int msgpack_list_write(void *data, const char *buf, size_t len)
return 0;
}
/// Convert readfile()-style list to a char * buffer with length
///
/// @param[in] list Converted list.
/// @param[out] ret_len Resulting buffer length.
/// @param[out] ret_buf Allocated buffer with the result or NULL if ret_len is
/// zero.
///
/// @return true in case of success, false in case of failure.
static inline bool vim_list_to_buf(const list_T *const list,
size_t *const ret_len, char **const ret_buf)
FUNC_ATTR_NONNULL_ARG(2,3) FUNC_ATTR_WARN_UNUSED_RESULT
{
size_t len = 0;
if (list != NULL) {
for (const listitem_T *li = list->lv_first;
li != NULL;
li = li->li_next) {
if (li->li_tv.v_type != VAR_STRING) {
return false;
}
len++;
if (li->li_tv.vval.v_string != 0) {
len += STRLEN(li->li_tv.vval.v_string);
}
}
if (len) {
len--;
}
}
*ret_len = len;
if (len == 0) {
*ret_buf = NULL;
return true;
}
ListReaderState lrstate = init_lrstate(list);
char *const buf = xmalloc(len);
size_t read_bytes;
if (read_from_list(&lrstate, buf, len, &read_bytes) != OK) {
assert(false);
}
assert(len == read_bytes);
*ret_buf = buf;
return true;
}
/// Convert one VimL value to msgpack
///
/// @param packer Messagepack packer.
/// @param[out] mpstack Stack with values to convert. Only used for pushing
/// values to it.
/// @param[in] tv Converted value.
///
/// @return OK in case of success, FAIL otherwise.
static int convert_one_value(msgpack_packer *const packer,
MPConvStack *const mpstack,
const typval_T *const tv)
FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
{
switch (tv->v_type) {
#define CHECK_SELF_REFERENCE(conv_type, vval_name, ptr) \
do { \
for (size_t i = 0; i < kv_size(*mpstack); i++) { \
if (kv_A(*mpstack, i).type == conv_type \
&& kv_A(*mpstack, i).data.vval_name == ptr) { \
EMSG2(_(e_invarg2), "container references itself"); \
return FAIL; \
} \
} \
} while (0)
case VAR_STRING: {
if (tv->vval.v_string == NULL) {
msgpack_pack_bin(packer, 0);
} else {
const size_t len = STRLEN(tv->vval.v_string);
msgpack_pack_bin(packer, len);
msgpack_pack_bin_body(packer, tv->vval.v_string, len);
}
break;
}
case VAR_NUMBER: {
msgpack_pack_int64(packer, (int64_t) tv->vval.v_number);
break;
}
case VAR_FLOAT: {
msgpack_pack_double(packer, (double) tv->vval.v_float);
break;
}
case VAR_FUNC: {
EMSG2(_(e_invarg2), "attempt to dump function reference");
return FAIL;
}
case VAR_LIST: {
if (tv->vval.v_list == NULL) {
msgpack_pack_array(packer, 0);
break;
}
CHECK_SELF_REFERENCE(kMPConvList, l.list, tv->vval.v_list);
msgpack_pack_array(packer, tv->vval.v_list->lv_len);
kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) {
.type = kMPConvList,
.data = {
.l = {
.list = tv->vval.v_list,
.li = tv->vval.v_list->lv_first,
},
},
}));
break;
}
case VAR_DICT: {
if (tv->vval.v_dict == NULL) {
msgpack_pack_map(packer, 0);
break;
}
const dictitem_T *type_di;
const dictitem_T *val_di;
if (tv->vval.v_dict->dv_hashtab.ht_used == 2
&& (type_di = dict_find((dict_T *) tv->vval.v_dict,
(char_u *) "_TYPE", -1)) != NULL
&& type_di->di_tv.v_type == VAR_LIST
&& (val_di = dict_find((dict_T *) tv->vval.v_dict,
(char_u *) "_VAL", -1)) != NULL) {
size_t i;
for (i = 0; i < ARRAY_SIZE(msgpack_type_lists); i++) {
if (type_di->di_tv.vval.v_list == msgpack_type_lists[i]) {
break;
}
}
if (i == ARRAY_SIZE(msgpack_type_lists)) {
goto vim_to_msgpack_regural_dict;
}
switch ((MessagePackType) i) {
case kMPNil: {
msgpack_pack_nil(packer);
break;
}
case kMPBoolean: {
if (val_di->di_tv.v_type != VAR_NUMBER) {
goto vim_to_msgpack_regural_dict;
}
if (val_di->di_tv.vval.v_number) {
msgpack_pack_true(packer);
} else {
msgpack_pack_false(packer);
}
break;
}
case kMPInteger: {
const list_T *val_list;
varnumber_T sign;
varnumber_T highest_bits;
varnumber_T high_bits;
varnumber_T low_bits;
// List of 4 integers; first is signed (should be 1 or -1, but this
// is not checked), second is unsigned and have at most one (sign is
// -1) or two (sign is 1) non-zero bits (number of bits is not
// checked), other unsigned and have at most 31 non-zero bits
// (number of bits is not checked).
if (val_di->di_tv.v_type != VAR_LIST
|| (val_list = val_di->di_tv.vval.v_list) == NULL
|| val_list->lv_len != 4
|| val_list->lv_first->li_tv.v_type != VAR_NUMBER
|| (sign = val_list->lv_first->li_tv.vval.v_number) == 0
|| val_list->lv_first->li_next->li_tv.v_type != VAR_NUMBER
|| (highest_bits =
val_list->lv_first->li_next->li_tv.vval.v_number) < 0
|| val_list->lv_last->li_prev->li_tv.v_type != VAR_NUMBER
|| (high_bits =
val_list->lv_last->li_prev->li_tv.vval.v_number) < 0
|| val_list->lv_last->li_tv.v_type != VAR_NUMBER
|| (low_bits = val_list->lv_last->li_tv.vval.v_number) < 0) {
goto vim_to_msgpack_regural_dict;
}
uint64_t number = ((uint64_t) (((uint64_t) highest_bits) << 62)
| (uint64_t) (((uint64_t) high_bits) << 31)
| (uint64_t) low_bits);
if (sign > 0) {
msgpack_pack_uint64(packer, number);
} else {
msgpack_pack_int64(packer, (int64_t) (-number));
}
break;
}
case kMPFloat: {
if (val_di->di_tv.v_type != VAR_FLOAT) {
goto vim_to_msgpack_regural_dict;
}
msgpack_pack_double(packer, val_di->di_tv.vval.v_float);
break;
}
case kMPString:
case kMPBinary: {
const bool is_string = ((MessagePackType) i == kMPString);
if (val_di->di_tv.v_type != VAR_LIST) {
goto vim_to_msgpack_regural_dict;
}
size_t len;
char *buf;
if (!vim_list_to_buf(val_di->di_tv.vval.v_list, &len, &buf)) {
goto vim_to_msgpack_regural_dict;
}
if (is_string) {
msgpack_pack_str(packer, len);
} else {
msgpack_pack_bin(packer, len);
}
if (len == 0) {
break;
}
if (is_string) {
msgpack_pack_str_body(packer, buf, len);
} else {
msgpack_pack_bin_body(packer, buf, len);
}
xfree(buf);
break;
}
case kMPArray: {
if (val_di->di_tv.v_type != VAR_LIST) {
goto vim_to_msgpack_regural_dict;
}
CHECK_SELF_REFERENCE(kMPConvList, l.list,
val_di->di_tv.vval.v_list);
msgpack_pack_array(packer, val_di->di_tv.vval.v_list->lv_len);
kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) {
.type = kMPConvList,
.data = {
.l = {
.list = val_di->di_tv.vval.v_list,
.li = val_di->di_tv.vval.v_list->lv_first,
},
},
}));
break;
}
case kMPMap: {
if (val_di->di_tv.v_type != VAR_LIST) {
goto vim_to_msgpack_regural_dict;
}
if (val_di->di_tv.vval.v_list == NULL) {
msgpack_pack_map(packer, 0);
break;
}
const list_T *const val_list = val_di->di_tv.vval.v_list;
for (const listitem_T *li = val_list->lv_first; li != NULL;
li = li->li_next) {
if (li->li_tv.v_type != VAR_LIST
|| li->li_tv.vval.v_list->lv_len != 2) {
goto vim_to_msgpack_regural_dict;
}
}
CHECK_SELF_REFERENCE(kMPConvPairs, l.list, val_list);
msgpack_pack_map(packer, val_list->lv_len);
kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) {
.type = kMPConvPairs,
.data = {
.l = {
.list = val_list,
.li = val_list->lv_first,
},
},
}));
break;
}
case kMPExt: {
const list_T *val_list;
varnumber_T type;
if (val_di->di_tv.v_type != VAR_LIST
|| (val_list = val_di->di_tv.vval.v_list) == NULL
|| val_list->lv_len != 2
|| (val_list->lv_first->li_tv.v_type != VAR_NUMBER)
|| (type = val_list->lv_first->li_tv.vval.v_number) > INT8_MAX
|| type < INT8_MIN
|| (val_list->lv_last->li_tv.v_type != VAR_LIST)) {
goto vim_to_msgpack_regural_dict;
}
size_t len;
char *buf;
if (!vim_list_to_buf(val_list->lv_last->li_tv.vval.v_list,
&len, &buf)) {
goto vim_to_msgpack_regural_dict;
}
msgpack_pack_ext(packer, len, (int8_t) type);
msgpack_pack_ext_body(packer, buf, len);
xfree(buf);
break;
}
}
break;
}
vim_to_msgpack_regural_dict:
CHECK_SELF_REFERENCE(kMPConvDict, d.dict, tv->vval.v_dict);
msgpack_pack_map(packer, tv->vval.v_dict->dv_hashtab.ht_used);
kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) {
.type = kMPConvDict,
.data = {
.d = {
.dict = tv->vval.v_dict,
.hi = tv->vval.v_dict->dv_hashtab.ht_array,
.todo = tv->vval.v_dict->dv_hashtab.ht_used,
},
},
}));
break;
}
}
#undef CHECK_SELF_REFERENCE
return OK;
}
/// Convert typval_T to messagepack
static int vim_to_msgpack(msgpack_packer *const packer,
const typval_T *const tv)
FUNC_ATTR_WARN_UNUSED_RESULT
{
MPConvStack mpstack;
kv_init(mpstack);
if (convert_one_value(packer, &mpstack, tv) == FAIL) {
goto vim_to_msgpack_error_ret;
}
while (kv_size(mpstack)) {
MPConvStackVal *cur_mpsv = &kv_A(mpstack, kv_size(mpstack) - 1);
const typval_T *cur_tv;
switch (cur_mpsv->type) {
case kMPConvDict: {
if (!cur_mpsv->data.d.todo) {
(void) kv_pop(mpstack);
continue;
}
while (HASHITEM_EMPTY(cur_mpsv->data.d.hi)) {
cur_mpsv->data.d.hi++;
}
const dictitem_T *const di = HI2DI(cur_mpsv->data.d.hi);
cur_mpsv->data.d.todo--;
cur_mpsv->data.d.hi++;
const size_t key_len = STRLEN(&di->di_key[0]);
msgpack_pack_str(packer, key_len);
msgpack_pack_str_body(packer, &di->di_key[0], key_len);
cur_tv = &di->di_tv;
break;
}
case kMPConvList: {
if (cur_mpsv->data.l.li == NULL) {
(void) kv_pop(mpstack);
continue;
}
cur_tv = &cur_mpsv->data.l.li->li_tv;
cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next;
break;
}
case kMPConvPairs: {
if (cur_mpsv->data.l.li == NULL) {
(void) kv_pop(mpstack);
continue;
}
const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list;
if (convert_one_value(packer, &mpstack, &kv_pair->lv_first->li_tv)
== FAIL) {
goto vim_to_msgpack_error_ret;
}
cur_tv = &kv_pair->lv_last->li_tv;
cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next;
break;
}
}
if (convert_one_value(packer, &mpstack, cur_tv) == FAIL) {
goto vim_to_msgpack_error_ret;
}
}
kv_destroy(mpstack);
return OK;
vim_to_msgpack_error_ret:
kv_destroy(mpstack);
return FAIL;
}
/// "msgpackdump()" function
static void f_msgpackdump(typval_T *argvars, typval_T *rettv)
FUNC_ATTR_NONNULL_ALL
{
if (argvars[0].v_type != VAR_LIST) {
EMSG2(_(e_listarg), "msgpackdump()");
return;
}
list_T *ret_list = rettv_list_alloc(rettv);
const list_T *list = argvars[0].vval.v_list;
@@ -11935,9 +12391,9 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv)
}
msgpack_packer *lpacker = msgpack_packer_new(ret_list, &msgpack_list_write);
for (const listitem_T *li = list->lv_first; li != NULL; li = li->li_next) {
Object obj = vim_to_object((typval_T *) &li->li_tv);
msgpack_rpc_from_object(obj, lpacker);
api_free_object(obj);
if (vim_to_msgpack(lpacker, &li->li_tv) == FAIL) {
break;
}
}
msgpack_packer_free(lpacker);
}
@@ -11983,7 +12439,257 @@ static int read_from_list(ListReaderState *const state, char *const buf,
}
}
*read_bytes = nbuf;
return NOTDONE;
return (state->offset < state->li_length || state->li->li_next != NULL
? NOTDONE
: OK);
}
/// Initialize ListReaderState structure
static inline ListReaderState init_lrstate(const list_T *const list)
FUNC_ATTR_NONNULL_ALL
{
return (ListReaderState) {
.li = list->lv_first,
.offset = 0,
.li_length = (list->lv_first->li_tv.vval.v_string == NULL
? 0
: STRLEN(list->lv_first->li_tv.vval.v_string)),
};
}
/// Convert msgpack object to a VimL one
static int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv)
FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
{
#define INIT_SPECIAL_DICT(tv, type, val) \
do { \
dict_T *const dict = dict_alloc(); \
dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); \
type_di->di_tv.v_type = VAR_LIST; \
type_di->di_tv.v_lock = 0; \
type_di->di_tv.vval.v_list = (list_T *) msgpack_type_lists[type]; \
type_di->di_tv.vval.v_list->lv_refcount++; \
dict_add(dict, type_di); \
dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); \
val_di->di_tv = val; \
dict_add(dict, val_di); \
tv->v_type = VAR_DICT; \
dict->dv_refcount++; \
tv->vval.v_dict = dict; \
} while (0)
switch (mobj.type) {
case MSGPACK_OBJECT_NIL: {
INIT_SPECIAL_DICT(rettv, kMPNil, ((typval_T) {
.v_type = VAR_NUMBER,
.v_lock = 0,
.vval = { .v_number = 0 },
}));
break;
}
case MSGPACK_OBJECT_BOOLEAN: {
INIT_SPECIAL_DICT(rettv, kMPBoolean,
((typval_T) {
.v_type = VAR_NUMBER,
.v_lock = 0,
.vval = {
.v_number = (varnumber_T) mobj.via.boolean,
},
}));
break;
}
case MSGPACK_OBJECT_POSITIVE_INTEGER: {
if (mobj.via.u64 <= VARNUMBER_MAX) {
*rettv = (typval_T) {
.v_type = VAR_NUMBER,
.v_lock = 0,
.vval = { .v_number = (varnumber_T) mobj.via.u64 },
};
} else {
list_T *const list = list_alloc();
list->lv_refcount++;
INIT_SPECIAL_DICT(rettv, kMPInteger,
((typval_T) {
.v_type = VAR_LIST,
.v_lock = 0,
.vval = { .v_list = list },
}));
uint64_t n = mobj.via.u64;
list_append_number(list, 1);
list_append_number(list, (varnumber_T) ((n >> 62) & 0x3));
list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF));
list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF));
}
break;
}
case MSGPACK_OBJECT_NEGATIVE_INTEGER: {
if (mobj.via.i64 >= VARNUMBER_MIN) {
*rettv = (typval_T) {
.v_type = VAR_NUMBER,
.v_lock = 0,
.vval = { .v_number = (varnumber_T) mobj.via.i64 },
};
} else {
list_T *const list = list_alloc();
list->lv_refcount++;
INIT_SPECIAL_DICT(rettv, kMPInteger,
((typval_T) {
.v_type = VAR_LIST,
.v_lock = 0,
.vval = { .v_list = list },
}));
uint64_t n = -((uint64_t) mobj.via.i64);
list_append_number(list, -1);
list_append_number(list, (varnumber_T) ((n >> 62) & 0x3));
list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF));
list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF));
}
break;
}
case MSGPACK_OBJECT_FLOAT: {
*rettv = (typval_T) {
.v_type = VAR_FLOAT,
.v_lock = 0,
.vval = { .v_float = mobj.via.f64 },
};
break;
}
case MSGPACK_OBJECT_STR: {
list_T *const list = list_alloc();
list->lv_refcount++;
INIT_SPECIAL_DICT(rettv, kMPString,
((typval_T) {
.v_type = VAR_LIST,
.v_lock = 0,
.vval = { .v_list = list },
}));
if (msgpack_list_write((void *) list, mobj.via.str.ptr, mobj.via.str.size)
== -1) {
return FAIL;
}
break;
}
case MSGPACK_OBJECT_BIN: {
if (memchr(mobj.via.bin.ptr, NUL, mobj.via.bin.size) == NULL) {
*rettv = (typval_T) {
.v_type = VAR_STRING,
.v_lock = 0,
.vval = { .v_string = xmemdupz(mobj.via.bin.ptr, mobj.via.bin.size) },
};
break;
}
list_T *const list = list_alloc();
list->lv_refcount++;
INIT_SPECIAL_DICT(rettv, kMPBinary,
((typval_T) {
.v_type = VAR_LIST,
.v_lock = 0,
.vval = { .v_list = list },
}));
if (msgpack_list_write((void *) list, mobj.via.bin.ptr, mobj.via.bin.size)
== -1) {
return FAIL;
}
break;
}
case MSGPACK_OBJECT_ARRAY: {
list_T *const list = list_alloc();
list->lv_refcount++;
*rettv = (typval_T) {
.v_type = VAR_LIST,
.v_lock = 0,
.vval = { .v_list = list },
};
for (size_t i = 0; i < mobj.via.array.size; i++) {
listitem_T *const li = listitem_alloc();
li->li_tv.v_type = VAR_UNKNOWN;
list_append(list, li);
if (msgpack_to_vim(mobj.via.array.ptr[i], &li->li_tv) == FAIL) {
return FAIL;
}
}
break;
}
case MSGPACK_OBJECT_MAP: {
for (size_t i = 0; i < mobj.via.map.size; i++) {
if (mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR
|| mobj.via.map.ptr[i].key.via.str.size == 0
|| memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL,
mobj.via.map.ptr[i].key.via.str.size) != NULL) {
goto msgpack_to_vim_generic_map;
}
}
dict_T *const dict = dict_alloc();
dict->dv_refcount++;
*rettv = (typval_T) {
.v_type = VAR_DICT,
.v_lock = 0,
.vval = { .v_dict = dict },
};
for (size_t i = 0; i < mobj.via.map.size; i++) {
dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key)
+ mobj.via.map.ptr[i].key.via.str.size);
memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr,
mobj.via.map.ptr[i].key.via.str.size);
di->di_tv.v_type = VAR_UNKNOWN;
if (dict_add(dict, di) == FAIL) {
// Duplicate key: fallback to generic map
clear_tv(rettv);
xfree(di);
goto msgpack_to_vim_generic_map;
}
if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) {
return FAIL;
}
}
break;
msgpack_to_vim_generic_map: {}
list_T *const list = list_alloc();
list->lv_refcount++;
INIT_SPECIAL_DICT(rettv, kMPMap,
((typval_T) {
.v_type = VAR_LIST,
.v_lock = 0,
.vval = { .v_list = list },
}));
for (size_t i = 0; i < mobj.via.map.size; i++) {
list_T *const kv_pair = list_alloc();
list_append_list(list, kv_pair);
listitem_T *const key_li = listitem_alloc();
key_li->li_tv.v_type = VAR_UNKNOWN;
list_append(kv_pair, key_li);
listitem_T *const val_li = listitem_alloc();
val_li->li_tv.v_type = VAR_UNKNOWN;
list_append(kv_pair, val_li);
if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_li->li_tv) == FAIL) {
return FAIL;
}
if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_li->li_tv) == FAIL) {
return FAIL;
}
}
break;
}
case MSGPACK_OBJECT_EXT: {
list_T *const list = list_alloc();
list->lv_refcount++;
list_append_number(list, mobj.via.ext.type);
list_T *const ext_val_list = list_alloc();
list_append_list(list, ext_val_list);
INIT_SPECIAL_DICT(rettv, kMPExt,
((typval_T) {
.v_type = VAR_LIST,
.v_lock = 0,
.vval = { .v_list = list },
}));
if (msgpack_list_write((void *) ext_val_list, mobj.via.ext.ptr,
mobj.via.ext.size) == -1) {
return FAIL;
}
break;
}
}
#undef INIT_SPECIAL_DICT
return OK;
}
/// "msgpackparse" function
@@ -12002,13 +12708,7 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv)
EMSG2(_(e_invarg2), "List item is not a string");
return;
}
ListReaderState lrstate = {
.li = list->lv_first,
.offset = 0,
.li_length = (list->lv_first->li_tv.vval.v_string == NULL
? 0
: STRLEN(list->lv_first->li_tv.vval.v_string)),
};
ListReaderState lrstate = init_lrstate(list);
msgpack_unpacker *const unpacker = msgpack_unpacker_new(IOSIZE);
if (unpacker == NULL) {
EMSG(_(e_outofmem));
@@ -12044,19 +12744,13 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv)
goto f_msgpackparse_exit;
}
if (result == MSGPACK_UNPACK_SUCCESS) {
Object obj;
if (!msgpack_rpc_to_object(&unpacked.data, &obj)) {
EMSG2(_(e_invarg2), "Failed to convert parsed string to Object");
goto f_msgpackparse_exit;
}
listitem_T *li = listitem_alloc();
Error err;
if (!object_to_vim(obj, &li->li_tv, &err)) {
EMSG2(_(e_invarg2), err.msg);
li->li_tv.v_type = VAR_UNKNOWN;
list_append(ret_list, li);
if (msgpack_to_vim(unpacked.data, &li->li_tv) == FAIL) {
EMSG2(_(e_invarg2), "Failed to convert msgpack string");
goto f_msgpackparse_exit;
}
list_append(ret_list, li);
api_free_object(obj);
}
if (result == MSGPACK_UNPACK_CONTINUE) {
if (rlret == OK) {

1
src/nvim/eval.h
View File

@@ -65,6 +65,7 @@ enum {
VV_PROGPATH,
VV_COMMAND_OUTPUT,
VV_COMPLETED_ITEM,
VV_MSGPACK_TYPES,
VV_LEN, /* number of v: vars */
};

5
src/nvim/eval_defs.h
View File

@@ -1,11 +1,16 @@
#ifndef NVIM_EVAL_DEFS_H
#define NVIM_EVAL_DEFS_H
#include <limits.h>
#include "nvim/hashtab.h"
typedef int varnumber_T;
typedef double float_T;
#define VARNUMBER_MAX INT_MAX
#define VARNUMBER_MIN INT_MIN
typedef struct listvar_S list_T;
typedef struct dictvar_S dict_T;

264
test/functional/viml/msgpack_functions_spec.lua
View File

@@ -325,20 +325,27 @@ describe('msgpack*() functions', function()
}
obj_test('are able to dump and restore rather big object', big_obj)
it('dump funcref as nil and restore as zero', function()
execute('let dumped = msgpackdump([function("tr")])')
eq({"\192"}, eval('dumped'))
eq({0}, eval('msgpackparse(dumped)'))
obj_test('are able to dump and restore floating-point value', {0.125})
it('restore nil as special dict', function()
execute('let dumped = ["\\xC0"]')
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL=0}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.nil'))
end)
it('restore boolean false as zero', function()
execute('let dumped = ["\\xC2"]')
eq({0}, eval('msgpackparse(dumped)'))
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL=0}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.boolean'))
end)
it('restore boolean true as one', function()
execute('let dumped = ["\\xC3"]')
eq({1}, eval('msgpackparse(dumped)'))
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL=1}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.boolean'))
end)
it('dump string as BIN 8', function()
@@ -346,13 +353,254 @@ describe('msgpack*() functions', function()
eq({"\196\004Test"}, eval('msgpackdump(obj)'))
end)
it('restore FIXSTR as string', function()
it('restore FIXSTR as special dict', function()
execute('let dumped = ["\\xa2ab"]')
eq({'ab'}, eval('msgpackparse(dumped)'))
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL={'ab'}}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.string'))
end)
it('restore BIN 8 as string', function()
execute('let dumped = ["\\xC4\\x02ab"]')
eq({'ab'}, eval('msgpackparse(dumped)'))
end)
it('restore FIXEXT1 as special dictionary', function()
execute('let dumped = ["\\xD4\\x10", ""]')
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.ext'))
end)
it('restore MAP with BIN key as special dictionary', function()
execute('let dumped = ["\\x81\\xC4\\x01a\\xC4\\n"]')
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL={{'a', ''}}}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map'))
end)
it('restore MAP with duplicate STR keys as special dictionary', function()
execute('let dumped = ["\\x82\\xA1a\\xC4\\n\\xA1a\\xC4\\n"]')
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'a'}}, ''},
{{_TYPE={}, _VAL={'a'}}, ''}}}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map'))
eq(1, eval('g:parsed[0]._VAL[0][0]._TYPE is v:msgpack_types.string'))
eq(1, eval('g:parsed[0]._VAL[1][0]._TYPE is v:msgpack_types.string'))
end)
it('restore MAP with MAP key as special dictionary', function()
execute('let dumped = ["\\x81\\x80\\xC4\\n"]')
execute('let parsed = msgpackparse(dumped)')
eq({{_TYPE={}, _VAL={{{}, ''}}}}, eval('parsed'))
eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map'))
end)
it('can restore and dump UINT64_MAX', function()
execute('let dumped = ["\\xCF" . repeat("\\xFF", 8)]')
execute('let parsed = msgpackparse(dumped)')
execute('let dumped2 = msgpackdump(parsed)')
eq(1, eval('type(parsed[0]) == type(0) ' ..
'|| parsed[0]._TYPE is v:msgpack_types.integer'))
if eval('type(parsed[0]) == type(0)') == 1 then
eq(1, eval('0xFFFFFFFFFFFFFFFF == parsed[0]'))
else
eq({_TYPE={}, _VAL={1, 3, 0x7FFFFFFF, 0x7FFFFFFF}}, eval('parsed[0]'))
end
eq(1, eval('dumped ==# dumped2'))
end)
it('can restore and dump INT64_MIN', function()
execute('let dumped = ["\\xD3\\x80" . repeat("\\n", 7)]')
execute('let parsed = msgpackparse(dumped)')
execute('let dumped2 = msgpackdump(parsed)')
eq(1, eval('type(parsed[0]) == type(0) ' ..
'|| parsed[0]._TYPE is v:msgpack_types.integer'))
if eval('type(parsed[0]) == type(0)') == 1 then
eq(1, eval('-0x8000000000000000 == parsed[0]'))
else
eq({_TYPE={}, _VAL={-1, 2, 0, 0}}, eval('parsed[0]'))
end
eq(1, eval('dumped ==# dumped2'))
end)
it('can restore and dump BIN string with zero byte', function()
execute('let dumped = ["\\xC4\\x01\\n"]')
execute('let parsed = msgpackparse(dumped)')
execute('let dumped2 = msgpackdump(parsed)')
eq({{_TYPE={}, _VAL={'\n'}}}, eval('parsed'))
eq(1, eval('parsed[0]._TYPE is v:msgpack_types.binary'))
eq(1, eval('dumped ==# dumped2'))
end)
it('can restore and dump STR string with zero byte', function()
execute('let dumped = ["\\xA1\\n"]')
execute('let parsed = msgpackparse(dumped)')
execute('let dumped2 = msgpackdump(parsed)')
eq({{_TYPE={}, _VAL={'\n'}}}, eval('parsed'))
eq(1, eval('parsed[0]._TYPE is v:msgpack_types.string'))
eq(1, eval('dumped ==# dumped2'))
end)
it('can restore and dump BIN string with NL', function()
execute('let dumped = ["\\xC4\\x01", ""]')
execute('let parsed = msgpackparse(dumped)')
execute('let dumped2 = msgpackdump(parsed)')
eq({"\n"}, eval('parsed'))
eq(1, eval('dumped ==# dumped2'))
end)
it('can dump generic mapping with generic mapping keys and values', function()
execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": []}')
execute('let todumpv1 = {"_TYPE": v:msgpack_types.map, "_VAL": []}')
execute('let todumpv2 = {"_TYPE": v:msgpack_types.map, "_VAL": []}')
execute('call add(todump._VAL, [todumpv1, todumpv2])')
eq({'\129\128\128'}, eval('msgpackdump([todump])'))
end)
it('can dump generic mapping with ext', function()
execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}')
eq({'\212\005', ''}, eval('msgpackdump([todump])'))
end)
it('can dump generic mapping with array', function()
execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}')
eq({'\146\005\145\196\n'}, eval('msgpackdump([todump])'))
end)
it('can dump generic mapping with UINT64_MAX', function()
execute('let todump = {"_TYPE": v:msgpack_types.integer}')
execute('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]')
eq({'\207\255\255\255\255\255\255\255\255'}, eval('msgpackdump([todump])'))
end)
it('can dump generic mapping with INT64_MIN', function()
execute('let todump = {"_TYPE": v:msgpack_types.integer}')
execute('let todump._VAL = [-1, 2, 0, 0]')
eq({'\211\128\n\n\n\n\n\n\n'}, eval('msgpackdump([todump])'))
end)
it('dump and restore generic mapping with floating-point value', function()
execute('let todump = {"_TYPE": v:msgpack_types.float, "_VAL": 0.125}')
eq({0.125}, eval('msgpackparse(msgpackdump([todump]))'))
end)
it('fails to dump a function reference', function()
execute('let Todump = function("tr")')
execute([[
try
let dumped = msgpackdump([Todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: attempt to dump function reference',
eval('exception'))
end)
it('fails to dump a function reference in a list', function()
execute('let todump = [function("tr")]')
execute([[
try
let dumped = msgpackdump([todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: attempt to dump function reference',
eval('exception'))
end)
it('fails to dump a recursive list', function()
execute('let todump = [[[]]]')
execute('call add(todump[0][0], todump)')
execute([[
try
let dumped = msgpackdump([todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: container references itself',
eval('exception'))
end)
it('fails to dump a recursive dict', function()
execute('let todump = {"d": {"d": {}}}')
execute('call extend(todump.d.d, {"d": todump})')
execute([[
try
let dumped = msgpackdump([todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: container references itself',
eval('exception'))
end)
it('fails to dump a recursive list in a special dict', function()
execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}')
execute('call add(todump._VAL, todump)')
execute([[
try
let dumped = msgpackdump([todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: container references itself',
eval('exception'))
end)
it('fails to dump a recursive (key) map in a special dict', function()
execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}')
execute('call add(todump._VAL, [todump, 0])')
execute([[
try
let dumped = msgpackdump([todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: container references itself',
eval('exception'))
end)
it('fails to dump a recursive (val) map in a special dict', function()
execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}')
execute('call add(todump._VAL, [0, todump])')
execute([[
try
let dumped = msgpackdump([todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: container references itself',
eval('exception'))
end)
it('fails to dump a recursive (val) special list in a special dict',
function()
execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}')
execute('call add(todump._VAL, [0, todump._VAL])')
execute([[
try
let dumped = msgpackdump([todump])
let exception = 0
catch
let exception = v:exception
endtry
]])
eq('Vim(let):E475: Invalid argument: container references itself',
eval('exception'))
end)
end)