mirrors/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2025-09-29 14:38:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7736 from jamessan/vim-8.0.0420

Browse Source 
[RFC] vim-patch:8.0.0420: text garbled when the system encoding differs from 'encoding'
This commit is contained in:
James McCoy
2017-12-17 20:15:12 -05:00
committed by GitHub
parent cca6d4b267 db0685a663
commit e6f8b105b0
14 changed files with 304 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/nvim/buffer.c
View File

@@ -1808,6 +1808,7 @@ void free_buf_options(buf_T *buf, int free_p_ff)
buf->b_p_ul = NO_LOCAL_UNDOLEVEL;
clear_string_option(&buf->b_p_lw);
clear_string_option(&buf->b_p_bkc);
clear_string_option(&buf->b_p_menc);
}

1
src/nvim/buffer_defs.h
View File

@@ -637,6 +637,7 @@ struct file_buffer {
uint32_t b_p_fex_flags; ///< flags for 'formatexpr'
char_u *b_p_kp; ///< 'keywordprg'
int b_p_lisp; ///< 'lisp'
char_u *b_p_menc; ///< 'makeencoding'
char_u *b_p_mps; ///< 'matchpairs'
int b_p_ml; ///< 'modeline'
int b_p_ml_nobin; ///< b_p_ml saved for binary mode

4
src/nvim/if_cscope.c
View File

@@ -1012,9 +1012,9 @@ static int cs_find_common(char *opt, char *pat, int forceit, int verbose,
fclose(f);
if (use_ll) /* Use location list */
wp = curwin;
/* '-' starts a new error list */
// '-' starts a new error list
if (qf_init(wp, tmp, (char_u *)"%f%*\\t%l%*\\t%m",
*qfpos == '-', cmdline) > 0) {
*qfpos == '-', cmdline, NULL) > 0) {
if (postponed_split != 0) {
(void)win_split(postponed_split > 0 ? postponed_split : 0,
postponed_split_flags);

2
src/nvim/main.c
View File

@@ -1369,7 +1369,7 @@ static void handle_quickfix(mparm_T *paramp)
set_string_option_direct((char_u *)"ef", -1,
paramp->use_ef, OPT_FREE, SID_CARG);
vim_snprintf((char *)IObuff, IOSIZE, "cfile %s", p_ef);
if (qf_init(NULL, p_ef, p_efm, true, IObuff) < 0) {
if (qf_init(NULL, p_ef, p_efm, true, IObuff, p_menc) < 0) {
ui_linefeed();
mch_exit(3);
}

12
src/nvim/option.c
View File

@@ -2217,6 +2217,7 @@ void check_buf_options(buf_T *buf)
check_string_option(&buf->b_p_tsr);
check_string_option(&buf->b_p_lw);
check_string_option(&buf->b_p_bkc);
check_string_option(&buf->b_p_menc);
}
/*
@@ -2635,8 +2636,8 @@ did_set_string_option (
else if (varp == &p_ei) {
if (check_ei() == FAIL)
errmsg = e_invarg;
/* 'encoding' and 'fileencoding' */
} else if (varp == &p_enc || gvarp == &p_fenc) {
// 'encoding', 'fileencoding' and 'makeencoding'
} else if (varp == &p_enc || gvarp == &p_fenc || gvarp == &p_menc) {
if (gvarp == &p_fenc) {
if (!MODIFIABLE(curbuf) && opt_flags != OPT_GLOBAL) {
errmsg = e_modifiable;
@@ -5400,6 +5401,9 @@ void unset_global_local_option(char *name, void *from)
case PV_LW:
clear_string_option(&buf->b_p_lw);
break;
case PV_MENC:
clear_string_option(&buf->b_p_menc);
break;
}
}
@@ -5433,6 +5437,7 @@ static char_u *get_varp_scope(vimoption_T *p, int opt_flags)
case PV_UL: return (char_u *)&(curbuf->b_p_ul);
case PV_LW: return (char_u *)&(curbuf->b_p_lw);
case PV_BKC: return (char_u *)&(curbuf->b_p_bkc);
case PV_MENC: return (char_u *)&(curbuf->b_p_menc);
}
return NULL; /* "cannot happen" */
}
@@ -5488,6 +5493,8 @@ static char_u *get_varp(vimoption_T *p)
? (char_u *)&(curbuf->b_p_ul) : p->var;
case PV_LW: return *curbuf->b_p_lw != NUL
? (char_u *)&(curbuf->b_p_lw) : p->var;
case PV_MENC: return *curbuf->b_p_menc != NUL
? (char_u *)&(curbuf->b_p_menc) : p->var;
case PV_ARAB: return (char_u *)&(curwin->w_p_arab);
case PV_LIST: return (char_u *)&(curwin->w_p_list);
@@ -5885,6 +5892,7 @@ void buf_copy_options(buf_T *buf, int flags)
buf->b_p_qe = vim_strsave(p_qe);
buf->b_p_udf = p_udf;
buf->b_p_lw = empty_option;
buf->b_p_menc = empty_option;
/*
* Don't copy the options set by ex_help(), use the saved values,

2
src/nvim/option_defs.h
View File

@@ -489,6 +489,7 @@ EXTERN char_u *p_lcs; // 'listchars'
EXTERN int p_lz; // 'lazyredraw'
EXTERN int p_lpl; // 'loadplugins'
EXTERN int p_magic; // 'magic'
EXTERN char_u *p_menc; // 'makeencoding'
EXTERN char_u *p_mef; // 'makeef'
EXTERN char_u *p_mp; // 'makeprg'
EXTERN char_u *p_cc; // 'colorcolumn'
@@ -736,6 +737,7 @@ enum {
, BV_KP
, BV_LISP
, BV_LW
, BV_MENC
, BV_MA
, BV_ML
, BV_MOD

7
src/nvim/options.lua
View File

@@ -1456,6 +1456,13 @@ return {
varname='p_mef',
defaults={if_true={vi=""}}
},
{
full_name='makeencoding', abbreviation='menc',
type='string', scope={'global', 'buffer'},
vi_def=true,
varname='p_menc',
defaults={if_true={vi=""}}
},
{
full_name='makeprg', abbreviation='mp',
type='string', scope={'global', 'buffer'},

106
src/nvim/quickfix.c
View File

@@ -160,6 +160,7 @@ typedef struct {
buf_T *buf;
linenr_T buflnum;
linenr_T lnumlast;
vimconv_T vc;
} qfstate_T;
typedef struct {
@@ -193,19 +194,19 @@ typedef struct {
static char_u *qf_last_bufname = NULL;
static bufref_T qf_last_bufref = { NULL, 0, 0 };
/*
* Read the errorfile "efile" into memory, line by line, building the error
* list. Set the error list's title to qf_title.
* Return -1 for error, number of errors for success.
*/
int
qf_init (
win_T *wp,
char_u *efile,
char_u *errorformat,
int newlist, /* TRUE: start a new error list */
char_u *qf_title
)
/// Read the errorfile "efile" into memory, line by line, building the error
/// list. Set the error list's title to qf_title.
///
/// @params wp If non-NULL, make a location list
/// @params efile If non-NULL, errorfile to parse
/// @params errorformat 'errorformat' string used to parse the error lines
/// @params newlist If true, create a new error list
/// @params qf_title If non-NULL, title of the error list
/// @params enc If non-NULL, encoding used to parse errors
///
/// @returns -1 for error, number of errors for success.
int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist,
char_u *qf_title, char_u *enc)
{
qf_info_T *qi = &ql_info;
@@ -214,8 +215,8 @@ qf_init (
}
return qf_init_ext(qi, efile, curbuf, NULL, errorformat, newlist,
(linenr_T)0, (linenr_T)0,
qf_title);
(linenr_T)0, (linenr_T)0,
qf_title, enc);
}
// Maximum number of bytes allowed per line while reading an errorfile.
@@ -638,6 +639,22 @@ retry:
} else {
state->linebuf = IObuff;
}
// Convert a line if it contains a non-ASCII character
if (state->vc.vc_type != CONV_NONE && has_non_ascii(state->linebuf)) {
char_u *line = string_convert(&state->vc, state->linebuf, &state->linelen);
if (line != NULL) {
if (state->linelen < IOSIZE) {
STRLCPY(state->linebuf, line, state->linelen + 1);
xfree(line);
} else {
xfree(state->growbuf);
state->linebuf = state->growbuf = line;
state->growbufsiz = state->linelen < LINE_MAXLEN
? state->linelen : LINE_MAXLEN;
}
}
}
return QF_OK;
}
@@ -976,15 +993,15 @@ qf_init_ext(
buf_T *buf,
typval_T *tv,
char_u *errorformat,
int newlist, /* TRUE: start a new error list */
linenr_T lnumfirst, /* first line number to use */
linenr_T lnumlast, /* last line number to use */
char_u *qf_title
int newlist, // TRUE: start a new error list
linenr_T lnumfirst, // first line number to use
linenr_T lnumlast, // last line number to use
char_u *qf_title,
char_u *enc
)
{
qfstate_T state = { NULL, 0, NULL, 0, NULL, NULL, NULL, NULL,
NULL, 0, 0 };
qffields_T fields = { NULL, NULL, 0, 0L, 0, false, NULL, 0, 0, 0 };
qfstate_T state;
qffields_T fields;
qfline_T *old_last = NULL;
bool adding = false;
static efm_T *fmt_first = NULL;
@@ -997,6 +1014,13 @@ qf_init_ext(
xfree(qf_last_bufname);
qf_last_bufname = NULL;
memset(&state, 0, sizeof(state));
memset(&fields, 0, sizeof(fields));
state.vc.vc_type = CONV_NONE;
if (enc != NULL && *enc != NUL) {
convert_setup(&state.vc, enc, p_enc);
}
fields.namebuf = xmalloc(CMDBUFFSIZE + 1);
fields.errmsglen = CMDBUFFSIZE + 1;
fields.errmsg = xmalloc(fields.errmsglen);
@@ -1147,6 +1171,10 @@ qf_init_end:
qf_update_buffer(qi, old_last);
if (state.vc.vc_type != CONV_NONE) {
convert_setup(&state.vc, NULL, NULL);
}
return retval;
}
@@ -3024,6 +3052,7 @@ void ex_make(exarg_T *eap)
qf_info_T *qi = &ql_info;
int res;
char_u *au_name = NULL;
char_u *enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
/* Redirect ":grep" to ":vimgrep" if 'grepprg' is "internal". */
if (grep_internal(eap->cmdidx)) {
@@ -3083,11 +3112,11 @@ void ex_make(exarg_T *eap)
res = qf_init(wp, fname, (eap->cmdidx != CMD_make
&& eap->cmdidx != CMD_lmake) ? p_gefm : p_efm,
(eap->cmdidx != CMD_grepadd
&& eap->cmdidx != CMD_lgrepadd),
*eap->cmdlinep);
if (wp != NULL)
(eap->cmdidx != CMD_grepadd && eap->cmdidx != CMD_lgrepadd),
*eap->cmdlinep, enc);
if (wp != NULL) {
qi = GET_LOC_LIST(wp);
}
if (au_name != NULL) {
apply_autocmds(EVENT_QUICKFIXCMDPOST, au_name,
curbuf->b_fname, TRUE, curbuf);
@@ -3429,19 +3458,18 @@ void ex_cfile(exarg_T *eap)
if (*eap->arg != NUL)
set_string_option_direct((char_u *)"ef", -1, eap->arg, OPT_FREE, 0);
/*
* This function is used by the :cfile, :cgetfile and :caddfile
* commands.
* :cfile always creates a new quickfix list and jumps to the
* first error.
* :cgetfile creates a new quickfix list but doesn't jump to the
* first error.
* :caddfile adds to an existing quickfix list. If there is no
* quickfix list then a new list is created.
*/
char_u *enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
// This function is used by the :cfile, :cgetfile and :caddfile
// commands.
// :cfile always creates a new quickfix list and jumps to the
// first error.
// :cgetfile creates a new quickfix list but doesn't jump to the
// first error.
// :caddfile adds to an existing quickfix list. If there is no
// quickfix list then a new list is created.
if (qf_init(wp, p_ef, p_efm, (eap->cmdidx != CMD_caddfile
&& eap->cmdidx != CMD_laddfile),
*eap->cmdlinep) > 0
*eap->cmdlinep, enc) > 0
&& (eap->cmdidx == CMD_cfile
|| eap->cmdidx == CMD_lfile)) {
if (au_name != NULL)
@@ -4338,7 +4366,7 @@ void ex_cbuffer(exarg_T *eap)
if (qf_init_ext(qi, NULL, buf, NULL, p_efm,
(eap->cmdidx != CMD_caddbuffer
&& eap->cmdidx != CMD_laddbuffer),
eap->line1, eap->line2, qf_title) > 0) {
eap->line1, eap->line2, qf_title, NULL) > 0) {
if (au_name != NULL) {
apply_autocmds(EVENT_QUICKFIXCMDPOST, (char_u *)au_name,
curbuf->b_fname, true, curbuf);
@@ -4403,7 +4431,7 @@ void ex_cexpr(exarg_T *eap)
if (qf_init_ext(qi, NULL, NULL, &tv, p_efm,
(eap->cmdidx != CMD_caddexpr
&& eap->cmdidx != CMD_laddexpr),
(linenr_T)0, (linenr_T)0, *eap->cmdlinep) > 0) {
(linenr_T)0, (linenr_T)0, *eap->cmdlinep, NULL) > 0) {
if (au_name != NULL) {
apply_autocmds(EVENT_QUICKFIXCMDPOST, (char_u *)au_name,
curbuf->b_fname, true, curbuf);

1
src/nvim/testdir/Makefile
View File

@@ -65,6 +65,7 @@ NEW_TESTS ?= \
test_increment_dbcs.res \
test_lambda.res \
test_langmap.res \
test_makeencoding.res \
test_marks.res \
test_match.res \
test_matchadd_conceal.res \

67
src/nvim/testdir/test_makeencoding.py Normal file
View File

@@ -0,0 +1,67 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Test program for :make, :grep and :cgetfile.
from __future__ import print_function, unicode_literals
import locale
import io
import sys
def set_output_encoding(enc=None):
"""Set the encoding of stdout and stderr
arguments:
enc -- Encoding name.
If omitted, locale.getpreferredencoding() is used.
"""
if enc is None:
enc = locale.getpreferredencoding()
def get_text_writer(fo, **kwargs):
kw = dict(kwargs)
kw.setdefault('errors', 'backslashreplace') # use \uXXXX style
kw.setdefault('closefd', False)
if sys.version_info[0] < 3:
# Work around for Python 2.x
# New line conversion isn't needed here. Done in somewhere else.
writer = io.open(fo.fileno(), mode='w', newline='', **kw)
write = writer.write # save the original write() function
enc = locale.getpreferredencoding()
def convwrite(s):
if isinstance(s, bytes):
write(s.decode(enc)) # convert to unistr
else:
write(s)
try:
writer.flush() # needed on Windows
except IOError:
pass
writer.write = convwrite
else:
writer = io.open(fo.fileno(), mode='w', **kw)
return writer
sys.stdout = get_text_writer(sys.stdout, encoding=enc)
sys.stderr = get_text_writer(sys.stderr, encoding=enc)
def main():
enc = 'utf-8'
if len(sys.argv) > 1:
enc = sys.argv[1]
set_output_encoding(enc)
message_tbl = {
'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
'latin1': 'ÀÈÌÒÙ',
'cp932': 'こんにちは',
'cp936': '你好',
}
print('Xfoobar.c(10) : %s (%s)' % (message_tbl[enc], enc))
if __name__ == "__main__":
main()

106
src/nvim/testdir/test_makeencoding.vim Normal file
View File

@@ -0,0 +1,106 @@
" Tests for 'makeencoding'.
if !has('multi_byte')
finish
endif
source shared.vim
let s:python = PythonProg()
if s:python == ''
" Can't run this test.
finish
endif
let s:script = 'test_makeencoding.py'
let s:message_tbl = {
\ 'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
\ 'latin1': 'ÀÈÌÒÙ',
\ 'cp932': 'こんにちは',
\ 'cp936': '你好',
\}
" Tests for :cgetfile and :lgetfile.
func Test_getfile()
set errorfile=Xerror.txt
set errorformat=%f(%l)\ :\ %m
" :cgetfile
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
cgetfile
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lgetfile
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
lgetfile
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
call delete(&errorfile)
endfunc
" Tests for :grep and :lgrep.
func Test_grep()
let &grepprg = s:python
set grepformat=%f(%l)\ :\ %m
" :grep
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent grep! " . s:script . " " . enc
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lgrep
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent lgrep! " . s:script . " " . enc
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
endfunc
" Tests for :make and :lmake.
func Test_make()
let &makeprg = s:python
set errorformat=%f(%l)\ :\ %m
" :make
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent make! " . s:script . " " . enc
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lmake
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent lmake! " . s:script . " " . enc
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
endfunc