Merge #2929 'Disallow "set encoding" after startup'

runtime/doc/mbyte.txt

@@ -89,8 +89,8 @@ See |encoding-values| for a list of acceptable values.
 
 The result is that all the text that is used inside Vim will be in this
 encoding.  Not only the text in the buffers, but also in registers, variables,
-etc.  This also means that changing the value of 'encoding' makes the existing
-text invalid!  The text doesn't change, but it will be displayed wrong.
+etc. 'encoding' cannot be changed after startup (at latest when |vimrc|
+is sourced) as changing the value later would make the existing text invalid.
 
 You can edit files in another encoding than what 'encoding' is set to.  Vim
 will convert the file when you read it and convert it back when you write it.

runtime/doc/message.txt

@@ -739,6 +739,12 @@ Example: >
 You tried to execute a command that is neither an Ex command nor
 a user-defined command.
 
+							*E905*  >
+  Cannot set this option after startup
+
+This option might only be changed in a |vimrc| file (or earlier). This means
+that plugins can rely on the value being constant after initialization.
+
 ==============================================================================
 3. Messages						*messages*
 

runtime/doc/options.txt

@@ -2217,21 +2217,13 @@ A jump table for the options with a short description can be found at |Q_op|.
 	viminfo file, etc.  It sets the kind of characters which Vim can work
 	with.  See |encoding-names| for the possible values.
 
-	NOTE: Changing this option will not change the encoding of the
-	existing text in Vim.  It may cause non-ASCII text to become invalid.
-	It should normally be kept at its default value, or set when Vim
-	starts up.  See |multibyte|.  To reload the menus see |:menutrans|.
+	This option cannot be changed after startup.
+	Otherwise it would cause non-ASCII text inside Vim to become
+	invalid. It should normally be kept at its default value, or be set
+	in vimrc. See |multibyte|.
 
-	This option cannot be set from a |modeline|.  It would most likely
-	corrupt the text.
-
-	NOTE: For GTK+ 2 it is highly recommended to set 'encoding' to
-	"utf-8".  Although care has been taken to allow different values of
-	'encoding', "utf-8" is the natural choice for the environment and
-	avoids unnecessary conversion overhead.  "utf-8" has not been made
-	the default to prevent different behavior of the GUI and terminal
-	versions, and to avoid changing the encoding of newly created files
-	without your knowledge (in case 'fileencodings' is empty).
+	The recommended 'encoding' is "utf-8". Remote plugins and GUI:s
+	only support utf-8.
 
 	The character encoding of files can be different from 'encoding'.
 	This is specified with 'fileencoding'.  The conversion is done with
@@ -2243,6 +2235,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 <
 	Normally 'encoding' will be equal to your current locale.  This will
 	be the default if Vim recognizes your environment settings.
+	"utf-8" is used when the locale encoding could not be detected.
 
 	When you set this option, it fires the |EncodingChanged| autocommand
 	event so that you can set up fonts if necessary.
@@ -2254,12 +2247,6 @@ A jump table for the options with a short description can be found at |Q_op|.
 	For example "Latin-1" becomes "latin1", "ISO_88592" becomes
 	"iso-8859-2" and "utf8" becomes "utf-8".
 
-	Note: "latin1" is also used when the encoding could not be detected.
-	This only works when editing files in the same encoding!  When the
-	actual character set is not latin1, make sure 'fileencoding' and
-	'fileencodings' are empty.  When conversion is needed, switch to using
-	utf-8.
-
 	When "unicode", "ucs-2" or "ucs-4" is used, Vim internally uses utf-8.
 	You don't notice this while editing, but it does matter for the
 	|viminfo-file|.  And Vim expects the terminal to use utf-8 too.  Thus

runtime/doc/vim_diff.txt

@@ -74,6 +74,8 @@ are always available and may be used simultaneously in separate plugins.  The
    'p')) mkdir() will silently exit. In Vim this was an error.
 3. mkdir() error messages now include strerror() text when mkdir fails.
 
+'encoding' cannot be changed after startup.
+
 ==============================================================================
 4. New Features						     *nvim-features-new*
 

src/nvim/globals.h

@@ -633,6 +633,10 @@ EXTERN int silent_mode INIT(= FALSE);
 /* set to TRUE when "-s" commandline argument
  * used for ex */
 
+// Set to true when sourcing of startup scripts (nvimrc) is done.
+// Used for options that cannot be changed after startup scripts.
+EXTERN bool did_source_startup_scripts INIT(= false);
+
 EXTERN pos_T VIsual;            /* start position of active Visual selection */
 EXTERN int VIsual_active INIT(= FALSE);
 /* whether Visual mode is active */
@@ -1078,6 +1082,8 @@ EXTERN garray_T error_ga
  * Excluded are errors that are only used once and debugging messages.
  */
 EXTERN char_u e_abort[] INIT(= N_("E470: Command aborted"));
+EXTERN char_u e_afterinit[] INIT(= N_(
+        "E905: Cannot set this option after startup"));
 EXTERN char_u e_api_spawn_failed[] INIT(= N_("E903: Could not spawn API job"));
 EXTERN char_u e_argreq[] INIT(= N_("E471: Argument required"));
 EXTERN char_u e_backslash[] INIT(= N_("E10: \\ should be followed by /, ? or &"));

src/nvim/main.c

@@ -1911,6 +1911,7 @@ static void source_startup_scripts(mparm_T *parmp)
       need_wait_return = TRUE;
     secure = 0;
   }
+  did_source_startup_scripts = true;
   TIME_MSG("sourcing vimrc file(s)");
 }
 

src/nvim/option.c

@@ -575,6 +575,12 @@ void set_init_1(void)
     mb_init();
   }
 
+  // Don't change &encoding when resetting to defaults with ":set all&".
+  opt_idx = findoption((char_u *)"encoding");
+  if (opt_idx >= 0) {
+    options[opt_idx].flags |= P_NODEFAULT;
+  }
+
   /* Set the default for 'helplang'. */
   set_helplang_default(get_mess_lang());
 }
@@ -2271,10 +2277,11 @@ did_set_string_option (
   else if (varp == &p_ei) {
     if (check_ei() == FAIL)
       errmsg = e_invarg;
-  }
   /* 'encoding' and 'fileencoding' */
-  else if (varp == &p_enc || gvarp == &p_fenc) {
-    if (gvarp == &p_fenc) {
+  } else if (varp == &p_enc || gvarp == &p_fenc) {
+    if (varp == &p_enc && did_source_startup_scripts) {
+       errmsg = e_afterinit;
+    } else if (gvarp == &p_fenc) {
       if (!MODIFIABLE(curbuf) && opt_flags != OPT_GLOBAL)
         errmsg = e_modifiable;
       else if (vim_strchr(*varp, ',') != NULL)
@@ -2305,10 +2312,6 @@ did_set_string_option (
        * (with another encoding). */
       if (varp == &p_enc && *curbuf->b_p_keymap != NUL)
         (void)keymap_init();
-
-      if (varp == &p_enc) {
-        ui_update_encoding();
-      }
     }
   } else if (varp == &p_penc) {
     /* Canonize printencoding if VIM standard one */

src/nvim/testdir/test10.in

@@ -4,7 +4,6 @@ STARTTEST
 :so small.vim
 :" Also test a BOM is ignored.
 :so mbyte.vim
-:set encoding=utf-8
 :7/start of errorfile/,/end of errorfile/w! Xerrorfile1
 :7/start of errorfile/,/end of errorfile/-1w! Xerrorfile2
 :/start of testfile/,/end of testfile/w! Xtestfile

src/nvim/testdir/test39.in

@@ -5,8 +5,6 @@ And test "U" in Visual mode, also on German sharp S.
 STARTTEST
 :so small.vim
 :so mbyte.vim
-:" This only works when 'encoding' is "latin1", don't depend on the environment
-:set enc=latin1
 /^abcde
 :" Test shift-right of a block
 jlllljj>wlljlll>
@@ -39,18 +37,18 @@ G$khhhhhkkcmno
 :exe ":norm! l\<C-V>j$hhAab\<Esc>"
 :.,/^$/w >> test.out
 :" Test for Visual block insert when virtualedit=all and utf-8 encoding
-:set ve=all enc=utf-8
+:set ve=all
 :/\t\tline
 :exe ":norm! 07l\<C-V>jjIx\<Esc>"
 :.,/^$/w >> test.out
 :" Test for Visual block append when virtualedit=all
 :exe ":norm! 012l\<C-v>jjAx\<Esc>"
-:set ve= enc=latin1
+:set ve=
 :.,/^$/w >> test.out
-:" gUe must uppercase a whole word, also when <EFBFBD> changes to SS
-Gothe youtu<EFBFBD>euu endYpk0wgUe
+:" gUe must uppercase a whole word, also when ß changes to SS
+Gothe youtußeuu endYpk0wgUe
 :" gUfx must uppercase until x, inclusive.
-O- you<EFBFBD>tu<EFBFBD>exu -0fogUfx
+O- youßtußexu -0fogUfx
 :" VU must uppercase a whole line
 YpkVU
 :" same, when it's the last line in the buffer
@@ -89,7 +87,7 @@ cccc
 cccc
 dddd
 
-yaaa
+yaaa
 ¿¿¿
 bbb
 

src/nvim/testdir/test69.in

@@ -5,7 +5,6 @@ Also test byteidx() and byteidxcomp()
 
 STARTTEST
 :so mbyte.vim
-:set encoding=utf-8
 ENDTEST
 
 Results of test69:

src/nvim/testdir/test83.in

@@ -2,7 +2,6 @@ Tests for tag search with !_TAG_FILE_ENCODING.
 
 STARTTEST
 :so mbyte.vim
-:set enc=utf8
 :if !has('iconv') || iconv("\x82\x60", "cp932", "utf-8") != "\uff21"
 : e! test.ok
 : w! test.out

src/nvim/testdir/test_eval.in

@@ -4,7 +4,6 @@ Note: system clipboard is saved, changed and restored.
 
 STARTTEST
 :so small.vim
-:set encoding=latin1
 :set noswapfile
 :lang C
 :fun AppendRegContents(reg)

src/nvim/testdir/unix.vim

@@ -1,3 +1,6 @@
 " Settings for test script execution
 " Always use "sh", don't use the value of "$SHELL".
 set shell=sh
+
+" Don't depend on system locale, always use utf-8
+set encoding=utf-8

src/nvim/tui/input.c

@@ -25,7 +25,9 @@ void term_input_init(TermInput *input, Loop *loop)
   if (!term) {
     term = "";  // termkey_new_abstract assumes non-null (#2745)
   }
-  input->tk = termkey_new_abstract(term, 0);
+  int enc_flag = enc_utf8 ? TERMKEY_FLAG_UTF8 : TERMKEY_FLAG_RAW;
+  input->tk = termkey_new_abstract(term, enc_flag);
+
   int curflags = termkey_get_canonflags(input->tk);
   termkey_set_canonflags(input->tk, curflags | TERMKEY_CANON_DELBS);
   // setup input handle
@@ -57,13 +59,6 @@ void term_input_stop(TermInput *input)
   time_watcher_stop(&input->timer_handle);
 }
 
-void term_input_set_encoding(TermInput *input, char* enc)
-{
-  int enc_flag = strcmp(enc, "utf-8") == 0 ? TERMKEY_FLAG_UTF8
-                                           : TERMKEY_FLAG_RAW;
-  termkey_set_flags(input->tk, enc_flag);
-}
-
 static void input_enqueue_event(void **argv)
 {
   char *buf = argv[0];

src/nvim/tui/tui.c

@@ -99,7 +99,6 @@ UI *tui_start(void)
   ui->suspend = tui_suspend;
   ui->set_title = tui_set_title;
   ui->set_icon = tui_set_icon;
-  ui->set_encoding = tui_set_encoding;
   return ui_bridge_attach(ui, tui_main, tui_scheduler);
 }
 
@@ -625,12 +624,6 @@ static void tui_set_icon(UI *ui, char *icon)
 {
 }
 
-static void tui_set_encoding(UI *ui, char* enc)
-{
-  TUIData *data = ui->data;
-  term_input_set_encoding(&data->input, enc);
-}
-
 static void invalidate(UI *ui, int top, int bot, int left, int right)
 {
   TUIData *data = ui->data;

src/nvim/ui.c

@@ -113,11 +113,6 @@ void ui_set_icon(char *icon)
   UI_CALL(flush);
 }
 
-void ui_update_encoding(void)
-{
-  UI_CALL(set_encoding, (char*)p_enc);
-}
-
 // May update the shape of the cursor.
 void ui_cursor_shape(void)
 {
@@ -188,7 +183,6 @@ void ui_attach(UI *ui)
   }
 
   uis[ui_count++] = ui;
-  ui_update_encoding();
   ui_refresh();
 }
 

src/nvim/ui.h

@@ -38,7 +38,6 @@ struct ui_t {
   void (*suspend)(UI *ui);
   void (*set_title)(UI *ui, char *title);
   void (*set_icon)(UI *ui, char *icon);
-  void (*set_encoding)(UI *ui, char *enc);
   void (*stop)(UI *ui);
 };
 

src/nvim/ui_bridge.c

@@ -52,7 +52,6 @@ UI *ui_bridge_attach(UI *ui, ui_main_fn ui_main, event_scheduler scheduler)
   rv->bridge.suspend = ui_bridge_suspend;
   rv->bridge.set_title = ui_bridge_set_title;
   rv->bridge.set_icon = ui_bridge_set_icon;
-  rv->bridge.set_encoding = ui_bridge_set_encoding;
   rv->scheduler = scheduler;
 
   rv->ui_main = ui_main;
@@ -334,14 +333,3 @@ static void ui_bridge_set_icon_event(void **argv)
   ui->set_icon(ui, argv[1]);
   xfree(argv[1]);
 }
-
-static void ui_bridge_set_encoding(UI *b, char* enc)
-{
-  UI_CALL(b, set_encoding, 2, b, xstrdup(enc));
-}
-static void ui_bridge_set_encoding_event(void **argv)
-{
-  UI *ui = UI(argv[0]);
-  ui->set_encoding(ui, argv[1]);
-  xfree(argv[1]);
-}

test/functional/ex_cmds/encoding_spec.lua

@@ -0,0 +1,40 @@
+local helpers = require('test.functional.helpers')
+local clear, execute, feed = helpers.clear, helpers.execute, helpers.feed
+local eq, neq, eval = helpers.eq, helpers.neq, helpers.eval
+
+describe('&encoding', function()
+
+  before_each(function()
+    clear()
+    -- sanity check: tests should run with encoding=utf-8
+    eq('utf-8', eval('&encoding'))
+    eq(3, eval('strwidth("Bär")'))
+  end)
+
+  it('cannot be changed after setup', function()
+    execute('set encoding=latin1')
+    -- error message expected
+    feed('<cr>')
+    neq(nil, string.find(eval('v:errmsg'), '^E905:'))
+    eq('utf-8', eval('&encoding'))
+    -- check nvim is still in utf-8 mode
+    eq(3, eval('strwidth("Bär")'))
+  end)
+
+  it('is not changed by `set all&`', function()
+    -- we need to set &encoding to something non-default
+    -- use 'latin1' when enc&vi is 'utf-8', 'utf-8' otherwise
+    execute('set fenc=default')
+    local enc_default, enc_other, width = eval('&fenc'), 'utf-8', 3
+    if enc_default == 'utf-8' then
+      enc_other = 'latin1'
+      width = 4 -- utf-8 string 'Bär' will count as 4 latin1 chars
+    end
+
+    clear('set enc=' .. enc_other)
+    execute('set all&')
+    eq(enc_other, eval('&encoding'))
+    eq(width, eval('strwidth("Bär")'))
+  end)
+
+end)

test/functional/helpers.lua

@@ -7,7 +7,7 @@ local Session = require('nvim.session')
 
 local nvim_prog = os.getenv('NVIM_PROG') or 'build/bin/nvim'
 local nvim_argv = {nvim_prog, '-u', 'NONE', '-i', 'NONE', '-N',
-                   '--cmd', 'set shortmess+=I background=light noswapfile noautoindent laststatus=1',
+                   '--cmd', 'set shortmess+=I background=light noswapfile noautoindent laststatus=1 encoding=utf-8',
                    '--embed'}
 
 -- Formulate a path to the directory containing nvim.  We use this to
@@ -183,11 +183,16 @@ local function spawn(argv)
   return session
 end
 
-local function clear()
+local function clear(extra_cmd)
   if session then
     session:exit(0)
   end
-  session = spawn(nvim_argv)
+  local args = {unpack(nvim_argv)}
+  if extra_cmd ~= nil then
+    table.insert(args, '--cmd')
+    table.insert(args, extra_cmd)
+  end
+  session = spawn(args)
 end
 
 local function insert(...)

test/functional/legacy/044_099_regexp_multibyte_magic_spec.lua

@@ -33,7 +33,6 @@ local function run_test_with_regexpengine(regexpengine)
     j 0123❤x
     k combinations]])
 
-  execute('set encoding=utf-8')
   execute('set re=' .. regexpengine)
 
   -- Lines 1-8. Exercise regexp search with various magic settings. On each

test/functional/legacy/075_maparg_spec.lua

@@ -10,7 +10,6 @@ describe('maparg()', function()
 
   it('is working', function()
     execute('set cpo-=<')
-    execute('set encoding=utf8')
 
     -- Test maparg() with a string result
     execute('map foo<C-V> is<F4>foo')

test/functional/legacy/082_string_comparison_spec.lua

@@ -9,7 +9,6 @@ describe('case-insensitive string comparison in UTF-8', function()
   setup(clear)
 
   it('is working', function()
-    execute('set enc=utf8')
     feed('ggdG<cr>')
     source([[
       function! Ch(a, op, b, expected)

test/functional/legacy/095_regexp_multibyte_spec.lua

@@ -15,7 +15,7 @@ describe('regex with multi-byte', function()
       Results of test95:]])
 
     source([=[
-      set encoding=utf-8 nomore
+      set nomore
       let tl = []
 
       call add(tl, [2, '[[:alpha:][=a=]]\+', '879 aiaãâaiuvna ', 'aiaãâaiuvna'])

test/functional/legacy/mapping_spec.lua

@@ -12,8 +12,6 @@ describe('mapping', function()
       test starts here:
       ]])
 
-    execute('set encoding=utf-8')
-
     -- Abbreviations with р (0x80) should work.
     execute('inoreab чкпр   vim')
     feed('GAчкпр <esc>')

test/functional/legacy/utf8_spec.lua

@@ -10,7 +10,6 @@ describe('utf8', function()
   it('is working', function()
     insert('start:')
 
-    execute('set encoding=utf-8')
     execute('new')
     execute('call setline(1, ["aaa", "あああ", "bbb"])')