Merge pull request #2506 from ZyX-I/shada

Replace viminfo with ShaDa files

.gitignore

@@ -13,6 +13,8 @@
 *.o
 *.so
 
+tags
+
 /src/nvim/po/vim.pot
 /src/nvim/po/*.ck
 

config/CMakeLists.txt

@@ -2,6 +2,8 @@ include(CheckTypeSize)
 include(CheckSymbolExists)
 include(CheckFunctionExists)
 include(CheckIncludeFiles)
+include(CheckCSourceRuns)
+include(CheckCSourceCompiles)
 
 check_type_size("int" SIZEOF_INT)
 check_type_size("long" SIZEOF_LONG)
@@ -71,6 +73,54 @@ if(HAVE_LANGINFO_H)
   check_symbol_exists(CODESET "langinfo.h" HAVE_NL_LANGINFO_CODESET)
 endif()
 
+set(SI "#include <stdint.h>\n")
+set(MS "int main(int argc,char**argv)\n{\n  uint64_t i=0x0102030405060708ULL;")
+set(ME "}")
+check_c_source_compiles("
+  #define _BSD_SOURCE 1
+  #define _DEFAULT_SOURCE 1
+  ${SI}
+  #include <endian.h>
+  #ifndef be64toh
+  # error No be64toh macros
+  #endif
+  ${MS}
+    uint64_t j = be64toh(i);
+    return (j == 0);  // j must not be zero
+  ${ME}"
+  HAVE_BE64TOH_MACROS)
+if(NOT "${HAVE_BE64TOH_MACROS}")
+  check_function_exists(be64toh HAVE_BE64TOH_FUNC)
+endif()
+if("${HAVE_BE64TOH_MACROS}" OR "${HAVE_BE64TOH_FUNC}")
+  set(HAVE_BE64TOH 1)
+endif()
+if (NOT "${HAVE_BE64TOH}")
+  if (NOT "${CMAKE_CROSSCOMPILING}")
+    # It is safe to make ORDER_BIG_ENDIAN not defined if
+    # - HAVE_BE64TOH is true. In this case be64toh will be used unconditionally in 
+    #   any case and ORDER_BIG_ENDIAN will not be examined.
+    # - CMAKE_CROSSCOMPILING *and* HAVE_BE64TOH are both false. In this case 
+    #   be64toh function which uses cycle and arithmetic operations is used which 
+    #   will work regardless of endianess. Function is sub-optimal though.
+    check_c_source_runs("
+      ${SI}
+      ${MS}
+        char *s = (char *) &i;
+        return (
+          s[0] == 0x01
+          && s[1] == 0x02
+          && s[2] == 0x03
+          && s[3] == 0x04
+          && s[4] == 0x05
+          && s[5] == 0x06
+          && s[6] == 0x07
+          && s[7] == 0x08) ? 0 : 1;
+      ${ME}"
+      ORDER_BIG_ENDIAN)
+  endif()
+endif()
+
 # generate configuration header and update include directories
 configure_file (
   "${PROJECT_SOURCE_DIR}/config/config.h.in"

config/config.h.in

@@ -61,4 +61,7 @@
 #cmakedefine HAVE_JEMALLOC
 #endif
 
+#cmakedefine HAVE_BE64TOH
+#cmakedefine ORDER_BIG_ENDIAN
+
 #endif  // AUTO_CONFIG_H

contrib/YouCompleteMe/ycm_extra_conf.py

@@ -9,7 +9,7 @@ def DirectoryOfThisScript():
 
 def GetDatabase():
     compilation_database_folder = os.path.join(DirectoryOfThisScript(),
-                                               '..', 'build')
+                                               '..', '..', 'build')
     if os.path.exists(compilation_database_folder):
         return ycm_core.CompilationDatabase(compilation_database_folder)
     return None

man/nvim.1

@@ -138,7 +138,7 @@ Sets the options 'hkmap' and 'rightleft'.
 .It Fl V Ns Oo Ar N Oc Ns Op Ar file
 Verbose mode.
 Print messages about which files are being sourced and for reading and
-writing an nviminfo file.
+writing a ShaDa file.
 .Ar N
 is the value for the 'verbose' option; defaults to
 .Cm 10
@@ -191,18 +191,18 @@ is
 loading plugins is also skipped.
 See
 .Ic :help initialization .
-.It Fl i Ar nviminfo
+.It Fl i Ar shada
 Use
-.Ar nviminfo
+.Ar shada
 instead of the default of
-.Pa ~/.nviminfo .
+.Pa ~/.nvim/shada/main.shada .
 If
-.Ar nviminfo
+.Ar shada
 is
 .Cm NONE ,
-do not read or write an nviminfo file.
+do not read or write a ShaDa file.
 See
-.Ic :help viminfo .
+.Ic :help shada .
 .It Fl -noplugin
 Skip loading plugins.
 Implied by

runtime/doc/autocmd.txt

@@ -265,8 +265,8 @@ Name			triggered by ~
 |TermResponse|		after the terminal response to |t_RV| is received
 
 |QuitPre|		when using `:quit`, before deciding whether to quit
-|VimLeavePre|		before exiting Vim, before writing the viminfo file
+|VimLeavePre|		before exiting Vim, before writing the shada file
-|VimLeave|		before exiting Vim, after writing the viminfo file
+|VimLeave|		before exiting Vim, after writing the shada file
 
 	Various
 |FileChangedShell|	Vim notices that a file changed since editing started
@@ -912,14 +912,14 @@ VimEnter			After doing all the startup stuff, including
 				the buffers in them.
 							*VimLeave*
 VimLeave			Before exiting Vim, just after writing the
-				.viminfo file.  Executed only once, like
+				.shada file.  Executed only once, like
 				VimLeavePre.
 				To detect an abnormal exit use |v:dying|.
 				When v:dying is 2 or more this event is not
 				triggered.
 							*VimLeavePre*
 VimLeavePre			Before exiting Vim, just before writing the
-				.viminfo file.  This is executed only once,
+				.shada file.  This is executed only once,
 				if there is a match with the name of what
 				happens to be the current buffer when exiting.
 				Mostly useful with a "*" pattern. >
@@ -1375,7 +1375,7 @@ use search patterns normally, e.g., with the "n" command.
 If you want an autocommand to set the search pattern, such that it is used
 after the autocommand finishes, use the ":let @/ =" command.
 The search-highlighting cannot be switched off with ":nohlsearch" in an
-autocommand.  Use the 'h' flag in the 'viminfo' option to disable search-
+autocommand.  Use the 'h' flag in the 'shada' option to disable search-
 highlighting when starting Vim.
 
 							*Cmd-event*

runtime/doc/cmdline.txt

@@ -729,7 +729,7 @@ function expand() |expand()|.
 	#<n	(where n is a number > 0) is replaced with old	  *:_#<* *c_#<*
 		file name n.  See |:oldfiles| or |v:oldfiles| to get the
 		number.							*E809*
-		{only when compiled with the |+eval| and |+viminfo| features}
+		{only when compiled with the |+eval| and |+shada| features}
 
 Note that these, except "#<n", give the file name as it was typed.  If an
 absolute path is needed (when using the file name from a different directory),

runtime/doc/eval.txt

@@ -557,9 +557,9 @@ Functions that can be used with a Dictionary: >
 If you need to know the type of a variable or expression, use the |type()|
 function.
 
-When the '!' flag is included in the 'viminfo' option, global variables that
+When the '!' flag is included in the 'shada' option, global variables that
 start with an uppercase letter, and don't contain a lowercase letter, are
-stored in the viminfo file |viminfo-file|.
+stored in the shada file |shada-file|.
 
 When the 'sessionoptions' option contains "global", global variables that
 start with an uppercase letter and contain at least one lowercase letter are
@@ -568,7 +568,7 @@ stored in the session file |session-file|.
 variable name		can be stored where ~
 my_var_6		not
 My_Var_6		session file
-MY_VAR_6		viminfo file
+MY_VAR_6		shada file
 
 
 It's possible to form a variable name with curly braces, see
@@ -1524,16 +1524,16 @@ v:msgpack_types	Dictionary containing msgpack types used by |msgpackparse()|
 		of msgpack types, use |is| operator.
 
 					*v:oldfiles* *oldfiles-variable*
-v:oldfiles	List of file names that is loaded from the |viminfo| file on
+v:oldfiles	List of file names that is loaded from the |shada| file on
 		startup.  These are the files that Vim remembers marks for.
 		The length of the List is limited by the ' argument of the
-		'viminfo' option (default is 100).
+		'shada' option (default is 100).
-		When the |viminfo| file is not used the List is empty.
+		When the |shada| file is not used the List is empty.
 		Also see |:oldfiles| and |c_#<|.
 		The List can be modified, but this has no effect on what is
-		stored in the |viminfo| file later.  If you use values other
+		stored in the |shada| file later.  If you use values other
 		than String this will cause trouble.
-		{only when compiled with the |+viminfo| feature}
+		{only when compiled with the |+shada| feature}
 
 					*v:operator* *operator-variable*
 v:operator	The last operator given in Normal mode.  This is a single
@@ -6936,6 +6936,7 @@ quickfix		Compiled with |quickfix| support.
 reltime			Compiled with |reltime()| support.
 rightleft		Compiled with 'rightleft' support.
 scrollbind		Compiled with 'scrollbind' support.
+shada			Compiled with shada support.
 showcmd			Compiled with 'showcmd' support.
 signs			Compiled with |:sign| support.
 smartindent		Compiled with 'smartindent' support.
@@ -6964,7 +6965,6 @@ unix			Unix version of Vim.
 user_commands		User-defined commands.
 vertsplit		Compiled with vertically split windows |:vsplit|.
 vim_starting		True while initial source'ing takes place. |startup|
-viminfo			Compiled with viminfo support.
 virtualedit		Compiled with 'virtualedit' option.
 visual			Compiled with Visual mode.
 visualextra		Compiled with extra Visual mode commands.

runtime/doc/index.txt

@@ -1359,7 +1359,7 @@ tag	      command	      action ~
 |:number|	:nu[mber]	print lines with line number
 |:nunmap|	:nun[map]	like ":unmap" but for Normal mode
 |:nunmenu|	:nunme[nu]	remove menu for Normal mode
-|:oldfiles|	:ol[dfiles]	list files that have marks in the viminfo file
+|:oldfiles|	:ol[dfiles]	list files that have marks in the ShaDa file
 |:open|		:o[pen]		start open mode (not implemented)
 |:omap|		:om[ap]		like ":map" but for Operator-pending mode
 |:omapclear|	:omapc[lear]	remove all mappings for Operator-pending mode
@@ -1418,9 +1418,9 @@ tag	      command	      action ~
 |:rewind|	:rew[ind]	go to the first file in the argument list
 |:right|	:ri[ght]	right align text
 |:rightbelow|	:rightb[elow]	make split window appear right or below
+|:rshada|	:rsh[ada]	read from ShaDa file
 |:rundo|	:rund[o]	read undo information from a file
 |:runtime|	:ru[ntime]	source vim scripts in 'runtimepath'
-|:rviminfo|	:rv[iminfo]	read from viminfo file
 |:substitute|	:s[ubstitute]	find and replace text
 |:sNext|	:sN[ext]	split window and go to previous file in
 				argument list
@@ -1579,8 +1579,8 @@ tag	      command	      action ~
 				argument list
 |:wq|		:wq		write to a file and quit window or Vim
 |:wqall|	:wqa[ll]	write all changed buffers and quit Vim
+|:wshada|	:wsh[ada]	write to ShaDa file
 |:wundo|	:wu[ndo]	write undo information to a file
-|:wviminfo|	:wv[iminfo]	write to viminfo file
 |:xit|		:x[it]		write if buffer changed and quit window or Vim
 |:xall|		:xa[ll]		same as ":wqall"
 |:xmapclear|	:xmapc[lear]	remove all mappings for Visual mode

runtime/doc/mbyte.txt

@@ -387,7 +387,7 @@ Vim will automatically convert from one to another encoding in several places:
   'encoding' (requires a gettext version that supports this).
 - When reading a Vim script where |:scriptencoding| is different from
   'encoding'.
-- When reading or writing a |viminfo| file.
+- When reading or writing a |shada| file.
 Most of these require the |+iconv| feature.  Conversion for reading and
 writing files may also be specified with the 'charconvert' option.
 

runtime/doc/motion.txt

@@ -805,7 +805,7 @@ unrelated.
 
 'a - 'z		lowercase marks, valid within one file
 'A - 'Z		uppercase marks, also called file marks, valid between files
-'0 - '9		numbered marks, set from .viminfo file
+'0 - '9		numbered marks, set from .shada file
 
 Lowercase marks 'a to 'z are remembered as long as the file remains in the
 buffer list.  If you remove the file from the buffer list, all its marks are
@@ -820,14 +820,14 @@ Uppercase marks 'A to 'Z include the file name.
 You can use them to jump from file to file.  You can only use an uppercase
 mark with an operator if the mark is in the current file.  The line number of
 the mark remains correct, even if you insert/delete lines or edit another file
-for a moment.  When the 'viminfo' option is not empty, uppercase marks are
+for a moment.  When the 'shada' option is not empty, uppercase marks are
-kept in the .viminfo file.  See |viminfo-file-marks|.
+kept in the .shada file.  See |shada-file-marks|.
 
 Numbered marks '0 to '9 are quite different.  They can not be set directly.
-They are only present when using a viminfo file |viminfo-file|.  Basically '0
+They are only present when using a shada file |shada-file|.  Basically '0
 is the location of the cursor when you last exited Vim, '1 the last but one
-time, etc.  Use the "r" flag in 'viminfo' to specify files for which no
+time, etc.  Use the "r" flag in 'shada' to specify files for which no
-Numbered mark should be stored.  See |viminfo-file-marks|.
+Numbered mark should be stored.  See |shada-file-marks|.
 
 
 							*'[* *`[*
@@ -1074,8 +1074,8 @@ if you stop editing a file without writing, like with ":n!".
 
 When you split a window, the jumplist will be copied to the new window.
 
-If you have included the ' item in the 'viminfo' option the jumplist will be
+If you have included the ' item in the 'shada' option the jumplist will be
-stored in the viminfo file and restored when starting Vim.
+stored in the ShaDa file and restored when starting Vim.
 
 
 CHANGE LIST JUMPS			*changelist* *change-list-jumps* *E664*

runtime/doc/options.txt

@@ -1290,8 +1290,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	used.
 	Conversion between "latin1", "unicode", "ucs-2", "ucs-4" and "utf-8"
 	is done internally by Vim, 'charconvert' is not used for this.
-	'charconvert' is also used to convert the viminfo file, if the 'c'
+	'charconvert' is also used to convert the shada file, if 'encoding' is 
-	flag is present in 'viminfo'.  Also used for Unicode conversion.
+	not "utf-8".  Also used for Unicode conversion.
 	Example: >
 		set charconvert=CharConvert()
 		fun CharConvert()
@@ -2163,7 +2163,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 			feature}
 	Sets the character encoding used inside Vim.  It applies to text in
 	the buffers, registers, Strings in expressions, text stored in the
-	viminfo file, etc.  It sets the kind of characters which Vim can work
+	shada file, etc.  It sets the kind of characters which Vim can work
 	with.  See |encoding-names| for the possible values.
 
 	'encoding' cannot be changed after startup, because (1) it causes
@@ -2195,7 +2195,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 
 	When "unicode", "ucs-2" or "ucs-4" is used, Vim internally uses utf-8.
 	You don't notice this while editing, but it does matter for the
-	|viminfo-file|.  And Vim expects the terminal to use utf-8 too.  Thus
+	|shada-file|.  And Vim expects the terminal to use utf-8 too.  Thus
 	setting 'encoding' to one of these values instead of utf-8 only has
 	effect for encoding used for files when 'fileencoding' is empty.
 
@@ -3407,7 +3407,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 	line below a closed fold.  A match in a previous line which is not
 	drawn may not continue in a newly drawn line.
 	You can specify whether the highlight status is restored on startup
-	with the 'h' flag in 'viminfo' |viminfo-h|.
+	with the 'h' flag in 'shada' |shada-h|.
 
 						*'history'* *'hi'*
 'history' 'hi'		number	(Vim default: 10000, Vi default: 0)
@@ -3793,7 +3793,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	option.
 	Careful: If you change this option, it might break expanding
 	environment variables.  E.g., when '/' is included and Vim tries to
-	expand "$HOME/.viminfo".  Maybe you should change 'iskeyword' instead.
+	expand "$HOME/.nvim/shada/main.shada".  Maybe you should change 
+	'iskeyword' instead.
 
 						*'iskeyword'* *'isk'*
 'iskeyword' 'isk'	string (Vim default for
@@ -5284,6 +5285,123 @@ A jump table for the options with a short description can be found at |Q_op|.
 	with Unix.  The Unix version of Vim cannot source dos format scripts,
 	but the Windows version of Vim can source unix format scripts.
 
+				*'shada'* *'sd'* *E526* *E527* *E528*
+'shada' 'sd'		string	(Vim default for
+				   Win32:  '!,100,<50,s10,h,rA:,rB:
+				   others: '!,100,<50,s10,h
+				 Vi default: "")
+			global
+	When non-empty, the shada file is read upon startup and written
+	when exiting Vim (see |shada-file|).  The string should be a comma
+	separated list of parameters, each consisting of a single character
+	identifying the particular parameter, followed by a number or string
+	which specifies the value of that parameter.  If a particular
+	character is left out, then the default value is used for that
+	parameter.  The following is a list of the identifying characters and
+	the effect of their value.
+	CHAR	VALUE	~
+							*shada-!*
+	!	When included, save and restore global variables that start
+		with an uppercase letter, and don't contain a lowercase
+		letter.  Thus "KEEPTHIS and "K_L_M" are stored, but "KeepThis"
+		and "_K_L_M" are not.  Nested List and Dict items may not be
+		read back correctly, you end up with an empty item.
+							*shada-quote*
+	"	Maximum number of lines saved for each register.  Old name of
+		the '<' item, with the disadvantage that you need to put a
+		backslash before the ", otherwise it will be recognized as the
+		start of a comment!
+							*shada-%*
+	%	When included, save and restore the buffer list.  If Vim is
+		started with a file name argument, the buffer list is not
+		restored.  If Vim is started without a file name argument, the
+		buffer list is restored from the shada file.  Buffers
+		without a file name and buffers for help files are not written
+		to the shada file.
+		When followed by a number, the number specifies the maximum
+		number of buffers that are stored.  Without a number all
+		buffers are stored.
+							*shada-'*
+	'	Maximum number of previously edited files for which the marks
+		are remembered.  This parameter must always be included when
+		'shada' is non-empty.
+		Including this item also means that the |jumplist| and the
+		|changelist| are stored in the shada file.
+							*shada-/*
+	/	Maximum number of items in the search pattern history to be
+		saved.  If non-zero, then the previous search and substitute
+		patterns are also saved.  When not included, the value of
+		'history' is used.
+							*shada-:*
+	:	Maximum number of items in the command-line history to be
+		saved.  When not included, the value of 'history' is used.
+							*shada-<*
+	<	Maximum number of lines saved for each register.  If zero then
+		registers are not saved.  When not included, all lines are
+		saved.  '"' is the old name for this item.
+		Also see the 's' item below: limit specified in KiB.
+							*shada-@*
+	@	Maximum number of items in the input-line history to be
+		saved.  When not included, the value of 'history' is used.
+							*shada-c*
+	c	Dummy option, kept for compatibility reasons.  Has no actual 
+		effect.  Current encoding state is described in 
+		|shada-encoding|.
+							*shada-f*
+	f	Whether file marks need to be stored.  If zero, file marks ('0
+		to '9, 'A to 'Z) are not stored.  When not present or when
+		non-zero, they are all stored.  '0 is used for the current
+		cursor position (when exiting or when doing |:wshada|).
+							*shada-h*
+	h	Disable the effect of 'hlsearch' when loading the shada
+		file.  When not included, it depends on whether ":nohlsearch"
+		has been used since the last search command.
+							*shada-n*
+	n	Name of the shada file.  The name must immediately follow
+		the 'n'.  Must be the last one!  If the "-i" argument was
+		given when starting Vim, that file name overrides the one
+		given here with 'shada'.  Environment variables are expanded
+		when opening the file, not when setting the option.
+							*shada-r*
+	r	Removable media.  The argument is a string (up to the next
+		',').  This parameter can be given several times.  Each
+		specifies the start of a path for which no marks will be
+		stored.  This is to avoid removable media.  For MS-DOS you
+		could use "ra:,rb:".  You can also use it for temp files,
+		e.g., for Unix: "r/tmp".  Case is ignored.
+							*shada-s*
+	s	Maximum size of an item contents in KiB.  If zero then nothing 
+		is saved.  Unlike Vim this applies to all items, except for 
+		the buffer list and header.  Full item size is off by three 
+		unsigned integers: with `s10` maximum item size may be 1 byte 
+		(type: 7-bit integer) + 9 bytes (timestamp: up to 64-bit 
+		integer) + 3 bytes (item size: up to 16-bit integer because 
+		2^8 < 10240 < 2^16) + 10240 bytes (requested maximum item 
+		contents size) = 10253 bytes.
+
+	Example: >
+	    :set shada='50,<1000,s100,:0,n~/nvim/shada
+<
+	'50		Marks will be remembered for the last 50 files you
+			edited.
+	<1000		Contents of registers (up to 1000 lines each) will be
+			remembered.
+	s100		Items with contents occupying more then 100 KiB are 
+			skipped.
+	:0		Command-line history will not be saved.
+	n~/nvim/shada	The name of the file to use is "~/nvim/shada".
+	no /		Since '/' is not specified, the default will be used,
+			that is, save all of the search history, and also the
+			previous search and substitute patterns.
+	no %		The buffer list will not be saved nor read back.
+	no h		'hlsearch' highlighting will be restored.
+
+	When setting 'shada' from an empty value you can use |:rshada| to
+	load the contents of the file, this is not done automatically.
+
+	This option cannot be set from a |modeline| or in the |sandbox|, for
+	security reasons.
+
 						*'shell'* *'sh'* *E91*
 'shell' 'sh'		string	(default $SHELL or "sh",
 					MS-DOS and Win32: "command.com" or
@@ -6616,7 +6734,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 			global
 	When bigger than zero, Vim will give messages about what it is doing.
 	Currently, these messages are given:
-	>= 1	When the viminfo file is read or written.
+	>= 1	When the shada file is read or written.
 	>= 2	When a file is ":source"'ed.
 	>= 5	Every searched tags file and include file.
 	>= 8	Files for which a group of autocommands is executed.
@@ -6676,120 +6794,11 @@ A jump table for the options with a short description can be found at |Q_op|.
 	with Unix.  The Unix version of Vim cannot source dos format scripts,
 	but the Windows version of Vim can source unix format scripts.
 
-				*'viminfo'* *'vi'* *E526* *E527* *E528*
+				*'viminfo'* *'vi'*
-'viminfo' 'vi'		string	(Vim default for
+'viminfo' 'vi'		string
-				   Win32:  '!,100,<50,s10,h,rA:,rB:
-				   others: '!,100,<50,s10,h
-				 Vi default: "")
 			global
-			{not available when compiled without the |+viminfo|
+	Deprecated alias for 'shada' option. Is kept for compatibility 
-			feature}
+	reasons.
-	When non-empty, the viminfo file is read upon startup and written
-	when exiting Vim (see |viminfo-file|).  The string should be a comma
-	separated list of parameters, each consisting of a single character
-	identifying the particular parameter, followed by a number or string
-	which specifies the value of that parameter.  If a particular
-	character is left out, then the default value is used for that
-	parameter.  The following is a list of the identifying characters and
-	the effect of their value.
-	CHAR	VALUE	~
-							*viminfo-!*
-	!	When included, save and restore global variables that start
-		with an uppercase letter, and don't contain a lowercase
-		letter.  Thus "KEEPTHIS and "K_L_M" are stored, but "KeepThis"
-		and "_K_L_M" are not.  Nested List and Dict items may not be
-		read back correctly, you end up with an empty item.
-							*viminfo-quote*
-	"	Maximum number of lines saved for each register.  Old name of
-		the '<' item, with the disadvantage that you need to put a
-		backslash before the ", otherwise it will be recognized as the
-		start of a comment!
-							*viminfo-%*
-	%	When included, save and restore the buffer list.  If Vim is
-		started with a file name argument, the buffer list is not
-		restored.  If Vim is started without a file name argument, the
-		buffer list is restored from the viminfo file.  Buffers
-		without a file name and buffers for help files are not written
-		to the viminfo file.
-		When followed by a number, the number specifies the maximum
-		number of buffers that are stored.  Without a number all
-		buffers are stored.
-							*viminfo-'*
-	'	Maximum number of previously edited files for which the marks
-		are remembered.  This parameter must always be included when
-		'viminfo' is non-empty.
-		Including this item also means that the |jumplist| and the
-		|changelist| are stored in the viminfo file.
-							*viminfo-/*
-	/	Maximum number of items in the search pattern history to be
-		saved.  If non-zero, then the previous search and substitute
-		patterns are also saved.  When not included, the value of
-		'history' is used.
-							*viminfo-:*
-	:	Maximum number of items in the command-line history to be
-		saved.  When not included, the value of 'history' is used.
-							*viminfo-<*
-	<	Maximum number of lines saved for each register.  If zero then
-		registers are not saved.  When not included, all lines are
-		saved.  '"' is the old name for this item.
-		Also see the 's' item below: limit specified in Kbyte.
-							*viminfo-@*
-	@	Maximum number of items in the input-line history to be
-		saved.  When not included, the value of 'history' is used.
-							*viminfo-c*
-	c	When included, convert the text in the viminfo file from the
-		'encoding' used when writing the file to the current
-		'encoding'.  See |viminfo-encoding|.
-							*viminfo-f*
-	f	Whether file marks need to be stored.  If zero, file marks ('0
-		to '9, 'A to 'Z) are not stored.  When not present or when
-		non-zero, they are all stored.  '0 is used for the current
-		cursor position (when exiting or when doing ":wviminfo").
-							*viminfo-h*
-	h	Disable the effect of 'hlsearch' when loading the viminfo
-		file.  When not included, it depends on whether ":nohlsearch"
-		has been used since the last search command.
-							*viminfo-n*
-	n	Name of the viminfo file.  The name must immediately follow
-		the 'n'.  Must be the last one!  If the "-i" argument was
-		given when starting Vim, that file name overrides the one
-		given here with 'viminfo'.  Environment variables are expanded
-		when opening the file, not when setting the option.
-							*viminfo-r*
-	r	Removable media.  The argument is a string (up to the next
-		',').  This parameter can be given several times.  Each
-		specifies the start of a path for which no marks will be
-		stored.  This is to avoid removable media.  For MS-DOS you
-		could use "ra:,rb:".  You can also use it for temp files,
-		e.g., for Unix: "r/tmp".  Case is ignored.  Maximum length of
-		each 'r' argument is 50 characters.
-							*viminfo-s*
-	s	Maximum size of an item in Kbyte.  If zero then registers are
-		not saved.  Currently only applies to registers.  The default
-		"s10" will exclude registers with more than 10 Kbyte of text.
-		Also see the '<' item above: line count limit.
-
-	Example: >
-	    :set viminfo='50,<1000,s100,:0,n~/vim/viminfo
-<
-	'50		Marks will be remembered for the last 50 files you
-			edited.
-	<1000		Contents of registers (up to 1000 lines each) will be
-			remembered.
-	s100		Registers with more than 100 Kbyte text are skipped.
-	:0		Command-line history will not be saved.
-	n~/vim/viminfo	The name of the file to use is "~/vim/viminfo".
-	no /		Since '/' is not specified, the default will be used,
-			that is, save all of the search history, and also the
-			previous search and substitute patterns.
-	no %		The buffer list will not be saved nor read back.
-	no h		'hlsearch' highlighting will be restored.
-
-	When setting 'viminfo' from an empty value you can use |:rviminfo| to
-	load the contents of the file, this is not done automatically.
-
-	This option cannot be set from a |modeline| or in the |sandbox|, for
-	security reasons.
 
 					    *'virtualedit'* *'ve'*
 'virtualedit' 've'	string	(default "")

runtime/doc/quickref.txt

@@ -838,6 +838,7 @@ Short explanation of each option:		*option-list*
 'selection'	  'sel'     what type of selection to use
 'selectmode'	  'slm'     when to use Select mode instead of Visual mode
 'sessionoptions'  'ssop'    options for |:mksession|
+'shada'		  'sd'	    use .shada file upon startup and exiting
 'shell'		  'sh'	    name of shell to use for external commands
 'shellcmdflag'	  'shcf'    flag to shell to execute one command
 'shellpipe'	  'sp'	    string to put output of ":make" in error file
@@ -911,7 +912,6 @@ Short explanation of each option:		*option-list*
 'verbosefile'	  'vfile'   file to write messages in
 'viewdir'	  'vdir'    directory where to store files with :mkview
 'viewoptions'	  'vop'     specifies what to save for :mkview
-'viminfo'	  'vi'	    use .viminfo file upon startup and exiting
 'virtualedit'	  've'	    when to use virtual editing
 'visualbell'	  'vb'	    use visual bell instead of beeping
 'warn'			    warn for shell command when buffer was changed
@@ -1131,7 +1131,7 @@ Context-sensitive completion on the command-line:
 |-w|	-w {scriptout}	    write typed chars to file {scriptout} (append)
 |-W|	-W {scriptout}	    write typed chars to file {scriptout} (overwrite)
 |-u|	-u {vimrc}	    read inits from {vimrc} instead of other inits
-|-i|	-i {viminfo}	    read info from {viminfo} instead of other files
+|-i|	-i {shada}	    read info from {shada} instead of other files
 |---|	--		    end of options, other arguments are file names
 |--help|    --help	    show list of arguments and exit
 |--version| --version	    show version info and exit
@@ -1215,12 +1215,12 @@ Context-sensitive completion on the command-line:
 ------------------------------------------------------------------------------
 *Q_ac*		Automatic Commands
 
-|viminfo-file|	read registers, marks, history at startup, save when exiting.
+|shada-file|	read registers, marks, history at startup, save when exiting.
 
-|:rviminfo|	:rv[iminfo] [file]	read info from viminfo file [file]
+|:rshada|	:rsh[ada] [file]	read info from ShaDa file [file]
-|:rviminfo|	:rv[iminfo]! [file]	idem, overwrite existing info
+|:rshada|	:rsh[ada]! [file]	idem, overwrite existing info
-|:wviminfo|	:wv[iminfo] [file]	add info to viminfo file [file]
+|:wshada|	:wsh[ada] [file]	add info to ShaDa file [file]
-|:wviminfo|	:wv[iminfo]! [file]	write info to viminfo file [file]
+|:wshada|	:wsh[ada]! [file]	write info to ShaDa file [file]
 
 |modeline|	Automatic option setting when editing a file
 

runtime/doc/starting.txt

@@ -12,7 +12,7 @@ Starting Vim						*starting*
 4. Suspending			|suspend|
 5. Saving settings		|save-settings|
 6. Views and Sessions		|views-sessions|
-7. The viminfo file		|viminfo-file|
+7. The ShaDa file		|shada-file|
 
 ==============================================================================
 1. Vim arguments					*vim-arguments*
@@ -128,7 +128,7 @@ argument.
 		"pat" in the first file being edited (see |pattern| for the
 		available search patterns).  The search starts at the cursor
 		position, which can be the first line or the cursor position
-		last used from |viminfo|. To force a search from the first
+		last used from |shada|. To force a search from the first
 		line use "+1 +/pat".
 
 +{command}						*-+c* *-c*
@@ -251,7 +251,7 @@ argument.
 							*-V* *verbose*
 -V[N]		Verbose.  Sets the 'verbose' option to [N] (default: 10).
 		Messages will be given for each file that is ":source"d and
-		for reading or writing a viminfo file.  Can be used to find
+		for reading or writing a ShaDa file.  Can be used to find
 		out what is happening upon startup and exit.
 		Example: >
 			vim -V8 foobar
@@ -322,10 +322,10 @@ argument.
 		same effect as "NONE", but loading plugins is not skipped.
 
 							*-i*
--i {viminfo}	The file "viminfo" is used instead of the default viminfo
+-i {shada}	The file {shada} is used instead of the default ShaDa
-		file.  If the name "NONE" is used (all uppercase), no viminfo
+		file.  If the name "NONE" is used (all uppercase), no ShaDa
-		file is read or written, even if 'viminfo' is set or when
+		file is read or written, even if 'shada' is set or when
-		":rv" or ":wv" are used.  See also |viminfo-file|.
+		":rsh" or ":wsh" are used.  See also |shada-file|.
 
 							*-s*
 -s {scriptin}	The script file "scriptin" is read.  The characters in the
@@ -475,9 +475,9 @@ accordingly.  Vim proceeds in this order:
 	Only when starting "gvim", the GUI initializations will be done.  See
 	|gui-init|.
 
-9. Read the viminfo file
+9. Read the ShaDa file
-	If the 'viminfo' option is not empty, the viminfo file is read.  See
+	If the 'shada' option is not empty, the ShaDa file is read.  See
-	|viminfo-file|.
+	|shada-file|.
 
 10. Read the quickfix file
 	If the "-q" flag was given to Vim, the quickfix file is read.  If this
@@ -564,10 +564,10 @@ just like executing a command from a vimrc/exrc in the current directory.
 If Vim takes a long time to start up, use the |--startuptime| argument to find
 out what happens.
 
-If you have "viminfo" enabled, the loading of the viminfo file may take a
+If you have 'shada' enabled, the loading of the ShaDa file may take a
-while.  You can find out if this is the problem by disabling viminfo for a
+while.  You can find out if this is the problem by disabling ShaDa for a
 moment (use the Vim argument "-i NONE", |-i|).  Try reducing the number of
-lines stored in a register with ":set viminfo='20,<50,s10".  |viminfo-file|.
+lines stored in a register with ":set shada='20,<50,s10".  |shada-file|.
 
 							*:intro*
 When Vim starts without a file name, an introductory message is displayed (for
@@ -768,8 +768,8 @@ save a Session and when you restore it later the window layout looks the same.
 You can use a Session to quickly switch between different projects,
 automatically loading the files you were last working on in that project.
 
-Views and Sessions are a nice addition to viminfo-files, which are used to
+Views and Sessions are a nice addition to ShaDa files, which are used to
-remember information for all Views and Sessions together |viminfo-file|.
+remember information for all Views and Sessions together |shada-file|.
 
 You can quickly start editing with a previously saved View or Session with the
 |-S| argument: >
@@ -865,7 +865,7 @@ The output of ":mkview" contains these items:
 Note that Views and Sessions are not perfect:
 - They don't restore everything.  For example, defined functions, autocommands
   and ":syntax on" are not included.  Things like register contents and
-  command line history are in viminfo, not in Sessions or Views.
+  command line history are in ShaDa, not in Sessions or Views.
 - Global option values are only set when they differ from the default value.
   When the current value is not the default value, loading a Session will not
   set it back to the default value.  Local options will be set back to the
@@ -896,15 +896,16 @@ To automatically save and restore views for *.c files: >
 	au BufWinEnter *.c silent loadview
 
 ==============================================================================
-8. The viminfo file				*viminfo* *viminfo-file* *E136*
+8. The ShaDa file				*shada* *shada-file*
-						*E575* *E576* *E577*
+
 If you exit Vim and later start it again, you would normally lose a lot of
-information.  The viminfo file can be used to remember that information, which
+information.  The ShaDa file can be used to remember that information, which
-enables you to continue where you left off.
+enables you to continue where you left off.  Its name is the abbreviation of 
+SHAred DAta because it is used for sharing data between Neovim sessions.
 
 This is introduced in section |21.3| of the user manual.
 
-The viminfo file is used to store:
+The ShaDa file is used to store:
 - The command line history.
 - The search string history.
 - The input-line history.
@@ -915,62 +916,59 @@ The viminfo file is used to store:
 - The buffer list.
 - Global variables.
 
-The viminfo file is not supported when the |+viminfo| feature has been
+You could also use a Session file.  The difference is that the ShaDa file
-disabled at compile time.
-
-You could also use a Session file.  The difference is that the viminfo file
 does not depend on what you are working on.  There normally is only one
-viminfo file.  Session files are used to save the state of a specific editing
+ShaDa file.  Session files are used to save the state of a specific editing
 Session.  You could have several Session files, one for each project you are
-working on.  Viminfo and Session files together can be used to effectively
+working on.  ShaDa and Session files together can be used to effectively
 enter Vim and directly start working in your desired setup. |session-file|
 
-							*viminfo-read*
+							*shada-read*
-When Vim is started and the 'viminfo' option is non-empty, the contents of
+When Vim is started and the 'shada' option is non-empty, the contents of
-the viminfo file are read and the info can be used in the appropriate places.
+the ShaDa file are read and the info can be used in the appropriate places.
 The |v:oldfiles| variable is filled.  The marks are not read in at startup
-(but file marks are).  See |initialization| for how to set the 'viminfo'
+(but file marks are).  See |initialization| for how to set the 'shada'
 option upon startup.
 
-							*viminfo-write*
+							*shada-write*
-When Vim exits and 'viminfo' is non-empty, the info is stored in the viminfo
+When Vim exits and 'shada' is non-empty, the info is stored in the ShaDa file 
-file (it's actually merged with the existing one, if one exists).  The
+(it's actually merged with the existing one, if one exists |shada-merging|).  
-'viminfo' option is a string containing information about what info should be
+The 'shada' option is a string containing information about what info should 
-stored, and contains limits on how much should be stored (see 'viminfo').
+be stored, and contains limits on how much should be stored (see 'shada').
 
 Notes for Unix:
-- The file protection for the viminfo file will be set to prevent other users
+- The file protection for the ShaDa file will be set to prevent other users
   from being able to read it, because it may contain any text or commands that
   you have worked with.
-- If you want to share the viminfo file with other users (e.g. when you "su"
+- If you want to share the ShaDa file with other users (e.g. when you "su"
   to another user), you can make the file writable for the group or everybody.
-  Vim will preserve this when writing new viminfo files.  Be careful, don't
+  Vim will preserve this when writing new ShaDa files.  Be careful, don't
-  allow just anybody to read and write your viminfo file!
+  allow just anybody to read and write your ShaDa file!
-- Vim will not overwrite a viminfo file that is not writable by the current
+- Vim will not overwrite a ShaDa file that is not writable by the current
   "real" user.  This helps for when you did "su" to become root, but your
   $HOME is still set to a normal user's home directory.  Otherwise Vim would
-  create a viminfo file owned by root that nobody else can read.
+  create a ShaDa file owned by root that nobody else can read.
-- The viminfo file cannot be a symbolic link.  This is to avoid security
+- The ShaDa file cannot be a symbolic link.  This is to avoid security
   issues.
 
-Marks are stored for each file separately.  When a file is read and 'viminfo'
+Marks are stored for each file separately.  When a file is read and 'shada'
-is non-empty, the marks for that file are read from the viminfo file.  NOTE:
+is non-empty, the marks for that file are read from the ShaDa file.  NOTE:
 The marks are only written when exiting Vim, which is fine because marks are
 remembered for all the files you have opened in the current editing session,
 unless ":bdel" is used.  If you want to save the marks for a file that you are
-about to abandon with ":bdel", use ":wv".  The '[' and ']' marks are not
+about to abandon with ":bdel", use ":wsh".  The '[' and ']' marks are not
 stored, but the '"' mark is.  The '"' mark is very useful for jumping to the
 cursor position when the file was last exited.  No marks are saved for files
-that start with any string given with the "r" flag in 'viminfo'.  This can be
+that start with any string given with the "r" flag in 'shada'.  This can be
 used to avoid saving marks for files on removable media (for MS-DOS you would
 use "ra:,rb:".
-The |v:oldfiles| variable is filled with the file names that the viminfo file
+The |v:oldfiles| variable is filled with the file names that the ShaDa file
 has marks for.
 
-							*viminfo-file-marks*
+							*shada-file-marks*
-Uppercase marks ('A to 'Z) are stored when writing the viminfo file.  The
+Uppercase marks ('A to 'Z) are stored when writing the ShaDa file.  The
-numbered marks ('0 to '9) are a bit special.  When the viminfo file is written
+numbered marks ('0 to '9) are a bit special.  When the ShaDa file is written
-(when exiting or with the ":wviminfo" command), '0 is set to the current cursor
+(when exiting or with the |:wshada| command), '0 is set to the current cursor
 position and file.  The old '0 is moved to '1, '1 to '2, etc.  This
 resembles what happens with the "1 to "9 delete registers.  If the current
 cursor position is already present in '0 to '9, it is moved to '0, to avoid
@@ -988,93 +986,196 @@ For a bash-like shell: >
 
 	alias lvim='vim -c "normal '\''0"'
 
-Use the "r" flag in 'viminfo' to specify for which files no marks should be
+Use the "r" flag in 'shada' to specify for which files no marks should be
 remembered.
 
+MERGING							*shada-merging*
+									{Nvim}
+When writing ShaDa files with |:wshada| without bang or at regular exit 
+information in the existing ShaDa file is merged with information from current 
+Neovim instance.  For this purpose ShaDa files store timestamps associated 
+with ShaDa entries.  Specifically the following is being done:
 
-VIMINFO FILE NAME					*viminfo-file-name*
+1. History lines are merged, ordered by timestamp.  Maximum amount of items in 
+   ShaDa file is defined by 'shada' option (|shada-/|, |shada-:|, |shada-@|, 
+   etc: one suboption for each character that represents history name 
+   (|:history|)).
+2. Local marks and changes for files that were not opened by Neovim are copied 
+   to new ShaDa file. Marks for files that were opened by Neovim are merged, 
+   changes to files opened by Neovim are ignored. |shada-'|
+3. Jump list is merged: jumps are ordered by timestamp, identical jumps 
+   (identical position AND timestamp) are squashed.
+4. Search patterns and substitute strings are not merged: search pattern or 
+   substitute string which has greatest timestamp will be the only one copied 
+   to ShaDa file.
+5. For each register entity with greatest timestamp is the only saved. 
+   |shada-<|
+6. All saved variables are saved from current Neovim instance. Additionally 
+   existing variable values are copied, meaning that the only way to remove 
+   variable from a ShaDa file is either removing it by hand or disabling 
+   writing variables completely. |shada-!|
+7. For each global mark entity with greatest timestamp is the only saved.
+8. Buffer list and header are the only entries which are not merged in any 
+   fashion: the only header and buffer list present are the ones from the 
+   Neovim instance which was last writing the file. |shada-%|
 
-- The default name of the viminfo file is "$HOME/.viminfo" for Unix,
+COMPATIBILITY						*shada-compatibility*
-  "$HOME\_viminfo" for MS-DOS and Win32.  For the last two, when $HOME is not
+									{Nvim}
-  set, "$VIM\_viminfo" is used.  When $VIM is also not set, "c:\_viminfo" is
+ShaDa files are forward and backward compatible.  This means that
-  used.
+
-- The 'n' flag in the 'viminfo' option can be used to specify another viminfo
+1. Entries which have unknown type (i.e. that hold unidentified data) are 
-  file name |'viminfo'|.
+   ignored when reading and blindly copied when writing.
+2. Register entries with unknown register name are ignored when reading and 
+   blindly copied when writing. Limitation: only registers that use name with 
+   code in interval [1, 255] are supported. |registers|
+3. Register entries with unknown register type are ignored when reading and 
+   merged as usual when writing. |getregtype()|
+4. Local and global mark entries with unknown mark names are ignored when 
+   reading. When writing global mark entries are blindly copied and local mark 
+   entries are also blindly copied, but only if file they are attached to fits 
+   in the |shada-'| limit. Unknown local mark entry's timestamp is also taken 
+   into account when calculating which files exactly should fit into this 
+   limit. Limitation: only marks that use name with code in interval [1, 255] 
+   are supported. |mark-motions|
+5. History entries with unknown history type are ignored when reading and 
+   blindly copied when writing. Limitation: there can be only up to 256 
+   history types. |history|
+6. Unknown keys found in register, local mark, global mark, change, jump and 
+   search pattern entries are saved internally and dumped when writing. 
+   Entries created during Neovim session never have such additions.
+7. Additional elements found in replacement string and history entries are 
+   saved internally and dumped. Entries created during Neovim session never 
+   have such additions.
+8. Additional elements found in variable entries are simply ignored when 
+   reading. When writing new variables they will be preserved during merging, 
+   but that's all. Variable values dumped from current Neovim session never 
+   have additional elements, even if variables themselves were obtained by 
+   reading ShaDa files.
+
+"Blindly" here means that there will be no attempts to somehow merge them, 
+even if other entries (with known name/type/etc) are merged. |shada-merging|
+
+SHADA FILE NAME						*shada-file-name*
+
+- The default name of the ShaDa file is "$HOME/.nvim/shada/main.shada" for 
+  Unix, "$HOME\_nvim\shada\main.shada" for MS-DOS and Win32.  For the last 
+  two, when $HOME is not set, "$VIM\_nvim\shada\main.shada" is used.  When 
+  $VIM is also not set, "c:\_nvim\shada\main.shada" is used.
+- The 'n' flag in the 'shada' option can be used to specify another ShaDa
+  file name |'shada'|.
 - The "-i" Vim argument can be used to set another file name, |-i|.  When the
-  file name given is "NONE" (all uppercase), no viminfo file is ever read or
+  file name given is "NONE" (all uppercase), no ShaDa file is ever read or
   written.  Also not for the commands below!
 - For the commands below, another file name can be given, overriding the
-  default and the name given with 'viminfo' or "-i" (unless it's NONE).
+  default and the name given with 'shada' or "-i" (unless it's NONE).
 
 
-CHARACTER ENCODING					*viminfo-encoding*
+CHARACTER ENCODING					*shada-encoding*
 
-The text in the viminfo file is encoded as specified with the 'encoding'
+The text in the ShaDa file is UTF-8-encoded.  Normally you will always work 
-option.  Normally you will always work with the same 'encoding' value, and
+with the same 'encoding' value, and this works just fine.  However, if you 
-this works just fine.  However, if you read the viminfo file with another
+read the ShaDa file with value for 'encoding' different from utf-8 and 
-value for 'encoding' than what it was written with, some of the text
+'encoding' used when writing ShaDa file, some of the text (non-ASCII 
-(non-ASCII characters) may be invalid.  If this is unacceptable, add the 'c'
+characters) may be invalid as Neovim always attempts to convert the text in 
-flag to the 'viminfo' option: >
+the ShaDa file from the UTF-8 to the current 'encoding' value.  Filenames are 
-	:set viminfo+=c
+never converted, affected elements are:
-Vim will then attempt to convert the text in the viminfo file from the
+
-'encoding' value it was written with to the current 'encoding' value.  This
+- history strings;
-requires Vim to be compiled with the |+iconv| feature.  Filenames are not
+- variable values;
-converted.
+- register values;
+- last used search and substitute patterns;
+- last used substitute replacement string.
 
 
-MANUALLY READING AND WRITING				*viminfo-read-write*
+MANUALLY READING AND WRITING				*shada-read-write*
 
-Two commands can be used to read and write the viminfo file manually.  This
+Two commands can be used to read and write the ShaDa file manually.  This
 can be used to exchange registers between two running Vim programs: First
-type ":wv" in one and then ":rv" in the other.  Note that if the register
+type ":wsh" in one and then ":rsh" in the other.  Note that if the register
-already contained something, then ":rv!" would be required.  Also note
+already contained something, then ":rsh!" would be required.  Also note
 however that this means everything will be overwritten with information from
 the first Vim, including the command line history, etc.
 
-The viminfo file itself can be edited by hand too, although we suggest you
+The ShaDa file itself can be edited by hand too, although we suggest you
-start with an existing one to get the format right.  It is reasonably
+start with an existing one to get the format right.  You need to understand 
-self-explanatory once you're in there.  This can be useful in order to
+MessagePack (or, more likely, find software that is able to use it) format to 
-create a second file, say "~/.my_viminfo" which could contain certain
+do this.  This can be useful in order to create a second file, say 
-settings that you always want when you first start Vim.  For example, you
+"~/.my.shada" which could contain certain settings that you always want when 
-can preload registers with particular data, or put certain commands in the
+you first start Neovim.  For example, you can preload registers with 
-command line history.  A line in your .vimrc file like >
+particular data, or put certain commands in the command line history.  A line 
-	:rviminfo! ~/.my_viminfo
+in your .nvimrc file like >
-can be used to load this information.  You could even have different viminfos
+	:rshada! ~/.my.shada
-for different types of files (e.g., C code) and load them based on the file
+can be used to load this information.  You could even have different ShaDa 
-name, using the ":autocmd" command (see |:autocmd|).
+files for different types of files (e.g., C code) and load them based on the 
+file name, using the ":autocmd" command (see |:autocmd|).  More information on 
+ShaDa file format is contained in |shada-format| section.
 
-							*viminfo-errors*
+					  *E136* *E138* *shada-error-handling*
-When Vim detects an error while reading a viminfo file, it will not overwrite
+Some errors make Neovim leave temporary file named `{basename}.tmp.X` (X is 
-that file.  If there are more than 10 errors, Vim stops reading the viminfo
+any free letter from `a` to `z`) while normally it will create this file, 
-file.  This was done to avoid accidentally destroying a file when the file
+write to it and then rename `{basename}.tmp.X` to `{basename}`. Such errors 
-name of the viminfo file is wrong.  This could happen when accidentally typing
+include:
-"vim -i file" when you wanted "vim -R file" (yes, somebody accidentally did
-that!).  If you want to overwrite a viminfo file with an error in it, you will
-either have to fix the error, or delete the file (while Vim is running, so
-most of the information will be restored).
 
-						   *:rv* *:rviminfo* *E195*
+- Errors which make Neovim think that read file is not a ShaDa file at all: 
-:rv[iminfo][!] [file]	Read from viminfo file [file] (default: see above).
+  non-ShaDa files are not overwritten for safety reasons to avoid accidentally 
+  destroying an unrelated file.  This could happen e.g. when typing "nvim -i 
+  file" in place of "nvim -R file" (yes, somebody did that at least with Vim).  
+  Such errors are listed at |shada-critical-contents-errors|.
+- If writing to the temporary file failed: e.g. because of the insufficient 
+  space left.
+- If renaming file failed: e.g. because of insufficient permissions.
+- If target ShaDa file has different from the Neovim instance's owners (user 
+  and group) and changing them failed.  Unix-specific, applies only when 
+  Neovim was launched from root.
+
+Do not forget to remove the temporary file or replace the target file with 
+temporary one after getting one of the above errors or all attempts to create 
+a ShaDa file may fail with |E138|.  If you got one of them when using 
+|:wshada| (and not when exiting Neovim: i.e. when you have Neovim session 
+running) you have additional options:
+
+- First thing which you should consider if you got any error, except failure 
+  to write to the temporary file: remove existing file and replace it with the 
+  temporary file.  Do it even if you have running Neovim instance.
+- Fix the permissions and/or file ownership, free some space and attempt to 
+  write again.  Do not remove the existing file.
+- Use |:wshada| with bang.  Does not help in case of permission error.  If 
+  target file was actually the ShaDa file some information may be lost in this 
+  case.  To make the matters slightly better use |:rshada| prior to writing, 
+  but this still will loose buffer-local marks and change list entries for any 
+  file which is not opened in the current Neovim instance.
+- Remove the target file from shell and use |:wshada|.  Consequences are not 
+  different from using |:wshada| with bang, but "rm -f" works in some cases 
+  when you don't have write permissions.
+
+						    *:rsh* *:rshada* *E886*
+:rsh[ada][!] [file]	Read from ShaDa file [file] (default: see above).
 			If [!] is given, then any information that is
 			already set (registers, marks, |v:oldfiles|, etc.)
 			will be overwritten.
 
-				*:wv* *:wviminfo* *E137* *E138* *E574* *E886*
+						   *:rv* *:rviminfo*
-:wv[iminfo][!] [file]	Write to viminfo file [file] (default: see above).
+:rv[iminfo][!] [file]	Deprecated alias to |:rshada| command.
+
+						    *:wsh* *:wshada* *E137*
+:wsh[ada][!] [file]	Write to ShaDa file [file] (default: see above).
 			The information in the file is first read in to make
 			a merge between old and new info.  When [!] is used,
 			the old information is not read first, only the
-			internal info is written.  If 'viminfo' is empty, marks
+			internal info is written (also disables safety checks 
-			for up to 100 files will be written.
+			described in |shada-error-handling|).  If 'shada' is 
-			When you get error "E138: Can't write viminfo file"
+			empty, marks for up to 100 files will be written.
-			check that no old temp files were left behind (e.g.
+			When you get error "E138: All .tmp.X files exist, 
-			~/.viminf*) and that you can write in the directory of
+			cannot write ShaDa file!" check that no old temp files 
-			the .viminfo file.
+			were left behind (e.g. ~/.nvim/shada/main.shada.tmp*).
+
+						   *:wv* *:wviminfo*
+:wv[iminfo][!] [file]	Deprecated alias to |:wshada| command.
 
 						*:ol* *:oldfiles*
-:ol[dfiles]		List the files that have marks stored in the viminfo
+:ol[dfiles]		List the files that have marks stored in the ShaDa
 			file.  This list is read on startup and only changes
-			afterwards with ":rviminfo!".  Also see |v:oldfiles|.
+			afterwards with ":rshada!".  Also see |v:oldfiles|.
 			The number can be used with |c_#<|.
 
 :bro[wse] ol[dfiles][!]
@@ -1085,4 +1186,159 @@ most of the information will be restored).
 			and still get the prompt to enter a file number.
 			Use ! to abandon a modified buffer. |abandon|
 
+SHADA FILE FORMAT						*shada-format*
+
+ShaDa files are concats of MessagePack entries.  Each entry is a concat of 
+exactly four MessagePack objects:
+
+1. First goes type of the entry.  Object type must be an unsigned integer.  
+   Object type must not be equal to zero.
+2. Second goes entry timestamp.  It must also be an unsigned integer.
+3. Third goes the length of the fourth entry.  Unsigned integer as well, used 
+   for fast skipping without parsing.
+4. Fourth is actual entry data.  All currently used ShaDa entries use 
+   containers to hold data: either map or array.  Exact format depends on the 
+   entry type:
+
+   Entry type (name)   Entry data ~
+   1 (Header)          Map containing data that describes the generator 
+                       instance that wrote this ShaDa file.  It is ignored 
+                       when reading ShaDa files.  Contains the following data:
+                       Key        Data ~
+                       generator  Binary, software used to generate ShaDa 
+                                  file. Is equal to "nvim" when ShaDa file was 
+                                  written by Neovim.
+                       version    Binary, generator version.
+                       encoding   Binary, effective 'encoding' value.
+                       max_kbyte  Integer, effective |shada-s| limit value.
+                       pid        Integer, instance process ID.
+                       *          It is allowed to have any number of 
+                                  additional keys with any data.
+   2 (SearchPattern)   Map containing data describing last used search or 
+                       substitute pattern.  Normally ShaDa file contains two 
+                       such entries: one with "ss" key set to true (describes 
+                       substitute pattern, see |:substitute|), and one set to 
+                       false (describes search pattern, see 
+                       |search-commands|). "su" key should be true on one of 
+                       the entries.  If key value is equal to default then it 
+                       is normally not present.  Keys:
+                       Key  Type     Default  Description ~
+                       sm   Boolean  true     Effective 'magic' value.
+                       sc   Boolean  false    Effective 'smartcase' value.
+                       sl   Boolean  true     True if search pattern comes 
+                                              with a line offset.  See 
+                                              |search-offset|.
+                       se   Boolean  false    True if |search-offset| 
+                                              requested to place cursor at 
+                                              (relative to) the end of the 
+                                              pattern.
+                       so   Integer  0        Offset value. |search-offset|
+                       su   Boolean  false    True if current entry was the 
+                                              last used search pattern.
+                       ss   Boolean  false    True if current entry describes 
+                                              |:substitute| pattern.
+                       sh   Boolean  false    True if |v:hlsearch| is on.
+                                              With |shada-h| or 'nohlsearch' 
+                                              this key is always false.
+                       sp   Binary   N/A      Actual pattern.  Required.
+                       *    any      none     Other keys are allowed for 
+                                              compatibility reasons, see 
+                                              |shada-compatibility|.
+   3 (SubString)       Array containing last |:substitute| replacement string.  
+                       Contains single entry: binary, replacement string used.  
+                       More entries are allowed for compatibility reasons, see 
+                       |shada-compatibility|.
+   4 (HistoryEntry)    Array containing one entry from history.  Should have 
+                       two or three entries.  First one is history type 
+                       (unsigned integer), second is history line (binary), 
+                       third is the separator character (unsigned integer, 
+                       must be in interval [0, 255]).  Third item is only 
+                       valid for search history.  Possible history types are 
+                       listed in |hist-names|, here are the corresponding 
+                       numbers: 0 - cmd, 1 - search, 2 - expr, 3 - input, 
+                       4 - debug.
+   5 (Register)        Map describing one register (|registers|).  If key 
+                       value is equal to default then it is normally not 
+                       present.  Keys:
+                       Key  Type             Def  Description ~
+                       rt   UInteger         0    Register type:
+                                                  No  Description ~
+                                                  0   |characterwise-register|
+                                                  1   |linewise-register|
+                                                  2   |blockwise-register|
+                       rw   UInteger         0    Register width. Only valid 
+                                                  for |blockwise-register|s.
+                       rc   Array of binary  N/A  Register contents.  Each 
+                                                  entry in the array 
+                                                  represents its own line.  
+                                                  NUL characters inside the 
+                                                  line should be represented 
+                                                  as NL according to 
+                                                  |NL-used-for-Nul|.
+                       n    UInteger         N/A  Register name: character 
+                                                  code in range [1, 255].  
+                                                  Example: |quote0| register 
+                                                  has name 48 (ASCII code for 
+                                                  zero character).
+                       *    any              none Other keys are allowed 
+                                                  for compatibility reasons, 
+                                                  see |shada-compatibility|.
+   6 (Variable)        Array containing two items: variable name (binary) and 
+                       variable value (any object).  Values are converted 
+                       using the same code |msgpackparse()| uses when reading, 
+                       |msgpackdump()| when writing, so there may appear 
+                       |msgpack-special-dict|s.  If there are more then two 
+                       entries then the rest are ignored 
+                       (|shada-compatibility|).
+   7 (GlobalMark)
+   8 (Jump)
+   10 (LocalMark)
+   11 (Change)         Map containing some position description:
+                       Entry      Position ~
+                       GlobaMark  Global mark position. |'A|
+                       LocalMark  Local mark position. |'a|
+                       Jump       One position from the |jumplist|.
+                       Change     One position from the |changelist|.
+
+                       Data contained in the map:
+                       Key  Type      Default  Description ~
+                       l    UInteger  1        Position line number.  Must be 
+                                               greater then zero.
+                       c    UInteger  0        Position column number.
+                       n    UInteger  34 ('"') Mark name.  Only valid for 
+                                               GlobalMark and LocalMark 
+                                               entries.
+                       f    Binary    N/A      File name.  Required.
+                       *    any       none     Other keys are allowed for 
+                                               compatibility reasons, see 
+                                               |shada-compatibility|.
+   9 (BufferList)      Array containing maps.  Each map in the array 
+                       represents one buffer.  Possible keys:
+                       Key  Type      Default  Description ~
+                       l    UInteger  1        Position line number.  Must be 
+                                               greater then zero.
+                       c    UInteger  0        Position column number.
+                       f    Binary    N/A      File name.  Required.
+                       *    any       none     Other keys are allowed for 
+                                               compatibility reasons, see 
+                                               |shada-compatibility|.
+   * (Unknown)         Any other entry type is allowed for compatibility 
+                       reasons, see |shada-compatibility|.
+
+								*E575* *E576*
+Errors in ShaDa file may have two types: E575 used for all “logical” errors 
+and E576 used for all “critical” errors.  Critical errors trigger behaviour 
+described in |shada-error-handling| when writing and skipping the rest of the 
+file when reading and include:
+					    *shada-critical-contents-errors*
+- Any of first three MessagePack objects being not an unsigned integer.
+- Third object requesting amount of bytes greater then bytes left in the ShaDa 
+  file.
+- Entry with zero type.  I.e. first object being equal to zero.
+- MessagePack parser failing to parse the entry data.
+- MessagePack parser consuming less or requesting greater bytes then described 
+  in the third object for parsing fourth object.  I.e. when fourth object 
+  either contains more then one MessagePack object or it does not contain 
+  complete MessagePack object.
+
  vim:tw=78:ts=8:ft=help:norl:

runtime/doc/usr_21.txt

@@ -12,7 +12,7 @@ it later.
 
 |21.1|	Suspend and resume
 |21.2|	Executing shell commands
-|21.3|	Remembering information; viminfo
+|21.3|	Remembering information; ShaDa
 |21.4|	Sessions
 |21.5|	Views
 |21.6|	Modelines
@@ -78,13 +78,14 @@ This is similar to using CTRL-Z to suspend Vim.  The difference is that a new
 shell is started.
 
 ==============================================================================
-*21.3*	Remembering information; viminfo
+*21.3*	Remembering information; ShaDa
 
 After editing for a while you will have text in registers, marks in various
 files, a command line history filled with carefully crafted commands.  When
 you exit Vim all of this is lost.  But you can get it back!
 
-The viminfo file is designed to store status information:
+The ShaDa (abbreviation of SHAred DAta) file is designed to store status 
+information:
 
 	Command-line and Search pattern history
 	Text in registers
@@ -92,38 +93,38 @@ The viminfo file is designed to store status information:
 	The buffer list
 	Global variables
 
-Each time you exit Vim it will store this information in a file, the viminfo
+Each time you exit Vim it will store this information in a file, the ShaDa
-file.  When Vim starts again, the viminfo file is read and the information
+file.  When Vim starts again, the ShaDa file is read and the information
 restored.
 
-The 'viminfo' option is set by default to restore a limited number of items.
+The 'shada' option is set by default to restore a limited number of items.
 You might want to set it to remember more information.  This is done through
 the following command: >
 
-	:set viminfo=string
+	:set shada=string
 
 The string specifies what to save.  The syntax of this string is an option
 character followed by an argument.  The option/argument pairs are separated by
 commas.
-   Take a look at how you can build up your own viminfo string.  First, the '
+   Take a look at how you can build up your own shada string.  First, the '
 option is used to specify how many files for which you save marks (a-z).  Pick
 a nice even number for this option (1000, for instance).  Your command now
 looks like this: >
 
-	:set viminfo='1000
+	:set shada='1000
 
 The f option controls whether global marks (A-Z and 0-9) are stored.  If this
 option is 0, none are stored.  If it is 1 or you do not specify an f option,
 the marks are stored.  You want this feature, so now you have this: >
 
-	:set viminfo='1000,f1
+	:set shada='1000,f1
 
 The < option controls how many lines are saved for each of the registers.  By
 default, all the lines are saved.  If 0, nothing is saved.  To avoid adding
-thousands of lines to your viminfo file (which might never get used and makes
+thousands of lines to your ShaDa file (which might never get used and makes
 starting Vim slower) you use a maximum of 500 lines: >
 
-	:set viminfo='1000,f1,<500
+	:set shada='1000,f1,<500
 <
 Other options you might want to use:
 	:	number of lines to save from the command line history
@@ -137,9 +138,9 @@ Other options you might want to use:
 	%	the buffer list (only restored when starting Vim without file
 		arguments)
 	c	convert the text using 'encoding'
-	n	name used for the viminfo file (must be the last option)
+	n	name used for the ShaDa file (must be the last option)
 
-See the 'viminfo' option and |viminfo-file| for more information.
+See the 'shada' option and |shada-file| for more information.
 
 When you run Vim multiple times, the last one exiting will store its
 information.  This may cause information that previously exiting Vims stored
@@ -168,7 +169,7 @@ exiting Vim, there is a slightly more complicated way.  You can see a list of
 files by typing the command: >
 
 	:oldfiles
-<	1: ~/.viminfo ~
+<	1: ~/.vimrc ~
 	2: ~/text/resume.txt ~
 	3: /tmp/draft ~
 
@@ -188,7 +189,7 @@ That #<123 thing is a bit complicated when you just want to edit a file.
 Fortunately there is a simpler way: >
 
 	:browse oldfiles
-<	1: ~/.viminfo ~
+<	1: ~/.vimrc ~
 	2: ~/text/resume.txt ~
 	3: /tmp/draft ~
 	-- More --
@@ -205,25 +206,25 @@ More info at |:oldfiles|, |v:oldfiles| and |c_#<|.
 
 MOVE INFO FROM ONE VIM TO ANOTHER
 
-You can use the ":wviminfo" and ":rviminfo" commands to save and restore the
+You can use the ":wshada" and ":rshada" commands to save and restore the
 information while still running Vim.  This is useful for exchanging register
 contents between two instances of Vim, for example.  In the first Vim do: >
 
-	:wviminfo! ~/tmp/viminfo
+	:wshada! ~/tmp/shada
 
 And in the second Vim do: >
 
-	:rviminfo! ~/tmp/viminfo
+	:rshada! ~/tmp/shada
 
 Obviously, the "w" stands for "write" and the "r" for "read".
-   The ! character is used by ":wviminfo" to forcefully overwrite an existing
+   The ! character is used by ":wshada" to forcefully overwrite an existing
 file.  When it is omitted, and the file exists, the information is merged into
 the file.
-   The ! character used for ":rviminfo" means that all the information is
+   The ! character used for ":rshada" means that all the information in ShaDa 
-used, this may overwrite existing information.  Without the ! only information
+file has priority over existing information, this may overwrite it.  Without 
-that wasn't set is used.
+the ! only information that wasn't set is used.
    These commands can also be used to store info and use it again later.  You
-could make a directory full of viminfo files, each containing info for a
+could make a directory full of ShaDa files, each containing info for a
 different purpose.
 
 ==============================================================================
@@ -355,12 +356,12 @@ Similarly, MS-Windows Vim understands file names with / to separate names, but
 Unix Vim doesn't understand \.
 
 
-SESSIONS AND VIMINFO
+SESSIONS AND SHADA
 
 Sessions store many things, but not the position of marks, contents of
-registers and the command line history.  You need to use the viminfo feature
+registers and the command line history.  You need to use the shada feature
 for these things.
-   In most situations you will want to use sessions separately from viminfo.
+   In most situations you will want to use sessions separately from shada.
 This can be used to switch to another session, but keep the command line
 history.  And yank text into registers in one session, and paste it back in
 another session.
@@ -368,12 +369,12 @@ another session.
 this yourself then.  Example: >
 
 	:mksession! ~/.vim/secret.vim
-	:wviminfo! ~/.vim/secret.viminfo
+	:wshada! ~/.vim/secret.shada
 
 And to restore this again: >
 
 	:source ~/.vim/secret.vim
-	:rviminfo! ~/.vim/secret.viminfo
+	:rshada! ~/.vim/secret.shada
 
 ==============================================================================
 *21.5*	Views

runtime/doc/usr_45.txt

@@ -235,7 +235,7 @@ actually view a file that way, if you have lots of time at hand.
 	Note:
 	Since 'encoding' is used for all text inside Vim, changing it makes
 	all non-ASCII text invalid.  You will notice this when using registers
-	and the 'viminfo' file (e.g., a remembered search pattern).  It's
+	and the |shada-file| (e.g., a remembered search pattern).  It's
 	recommended to set 'encoding' in your vimrc file, and leave it alone.
 
 ==============================================================================

runtime/doc/usr_toc.txt

@@ -181,7 +181,7 @@ Subjects that can be read independently.
 |usr_21.txt|  Go away and come back
 		|21.1|	Suspend and resume
 		|21.2|	Executing shell commands
-		|21.3|	Remembering information; viminfo
+		|21.3|	Remembering information; ShaDa
 		|21.4|	Sessions
 		|21.5|	Views
 		|21.6|	Modelines

runtime/doc/various.txt

@@ -370,6 +370,7 @@ N  *+reltime*		|reltime()| function, 'hlsearch'/'incsearch' timeout,
 			'redrawtime' option
 B  *+rightleft*		Right to left typing |'rightleft'|
 N  *+scrollbind*	|'scrollbind'|
+N  *+shada*		|'shada'|
 B  *+signs*		|:sign|
 N  *+smartindent*	|'smartindent'|
 N  *+startuptime*	|--startuptime| argument
@@ -387,7 +388,6 @@ N  *+textobjects*	|text-objects| selection
 N  *+title*		Setting the window 'title' and 'icon'
 N  *+toolbar*		|gui-toolbar|
 N  *+user_commands*	User-defined commands. |user-commands|
-N  *+viminfo*		|'viminfo'|
 N  *+vertsplit*		Vertically split windows |:vsplit|
 N  *+virtualedit*	|'virtualedit'|
 S  *+visual*		Visual mode |Visual-mode| Always enabled since 7.4.200.

runtime/doc/vim_diff.txt

@@ -23,7 +23,8 @@ these differences.
 
 - Use `.nvimrc` instead of `.vimrc` for storing configuration.
 - Use `.nvim` instead of `.vim` to store configuration files.
-- Use `.nviminfo` instead of `.viminfo` for persistent session information.
+- Use `.nvim/shada/main.shada` instead of `.viminfo` for persistent session 
+  information.
 
 ==============================================================================
 2. Option defaults					  *nvim-option-defaults*
@@ -93,6 +94,28 @@ are always available and may be used simultaneously in separate plugins.  The
    "{E724@level}"), but this is not reliable because |string()| continues to 
    error out.
 
+Viminfo text files were replaced with binary (messagepack) ShaDa files. 
+Additional differences:
+
+- |shada-c| has no effect.
+- |shada-s| now limits size of every item and not just registers.
+- When reading ShaDa files items are merged according to the timestamp. 
+  |shada-merging|
+- 'viminfo' option got renamed to 'shada'. Old option is kept as an alias for 
+  compatibility reasons.
+- |:wviminfo| was renamed to |:wshada|, |:rviminfo| to |:rshada|.  Old 
+  commands are still kept.
+- When writing (|:wshada| without bang or at exit) it merges much more data, 
+  and does this according to the timestamp.  Vim merges only marks. 
+  |shada-merging|
+- ShaDa file format was designed with forward and backward compatibility in 
+  mind. |shada-compatibility|
+- Some errors make ShaDa code keep temporary file in-place for user to decide 
+  what to do with it.  Vim deletes temporary file in these cases. 
+  |shada-error-handling|
+- Vim keeps no timestamps at all, neither in viminfo file nor in the instance 
+  itself.
+
 ==============================================================================
 4. New Features						     *nvim-features-new*
 

runtime/doc/windows.txt

@@ -53,7 +53,7 @@ hidden:   The buffer is not displayed.  If there is a file for this buffer, it
 							*inactive-buffer*
 inactive: The buffer is not displayed and does not contain anything.  Options
 	  for the buffer are remembered if the file was once loaded.  It can
-	  contain marks from the |viminfo| file.  But the buffer doesn't
+	  contain marks from the |shada| file.  But the buffer doesn't
 	  contain text.
 
 In a table:

runtime/macros/less.vim

@@ -42,7 +42,7 @@ set hlsearch
 set incsearch
 nohlsearch
 " Don't remember file names and positions
-set viminfo=
+set shada=
 set nows
 " Inhibit screen updates while searching
 let s:lz = &lz

runtime/optwin.vim

@@ -1226,8 +1226,8 @@ if has("mksession")
   call append("$", "viewdir\tdirectory where to store files with :mkview")
   call <SID>OptionG("vdir", &vdir)
 endif
-if has("viminfo")
+if has("shada")
-  call append("$", "viminfo\tlist that specifies what to write in the viminfo file")
+  call append("$", "viminfo\tlist that specifies what to write in the ShaDa file")
   call <SID>OptionG("vi", &vi)
 endif
 if has("quickfix")

scripts/shadacat.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3.4
+
+import os
+import sys
+import codecs
+
+from enum import Enum
+from datetime import datetime
+from functools import reduce
+
+import msgpack
+
+
+class EntryTypes(Enum):
+  Unknown = -1
+  Missing = 0
+  Header = 1
+  SearchPattern = 2
+  SubString = 3
+  HistoryEntry = 4
+  Register = 5
+  Variable = 6
+  GlobalMark = 7
+  Jump = 8
+  BufferList = 9
+  LocalMark = 10
+  Change = 11
+
+
+def strtrans_errors(e):
+  if not isinstance(e, UnicodeDecodeError):
+    raise NotImplementedError('don’t know how to handle {0} error'.format(
+      e.__class__.__name__))
+  return '<{0:x}>'.format(reduce((lambda a, b: a*0x100+b),
+                                 list(e.object[e.start:e.end]))), e.end
+
+
+codecs.register_error('strtrans', strtrans_errors)
+
+
+def idfunc(o):
+    return o
+
+
+class CharInt(int):
+    def __repr__(self):
+        return super(CharInt, self).__repr__() + ' (\'%s\')' % chr(self)
+
+
+ctable = {
+    bytes: lambda s: s.decode('utf-8', 'strtrans'),
+    dict: lambda d: dict((mnormalize(k), mnormalize(v)) for k, v in d.items()),
+    list: lambda l: list(mnormalize(i) for i in l),
+    int: lambda n: CharInt(n) if 0x20 <= n <= 0x7E else n,
+}
+
+
+def mnormalize(o):
+  return ctable.get(type(o), idfunc)(o)
+
+
+fname = sys.argv[1]
+poswidth = len(str(os.stat(fname).st_size or 1000))
+
+
+with open(fname, 'rb') as fp:
+  unpacker = msgpack.Unpacker(file_like=fp, read_size=1)
+  max_type = max(typ.value for typ in EntryTypes)
+  while True:
+    try:
+      pos = fp.tell()
+      typ = unpacker.unpack()
+    except msgpack.OutOfData:
+      break
+    else:
+      timestamp = unpacker.unpack()
+      time = datetime.fromtimestamp(timestamp)
+      length = unpacker.unpack()
+      if typ > max_type:
+        entry = fp.read(length)
+        typ = EntryTypes.Unknown
+      else:
+        entry = unpacker.unpack()
+        typ = EntryTypes(typ)
+      print('%*u %13s %s %5u %r' % (
+        poswidth, pos, typ.name, time.isoformat(), length, mnormalize(entry)))

src/nvim/api/private/helpers.c

@@ -597,7 +597,7 @@ static void init_type_metadata(Dictionary *metadata)
 }
 
 /// Creates a deep clone of an object
-static Object copy_object(Object obj)
+Object copy_object(Object obj)
 {
   switch (obj.type) {
     case kObjectTypeNil:

src/nvim/api/private/helpers.h

@@ -69,6 +69,8 @@
 #define ADD(array, item)                                                      \
   kv_push(Object, array, item)
 
+#define STATIC_CSTR_AS_STRING(s) ((String) {.data = s, .size = sizeof(s) - 1})
+
 // Helpers used by the generated msgpack-rpc api wrappers
 #define api_init_boolean
 #define api_init_integer

src/nvim/auevents.lua

@@ -84,7 +84,7 @@ return {
     'User',                   -- user defined autocommand
     'VimEnter',               -- after starting Vim
     'VimLeave',               -- before exiting Vim
-    'VimLeavePre',            -- before exiting Vim and writing .viminfo
+    'VimLeavePre',            -- before exiting Vim and writing ShaDa file
     'VimResized',             -- after Vim window was resized
     'WinEnter',               -- after entering a window
     'WinLeave',               -- before leaving a window

src/nvim/buffer.c

@@ -73,6 +73,7 @@
 #include "nvim/undo.h"
 #include "nvim/version.h"
 #include "nvim/window.h"
+#include "nvim/shada.h"
 #include "nvim/os/os.h"
 #include "nvim/os/time.h"
 #include "nvim/os/input.h"
@@ -555,9 +556,21 @@ static void free_buffer(buf_T *buf)
   free_buffer_stuff(buf, TRUE);
   unref_var_dict(buf->b_vars);
   aubuflocal_remove(buf);
+  dict_unref(buf->additional_data);
+  clear_fmark(&buf->b_last_cursor);
+  clear_fmark(&buf->b_last_insert);
+  clear_fmark(&buf->b_last_change);
+  for (size_t i = 0; i < NMARKS; i++) {
+    free_fmark(buf->b_namedm[i]);
+  }
+  for (int i = 0; i < buf->b_changelistlen; i++) {
+    free_fmark(buf->b_changelist[i]);
+  }
   if (autocmd_busy) {
     // Do not free the buffer structure while autocommands are executing,
     // it's still needed. Free it when autocmd_busy is reset.
+    memset(&buf->b_namedm[0], 0, sizeof(buf->b_namedm));
+    memset(&buf->b_changelist[0], 0, sizeof(buf->b_changelist));
     buf->b_next = au_pending_free_buf;
     au_pending_free_buf = buf;
   } else {
@@ -1978,12 +1991,18 @@ buflist_nr2name (
       fullname ? buf->b_ffname : buf->b_fname);
 }
 
-/*
+/// Set the line and column numbers for the given buffer and window
- * Set the "lnum" and "col" for the buffer "buf" and the current window.
+///
- * When "copy_options" is TRUE save the local window option values.
+/// @param[in,out]  buf           Buffer for which line and column are set.
- * When "lnum" is 0 only do the options.
+/// @param[in,out]  win           Window for which line and column are set.
- */
+/// @param[in]      lnum          Line number to be set. If it is zero then only
-static void buflist_setfpos(buf_T *buf, win_T *win, linenr_T lnum, colnr_T col, int copy_options)
+///                               options are touched.
+/// @param[in]      col           Column number to be set.
+/// @param[in]      copy_options  If true save the local window option values.
+void buflist_setfpos(buf_T *const buf, win_T *const win,
+                     linenr_T lnum, colnr_T col,
+                     bool copy_options)
+  FUNC_ATTR_NONNULL_ALL
 {
   wininfo_T   *wip;
 
@@ -4164,93 +4183,6 @@ chk_modeline (
   return retval;
 }
 
-int read_viminfo_bufferlist(vir_T *virp, int writing)
-{
-  char_u      *tab;
-  linenr_T lnum;
-  colnr_T col;
-  buf_T       *buf;
-  char_u      *sfname;
-  char_u      *xline;
-
-  /* Handle long line and escaped characters. */
-  xline = viminfo_readstring(virp, 1, FALSE);
-
-  /* don't read in if there are files on the command-line or if writing: */
-  if (xline != NULL && !writing && ARGCOUNT == 0
-      && find_viminfo_parameter('%') != NULL) {
-    /* Format is: <fname> Tab <lnum> Tab <col>.
-     * Watch out for a Tab in the file name, work from the end. */
-    lnum = 0;
-    col = 0;
-    tab = vim_strrchr(xline, '\t');
-    if (tab != NULL) {
-      *tab++ = '\0';
-      col = (colnr_T)atoi((char *)tab);
-      tab = vim_strrchr(xline, '\t');
-      if (tab != NULL) {
-        *tab++ = '\0';
-        lnum = atol((char *)tab);
-      }
-    }
-
-    /* Expand "~/" in the file name at "line + 1" to a full path.
-     * Then try shortening it by comparing with the current directory */
-    expand_env(xline, NameBuff, MAXPATHL);
-    sfname = path_shorten_fname_if_possible(NameBuff);
-
-    buf = buflist_new(NameBuff, sfname, (linenr_T)0, BLN_LISTED);
-    if (buf != NULL) {          /* just in case... */
-      buf->b_last_cursor.lnum = lnum;
-      buf->b_last_cursor.col = col;
-      buflist_setfpos(buf, curwin, lnum, col, FALSE);
-    }
-  }
-  xfree(xline);
-
-  return viminfo_readline(virp);
-}
-
-void write_viminfo_bufferlist(FILE *fp)
-{
-  char_u      *line;
-  int max_buffers;
-
-  if (find_viminfo_parameter('%') == NULL)
-    return;
-
-  /* Without a number -1 is returned: do all buffers. */
-  max_buffers = get_viminfo_parameter('%');
-
-  /* Allocate room for the file name, lnum and col. */
-#define LINE_BUF_LEN (MAXPATHL + 40)
-  line = xmalloc(LINE_BUF_LEN);
-
-  FOR_ALL_TAB_WINDOWS(tp, win) {
-    set_last_cursor(win);
-  }
-
-  fputs(_("\n# Buffer list:\n"), fp);
-  FOR_ALL_BUFFERS(buf) {
-    if (buf->b_fname == NULL
-        || !buf->b_p_bl
-        || bt_quickfix(buf)
-        || removable(buf->b_ffname))
-      continue;
-
-    if (max_buffers-- == 0)
-      break;
-    putc('%', fp);
-    home_replace(NULL, buf->b_ffname, line, MAXPATHL, TRUE);
-    vim_snprintf_add((char *)line, LINE_BUF_LEN, "\t%" PRId64 "\t%d",
-        (int64_t)buf->b_last_cursor.lnum,
-        buf->b_last_cursor.col);
-    viminfo_writestring(fp, line);
-  }
-  xfree(line);
-}
-
-
 /*
  * Return special buffer name.
  * Returns NULL when the buffer has a normal file name.

src/nvim/buffer_defs.h

@@ -327,15 +327,6 @@ typedef struct {
   bool vc_fail;                 /* fail for invalid char, don't use '?' */
 } vimconv_T;
 
-/*
- * Structure used for reading from the viminfo file.
- */
-typedef struct {
-  char_u      *vir_line;        /* text of the current line */
-  FILE        *vir_fd;          /* file descriptor */
-  vimconv_T vir_conv;           /* encoding conversion */
-} vir_T;
-
 #define CONV_NONE               0
 #define CONV_TO_UTF8            1
 #define CONV_9_TO_UTF8          2
@@ -515,21 +506,21 @@ struct file_buffer {
   uint64_t b_orig_size;         /* size of original file in bytes */
   int b_orig_mode;              /* mode of original file */
 
-  pos_T b_namedm[NMARKS];         /* current named marks (mark.c) */
+  fmark_T b_namedm[NMARKS];     /* current named marks (mark.c) */
 
   /* These variables are set when VIsual_active becomes FALSE */
   visualinfo_T b_visual;
   int b_visual_mode_eval;            /* b_visual.vi_mode for visualmode() */
 
-  pos_T b_last_cursor;          /* cursor position when last unloading this
+  fmark_T b_last_cursor;        // cursor position when last unloading this
-                                   buffer */
+                                // buffer
-  pos_T b_last_insert;          /* where Insert mode was left */
+  fmark_T b_last_insert;        // where Insert mode was left
-  pos_T b_last_change;          /* position of last change: '. mark */
+  fmark_T b_last_change;        // position of last change: '. mark
 
   /*
    * the changelist contains old change positions
    */
-  pos_T b_changelist[JUMPLISTSIZE];
+  fmark_T b_changelist[JUMPLISTSIZE];
   int b_changelistlen;                  /* number of active entries */
   bool b_new_change;                    /* set by u_savecommon() */
 
@@ -553,7 +544,7 @@ struct file_buffer {
   pos_T b_op_start_orig;  // used for Insstart_orig
   pos_T b_op_end;
 
-  bool b_marks_read;            /* Have we read viminfo marks yet? */
+  bool b_marks_read;            /* Have we read ShaDa marks yet? */
 
   /*
    * The following only used in undo.c.
@@ -757,6 +748,8 @@ struct file_buffer {
   signlist_T *b_signlist;       /* list of signs to draw */
 
   Terminal *terminal;           // Terminal instance associated with the buffer
+
+  dict_T *additional_data;      // Additional data from shada file if any.
 };
 
 /*

src/nvim/edit.c

@@ -60,6 +60,7 @@
 #include "nvim/undo.h"
 #include "nvim/window.h"
 #include "nvim/event/loop.h"
+#include "nvim/mark.h"
 #include "nvim/os/input.h"
 #include "nvim/os/time.h"
 
@@ -6991,8 +6992,9 @@ ins_esc (
     curwin->w_set_curswant = TRUE;
 
   /* Remember the last Insert position in the '^ mark. */
-  if (!cmdmod.keepjumps)
+  if (!cmdmod.keepjumps) {
-    curbuf->b_last_insert = curwin->w_cursor;
+    RESET_FMARK(&curbuf->b_last_insert, curwin->w_cursor, curbuf->b_fnum);
+  }
 
   /*
    * The cursor should end up on the last inserted character.

src/nvim/eval.c

@@ -107,18 +107,6 @@
 #define AUTOLOAD_CHAR '#'       /* Character used as separator in autoload 
                                    function/variable names. */
 
-/*
- * In a hashtab item "hi_key" points to "di_key" in a dictitem.
- * This avoids adding a pointer to the hashtab item.
- * DI2HIKEY() converts a dictitem pointer to a hashitem key pointer.
- * HIKEY2DI() converts a hashitem key pointer to a dictitem pointer.
- * HI2DI() converts a hashitem pointer to a dictitem pointer.
- */
-static dictitem_T dumdi;
-#define DI2HIKEY(di) ((di)->di_key)
-#define HIKEY2DI(p)  ((dictitem_T *)(p - (dumdi.di_key - (char_u *)&dumdi)))
-#define HI2DI(hi)     HIKEY2DI((hi)->hi_key)
-
 /*
  * Structure returned by get_lval() and used by set_var_lval().
  * For a plain name:
@@ -354,7 +342,7 @@ typedef struct {
 typedef enum {
   VAR_FLAVOUR_DEFAULT,          /* doesn't start with uppercase */
   VAR_FLAVOUR_SESSION,          /* starts with uppercase, some lower */
-  VAR_FLAVOUR_VIMINFO           /* all uppercase */
+  VAR_FLAVOUR_SHADA             /* all uppercase */
 } var_flavour_T;
 
 /* values for vv_flags: */
@@ -5352,7 +5340,7 @@ static int list_concat(list_T *l1, list_T *l2, typval_T *tv)
     return FAIL;
 
   /* make a copy of the first list. */
-  l = list_copy(l1, FALSE, 0);
+  l = list_copy(NULL, l1, false, 0);
   if (l == NULL)
     return FAIL;
   tv->v_type = VAR_LIST;
@@ -5363,13 +5351,20 @@ static int list_concat(list_T *l1, list_T *l2, typval_T *tv)
   return OK;
 }
 
-/*
+/// Make a copy of list
- * Make a copy of list "orig".  Shallow if "deep" is FALSE.
+///
- * The refcount of the new list is set to 1.
+/// @param[in]  conv  If non-NULL, then all internal strings will be converted.
- * See item_copy() for "copyID".
+/// @param[in]  orig  Original list to copy.
- * Returns NULL if orig is NULL or some failure happens.
+/// @param[in]  deep  If false, then shallow copy will be done.
- */
+/// @param[in]  copyID  See var_item_copy().
-static list_T *list_copy(list_T *orig, int deep, int copyID)
+///
+/// @return Copied list. May be NULL in case original list is NULL or some
+///         failure happens. The refcount of the new list is set to 1.
+static list_T *list_copy(const vimconv_T *const conv,
+                         list_T *const orig,
+                         const bool deep,
+                         const int copyID)
+  FUNC_ATTR_WARN_UNUSED_RESULT
 {
   listitem_T  *item;
   listitem_T  *ni;
@@ -5388,7 +5383,7 @@ static list_T *list_copy(list_T *orig, int deep, int copyID)
        item = item->li_next) {
     ni = listitem_alloc();
     if (deep) {
-      if (item_copy(&item->li_tv, &ni->li_tv, deep, copyID) == FAIL) {
+      if (var_item_copy(conv, &item->li_tv, &ni->li_tv, deep, copyID) == FAIL) {
         xfree(ni);
         break;
       }
@@ -5546,6 +5541,7 @@ static int list_join(garray_T *const gap, list_T *const l,
 bool garbage_collect(void)
 {
   bool abort = false;
+#define ABORTING(func) abort = abort || func
 
   // Only do this once.
   want_garbage_collect = false;
@@ -5564,45 +5560,117 @@ bool garbage_collect(void)
   // referenced through previous_funccal.  This must be first, because if
   // the item is referenced elsewhere the funccal must not be freed.
   for (funccall_T *fc = previous_funccal; fc != NULL; fc = fc->caller) {
-    abort = abort || set_ref_in_ht(&fc->l_vars.dv_hashtab, copyID + 1, NULL);
+    ABORTING(set_ref_in_ht)(&fc->l_vars.dv_hashtab, copyID + 1, NULL);
-    abort = abort || set_ref_in_ht(&fc->l_avars.dv_hashtab, copyID + 1, NULL);
+    ABORTING(set_ref_in_ht)(&fc->l_avars.dv_hashtab, copyID + 1, NULL);
   }
 
   // script-local variables
   for (int i = 1; i <= ga_scripts.ga_len; ++i) {
-    abort = abort || set_ref_in_ht(&SCRIPT_VARS(i), copyID, NULL);
+    ABORTING(set_ref_in_ht)(&SCRIPT_VARS(i), copyID, NULL);
   }
 
-  // buffer-local variables
   FOR_ALL_BUFFERS(buf) {
-    abort = abort || set_ref_in_item(&buf->b_bufvar.di_tv, copyID, NULL, NULL);
+    // buffer-local variables
+    ABORTING(set_ref_in_item)(&buf->b_bufvar.di_tv, copyID, NULL, NULL);
+    // buffer marks (ShaDa additional data)
+    ABORTING(set_ref_in_fmark)(buf->b_last_cursor, copyID);
+    ABORTING(set_ref_in_fmark)(buf->b_last_insert, copyID);
+    ABORTING(set_ref_in_fmark)(buf->b_last_change, copyID);
+    for (size_t i = 0; i < NMARKS; i++) {
+      ABORTING(set_ref_in_fmark)(buf->b_namedm[i], copyID);
+    }
+    // buffer change list (ShaDa additional data)
+    for (int i = 0; i < buf->b_changelistlen; i++) {
+      ABORTING(set_ref_in_fmark)(buf->b_changelist[i], copyID);
+    }
+    // buffer ShaDa additional data
+    ABORTING(set_ref_dict)(buf->additional_data, copyID);
   }
 
-  // window-local variables
   FOR_ALL_TAB_WINDOWS(tp, wp) {
-    abort = abort || set_ref_in_item(&wp->w_winvar.di_tv, copyID, NULL, NULL);
+    // window-local variables
+    ABORTING(set_ref_in_item)(&wp->w_winvar.di_tv, copyID, NULL, NULL);
+    // window jump list (ShaDa additional data)
+    for (int i = 0; i < wp->w_jumplistlen; i++) {
+      ABORTING(set_ref_in_fmark)(wp->w_jumplist[i].fmark, copyID);
+    }
   }
   if (aucmd_win != NULL) {
-    abort = abort ||
+    ABORTING(set_ref_in_item)(&aucmd_win->w_winvar.di_tv, copyID, NULL, NULL);
-            set_ref_in_item(&aucmd_win->w_winvar.di_tv, copyID, NULL, NULL);
+  }
+
+  // registers (ShaDa additional data)
+  {
+    const void *reg_iter = NULL;
+    do {
+      yankreg_T reg;
+      char name = NUL;
+      reg_iter = op_register_iter(reg_iter, &name, &reg);
+      if (name != NUL) {
+        ABORTING(set_ref_dict)(reg.additional_data, copyID);
+      }
+    } while (reg_iter != NULL);
+  }
+
+  // global marks (ShaDa additional data)
+  {
+    const void *mark_iter = NULL;
+    do {
+      xfmark_T fm;
+      char name = NUL;
+      mark_iter = mark_global_iter(mark_iter, &name, &fm);
+      if (name != NUL) {
+        ABORTING(set_ref_dict)(fm.fmark.additional_data, copyID);
+      }
+    } while (mark_iter != NULL);
   }
 
   // tabpage-local variables
   FOR_ALL_TABS(tp) {
-    abort = abort || set_ref_in_item(&tp->tp_winvar.di_tv, copyID, NULL, NULL);
+    ABORTING(set_ref_in_item)(&tp->tp_winvar.di_tv, copyID, NULL, NULL);
   }
 
   // global variables
-  abort = abort || set_ref_in_ht(&globvarht, copyID, NULL);
+  ABORTING(set_ref_in_ht)(&globvarht, copyID, NULL);
 
   // function-local variables
   for (funccall_T *fc = current_funccal; fc != NULL; fc = fc->caller) {
-    abort = abort || set_ref_in_ht(&fc->l_vars.dv_hashtab, copyID, NULL);
+    ABORTING(set_ref_in_ht)(&fc->l_vars.dv_hashtab, copyID, NULL);
-    abort = abort || set_ref_in_ht(&fc->l_avars.dv_hashtab, copyID, NULL);
+    ABORTING(set_ref_in_ht)(&fc->l_avars.dv_hashtab, copyID, NULL);
   }
 
   // v: vars
-  abort = abort || set_ref_in_ht(&vimvarht, copyID, NULL);
+  ABORTING(set_ref_in_ht)(&vimvarht, copyID, NULL);
+
+  // history items (ShaDa additional elements)
+  if (p_hi) {
+    for (uint8_t i = 0; i < HIST_COUNT; i++) {
+      const void *iter = NULL;
+      do {
+        histentry_T hist;
+        iter = hist_iter(iter, i, false, &hist);
+        if (hist.hisstr != NULL) {
+          ABORTING(set_ref_list)(hist.additional_elements, copyID);
+        }
+      } while (iter != NULL);
+    }
+  }
+
+  // previously used search/substitute patterns (ShaDa additional data)
+  {
+    SearchPattern pat;
+    get_search_pattern(&pat);
+    ABORTING(set_ref_dict)(pat.additional_data, copyID);
+    get_substitute_pattern(&pat);
+    ABORTING(set_ref_dict)(pat.additional_data, copyID);
+  }
+
+  // previously used replacement string
+  {
+    SubReplacementString sub;
+    sub_get_replacement(&sub);
+    ABORTING(set_ref_list)(sub.additional_elements, copyID);
+  }
 
   bool did_free = false;
   if (!abort) {
@@ -5631,6 +5699,7 @@ bool garbage_collect(void)
     verb_msg((char_u *)_(
         "Not enough memory to set references, garbage collection aborted!"));
   }
+#undef ABORTING
   return did_free;
 }
 
@@ -5690,6 +5759,7 @@ static int free_unref_items(int copyID)
 ///
 /// @returns             true if setting references failed somehow.
 bool set_ref_in_ht(hashtab_T *ht, int copyID, list_stack_T **list_stack)
+  FUNC_ATTR_WARN_UNUSED_RESULT
 {
   bool abort = false;
   ht_stack_T *ht_stack = NULL;
@@ -5732,6 +5802,7 @@ bool set_ref_in_ht(hashtab_T *ht, int copyID, list_stack_T **list_stack)
 ///
 /// @returns             true if setting references failed somehow.
 bool set_ref_in_list(list_T *l, int copyID, ht_stack_T **ht_stack)
+  FUNC_ATTR_WARN_UNUSED_RESULT
 {
   bool abort = false;
   list_stack_T *list_stack = NULL;
@@ -5772,6 +5843,7 @@ bool set_ref_in_list(list_T *l, int copyID, ht_stack_T **ht_stack)
 /// @returns             true if setting references failed somehow.
 bool set_ref_in_item(typval_T *tv, int copyID, ht_stack_T **ht_stack,
                      list_stack_T **list_stack)
+  FUNC_ATTR_WARN_UNUSED_RESULT
 {
   bool abort = false;
 
@@ -5821,6 +5893,52 @@ bool set_ref_in_item(typval_T *tv, int copyID, ht_stack_T **ht_stack,
   return abort;
 }
 
+/// Mark all lists and dicts referenced in given mark
+///
+/// @returns true if setting references failed somehow.
+static inline bool set_ref_in_fmark(fmark_T fm, int copyID)
+  FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  if (fm.additional_data != NULL
+      && fm.additional_data->dv_copyID != copyID) {
+    fm.additional_data->dv_copyID = copyID;
+    return set_ref_in_ht(&fm.additional_data->dv_hashtab, copyID, NULL);
+  }
+  return false;
+}
+
+/// Mark all lists and dicts referenced in given list and the list itself
+///
+/// @returns true if setting references failed somehow.
+static inline bool set_ref_list(list_T *list, int copyID)
+  FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  if (list != NULL) {
+    typval_T tv = (typval_T) {
+      .v_type = VAR_LIST,
+      .vval = { .v_list = list }
+    };
+    return set_ref_in_item(&tv, copyID, NULL, NULL);
+  }
+  return false;
+}
+
+/// Mark all lists and dicts referenced in given dict and the dict itself
+///
+/// @returns true if setting references failed somehow.
+static inline bool set_ref_dict(dict_T *dict, int copyID)
+  FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  if (dict != NULL) {
+    typval_T tv = (typval_T) {
+      .v_type = VAR_DICT,
+      .vval = { .v_dict = dict }
+    };
+    return set_ref_in_item(&tv, copyID, NULL, NULL);
+  }
+  return false;
+}
+
 /*
  * Allocate an empty header for a dictionary.
  */
@@ -5964,13 +6082,20 @@ void dictitem_free(dictitem_T *item)
   xfree(item);
 }
 
-/*
+/// Make a copy of dictionary
- * Make a copy of dict "d".  Shallow if "deep" is FALSE.
+///
- * The refcount of the new dict is set to 1.
+/// @param[in]  conv  If non-NULL, then all internal strings will be converted.
- * See item_copy() for "copyID".
+/// @param[in]  orig  Original dictionary to copy.
- * Returns NULL if orig is NULL or some other failure.
+/// @param[in]  deep  If false, then shallow copy will be done.
- */
+/// @param[in]  copyID  See var_item_copy().
-static dict_T *dict_copy(dict_T *orig, int deep, int copyID)
+///
+/// @return Copied dictionary. May be NULL in case original dictionary is NULL
+///         or some failure happens. The refcount of the new dictionary is set
+///         to 1.
+static dict_T *dict_copy(const vimconv_T *const conv,
+                         dict_T *const orig,
+                         const bool deep,
+                         const int copyID)
 {
   dictitem_T  *di;
   int todo;
@@ -5990,9 +6115,20 @@ static dict_T *dict_copy(dict_T *orig, int deep, int copyID)
       if (!HASHITEM_EMPTY(hi)) {
         --todo;
 
+        if (conv == NULL || conv->vc_type == CONV_NONE) {
           di = dictitem_alloc(hi->hi_key);
+        } else {
+          char *const key = (char *) string_convert((vimconv_T *) conv,
+                                                    hi->hi_key, NULL);
+          if (key == NULL) {
+            di = dictitem_alloc(hi->hi_key);
+          } else {
+            di = dictitem_alloc((char_u *) key);
+            xfree(key);
+          }
+        }
         if (deep) {
-          if (item_copy(&HI2DI(hi)->di_tv, &di->di_tv, deep,
+          if (var_item_copy(conv, &HI2DI(hi)->di_tv, &di->di_tv, deep,
                             copyID) == FAIL) {
             xfree(di);
             break;
@@ -6305,7 +6441,7 @@ failret:
 ///                       the results.
 /// @param  firstargname  Name of the first argument.
 /// @param  name  Name of the target converter.
-#define DEFINE_VIML_CONV_FUNCTIONS(name, firstargtype, firstargname) \
+#define DEFINE_VIML_CONV_FUNCTIONS(scope, name, firstargtype, firstargname) \
 static int name##_convert_one_value(firstargtype firstargname, \
                                     MPConvStack *const mpstack, \
                                     typval_T *const tv, \
@@ -6543,7 +6679,7 @@ name##_convert_one_value_regular_dict: \
   return OK; \
 } \
 \
-static int vim_to_##name(firstargtype firstargname, typval_T *const tv) \
+scope int vim_to_##name(firstargtype firstargname, typval_T *const tv) \
   FUNC_ATTR_WARN_UNUSED_RESULT \
 { \
   current_copyID += COPYID_INC; \
@@ -6739,7 +6875,7 @@ vim_to_msgpack_error_ret: \
 
 #define CONV_ALLOW_SPECIAL false
 
-DEFINE_VIML_CONV_FUNCTIONS(string, garray_T *const, gap)
+DEFINE_VIML_CONV_FUNCTIONS(static, string, garray_T *const, gap)
 
 #undef CONV_RECURSE
 #define CONV_RECURSE(val, conv_type) \
@@ -6769,7 +6905,7 @@ DEFINE_VIML_CONV_FUNCTIONS(string, garray_T *const, gap)
       return OK; \
     } while (0)
 
-DEFINE_VIML_CONV_FUNCTIONS(echo, garray_T *const, gap)
+DEFINE_VIML_CONV_FUNCTIONS(static, echo, garray_T *const, gap)
 
 #undef CONV_STRING
 #undef CONV_STR_STRING
@@ -8344,7 +8480,7 @@ static void f_confirm(typval_T *argvars, typval_T *rettv)
  */
 static void f_copy(typval_T *argvars, typval_T *rettv)
 {
-  item_copy(&argvars[0], rettv, FALSE, 0);
+  var_item_copy(NULL, &argvars[0], rettv, false, 0);
 }
 
 /*
@@ -8513,7 +8649,9 @@ static void f_deepcopy(typval_T *argvars, typval_T *rettv)
     EMSG(_(e_invarg));
   else {
     current_copyID += COPYID_INC;
-    item_copy(&argvars[0], rettv, TRUE, noref == 0 ? current_copyID : 0);
+    var_item_copy(NULL, &argvars[0], rettv, true, (noref == 0
+                                                   ? current_copyID
+                                                   : 0));
   }
 }
 
@@ -10482,6 +10620,7 @@ static void f_has(typval_T *argvars, typval_T *rettv)
     "scrollbind",
     "showcmd",
     "cmdline_info",
+    "shada",
     "signs",
     "smartindent",
     "startuptime",
@@ -10498,7 +10637,6 @@ static void f_has(typval_T *argvars, typval_T *rettv)
     "title",
     "user-commands",        /* was accidentally included in 5.4 */
     "user_commands",
-    "viminfo",
     "vertsplit",
     "virtualedit",
     "visual",
@@ -12486,7 +12624,7 @@ static inline bool vim_list_to_buf(const list_T *const list,
 
 #define CONV_ALLOW_SPECIAL true
 
-DEFINE_VIML_CONV_FUNCTIONS(msgpack, msgpack_packer *const, packer)
+DEFINE_VIML_CONV_FUNCTIONS(, msgpack, msgpack_packer *const, packer)
 
 #undef CONV_STRING
 #undef CONV_STR_STRING
@@ -12592,7 +12730,7 @@ static inline ListReaderState init_lrstate(const list_T *const list)
 }
 
 /// Convert msgpack object to a VimL one
-static int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv)
+int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
 #define INIT_SPECIAL_DICT(tv, type, val) \
@@ -17353,14 +17491,14 @@ char_u *get_vim_var_str(int idx) FUNC_ATTR_PURE FUNC_ATTR_NONNULL_RET
  * Get List v: variable value.  Caller must take care of reference count when
  * needed.
  */
-list_T *get_vim_var_list(int idx) FUNC_ATTR_PURE FUNC_ATTR_NONNULL_RET
+list_T *get_vim_var_list(int idx) FUNC_ATTR_PURE
 {
   return vimvars[idx].vv_list;
 }
 
 /// Get Dictionary v: variable value.  Caller must take care of reference count
 /// when needed.
-dict_T *get_vim_var_dict(int idx) FUNC_ATTR_PURE FUNC_ATTR_NONNULL_RET
+dict_T *get_vim_var_dict(int idx) FUNC_ATTR_PURE
 {
   return vimvars[idx].vv_dict;
 }
@@ -18446,14 +18584,28 @@ void copy_tv(typval_T *from, typval_T *to)
   }
 }
 
-/*
+/// Make a copy of an item
- * Make a copy of an item.
+///
- * Lists and Dictionaries are also copied.  A deep copy if "deep" is set.
+/// Lists and Dictionaries are also copied.
- * For deepcopy() "copyID" is zero for a full copy or the ID for when a
+///
- * reference to an already copied list/dict can be used.
+/// @param[in]  conv  If not NULL, convert all copied strings.
- * Returns FAIL or OK.
+/// @param[in]  from  Value to copy.
- */
+/// @param[out]  to  Location where to copy to.
-static int item_copy(typval_T *from, typval_T *to, int deep, int copyID)
+/// @param[in]  deep  If true, use copy the container and all of the contained
+///                   containers (nested).
+/// @param[in]  copyID  If non-zero then when container is referenced more then
+///                     once then copy of it that was already done is used. E.g.
+///                     when copying list `list = [list2, list2]` (`list[0] is
+///                     list[1]`) var_item_copy with zero copyID will emit
+///                     a copy with (`copy[0] isnot copy[1]`), with non-zero it
+///                     will emit a copy with (`copy[0] is copy[1]`) like in the
+///                     original list. Not use when deep is false.
+int var_item_copy(const vimconv_T *const conv,
+                  typval_T *const from,
+                  typval_T *const to,
+                  const bool deep,
+                  const int copyID)
+  FUNC_ATTR_NONNULL_ARG(2, 3)
 {
   static int recurse = 0;
   int ret = OK;
@@ -18467,10 +18619,23 @@ static int item_copy(typval_T *from, typval_T *to, int deep, int copyID)
   switch (from->v_type) {
   case VAR_NUMBER:
   case VAR_FLOAT:
-  case VAR_STRING:
   case VAR_FUNC:
     copy_tv(from, to);
     break;
+  case VAR_STRING:
+    if (conv == NULL || conv->vc_type == CONV_NONE) {
+      copy_tv(from, to);
+    } else {
+      to->v_type = VAR_STRING;
+      to->v_lock = 0;
+      if ((to->vval.v_string = string_convert((vimconv_T *)conv,
+                                              from->vval.v_string,
+                                              NULL))
+          == NULL) {
+        to->vval.v_string = (char_u *) xstrdup((char *) from->vval.v_string);
+      }
+    }
+    break;
   case VAR_LIST:
     to->v_type = VAR_LIST;
     to->v_lock = 0;
@@ -18480,8 +18645,9 @@ static int item_copy(typval_T *from, typval_T *to, int deep, int copyID)
       /* use the copy made earlier */
       to->vval.v_list = from->vval.v_list->lv_copylist;
       ++to->vval.v_list->lv_refcount;
-    } else
+    } else {
-      to->vval.v_list = list_copy(from->vval.v_list, deep, copyID);
+      to->vval.v_list = list_copy(conv, from->vval.v_list, deep, copyID);
+    }
     if (to->vval.v_list == NULL)
       ret = FAIL;
     break;
@@ -18494,13 +18660,14 @@ static int item_copy(typval_T *from, typval_T *to, int deep, int copyID)
       /* use the copy made earlier */
       to->vval.v_dict = from->vval.v_dict->dv_copydict;
       ++to->vval.v_dict->dv_refcount;
-    } else
+    } else {
-      to->vval.v_dict = dict_copy(from->vval.v_dict, deep, copyID);
+      to->vval.v_dict = dict_copy(conv, from->vval.v_dict, deep, copyID);
+    }
     if (to->vval.v_dict == NULL)
       ret = FAIL;
     break;
   default:
-    EMSG2(_(e_intern2), "item_copy()");
+    EMSG2(_(e_intern2), "var_item_copy()");
     ret = FAIL;
   }
   --recurse;
@@ -20710,109 +20877,64 @@ static var_flavour_T var_flavour(char_u *varname)
 
   if (ASCII_ISUPPER(*p)) {
     while (*(++p))
-      if (ASCII_ISLOWER(*p))
+      if (ASCII_ISLOWER(*p)) {
         return VAR_FLAVOUR_SESSION;
-    return VAR_FLAVOUR_VIMINFO;
+      }
-  } else
+    return VAR_FLAVOUR_SHADA;
+  } else {
     return VAR_FLAVOUR_DEFAULT;
   }
+}
 
-/*
+/// Iterate over global variables
- * Restore global vars that start with a capital from the viminfo file
+///
- */
+/// @warning No modifications to global variable dictionary must be performed
-int read_viminfo_varlist(vir_T *virp, int writing)
+///          while iteration is in progress.
+///
+/// @param[in]   iter   Iterator. Pass NULL to start iteration.
+/// @param[out]  name   Variable name.
+/// @param[out]  rettv  Variable value.
+///
+/// @return Pointer that needs to be passed to next `var_shada_iter` invocation
+///         or NULL to indicate that iteration is over.
+const void *var_shada_iter(const void *const iter, const char **const name,
+                           typval_T *rettv)
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ARG(2, 3)
 {
-  char_u      *tab;
+  const hashitem_T *hi;
-  int type = VAR_NUMBER;
+  const hashitem_T *hifirst = globvarht.ht_array;
-  typval_T tv;
+  const size_t hinum = (size_t) globvarht.ht_mask + 1;
-
+  *name = NULL;
-  if (!writing && (find_viminfo_parameter('!') != NULL)) {
+  if (iter == NULL) {
-    tab = vim_strchr(virp->vir_line + 1, '\t');
+    hi = globvarht.ht_array;
-    if (tab != NULL) {
+    while ((size_t) (hi - hifirst) < hinum
-      *tab++ = NUL;            /* isolate the variable name */
+           && (HASHITEM_EMPTY(hi)
-      switch (*tab) {
+               || var_flavour(HI2DI(hi)->di_key) != VAR_FLAVOUR_SHADA)) {
-      case 'S': type = VAR_STRING; break;
+      hi++;
-      case 'F': type = VAR_FLOAT; break;
-      case 'D': type = VAR_DICT; break;
-      case 'L': type = VAR_LIST; break;
     }
-
+    if ((size_t) (hi - hifirst) == hinum) {
-      tab = vim_strchr(tab, '\t');
+      return NULL;
-      if (tab != NULL) {
+    }
-        tv.v_type = type;
+  } else {
-        if (type == VAR_STRING || type == VAR_DICT || type == VAR_LIST)
+    hi = (const hashitem_T *) iter;
-          tv.vval.v_string = viminfo_readstring(virp,
+  }
-              (int)(tab - virp->vir_line + 1), TRUE);
+  *name = (char *) HI2DI(hi)->di_key;
-        else if (type == VAR_FLOAT)
+  copy_tv(&(HI2DI(hi)->di_tv), rettv);
-          (void)string2float(tab + 1, &tv.vval.v_float);
+  while ((size_t) (++hi - hifirst) < hinum) {
-        else
+    if (!HASHITEM_EMPTY(hi)
-          tv.vval.v_number = atol((char *)tab + 1);
+        && var_flavour(HI2DI(hi)->di_key) == VAR_FLAVOUR_SHADA) {
-        if (type == VAR_DICT || type == VAR_LIST) {
+      return hi;
-          typval_T *etv = eval_expr(tv.vval.v_string, NULL);
-
-          if (etv == NULL)
-            /* Failed to parse back the dict or list, use it as a
-             * string. */
-            tv.v_type = VAR_STRING;
-          else {
-            xfree(tv.vval.v_string);
-            tv = *etv;
-            xfree(etv);
-          }
-        }
-
-        set_var(virp->vir_line + 1, &tv, FALSE);
-
-        if (tv.v_type == VAR_STRING)
-          xfree(tv.vval.v_string);
-        else if (tv.v_type == VAR_DICT || tv.v_type == VAR_LIST)
-          clear_tv(&tv);
     }
   }
+  return NULL;
 }
 
-  return viminfo_readline(virp);
+void var_set_global(const char *const name, typval_T vartv)
-}
-
-/*
- * Write global vars that start with a capital to the viminfo file
- */
-void write_viminfo_varlist(FILE *fp)
 {
-  hashitem_T  *hi;
+  funccall_T *const saved_current_funccal = current_funccal;
-  dictitem_T  *this_var;
+  current_funccal = NULL;
-  int todo;
+  set_var((char_u *) name, &vartv, false);
-  char        *s;
+  current_funccal = saved_current_funccal;
-  char_u      *p;
-
-  if (find_viminfo_parameter('!') == NULL)
-    return;
-
-  fputs(_("\n# global variables:\n"), fp);
-
-  todo = (int)globvarht.ht_used;
-  for (hi = globvarht.ht_array; todo > 0; ++hi) {
-    if (!HASHITEM_EMPTY(hi)) {
-      --todo;
-      this_var = HI2DI(hi);
-      if (var_flavour(this_var->di_key) == VAR_FLAVOUR_VIMINFO) {
-        switch (this_var->di_tv.v_type) {
-        case VAR_STRING: s = "STR"; break;
-        case VAR_NUMBER: s = "NUM"; break;
-        case VAR_FLOAT:  s = "FLO"; break;
-        case VAR_DICT:   s = "DIC"; break;
-        case VAR_LIST:   s = "LIS"; break;
-        default: continue;
-        }
-        fprintf(fp, "!%s\t%s\t", this_var->di_key, s);
-        p = (char_u *) echo_string(&this_var->di_tv, NULL);
-        if (p != NULL) {
-          viminfo_writestring(fp, p);
-        }
-        xfree(p);
-      }
-    }
-  }
 }
 
 int store_session_globals(FILE *fd)

src/nvim/eval.h

@@ -1,6 +1,8 @@
 #ifndef NVIM_EVAL_H
 #define NVIM_EVAL_H
 
+#include <msgpack.h>
+
 #include "nvim/profile.h"
 
 /* Defines for Vim variables.  These must match vimvars[] in eval.c! */
@@ -72,6 +74,8 @@ enum {
 /// Maximum number of function arguments
 #define MAX_FUNC_ARGS   20
 
+int vim_to_msgpack(msgpack_packer *const, typval_T *const);
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "eval.h.generated.h"
 #endif

src/nvim/eval_defs.h

@@ -2,6 +2,7 @@
 #define NVIM_EVAL_DEFS_H
 
 #include <limits.h>
+#include <stddef.h>
 
 #include "nvim/hashtab.h"
 
@@ -132,4 +133,16 @@ typedef struct list_stack_S {
   struct list_stack_S *prev;
 } list_stack_T;
 
+// In a hashtab item "hi_key" points to "di_key" in a dictitem.
+// This avoids adding a pointer to the hashtab item.
+
+/// Convert a dictitem pointer to a hashitem key pointer
+#define DI2HIKEY(di) ((di)->di_key)
+
+/// Convert a hashitem key pointer to a dictitem pointer
+#define HIKEY2DI(p)  ((dictitem_T *)(p - offsetof(dictitem_T, di_key)))
+
+/// Convert a hashitem pointer to a dictitem pointer
+#define HI2DI(hi)     HIKEY2DI((hi)->hi_key)
+
 #endif  // NVIM_EVAL_DEFS_H

src/nvim/ex_cmds.c

@@ -67,6 +67,7 @@
 #include "nvim/os/os.h"
 #include "nvim/os/shell.h"
 #include "nvim/os/input.h"
+#include "nvim/os/time.h"
 
 /*
  * Struct to hold the sign properties.
@@ -1391,550 +1392,6 @@ void append_redir(char_u *buf, int buflen, char_u *opt, char_u *fname)
         (char *)opt, (char *)fname);
 }
 
-
-static int viminfo_errcnt;
-
-static int no_viminfo(void)
-{
-  /* "vim -i NONE" does not read or write a viminfo file */
-  return use_viminfo != NULL && STRCMP(use_viminfo, "NONE") == 0;
-}
-
-/*
- * Report an error for reading a viminfo file.
- * Count the number of errors.	When there are more than 10, return TRUE.
- */
-int viminfo_error(char *errnum, char *message, char_u *line)
-{
-  vim_snprintf((char *)IObuff, IOSIZE, _("%sviminfo: %s in line: "),
-      errnum, message);
-  STRNCAT(IObuff, line, IOSIZE - STRLEN(IObuff) - 1);
-  if (IObuff[STRLEN(IObuff) - 1] == '\n')
-    IObuff[STRLEN(IObuff) - 1] = NUL;
-  emsg(IObuff);
-  if (++viminfo_errcnt >= 10) {
-    EMSG(_("E136: viminfo: Too many errors, skipping rest of file"));
-    return TRUE;
-  }
-  return FALSE;
-}
-
-/*
- * read_viminfo() -- Read the viminfo file.  Registers etc. which are already
- * set are not over-written unless "flags" includes VIF_FORCEIT. -- webb
- */
-int 
-read_viminfo (
-    char_u *file,          /* file name or NULL to use default name */
-    int flags                  /* VIF_WANT_INFO et al. */
-)
-{
-  FILE        *fp;
-  char_u      *fname;
-
-  if (no_viminfo())
-    return FAIL;
-
-  fname = viminfo_filename(file);       /* get file name in allocated buffer */
-  fp = mch_fopen((char *)fname, READBIN);
-
-  if (p_verbose > 0) {
-    verbose_enter();
-    smsg(_("Reading viminfo file \"%s\"%s%s%s"),
-        fname,
-        (flags & VIF_WANT_INFO) ? _(" info") : "",
-        (flags & VIF_WANT_MARKS) ? _(" marks") : "",
-        (flags & VIF_GET_OLDFILES) ? _(" oldfiles") : "",
-        fp == NULL ? _(" FAILED") : "");
-    verbose_leave();
-  }
-
-  xfree(fname);
-  if (fp == NULL)
-    return FAIL;
-
-  viminfo_errcnt = 0;
-  do_viminfo(fp, NULL, flags);
-
-  fclose(fp);
-  return OK;
-}
-
-/*
- * Write the viminfo file.  The old one is read in first so that effectively a
- * merge of current info and old info is done.  This allows multiple vims to
- * run simultaneously, without losing any marks etc.
- * If "forceit" is TRUE, then the old file is not read in, and only internal
- * info is written to the file.
- */
-void write_viminfo(char_u *file, int forceit)
-{
-  char_u      *fname;
-  FILE        *fp_in = NULL;    /* input viminfo file, if any */
-  FILE        *fp_out = NULL;   /* output viminfo file */
-  char_u      *tempname = NULL;         /* name of temp viminfo file */
-  char_u      *wp;
-#if defined(UNIX)
-  mode_t umask_save;
-#endif
-
-  if (no_viminfo())
-    return;
-
-  fname = viminfo_filename(file);       /* may set to default if NULL */
-
-  fp_in = mch_fopen((char *)fname, READBIN);
-  if (fp_in == NULL) {
-    /* if it does exist, but we can't read it, don't try writing */
-    if (os_file_exists(fname))
-      goto end;
-#if defined(UNIX)
-    /*
-     * For Unix we create the .viminfo non-accessible for others,
-     * because it may contain text from non-accessible documents.
-     */
-    umask_save = umask(077);
-#endif
-    fp_out = mch_fopen((char *)fname, WRITEBIN);
-#if defined(UNIX)
-    (void)umask(umask_save);
-#endif
-  } else {
-    /*
-     * There is an existing viminfo file.  Create a temporary file to
-     * write the new viminfo into, in the same directory as the
-     * existing viminfo file, which will be renamed later.
-     */
-#ifdef UNIX
-    /*
-     * For Unix we check the owner of the file.  It's not very nice to
-     * overwrite a user's viminfo file after a "su root", with a
-     * viminfo file that the user can't read.
-     */
-
-    FileInfo old_info;  // FileInfo of existing viminfo file
-    if (os_fileinfo((char *)fname, &old_info)
-        && getuid() != ROOT_UID
-        && !(old_info.stat.st_uid == getuid()
-             ? (old_info.stat.st_mode & 0200)
-             : (old_info.stat.st_gid == getgid()
-                ? (old_info.stat.st_mode & 0020)
-                : (old_info.stat.st_mode & 0002)))) {
-      int tt = msg_didany;
-
-      /* avoid a wait_return for this message, it's annoying */
-      EMSG2(_("E137: Viminfo file is not writable: %s"), fname);
-      msg_didany = tt;
-      fclose(fp_in);
-      goto end;
-    }
-#endif
-
-    // Make tempname
-    tempname = (char_u *)modname((char *)fname, ".tmp", FALSE);
-    if (tempname != NULL) {
-      /*
-       * Check if tempfile already exists.  Never overwrite an
-       * existing file!
-       */
-      if (os_file_exists(tempname)) {
-        /*
-         * Try another name.  Change one character, just before
-         * the extension.
-         */
-        wp = tempname + STRLEN(tempname) - 5;
-        if (wp < path_tail(tempname))                 /* empty file name? */
-          wp = path_tail(tempname);
-        for (*wp = 'z'; os_file_exists(tempname); --*wp) {
-          /*
-           * They all exist?  Must be something wrong! Don't
-           * write the viminfo file then.
-           */
-          if (*wp == 'a') {
-            xfree(tempname);
-            tempname = NULL;
-            break;
-          }
-        }
-      }
-    }
-
-    if (tempname != NULL) {
-      int fd;
-
-      /* Use os_open() to be able to use O_NOFOLLOW and set file
-       * protection:
-       * Unix: same as original file, but strip s-bit.  Reset umask to
-       * avoid it getting in the way.
-       * Others: r&w for user only. */
-# ifdef UNIX
-      umask_save = umask(0);
-      fd = os_open((char *)tempname,
-          O_CREAT|O_EXCL|O_WRONLY|O_NOFOLLOW,
-          (int)((old_info.stat.st_mode & 0777) | 0600));
-      (void)umask(umask_save);
-# else
-      fd = os_open((char *)tempname,
-          O_CREAT|O_EXCL|O_WRONLY|O_NOFOLLOW, 0600);
-# endif
-      if (fd < 0)
-        fp_out = NULL;
-      else
-        fp_out = fdopen(fd, WRITEBIN);
-
-      /*
-       * If we can't create in the same directory, try creating a
-       * "normal" temp file.
-       */
-      if (fp_out == NULL) {
-        xfree(tempname);
-        if ((tempname = vim_tempname()) != NULL)
-          fp_out = mch_fopen((char *)tempname, WRITEBIN);
-      }
-
-#ifdef UNIX
-      /*
-       * Make sure the owner can read/write it.  This only works for
-       * root.
-       */
-      if (fp_out != NULL) {
-        os_fchown(fileno(fp_out), old_info.stat.st_uid, old_info.stat.st_gid);
-      }
-#endif
-    }
-  }
-
-  /*
-   * Check if the new viminfo file can be written to.
-   */
-  if (fp_out == NULL) {
-    EMSG2(_("E138: Can't write viminfo file %s!"),
-        (fp_in == NULL || tempname == NULL) ? fname : tempname);
-    if (fp_in != NULL)
-      fclose(fp_in);
-    goto end;
-  }
-
-  if (p_verbose > 0) {
-    verbose_enter();
-    smsg(_("Writing viminfo file \"%s\""), fname);
-    verbose_leave();
-  }
-
-  viminfo_errcnt = 0;
-  do_viminfo(fp_in, fp_out, forceit ? 0 : (VIF_WANT_INFO | VIF_WANT_MARKS));
-
-  fclose(fp_out);           /* errors are ignored !? */
-  if (fp_in != NULL) {
-    fclose(fp_in);
-
-    /* In case of an error keep the original viminfo file. Otherwise
-     * rename the newly written file. Give an error if that fails. */
-    if (viminfo_errcnt == 0 && vim_rename(tempname, fname) == -1) {
-      viminfo_errcnt++;
-      EMSG2(_("E886: Can't rename viminfo file to %s!"), fname);
-    }
-    if (viminfo_errcnt > 0) {
-      os_remove((char *)tempname);
-    }
-  }
-
-end:
-  xfree(fname);
-  xfree(tempname);
-}
-
-/*
- * Get the viminfo file name to use.
- * If "file" is given and not empty, use it (has already been expanded by
- * cmdline functions).
- * Otherwise use "-i file_name", value from 'viminfo' or the default, and
- * expand environment variables.
- * Returns an allocated string.
- */
-static char_u *viminfo_filename(char_u *file)
-{
-  if (file == NULL || *file == NUL) {
-    if (use_viminfo != NULL)
-      file = use_viminfo;
-    else if ((file = find_viminfo_parameter('n')) == NULL || *file == NUL) {
-#ifdef VIMINFO_FILE2
-      // don't use $HOME when not defined (turned into "c:/"!).
-      if (!os_env_exists("HOME")) {
-        // don't use $VIM when not available.
-        expand_env((char_u *)"$VIM", NameBuff, MAXPATHL);
-        if (STRCMP("$VIM", NameBuff) != 0)          /* $VIM was expanded */
-          file = (char_u *)VIMINFO_FILE2;
-        else
-          file = (char_u *)VIMINFO_FILE;
-      } else
-#endif
-      file = (char_u *)VIMINFO_FILE;
-    }
-    expand_env(file, NameBuff, MAXPATHL);
-    file = NameBuff;
-  }
-  return vim_strsave(file);
-}
-
-/*
- * do_viminfo() -- Should only be called from read_viminfo() & write_viminfo().
- */
-static void do_viminfo(FILE *fp_in, FILE *fp_out, int flags)
-{
-  int count = 0;
-  int eof = FALSE;
-  vir_T vir;
-  int merge = FALSE;
-
-  vir.vir_line = xmalloc(LSIZE);
-  vir.vir_fd = fp_in;
-  vir.vir_conv.vc_type = CONV_NONE;
-
-  if (fp_in != NULL) {
-    if (flags & VIF_WANT_INFO) {
-      eof = read_viminfo_up_to_marks(&vir,
-          flags & VIF_FORCEIT, fp_out != NULL);
-      merge = TRUE;
-    } else if (flags != 0)
-      /* Skip info, find start of marks */
-      while (!(eof = viminfo_readline(&vir))
-             && vir.vir_line[0] != '>')
-        ;
-  }
-  if (fp_out != NULL) {
-    /* Write the info: */
-    fprintf(fp_out, _("# This viminfo file was generated by Nvim %s.\n"),
-        mediumVersion);
-    fputs(_("# You may edit it if you're careful!\n\n"), fp_out);
-    fputs(_("# Value of 'encoding' when this file was written\n"), fp_out);
-    fprintf(fp_out, "*encoding=%s\n\n", p_enc);
-    write_viminfo_search_pattern(fp_out);
-    write_viminfo_sub_string(fp_out);
-    write_viminfo_history(fp_out, merge);
-    write_viminfo_registers(fp_out);
-    write_viminfo_varlist(fp_out);
-    write_viminfo_filemarks(fp_out);
-    write_viminfo_bufferlist(fp_out);
-    count = write_viminfo_marks(fp_out);
-  }
-  if (fp_in != NULL
-      && (flags & (VIF_WANT_MARKS | VIF_GET_OLDFILES | VIF_FORCEIT)))
-    copy_viminfo_marks(&vir, fp_out, count, eof, flags);
-
-  xfree(vir.vir_line);
-  if (vir.vir_conv.vc_type != CONV_NONE)
-    convert_setup(&vir.vir_conv, NULL, NULL);
-}
-
-/*
- * read_viminfo_up_to_marks() -- Only called from do_viminfo().  Reads in the
- * first part of the viminfo file which contains everything but the marks that
- * are local to a file.  Returns TRUE when end-of-file is reached. -- webb
- */
-static int read_viminfo_up_to_marks(vir_T *virp, int forceit, int writing)
-{
-  int eof;
-
-  prepare_viminfo_history(forceit ? 9999 : 0, writing);
-  eof = viminfo_readline(virp);
-  while (!eof && virp->vir_line[0] != '>') {
-    switch (virp->vir_line[0]) {
-    /* Characters reserved for future expansion, ignored now */
-    case '+':         /* "+40 /path/dir file", for running vim without args */
-    case '|':         /* to be defined */
-    case '^':         /* to be defined */
-    case '<':         /* long line - ignored */
-    /* A comment or empty line. */
-    case NUL:
-    case '\r':
-    case '\n':
-    case '#':
-      eof = viminfo_readline(virp);
-      break;
-    case '*':         /* "*encoding=value" */
-      eof = viminfo_encoding(virp);
-      break;
-    case '!':         /* global variable */
-      eof = read_viminfo_varlist(virp, writing);
-      break;
-    case '%':         /* entry for buffer list */
-      eof = read_viminfo_bufferlist(virp, writing);
-      break;
-    case '"':
-      eof = read_viminfo_register(virp, forceit);
-      break;
-    case '/':               /* Search string */
-    case '&':               /* Substitute search string */
-    case '~':               /* Last search string, followed by '/' or '&' */
-      eof = read_viminfo_search_pattern(virp, forceit);
-      break;
-    case '$':
-      eof = read_viminfo_sub_string(virp, forceit);
-      break;
-    case ':':
-    case '?':
-    case '=':
-    case '@':
-      eof = read_viminfo_history(virp, writing);
-      break;
-    case '-':
-    case '\'':
-      eof = read_viminfo_filemark(virp, forceit);
-      break;
-    default:
-      if (viminfo_error("E575: ", _("Illegal starting char"),
-              virp->vir_line))
-        eof = TRUE;
-      else
-        eof = viminfo_readline(virp);
-      break;
-    }
-  }
-
-  /* Finish reading history items. */
-  if (!writing)
-    finish_viminfo_history();
-
-  /* Change file names to buffer numbers for fmarks. */
-  FOR_ALL_BUFFERS(buf) {
-    fmarks_check_names(buf);
-  }
-
-  return eof;
-}
-
-/*
- * Compare the 'encoding' value in the viminfo file with the current value of
- * 'encoding'.  If different and the 'c' flag is in 'viminfo', setup for
- * conversion of text with iconv() in viminfo_readstring().
- */
-static int viminfo_encoding(vir_T *virp)
-{
-  char_u      *p;
-  int i;
-
-  if (get_viminfo_parameter('c') != 0) {
-    p = vim_strchr(virp->vir_line, '=');
-    if (p != NULL) {
-      /* remove trailing newline */
-      ++p;
-      for (i = 0; vim_isprintc(p[i]); ++i)
-        ;
-      p[i] = NUL;
-
-      convert_setup(&virp->vir_conv, p, p_enc);
-    }
-  }
-  return viminfo_readline(virp);
-}
-
-/*
- * Read a line from the viminfo file.
- * Returns TRUE for end-of-file;
- */
-int viminfo_readline(vir_T *virp)
-{
-  return vim_fgets(virp->vir_line, LSIZE, virp->vir_fd);
-}
-
-/*
- * check string read from viminfo file
- * remove '\n' at the end of the line
- * - replace CTRL-V CTRL-V with CTRL-V
- * - replace CTRL-V 'n'    with '\n'
- *
- * Check for a long line as written by viminfo_writestring().
- *
- * Return the string in allocated memory.
- */
-char_u *
-viminfo_readstring (
-    vir_T *virp,
-    int off,                            /* offset for virp->vir_line */
-    int convert                 /* convert the string */
-)
-  FUNC_ATTR_NONNULL_RET
-{
-  char_u      *retval;
-  char_u      *s, *d;
-
-  if (virp->vir_line[off] == Ctrl_V && ascii_isdigit(virp->vir_line[off + 1])) {
-    ssize_t len = atol((char *)virp->vir_line + off + 1);
-    retval = xmalloc(len);
-    // TODO(philix): change type of vim_fgets() size argument to size_t
-    (void)vim_fgets(retval, (int)len, virp->vir_fd);
-    s = retval + 1;         /* Skip the leading '<' */
-  } else {
-    retval = vim_strsave(virp->vir_line + off);
-    s = retval;
-  }
-
-  /* Change CTRL-V CTRL-V to CTRL-V and CTRL-V n to \n in-place. */
-  d = retval;
-  while (*s != NUL && *s != '\n') {
-    if (s[0] == Ctrl_V && s[1] != NUL) {
-      if (s[1] == 'n')
-        *d++ = '\n';
-      else
-        *d++ = Ctrl_V;
-      s += 2;
-    } else
-      *d++ = *s++;
-  }
-  *d = NUL;
-
-  if (convert && virp->vir_conv.vc_type != CONV_NONE && *retval != NUL) {
-    d = string_convert(&virp->vir_conv, retval, NULL);
-    if (d != NULL) {
-      xfree(retval);
-      retval = d;
-    }
-  }
-
-  return retval;
-}
-
-/*
- * write string to viminfo file
- * - replace CTRL-V with CTRL-V CTRL-V
- * - replace '\n'   with CTRL-V 'n'
- * - add a '\n' at the end
- *
- * For a long line:
- * - write " CTRL-V <length> \n " in first line
- * - write " < <string> \n "	  in second line
- */
-void viminfo_writestring(FILE *fd, char_u *p)
-{
-  int c;
-  char_u      *s;
-  int len = 0;
-
-  for (s = p; *s != NUL; ++s) {
-    if (*s == Ctrl_V || *s == '\n')
-      ++len;
-    ++len;
-  }
-
-  /* If the string will be too long, write its length and put it in the next
-   * line.  Take into account that some room is needed for what comes before
-   * the string (e.g., variable name).  Add something to the length for the
-   * '<', NL and trailing NUL. */
-  if (len > LSIZE / 2)
-    fprintf(fd, "\026%d\n<", len + 3);
-
-  while ((c = *p++) != NUL) {
-    if (c == Ctrl_V || c == '\n') {
-      putc(Ctrl_V, fd);
-      if (c == '\n')
-        c = 'n';
-    }
-    putc(c, fd);
-  }
-  putc('\n', fd);
-}
-
 void print_line_no_prefix(linenr_T lnum, int use_number, int list)
 {
   char_u numbuf[30];
@@ -3364,8 +2821,33 @@ int check_secure(void)
   return FALSE;
 }
 
-static char_u   *old_sub = NULL;        /* previous substitute pattern */
+/// Previous substitute replacement string
-static int global_need_beginline;       /* call beginline() after ":g" */
+static SubReplacementString old_sub = {NULL, 0, NULL};
+
+static int global_need_beginline;       // call beginline() after ":g"
+
+/// Get old substitute replacement string
+///
+/// @param[out]  ret_sub    Location where old string will be saved.
+void sub_get_replacement(SubReplacementString *const ret_sub)
+  FUNC_ATTR_NONNULL_ALL
+{
+  *ret_sub = old_sub;
+}
+
+/// Set substitute string and timestamp
+///
+/// @warning `sub` must be in allocated memory. It is not copied.
+///
+/// @param[in]  sub  New replacement string.
+void sub_set_replacement(SubReplacementString sub)
+{
+  xfree(old_sub.sub);
+  if (sub.additional_elements != old_sub.additional_elements) {
+    list_unref(old_sub.additional_elements);
+  }
+  old_sub = sub;
+}
 
 /* do_sub()
  *
@@ -3473,16 +2955,19 @@ void do_sub(exarg_T *eap)
     }
 
     if (!eap->skip) {
-      xfree(old_sub);
+      sub_set_replacement((SubReplacementString) {
-      old_sub = vim_strsave(sub);
+        .sub = xstrdup((char *) sub),
+        .timestamp = os_time(),
+        .additional_elements = NULL,
+      });
     }
   } else if (!eap->skip) {    /* use previous pattern and substitution */
-    if (old_sub == NULL) {      /* there is no previous command */
+    if (old_sub.sub == NULL) {      /* there is no previous command */
       EMSG(_(e_nopresub));
       return;
     }
     pat = NULL;                 /* search_regcomp() will use previous pattern */
-    sub = old_sub;
+    sub = (char_u *) old_sub.sub;
 
     /* Vi compatibility quirk: repeating with ":s" keeps the cursor in the
      * last column after using "$". */
@@ -4501,27 +3986,10 @@ void global_exe(char_u *cmd)
     msgmore(curbuf->b_ml.ml_line_count - old_lcount);
 }
 
-int read_viminfo_sub_string(vir_T *virp, int force)
-{
-  if (force)
-    xfree(old_sub);
-  if (force || old_sub == NULL)
-    old_sub = viminfo_readstring(virp, 1, TRUE);
-  return viminfo_readline(virp);
-}
-
-void write_viminfo_sub_string(FILE *fp)
-{
-  if (get_viminfo_parameter('/') != 0 && old_sub != NULL) {
-    fputs(_("\n# Last Substitute String:\n$"), fp);
-    viminfo_writestring(fp, old_sub);
-  }
-}
-
 #if defined(EXITFREE)
 void free_old_sub(void)
 {
-  xfree(old_sub);
+  sub_set_replacement((SubReplacementString) {NULL, 0, NULL});
 }
 
 #endif

src/nvim/ex_cmds.h

@@ -3,6 +3,9 @@
 
 #include <stdbool.h>
 
+#include "nvim/os/time.h"
+#include "nvim/eval_defs.h"
+
 /* flags for do_ecmd() */
 #define ECMD_HIDE       0x01    /* don't free the current buffer */
 #define ECMD_SET_HELP   0x02    /* set b_help flag of (new) buffer before
@@ -16,11 +19,12 @@
 #define ECMD_LAST       (linenr_T)-1    /* use last position in all files */
 #define ECMD_ONE        (linenr_T)1     /* use first line */
 
-/* flags for read_viminfo() and children */
+/// Previous :substitute replacement string definition
-#define VIF_WANT_INFO           1       /* load non-mark info */
+typedef struct {
-#define VIF_WANT_MARKS          2       /* load file marks */
+  char *sub;            ///< Previous replacement string.
-#define VIF_FORCEIT             4       /* overwrite info already read */
+  Timestamp timestamp;  ///< Time when it was last set.
-#define VIF_GET_OLDFILES        8       /* load v:oldfiles */
+  list_T *additional_elements;  ///< Additional data left from ShaDa file.
+} SubReplacementString;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ex_cmds.h.generated.h"

src/nvim/ex_cmds.lua

@@ -2119,6 +2119,12 @@ return {
     addr_type=ADDR_LINES,
     func='ex_wrongmodifier',
   },
+  {
+    command='rshada',
+    flags=bit.bor(BANG, FILE1, TRLBAR, CMDWIN),
+    addr_type=ADDR_LINES,
+    func='ex_shada',
+  },
   {
     command='runtime',
     flags=bit.bor(BANG, NEEDARG, FILES, TRLBAR, SBOXOK, CMDWIN),
@@ -2153,7 +2159,7 @@ return {
     command='rviminfo',
     flags=bit.bor(BANG, FILE1, TRLBAR, CMDWIN),
     addr_type=ADDR_LINES,
-    func='ex_viminfo',
+    func='ex_shada',
   },
   {
     command='substitute',
@@ -3031,6 +3037,12 @@ return {
     addr_type=ADDR_LINES,
     func='ex_wsverb',
   },
+  {
+    command='wshada',
+    flags=bit.bor(BANG, FILE1, TRLBAR, CMDWIN),
+    addr_type=ADDR_LINES,
+    func='ex_shada',
+  },
   {
     command='wundo',
     flags=bit.bor(BANG, NEEDARG, FILE1),
@@ -3041,7 +3053,7 @@ return {
     command='wviminfo',
     flags=bit.bor(BANG, FILE1, TRLBAR, CMDWIN),
     addr_type=ADDR_LINES,
-    func='ex_viminfo',
+    func='ex_shada',
   },
   {
     command='xit',

src/nvim/ex_docmd.c

@@ -75,6 +75,7 @@
 #include "nvim/mouse.h"
 #include "nvim/event/rstream.h"
 #include "nvim/event/wstream.h"
+#include "nvim/shada.h"
 
 static int quitmore = 0;
 static int ex_pressedreturn = FALSE;
@@ -9139,22 +9140,21 @@ int put_line(FILE *fd, char *s)
 }
 
 /*
- * ":rviminfo" and ":wviminfo".
+ * ":rshada" and ":wshada".
  */
-static void ex_viminfo(exarg_T *eap)
+static void ex_shada(exarg_T *eap)
 {
-  char_u      *save_viminfo;
+  char_u      *save_shada;
 
-  save_viminfo = p_viminfo;
+  save_shada = p_shada;
-  if (*p_viminfo == NUL)
+  if (*p_shada == NUL)
-    p_viminfo = (char_u *)"'100";
+    p_shada = (char_u *)"'100";
-  if (eap->cmdidx == CMD_rviminfo) {
+  if (eap->cmdidx == CMD_rviminfo || eap->cmdidx == CMD_rshada) {
-    if (read_viminfo(eap->arg, VIF_WANT_INFO | VIF_WANT_MARKS
+    (void) shada_read_everything((char *) eap->arg, eap->forceit, false);
-            | (eap->forceit ? VIF_FORCEIT : 0)) == FAIL)
+  } else {
-      EMSG(_("E195: Cannot open viminfo file for reading"));
+    shada_write_file((char *) eap->arg, eap->forceit);
-  } else
+  }
-    write_viminfo(eap->arg, eap->forceit);
+  p_shada = save_shada;
-  p_viminfo = save_viminfo;
 }
 
 /*

src/nvim/ex_getln.c

@@ -65,6 +65,7 @@
 #include "nvim/os/input.h"
 #include "nvim/os/os.h"
 #include "nvim/event/loop.h"
+#include "nvim/os/time.h"
 
 /*
  * Variables shared between getcmdline(), redrawcmdline() and others.
@@ -100,12 +101,6 @@ static int cmd_showtail;                /* Only show path tail in lists ? */
 
 static int new_cmdpos;          /* position set by set_cmdline_pos() */
 
-typedef struct hist_entry {
-  int hisnum;                   /* identifying number */
-  int viminfo;                  /* when TRUE hisstr comes from viminfo */
-  char_u      *hisstr;          /* actual entry, separator char after the NUL */
-} histentry_T;
-
 /*
  * Type used by call_user_expand_func
  */
@@ -4230,12 +4225,10 @@ void init_history(void)
 
         // delete entries that don't fit in newlen, if any
         for (int i = 0; i < i1; i++) {
-          xfree(history[type][i].hisstr);
+          hist_free_entry(history[type] + i);
-          history[type][i].hisstr = NULL;
         }
         for (int i = i1 + l1; i < i2; i++) {
-          xfree(history[type][i].hisstr);
+          hist_free_entry(history[type] + i);
-          history[type][i].hisstr = NULL;
         }
       }
 
@@ -4253,11 +4246,18 @@ void init_history(void)
   }
 }
 
-static void clear_hist_entry(histentry_T *hisptr)
+static inline void hist_free_entry(histentry_T *hisptr)
+  FUNC_ATTR_NONNULL_ALL
 {
-  hisptr->hisnum = 0;
+  xfree(hisptr->hisstr);
-  hisptr->viminfo = FALSE;
+  list_unref(hisptr->additional_elements);
-  hisptr->hisstr = NULL;
+  clear_hist_entry(hisptr);
+}
+
+static inline void clear_hist_entry(histentry_T *hisptr)
+  FUNC_ATTR_NONNULL_ALL
+{
+  memset(hisptr, 0, sizeof(*hisptr));
 }
 
 /*
@@ -4268,9 +4268,8 @@ static int
 in_history (
     int type,
     char_u *str,
-    int move_to_front,              /* Move the entry to the front if it exists */
+    int move_to_front,              // Move the entry to the front if it exists
-    int sep,
+    int sep
-    int writing                    /* ignore entries read from viminfo */
 )
 {
   int i;
@@ -4288,7 +4287,6 @@ in_history (
      * well. */
     p = history[type][i].hisstr;
     if (STRCMP(str, p) == 0
-        && !(writing && history[type][i].viminfo)
         && (type != HIST_SEARCH || sep == p[STRLEN(p) + 1])) {
       if (!move_to_front)
         return TRUE;
@@ -4300,6 +4298,7 @@ in_history (
   } while (i != hisidx[type]);
 
   if (last_i >= 0) {
+    list_T *const list = history[type][i].additional_elements;
     str = history[type][i].hisstr;
     while (i != hisidx[type]) {
       if (++i >= hislen)
@@ -4307,12 +4306,14 @@ in_history (
       history[type][last_i] = history[type][i];
       last_i = i;
     }
+    list_unref(list);
     history[type][i].hisnum = ++hisnum[type];
-    history[type][i].viminfo = FALSE;
     history[type][i].hisstr = str;
-    return TRUE;
+    history[type][i].timestamp = os_time();
+    history[type][i].additional_elements = NULL;
+    return true;
   }
-  return FALSE;
+  return false;
 }
 
 /*
@@ -4372,27 +4373,27 @@ add_to_history (
     if (maptick == last_maptick) {
       /* Current line is from the same mapping, remove it */
       hisptr = &history[HIST_SEARCH][hisidx[HIST_SEARCH]];
-      xfree(hisptr->hisstr);
+      hist_free_entry(hisptr);
-      clear_hist_entry(hisptr);
       --hisnum[histype];
       if (--hisidx[HIST_SEARCH] < 0)
         hisidx[HIST_SEARCH] = hislen - 1;
     }
     last_maptick = -1;
   }
-  if (!in_history(histype, new_entry, TRUE, sep, FALSE)) {
+  if (!in_history(histype, new_entry, true, sep)) {
     if (++hisidx[histype] == hislen)
       hisidx[histype] = 0;
     hisptr = &history[histype][hisidx[histype]];
-    xfree(hisptr->hisstr);
+    hist_free_entry(hisptr);
 
     /* Store the separator after the NUL of the string. */
     len = (int)STRLEN(new_entry);
     hisptr->hisstr = vim_strnsave(new_entry, len + 2);
+    hisptr->timestamp = os_time();
+    hisptr->additional_elements = NULL;
     hisptr->hisstr[len + 1] = sep;
 
     hisptr->hisnum = ++hisnum[histype];
-    hisptr->viminfo = FALSE;
     if (histype == HIST_SEARCH && in_map)
       last_maptick = maptick;
   }
@@ -4545,23 +4546,21 @@ char_u *get_history_entry(int histype, int idx)
     return (char_u *)"";
 }
 
-/*
+/// Clear all entries in a history
- * Clear all entries of a history.
+///
- * "histype" may be one of the HIST_ values.
+/// @param[in]  histype  One of the HIST_ values.
- */
+///
-int clr_history(int histype)
+/// @return OK if there was something to clean and histype was one of HIST_
+///         values, FAIL otherwise.
+int clr_history(const int histype)
 {
-  int i;
-  histentry_T *hisptr;
-
   if (hislen != 0 && histype >= 0 && histype < HIST_COUNT) {
-    hisptr = history[histype];
+    histentry_T *hisptr = history[histype];
-    for (i = hislen; i--; ) {
+    for (int i = hislen; i--; hisptr++) {
-      xfree(hisptr->hisstr);
+      hist_free_entry(hisptr);
-      clear_hist_entry(hisptr);
     }
-    hisidx[histype] = -1;       /* mark history as cleared */
+    hisidx[histype] = -1;  // mark history as cleared
-    hisnum[histype] = 0;        /* reset identifier counter */
+    hisnum[histype] = 0;   // reset identifier counter
     return OK;
   }
   return FAIL;
@@ -4578,7 +4577,7 @@ int del_history_entry(int histype, char_u *str)
   int idx;
   int i;
   int last;
-  int found = FALSE;
+  bool found = false;
 
   regmatch.regprog = NULL;
   regmatch.rm_ic = FALSE;       /* always match case */
@@ -4595,9 +4594,8 @@ int del_history_entry(int histype, char_u *str)
       if (hisptr->hisstr == NULL)
         break;
       if (vim_regexec(&regmatch, hisptr->hisstr, (colnr_T)0)) {
-        found = TRUE;
+        found = true;
-        xfree(hisptr->hisstr);
+        hist_free_entry(hisptr);
-        clear_hist_entry(hisptr);
       } else {
         if (i != last) {
           history[histype][last] = *hisptr;
@@ -4628,7 +4626,7 @@ int del_history_idx(int histype, int idx)
   if (i < 0)
     return FALSE;
   idx = hisidx[histype];
-  xfree(history[histype][i].hisstr);
+  hist_free_entry(&history[histype][i]);
 
   /* When deleting the last added search string in a mapping, reset
    * last_maptick, so that the last added search string isn't deleted again.
@@ -4641,9 +4639,10 @@ int del_history_idx(int histype, int idx)
     history[histype][i] = history[histype][j];
     i = j;
   }
-  clear_hist_entry(&history[histype][i]);
+  clear_hist_entry(&history[histype][idx]);
-  if (--i < 0)
+  if (--i < 0) {
     i += hislen;
+  }
   hisidx[histype] = i;
   return TRUE;
 }
@@ -4762,248 +4761,31 @@ void ex_history(exarg_T *eap)
   }
 }
 
-/*
+/// Translate a history type number to the associated character
- * Buffers for history read from a viminfo file.  Only valid while reading.
+int hist_type2char(int type)
- */
+  FUNC_ATTR_CONST
-static char_u **viminfo_history[HIST_COUNT] = {NULL, NULL, NULL, NULL};
-static int viminfo_hisidx[HIST_COUNT] = {0, 0, 0, 0};
-static int viminfo_hislen[HIST_COUNT] = {0, 0, 0, 0};
-static int viminfo_add_at_front = FALSE;
-
-
-/*
- * Translate a history type number to the associated character.
- */
-static int 
-hist_type2char (
-    int type,
-    int use_question                   /* use '?' instead of '/' */
-)
 {
-  if (type == HIST_CMD)
+  switch (type) {
+    case HIST_CMD: {
       return ':';
-  if (type == HIST_SEARCH) {
+    }
-    if (use_question)
+    case HIST_SEARCH: {
-      return '?';
-    else
       return '/';
     }
-  if (type == HIST_EXPR)
+    case HIST_EXPR: {
       return '=';
+    }
+    case HIST_INPUT: {
       return '@';
     }
-
+    case HIST_DEBUG: {
-/*
+      return '>';
- * Prepare for reading the history from the viminfo file.
+    }
- * This allocates history arrays to store the read history lines.
+    default: {
- */
+      assert(false);
-void prepare_viminfo_history(int asklen, int writing)
-{
-  int i;
-  int num;
-
-  init_history();
-  viminfo_add_at_front = (asklen != 0 && !writing);
-  if (asklen > hislen)
-    asklen = hislen;
-
-  for (int type = 0; type < HIST_COUNT; ++type) {
-    /* Count the number of empty spaces in the history list.  Entries read
-     * from viminfo previously are also considered empty.  If there are
-     * more spaces available than we request, then fill them up. */
-    for (i = 0, num = 0; i < hislen; i++)
-      if (history[type][i].hisstr == NULL || history[type][i].viminfo)
-        num++;
-    int len = asklen;
-    if (num > len)
-      len = num;
-    if (len <= 0)
-      viminfo_history[type] = NULL;
-    else
-      viminfo_history[type] = xmalloc(len * sizeof(char_u *));
-    if (viminfo_history[type] == NULL)
-      len = 0;
-    viminfo_hislen[type] = len;
-    viminfo_hisidx[type] = 0;
     }
   }
-
+  return NUL;
-/*
- * Accept a line from the viminfo, store it in the history array when it's
- * new.
- */
-int read_viminfo_history(vir_T *virp, int writing)
-{
-  int type;
-  char_u      *val;
-
-  type = hist_char2type(virp->vir_line[0]);
-  if (viminfo_hisidx[type] < viminfo_hislen[type]) {
-    val = viminfo_readstring(virp, 1, TRUE);
-    if (val != NULL && *val != NUL) {
-      int sep = (*val == ' ' ? NUL : *val);
-
-      if (!in_history(type, val + (type == HIST_SEARCH),
-              viminfo_add_at_front, sep, writing)) {
-        /* Need to re-allocate to append the separator byte. */
-        size_t len = STRLEN(val);
-        char_u *p = xmalloc(len + 2);
-        if (type == HIST_SEARCH) {
-          /* Search entry: Move the separator from the first
-           * column to after the NUL. */
-          memmove(p, val + 1, len);
-          p[len] = sep;
-        } else {
-          /* Not a search entry: No separator in the viminfo
-           * file, add a NUL separator. */
-          memmove(p, val, len + 1);
-          p[len + 1] = NUL;
-        }
-        viminfo_history[type][viminfo_hisidx[type]++] = p;
-      }
-    }
-    xfree(val);
-  }
-  return viminfo_readline(virp);
-}
-
-/*
- * Finish reading history lines from viminfo.  Not used when writing viminfo.
- */
-void finish_viminfo_history(void)
-{
-  int idx;
-  int i;
-  int type;
-
-  for (type = 0; type < HIST_COUNT; ++type) {
-    if (history[type] == NULL)
-      continue;
-    idx = hisidx[type] + viminfo_hisidx[type];
-    if (idx >= hislen)
-      idx -= hislen;
-    else if (idx < 0)
-      idx = hislen - 1;
-    if (viminfo_add_at_front)
-      hisidx[type] = idx;
-    else {
-      if (hisidx[type] == -1)
-        hisidx[type] = hislen - 1;
-      do {
-        if (history[type][idx].hisstr != NULL
-            || history[type][idx].viminfo)
-          break;
-        if (++idx == hislen)
-          idx = 0;
-      } while (idx != hisidx[type]);
-      if (idx != hisidx[type] && --idx < 0)
-        idx = hislen - 1;
-    }
-    for (i = 0; i < viminfo_hisidx[type]; i++) {
-      xfree(history[type][idx].hisstr);
-      history[type][idx].hisstr = viminfo_history[type][i];
-      history[type][idx].viminfo = TRUE;
-      if (--idx < 0)
-        idx = hislen - 1;
-    }
-    idx += 1;
-    idx %= hislen;
-    for (i = 0; i < viminfo_hisidx[type]; i++) {
-      history[type][idx++].hisnum = ++hisnum[type];
-      idx %= hislen;
-    }
-    xfree(viminfo_history[type]);
-    viminfo_history[type] = NULL;
-    viminfo_hisidx[type] = 0;
-  }
-}
-
-/*
- * Write history to viminfo file in "fp".
- * When "merge" is TRUE merge history lines with a previously read viminfo
- * file, data is in viminfo_history[].
- * When "merge" is FALSE just write all history lines.  Used for ":wviminfo!".
- */
-void write_viminfo_history(FILE *fp, int merge)
-{
-  int i;
-  int type;
-  int num_saved;
-  char_u  *p;
-  int c;
-  int round;
-
-  init_history();
-  if (hislen == 0)
-    return;
-  for (type = 0; type < HIST_COUNT; ++type) {
-    num_saved = get_viminfo_parameter(hist_type2char(type, FALSE));
-    if (num_saved == 0)
-      continue;
-    if (num_saved < 0)      /* Use default */
-      num_saved = hislen;
-    fprintf(fp, _("\n# %s History (newest to oldest):\n"),
-        type == HIST_CMD ? _("Command Line") :
-        type == HIST_SEARCH ? _("Search String") :
-        type == HIST_EXPR ?  _("Expression") :
-        _("Input Line"));
-    if (num_saved > hislen)
-      num_saved = hislen;
-
-    /*
-     * Merge typed and viminfo history:
-     * round 1: history of typed commands.
-     * round 2: history from recently read viminfo.
-     */
-    for (round = 1; round <= 2; ++round) {
-      if (round == 1)
-        /* start at newest entry, somewhere in the list */
-        i = hisidx[type];
-      else if (viminfo_hisidx[type] > 0)
-        /* start at newest entry, first in the list */
-        i = 0;
-      else
-        /* empty list */
-        i = -1;
-      if (i >= 0)
-        while (num_saved > 0
-               && !(round == 2 && i >= viminfo_hisidx[type])) {
-          p = round == 1 ? history[type][i].hisstr
-              : viminfo_history[type] == NULL ? NULL
-              : viminfo_history[type][i];
-          if (p != NULL && (round == 2
-                            || !merge
-                            || !history[type][i].viminfo)) {
-            --num_saved;
-            fputc(hist_type2char(type, TRUE), fp);
-            /* For the search history: put the separator in the
-            * second column; use a space if there isn't one. */
-            if (type == HIST_SEARCH) {
-              c = p[STRLEN(p) + 1];
-              putc(c == NUL ? ' ' : c, fp);
-            }
-            viminfo_writestring(fp, p);
-          }
-          if (round == 1) {
-            /* Decrement index, loop around and stop when back at
-             * the start. */
-            if (--i < 0)
-              i = hislen - 1;
-            if (i == hisidx[type])
-              break;
-          } else {
-            /* Increment index. Stop at the end in the while. */
-            ++i;
-          }
-        }
-    }
-    for (i = 0; i < viminfo_hisidx[type]; ++i)
-      if (viminfo_history[type] != NULL)
-        xfree(viminfo_history[type][i]);
-    xfree(viminfo_history[type]);
-    viminfo_history[type] = NULL;
-    viminfo_hisidx[type] = 0;
-  }
 }
 
 /*
@@ -5294,3 +5076,87 @@ char_u *script_get(exarg_T *eap, char_u *cmd)
 
   return (char_u *)ga.ga_data;
 }
+
+/// Iterate over history items
+///
+/// @warning No history-editing functions must be run while iteration is in
+///          progress.
+///
+/// @param[in]   iter          Pointer to the last history entry.
+/// @param[in]   history_type  Type of the history (HIST_*). Ignored if iter
+///                            parameter is not NULL.
+/// @param[in]   zero          If true then zero (but not free) returned items.
+///
+///                            @warning When using this parameter user is
+///                                     responsible for calling clr_history()
+///                                     itself after iteration is over. If
+///                                     clr_history() is not called behaviour is
+///                                     undefined. No functions that work with
+///                                     history must be called during iteration
+///                                     in this case.
+/// @param[out]  hist          Next history entry.
+///
+/// @return Pointer used in next iteration or NULL to indicate that iteration
+///         was finished.
+const void *hist_iter(const void *const iter, const uint8_t history_type,
+                      const bool zero, histentry_T *const hist)
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ARG(4)
+{
+  *hist = (histentry_T) {
+    .hisstr = NULL
+  };
+  if (hisidx[history_type] == -1) {
+    return NULL;
+  }
+  histentry_T *const hstart = &(history[history_type][0]);
+  histentry_T *const hlast = (
+      &(history[history_type][hisidx[history_type]]));
+  const histentry_T *const hend = &(history[history_type][hislen - 1]);
+  histentry_T *hiter;
+  if (iter == NULL) {
+    histentry_T *hfirst = hlast;
+    do {
+      hfirst++;
+      if (hfirst > hend) {
+        hfirst = hstart;
+      }
+      if (hfirst->hisstr != NULL) {
+        break;
+      }
+    } while (hfirst != hlast);
+    hiter = hfirst;
+  } else {
+    hiter = (histentry_T *) iter;
+  }
+  if (hiter == NULL) {
+    return NULL;
+  }
+  *hist = *hiter;
+  if (zero) {
+    memset(hiter, 0, sizeof(*hiter));
+  }
+  if (hiter == hlast) {
+    return NULL;
+  }
+  hiter++;
+  return (const void *) ((hiter > hend) ? hstart : hiter);
+}
+
+/// Get array of history items
+///
+/// @param[in]   history_type  Type of the history to get array for.
+/// @param[out]  new_hisidx    Location where last index in the new array should
+///                            be saved.
+/// @param[out]  new_hisnum    Location where last history number in the new
+///                            history should be saved.
+///
+/// @return Pointer to the array or NULL.
+histentry_T *hist_get_array(const uint8_t history_type, int **const new_hisidx,
+                            int **const new_hisnum)
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
+{
+  init_history();
+  *new_hisidx = &(hisidx[history_type]);
+  *new_hisnum = &(hisnum[history_type]);
+  return history[history_type];
+}

src/nvim/ex_getln.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_EX_GETLN_H
 #define NVIM_EX_GETLN_H
 
+#include "nvim/eval_defs.h"
 #include "nvim/ex_cmds.h"
 
 /* Values for nextwild() and ExpandOne().  See ExpandOne() for meaning. */
@@ -23,18 +24,28 @@
 #define WILD_ESCAPE             128
 #define WILD_ICASE              256
 
-/*
+/// Present history tables
- * There are four history tables:
+typedef enum {
- */
+  HIST_CMD,     ///< Colon commands.
-#define HIST_CMD        0       /* colon commands */
+  HIST_SEARCH,  ///< Search commands.
-#define HIST_SEARCH     1       /* search commands */
+  HIST_EXPR,    ///< Expressions (e.g. from entering = register).
-#define HIST_EXPR       2       /* expressions (from entering = register) */
+  HIST_INPUT,   ///< input() lines.
-#define HIST_INPUT      3       /* input() lines */
+  HIST_DEBUG,   ///< Debug commands.
-#define HIST_DEBUG      4       /* debug commands */
+} HistoryType;
-#define HIST_COUNT      5       /* number of history tables */
+
+/// Number of history tables
+#define HIST_COUNT      (HIST_DEBUG + 1)
 
 typedef char_u *(*CompleteListItemGetter)(expand_T *, int);
 
+/// History entry definition
+typedef struct hist_entry {
+  int hisnum;           ///< Entry identifier number.
+  char_u *hisstr;       ///< Actual entry, separator char after the NUL.
+  Timestamp timestamp;  ///< Time when entry was added.
+  list_T *additional_elements;  ///< Additional entries from ShaDa file.
+} histentry_T;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ex_getln.h.generated.h"
 #endif

src/nvim/fileio.c

@@ -57,6 +57,7 @@
 #include "nvim/types.h"
 #include "nvim/undo.h"
 #include "nvim/window.h"
+#include "nvim/shada.h"
 #include "nvim/os/os.h"
 #include "nvim/os/time.h"
 #include "nvim/os/input.h"
@@ -2166,16 +2167,17 @@ readfile_charconvert (
 
 
 /*
- * Read marks for the current buffer from the viminfo file, when we support
+ * Read marks for the current buffer from the ShaDa file, when we support
  * buffer marks and the buffer has a name.
  */
 static void check_marks_read(void)
 {
-  if (!curbuf->b_marks_read && get_viminfo_parameter('\'') > 0
+  if (!curbuf->b_marks_read && get_shada_parameter('\'') > 0
-      && curbuf->b_ffname != NULL)
+      && curbuf->b_ffname != NULL) {
-    read_viminfo(NULL, VIF_WANT_MARKS);
+    shada_read_marks();
+  }
 
-  /* Always set b_marks_read; needed when 'viminfo' is changed to include
+  /* Always set b_marks_read; needed when 'shada' is changed to include
    * the ' parameter after opening a buffer. */
   curbuf->b_marks_read = true;
 }

src/nvim/globals.h

@@ -891,8 +891,8 @@ EXTERN int must_redraw INIT(= 0);           /* type of redraw necessary */
 EXTERN int skip_redraw INIT(= FALSE);       /* skip redraw once */
 EXTERN int do_redraw INIT(= FALSE);         /* extra redraw once */
 
-EXTERN int need_highlight_changed INIT(= TRUE);
+EXTERN int need_highlight_changed INIT(= true);
-EXTERN char_u   *use_viminfo INIT(= NULL);  /* name of viminfo file to use */
+EXTERN char *used_shada_file INIT(= NULL);  // name of the ShaDa file to use
 
 #define NSCRIPT 15
 EXTERN FILE     *scriptin[NSCRIPT];         /* streams to read script from */

src/nvim/lib/khash.h

@@ -114,8 +114,8 @@ int main() {
 */
 
 
-#ifndef __AC_KHASH_H
+#ifndef NVIM_LIB_KHASH_H
-#define __AC_KHASH_H
+#define NVIM_LIB_KHASH_H
 
 /*!
   @header
@@ -196,6 +196,7 @@ static const double __ac_HASH_UPPER = 0.77;
 
 #define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \
   extern kh_##name##_t *kh_init_##name(void); \
+  extern void kh_dealloc_##name(kh_##name##_t *h); \
   extern void kh_destroy_##name(kh_##name##_t *h); \
   extern void kh_clear_##name(kh_##name##_t *h); \
   extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
@@ -203,17 +204,27 @@ static const double __ac_HASH_UPPER = 0.77;
   extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
   extern void kh_del_##name(kh_##name##_t *h, khint_t x);
 
-#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, \
+                     __hash_equal) \
+  SCOPE kh_##name##_t *kh_init_##name(void) \
+    REAL_FATTR_UNUSED; \
   SCOPE kh_##name##_t *kh_init_##name(void) { \
     return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \
   } \
+  SCOPE void kh_dealloc_##name(kh_##name##_t *h) \
+    REAL_FATTR_UNUSED; \
+  SCOPE void kh_dealloc_##name(kh_##name##_t *h) \
+  { \
+    kfree(h->keys); \
+    kfree(h->flags); \
+    kfree(h->vals); \
+  } \
   SCOPE void kh_destroy_##name(kh_##name##_t *h) \
     REAL_FATTR_UNUSED; \
   SCOPE void kh_destroy_##name(kh_##name##_t *h) \
   { \
     if (h) { \
-			kfree((void *)h->keys); kfree(h->flags);					\
+      kh_dealloc_##name(h); \
-			kfree((void *)h->vals);										\
       kfree(h); \
     } \
   } \
@@ -235,31 +246,45 @@ static const double __ac_HASH_UPPER = 0.77;
       mask = h->n_buckets - 1; \
       k = __hash_func(key); i = k & mask; \
       last = i; \
-			while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+      while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || \
+                                            !__hash_equal(h->keys[i], key))) { \
         i = (i + (++step)) & mask; \
-				if (i == last) return h->n_buckets;						\
+        if (i == last) { \
+          return h->n_buckets; \
+        } \
       } \
       return __ac_iseither(h->flags, i) ? h->n_buckets : i; \
-		} else return 0;												\
+    } else { \
+      return 0; \
+    } \
   } \
   SCOPE void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
     REAL_FATTR_UNUSED; \
   SCOPE void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
-	{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
+  { /* This function uses 0.25*n_buckets bytes of working space instead of */ \
+    /* [sizeof(key_t+val_t)+.25]*n_buckets. */ \
     khint32_t *new_flags = 0; \
     khint_t j = 1; \
     { \
       kroundup32(new_n_buckets); \
-			if (new_n_buckets < 4) new_n_buckets = 4;					\
+      if (new_n_buckets < 4) { \
-			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	/* requested size is too small */ \
+        new_n_buckets = 4; \
-			else { /* hash table size to be changed (shrink or expand); rehash */ \
+      } \
-				new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t));	\
+      /* requested size is too small */ \
-				memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+      if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) { \
+        j = 0; \
+      } else { /* hash table size to be changed (shrink or expand); rehash */ \
+        new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) \
+                                        * sizeof(khint32_t)); \
+        memset(new_flags, 0xaa, \
+               __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
         if (h->n_buckets < new_n_buckets) { /* expand */ \
-					khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+          khkey_t *new_keys = (khkey_t*)krealloc( \
+              (void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
           h->keys = new_keys; \
           if (kh_is_map) { \
-						khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+            khval_t *new_vals = (khval_t*)krealloc( \
+                (void *)h->vals, new_n_buckets * sizeof(khval_t)); \
             h->vals = new_vals; \
           } \
         } /* otherwise shrink */ \
@@ -272,29 +297,50 @@ static const double __ac_HASH_UPPER = 0.77;
           khval_t val; \
           khint_t new_mask; \
           new_mask = new_n_buckets - 1; \
-					if (kh_is_map) val = h->vals[j];					\
+          if (kh_is_map) { \
+            val = h->vals[j]; \
+          } \
           __ac_set_isdel_true(h->flags, j); \
-					while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+          /* kick-out process; sort of like in Cuckoo hashing */ \
+          while (1) { \
             khint_t k, i, step = 0; \
             k = __hash_func(key); \
             i = k & new_mask; \
-						while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \
+            while (!__ac_isempty(new_flags, i)) { \
+              i = (i + (++step)) & new_mask; \
+            } \
             __ac_set_isempty_false(new_flags, i); \
-						if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
+            /* kick out the existing element */ \
-							{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+            if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { \
-							if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+              { \
-							__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
+                khkey_t tmp = h->keys[i]; \
+                h->keys[i] = key; \
+                key = tmp; \
+              } \
+              if (kh_is_map) { \
+                khval_t tmp = h->vals[i]; \
+                h->vals[i] = val; \
+                val = tmp; \
+              } \
+              /* mark it as deleted in the old hash table */ \
+              __ac_set_isdel_true(h->flags, i); \
             } else { /* write the element and jump out of the loop */ \
               h->keys[i] = key; \
-							if (kh_is_map) h->vals[i] = val;			\
+              if (kh_is_map) { \
+                h->vals[i] = val; \
+              } \
               break; \
             } \
           } \
         } \
       } \
       if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
-				h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+        h->keys = (khkey_t*)krealloc((void *)h->keys, \
-				if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+                                     new_n_buckets * sizeof(khkey_t)); \
+        if (kh_is_map) { \
+          h->vals = (khval_t*)krealloc((void *)h->vals, \
+                                       new_n_buckets * sizeof(khval_t)); \
+        } \
       } \
       kfree(h->flags); /* free the working space */ \
       h->flags = new_flags; \
@@ -314,35 +360,52 @@ static const double __ac_HASH_UPPER = 0.77;
       } else { \
         kh_resize_##name(h, h->n_buckets + 1); /* expand the hash table */ \
       } \
-		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+    } /* TODO: implement automatically shrinking; */ \
+      /* resize() already support shrinking */ \
     { \
       khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \
-			x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
+      x = site = h->n_buckets; \
-			if (__ac_isempty(h->flags, i)) x = i; /* for speed up */	\
+      k = __hash_func(key); \
-			else {														\
+      i = k & mask; \
+      if (__ac_isempty(h->flags, i)) { \
+        x = i; /* for speed up */ \
+      } else { \
         last = i; \
-				while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+        while (!__ac_isempty(h->flags, i) \
-					if (__ac_isdel(h->flags, i)) site = i;				\
+               && (__ac_isdel(h->flags, i) \
+                   || !__hash_equal(h->keys[i], key))) { \
+          if (__ac_isdel(h->flags, i)) { \
+            site = i; \
+          } \
           i = (i + (++step)) & mask; \
-					if (i == last) { x = site; break; }					\
+          if (i == last) { \
+            x = site; \
+            break; \
+          } \
         } \
         if (x == h->n_buckets) { \
-					if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+          if (__ac_isempty(h->flags, i) && site != h->n_buckets) { \
-					else x = i;											\
+            x = site; \
+          } else { \
+            x = i; \
+          } \
         } \
       } \
     } \
     if (__ac_isempty(h->flags, x)) { /* not present at all */ \
       h->keys[x] = key; \
       __ac_set_isboth_false(h->flags, x); \
-			++h->size; ++h->n_occupied;									\
+      h->size++; \
+      h->n_occupied++; \
       *ret = 1; \
     } else if (__ac_isdel(h->flags, x)) { /* deleted */ \
       h->keys[x] = key; \
       __ac_set_isboth_false(h->flags, x); \
-			++h->size;													\
+      h->size++; \
       *ret = 2; \
-		} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+    } else { \
+      *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+    } \
     return x; \
   } \
   SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \
@@ -446,6 +509,13 @@ static kh_inline khint_t __ac_Wang_hash(khint_t key)
  */
 #define kh_destroy(name, h) kh_destroy_##name(h)
 
+/*! @function
+  @abstract     Free memory referenced directly inside a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_dealloc(name, h) kh_dealloc_##name(h)
+
 /*! @function
   @abstract     Reset a hash table without deallocating memory.
   @param  name  Name of the hash table [symbol]
@@ -577,6 +647,24 @@ static kh_inline khint_t __ac_Wang_hash(khint_t key)
 		code;												\
 	} }
 
+/*! @function
+  @abstract     Iterate over the keys in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  kvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach_key(h, kvar, code) \
+  { \
+    khint_t __i; \
+    for (__i = kh_begin(h); __i != kh_end(h); __i++) { \
+      if (!kh_exist(h, __i)) { \
+        continue; \
+      } \
+      (kvar) = kh_key(h, __i); \
+      code; \
+    } \
+  }
+
 /* More conenient interfaces */
 
 /*! @function
@@ -624,4 +712,19 @@ typedef const char *kh_cstr_t;
  */
 #define KHASH_MAP_INIT_STR(name, khval_t) \
   KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
-#endif /* __AC_KHASH_H */
+
+/*! @function
+  @abstract     Return a literal for an empty hash table.
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_EMPTY_TABLE(name) \
+  ((kh_##name##_t) { \
+    .n_buckets = 0, \
+    .size = 0, \
+    .n_occupied = 0, \
+    .upper_bound = 0, \
+    .flags = NULL, \
+    .keys = NULL, \
+    .vals = NULL, \
+  })
+#endif  // NVIM_LIB_KHASH_H

src/nvim/lib/ringbuf.h

@@ -0,0 +1,281 @@
+/// Macros-based ring buffer implementation.
+///
+/// Supported functions:
+///
+/// - new: allocates new ring buffer.
+/// - dealloc: free ring buffer itself.
+/// - free: free ring buffer and all its elements.
+/// - push: adds element to the end of the buffer.
+/// - length: get buffer length.
+/// - size: size of the ring buffer.
+/// - idx: get element at given index.
+/// - idx_p: get pointer to the element at given index.
+/// - insert: insert element at given position.
+/// - remove: remove element from given position.
+#ifndef NVIM_LIB_RINGBUF_H
+#define NVIM_LIB_RINGBUF_H
+
+#include <string.h>
+#include <assert.h>
+#include <stdint.h>
+
+#include "nvim/memory.h"
+#include "nvim/func_attr.h"
+
+#define _RINGBUF_LENGTH(rb) \
+    ((rb)->first == NULL ? 0 \
+     : ((rb)->next == (rb)->first) ? (size_t) ((rb)->buf_end - (rb)->buf) + 1 \
+     : ((rb)->next > (rb)->first) ? (size_t) ((rb)->next - (rb)->first) \
+     : (size_t) ((rb)->next - (rb)->buf + (rb)->buf_end - (rb)->first + 1))
+
+#define _RINGBUF_NEXT(rb, var) \
+    ((var) == (rb)->buf_end ? (rb)->buf : (var) + 1)
+#define _RINGBUF_PREV(rb, var) \
+    ((var) == (rb)->buf ? (rb)->buf_end : (var) - 1)
+
+/// Iterate over all ringbuf values
+///
+/// @param  rb       Ring buffer to iterate over.
+/// @param  RBType   Type of the ring buffer element.
+/// @param  varname  Variable name.
+#define RINGBUF_FORALL(rb, RBType, varname) \
+    size_t varname##_length_fa_ = _RINGBUF_LENGTH(rb); \
+    for (RBType *varname = ((rb)->first == NULL ? (rb)->next : (rb)->first); \
+         varname##_length_fa_; \
+         (varname = _RINGBUF_NEXT(rb, varname)), \
+         varname##_length_fa_--)
+
+/// Iterate over all ringbuf values, from end to the beginning
+///
+/// Unlike previous RINGBUF_FORALL uses already defined variable, in place of
+/// defining variable in the cycle body.
+///
+/// @param  rb       Ring buffer to iterate over.
+/// @param  RBType   Type of the ring buffer element.
+/// @param  varname  Variable name.
+#define RINGBUF_ITER_BACK(rb, RBType, varname) \
+    size_t varname##_length_ib_ = _RINGBUF_LENGTH(rb); \
+    for (varname = ((rb)->next == (rb)->buf ? (rb)->buf_end : (rb)->next - 1); \
+         varname##_length_ib_; \
+         (varname = _RINGBUF_PREV(rb, varname)), \
+         varname##_length_ib_--)
+
+/// Define a ring buffer structure
+///
+/// @param TypeName    Ring buffer type name. Actual type name will be
+///                    `{TypeName}RingBuffer`.
+/// @param RBType      Type of the single ring buffer element.
+#define RINGBUF_TYPEDEF(TypeName, RBType)                                      \
+typedef struct {                                                               \
+  RBType *buf;                                                                 \
+  RBType *next;                                                                \
+  RBType *first;                                                               \
+  RBType *buf_end;                                                             \
+} TypeName##RingBuffer;
+
+/// Initialize a new ring buffer
+///
+/// @param TypeName    Ring buffer type name. Actual type name will be
+///                    `{TypeName}RingBuffer`.
+/// @param funcprefix  Prefix for all ring buffer functions. Function name will
+///                    look like `{funcprefix}_rb_{function_name}`.
+/// @param RBType      Type of the single ring buffer element.
+/// @param rbfree      Function used to free ring buffer element. May be
+///                    a macros like `#define RBFREE(item)` (to skip freeing).
+///
+///                    Intended function signature: `void *rbfree(RBType *)`;
+#define RINGBUF_INIT(TypeName, funcprefix, RBType, rbfree)                     \
+                                                                               \
+                                                                               \
+static inline TypeName##RingBuffer funcprefix##_rb_new(const size_t size)      \
+  REAL_FATTR_WARN_UNUSED_RESULT;                                               \
+static inline TypeName##RingBuffer funcprefix##_rb_new(const size_t size)      \
+{                                                                              \
+  assert(size != 0);                                                           \
+  RBType *buf = xmalloc(size * sizeof(RBType));                                \
+  return (TypeName##RingBuffer) {                                              \
+    .buf = buf,                                                                \
+    .next = buf,                                                               \
+    .first = NULL,                                                             \
+    .buf_end = buf + size - 1,                                                 \
+  };                                                                           \
+}                                                                              \
+                                                                               \
+static inline void funcprefix##_rb_free(TypeName##RingBuffer *const rb)        \
+  REAL_FATTR_UNUSED;                                                           \
+static inline void funcprefix##_rb_free(TypeName##RingBuffer *const rb)        \
+{                                                                              \
+  if (rb == NULL) {                                                            \
+    return;                                                                    \
+  }                                                                            \
+  RINGBUF_FORALL(rb, RBType, rbitem) {                                         \
+    rbfree(rbitem);                                                            \
+  }                                                                            \
+  xfree(rb->buf);                                                              \
+}                                                                              \
+                                                                               \
+static inline void funcprefix##_rb_dealloc(TypeName##RingBuffer *const rb)     \
+  REAL_FATTR_UNUSED;                                                           \
+static inline void funcprefix##_rb_dealloc(TypeName##RingBuffer *const rb)     \
+{                                                                              \
+  xfree(rb->buf);                                                              \
+}                                                                              \
+                                                                               \
+static inline void funcprefix##_rb_push(TypeName##RingBuffer *const rb,        \
+                                        RBType item)                           \
+  REAL_FATTR_NONNULL_ARG(1);                                                   \
+static inline void funcprefix##_rb_push(TypeName##RingBuffer *const rb,        \
+                                        RBType item)                           \
+{                                                                              \
+  if (rb->next == rb->first) {                                                 \
+    rbfree(rb->first);                                                         \
+    rb->first = _RINGBUF_NEXT(rb, rb->first);                                  \
+  } else if (rb->first == NULL) {                                              \
+    rb->first = rb->next;                                                      \
+  }                                                                            \
+  *rb->next = item;                                                            \
+  rb->next = _RINGBUF_NEXT(rb, rb->next);                                      \
+}                                                                              \
+                                                                               \
+static inline ptrdiff_t funcprefix##_rb_find_idx(                              \
+    const TypeName##RingBuffer *const rb, const RBType *const item_p)          \
+  REAL_FATTR_NONNULL_ALL REAL_FATTR_PURE REAL_FATTR_UNUSED;                    \
+static inline ptrdiff_t funcprefix##_rb_find_idx(                              \
+    const TypeName##RingBuffer *const rb, const RBType *const item_p)          \
+{                                                                              \
+  assert(rb->buf <= item_p);                                                   \
+  assert(rb->buf_end >= item_p);                                               \
+  if (rb->first == NULL) {                                                     \
+    return -1;                                                                 \
+  } else if (item_p >= rb->first) {                                            \
+    return item_p - rb->first;                                                 \
+  } else {                                                                     \
+    return item_p - rb->buf + rb->buf_end - rb->first + 1;                     \
+  }                                                                            \
+}                                                                              \
+                                                                               \
+static inline size_t funcprefix##_rb_size(                                     \
+    const TypeName##RingBuffer *const rb)                                      \
+  REAL_FATTR_NONNULL_ALL REAL_FATTR_PURE;                                      \
+static inline size_t funcprefix##_rb_size(                                     \
+    const TypeName##RingBuffer *const rb)                                      \
+{                                                                              \
+  return (size_t) (rb->buf_end - rb->buf) + 1;                                 \
+}                                                                              \
+                                                                               \
+static inline size_t funcprefix##_rb_length(                                   \
+    const TypeName##RingBuffer *const rb)                                      \
+  REAL_FATTR_NONNULL_ALL REAL_FATTR_PURE;                                      \
+static inline size_t funcprefix##_rb_length(                                   \
+    const TypeName##RingBuffer *const rb)                                      \
+{                                                                              \
+  return _RINGBUF_LENGTH(rb);                                                  \
+}                                                                              \
+                                                                               \
+static inline RBType *funcprefix##_rb_idx_p(                                   \
+    const TypeName##RingBuffer *const rb, const size_t idx)                    \
+  REAL_FATTR_NONNULL_ALL REAL_FATTR_PURE;                                      \
+static inline RBType *funcprefix##_rb_idx_p(                                   \
+    const TypeName##RingBuffer *const rb, const size_t idx)                    \
+{                                                                              \
+  assert(idx <= funcprefix##_rb_size(rb));                                     \
+  assert(idx <= funcprefix##_rb_length(rb));                                   \
+  if (rb->first + idx > rb->buf_end) {                                         \
+    return rb->buf + ((rb->first + idx) - (rb->buf_end + 1));                  \
+  } else {                                                                     \
+    return rb->first + idx;                                                    \
+  }                                                                            \
+}                                                                              \
+                                                                               \
+static inline RBType funcprefix##_rb_idx(const TypeName##RingBuffer *const rb, \
+                                         const size_t idx)                     \
+  REAL_FATTR_NONNULL_ALL REAL_FATTR_PURE REAL_FATTR_UNUSED;                    \
+static inline RBType funcprefix##_rb_idx(const TypeName##RingBuffer *const rb, \
+                                         const size_t idx)                     \
+{                                                                              \
+  return *funcprefix##_rb_idx_p(rb, idx);                                      \
+}                                                                              \
+                                                                               \
+static inline void funcprefix##_rb_insert(TypeName##RingBuffer *const rb,      \
+                                          const size_t idx,                    \
+                                          RBType item)                         \
+  REAL_FATTR_NONNULL_ARG(1) REAL_FATTR_UNUSED;                                 \
+static inline void funcprefix##_rb_insert(TypeName##RingBuffer *const rb,      \
+                                          const size_t idx,                    \
+                                          RBType item)                         \
+{                                                                              \
+  assert(idx <= funcprefix##_rb_size(rb));                                     \
+  assert(idx <= funcprefix##_rb_length(rb));                                   \
+  const size_t length = funcprefix##_rb_length(rb);                            \
+  if (idx == length) {                                                         \
+    funcprefix##_rb_push(rb, item);                                            \
+    return;                                                                    \
+  }                                                                            \
+  RBType *const insertpos = funcprefix##_rb_idx_p(rb, idx);                    \
+  if (insertpos == rb->next) {                                                 \
+    funcprefix##_rb_push(rb, item);                                            \
+    return;                                                                    \
+  }                                                                            \
+  if (length == funcprefix##_rb_size(rb)) {                                    \
+    rbfree(rb->first);                                                         \
+  }                                                                            \
+  if (insertpos < rb->next) {                                                  \
+    memmove(insertpos + 1, insertpos,                                          \
+            (size_t) ((uintptr_t) rb->next - (uintptr_t) insertpos));          \
+  } else {                                                                     \
+    assert(insertpos > rb->first);                                             \
+    assert(rb->next <= rb->first);                                             \
+    memmove(rb->buf + 1, rb->buf,                                              \
+            (size_t) ((uintptr_t) rb->next - (uintptr_t) rb->buf));            \
+    *rb->buf = *rb->buf_end;                                                   \
+    memmove(insertpos + 1, insertpos,                                          \
+            (size_t) ((uintptr_t) (rb->buf_end + 1) - (uintptr_t) insertpos)); \
+  }                                                                            \
+  *insertpos = item;                                                           \
+  if (length == funcprefix##_rb_size(rb)) {                                    \
+    rb->first = _RINGBUF_NEXT(rb, rb->first);                                  \
+  }                                                                            \
+  rb->next = _RINGBUF_NEXT(rb, rb->next);                                      \
+}                                                                              \
+                                                                               \
+static inline void funcprefix##_rb_remove(TypeName##RingBuffer *const rb,      \
+                                          const size_t idx)                    \
+  REAL_FATTR_NONNULL_ARG(1) REAL_FATTR_UNUSED;                                 \
+static inline void funcprefix##_rb_remove(TypeName##RingBuffer *const rb,      \
+                                          const size_t idx)                    \
+{                                                                              \
+  assert(idx < funcprefix##_rb_size(rb));                                      \
+  assert(idx < funcprefix##_rb_length(rb));                                    \
+  RBType *const rmpos = funcprefix##_rb_idx_p(rb, idx);                        \
+  rbfree(rmpos);                                                               \
+  if (rmpos == rb->next - 1) {                                                 \
+    rb->next--;                                                                \
+    if (rb->first == rb->next) {                                               \
+      rb->first = NULL;                                                        \
+      rb->next = rb->buf;                                                      \
+    }                                                                          \
+  } else if (rmpos == rb->first) {                                             \
+    rb->first = _RINGBUF_NEXT(rb, rb->first);                                  \
+    if (rb->first == rb->next) {                                               \
+      rb->first = NULL;                                                        \
+      rb->next = rb->buf;                                                      \
+    }                                                                          \
+  } else if (rb->first < rb->next || rb->next == rb->buf) {                    \
+    assert(rmpos > rb->first);                                                 \
+    assert(rmpos <= _RINGBUF_PREV(rb, rb->next));                              \
+    memmove(rb->first + 1, rb->first,                                          \
+            (size_t) ((uintptr_t) rmpos - (uintptr_t) rb->first));             \
+    rb->first = _RINGBUF_NEXT(rb, rb->first);                                  \
+  } else if (rmpos < rb->next) {                                               \
+    memmove(rmpos, rmpos + 1,                                                  \
+            (size_t) ((uintptr_t) rb->next - (uintptr_t) rmpos));              \
+    rb->next = _RINGBUF_PREV(rb, rb->next);                                    \
+  } else {                                                                     \
+    assert(rb->first < rb->buf_end);                                           \
+    memmove(rb->first + 1, rb->first,                                          \
+            (size_t) ((uintptr_t) rmpos - (uintptr_t) rb->first));             \
+    rb->first = _RINGBUF_NEXT(rb, rb->first);                                  \
+  }                                                                            \
+}
+
+#endif  // NVIM_LIB_RINGBUF_H

src/nvim/main.c

@@ -49,6 +49,7 @@
 #include "nvim/ops.h"
 #include "nvim/option.h"
 #include "nvim/os_unix.h"
+#include "nvim/os/os_defs.h"
 #include "nvim/path.h"
 #include "nvim/profile.h"
 #include "nvim/quickfix.h"
@@ -58,6 +59,7 @@
 #include "nvim/ui.h"
 #include "nvim/version.h"
 #include "nvim/window.h"
+#include "nvim/shada.h"
 #include "nvim/os/input.h"
 #include "nvim/os/os.h"
 #include "nvim/os/time.h"
@@ -377,12 +379,12 @@ int main(int argc, char **argv)
   }
 
   /*
-   * Read in registers, history etc, but not marks, from the viminfo file.
+   * Read in registers, history etc, from the ShaDa file.
    * This is where v:oldfiles gets filled.
    */
-  if (*p_viminfo != NUL) {
+  if (*p_shada != NUL) {
-    read_viminfo(NULL, VIF_WANT_INFO | VIF_GET_OLDFILES);
+    shada_read_everything(NULL, false, true);
-    TIME_MSG("reading viminfo");
+    TIME_MSG("reading ShaDa");
   }
   /* It's better to make v:oldfiles an empty list than NULL. */
   if (get_vim_var_list(VV_OLDFILES) == NULL)
@@ -803,9 +805,10 @@ void getout(int exitval)
     apply_autocmds(EVENT_VIMLEAVEPRE, NULL, NULL, FALSE, curbuf);
   }
 
-  if (p_viminfo && *p_viminfo != NUL)
+  if (p_shada && *p_shada != NUL) {
-    /* Write out the registers, history, marks etc, to the viminfo file */
+    // Write out the registers, history, marks etc, to the ShaDa file
-    write_viminfo(NULL, FALSE);
+    shada_write_file(NULL, false);
+  }
 
   if (get_vim_var_nr(VV_DYING) <= 1)
     apply_autocmds(EVENT_VIMLEAVE, NULL, NULL, FALSE, curbuf);
@@ -1164,7 +1167,7 @@ static void command_line_scan(mparm_T *parmp)
           }
           /*FALLTHROUGH*/
         case 'S':                 /* "-S {file}" execute Vim script */
-        case 'i':                 /* "-i {viminfo}" use for viminfo */
+        case 'i':                 /* "-i {shada}" use for ShaDa file */
         case 'u':                 /* "-u {vimrc}" vim inits file */
         case 'U':                 /* "-U {gvimrc}" gvim inits file */
         case 'W':                 /* "-W {scriptout}" overwrite */
@@ -1235,8 +1238,8 @@ static void command_line_scan(mparm_T *parmp)
             parmp->use_ef = (char_u *)argv[0];
             break;
 
-          case 'i':               /* "-i {viminfo}" use for viminfo */
+          case 'i':               /* "-i {shada}" use for shada */
-            use_viminfo = (char_u *)argv[0];
+            used_shada_file = argv[0];
             break;
 
           case 's':               /* "-s {scriptin}" read from script file */
@@ -2039,7 +2042,7 @@ static void usage(void)
   mch_msg(_("  -r, -L                List swap files and exit\n"));
   mch_msg(_("  -r <file>             Recover crashed session\n"));
   mch_msg(_("  -u <nvimrc>           Use <nvimrc> instead of the default\n"));
-  mch_msg(_("  -i <nviminfo>         Use <nviminfo> instead of the default\n"));
+  mch_msg(_("  -i <shada>            Use <shada> instead of the default " SHADA_FILE "\n"));  // NOLINT(whitespace/line_length)
   mch_msg(_("  --noplugin            Don't load plugin scripts\n"));
   mch_msg(_("  -o[N]                 Open N windows (default: one for each file)\n"));
   mch_msg(_("  -O[N]                 Like -o but split vertically\n"));

src/nvim/mark.h

@@ -1,9 +1,78 @@
 #ifndef NVIM_MARK_H
 #define NVIM_MARK_H
 
+#include "nvim/macros.h"
+#include "nvim/ascii.h"
 #include "nvim/buffer_defs.h"
 #include "nvim/mark_defs.h"
+#include "nvim/memory.h"
 #include "nvim/pos.h"
+#include "nvim/os/time.h"
+
+/// Set fmark using given value
+#define SET_FMARK(fmarkp_, mark_, fnum_) \
+    do { \
+      fmark_T *const fmarkp__ = fmarkp_; \
+      fmarkp__->mark = mark_; \
+      fmarkp__->fnum = fnum_; \
+      fmarkp__->timestamp = os_time(); \
+      fmarkp__->additional_data = NULL; \
+    } while (0)
+
+/// Free and set fmark using given value
+#define RESET_FMARK(fmarkp_, mark_, fnum_) \
+    do { \
+      fmark_T *const fmarkp___ = fmarkp_; \
+      free_fmark(*fmarkp___); \
+      SET_FMARK(fmarkp___, mark_, fnum_); \
+    } while (0)
+
+/// Clear given fmark
+#define CLEAR_FMARK(fmarkp_) \
+    RESET_FMARK(fmarkp_, ((pos_T) {0, 0, 0}), 0)
+
+/// Set given extended mark (regular mark + file name)
+#define SET_XFMARK(xfmarkp_, mark_, fnum_, fname_) \
+    do { \
+      xfmark_T *const xfmarkp__ = xfmarkp_; \
+      xfmarkp__->fname = fname_; \
+      SET_FMARK(&(xfmarkp__->fmark), mark_, fnum_); \
+    } while (0)
+
+/// Free and set given extended mark (regular mark + file name)
+#define RESET_XFMARK(xfmarkp_, mark_, fnum_, fname_) \
+    do { \
+      xfmark_T *const xfmarkp__ = xfmarkp_; \
+      free_xfmark(*xfmarkp__); \
+      xfmarkp__->fname = fname_; \
+      SET_FMARK(&(xfmarkp__->fmark), mark_, fnum_); \
+    } while (0)
+
+/// Convert mark name to the offset
+static inline int mark_global_index(const char name)
+  FUNC_ATTR_CONST
+{
+  return (ASCII_ISUPPER(name)
+          ? (name - 'A')
+          : (ascii_isdigit(name)
+             ? (NMARKS + (name - '0'))
+             : -1));
+}
+
+/// Convert local mark name to the offset
+static inline int mark_local_index(const char name)
+  FUNC_ATTR_CONST
+{
+  return (ASCII_ISLOWER(name)
+          ? (name - 'a')
+          : (name == '"'
+             ? NMARKS
+             : (name == '^'
+                ? NMARKS + 1
+                : (name == '.'
+                   ? NMARKS + 2
+                   : -1))));
+}
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "mark.h.generated.h"

src/nvim/mark_defs.h

@@ -2,25 +2,47 @@
 #define NVIM_MARK_DEFS_H
 
 #include "nvim/pos.h"
+#include "nvim/os/time.h"
+#include "nvim/eval_defs.h"
 
 /*
  * marks: positions in a file
  * (a normal mark is a lnum/col pair, the same as a file position)
  */
 
-#define NMARKS          ('z' - 'a' + 1) /* max. # of named marks */
+/// Number of possible numbered global marks
-#define JUMPLISTSIZE    100             /* max. # of marks in jump list */
+#define EXTRA_MARKS     ('9' - '0' + 1)
-#define TAGSTACKSIZE    20              /* max. # of tags in tag stack */
 
+/// Maximum possible number of letter marks
+#define NMARKS          ('z' - 'a' + 1)
+
+/// Total possible number of global marks
+#define NGLOBALMARKS    (NMARKS + EXTRA_MARKS)
+
+/// Total possible number of local marks
+///
+/// That are uppercase marks plus '"', '^' and '.'. There are other local marks,
+/// but they are not saved in ShaDa files.
+#define NLOCALMARKS     (NMARKS + 3)
+
+/// Maximum number of marks in jump list
+#define JUMPLISTSIZE    100
+
+/// Maximum number of tags in tag stack
+#define TAGSTACKSIZE    20
+
+/// Structure defining single local mark
 typedef struct filemark {
-  pos_T mark;                   /* cursor position */
+  pos_T mark;           ///< Cursor position.
-  int fnum;                     /* file number */
+  int fnum;             ///< File number.
+  Timestamp timestamp;  ///< Time when this mark was last set.
+  dict_T *additional_data;  ///< Additional data from ShaDa file.
 } fmark_T;
 
-/* Xtended file mark: also has a file name */
+/// Structure defining extended mark (mark with file name attached)
 typedef struct xfilemark {
-  fmark_T fmark;
+  fmark_T fmark;       ///< Actual mark.
-  char_u      *fname;           /* file name, used when fnum == 0 */
+  char_u      *fname;  ///< File name, used when fnum == 0.
 } xfmark_T;
 
 #endif // NVIM_MARK_DEFS_H

src/nvim/mbyte.c

@@ -66,7 +66,7 @@
  * (4) The encoding of the file is specified with 'fileencoding'.  Conversion
  *     is to be done when it's different from 'encoding'.
  *
- * The viminfo file is a special case: Only text is converted, not file names.
+ * The ShaDa file is a special case: Only text is converted, not file names.
  * Vim scripts may contain an ":encoding" command.  This has an effect for
  * some commands, like ":menutrans"
  */

src/nvim/memory.c

@@ -30,7 +30,7 @@
 
 /// Try to free memory. Used when trying to recover from out of memory errors.
 /// @see {xmalloc}
-static void try_to_free_memory(void)
+void try_to_free_memory(void)
 {
   static bool trying_to_free = false;
   // avoid recursive calls

src/nvim/misc1.c

@@ -2042,8 +2042,7 @@ static void changed_common(linenr_T lnum, colnr_T col, linenr_T lnume, long xtra
 
   /* set the '. mark */
   if (!cmdmod.keepjumps) {
-    curbuf->b_last_change.lnum = lnum;
+    RESET_FMARK(&curbuf->b_last_change, ((pos_T) {lnum, col, 0}), 0);
-    curbuf->b_last_change.col = col;
 
     /* Create a new entry if a new undo-able change was started or we
      * don't have an entry yet. */
@@ -2054,7 +2053,7 @@ static void changed_common(linenr_T lnum, colnr_T col, linenr_T lnume, long xtra
         /* Don't create a new entry when the line number is the same
          * as the last one and the column is not too far away.  Avoids
          * creating many entries for typing "xxxxx". */
-        p = &curbuf->b_changelist[curbuf->b_changelistlen - 1];
+        p = &curbuf->b_changelist[curbuf->b_changelistlen - 1].mark;
         if (p->lnum != lnum)
           add = TRUE;
         else {
@@ -2074,7 +2073,7 @@ static void changed_common(linenr_T lnum, colnr_T col, linenr_T lnume, long xtra
           /* changelist is full: remove oldest entry */
           curbuf->b_changelistlen = JUMPLISTSIZE - 1;
           memmove(curbuf->b_changelist, curbuf->b_changelist + 1,
-              sizeof(pos_T) * (JUMPLISTSIZE - 1));
+              sizeof(curbuf->b_changelist[0]) * (JUMPLISTSIZE - 1));
           FOR_ALL_TAB_WINDOWS(tp, wp) {
             /* Correct position in changelist for other windows on
              * this buffer. */

src/nvim/normal.c

@@ -6327,8 +6327,8 @@ static void nv_g_cmd(cmdarg_T *cap)
    * "gi": start Insert at the last position.
    */
   case 'i':
-    if (curbuf->b_last_insert.lnum != 0) {
+    if (curbuf->b_last_insert.mark.lnum != 0) {
-      curwin->w_cursor = curbuf->b_last_insert;
+      curwin->w_cursor = curbuf->b_last_insert.mark;
       check_cursor_lnum();
       i = (int)STRLEN(get_cursor_line_ptr());
       if (curwin->w_cursor.col > (colnr_T)i) {

src/nvim/ops.c

@@ -50,22 +50,7 @@
 #include "nvim/undo.h"
 #include "nvim/window.h"
 #include "nvim/os/input.h"
-
+#include "nvim/os/time.h"
-/*
- * Registers:
- *      0 = register for latest (unnamed) yank
- *   1..9 = registers '1' to '9', for deletes
- * 10..35 = registers 'a' to 'z'
- *     36 = delete register '-'
- *     37 = selection register '*'
- *     38 = clipboard register '+'
- */
-#define DELETION_REGISTER 36
-#define NUM_SAVED_REGISTERS 37
-// The following registers should not be saved in viminfo:
-#define STAR_REGISTER 37
-#define PLUS_REGISTER 38
-#define NUM_REGISTERS 39
 
 static yankreg_T y_regs[NUM_REGISTERS];
 
@@ -778,19 +763,11 @@ yankreg_T *get_yank_register(int regname, int mode)
     return y_previous;
   }
 
-  int i = 0; // when not 0-9, a-z, A-Z or '-'/'+'/'*': use register 0
+  int i = op_reg_index(regname);
-  if (ascii_isdigit(regname))
+  // when not 0-9, a-z, A-Z or '-'/'+'/'*': use register 0
-    i = regname - '0';
+  if (i == -1) {
-  else if (ASCII_ISLOWER(regname))
+    i = 0;
-    i = CharOrdLow(regname) + 10;
+  }
-  else if (ASCII_ISUPPER(regname)) {
-    i = CharOrdUp(regname) + 10;
-  } else if (regname == '-')
-    i = DELETION_REGISTER;
-  else if (regname == '*')
-    i = STAR_REGISTER;
-  else if (regname == '+')
-    i = PLUS_REGISTER;
   reg = &y_regs[i];
 
   if (mode == YREG_YANK) {
@@ -890,6 +867,16 @@ int do_record(int c)
   return retval;
 }
 
+static void set_yreg_additional_data(yankreg_T *reg, dict_T *additional_data)
+  FUNC_ATTR_NONNULL_ARG(1)
+{
+  if (reg->additional_data == additional_data) {
+    return;
+  }
+  dict_unref(reg->additional_data);
+  reg->additional_data = additional_data;
+}
+
 /*
  * Stuff string "p" into yank register "regname" as a single line (append if
  * uppercase).	"p" must have been alloced.
@@ -919,11 +906,13 @@ static int stuff_yank(int regname, char_u *p)
     *pp = lp;
   } else {
     free_register(reg);
+    set_yreg_additional_data(reg, NULL);
     reg->y_array = (char_u **)xmalloc(sizeof(char_u *));
     reg->y_array[0] = p;
     reg->y_size = 1;
     reg->y_type = MCHAR;      /* used to be MLINE, why? */
   }
+  reg->timestamp = os_time();
   return OK;
 }
 
@@ -2266,10 +2255,7 @@ int op_change(oparg_T *oap)
  */
 void init_yank(void)
 {
-  int i;
+  memset(&(y_regs[0]), 0, sizeof(y_regs));
-
-  for (i = 0; i < NUM_REGISTERS; i++)
-    y_regs[i].y_array = NULL;
 }
 
 #if defined(EXITFREE)
@@ -2291,6 +2277,7 @@ void clear_registers(void)
 void free_register(yankreg_T *reg)
   FUNC_ATTR_NONNULL_ALL
 {
+  set_yreg_additional_data(reg, NULL);
   if (reg->y_array != NULL) {
     long i;
 
@@ -2369,6 +2356,8 @@ static void op_yank_reg(oparg_T *oap, bool message, yankreg_T *reg, bool append)
   reg->y_type = yanktype;     /* set the yank register type */
   reg->y_width = 0;
   reg->y_array = xcalloc(yanklines, sizeof(char_u *));
+  reg->additional_data = NULL;
+  reg->timestamp = os_time();
 
   y_idx = 0;
   lnum = oap->start.lnum;
@@ -4433,171 +4422,6 @@ int do_addsub(int command, linenr_T Prenum1)
   return OK;
 }
 
-int read_viminfo_register(vir_T *virp, int force)
-{
-  int eof;
-  int do_it = TRUE;
-  int size;
-  int limit;
-  int set_prev = FALSE;
-  char_u      *str;
-  char_u      **array = NULL;
-
-  /* We only get here (hopefully) if line[0] == '"' */
-  str = virp->vir_line + 1;
-
-  /* If the line starts with "" this is the y_previous register. */
-  if (*str == '"') {
-    set_prev = TRUE;
-    str++;
-  }
-
-  if (!ASCII_ISALNUM(*str) && *str != '-') {
-    if (viminfo_error("E577: ", _("Illegal register name"), virp->vir_line))
-      return TRUE;              /* too many errors, pretend end-of-file */
-    do_it = FALSE;
-  }
-  yankreg_T *reg = get_yank_register(*str++, YREG_PUT);
-  if (!force && reg->y_array != NULL)
-    do_it = FALSE;
-
-  if (*str == '@') {
-    /* "x@: register x used for @@ */
-    if (force || execreg_lastc == NUL)
-      execreg_lastc = str[-1];
-  }
-
-  size = 0;
-  limit = 100;          /* Optimized for registers containing <= 100 lines */
-  if (do_it) {
-    if (set_prev) {
-      y_previous = reg;
-    }
-
-    free_register(reg);
-    array = xmalloc(limit * sizeof(char_u *));
-
-    str = skipwhite(skiptowhite(str));
-    if (STRNCMP(str, "CHAR", 4) == 0) {
-      reg->y_type = MCHAR;
-    } else if (STRNCMP(str, "BLOCK", 5) == 0) {
-      reg->y_type = MBLOCK;
-    } else {
-      reg->y_type = MLINE;
-    }
-    /* get the block width; if it's missing we get a zero, which is OK */
-    str = skipwhite(skiptowhite(str));
-    reg->y_width = getdigits_int(&str);
-  }
-
-  while (!(eof = viminfo_readline(virp))
-         && (virp->vir_line[0] == TAB || virp->vir_line[0] == '<')) {
-    if (do_it) {
-      if (size >= limit) {
-        limit *= 2;
-        array = xrealloc(array, limit * sizeof(char_u *));
-      }
-      array[size++] = viminfo_readstring(virp, 1, TRUE);
-    }
-  }
-
-  if (do_it) {
-    if (size == 0) {
-      xfree(array);
-    } else if (size < limit) {
-      reg->y_array = xrealloc(array, size * sizeof(char_u *));
-    } else {
-      reg->y_array = array;
-    }
-    reg->y_size = size;
-  }
-  return eof;
-}
-
-void write_viminfo_registers(FILE *fp)
-{
-  int i, j;
-  char_u  *type;
-  char_u c;
-  int num_lines;
-  int max_num_lines;
-  int max_kbyte;
-  long len;
-
-  fputs(_("\n# Registers:\n"), fp);
-
-  /* Get '<' value, use old '"' value if '<' is not found. */
-  max_num_lines = get_viminfo_parameter('<');
-  if (max_num_lines < 0)
-    max_num_lines = get_viminfo_parameter('"');
-  if (max_num_lines == 0)
-    return;
-  max_kbyte = get_viminfo_parameter('s');
-  if (max_kbyte == 0)
-    return;
-
-  // don't include clipboard registers '*'/'+'
-  for (i = 0; i < NUM_SAVED_REGISTERS; i++) {
-    if (y_regs[i].y_array == NULL)
-      continue;
-
-    /* Skip empty registers. */
-    num_lines = y_regs[i].y_size;
-    if (num_lines == 0
-        || (num_lines == 1 && y_regs[i].y_type == MCHAR
-            && *y_regs[i].y_array[0] == NUL))
-      continue;
-
-    if (max_kbyte > 0) {
-      /* Skip register if there is more text than the maximum size. */
-      len = 0;
-      for (j = 0; j < num_lines; j++)
-        len += (long)STRLEN(y_regs[i].y_array[j]) + 1L;
-      if (len > (long)max_kbyte * 1024L)
-        continue;
-    }
-
-    switch (y_regs[i].y_type) {
-    case MLINE:
-      type = (char_u *)"LINE";
-      break;
-    case MCHAR:
-      type = (char_u *)"CHAR";
-      break;
-    case MBLOCK:
-      type = (char_u *)"BLOCK";
-      break;
-    default:
-      sprintf((char *)IObuff, _("E574: Unknown register type %d"),
-          y_regs[i].y_type);
-      emsg(IObuff);
-      type = (char_u *)"LINE";
-      break;
-    }
-    if (y_previous == &y_regs[i])
-      fprintf(fp, "\"");
-    c = get_register_name(i);
-    fprintf(fp, "\"%c", c);
-    if (c == execreg_lastc)
-      fprintf(fp, "@");
-    fprintf(fp, "\t%s\t%d\n", type,
-        (int)y_regs[i].y_width
-        );
-
-    /* If max_num_lines < 0, then we save ALL the lines in the register */
-    if (max_num_lines > 0 && num_lines > max_num_lines)
-      num_lines = max_num_lines;
-    for (j = 0; j < num_lines; j++) {
-      putc('\t', fp);
-      viminfo_writestring(fp, y_regs[i].y_array[j]);
-    }
-  }
-}
-
-
-
-
-
 /*
  * Return the type of a register.
  * Used for getregtype()
@@ -4739,7 +4563,6 @@ void *get_reg_contents(int regname, int flags)
   return retval;
 }
 
-
 static yankreg_T *init_write_reg(int name, yankreg_T **old_y_previous, bool must_append)
 {
   if (!valid_yank_reg(name, true)) {  // check for valid reg name
@@ -4973,6 +4796,8 @@ static void str_to_reg(yankreg_T *y_ptr, int yank_type, const char_u *str,
   }
   y_ptr->y_type = type;
   y_ptr->y_size = lnum;
+  set_yreg_additional_data(y_ptr, NULL);
+  y_ptr->timestamp = os_time();
   if (type == MBLOCK) {
     y_ptr->y_width = (blocklen == -1 ? (colnr_T) maxlen - 1 : blocklen);
   } else {
@@ -5363,6 +5188,10 @@ static bool get_clipboard(int name, yankreg_T **target, bool quiet)
 
   reg->y_array = xcalloc(lines->lv_len, sizeof(uint8_t *));
   reg->y_size = lines->lv_len;
+  reg->additional_data = NULL;
+  reg->timestamp = 0;
+  // Timestamp is not saved for clipboard registers because clipboard registers
+  // are not saved in the ShaDa file.
 
   int i = 0;
   for (listitem_T *li = lines->lv_first; li != NULL; li = li->li_next) {
@@ -5411,6 +5240,8 @@ err:
   }
   reg->y_array = NULL;
   reg->y_size = 0;
+  reg->additional_data = NULL;
+  reg->timestamp = 0;
   if (errmsg) {
     EMSG("clipboard: provider returned invalid data");
   }
@@ -5478,3 +5309,91 @@ void end_global_changes(void)
     clipboard_needs_update = false;
   }
 }
+
+/// Check whether register is empty
+static inline bool reg_empty(const yankreg_T *const reg)
+  FUNC_ATTR_PURE
+{
+  return (reg->y_array == NULL
+          || reg->y_size == 0
+          || (reg->y_size == 1
+              && reg->y_type == MCHAR
+              && *(reg->y_array[0]) == NUL));
+}
+
+/// Iterate over registerrs
+///
+/// @param[in]   iter      Iterator. Pass NULL to start iteration.
+/// @param[out]  name      Register name.
+/// @param[out]  reg       Register contents.
+///
+/// @return Pointer that needs to be passed to next `op_register_iter` call or
+///         NULL if iteration is over.
+const void *op_register_iter(const void *const iter, char *const name,
+                             yankreg_T *const reg)
+  FUNC_ATTR_NONNULL_ARG(2, 3) FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  *name = NUL;
+  const yankreg_T *iter_reg = (iter == NULL
+                               ? &(y_regs[0])
+                               : (const yankreg_T *const) iter);
+  while (iter_reg - &(y_regs[0]) < NUM_SAVED_REGISTERS && reg_empty(iter_reg)) {
+    iter_reg++;
+  }
+  if (iter_reg - &(y_regs[0]) == NUM_SAVED_REGISTERS || reg_empty(iter_reg)) {
+    return NULL;
+  }
+  size_t iter_off = iter_reg - &(y_regs[0]);
+  *name = (char) get_register_name(iter_off);
+  *reg = *iter_reg;
+  while (++iter_reg - &(y_regs[0]) < NUM_SAVED_REGISTERS) {
+    if (!reg_empty(iter_reg)) {
+      return (void *) iter_reg;
+    }
+  }
+  return NULL;
+}
+
+/// Get a number of non-empty registers
+size_t op_register_amount(void)
+  FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  size_t ret = 0;
+  for (size_t i = 0; i < NUM_SAVED_REGISTERS; i++) {
+    if (!reg_empty(y_regs + i)) {
+      ret++;
+    }
+  }
+  return ret;
+}
+
+/// Set register to a given value
+///
+/// @param[in]  name  Register name.
+/// @param[in]  reg   Register value.
+///
+/// @return true on success, false on failure.
+bool op_register_set(const char name, const yankreg_T reg)
+{
+  int i = op_reg_index(name);
+  if (i == -1) {
+    return false;
+  }
+  free_register(&y_regs[i]);
+  y_regs[i] = reg;
+  return true;
+}
+
+/// Get register with the given name
+///
+/// @param[in]  name  Register name.
+///
+/// @return Pointer to the register contents or NULL.
+const yankreg_T *op_register_get(const char name)
+{
+  int i = op_reg_index(name);
+  if (i == -1) {
+    return NULL;
+  }
+  return &y_regs[i];
+}

src/nvim/ops.h

@@ -3,7 +3,11 @@
 
 #include <stdbool.h>
 
+#include "nvim/macros.h"
+#include "nvim/ascii.h"
 #include "nvim/types.h"
+#include "nvim/eval_defs.h"
+#include "nvim/os/time.h"
 
 typedef int (*Indenter)(void);
 
@@ -15,6 +19,22 @@ typedef int (*Indenter)(void);
 #define PUT_LINE_SPLIT   16     /* split line for linewise register */
 #define PUT_LINE_FORWARD 32     /* put linewise register below Visual sel. */
 
+/*
+ * Registers:
+ *      0 = register for latest (unnamed) yank
+ *   1..9 = registers '1' to '9', for deletes
+ * 10..35 = registers 'a' to 'z'
+ *     36 = delete register '-'
+ *     37 = selection register '*'
+ *     38 = clipboard register '+'
+ */
+#define DELETION_REGISTER 36
+#define NUM_SAVED_REGISTERS 37
+// The following registers should not be saved in ShaDa file:
+#define STAR_REGISTER 37
+#define PLUS_REGISTER 38
+#define NUM_REGISTERS 39
+
 /*
  * Operator IDs; The order must correspond to opchars[] in ops.c!
  */
@@ -47,14 +67,6 @@ typedef int (*Indenter)(void);
 #define OP_FORMAT2      26      /* "gw" format operator, keeps cursor pos */
 #define OP_FUNCTION     27      /* "g@" call 'operatorfunc' */
 
-/// Contents of a yank (read-write) register
-typedef struct yankreg {
-  char_u      **y_array;        ///< pointer to array of line pointers
-  linenr_T y_size;              ///< number of lines in y_array
-  char_u y_type;                ///< MLINE, MCHAR or MBLOCK
-  colnr_T y_width;              ///< only set if y_type == MBLOCK
-} yankreg_T;
-
 /// Flags for get_reg_contents().
 enum GRegFlags {
   kGRegNoExpr  = 1,  ///< Do not allow expression register.
@@ -62,6 +74,41 @@ enum GRegFlags {
   kGRegList    = 4   ///< Return list.
 };
 
+/// Definition of one register
+typedef struct yankreg {
+  char_u **y_array;  ///< Pointer to an array of line pointers.
+  linenr_T y_size;   ///< Number of lines in y_array.
+  char_u y_type;     ///< Register type: MLINE, MCHAR or MBLOCK.
+  colnr_T y_width;   ///< Register width (only valid for y_type == MBLOCK).
+  Timestamp timestamp;  ///< Time when register was last modified.
+  dict_T *additional_data;  ///< Additional data from ShaDa file.
+} yankreg_T;
+
+/// Convert register name into register index
+///
+/// @param[in]  regname  Register name.
+///
+/// @return Index in y_regs array or -1 if register name was not recognized.
+static inline int op_reg_index(const int regname)
+  FUNC_ATTR_CONST
+{
+  if (ascii_isdigit(regname)) {
+    return regname - '0';
+  } else if (ASCII_ISLOWER(regname)) {
+    return CharOrdLow(regname) + 10;
+  } else if (ASCII_ISUPPER(regname)) {
+    return CharOrdUp(regname) + 10;
+  } else if (regname == '-') {
+    return DELETION_REGISTER;
+  } else if (regname == '*') {
+    return STAR_REGISTER;
+  } else if (regname == '+') {
+    return PLUS_REGISTER;
+  } else {
+    return -1;
+  }
+}
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ops.h.generated.h"
 #endif

src/nvim/option.c

@@ -1738,16 +1738,16 @@ set_options_bin (
 
 /*
  * Find the parameter represented by the given character (eg ', :, ", or /),
- * and return its associated value in the 'viminfo' string.
+ * and return its associated value in the 'shada' string.
  * Only works for number parameters, not for 'r' or 'n'.
  * If the parameter is not specified in the string or there is no following
  * number, return -1.
  */
-int get_viminfo_parameter(int type)
+int get_shada_parameter(int type)
 {
   char_u  *p;
 
-  p = find_viminfo_parameter(type);
+  p = find_shada_parameter(type);
   if (p != NULL && ascii_isdigit(*p))
     return atoi((char *)p);
   return -1;
@@ -1755,14 +1755,14 @@ int get_viminfo_parameter(int type)
 
 /*
  * Find the parameter represented by the given character (eg ''', ':', '"', or
- * '/') in the 'viminfo' option and return a pointer to the string after it.
+ * '/') in the 'shada' option and return a pointer to the string after it.
  * Return NULL if the parameter is not specified in the string.
  */
-char_u *find_viminfo_parameter(int type)
+char_u *find_shada_parameter(int type)
 {
   char_u  *p;
 
-  for (p = p_viminfo; *p; ++p) {
+  for (p = p_shada; *p; ++p) {
     if (*p == type)
       return p + 1;
     if (*p == 'n')                  /* 'n' is always the last one */
@@ -1968,6 +1968,8 @@ static void redraw_titles(void) {
   redraw_tabline = TRUE;
 }
 
+static int shada_idx = -1;
+
 /*
  * Set a string option to a new value (without checking the effect).
  * The string is copied into allocated memory.
@@ -2001,6 +2003,8 @@ set_string_option_direct (
   if (options[idx].var == NULL)         /* can't set hidden option */
     return;
 
+  assert((void *) options[idx].var != (void *) &p_shada);
+
   s = vim_strsave(val);
   {
     varp = (char_u **)get_varp_scope(&(options[idx]),
@@ -2441,10 +2445,16 @@ did_set_string_option (
     verbose_stop();
     if (*p_vfile != NUL && verbose_open() == FAIL)
       errmsg = e_invarg;
-  }
+  /* 'shada' */
-  /* 'viminfo' */
+  } else if (varp == &p_shada) {
-  else if (varp == &p_viminfo) {
+    // TODO(ZyX-I): Remove this code in the future, alongside with &viminfo
-    for (s = p_viminfo; *s; ) {
+    //              option.
+    opt_idx = ((options[opt_idx].fullname[0] == 'v')
+               ? (shada_idx == -1
+                  ? ((shada_idx = findoption((char_u *) "shada")))
+                  : shada_idx)
+               : opt_idx);
+    for (s = p_shada; *s; ) {
       /* Check it's a valid character */
       if (vim_strchr((char_u *)"!\"%'/:<@cfhnrs", *s) == NULL) {
         errmsg = illegal_char(errbuf, *s);
@@ -2486,7 +2496,7 @@ did_set_string_option (
         break;
       }
     }
-    if (*p_viminfo && errmsg == NULL && get_viminfo_parameter('\'') < 0)
+    if (*p_shada && errmsg == NULL && get_shada_parameter('\'') < 0)
       errmsg = (char_u *)N_("E528: Must specify a ' value");
   }
   /* 'showbreak' */

src/nvim/option_defs.h

@@ -558,7 +558,7 @@ EXTERN long p_ur;               /* 'undoreload' */
 EXTERN long p_uc;               /* 'updatecount' */
 EXTERN long p_ut;               /* 'updatetime' */
 EXTERN char_u   *p_fcs;         /* 'fillchar' */
-EXTERN char_u   *p_viminfo;     /* 'viminfo' */
+EXTERN char_u   *p_shada;       /* 'shada' */
 EXTERN char_u   *p_vdir;        /* 'viewdir' */
 EXTERN char_u   *p_vop;         /* 'viewoptions' */
 EXTERN unsigned vop_flags;      /* uses SSOP_ flags */

src/nvim/options.lua

@@ -1991,6 +1991,14 @@ return {
         vim="blank,buffers,curdir,folds,help,tabpages,winsize"
       }}
     },
+    {
+      full_name='shada', abbreviation='sd',
+      type='string', list='comma', scope={'global'},
+      deny_duplicates=true,
+      secure=true,
+      varname='p_shada',
+      defaults={if_true={vi="", vim="!,'100,<50,s10,h"}}
+    },
     {
       full_name='shell', abbreviation='sh',
       type='string', scope={'global'},
@@ -2584,7 +2592,7 @@ return {
       type='string', list='comma', scope={'global'},
       deny_duplicates=true,
       secure=true,
-      varname='p_viminfo',
+      varname='p_shada',
       defaults={if_true={vi="", vim="!,'100,<50,s10,h"}}
     },
     {

src/nvim/os/fs_defs.h

@@ -21,4 +21,9 @@ typedef struct {
   uv_dirent_t ent;  ///< @private The entry information.
 } Directory;
 
+/// Function to convert -errno error to char * error description
+///
+/// -errno errors are returned by a number of os functions.
+#define os_strerror uv_strerror
+
 #endif  // NVIM_OS_FS_DEFS_H

src/nvim/os/time.c

@@ -103,3 +103,12 @@ struct tm *os_get_localtime(struct tm *result) FUNC_ATTR_NONNULL_ALL
   time_t rawtime = time(NULL);
   return os_localtime_r(&rawtime, result);
 }
+
+/// Obtains the current UNIX timestamp
+///
+/// @return Seconds since epoch.
+Timestamp os_time(void)
+  FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  return (Timestamp) time(NULL);
+}

src/nvim/os/time.h

@@ -5,6 +5,8 @@
 #include <stdbool.h>
 #include <time.h>
 
+typedef uint64_t Timestamp;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "os/time.h.generated.h"
 #endif

src/nvim/os/unix_defs.h

@@ -43,8 +43,8 @@
 #ifndef VIMRC_FILE
 # define VIMRC_FILE     ".nvimrc"
 #endif
-#ifndef VIMINFO_FILE
+#ifndef SHADA_FILE
-# define VIMINFO_FILE "~/.nviminfo"
+# define SHADA_FILE "~/.nvim/shada/main.shada"
 #endif
 
 // Default for 'backupdir'.

src/nvim/os/win_defs.h

@@ -9,7 +9,7 @@
 // Defines needed to fix the build on Windows:
 // - USR_EXRC_FILE
 // - USR_VIMRC_FILE
-// - VIMINFO_FILE
+// - SHADA_FILE
 // - DFLT_DIR
 // - DFLT_BDIR
 // - DFLT_VDIR

src/nvim/search.c

@@ -79,23 +79,6 @@
  * Henry Spencer's regular expression library.  See regexp.c.
  */
 
-/* The offset for a search command is store in a soff struct */
-/* Note: only spats[0].off is really used */
-struct soffset {
-  int dir;                      /* search direction, '/' or '?' */
-  int line;                     /* search has line offset */
-  int end;                      /* search set cursor at end */
-  long off;                     /* line or char offset */
-};
-
-/* A search pattern and its attributes are stored in a spat struct */
-struct spat {
-  char_u          *pat;         /* the pattern (in allocated memory) or NULL */
-  int magic;                    /* magicness of the pattern */
-  int no_scs;                   /* no smartcase for this pattern */
-  struct soffset off;
-};
-
 /*
  * Two search patterns are remembered: One for the :substitute command and
  * one for other searches.  last_idx points to the one that was used the last
@@ -103,8 +86,10 @@ struct spat {
  */
 static struct spat spats[2] =
 {
-  {NULL, TRUE, FALSE, {'/', 0, 0, 0L}},         /* last used search pat */
+  // Last used search pattern
-  {NULL, TRUE, FALSE, {'/', 0, 0, 0L}}          /* last used substitute pat */
+  [0] = {NULL, true, false, 0, {'/', false, false, 0L}, NULL},
+  // Last used substitute pattern
+  [1] = {NULL, true, false, 0, {'/', false, false, 0L}, NULL}
 };
 
 static int last_idx = 0;        /* index in spats[] for RE_LAST */
@@ -256,10 +241,12 @@ char_u *reverse_text(char_u *s) FUNC_ATTR_NONNULL_RET
 void save_re_pat(int idx, char_u *pat, int magic)
 {
   if (spats[idx].pat != pat) {
-    xfree(spats[idx].pat);
+    free_spat(&spats[idx]);
     spats[idx].pat = vim_strsave(pat);
     spats[idx].magic = magic;
     spats[idx].no_scs = no_smartcase;
+    spats[idx].timestamp = os_time();
+    spats[idx].additional_data = NULL;
     last_idx = idx;
     /* If 'hlsearch' set and search pat changed: need redraw. */
     if (p_hls)
@@ -291,21 +278,29 @@ void save_search_patterns(void)
 void restore_search_patterns(void)
 {
   if (--save_level == 0) {
-    xfree(spats[0].pat);
+    free_spat(&spats[0]);
     spats[0] = saved_spats[0];
     set_vv_searchforward();
-    xfree(spats[1].pat);
+    free_spat(&spats[1]);
     spats[1] = saved_spats[1];
     last_idx = saved_last_idx;
     SET_NO_HLSEARCH(saved_no_hlsearch);
   }
 }
 
+static inline void free_spat(struct spat *const spat)
+{
+  xfree(spat->pat);
+  dict_unref(spat->additional_data);
+}
+
 #if defined(EXITFREE)
 void free_search_patterns(void)
 {
-  xfree(spats[0].pat);
+  free_spat(&spats[0]);
-  xfree(spats[1].pat);
+  free_spat(&spats[1]);
+
+  memset(spats, 0, sizeof(spats));
 
   if (mr_pattern_alloced) {
     xfree(mr_pattern);
@@ -414,17 +409,19 @@ void reset_search_dir(void)
 }
 
 /*
- * Set the last search pattern.  For ":let @/ =" and viminfo.
+ * Set the last search pattern.  For ":let @/ =" and ShaDa file.
  * Also set the saved search pattern, so that this works in an autocommand.
  */
 void set_last_search_pat(const char_u *s, int idx, int magic, int setlast)
 {
-  xfree(spats[idx].pat);
+  free_spat(&spats[idx]);
   /* An empty string means that nothing should be matched. */
   if (*s == NUL)
     spats[idx].pat = NULL;
   else
     spats[idx].pat = (char_u *) xstrdup((char *) s);
+  spats[idx].timestamp = os_time();
+  spats[idx].additional_data = NULL;
   spats[idx].magic = magic;
   spats[idx].no_scs = FALSE;
   spats[idx].off.dir = '/';
@@ -435,7 +432,7 @@ void set_last_search_pat(const char_u *s, int idx, int magic, int setlast)
   if (setlast)
     last_idx = idx;
   if (save_level) {
-    xfree(saved_spats[idx].pat);
+    free_spat(&saved_spats[idx]);
     saved_spats[idx] = spats[0];
     if (spats[idx].pat == NULL)
       saved_spats[idx].pat = NULL;
@@ -1053,7 +1050,7 @@ int do_search(
       else if ((options & SEARCH_OPT) &&
                (*p == 'e' || *p == 's' || *p == 'b')) {
         if (*p == 'e')                  /* end */
-          spats[0].off.end = SEARCH_END;
+          spats[0].off.end = true;
         ++p;
       }
       if (ascii_isdigit(*p) || *p == '+' || *p == '-') {      /* got an offset */
@@ -1166,12 +1163,13 @@ int do_search(
       lrFswap(searchstr,0);
 
     c = searchit(curwin, curbuf, &pos, dirc == '/' ? FORWARD : BACKWARD,
-        searchstr, count, spats[0].off.end + (options &
+        searchstr, count, (spats[0].off.end * SEARCH_END
+                           + (options &
                               (SEARCH_KEEP + SEARCH_PEEK +
                                SEARCH_HIS
                                + SEARCH_MSG + SEARCH_START
                                + ((pat != NULL && *pat ==
-                                                   ';') ? 0 : SEARCH_NOOF))),
+                                   ';') ? 0 : SEARCH_NOOF)))),
         RE_LAST, (linenr_T)0, tm);
 
     if (dircp != NULL)
@@ -4605,105 +4603,45 @@ static void show_pat_in_path(char_u *line, int type, int did_show, int action, F
   }
 }
 
-int read_viminfo_search_pattern(vir_T *virp, int force)
+/// Get last search pattern
+void get_search_pattern(SearchPattern *const pat)
 {
-  char_u      *lp;
+  memcpy(pat, &(spats[0]), sizeof(spats[0]));
-  int idx = -1;
-  int magic = FALSE;
-  int no_scs = FALSE;
-  int off_line = FALSE;
-  int off_end = 0;
-  long off = 0;
-  int setlast = FALSE;
-  static int hlsearch_on = FALSE;
-  char_u      *val;
-
-  /*
-   * Old line types:
-   * "/pat", "&pat": search/subst. pat
-   * "~/pat", "~&pat": last used search/subst. pat
-   * New line types:
-   * "~h", "~H": hlsearch highlighting off/on
-   * "~<magic><smartcase><line><end><off><last><which>pat"
-   * <magic>: 'm' off, 'M' on
-   * <smartcase>: 's' off, 'S' on
-   * <line>: 'L' line offset, 'l' char offset
-   * <end>: 'E' from end, 'e' from start
-   * <off>: decimal, offset
-   * <last>: '~' last used pattern
-   * <which>: '/' search pat, '&' subst. pat
-   */
-  lp = virp->vir_line;
-  if (lp[0] == '~' && (lp[1] == 'm' || lp[1] == 'M')) { /* new line type */
-    if (lp[1] == 'M')                   /* magic on */
-      magic = TRUE;
-    if (lp[2] == 's')
-      no_scs = TRUE;
-    if (lp[3] == 'L')
-      off_line = TRUE;
-    if (lp[4] == 'E')
-      off_end = SEARCH_END;
-    lp += 5;
-    off = getdigits_long(&lp);
-  }
-  if (lp[0] == '~') {           /* use this pattern for last-used pattern */
-    setlast = TRUE;
-    lp++;
-  }
-  if (lp[0] == '/')
-    idx = RE_SEARCH;
-  else if (lp[0] == '&')
-    idx = RE_SUBST;
-  else if (lp[0] == 'h')        /* ~h: 'hlsearch' highlighting off */
-    hlsearch_on = FALSE;
-  else if (lp[0] == 'H')        /* ~H: 'hlsearch' highlighting on */
-    hlsearch_on = TRUE;
-  if (idx >= 0) {
-    if (force || spats[idx].pat == NULL) {
-      val = viminfo_readstring(virp, (int)(lp - virp->vir_line + 1), TRUE);
-      set_last_search_pat(val, idx, magic, setlast);
-      xfree(val);
-      spats[idx].no_scs = no_scs;
-      spats[idx].off.line = off_line;
-      spats[idx].off.end = off_end;
-      spats[idx].off.off = off;
-      if (setlast) {
-        SET_NO_HLSEARCH(!hlsearch_on);
-      }
-    }
-  }
-  return viminfo_readline(virp);
 }
 
-void write_viminfo_search_pattern(FILE *fp)
+/// Get last substitute pattern
+void get_substitute_pattern(SearchPattern *const pat)
 {
-  if (get_viminfo_parameter('/') != 0) {
+  memcpy(pat, &(spats[1]), sizeof(spats[1]));
-    fprintf(fp, "\n# hlsearch on (H) or off (h):\n~%c",
+  memset(&(pat->off), 0, sizeof(pat->off));
-        (no_hlsearch || find_viminfo_parameter('h') != NULL) ? 'h' : 'H');
-    wvsp_one(fp, RE_SEARCH, "", '/');
-    wvsp_one(fp, RE_SUBST, _("Substitute "), '&');
-  }
 }
 
-static void 
+/// Set last search pattern
-wvsp_one (
+void set_search_pattern(const SearchPattern pat)
-    FILE *fp,        /* file to write to */
-    int idx,                /* spats[] index */
-    char *s,         /* search pat */
-    int sc                 /* dir char */
-)
 {
-  if (spats[idx].pat != NULL) {
+  free_spat(&spats[0]);
-    fprintf(fp, _("\n# Last %sSearch Pattern:\n~"), s);
+  memcpy(&(spats[0]), &pat, sizeof(spats[0]));
-    /* off.dir is not stored, it's reset to forward */
-    fprintf(fp, "%c%c%c%c%" PRId64 "%s%c",
-        spats[idx].magic    ? 'M' : 'm',                /* magic */
-        spats[idx].no_scs   ? 's' : 'S',                /* smartcase */
-        spats[idx].off.line ? 'L' : 'l',                /* line offset */
-        spats[idx].off.end  ? 'E' : 'e',                /* offset from end */
-        (int64_t)spats[idx].off.off,                    /* offset */
-        last_idx == idx     ? "~" : "",                 /* last used pat */
-        sc);
-    viminfo_writestring(fp, spats[idx].pat);
 }
+
+/// Set last substitute pattern
+void set_substitute_pattern(const SearchPattern pat)
+{
+  free_spat(&spats[1]);
+  memcpy(&(spats[1]), &pat, sizeof(spats[1]));
+  memset(&(spats[1].off), 0, sizeof(spats[1].off));
+}
+
+/// Set last used search pattern
+///
+/// @param[in]  is_substitute_pattern  If true set substitute pattern as last
+///                                    used. Otherwise sets search pattern.
+void set_last_used_pattern(const bool is_substitute_pattern)
+{
+  last_idx = (is_substitute_pattern ? 1 : 0);
+}
+
+/// Returns true if search pattern was the last used one
+bool search_was_last_used(void)
+{
+  return last_idx == 0;
 }

src/nvim/search.h

@@ -1,6 +1,9 @@
 #ifndef NVIM_SEARCH_H
 #define NVIM_SEARCH_H
 
+#include <stdbool.h>
+#include <stdint.h>
+
 /* Values for the find_pattern_in_path() function args 'type' and 'action': */
 #define FIND_ANY        1
 #define FIND_DEFINE     2
@@ -39,6 +42,27 @@
 #define RE_BOTH         2       /* save pat in both patterns */
 #define RE_LAST         2       /* use last used pattern if "pat" is NULL */
 
+/// Structure containing offset definition for the last search pattern
+///
+/// @note Only offset for the last search pattern is used, not for the last
+///       substitute pattern.
+typedef struct soffset {
+  char dir;     ///< Search direction: forward ('/') or backward ('?')
+  bool line;    ///< True if search has line offset.
+  bool end;     ///< True if search sets cursor at the end.
+  int64_t off;  ///< Actual offset value.
+} SearchOffset;
+
+/// Structure containing last search pattern and its attributes.
+typedef struct spat {
+  char_u *pat;          ///< The pattern (in allocated memory) or NULL.
+  bool magic;           ///< Magicness of the pattern.
+  bool no_scs;          ///< No smartcase for this pattern.
+  Timestamp timestamp;  ///< Time of the last change.
+  SearchOffset off;     ///< Pattern offset.
+  dict_T *additional_data;  ///< Additional data from ShaDa file.
+} SearchPattern;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "search.h.generated.h"
 #endif

src/nvim/shada.h

@@ -0,0 +1,7 @@
+#ifndef NVIM_SHADA_H
+#define NVIM_SHADA_H
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "shada.h.generated.h"
+#endif
+#endif  // NVIM_SHADA_H

src/nvim/strings.c

@@ -483,6 +483,21 @@ bool has_non_ascii(const char_u *s)
   return false;
 }
 
+/// Return true if string "s" contains a non-ASCII character (128 or higher).
+/// When "s" is NULL false is returned.
+bool has_non_ascii_len(const char *const s, const size_t len)
+  FUNC_ATTR_PURE
+{
+  if (s != NULL) {
+    for (size_t i = 0; i < len; i++) {
+      if ((uint8_t) s[i] >= 128) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 /*
  * Concatenate two strings and return the result in allocated memory.
  */

src/nvim/tag.c

@@ -130,7 +130,7 @@ static char_u   *tagmatchname = NULL;   /* name of last used tag */
  * Tag for preview window is remembered separately, to avoid messing up the
  * normal tagstack.
  */
-static taggy_T ptag_entry = {NULL, {INIT_POS_T(0, 0, 0), 0}, 0, 0};
+static taggy_T ptag_entry = {NULL, {INIT_POS_T(0, 0, 0), 0, 0, NULL}, 0, 0};
 
 /*
  * Jump to tag; handling of tag commands and tag stack

src/nvim/testdir/test8.in

@@ -32,7 +32,7 @@ endfunc
   $put ='VimLeave done'
   write
 endfunc
-:set viminfo='100
+:set shada='100
 :au BufUnload * call CloseAll()
 :au VimLeave * call WriteToOut()
 :e small.vim

src/nvim/undo.c

@@ -325,6 +325,14 @@ static long get_undolevel(void)
   return curbuf->b_p_ul;
 }
 
+static inline void zero_fmark_additional_data(fmark_T *fmarks)
+{
+  for (size_t i = 0; i < NMARKS; i++) {
+    dict_unref(fmarks[i].additional_data);
+    fmarks[i].additional_data = NULL;
+  }
+}
+
 /*
  * Common code for various ways to save text before a change.
  * "top" is the line above the first changed line.
@@ -467,7 +475,9 @@ int u_savecommon(linenr_T top, linenr_T bot, linenr_T newbot, int reload)
                     ((curbuf->b_ml.ml_flags & ML_EMPTY) ? UH_EMPTYBUF : 0);
 
     /* save named marks and Visual marks for undo */
-    memmove(uhp->uh_namedm, curbuf->b_namedm, sizeof(pos_T) * NMARKS);
+    zero_fmark_additional_data(curbuf->b_namedm);
+    memmove(uhp->uh_namedm, curbuf->b_namedm,
+            sizeof(curbuf->b_namedm[0]) * NMARKS);
     uhp->uh_visual = curbuf->b_visual;
 
     curbuf->b_u_newhead = uhp;
@@ -785,7 +795,7 @@ static bool serialize_uhp(bufinfo_T *bi, u_header_T *uhp)
   undo_write_bytes(bi, (uintmax_t)uhp->uh_flags, 2);
   // Assume NMARKS will stay the same.
   for (size_t i = 0; i < (size_t)NMARKS; i++) {
-    serialize_pos(bi, uhp->uh_namedm[i]);
+    serialize_pos(bi, uhp->uh_namedm[i].mark);
   }
   serialize_visualinfo(bi, &uhp->uh_visual);
   uint8_t time_buf[8];
@@ -831,8 +841,11 @@ static u_header_T *unserialize_uhp(bufinfo_T *bi, char_u *file_name)
   unserialize_pos(bi, &uhp->uh_cursor);
   uhp->uh_cursor_vcol = undo_read_4c(bi);
   uhp->uh_flags = undo_read_2c(bi);
+  const Timestamp cur_timestamp = os_time();
   for (size_t i = 0; i < (size_t)NMARKS; i++) {
-    unserialize_pos(bi, &uhp->uh_namedm[i]);
+    unserialize_pos(bi, &uhp->uh_namedm[i].mark);
+    uhp->uh_namedm[i].timestamp = cur_timestamp;
+    uhp->uh_namedm[i].fnum = 0;
   }
   unserialize_visualinfo(bi, &uhp->uh_visual);
   uhp->uh_time = undo_read_time(bi);
@@ -2009,7 +2022,7 @@ static void u_undoredo(int undo)
   u_entry_T   *newlist = NULL;
   int old_flags;
   int new_flags;
-  pos_T namedm[NMARKS];
+  fmark_T namedm[NMARKS];
   visualinfo_T visualinfo;
   int empty_buffer;                         /* buffer became empty */
   u_header_T  *curhead = curbuf->b_u_curhead;
@@ -2029,7 +2042,8 @@ static void u_undoredo(int undo)
   /*
    * save marks before undo/redo
    */
-  memmove(namedm, curbuf->b_namedm, sizeof(pos_T) * NMARKS);
+  zero_fmark_additional_data(curbuf->b_namedm);
+  memmove(namedm, curbuf->b_namedm, sizeof(curbuf->b_namedm[0]) * NMARKS);
   visualinfo = curbuf->b_visual;
   curbuf->b_op_start.lnum = curbuf->b_ml.ml_line_count;
   curbuf->b_op_start.col = 0;
@@ -2158,7 +2172,8 @@ static void u_undoredo(int undo)
    * restore marks from before undo/redo
    */
   for (i = 0; i < NMARKS; ++i)
-    if (curhead->uh_namedm[i].lnum != 0) {
+    if (curhead->uh_namedm[i].mark.lnum != 0) {
+      free_fmark(curbuf->b_namedm[i]);
       curbuf->b_namedm[i] = curhead->uh_namedm[i];
       curhead->uh_namedm[i] = namedm[i];
     }

src/nvim/undo_defs.h

@@ -5,6 +5,7 @@
 
 #include "nvim/pos.h"
 #include "nvim/buffer_defs.h"
+#include "nvim/mark_defs.h"
 
 /* Structure to store info about the Visual area. */
 typedef struct {
@@ -54,7 +55,7 @@ struct u_header {
   pos_T uh_cursor;              /* cursor position before saving */
   long uh_cursor_vcol;
   int uh_flags;                 /* see below */
-  pos_T uh_namedm[NMARKS];              /* marks before undo/after redo */
+  fmark_T uh_namedm[NMARKS];    /* marks before undo/after redo */
   visualinfo_T uh_visual;       /* Visual areas before undo/after redo */
   time_t uh_time;               /* timestamp when the change was made */
   long uh_save_nr;              /* set when the file was saved after the

src/nvim/window.c

@@ -1925,7 +1925,7 @@ int win_close(win_T *win, int free_buf)
       && (last_window() || curtab != prev_curtab
           || close_last_window_tabpage(win, free_buf, prev_curtab))) {
     /* Autocommands have close all windows, quit now.  Restore
-    * curwin->w_buffer, otherwise writing viminfo may fail. */
+    * curwin->w_buffer, otherwise writing ShaDa file may fail. */
     if (curwin->w_buffer == NULL)
       curwin->w_buffer = curbuf;
     getout(0);

test/config/paths.lua.in

@@ -7,6 +7,7 @@ end
 
 module.test_include_path = "${CMAKE_BINARY_DIR}/test/includes/post"
 module.test_libnvim_path = "${TEST_LIBNVIM_PATH}"
+module.test_source_path = "${CMAKE_SOURCE_DIR}"
 table.insert(module.include_paths, "${CMAKE_BINARY_DIR}/include")
 
 return module

test/functional/ex_cmds/recover_spec.lua

@@ -47,7 +47,8 @@ describe(':preserve', function()
 
     --TODO(justinmk): this is an ugly hack to force `helpers` to support
     --multiple sessions.
-    local nvim2 = helpers.spawn({helpers.nvim_prog, '-u', 'NONE', '--embed'})
+    local nvim2 = helpers.spawn({helpers.nvim_prog, '-u', 'NONE', '--embed'},
+                                true)
     helpers.set_session(nvim2)
 
     source(init)

test/functional/ex_cmds/wviminfo_spec.lua

@@ -3,8 +3,8 @@ local clear, execute, eq, neq, spawn, nvim_prog, set_session, wait, write_file
   = helpers.clear, helpers.execute, helpers.eq, helpers.neq, helpers.spawn,
   helpers.nvim_prog, helpers.set_session, helpers.wait, helpers.write_file
 
-describe(':wviminfo', function()
+describe(':wshada', function()
-  local viminfo_file = 'wviminfo_test'
+  local shada_file = 'wshada_test'
   local session
 
   before_each(function()
@@ -17,38 +17,41 @@ describe(':wviminfo', function()
                            '--cmd', 'set swapfile'})
     set_session(session)
 
-    os.remove(viminfo_file)
+    os.remove(shada_file)
   end)
 
-  it('creates a viminfo file', function()
+  it('creates a shada file', function()
     -- file should _not_ exist
-    eq(nil, lfs.attributes(viminfo_file))
+    eq(nil, lfs.attributes(shada_file))
-    execute('wv! '..viminfo_file)
+    execute('wsh! '..shada_file)
     wait()
     -- file _should_ exist
-    neq(nil, lfs.attributes(viminfo_file))
+    neq(nil, lfs.attributes(shada_file))
   end)
 
   it('overwrites existing files', function()
-    local text = 'wviminfo test'
+    local text = 'wshada test'
 
     -- Create a dummy file
-    write_file(viminfo_file, text)
+    write_file(shada_file, text)
 
     -- sanity check
-    eq(text, io.open(viminfo_file):read())
+    eq(text, io.open(shada_file):read())
-    neq(nil, lfs.attributes(viminfo_file))
+    neq(nil, lfs.attributes(shada_file))
 
-    execute('wv! '..viminfo_file)
+    execute('wsh! '..shada_file)
     wait()
 
-    -- File should have been overwritten with a viminfo file.
+    -- File should have been overwritten with a shada file.
-    local line1 = io.lines(viminfo_file)()
+    local fp = io.open(shada_file, 'r')
-    assert(nil ~= string.find(line1, 'This viminfo file was generated by Nvim'),
+    local char1 = fp:read(1)
-      viminfo_file..' should be a viminfo-formatted file')
+    fp:close()
+    -- ShaDa file starts with a “header” entry
+    assert(char1:byte() == 0x01,
+      shada_file..' should be a shada file')
   end)
 
   teardown(function()
-    os.remove(viminfo_file)
+    os.remove(shada_file)
   end)
 end)

test/functional/ex_getln/history_spec.lua

@@ -0,0 +1,36 @@
+local helpers = require('test.functional.helpers')
+local clear, meths, funcs, eq =
+  helpers.clear, helpers.meths, helpers.funcs, helpers.eq
+
+describe('history support code', function()
+  before_each(clear)
+
+  it('correctly clears start of the history', function()
+    -- Regression test: check absense of the memory leak when clearing start of 
+    -- the history using ex_getln.c/clr_history().
+    eq(1, funcs.histadd(':', 'foo'))
+    eq(1, funcs.histdel(':'))
+    eq('', funcs.histget(':', -1))
+  end)
+
+  it('correctly clears end of the history', function()
+    -- Regression test: check absense of the memory leak when clearing end of 
+    -- the history using ex_getln.c/clr_history().
+    meths.set_option('history', 1)
+    eq(1, funcs.histadd(':', 'foo'))
+    eq(1, funcs.histdel(':'))
+    eq('', funcs.histget(':', -1))
+  end)
+
+  it('correctly removes item from history', function()
+    -- Regression test: check that ex_getln.c/del_history_idx() correctly clears 
+    -- history index after removing history entry. If it does not then deleting 
+    -- history will result in a double free.
+    eq(1, funcs.histadd(':', 'foo'))
+    eq(1, funcs.histadd(':', 'bar'))
+    eq(1, funcs.histadd(':', 'baz'))
+    eq(1, funcs.histdel(':', -2))
+    eq(1, funcs.histdel(':'))
+    eq('', funcs.histget(':', -1))
+  end)
+end)

test/functional/helpers.lua

@@ -44,11 +44,15 @@ elseif os.getenv('GDB') then
 end
 
 if prepend_argv then
+  local new_nvim_argv = {}
   local len = #prepend_argv
-  for i = 1, #nvim_argv do
+  for i = 1, len do
-    prepend_argv[i + len] = nvim_argv[i]
+    new_nvim_argv[i] = prepend_argv[i]
   end
-  nvim_argv = prepend_argv
+  for i = 1, #nvim_argv do
+    new_nvim_argv[i + len] = nvim_argv[i]
+  end
+  nvim_argv = new_nvim_argv
 end
 
 local session, loop_running, loop_stopped, last_error
@@ -174,12 +178,27 @@ local function rawfeed(...)
   end
 end
 
-local function spawn(argv)
+local function merge_args(...)
+  local i = 1
+  local argv = {}
+  for anum = 1,select('#', ...) do
+    local args = select(anum, ...)
+    if args then
+      for _, arg in ipairs(args) do
+        argv[i] = arg
+        i = i + 1
+      end
+    end
+  end
+  return argv
+end
+
+local function spawn(argv, merge)
   local loop = Loop.new()
   local msgpack_stream = MsgpackStream.new(loop)
   local async_session = AsyncSession.new(msgpack_stream)
   local session = Session.new(async_session)
-  loop:spawn(argv)
+  loop:spawn(merge and merge_args(prepend_argv, argv) or argv)
   return session
 end
 
@@ -216,9 +235,12 @@ local function execute(...)
 end
 
 -- Dedent the given text and write it to the file name.
-local function write_file(name, text)
+local function write_file(name, text, dont_dedent)
   local file = io.open(name, 'w')
-  file:write(dedent(text))
+  if not dont_dedent then
+    text = dedent(text)
+  end
+  file:write(text)
   file:flush()
   file:close()
 end
@@ -337,7 +359,40 @@ local exc_exec = function(cmd)
   return ret
 end
 
+local function redir_exec(cmd)
+  nvim_command(([[
+    redir => g:__output
+      silent! execute "%s"
+    redir END
+  ]]):format(cmd:gsub('\n', '\\n'):gsub('[\\"]', '\\%0')))
+  local ret = nvim_eval('get(g:, "__output", 0)')
+  nvim_command('unlet! g:__output')
+  return ret
+end
+
+local function create_callindex(func)
+  local tbl = {}
+  setmetatable(tbl, {
+    __index = function(tbl, arg1)
+      ret = function(...) return func(arg1, ...) end
+      tbl[arg1] = ret
+      return ret
+    end,
+  })
+  return tbl
+end
+
+local funcs = create_callindex(nvim_call)
+local meths = create_callindex(nvim)
+local bufmeths = create_callindex(buffer)
+local winmeths = create_callindex(window)
+local tabmeths = create_callindex(tabpage)
+local curbufmeths = create_callindex(curbuf)
+local curwinmeths = create_callindex(curwin)
+local curtabmeths = create_callindex(curtab)
+
 return {
+  prepend_argv = prepend_argv,
   clear = clear,
   spawn = spawn,
   dedent = dedent,
@@ -374,4 +429,14 @@ return {
   rmdir = rmdir,
   mkdir = lfs.mkdir,
   exc_exec = exc_exec,
+  redir_exec = redir_exec,
+  merge_args = merge_args,
+  funcs = funcs,
+  meths = meths,
+  bufmeths = bufmeths,
+  winmeths = winmeths,
+  tabmeths = tabmeths,
+  curbufmeths = curbufmeths,
+  curwinmeths = curwinmeths,
+  curtabmeths = curtabmeths,
 }

test/functional/legacy/074_global_var_in_viminfo_spec.lua

@@ -1,14 +1,15 @@
--- Tests for storing global variables in the .viminfo file
+-- Tests for storing global variables in the .shada file
 
 local helpers, lfs = require('test.functional.helpers'), require('lfs')
 local clear, execute, eq, neq, eval, wait, spawn =
   helpers.clear, helpers.execute, helpers.eq, helpers.neq, helpers.eval,
   helpers.wait, helpers.spawn
 
-describe('storing global variables in viminfo files', function()
+describe('storing global variables in ShaDa files', function()
+  local tempname = 'Xtest-functional-legacy-074'
   setup(function()
     clear()
-    os.remove("Xviminfo")
+    os.remove(tempname)
   end)
 
   it('is working', function()
@@ -27,33 +28,34 @@ describe('storing global variables in viminfo files', function()
     execute(
       -- This will cause a few errors, do it silently.
       'set visualbell',
-      'set viminfo+=!',
+      'set shada+=!',
       "let MY_GLOBAL_DICT={'foo': 1, 'bar': 0, 'longvarible': 1000}",
-      -- Store a really long list, so line wrapping will occur in viminfo
+      -- Store a really long list. Initially this was testing line wrapping in 
-      -- file.
+      -- viminfo, but shada files has no line wrapping, no matter how long the 
+      -- list is.
       'let MY_GLOBAL_LIST=range(1,100)'
     )
     eq(test_dict, eval('MY_GLOBAL_DICT'))
     eq(test_list, eval('MY_GLOBAL_LIST'))
 
-    execute('wv! Xviminfo')
+    execute('wsh! ' .. tempname)
     wait()
 
-    -- Assert that the viminfo file exists.
+    -- Assert that the shada file exists.
-    neq(nil, lfs.attributes('Xviminfo'))
+    neq(nil, lfs.attributes(tempname))
     execute('unlet MY_GLOBAL_DICT',
             'unlet MY_GLOBAL_LIST')
     -- Assert that the variables where deleted.
     eq(0, eval('exists("MY_GLOBAL_DICT")'))
     eq(0, eval('exists("MY_GLOBAL_LIST")'))
 
-    execute('rv! Xviminfo')
+    execute('rsh! ' .. tempname)
 
     eq(test_list, eval('MY_GLOBAL_LIST'))
     eq(test_dict, eval('MY_GLOBAL_DICT'))
   end)
 
   teardown(function()
-    os.remove('Xviminfo')
+    os.remove(tempname)
   end)
 end)

test/functional/shada/buffers_spec.lua

@@ -0,0 +1,51 @@
+-- ShaDa buffer list saving/reading support
+local helpers = require('test.functional.helpers')
+local nvim_command, funcs, eq =
+  helpers.command, helpers.funcs, helpers.eq
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, set_additional_cmd, clear =
+  shada_helpers.reset, shada_helpers.set_additional_cmd,
+  shada_helpers.clear
+
+describe('ShaDa support code', function()
+  testfilename = 'Xtestfile-functional-shada-buffers'
+  testfilename_2 = 'Xtestfile-functional-shada-buffers-2'
+  before_each(reset)
+  after_each(clear)
+
+  it('is able to dump and restore buffer list', function()
+    set_additional_cmd('set shada+=%')
+    reset()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('edit ' .. testfilename_2)
+    nvim_command('qall')
+    reset()
+    eq(3, funcs.bufnr('$'))
+    eq('', funcs.bufname(1))
+    eq(testfilename, funcs.bufname(2))
+    eq(testfilename_2, funcs.bufname(3))
+  end)
+
+  it('does not restore buffer list without % in &shada', function()
+    set_additional_cmd('set shada+=%')
+    reset()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('edit ' .. testfilename_2)
+    set_additional_cmd('')
+    nvim_command('qall')
+    reset()
+    eq(1, funcs.bufnr('$'))
+    eq('', funcs.bufname(1))
+  end)
+
+  it('does not dump buffer list without % in &shada', function()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('edit ' .. testfilename_2)
+    set_additional_cmd('set shada+=%')
+    nvim_command('qall')
+    reset()
+    eq(1, funcs.bufnr('$'))
+    eq('', funcs.bufname(1))
+  end)
+end)

test/functional/shada/compatibility_spec.lua

@@ -0,0 +1,453 @@
+-- ShaDa compatibility support
+local helpers = require('test.functional.helpers')
+local nvim_command, funcs, eq = helpers.command, helpers.funcs, helpers.eq
+local exc_exec = helpers.exc_exec
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, set_additional_cmd, clear, get_shada_rw =
+  shada_helpers.reset, shada_helpers.set_additional_cmd,
+  shada_helpers.clear, shada_helpers.get_shada_rw
+local read_shada_file = shada_helpers.read_shada_file
+
+local wshada, sdrcmd, shada_fname = get_shada_rw('Xtest-functional-shada-compatibility.shada')
+
+describe('ShaDa forward compatibility support code', function()
+  before_each(reset)
+  after_each(function()
+    clear()
+    os.remove(shada_fname)
+  end)
+
+  it('works with search pattern item with BOOL unknown (sX) key value', function()
+    wshada('\002\001\011\130\162sX\194\162sp\196\001-')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and not v.value.ss then
+        eq(false, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('silent! /---/')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and not v.value.ss then
+        eq(nil, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada! ' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with s/search pattern item with BOOL unknown (sX) key value', function()
+    wshada('\002\001\015\131\162sX\194\162ss\195\162sp\196\001-')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and v.value.ss then
+        eq(false, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('silent! s/--/---/ge')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and v.value.ss then
+        eq(nil, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with replacement item with BOOL additional value in list', function()
+    wshada('\003\000\005\146\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 3 then
+        eq(2, #v.value)
+        eq(false, v.value[2])
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('silent! s/--/---/ge')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 3 then
+        eq(1, #v.value)
+        found = true
+      end
+    end
+    eq(true, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  for _, v in ipairs({{name='global mark', mpack='\007\001\018\131\162mX\195\161f\196\006/a/b/c\161nA'},
+                      {name='jump', mpack='\008\001\018\131\162mX\195\161f\196\006/a/b/c\161l\002'},
+                      {name='local mark', mpack='\010\001\018\131\162mX\195\161f\196\006/a/b/c\161na'},
+                      {name='change', mpack='\011\001\015\130\162mX\195\161f\196\006/a/b/c'},
+                     }) do
+    it('works with ' .. v.name .. ' item with BOOL unknown (mX) key value', function()
+      nvim_command('silent noautocmd edit /a/b/c')
+      eq('/a/b/c', funcs.bufname('%'))
+      funcs.setline('.', {'1', '2', '3'})
+      wshada(v.mpack)
+      eq(0, exc_exec(sdrcmd(true)))
+      os.remove(shada_fname)
+      nvim_command('wshada ' .. shada_fname)
+      local found = false
+      for _, subv in ipairs(read_shada_file(shada_fname)) do
+        if subv.type == v.mpack:byte() then
+          if subv.value.mX == true then
+            found = true
+          end
+        end
+      end
+      eq(true, found)
+      eq(0, exc_exec(sdrcmd()))
+      nvim_command('bwipeout!')
+      funcs.setpos('\'A', {0, 1, 1, 0})
+      os.remove(shada_fname)
+      nvim_command('wshada ' .. shada_fname)
+      found = false
+      for _, subv in ipairs(read_shada_file(shada_fname)) do
+        if subv.type == v.mpack:byte() then
+          if subv.value.mX == true then
+            found = true
+          end
+        end
+      end
+      eq(false, found)
+      funcs.garbagecollect(1)
+      funcs.garbagecollect(1)
+      nvim_command('rshada!' .. shada_fname)
+      funcs.garbagecollect(1)
+      funcs.garbagecollect(1)
+    end)
+
+    if v.name == 'global mark' or v.name == 'local mark' then
+      it('works with ' .. v.name .. ' item with <C-a> name', function()
+        nvim_command('silent noautocmd edit /a/b/c')
+        eq('/a/b/c', funcs.bufname('%'))
+        funcs.setline('.', {'1', '2', '3'})
+        wshada(v.mpack:gsub('n.$', 'n\001')
+               .. v.mpack:gsub('n.$', 'n\002')
+               .. v.mpack:gsub('n.$', 'n\003'):gsub('/a/b/c', '/d/e/f'))
+        eq(0, exc_exec(sdrcmd(true)))
+        nvim_command('wshada ' .. shada_fname)
+        local found = 0
+        for i, subv in ipairs(read_shada_file(shada_fname)) do
+          if i == 1 then
+            eq(1, subv.type)
+          end
+          if subv.type == v.mpack:byte() then
+            if subv.value.mX == true and subv.value.n <= 3 then
+              found = found + 1
+            end
+          end
+        end
+        eq(3, found)
+        nvim_command('wshada! ' .. shada_fname)
+        local found = 0
+        for i, subv in ipairs(read_shada_file(shada_fname)) do
+          if i == 1 then
+            eq(1, subv.type)
+          end
+          if subv.type == v.mpack:byte() then
+            if subv.value.mX == true and subv.value.n <= 3 then
+              found = found + 1
+            end
+          end
+        end
+        eq(0, found)
+        funcs.garbagecollect(1)
+        funcs.garbagecollect(1)
+        nvim_command('rshada!' .. shada_fname)
+        funcs.garbagecollect(1)
+        funcs.garbagecollect(1)
+      end)
+    end
+  end
+
+  it('works with register item with BOOL unknown (rX) key', function()
+    wshada('\005\001\015\131\161na\162rX\194\162rc\145\196\001-')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 5 and v.value.rX == false then
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('let @a = "Test"')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 5 and v.value.rX == false then
+        found = true
+      end
+    end
+    eq(false, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with register item with <C-a> name', function()
+    wshada('\005\001\015\131\161n\001\162rX\194\162rc\145\196\001-')
+    eq(0, exc_exec(sdrcmd(true)))
+    nvim_command('wshada ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.n == 1 then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.n == 1 then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with register item with type 10', function()
+    wshada('\005\001\019\132\161na\162rX\194\162rc\145\196\001-\162rt\010')
+    eq(0, exc_exec(sdrcmd(true)))
+    -- getreg may return empty list as list with NULL pointer which API 
+    -- translates into nil for some reason.
+    eq({}, funcs.getreg('a', 1, 1) or {})
+    eq('', funcs.getregtype('a'))
+    nvim_command('wshada ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.rt == 10 then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.rt == 10 then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with buffer list item with BOOL unknown (bX) key', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\016\145\130\161f\196\006/a/b/c\162bX\195')
+    eq(0, exc_exec(sdrcmd()))
+    eq(2, funcs.bufnr('$'))
+    eq('/a/b/c', funcs.bufname(2))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 9 and #v.value == 1 and v.value[1].bX == true then
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('buffer 2')
+    nvim_command('edit!')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 5 and v.value.rX == false then
+        found = true
+      end
+    end
+    eq(false, found)
+    nvim_command('bwipeout!')
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with history item with BOOL additional value in list', function()
+    wshada('\004\000\006\147\000\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 4 and v.value[1] == 0 and v.value[2] == '-' then
+        eq(false, v.value[3])
+        eq(3, #v.value)
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    funcs.histadd(':', '--')
+    funcs.histadd(':', '-')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 4 and v.value[1] == 0 and v.value[2] == '-' then
+        eq(2, #v.value)
+        found = true
+      end
+    end
+    eq(true, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with history item with type 10', function()
+    wshada('\004\000\006\147\010\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    nvim_command('wshada ' .. shada_fname)
+    eq(0, exc_exec(sdrcmd()))
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 4 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 4 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+
+  it('works with item with 100 type', function()
+    wshada('\100\000\006\147\010\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    nvim_command('wshada ' .. shada_fname)
+    eq(0, exc_exec(sdrcmd()))
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 100 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 100 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+    nvim_command('rshada!' .. shada_fname)
+    funcs.garbagecollect(1)
+    funcs.garbagecollect(1)
+  end)
+end)

test/functional/shada/errors_spec.lua

@@ -0,0 +1,490 @@
+-- ShaDa errors handling support
+local helpers = require('test.functional.helpers')
+local nvim_command, eq, exc_exec = helpers.command, helpers.eq, helpers.exc_exec
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, clear, get_shada_rw =
+  shada_helpers.reset, shada_helpers.clear, shada_helpers.get_shada_rw
+
+local wshada, sdrcmd, shada_fname, clean =
+  get_shada_rw('Xtest-functional-shada-errors.shada')
+
+describe('ShaDa error handling', function()
+  before_each(reset)
+  after_each(function()
+    clear()
+    clean()
+  end)
+
+  -- Note: most of tests have additional items like sX, mX, rX. These are for 
+  -- valgrind tests, to check for memory leaks (i.e. whether error handling code 
+  -- does (not) forget to call ga_clear). Not needed for array-based items like 
+  -- history because they are not using ad_ga.
+
+  it('does not fail on empty file', function()
+    wshada('')
+    eq(0, exc_exec(sdrcmd()))
+  end)
+
+  it('fails on zero', function()
+    wshada('\000')
+    eq('Vim(rshada):E576: Error while reading ShaDa file: expected positive integer at position 0, but got nothing', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on missing item', function()
+    wshada('\000\000\000')
+    eq('Vim(rshada):E576: Error while reading ShaDa file: there is an item at position 0 that must not be there: Missing items are for internal uses only', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on -2 type', function()
+    wshada('\254\000\000')
+    eq('Vim(rshada):E576: Error while reading ShaDa file: expected positive integer at position 0', exc_exec(sdrcmd()))
+  end)
+
+  it('does not fail on header with zero length', function()
+    -- Header items are skipped when reading.
+    wshada('\001\000\000')
+    eq(0, exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with zero length', function()
+    wshada('\002\000\000')
+    eq('Vim(rshada):E576: Failed to parse ShaDa file: incomplete msgpack string at position 3', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with -2 timestamp', function()
+    wshada('\002\254\000')
+    eq('Vim(rshada):E576: Error while reading ShaDa file: expected positive integer at position 1', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with -2 length', function()
+    wshada('\002\000\254')
+    eq('Vim(rshada):E576: Error while reading ShaDa file: expected positive integer at position 2', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with length greater then file length', function()
+    wshada('\002\000\002\000')
+    eq('Vim(rshada):E576: Error while reading ShaDa file: last entry specified that it occupies 2 bytes, but file ended earlier', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with invalid byte', function()
+    -- 195 (== 0xC1) cannot start any valid messagepack entry (the only byte 
+    -- that cannot do this). Specifically unpack_template.h contains
+    --
+    --     //case 0xc1:  // string
+    --     //  again_terminal_trail(NEXT_CS(p), p+1);
+    --
+    -- (literally: commented out code) which means that in place of this code 
+    -- `goto _failed` is used from default: case. I do not know any other way to 
+    -- get MSGPACK_UNPACK_PARSE_ERROR and not MSGPACK_UNPACK_CONTINUE or 
+    -- MSGPACK_UNPACK_EXTRA_BYTES.
+    wshada('\002\000\001\193')
+    eq('Vim(rshada):E576: Failed to parse ShaDa file due to a msgpack parser error at position 3', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with incomplete map', function()
+    wshada('\002\000\001\129')
+    eq('Vim(rshada):E576: Failed to parse ShaDa file: incomplete msgpack string at position 3', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item without a pattern', function()
+    wshada('\002\000\005\129\162sX\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has no pattern', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern with extra bytes', function()
+    wshada('\002\000\002\128\000')
+    eq('Vim(rshada):E576: Failed to parse ShaDa file: extra bytes in msgpack string at position 3', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL value', function()
+    wshada('\002\000\001\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 is not a dictionary', exc_exec(sdrcmd()))
+  end)
+
+  -- sp entry is here because it causes an allocation.
+  it('fails on search pattern item with BIN key', function()
+    wshada('\002\000\014\131\162sp\196\001a\162sX\192\196\000\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has key which is not a string', exc_exec(sdrcmd()))
+  end)
+
+  -- sp entry is here because it causes an allocation.
+  it('fails on search pattern item with empty key', function()
+    wshada('\002\000\013\131\162sp\196\001a\162sX\192\160\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has empty key', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL magic key value', function()
+    wshada('\002\000\009\130\162sX\192\162sm\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sm key value which is not a boolean', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL smartcase key value', function()
+    wshada('\002\000\009\130\162sX\192\162sc\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sc key value which is not a boolean', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL has_line_offset key value', function()
+    wshada('\002\000\009\130\162sX\192\162sl\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sl key value which is not a boolean', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL place_cursor_at_end key value', function()
+    wshada('\002\000\009\130\162sX\192\162se\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has se key value which is not a boolean', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL is_last_used key value', function()
+    wshada('\002\000\009\130\162sX\192\162su\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has su key value which is not a boolean', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL is_substitute_pattern key value', function()
+    wshada('\002\000\009\130\162sX\192\162ss\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has ss key value which is not a boolean', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL highlighted key value', function()
+    wshada('\002\000\009\130\162sX\192\162sh\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sh key value which is not a boolean', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL offset key value', function()
+    wshada('\002\000\009\130\162sX\192\162so\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has so key value which is not an integer', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with NIL pat key value', function()
+    wshada('\002\000\009\130\162sX\192\162sp\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sp key value which is not a binary', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search pattern item with STR pat key value', function()
+    wshada('\002\000\011\130\162sX\192\162sp\162sp')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sp key value which is not a binary', exc_exec(sdrcmd()))
+  end)
+
+  for _, v in ipairs({{name='global mark', mpack='\007'},
+                      {name='jump', mpack='\008'},
+                      {name='local mark', mpack='\010'},
+                      {name='change', mpack='\011'},
+                     }) do
+    local is_mark_test = ({['global mark']=true, ['local mark']=true})[v.name]
+
+    it('fails on ' .. v.name .. ' item with NIL value', function()
+      wshada(v.mpack .. '\000\001\192')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 is not a dictionary', exc_exec(sdrcmd()))
+    end)
+
+    -- f entry is here because it causes an allocation.
+    it('fails on ' .. v.name .. ' item with BIN key', function()
+      wshada(v.mpack .. '\000\013\131\161f\196\001/\162mX\192\196\000\000')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has key which is not a string', exc_exec(sdrcmd()))
+    end)
+
+    -- f entry is here because it causes an allocation.
+    it('fails on ' .. v.name .. ' item with empty key', function()
+      wshada(v.mpack .. '\000\012\131\161f\196\001/\162mX\192\160\000')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has empty key', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item without f key', function()
+      wshada(v.mpack .. '\000\008\130\162mX\192\161l\001')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 is missing file name', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item with zero l key', function()
+      wshada(v.mpack .. '\000\013\131\162mX\192\161f\196\001/\161l\000')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has invalid line number', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item with negative l key', function()
+      wshada(v.mpack .. '\000\013\131\162mX\192\161f\196\001/\161l\255')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has invalid line number', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item with negative c key', function()
+      wshada(v.mpack .. '\000\013\131\162mX\192\161f\196\001/\161c\255')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has invalid column number', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item with STR n key value', function()
+      wshada(v.mpack .. '\000\011\130\162mX\192\161n\163spa')
+      eq(is_mark_test and 'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key value which is not an unsigned integer' or 'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key which is only valid for local and global mark entries', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item with STR l key value', function()
+      wshada(v.mpack .. '\000\010\130\162mX\192\161l\162sp')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has l key value which is not an integer', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item with STR c key value', function()
+      wshada(v.mpack .. '\000\010\130\162mX\192\161c\162sp')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has c key value which is not an integer', exc_exec(sdrcmd()))
+    end)
+
+    it('fails on ' .. v.name .. ' item with STR f key value', function()
+      wshada(v.mpack .. '\000\010\130\162mX\192\161f\162sp')
+      eq('Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has f key value which is not a binary', exc_exec(sdrcmd()))
+    end)
+  end
+
+  it('fails on register item with NIL value', function()
+    wshada('\005\000\001\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 is not a dictionary', exc_exec(sdrcmd()))
+  end)
+
+  -- rc entry is here because it causes an allocation
+  it('fails on register item with BIN key', function()
+    wshada('\005\000\015\131\162rc\145\196\001a\162rX\192\196\000\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has key which is not a string', exc_exec(sdrcmd()))
+  end)
+
+  -- rc entry is here because it causes an allocation
+  it('fails on register item with BIN key', function()
+    wshada('\005\000\014\131\162rc\145\196\001a\162rX\192\160\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has empty key', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on register item with NIL rt key value', function()
+    wshada('\005\000\009\130\162rX\192\162rt\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rt key value which is not an unsigned integer', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on register item with NIL rw key value', function()
+    wshada('\005\000\009\130\162rX\192\162rw\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rw key value which is not an unsigned integer', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on register item with NIL rc key value', function()
+    wshada('\005\000\009\130\162rX\192\162rc\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with non-array value', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on register item with empty rc key value', function()
+    wshada('\005\000\009\130\162rX\192\162rc\144')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with empty array', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on register item with NIL in rc array', function()
+    wshada('\005\000\013\130\162rX\192\162rc\146\196\001a\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc array with non-binary value', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on register item without rc array', function()
+    wshada('\005\000\009\129\162rX\146\196\001a\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has missing rc array', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on history item with NIL value', function()
+    wshada('\004\000\001\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on history item with empty value', function()
+    wshada('\004\000\001\144')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 does not have enough elements', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on history item with single element value', function()
+    wshada('\004\000\002\145\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 does not have enough elements', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on history item with NIL first item', function()
+    wshada('\004\000\003\146\192\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 has wrong history type type', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on history item with FIXUINT second item', function()
+    wshada('\004\000\003\146\000\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 has wrong history string type', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on history item with second item with zero byte', function()
+    wshada('\004\000\007\146\000\196\003ab\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 contains string with zero byte inside', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search history item without third item', function()
+    wshada('\004\000\007\146\001\196\003abc')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search history entry at position 0 does not have separator character', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on search history item with NIL third item', function()
+    wshada('\004\000\007\147\001\196\002ab\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: search history entry at position 0 has wrong history separator type', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on variable item with NIL value', function()
+    wshada('\006\000\001\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on variable item with empty value', function()
+    wshada('\006\000\001\144')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 does not have enough elements', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on variable item with single element value', function()
+    wshada('\006\000\002\145\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 does not have enough elements', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on variable item with NIL first item', function()
+    wshada('\006\000\003\146\192\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 has wrong variable name type', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on replacement item with NIL value', function()
+    wshada('\003\000\001\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on replacement item with empty value', function()
+    wshada('\003\000\001\144')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 does not have enough elements', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on replacement item with NIL first item', function()
+    wshada('\003\000\002\145\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 has wrong sub string type', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with NIL value', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\001\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list entry at position 0 is not an array', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with NIL item in the array', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\008\146\129\161f\196\001/\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry that is not a dictionary', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with empty item', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\008\146\129\161f\196\001/\128')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry that does not have a file name', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with NIL l key', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161l\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list entry entry at position 0 has l key value which is not an integer', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with zero l key', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161l\000')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry with invalid line number', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with negative l key', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161l\255')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry with invalid line number', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with negative c key', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161c\255')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry with invalid column number', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on buffer list item with NIL c key', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161c\192')
+    eq('Vim(rshada):E575: Error while reading ShaDa file: buffer list entry entry at position 0 has c key value which is not an integer', exc_exec(sdrcmd()))
+  end)
+
+  it('fails on invalid ShaDa file (viminfo file)', function()
+    wshada([[# This viminfo file was generated by Vim 7.4.
+# You may edit it if you're careful!
+
+# Value of 'encoding' when this file was written
+*encoding=utf-8
+
+
+# hlsearch on (H) or off (h):
+~h
+# Last Search Pattern:
+~MSle0~/buffer=abuf
+
+# Last Substitute Search Pattern:
+~MSle0&^$
+
+# Last Substitute String:
+$
+
+# Command Line History (newest to oldest):
+:cq
+
+# Search String History (newest to oldest):
+? \<TMUX\>
+
+# Expression History (newest to oldest):
+=system('echo "\xAB"')
+
+# Input Line History (newest to oldest):
+@i
+
+# Input Line History (newest to oldest):
+
+# Registers:
+"0	LINE	0
+	        case FLAG_B: puts("B"); break;
+"1	LINE	0
+	pick 874a489 shada,functests: Test compatibility support
+""-	CHAR	0
+	.
+
+# global variables:
+!STUF_HISTORY_TRANSLIT	LIS	[]
+!TR3_INPUT_HISTORY	LIS	[]
+
+# File marks:
+'A  8320  12  ~/a.a/Proj/c/neovim-2076/src/nvim/ex_docmd.c
+'0  66  5  ~/a.a/Proj/c/neovim/.git/rebase-merge/git-rebase-todo
+'1  7  0  ~/.vam/powerline/.git/MERGE_MSG
+'2  64  4  ~/a.a/Proj/c/neovim/.git/rebase-merge/git-rebase-todo
+'3  9  0  ~/a.a/Proj/c/neovim/.git/COMMIT_EDITMSG
+'4  62  0  ~/a.a/Proj/c/neovim/.git/rebase-merge/git-rebase-todo
+'5  57  4  ~/a.a/Proj/c/neovim/.git/rebase-merge/git-rebase-todo
+'6  1  0  ~/a.a/Proj/c/neovim/.git/rebase-merge/git-rebase-todo
+'7  399  7  /usr/share/vim/vim74/doc/motion.txt
+'8  1  0  ~/a.a/Proj/c/zpython/build/CMakeFiles/3.2.2/CMakeCCompiler.cmake
+'9  1  0  ~/a.a/Proj/c/vim/README.txt
+
+# Jumplist (newest first):
+-'  66  5  ~/a.a/Proj/c/neovim/.git/rebase-merge/git-rebase-todo
+
+# History of marks within files (newest to oldest):
+
+> ~/a.a/Proj/c/neovim/.git/rebase-merge/git-rebase-todo
+	"	66	5
+	^	66	6
+	.	66	5
+	+	65	0
+	+	65	0
+]])
+    eq('Vim(rshada):E576: Failed to parse ShaDa file: extra bytes in msgpack string at position 3', exc_exec(sdrcmd()))
+    eq('Vim(wshada):E576: Failed to parse ShaDa file: extra bytes in msgpack string at position 3', exc_exec('wshada ' .. shada_fname))
+    eq(0, exc_exec('wshada! ' .. shada_fname))
+  end)
+
+  it('fails on invalid ShaDa file (wrapper script)', function()
+    wshada('#!/bin/sh\n\npowerline "$@" 2>&1 | tee -a powerline\n')
+    eq('Vim(rshada):E576: Failed to parse ShaDa file: extra bytes in msgpack string at position 3', exc_exec(sdrcmd()))
+    eq('Vim(wshada):E576: Failed to parse ShaDa file: extra bytes in msgpack string at position 3', exc_exec('wshada ' .. shada_fname))
+    eq(0, exc_exec('wshada! ' .. shada_fname))
+  end)
+
+  it('fails on invalid ShaDa file (failing skip in second item)', function()
+    wshada('\001\000\001\128#!/')
+    eq('Vim(rshada):E576: Error while reading ShaDa file: last entry specified that it occupies 47 bytes, but file ended earlier', exc_exec(sdrcmd()))
+    eq('Vim(wshada):E576: Error while reading ShaDa file: last entry specified that it occupies 47 bytes, but file ended earlier', exc_exec('wshada ' .. shada_fname))
+    eq(0, exc_exec('wshada! ' .. shada_fname))
+  end)
+end)

test/functional/shada/helpers.lua

@@ -0,0 +1,94 @@
+local helpers = require('test.functional.helpers')
+local spawn, set_session, meths, nvim_prog =
+  helpers.spawn, helpers.set_session, helpers.meths, helpers.nvim_prog
+local write_file, merge_args = helpers.write_file, helpers.merge_args
+
+local msgpack = require('MessagePack')
+
+local tmpname = os.tmpname()
+local additional_cmd = ''
+
+local function nvim_argv()
+  local ret
+  local nvim_argv = {nvim_prog, '-u', 'NONE', '-i', tmpname, '-N',
+                     '--cmd', 'set shortmess+=I background=light noswapfile',
+                     '--cmd', additional_cmd,
+                     '--embed'}
+  if helpers.prepend_argv then
+    return merge_args(helpers.prepend_argv, nvim_argv)
+  else
+    return nvim_argv
+  end
+end
+
+local session = nil
+
+local reset = function()
+  if session then
+    session:exit(0)
+  end
+  session = spawn(nvim_argv())
+  set_session(session)
+  meths.set_var('tmpname', tmpname)
+end
+
+local set_additional_cmd = function(s)
+  additional_cmd = s
+end
+
+local clear = function()
+  os.remove(tmpname)
+  set_additional_cmd('')
+end
+
+local get_shada_rw = function(fname)
+  local wshada = function(text)
+    write_file(fname, text, true)
+  end
+  local sdrcmd = function(bang)
+    return 'rshada' .. (bang and '!' or '') .. ' ' .. fname
+  end
+  local clean = function()
+    os.remove(fname)
+    local i = ('a'):byte()
+    while i <= ('z'):byte() do
+      if not os.remove(fname .. ('.tmp.%c'):format(i)) then
+        break
+      end
+      i = i + 1
+    end
+  end
+  return wshada, sdrcmd, fname, clean
+end
+
+local mpack_keys = {'type', 'timestamp', 'length', 'value'}
+
+local read_shada_file = function(fname)
+  local fd = io.open(fname, 'r')
+  local mstring = fd:read('*a')
+  fd:close()
+  local unpacker = msgpack.unpacker(mstring)
+  local ret = {}
+  local cur
+  local i = 0
+  while true do
+    local off, val = unpacker()
+    if not off then break end
+    if i % 4 == 0 then
+      cur = {}
+      ret[#ret + 1] = cur
+    end
+    cur[mpack_keys[(i % 4) + 1]] = val
+    i = i + 1
+  end
+  return ret
+end
+
+return {
+  reset=reset,
+  set_additional_cmd=set_additional_cmd,
+  clear=clear,
+  exc_exec=exc_exec,
+  get_shada_rw=get_shada_rw,
+  read_shada_file=read_shada_file,
+}

test/functional/shada/history_spec.lua

@@ -0,0 +1,312 @@
+-- ShaDa history saving/reading support
+local helpers = require('test.functional.helpers')
+local nvim_command, funcs, meths, nvim_feed, eq =
+  helpers.command, helpers.funcs, helpers.meths, helpers.feed, helpers.eq
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, set_additional_cmd, clear =
+  shada_helpers.reset, shada_helpers.set_additional_cmd,
+  shada_helpers.clear
+
+describe('ShaDa support code', function()
+  before_each(reset)
+  after_each(clear)
+
+  it('is able to dump and read back command-line history', function()
+    nvim_command('set shada=\'0')
+    nvim_feed(':" Test\n')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada=\'0')
+    nvim_command('rshada')
+    eq('" Test', funcs.histget(':', -1))
+  end)
+
+  it('is able to dump and read back 2 items in command-line history', function()
+    nvim_command('set shada=\'0 history=2')
+    nvim_feed(':" Test\n')
+    nvim_feed(':" Test 2\n')
+    nvim_command('qall')
+    reset()
+    nvim_command('set shada=\'0 history=2')
+    nvim_command('rshada')
+    eq('" Test 2', funcs.histget(':', -1))
+    eq('" Test', funcs.histget(':', -2))
+    nvim_command('qall')
+  end)
+
+  it('respects &history when dumping',
+  function()
+    nvim_command('set shada=\'0 history=1')
+    nvim_feed(':" Test\n')
+    nvim_feed(':" Test 2\n')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada=\'0 history=2')
+    nvim_command('rshada')
+    eq('" Test 2', funcs.histget(':', -1))
+    eq('', funcs.histget(':', -2))
+  end)
+
+  it('respects &history when loading',
+  function()
+    nvim_command('set shada=\'0 history=2')
+    nvim_feed(':" Test\n')
+    nvim_feed(':" Test 2\n')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada=\'0 history=1')
+    nvim_command('rshada')
+    eq('" Test 2', funcs.histget(':', -1))
+    eq('', funcs.histget(':', -2))
+  end)
+
+  it('dumps only requested amount of command-line history items', function()
+    nvim_command('set shada=\'0,:1')
+    nvim_feed(':" Test\n')
+    nvim_feed(':" Test 2\n')
+    nvim_command('wshada')
+    -- Regression test: :wshada should not alter or free history.
+    eq('" Test 2', funcs.histget(':', -1))
+    eq('" Test', funcs.histget(':', -2))
+    reset()
+    nvim_command('set shada=\'0')
+    nvim_command('rshada')
+    eq('" Test 2', funcs.histget(':', -1))
+    eq('', funcs.histget(':', -2))
+  end)
+
+  it('does not respect number in &shada when loading history', function()
+    nvim_command('set shada=\'0')
+    nvim_feed(':" Test\n')
+    nvim_feed(':" Test 2\n')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada=\'0,:1')
+    nvim_command('rshada')
+    eq('" Test 2', funcs.histget(':', -1))
+    eq('" Test', funcs.histget(':', -2))
+  end)
+
+  it('dumps and loads all kinds of histories', function()
+    nvim_command('debuggreedy')
+    nvim_feed(':debug echo "Test"\n" Test 2\nc\n')  -- Debug history.
+    nvim_feed(':call input("")\nTest 2\n')  -- Input history.
+    nvim_feed('"="Test"\nyy')  -- Expression history.
+    nvim_feed('/Test\n')  -- Search history
+    nvim_feed(':" Test\n')  -- Command-line history
+    nvim_command('0debuggreedy')
+    nvim_command('wshada')
+    reset()
+    nvim_command('rshada')
+    eq('" Test', funcs.histget(':', -1))
+    eq('Test', funcs.histget('/', -1))
+    eq('"Test"', funcs.histget('=', -1))
+    eq('Test 2', funcs.histget('@', -1))
+    eq('c', funcs.histget('>', -1))
+  end)
+
+  it('dumps and loads last search pattern with offset', function()
+    funcs.setline('.', {'foo', 'bar'})
+    nvim_feed('gg0/a/e+1\n')
+    eq({0, 2, 3, 0}, funcs.getpos('.'))
+    nvim_command('wshada')
+    reset()
+    funcs.setline('.', {'foo', 'bar'})
+    nvim_feed('gg0n')
+    eq({0, 2, 3, 0}, funcs.getpos('.'))
+  end)
+
+  it('saves v:hlsearch=1', function()
+    nvim_command('set hlsearch shada-=h')
+    nvim_feed('/test\n')
+    eq(1, meths.get_vvar('hlsearch'))
+    nvim_command('qall')
+    reset()
+    eq(1, meths.get_vvar('hlsearch'))
+  end)
+
+  it('saves v:hlsearch=0 with :nohl', function()
+    nvim_command('set hlsearch shada-=h')
+    nvim_feed('/test\n')
+    nvim_command('nohlsearch')
+    nvim_command('qall')
+    reset()
+    eq(0, meths.get_vvar('hlsearch'))
+  end)
+
+  it('saves v:hlsearch=0 with default &shada', function()
+    nvim_command('set hlsearch')
+    nvim_feed('/test\n')
+    eq(1, meths.get_vvar('hlsearch'))
+    nvim_command('qall')
+    reset()
+    eq(0, meths.get_vvar('hlsearch'))
+  end)
+
+  it('dumps and loads last substitute pattern and replacement string', function()
+    funcs.setline('.', {'foo', 'bar'})
+    nvim_command('%s/f/g/g')
+    eq('goo', funcs.getline(1))
+    nvim_command('wshada')
+    reset()
+    funcs.setline('.', {'foo', 'bar'})
+    nvim_command('&')
+    eq('goo', funcs.getline(1))
+  end)
+
+  it('dumps and loads history correctly when &encoding is not UTF-8', function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_feed(':echo "\171"\n')
+    nvim_command('qall')
+    reset()
+    eq('echo "\171"', funcs.histget(':', -1))
+  end)
+
+  it('dumps and loads history correctly when &encoding /= UTF-8 when dumping',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_feed(':echo "\171"\n')
+    set_additional_cmd('')
+    nvim_command('qall')
+    reset()
+    eq('echo "«"', funcs.histget(':', -1))
+  end)
+
+  it('dumps and loads history correctly when &encoding /= UTF-8 when loading',
+  function()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_feed(':echo "«"\n')
+    set_additional_cmd('set encoding=latin1')
+    nvim_command('qall')
+    reset()
+    eq('echo "\171"', funcs.histget(':', -1))
+  end)
+
+  it('dumps and loads replacement correctly when &encoding is not UTF-8',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/./\171/ge')
+    nvim_command('qall!')
+    reset()
+    funcs.setline('.', {'.'})
+    nvim_command('&')
+    eq('\171', funcs.getline('.'))
+  end)
+
+  it('dumps&loads replacement correctly when &encoding /= UTF-8 when dumping',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/./\171/ge')
+    set_additional_cmd('')
+    nvim_command('qall')
+    reset()
+    funcs.setline('.', {'.'})
+    nvim_command('&')
+    eq('«', funcs.getline('.'))
+  end)
+
+  it('dumps&loads replacement correctly when &encoding /= UTF-8 when loading',
+  function()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/./«/ge')
+    set_additional_cmd('set encoding=latin1')
+    nvim_command('qall')
+    reset()
+    funcs.setline('.', {'.'})
+    nvim_command('&')
+    eq('\171', funcs.getline('.'))
+  end)
+
+  it('dumps and loads substitute pattern correctly when &encoding is not UTF-8',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/\171/./ge')
+    nvim_command('qall!')
+    reset()
+    funcs.setline('.', {'\171«'})
+    nvim_command('&')
+    eq('.«', funcs.getline('.'))
+  end)
+
+  it('dumps&loads s/pattern correctly when &encoding /= UTF-8 when dumping',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/\171/./ge')
+    set_additional_cmd('')
+    nvim_command('qall')
+    reset()
+    funcs.setline('.', {'«\171'})
+    nvim_command('&')
+    eq('.\171', funcs.getline('.'))
+  end)
+
+  it('dumps&loads s/pattern correctly when &encoding /= UTF-8 when loading',
+  function()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/«/./ge')
+    set_additional_cmd('set encoding=latin1')
+    nvim_command('qall')
+    reset()
+    funcs.setline('.', {'\171«'})
+    nvim_command('&')
+    eq('.«', funcs.getline('.'))
+  end)
+
+  it('dumps and loads search pattern correctly when &encoding is not UTF-8',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('silent! /\171/')
+    nvim_command('set shada+=/0')
+    nvim_command('qall!')
+    reset()
+    funcs.setline('.', {'\171«'})
+    nvim_command('~&')
+    eq('«', funcs.getline('.'))
+    eq('', funcs.histget('/', -1))
+  end)
+
+  it('dumps&loads /pattern correctly when &encoding /= UTF-8 when dumping',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('silent! /\171/')
+    nvim_command('set shada+=/0')
+    set_additional_cmd('')
+    nvim_command('qall')
+    reset()
+    funcs.setline('.', {'«\171'})
+    nvim_command('~&')
+    eq('\171', funcs.getline('.'))
+    eq('', funcs.histget('/', -1))
+  end)
+
+  it('dumps&loads /pattern correctly when &encoding /= UTF-8 when loading',
+  function()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('silent! /«/')
+    nvim_command('set shada+=/0')
+    set_additional_cmd('set encoding=latin1')
+    nvim_command('qall')
+    reset()
+    funcs.setline('.', {'\171«'})
+    nvim_command('~&')
+    eq('«', funcs.getline('.'))
+    eq('', funcs.histget('/', -1))
+  end)
+end)

test/functional/shada/marks_spec.lua

@@ -0,0 +1,180 @@
+-- ShaDa marks saving/reading support
+local helpers = require('test.functional.helpers')
+local meths, curwinmeths, curbufmeths, nvim_command, funcs, eq =
+  helpers.meths, helpers.curwinmeths, helpers.curbufmeths, helpers.command,
+  helpers.funcs, helpers.eq
+local exc_exec, redir_exec = helpers.exc_exec, helpers.redir_exec
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, set_additional_cmd, clear =
+  shada_helpers.reset, shada_helpers.set_additional_cmd,
+  shada_helpers.clear
+
+local nvim_current_line = function()
+  return curwinmeths.get_cursor()[1]
+end
+
+describe('ShaDa support code', function()
+  testfilename = 'Xtestfile-functional-shada-marks'
+  testfilename_2 = 'Xtestfile-functional-shada-marks-2'
+  before_each(function()
+    reset()
+    local fd = io.open(testfilename, 'w')
+    fd:write('test\n')
+    fd:write('test2\n')
+    fd:close()
+    local fd = io.open(testfilename_2, 'w')
+    fd:write('test3\n')
+    fd:write('test4\n')
+    fd:close()
+  end)
+  after_each(function()
+    clear()
+    os.remove(testfilename)
+    os.remove(testfilename_2)
+  end)
+
+  it('is able to dump and read back global mark', function()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('mark A')
+    nvim_command('2')
+    nvim_command('kB')
+    nvim_command('wshada')
+    reset()
+    nvim_command('rshada')
+    nvim_command('normal! `A')
+    eq(testfilename, funcs.fnamemodify(curbufmeths.get_name(), ':t'))
+    eq(1, nvim_current_line())
+    nvim_command('normal! `B')
+    eq(2, nvim_current_line())
+  end)
+
+  it('does not dump global mark with `f0` in shada', function()
+    nvim_command('set shada+=f0')
+    nvim_command('edit ' .. testfilename)
+    nvim_command('mark A')
+    nvim_command('2')
+    nvim_command('kB')
+    nvim_command('wshada')
+    reset()
+    nvim_command('language C')
+    eq('Vim(normal):E20: Mark not set', exc_exec('normal! `A'))
+  end)
+
+  it('does read back global mark even with `\'0` and `f0` in shada', function()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('mark A')
+    nvim_command('2')
+    nvim_command('kB')
+    nvim_command('wshada')
+    set_additional_cmd('set shada=\'0,f0')
+    reset()
+    nvim_command('language C')
+    nvim_command('normal! `A')
+    eq(testfilename, funcs.fnamemodify(curbufmeths.get_name(), ':t'))
+    eq(1, nvim_current_line())
+  end)
+
+  it('is able to dump and read back local mark', function()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('mark a')
+    nvim_command('2')
+    nvim_command('kb')
+    nvim_command('qall')
+    reset()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('normal! `a')
+    eq(testfilename, funcs.fnamemodify(curbufmeths.get_name(), ':t'))
+    eq(1, nvim_current_line())
+    nvim_command('normal! `b')
+    eq(2, nvim_current_line())
+  end)
+
+  it('is able to populate v:oldfiles', function()
+    nvim_command('edit ' .. testfilename)
+    local tf_full = curbufmeths.get_name()
+    nvim_command('edit ' .. testfilename_2)
+    local tf_full_2 = curbufmeths.get_name()
+    nvim_command('qall')
+    reset()
+    local oldfiles = meths.get_vvar('oldfiles')
+    eq(2, #oldfiles)
+    eq(testfilename, oldfiles[1]:sub(-#testfilename))
+    eq(testfilename_2, oldfiles[2]:sub(-#testfilename_2))
+    eq(tf_full, oldfiles[1])
+    eq(tf_full_2, oldfiles[2])
+    nvim_command('rshada!')
+    local oldfiles = meths.get_vvar('oldfiles')
+    eq(2, #oldfiles)
+    eq(testfilename, oldfiles[1]:sub(-#testfilename))
+    eq(testfilename_2, oldfiles[2]:sub(-#testfilename_2))
+    eq(tf_full, oldfiles[1])
+    eq(tf_full_2, oldfiles[2])
+  end)
+
+  it('is able to dump and restore jump list', function()
+    nvim_command('edit ' .. testfilename_2)
+    nvim_command('normal! G')
+    nvim_command('normal! gg')
+    nvim_command('edit ' .. testfilename)
+    nvim_command('normal! G')
+    nvim_command('normal! gg')
+    nvim_command('enew')
+    nvim_command('normal! gg')
+    local saved = redir_exec('jumps')
+    nvim_command('qall')
+    reset()
+    eq(saved, redir_exec('jumps'))
+  end)
+
+  it('is able to dump and restore jump list with different times (slow!)',
+  function()
+    nvim_command('edit ' .. testfilename_2)
+    nvim_command('sleep 2')
+    nvim_command('normal! G')
+    nvim_command('sleep 2')
+    nvim_command('normal! gg')
+    nvim_command('sleep 2')
+    nvim_command('edit ' .. testfilename)
+    nvim_command('sleep 2')
+    nvim_command('normal! G')
+    nvim_command('sleep 2')
+    nvim_command('normal! gg')
+    nvim_command('qall')
+    reset()
+    nvim_command('redraw')
+    nvim_command('edit ' .. testfilename)
+    eq(testfilename, funcs.bufname('%'))
+    eq(1, nvim_current_line())
+    nvim_command('execute "normal! \\<C-o>"')
+    eq(testfilename, funcs.bufname('%'))
+    eq(1, nvim_current_line())
+    nvim_command('execute "normal! \\<C-o>"')
+    eq(testfilename, funcs.bufname('%'))
+    eq(2, nvim_current_line())
+    nvim_command('execute "normal! \\<C-o>"')
+    eq(testfilename_2, funcs.bufname('%'))
+    eq(1, nvim_current_line())
+    nvim_command('execute "normal! \\<C-o>"')
+    eq(testfilename_2, funcs.bufname('%'))
+    eq(2, nvim_current_line())
+  end)
+
+  it('is able to dump and restore change list', function()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('normal! Gra')
+    nvim_command('normal! ggrb')
+    nvim_command('qall!')
+    reset()
+    nvim_command('edit ' .. testfilename)
+    nvim_command('normal! Gg;')
+    -- Note: without “sync” “commands” test has good changes to fail for unknown 
+    -- reason (in first eq expected 1 is compared with 2). Any command inserted 
+    -- causes this to work properly.
+    nvim_command('" sync')
+    eq(1, nvim_current_line())
+    nvim_command('normal! g;')
+    nvim_command('" sync 2')
+    eq(2, nvim_current_line())
+  end)
+end)

test/functional/shada/registers_spec.lua

@@ -0,0 +1,163 @@
+-- ShaDa registers saving/reading support
+local helpers = require('test.functional.helpers')
+local nvim_command, funcs, eq = helpers.command, helpers.funcs, helpers.eq
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, set_additional_cmd, clear =
+  shada_helpers.reset, shada_helpers.set_additional_cmd,
+  shada_helpers.clear
+
+local setreg = function(name, contents, typ)
+  if type(contents) == 'string' then
+    contents = {contents}
+  end
+  funcs.setreg(name, contents, typ)
+end
+
+local getreg = function(name)
+  return {
+    funcs.getreg(name, 1, 1),
+    funcs.getregtype(name),
+  }
+end
+
+describe('ShaDa support code', function()
+  before_each(reset)
+  after_each(clear)
+
+  it('is able to dump and restore registers and their type', function()
+    setreg('c', {'d', 'e', ''}, 'c')
+    setreg('l', {'a', 'b', 'cde'}, 'l')
+    setreg('b', {'bca', 'abc', 'cba'}, 'b3')
+    nvim_command('qall')
+    reset()
+    eq({{'d', 'e', ''}, 'v'}, getreg('c'))
+    eq({{'a', 'b', 'cde'}, 'V'}, getreg('l'))
+    eq({{'bca', 'abc', 'cba'}, '\0223'}, getreg('b'))
+  end)
+
+  it('does not dump registers with zero <', function()
+    nvim_command('set shada=\'0,<0')
+    setreg('c', {'d', 'e', ''}, 'c')
+    setreg('l', {'a', 'b', 'cde'}, 'l')
+    setreg('b', {'bca', 'abc', 'cba'}, 'b3')
+    nvim_command('qall')
+    reset()
+    eq({nil, ''}, getreg('c'))
+    eq({nil, ''}, getreg('l'))
+    eq({nil, ''}, getreg('b'))
+  end)
+
+  it('does restore registers with zero <', function()
+    setreg('c', {'d', 'e', ''}, 'c')
+    setreg('l', {'a', 'b', 'cde'}, 'l')
+    setreg('b', {'bca', 'abc', 'cba'}, 'b3')
+    set_additional_cmd('set shada=\'0,<0')
+    nvim_command('qall')
+    reset()
+    eq({{'d', 'e', ''}, 'v'}, getreg('c'))
+    eq({{'a', 'b', 'cde'}, 'V'}, getreg('l'))
+    eq({{'bca', 'abc', 'cba'}, '\0223'}, getreg('b'))
+  end)
+
+  it('does not dump registers with zero "', function()
+    nvim_command('set shada=\'0,\\"0')
+    setreg('c', {'d', 'e', ''}, 'c')
+    setreg('l', {'a', 'b', 'cde'}, 'l')
+    setreg('b', {'bca', 'abc', 'cba'}, 'b3')
+    nvim_command('qall')
+    reset()
+    eq({nil, ''}, getreg('c'))
+    eq({nil, ''}, getreg('l'))
+    eq({nil, ''}, getreg('b'))
+  end)
+
+  it('does restore registers with zero "', function()
+    setreg('c', {'d', 'e', ''}, 'c')
+    setreg('l', {'a', 'b', 'cde'}, 'l')
+    setreg('b', {'bca', 'abc', 'cba'}, 'b3')
+    set_additional_cmd('set shada=\'0,\\"0')
+    nvim_command('qall')
+    reset()
+    eq({{'d', 'e', ''}, 'v'}, getreg('c'))
+    eq({{'a', 'b', 'cde'}, 'V'}, getreg('l'))
+    eq({{'bca', 'abc', 'cba'}, '\0223'}, getreg('b'))
+  end)
+
+  it('does dump registers with zero ", but non-zero <', function()
+    nvim_command('set shada=\'0,\\"0,<50')
+    setreg('c', {'d', 'e', ''}, 'c')
+    setreg('l', {'a', 'b', 'cde'}, 'l')
+    setreg('b', {'bca', 'abc', 'cba'}, 'b3')
+    nvim_command('qall')
+    reset()
+    eq({{'d', 'e', ''}, 'v'}, getreg('c'))
+    eq({{'a', 'b', 'cde'}, 'V'}, getreg('l'))
+    eq({{'bca', 'abc', 'cba'}, '\0223'}, getreg('b'))
+  end)
+
+  it('does limit number of lines according to <', function()
+    nvim_command('set shada=\'0,<2')
+    setreg('o', {'d'}, 'c')
+    setreg('t', {'a', 'b', 'cde'}, 'l')
+    nvim_command('qall')
+    reset()
+    eq({{'d'}, 'v'}, getreg('o'))
+    eq({nil, ''}, getreg('t'))
+  end)
+
+  it('does limit number of lines according to "', function()
+    nvim_command('set shada=\'0,\\"2')
+    setreg('o', {'d'}, 'c')
+    setreg('t', {'a', 'b', 'cde'}, 'l')
+    nvim_command('qall')
+    reset()
+    eq({{'d'}, 'v'}, getreg('o'))
+    eq({nil, ''}, getreg('t'))
+  end)
+
+  it('does limit number of lines according to < rather then "', function()
+    nvim_command('set shada=\'0,\\"2,<3')
+    setreg('o', {'d'}, 'c')
+    setreg('t', {'a', 'b', 'cde'}, 'l')
+    setreg('h', {'abc', 'acb', 'bac', 'bca', 'cab', 'cba'}, 'b3')
+    nvim_command('qall')
+    reset()
+    eq({{'d'}, 'v'}, getreg('o'))
+    eq({{'a', 'b', 'cde'}, 'V'}, getreg('t'))
+    eq({nil, ''}, getreg('h'))
+  end)
+
+  it('dumps and loads register correctly when &encoding is not UTF-8',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    setreg('e', {'\171'}, 'c')
+    nvim_command('qall')
+    reset()
+    eq({{'\171'}, 'v'}, getreg('e'))
+  end)
+
+  it('dumps and loads history correctly when &encoding /= UTF-8 when dumping',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    setreg('e', {'\171'}, 'c')
+    set_additional_cmd('')
+    nvim_command('qall')
+    reset()
+    eq({{'«'}, 'v'}, getreg('e'))
+  end)
+
+  it('dumps and loads history correctly when &encoding /= UTF-8 when loading',
+  function()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    setreg('e', {'«'}, 'c')
+    set_additional_cmd('set encoding=latin1')
+    nvim_command('qall')
+    reset()
+    eq({{'\171'}, 'v'}, getreg('e'))
+  end)
+end)

test/functional/shada/shada_spec.lua

@@ -0,0 +1,232 @@
+-- Other ShaDa tests
+local helpers = require('test.functional.helpers')
+local meths, nvim_command, funcs, eq =
+  helpers.meths, helpers.command, helpers.funcs, helpers.eq
+local write_file, spawn, set_session, nvim_prog, exc_exec =
+  helpers.write_file, helpers.spawn, helpers.set_session, helpers.nvim_prog,
+  helpers.exc_exec
+local lfs = require('lfs')
+local paths = require('test.config.paths')
+
+local msgpack = require('MessagePack')
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, clear, get_shada_rw =
+  shada_helpers.reset, shada_helpers.clear, shada_helpers.get_shada_rw
+local read_shada_file = shada_helpers.read_shada_file
+
+local wshada, _, shada_fname, clean =
+  get_shada_rw('Xtest-functional-shada-shada.shada')
+
+describe('ShaDa support code', function()
+  before_each(reset)
+  after_each(function()
+    clear()
+    clean()
+  end)
+
+  it('preserves `s` item size limit with unknown entries', function()
+    wshada('\100\000\207\000\000\000\000\000\000\004\000\218\003\253' .. ('-'):rep(1024 - 3)
+           .. '\100\000\207\000\000\000\000\000\000\004\001\218\003\254' .. ('-'):rep(1025 - 3))
+    eq(0, exc_exec('wshada ' .. shada_fname))
+    local found = 0
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 100 then
+        found = found + 1
+      end
+    end
+    eq(2, found)
+    eq(0, exc_exec('set shada-=s10 shada+=s1'))
+    eq(0, exc_exec('wshada ' .. shada_fname))
+    found = 0
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 100 then
+        found = found + 1
+      end
+    end
+    eq(1, found)
+  end)
+
+  it('preserves `s` item size limit with instance history entries', function()
+    local hist1 = ('-'):rep(1024 - 5)
+    local hist2 = ('-'):rep(1025 - 5)
+    nvim_command('set shada-=s10 shada+=s1')
+    funcs.histadd(':', hist1)
+    funcs.histadd(':', hist2)
+    eq(0, exc_exec('wshada ' .. shada_fname))
+    local found = 0
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 4 then
+        found = found + 1
+        eq(hist1, v.value[2])
+      end
+    end
+    eq(1, found)
+  end)
+
+  it('leaves .tmp.a in-place when there is error in original ShaDa', function()
+    wshada('Some text file')
+    eq('Vim(wshada):E576: Error while reading ShaDa file: last entry specified that it occupies 109 bytes, but file ended earlier', exc_exec('wshada ' .. shada_fname))
+    eq(1, read_shada_file(shada_fname .. '.tmp.a')[1].type)
+  end)
+
+  it('does not leave .tmp.a in-place when there is error in original ShaDa, but writing with bang', function()
+    wshada('Some text file')
+    eq(0, exc_exec('wshada! ' .. shada_fname))
+    eq(1, read_shada_file(shada_fname)[1].type)
+    eq(nil, lfs.attributes(shada_fname .. '.tmp.a'))
+  end)
+
+  it('leaves .tmp.b in-place when there is error in original ShaDa and it has .tmp.a', function()
+    wshada('Some text file')
+    eq('Vim(wshada):E576: Error while reading ShaDa file: last entry specified that it occupies 109 bytes, but file ended earlier', exc_exec('wshada ' .. shada_fname))
+    eq('Vim(wshada):E576: Error while reading ShaDa file: last entry specified that it occupies 109 bytes, but file ended earlier', exc_exec('wshada ' .. shada_fname))
+    eq(1, read_shada_file(shada_fname .. '.tmp.a')[1].type)
+    eq(1, read_shada_file(shada_fname .. '.tmp.b')[1].type)
+  end)
+
+  it('leaves .tmp.z in-place when there is error in original ShaDa and it has .tmp.a … .tmp.x', function()
+    wshada('Some text file')
+    local i = ('a'):byte()
+    while i < ('z'):byte() do
+      write_file(shada_fname .. ('.tmp.%c'):format(i), 'Some text file', true)
+      i = i + 1
+    end
+    eq('Vim(wshada):E576: Error while reading ShaDa file: last entry specified that it occupies 109 bytes, but file ended earlier', exc_exec('wshada ' .. shada_fname))
+    eq(1, read_shada_file(shada_fname .. '.tmp.z')[1].type)
+  end)
+
+  it('errors out when there are .tmp.a … .tmp.z ShaDa files', function()
+    wshada('')
+    local i = ('a'):byte()
+    while i <= ('z'):byte() do
+      write_file(shada_fname .. ('.tmp.%c'):format(i), '', true)
+      i = i + 1
+    end
+    eq('Vim(wshada):E138: All Xtest-functional-shada-shada.shada.tmp.X files exist, cannot write ShaDa file!', exc_exec('wshada ' .. shada_fname))
+  end)
+
+  it('reads correctly various timestamps', function()
+    local mpack = {
+      '\100',  -- Positive fixnum 100
+      '\204\255',  -- uint 8 255
+      '\205\010\003',  -- uint 16 2563
+      '\206\255\010\030\004',  -- uint 32 4278853124
+      '\207\005\100\060\250\255\010\030\004',  -- uint 64 388502516579048964
+    }
+    local s = '\100'
+    local e = '\001\192'
+    wshada(s .. table.concat(mpack, e .. s) .. e)
+    eq(0, exc_exec('wshada ' .. shada_fname))
+    local found = 0
+    local typ = select(2, msgpack.unpacker(s)())
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == typ then
+        found = found + 1
+        eq(select(2, msgpack.unpacker(mpack[found])()), v.timestamp)
+      end
+    end
+    eq(#mpack, found)
+  end)
+
+  it('does not write NONE file', function()
+    local session = spawn({nvim_prog, '-u', 'NONE', '-i', 'NONE', '--embed',
+                           '--cmd', 'qall'}, true)
+    session:exit(0)
+    eq(nil, lfs.attributes('NONE'))
+    eq(nil, lfs.attributes('NONE.tmp.a'))
+  end)
+
+  it('does not read NONE file', function()
+    write_file('NONE', '\005\001\015\131\161na\162rX\194\162rc\145\196\001-')
+    local session = spawn({nvim_prog, '-u', 'NONE', '-i', 'NONE', '--embed'},
+                          true)
+    set_session(session)
+    eq('', funcs.getreg('a'))
+    session:exit(0)
+    os.remove('NONE')
+  end)
+
+  local marklike = {[7]=true, [8]=true, [10]=true, [11]=true}
+  local find_file = function(fname)
+    local found = {}
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if marklike[v.type] and v.value.f == fname then
+        found[v.type] = (found[v.type] or 0) + 1
+      elseif v.type == 9 then
+        for _, b in ipairs(v.value) do
+          if b.f == fname then
+            found[v.type] = (found[v.type] or 0) + 1
+          end
+        end
+      end
+    end
+    return found
+  end
+
+  it('correctly uses shada-r option', function()
+    meths.set_var('__home', paths.test_source_path)
+    nvim_command('let $HOME = __home')
+    nvim_command('unlet __home')
+    nvim_command('edit ~/README.md')
+    nvim_command('normal! GmAggmaAabc')
+    nvim_command('undo')
+    nvim_command('set shada+=%')
+    nvim_command('wshada! ' .. shada_fname)
+    local readme_fname = paths.test_source_path .. '/README.md'
+    eq({[7]=1, [8]=2, [9]=1, [10]=4, [11]=1}, find_file(readme_fname))
+    nvim_command('set shada+=r~')
+    nvim_command('wshada! ' .. shada_fname)
+    eq({}, find_file(readme_fname))
+    nvim_command('set shada-=r~')
+    nvim_command('wshada! ' .. shada_fname)
+    eq({[7]=1, [8]=2, [9]=1, [10]=4, [11]=1}, find_file(readme_fname))
+    nvim_command('set shada+=r' .. paths.test_source_path)
+    nvim_command('wshada! ' .. shada_fname)
+    eq({}, find_file(readme_fname))
+  end)
+
+  it('correctly ignores case with shada-r option', function()
+    local pwd = funcs.getcwd()
+    local relfname = 'абв/test'
+    local fname = pwd .. '/' .. relfname
+    meths.set_var('__fname', fname)
+    nvim_command('silent! edit `=__fname`')
+    funcs.setline(1, {'a', 'b', 'c', 'd'})
+    nvim_command('normal! GmAggmaAabc')
+    nvim_command('undo')
+    nvim_command('set shada+=%')
+    nvim_command('wshada! ' .. shada_fname)
+    eq({[7]=1, [8]=2, [9]=1, [10]=4, [11]=2}, find_file(fname))
+    nvim_command('set shada+=r' .. pwd .. '/АБВ')
+    nvim_command('wshada! ' .. shada_fname)
+    eq({}, find_file(fname))
+  end)
+
+  it('is able to set &shada after &viminfo', function()
+    meths.set_option('viminfo', '\'10')
+    eq('\'10', meths.get_option('viminfo'))
+    eq('\'10', meths.get_option('shada'))
+    meths.set_option('shada', '')
+    eq('', meths.get_option('viminfo'))
+    eq('', meths.get_option('shada'))
+  end)
+
+  it('is able to set all& after setting &shada', function()
+    meths.set_option('shada', '\'10')
+    eq('\'10', meths.get_option('viminfo'))
+    eq('\'10', meths.get_option('shada'))
+    nvim_command('set all&')
+    eq('!,\'100,<50,s10,h', meths.get_option('viminfo'))
+    eq('!,\'100,<50,s10,h', meths.get_option('shada'))
+  end)
+
+  it('is able to set &shada after &viminfo using :set', function()
+    nvim_command('set viminfo=\'10')
+    eq('\'10', meths.get_option('viminfo'))
+    eq('\'10', meths.get_option('shada'))
+    nvim_command('set shada=')
+    eq('', meths.get_option('viminfo'))
+    eq('', meths.get_option('shada'))
+  end)
+end)

test/functional/shada/variables_spec.lua

@@ -0,0 +1,139 @@
+-- ShaDa variables saving/reading support
+local helpers = require('test.functional.helpers')
+local meths, funcs, nvim_command, eq =
+  helpers.meths, helpers.funcs, helpers.command, helpers.eq
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, set_additional_cmd, clear =
+  shada_helpers.reset, shada_helpers.set_additional_cmd,
+  shada_helpers.clear
+
+describe('ShaDa support code', function()
+  before_each(reset)
+  after_each(clear)
+
+  it('is able to dump and read back string variable', function()
+    meths.set_var('STRVAR', 'foo')
+    nvim_command('set shada+=!')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada+=!')
+    nvim_command('rshada')
+    eq('foo', meths.get_var('STRVAR'))
+  end)
+
+  local autotest = function(tname, varname, varval)
+    it('is able to dump and read back ' .. tname .. ' variable automatically',
+    function()
+      set_additional_cmd('set shada+=!')
+      reset()
+      meths.set_var(varname, varval)
+      -- Exit during `reset` is not a regular exit: it does not write shada 
+      -- automatically
+      nvim_command('qall')
+      reset()
+      eq(varval, meths.get_var(varname))
+    end)
+  end
+
+  autotest('string', 'STRVAR', 'foo')
+  autotest('number', 'NUMVAR', 42)
+  autotest('float', 'FLTVAR', 42.5)
+  autotest('dictionary', 'DCTVAR', {a=10})
+  autotest('list', 'LSTVAR', {{a=10}, {b=10.5}, {c='str'}})
+
+  it('does not read back variables without `!` in &shada', function()
+    meths.set_var('STRVAR', 'foo')
+    nvim_command('set shada+=!')
+    nvim_command('wshada')
+    set_additional_cmd('set shada-=!')
+    reset()
+    nvim_command('rshada')
+    eq(0, funcs.exists('g:STRVAR'))
+  end)
+
+  it('does not dump variables without `!` in &shada', function()
+    nvim_command('set shada-=!')
+    meths.set_var('STRVAR', 'foo')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada+=!')
+    nvim_command('rshada')
+    eq(0, funcs.exists('g:STRVAR'))
+  end)
+
+  it('does not dump session variables', function()
+    nvim_command('set shada+=!')
+    meths.set_var('StrVar', 'foo')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada+=!')
+    nvim_command('rshada')
+    eq(0, funcs.exists('g:StrVar'))
+  end)
+
+  it('does not dump regular variables', function()
+    nvim_command('set shada+=!')
+    meths.set_var('str_var', 'foo')
+    nvim_command('wshada')
+    reset()
+    nvim_command('set shada+=!')
+    nvim_command('rshada')
+    eq(0, funcs.exists('g:str_var'))
+  end)
+
+  it('dumps and loads variables correctly when &encoding is not UTF-8',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    meths.set_var('STRVAR', '\171')
+    meths.set_var('LSTVAR', {'\171'})
+    meths.set_var('DCTVAR', {['\171']='\171'})
+    meths.set_var('NESTEDVAR', {['\171']={{'\171'}, {['\171']='\171'},
+                                {a='Test'}}})
+    nvim_command('qall')
+    reset()
+    eq('\171', meths.get_var('STRVAR'))
+    eq({'\171'}, meths.get_var('LSTVAR'))
+    eq({['\171']='\171'}, meths.get_var('DCTVAR'))
+    eq({['\171']={{'\171'}, {['\171']='\171'}, {a='Test'}}},
+       meths.get_var('NESTEDVAR'))
+  end)
+
+  it('dumps and loads variables correctly when &encoding /= UTF-8 when dumping',
+  function()
+    set_additional_cmd('set encoding=latin1')
+    reset()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    meths.set_var('STRVAR', '\171')
+    meths.set_var('LSTVAR', {'\171'})
+    meths.set_var('DCTVAR', {['\171']='\171'})
+    meths.set_var('NESTEDVAR', {['\171']={{'\171'}, {['\171']='\171'},
+                                {a='Test'}}})
+    set_additional_cmd('')
+    nvim_command('qall')
+    reset()
+    eq('«', meths.get_var('STRVAR'))
+    eq({'«'}, meths.get_var('LSTVAR'))
+    eq({['«']='«'}, meths.get_var('DCTVAR'))
+    eq({['«']={{'«'}, {['«']='«'}, {a='Test'}}}, meths.get_var('NESTEDVAR'))
+  end)
+
+  it('dumps and loads variables correctly when &encoding /= UTF-8 when loading',
+  function()
+    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    meths.set_var('STRVAR', '«')
+    meths.set_var('LSTVAR', {'«'})
+    meths.set_var('DCTVAR', {['«']='«'})
+    meths.set_var('NESTEDVAR', {['«']={{'«'}, {['«']='«'}, {a='Test'}}})
+    set_additional_cmd('set encoding=latin1')
+    nvim_command('qall')
+    reset()
+    eq('\171', meths.get_var('STRVAR'))
+    eq({'\171'}, meths.get_var('LSTVAR'))
+    eq({['\171']='\171'}, meths.get_var('DCTVAR'))
+    eq({['\171']={{'\171'}, {['\171']='\171'}, {a='Test'}}},
+       meths.get_var('NESTEDVAR'))
+  end)
+end)