vim-patch:7.4.1142

Problem:    Cannot define keyword characters for a syntax file.
Solution:   Add the ":syn iskeyword" command. (Christian Brabandt)

b8060fe862

runtime/doc/options.txt

@@ -1,4 +1,4 @@
-*options.txt*	For Vim version 7.4.  Last change: 2016 Jan 03
+*options.txt*	For Vim version 7.4.  Last change: 2016 Jan 19
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -3696,6 +3696,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	'*', '"' and '|' (so that CTRL-] on a command finds the help for that
 	command).
 	When the 'lisp' option is on the '-' character is always included.
+	This option also influences syntax highlighting, unless the syntax
+	uses |:syn-iskeyword|.
 
 						*'isprint'* *'isp'*
 'isprint' 'isp'	string	(default: "@,161-255")

runtime/doc/syntax.txt

@@ -1,4 +1,4 @@
-*syntax.txt*	For Vim version 7.4.  Last change: 2015 Dec 19
+*syntax.txt*	For Vim version 7.4.  Last change: 2016 Jan 19
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -3422,6 +3422,32 @@ SPELL CHECKING						*:syn-spell*
 
 	To activate spell checking the 'spell' option must be set.
 
+SYNTAX ISKEYWORD SETTING				*:syn-iskeyword*
+
+:sy[ntax] iskeyword [clear | {option}]
+	This defines the keyword characters.  It's like the 'iskeyword' option
+	for but only applies to syntax highlighting.
+
+	clear:		Syntax specific iskeyword setting is disabled and the
+			buffer-local 'iskeyword' setting is used.
+	{option}        Set the syntax 'iskeyword' option to a new value. 
+
+	Example: >
+  :syntax iskeyword @,48-57,192-255,$,_
+<
+	This would set the syntax specific iskeyword option to include all
+	alphabetic characters, plus the numeric characters, all accented
+	characters and also includes the "_" and the "$".
+
+	If no argument is given, the current value will be output.
+
+	Setting this option influences what |/\k| matches in syntax patterns
+	and also determines where |:syn-keywords| will be checked for a new
+	match.
+
+	It is recommended when writing syntax files, to use this command
+	to the correct value for the specific syntax language and not change
+	the 'iskeyword' option.
 
 DEFINING KEYWORDS					*:syn-keyword*
 
@@ -3453,6 +3479,7 @@ DEFINING KEYWORDS					*:syn-keyword*
 	isn't, the keyword will never be recognized.
 	Multi-byte characters can also be used.  These do not have to be in
 	'iskeyword'.
+	See |:syn-iskeyword| for defining syntax specific iskeyword settings.
 
 	A keyword always has higher priority than a match or region, the
 	keyword is used if more than one item matches.	Keywords do not nest

src/nvim/buffer.c

@@ -1534,6 +1534,7 @@ void free_buf_options(buf_T *buf, int free_p_ff)
   clear_string_option(&buf->b_p_cms);
   clear_string_option(&buf->b_p_nf);
   clear_string_option(&buf->b_p_syn);
+  clear_string_option(&buf->b_s.b_syn_isk);
   clear_string_option(&buf->b_s.b_p_spc);
   clear_string_option(&buf->b_s.b_p_spf);
   vim_regfree(buf->b_s.b_cap_prog);

src/nvim/buffer_defs.h

@@ -447,6 +447,8 @@ typedef struct {
   char_u      *b_p_spf;         /* 'spellfile' */
   char_u      *b_p_spl;         /* 'spelllang' */
   int b_cjk;                    /* all CJK letters as OK */
+  char_u b_syn_chartab[32];     // syntax iskeyword option
+  char_u *b_syn_isk;            // iskeyword option
 } synblock_T;
 
 

src/nvim/option.c

@@ -2129,6 +2129,7 @@ void check_buf_options(buf_T *buf)
   check_string_option(&buf->b_p_nf);
   check_string_option(&buf->b_p_qe);
   check_string_option(&buf->b_p_syn);
+  check_string_option(&buf->b_s.b_syn_isk);
   check_string_option(&buf->b_s.b_p_spc);
   check_string_option(&buf->b_s.b_p_spf);
   check_string_option(&buf->b_s.b_p_spl);
@@ -5606,6 +5607,7 @@ void buf_copy_options(buf_T *buf, int flags)
       /* Don't copy 'syntax', it must be set */
       buf->b_p_syn = empty_option;
       buf->b_p_smc = p_smc;
+      buf->b_s.b_syn_isk = empty_option;
       buf->b_s.b_p_spc = vim_strsave(p_spc);
       (void)compile_cap_prog(&buf->b_s);
       buf->b_s.b_p_spf = vim_strsave(p_spf);

src/nvim/syntax.c

@@ -812,19 +812,39 @@ static void syn_sync(win_T *wp, linenr_T start_lnum, synstate_T *last_valid)
   validate_current_state();
 }
 
+static void save_chartab(char_u *chartab)
+{
+  if (syn_block->b_syn_isk != empty_option) {
+    memmove(chartab, syn_buf->b_chartab, (size_t)32);
+    memmove(syn_buf->b_chartab, syn_win->w_s->b_syn_chartab, (size_t)32);
+  }
+}
+
+static void restore_chartab(char_u *chartab)
+{
+  if (syn_win->w_s->b_syn_isk != empty_option) {
+    memmove(syn_buf->b_chartab, chartab, (size_t)32);
+  }
+}
+
 /*
  * Return TRUE if the line-continuation pattern matches in line "lnum".
  */
 static int syn_match_linecont(linenr_T lnum)
 {
-  regmmatch_T regmatch;
-
   if (syn_block->b_syn_linecont_prog != NULL) {
+    regmmatch_T regmatch;
+    // chartab array for syn iskeyword
+    char_u buf_chartab[32];
+    save_chartab(buf_chartab);
+
     regmatch.rmm_ic = syn_block->b_syn_linecont_ic;
     regmatch.regprog = syn_block->b_syn_linecont_prog;
     int r = syn_regexec(&regmatch, lnum, (colnr_T)0,
                         IF_SYN_TIME(&syn_block->b_syn_linecont_time));
     syn_block->b_syn_linecont_prog = regmatch.regprog;
+
+    restore_chartab(buf_chartab);
     return r;
   }
   return FALSE;
@@ -1617,6 +1637,7 @@ syn_current_attr (
   lpos_T pos;
   int lc_col;
   reg_extmatch_T *cur_extmatch = NULL;
+  char_u      buf_chartab[32];  // chartab array for syn iskeyword
   char_u      *line;            /* current line.  NOTE: becomes invalid after
                                    looking for a pattern match! */
 
@@ -1668,6 +1689,9 @@ syn_current_attr (
    * avoid matching the same item in the same position twice. */
   ga_init(&zero_width_next_ga, (int)sizeof(int), 10);
 
+  // use syntax iskeyword option
+  save_chartab(buf_chartab);
+
   /*
    * Repeat matching keywords and patterns, to find contained items at the
    * same column.  This stops when there are no extra matches at the current
@@ -1992,6 +2016,8 @@ syn_current_attr (
 
   } while (found_match);
 
+  restore_chartab(buf_chartab);
+
   /*
    * Use attributes from the current state, if within its highlighting.
    * If not, use attributes from the current-but-one state, etc.
@@ -2523,6 +2549,7 @@ find_endpos (
   lpos_T pos;
   char_u      *line;
   int had_match = FALSE;
+  char_u buf_chartab[32];  // chartab array for syn option iskeyword
 
   /* just in case we are invoked for a keyword */
   if (idx < 0)
@@ -2565,6 +2592,10 @@ find_endpos (
   matchcol = startpos->col;     /* start looking for a match at sstart */
   start_idx = idx;              /* remember the first END pattern. */
   best_regmatch.startpos[0].col = 0;            /* avoid compiler warning */
+
+  // use syntax iskeyword option
+  save_chartab(buf_chartab);
+
   for (;; ) {
     /*
      * Find end pattern that matches first after "matchcol".
@@ -2707,6 +2738,8 @@ find_endpos (
   if (!had_match)
     m_endpos->lnum = 0;
 
+  restore_chartab(buf_chartab);
+
   /* Remove external matches. */
   unref_extmatch(re_extmatch_in);
   re_extmatch_in = NULL;
@@ -3027,6 +3060,46 @@ static void syn_cmd_spell(exarg_T *eap, int syncing)
   redraw_win_later(curwin, NOT_VALID);
 }
 
+/// Handle ":syntax iskeyword" command.
+static void syn_cmd_iskeyword(exarg_T *eap, int syncing)
+{
+  char_u *arg = eap->arg;
+  char_u save_chartab[32];
+  char_u *save_isk;
+
+  if (eap->skip) {
+    return;
+  }
+
+  arg = skipwhite(arg);
+  if (*arg == NUL) {
+    MSG_PUTS("\n");
+    MSG_PUTS(_("syntax iskeyword "));
+    if (curwin->w_s->b_syn_isk != empty_option) {
+      msg_outtrans(curwin->w_s->b_syn_isk);
+    } else {
+      msg_outtrans((char_u *)"not set");
+    }
+  } else {
+    if (STRNICMP(arg, "clear", 5) == 0) {
+      memmove(curwin->w_s->b_syn_chartab, curbuf->b_chartab, (size_t)32);
+      clear_string_option(&curwin->w_s->b_syn_isk);
+    } else {
+      memmove(save_chartab, curbuf->b_chartab, (size_t)32);
+      save_isk = curbuf->b_p_isk;
+      curbuf->b_p_isk = vim_strsave(arg);
+
+      buf_init_chartab(curbuf, false);
+      memmove(curwin->w_s->b_syn_chartab, curbuf->b_chartab, (size_t)32);
+      memmove(curbuf->b_chartab, save_chartab, (size_t)32);
+      clear_string_option(&curwin->w_s->b_syn_isk);
+      curwin->w_s->b_syn_isk = curbuf->b_p_isk;
+      curbuf->b_p_isk = save_isk;
+    }
+  }
+  redraw_win_later(curwin, NOT_VALID);
+}
+
 /*
  * Clear all syntax info for one buffer.
  */
@@ -3065,6 +3138,7 @@ void syntax_clear(synblock_T *block)
   xfree(block->b_syn_linecont_pat);
   block->b_syn_linecont_pat = NULL;
   block->b_syn_folditems = 0;
+  clear_string_option(&block->b_syn_isk);
 
   /* free the stored states */
   syn_stack_free_all(block);
@@ -3107,6 +3181,7 @@ static void syntax_sync_clear(void)
   curwin->w_s->b_syn_linecont_prog = NULL;
   xfree(curwin->w_s->b_syn_linecont_pat);
   curwin->w_s->b_syn_linecont_pat = NULL;
+  clear_string_option(&curwin->w_s->b_syn_isk);
 
   syn_stack_free_all(curwin->w_s);              /* Need to recompute all syntax. */
 }
@@ -3271,6 +3346,7 @@ static void syn_cmd_reset(exarg_T *eap, int syncing)
 {
   eap->nextcmd = check_nextcmd(eap->arg);
   if (!eap->skip) {
+    clear_string_option(&curwin->w_s->b_syn_isk);
     set_internal_string_var((char_u *)"syntax_cmd", (char_u *)"reset");
     do_cmdline_cmd("runtime! syntax/syncolor.vim");
     do_unlet((char_u *)"g:syntax_cmd", TRUE);
@@ -5369,6 +5445,7 @@ static struct subcommand subcommands[] =
   {"conceal",         syn_cmd_conceal},
   {"enable",          syn_cmd_enable},
   {"include",         syn_cmd_include},
+  {"iskeyword",       syn_cmd_iskeyword},
   {"keyword",         syn_cmd_keyword},
   {"list",            syn_cmd_list},
   {"manual",          syn_cmd_manual},
@@ -5434,6 +5511,7 @@ void ex_ownsyntax(exarg_T *eap)
     clear_string_option(&curwin->w_s->b_p_spc);
     clear_string_option(&curwin->w_s->b_p_spf);
     clear_string_option(&curwin->w_s->b_p_spl);
+    clear_string_option(&curwin->w_s->b_syn_isk);
   }
 
   /* save value of b:current_syntax */

src/nvim/testdir/Makefile

@@ -39,6 +39,7 @@ NEW_TESTS = \
 	    test_cursor_func.res \
 	    test_help_tagjump.res \
 	    test_menu.res \
+	    test_syntax.res \
 	    test_timers.res \
 	    test_viml.res \
 	    test_alot.res

src/nvim/testdir/test_syntax.vim

@@ -0,0 +1,63 @@
+" Test for syntax and syntax iskeyword option
+
+func GetSyntaxItem(pat)
+  let c = ''
+  let a = ['a', getreg('a'), getregtype('a')]
+  0
+  redraw!
+  call search(a:pat, 'W')
+  let synid = synID(line('.'), col('.'), 1)
+  while synid == synID(line('.'), col('.'), 1)
+    norm! v"ay
+    " stop at whitespace
+    if @a =~# '\s'
+      break
+    endif
+    let c .= @a
+    norm! l
+  endw
+  call call('setreg', a)
+  0
+  return c
+endfunc
+
+func Test_syn_iskeyword()
+  new
+  call setline(1, [
+	\ 'CREATE TABLE FOOBAR(',
+	\ '    DLTD_BY VARCHAR2(100)',
+	\ ');',
+  	\ ''])
+
+  syntax on
+  set ft=sql
+  syn match SYN /C\k\+\>/
+  hi link SYN ErrorMsg
+  call assert_equal('DLTD_BY', GetSyntaxItem('DLTD'))
+  /\<D\k\+\>/:norm! ygn
+  call assert_equal('DLTD_BY', @0)
+  redir @c
+  syn iskeyword
+  redir END
+  call assert_equal("\nsyntax iskeyword not set", @c)
+
+  syn iskeyword @,48-57,_,192-255
+  redir @c
+  syn iskeyword
+  redir END
+  call assert_equal("\nsyntax iskeyword @,48-57,_,192-255", @c)
+
+  setlocal isk-=_
+  call assert_equal('DLTD_BY', GetSyntaxItem('DLTD'))
+  /\<D\k\+\>/:norm! ygn
+  let b2=@0
+  call assert_equal('DLTD', @0)
+
+  syn iskeyword clear
+  redir @c
+  syn iskeyword
+  redir END
+  call assert_equal("\nsyntax iskeyword not set", @c)
+
+  quit!
+endfunc

src/nvim/version.c

@@ -550,7 +550,7 @@ static int included_patches[] = {
   // 1145 NA
   1144,
   1143,
-  // 1142,
+  1142,
   1141,
   // 1140,
   // 1139 NA