Support Kitty Unicode placeholder image transmission

- Parse Kitty graphics commands wrapped in tmux passthrough sequences.
  - Accept unpadded base64 payloads emitted by icat.
  - Handle chunked transfers whose final chunk omits m.
  - Track virtual image placements.
  - Convert Unicode placeholders into shared image cells for Kitty and SIXEL clients.
  - Add regression coverage for wrapped, chunked, unpadded input.

image-kitty.c

@@ -83,6 +83,7 @@ struct kitty_state {
 	u_int	 image_id;
 	u_int	 quiet;
 	int	 no_cursor;
+	int	 virtual;
 	u_int	 data_size;
 	int	 more;
 
@@ -93,6 +94,7 @@ struct kitty_state {
 struct kitty_source {
 	u_int			 app_id;
 	u_int			 server_id;
+	u_int			 virtual_id;
 	struct kitty_source	*next;
 };
 
@@ -443,6 +445,7 @@ kitty_control(struct kitty_state *ks, const u_char *buf, size_t len)
 		case 'm':
 		case 'S':
 		case 'C':
+		case 'U':
 			if (kitty_number((const char *)value, valuelen,
 			    &number) != 0)
 				return (-1);
@@ -461,6 +464,7 @@ kitty_control(struct kitty_state *ks, const u_char *buf, size_t len)
 			case 'm': ks->more = (number != 0); break;
 			case 'S': ks->data_size = number; break;
 			case 'C': ks->no_cursor = (number != 0); break;
+			case 'U': ks->virtual = (number != 0); break;
 			}
 			break;
 		}
@@ -488,6 +492,8 @@ kitty_free_state(void *state)
 	kitty_state_free(kc->transfer);
 	for (source = kc->sources; source != NULL; source = next) {
 		next = source->next;
+		if (source->virtual_id != 0)
+			image_free(source->virtual_id);
 		image_free(source->server_id);
 		free(source);
 	}
@@ -519,12 +525,31 @@ kitty_source_set(struct kitty_context *kc, u_int id, struct image *im)
 		source->app_id = id;
 		source->next = kc->sources;
 		kc->sources = source;
-	} else
+	} else {
+		if (source->virtual_id != 0) {
+			image_free(source->virtual_id);
+			source->virtual_id = 0;
+		}
 		image_free(source->server_id);
+	}
 	image_ref(im->id);
 	source->server_id = im->id;
 }
 
+static void
+kitty_virtual_set(struct kitty_context *kc, u_int id, struct image *im)
+{
+	struct kitty_source	*source;
+
+	source = kitty_source_find(kc, id);
+	if (source == NULL)
+		return;
+	if (source->virtual_id != 0)
+		image_free(source->virtual_id);
+	image_ref(im->id);
+	source->virtual_id = im->id;
+}
+
 static struct image *
 kitty_source_remove(struct kitty_context *kc, u_int id)
 {
@@ -538,6 +563,8 @@ kitty_source_remove(struct kitty_context *kc, u_int id)
 		if (im != NULL)
 			image_ref(im->id);
 		*pp = source->next;
+		if (source->virtual_id != 0)
+			image_free(source->virtual_id);
 		image_free(source->server_id);
 		free(source);
 		return (im);
@@ -738,6 +765,7 @@ kitty_parse_image(void **state, const u_char *buf, size_t len, u_int xpixel,
 		ks->format = 32;
 		ks->medium = 'd';
 	}
+	ks->more = 0;
 	*image_id = ks->image_id;
 	*quiet = ks->quiet;
 	*action = ks->action;
@@ -761,6 +789,16 @@ kitty_parse_image(void **state, const u_char *buf, size_t len, u_int xpixel,
 		if (source == NULL) {
 			*status = KITTY_PARSE_MISSING;
 			im = NULL;
+		} else if (ks->virtual) {
+			im = kitty_place_image(source, ks, xpixel, ypixel);
+			image_free(source->id);
+			if (im != NULL) {
+				kitty_virtual_set(kc, ks->image_id, im);
+				image_free(im->id);
+				im = NULL;
+				*action = 'u';
+				*status = KITTY_PARSE_OK;
+			}
 		} else {
 			im = kitty_place_image(source, ks, xpixel, ypixel);
 			image_free(source->id);
@@ -814,8 +852,8 @@ kitty_parse_image(void **state, const u_char *buf, size_t len, u_int xpixel,
 			decoded = uncompressed;
 			decodedlen = uncompressedlen;
 		} else if (ks->compression != '\0') {
-				free(decoded);
-				goto fail;
+			free(decoded);
+			goto fail;
 		}
 		pixels = image_png_decode(decoded, decodedlen, IMAGE_SIZE_LIMIT,
 		    &ks->width, &ks->height);
@@ -848,12 +886,23 @@ kitty_parse_image(void **state, const u_char *buf, size_t len, u_int xpixel,
 		*status = KITTY_PARSE_OK;
 		if (ks->action != 'q')
 			kitty_source_set(kc, ks->image_id, source);
-		if (ks->action == 'T') {
+		if (ks->action == 'T' && !ks->virtual) {
 			im = kitty_place_image(source, ks, xpixel, ypixel);
 			if (im == NULL)
 				*status = KITTY_PARSE_ERROR;
-		} else
+		} else if (ks->virtual) {
+			im = kitty_place_image(source, ks, xpixel, ypixel);
+			if (im == NULL)
+				*status = KITTY_PARSE_ERROR;
+			else {
+				kitty_virtual_set(kc, ks->image_id, im);
+				image_free(im->id);
+				im = NULL;
+				*action = 'u';
+			}
+		} else {
 			im = NULL;
+		}
 		image_free(source->id);
 	}
 	kitty_state_free(ks);
@@ -864,3 +913,128 @@ fail:
 	kitty_state_free(ks);
 	return (NULL);
 }
+
+static int
+kitty_placeholder_character(const u_char *data, size_t size, size_t *offset,
+    uint32_t *value)
+{
+	u_char	 ch;
+	u_int	 needed, i;
+	uint32_t result;
+
+	if (*offset >= size)
+		return (0);
+	ch = data[(*offset)++];
+	if (ch < 0x80) {
+		*value = ch;
+		return (1);
+	}
+	if ((ch & 0xe0) == 0xc0) {
+		needed = 1;
+		result = ch & 0x1f;
+	} else if ((ch & 0xf0) == 0xe0) {
+		needed = 2;
+		result = ch & 0x0f;
+	} else if ((ch & 0xf8) == 0xf0) {
+		needed = 3;
+		result = ch & 0x07;
+	} else
+		return (0);
+	if (needed > size - *offset)
+		return (0);
+	for (i = 0; i < needed; i++) {
+		ch = data[(*offset)++];
+		if ((ch & 0xc0) != 0x80)
+			return (0);
+		result = (result << 6)|(ch & 0x3f);
+	}
+	*value = result;
+	return (1);
+}
+
+static int
+kitty_placeholder_index(uint32_t value, u_int *index)
+{
+	u_int	i;
+
+	for (i = 0; i < nitems(kitty_diacritics); i++) {
+		if (kitty_diacritics[i] == value) {
+			*index = i;
+			return (1);
+		}
+	}
+	return (0);
+}
+
+/* Replace a Kitty Unicode placeholder with a shared image marker cell. */
+int
+kitty_placeholder_to_cell(void *state, struct grid_cell *gc,
+    const struct grid_cell *left)
+{
+	struct kitty_context	*kc = state;
+	struct kitty_source	*source;
+	struct image		*im;
+	uint32_t		 value;
+	size_t			 offset = 0;
+	u_int			 values[3], nvalues = 0, id, x, y;
+	struct utf8_data	 data;
+	int			 fg, bg, us;
+
+	if (kc == NULL ||
+	    !kitty_placeholder_character(gc->data.data, gc->data.size, &offset,
+	    &value) || value != 0x10eeee)
+		return (0);
+	while (offset < gc->data.size && nvalues < nitems(values)) {
+		if (!kitty_placeholder_character(gc->data.data, gc->data.size,
+		    &offset, &value) ||
+		    !kitty_placeholder_index(value, &values[nvalues]))
+			return (0);
+		nvalues++;
+	}
+	if (offset != gc->data.size)
+		return (0);
+
+	if (gc->fg & COLOUR_FLAG_RGB)
+		id = gc->fg & 0xffffff;
+	else if (gc->fg >= 0 && gc->fg <= 255)
+		id = gc->fg;
+	else
+		return (0);
+	if (nvalues == 3)
+		id |= values[2] << 24;
+	source = kitty_source_find(kc, id);
+	if (source == NULL)
+		return (0);
+	im = image_find(source->virtual_id != 0 ? source->virtual_id :
+	    source->server_id);
+	if (im == NULL)
+		return (0);
+
+	if (nvalues >= 1)
+		y = values[0];
+	else if (left != NULL && left->flags & GRID_FLAG_IMAGE &&
+	    left->image_id == im->id)
+		y = left->image_y;
+	else
+		return (0);
+	if (nvalues >= 2)
+		x = values[1];
+	else if (left != NULL && left->flags & GRID_FLAG_IMAGE &&
+	    left->image_id == im->id && left->image_x != UINT_MAX)
+		x = left->image_x + 1;
+	else
+		return (0);
+	if (x >= im->sx || y >= im->sy)
+		return (0);
+
+	utf8_copy(&data, &gc->data);
+	fg = gc->fg;
+	bg = gc->bg;
+	us = gc->us;
+	image_set_cell(gc, im, x, y);
+	utf8_copy(&gc->data, &data);
+	gc->fg = fg;
+	gc->bg = bg;
+	gc->us = us;
+	return (1);
+}

image.c

@@ -394,18 +394,23 @@ image_base64_decode(const char *data, size_t len, size_t limit, size_t *size)
 {
 	char	*copy;
 	u_char	*out;
-	size_t	 needed;
+	size_t	 needed, padded, padding;
 	int	 result;
 
 	if (len > SIZE_MAX - 3)
 		return (NULL);
-	needed = (len + 3) / 4 * 3;
+	padding = (4 - len % 4) % 4;
+	if (padding == 3)
+		return (NULL);
+	padded = len + padding;
+	needed = padded / 4 * 3;
 	if (needed > limit || needed > INT_MAX)
 		return (NULL);
 
-	copy = xmalloc(len + 1);
+	copy = xmalloc(padded + 1);
 	memcpy(copy, data, len);
-	copy[len] = '\0';
+	memset(copy + len, '=', padding);
+	copy[padded] = '\0';
 	out = xmalloc(needed == 0 ? 1 : needed);
 	result = b64_pton(copy, out, needed);
 	free(copy);

input.c

@@ -186,6 +186,9 @@ static void	input_enter_osc(struct input_ctx *);
 static void	input_exit_osc(struct input_ctx *);
 static void	input_enter_apc(struct input_ctx *);
 static void	input_exit_apc(struct input_ctx *);
+#ifdef ENABLE_IMAGES
+static int	input_handle_kitty(struct input_ctx *, const u_char *, size_t);
+#endif
 static void	input_enter_rename(struct input_ctx *);
 static void	input_exit_rename(struct input_ctx *);
 
@@ -2648,6 +2651,18 @@ input_dcs_dispatch(struct input_ctx *ictx)
 		return (0);
 	}
 
+#ifdef ENABLE_IMAGES
+	/* Kitty uses this wrapper automatically when it detects tmux. */
+	if (wp != NULL && len >= prefixlen + 5 &&
+	    memcmp(buf, prefix, prefixlen) == 0 &&
+	    memcmp(buf + prefixlen, "\033_G", 3) == 0 &&
+	    memcmp(buf + len - 2, "\033\\", 2) == 0) {
+		input_handle_kitty(ictx, buf + prefixlen + 3,
+		    len - prefixlen - 5);
+		return (0);
+	}
+#endif
+
 #ifdef ENABLE_SIXEL
 	if (wp != NULL && buf[0] == 'q' && ictx->interm_len == 0) {
 		w = wp->window;
@@ -2795,57 +2810,66 @@ input_enter_apc(struct input_ctx *ictx)
 }
 
 /* APC terminator (ST) received. */
+#ifdef ENABLE_IMAGES
+static int
+input_handle_kitty(struct input_ctx *ictx, const u_char *buf, size_t len)
+{
+	struct screen_write_ctx	*sctx = &ictx->ctx;
+	struct window_pane	*wp = ictx->wp;
+	struct image		*im;
+	u_int			 image_id = 0, quiet = 0;
+	char			 action = '\0';
+	int			 status;
+
+	if (wp == NULL)
+		return (0);
+	im = kitty_parse_image(&ictx->kitty_state, buf, len,
+		    wp->window->xpixel,
+		    wp->window->ypixel, &image_id, &quiet, &action, &status);
+	if (status == KITTY_PARSE_MORE)
+		return (1);
+	if (status != KITTY_PARSE_OK) {
+		if (quiet < 2 && action != '\0') {
+			if (status == KITTY_PARSE_MISSING)
+				input_reply(ictx, 0, "\033_Gi=%u;ENOENT\033\\",
+				    image_id);
+			else
+				input_reply(ictx, 0, "\033_Gi=%u;EINVAL\033\\",
+				    image_id);
+		}
+		return (1);
+	}
+	if (im != NULL) {
+		if (action == 'd')
+			image_clear(sctx, im->id);
+		else
+			image_write(sctx, im, ictx->cell.cell.bg);
+		image_free(im->id);
+		if (quiet == 0 && image_id != 0)
+			input_reply(ictx, 0, "\033_Gi=%u;OK\033\\", image_id);
+	} else if (action == 'd' && image_id == 0)
+		image_clear(sctx, 0);
+	else if ((action == 't' || action == 'q' || action == 'u') && quiet == 0)
+		input_reply(ictx, 0, "\033_Gi=%u;OK\033\\", image_id);
+	return (1);
+}
+#endif
+
 static void
 input_exit_apc(struct input_ctx *ictx)
 {
 	struct screen_write_ctx	*sctx = &ictx->ctx;
 	struct window_pane	*wp = ictx->wp;
-#ifdef ENABLE_IMAGES
-	struct image		*im;
-	u_int			 image_id = 0, quiet = 0;
-	char			 action = '\0';
-	int			 status;
-#endif
 
 	if (ictx->flags & INPUT_DISCARD)
 		return;
 	log_debug("%s: \"%s\"", __func__, ictx->input_buf);
 
 #ifdef ENABLE_IMAGES
-	if (wp != NULL && ictx->input_len > 1 && ictx->input_buf[0] == 'G') {
-		im = kitty_parse_image(&ictx->kitty_state, ictx->input_buf + 1,
-		    ictx->input_len - 1, wp->window->xpixel,
-		    wp->window->ypixel, &image_id, &quiet, &action, &status);
-		if (status == KITTY_PARSE_MORE)
-			return;
-		if (status != KITTY_PARSE_OK) {
-			if (quiet < 2 && action != '\0') {
-				if (status == KITTY_PARSE_MISSING)
-					input_reply(ictx, 0,
-					    "\033_Gi=%u;ENOENT\033\\",
-					    image_id);
-				else
-					input_reply(ictx, 0,
-					    "\033_Gi=%u;EINVAL\033\\",
-					    image_id);
-			}
-			return;
-		}
-		if (im != NULL) {
-			if (action == 'd')
-				image_clear(sctx, im->id);
-			else
-				image_write(sctx, im, ictx->cell.cell.bg);
-			image_free(im->id);
-			if (quiet == 0 && image_id != 0)
-				input_reply(ictx, 0, "\033_Gi=%u;OK\033\\",
-				    image_id);
-		} else if (action == 'd' && image_id == 0) {
-			image_clear(sctx, 0);
-		} else if ((action == 't' || action == 'q') && quiet == 0)
-			input_reply(ictx, 0, "\033_Gi=%u;OK\033\\", image_id);
+	if (ictx->input_len > 1 && ictx->input_buf[0] == 'G' &&
+	    input_handle_kitty(ictx, ictx->input_buf + 1,
+	    ictx->input_len - 1))
 		return;
-	}
 #endif
 
 	if (wp != NULL &&
@@ -2935,6 +2959,23 @@ input_top_bit_set(struct input_ctx *ictx)
 	utf8_copy(&ictx->cell.cell.data, ud);
 	screen_write_collect_add(sctx, &ictx->cell.cell);
 
+#ifdef ENABLE_IMAGES
+	if (sctx->s->cx != 0) {
+		struct grid_cell	 gc, left;
+		u_int			 x = sctx->s->cx - 1;
+
+		grid_view_get_cell(sctx->s->grid, x, sctx->s->cy, &gc);
+		if (x != 0)
+			grid_view_get_cell(sctx->s->grid, x - 1, sctx->s->cy,
+			    &left);
+		if (kitty_placeholder_to_cell(ictx->kitty_state, &gc,
+		    x == 0 ? NULL : &left)) {
+			grid_view_set_cell(sctx->s->grid, x, sctx->s->cy, &gc);
+			image_redraw_area(sctx, x, sctx->s->cy, 1, 1);
+		}
+	}
+#endif
+
 	utf8_copy(&ictx->last, &ictx->cell.cell.data);
 	ictx->flags |= INPUT_LAST;
 

regress/image-support.sh

@@ -54,20 +54,31 @@ $TMUX new-window -d "
 sleep 1
 [ "$($TMUX display-message -pt:2 '#{cursor_y}')" = 1 ] || exit 1
 
+# A transmitted Kitty image followed by a Unicode placeholder placement is
+# converted into shared image cells rather than retained as placeholder text.
+$TMUX new-window -d "
+	printf '\033Ptmux;\033\033_Ga=t,q=2,f=32,o=z,m=1,s=2,v=1,i=588707642;eJz7z8Dw\033\033\\\033\\'
+	printf '\033Ptmux;\033\033_Ga=t,q=2;HwQAFvEF+w\033\033\\\033\\'
+	printf '\033Ptmux;\033\033_Ga=p,U=1,q=2,i=588707642,c=2,r=1\033\033\\\033\\'
+	printf '\033[38;2;22;247;58m\364\216\273\256\314\205\314\205\326\222\364\216\273\256\033[39m'
+	sleep 10"
+sleep 1
+[ -z "$($TMUX capture-pane -pt:3 -S0 -E0)" ] || exit 1
+
 # PNG Kitty input uses the shared image decoder and canonical cell sizing.
 $TMUX new-window -d "
 	printf '\033_Ga=T,q=2,f=100;iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGUExURf8AAP///0EdNBEAAAABYktHRAH/Ai3eAAAAB3RJTUUH6ggCDAECH324BwAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=\033\\'
 	sleep 10"
 sleep 1
-[ "$($TMUX display-message -pt:3 '#{cursor_y}')" = 1 ] || exit 1
+[ "$($TMUX display-message -pt:4 '#{cursor_y}')" = 1 ] || exit 1
 
 # SIXEL input reaches the same grid marker and copy-mode paths.
 $TMUX new-window -d "cat '$FIXTURE'; sleep 10"
 sleep 1
-[ "$($TMUX display-message -pt:4 '#{cursor_y}')" -gt 0 ] || exit 1
-$TMUX copy-mode -t:4 || exit 1
-$TMUX send-keys -t:4 -X history-top || exit 1
-$TMUX capture-pane -pt:4 >$TMP || exit 1
+[ "$($TMUX display-message -pt:5 '#{cursor_y}')" -gt 0 ] || exit 1
+$TMUX copy-mode -t:5 || exit 1
+$TMUX send-keys -t:5 -X history-top || exit 1
+$TMUX capture-pane -pt:5 >$TMP || exit 1
 
 # A 26-pixel raster occupies two default 16-pixel cells, but the final cell
 # remains only partially filled instead of stretching the raster to 32 pixels.

tmux.h

@@ -4328,6 +4328,8 @@ void		 image_clear(struct screen_write_ctx *, u_int);
 #define KITTY_PARSE_MISSING 2
 struct image	*kitty_parse_image(void **, const u_char *, size_t, u_int,
 		     u_int, u_int *, u_int *, char *, int *);
+int		 kitty_placeholder_to_cell(void *, struct grid_cell *,
+		     const struct grid_cell *);
 void		 kitty_free_state(void *);
 void		 kitty_draw_rectangle(struct tty *,
 		     const struct image_rectangle *, const struct tty_style_ctx *);