Refactor to centralize image-specific rendering and lifecycle code in image.c and the image-* backends.

2026-08-04 14:38:35 +00:00 · 2026-08-03 09:13:45 +01:00
parent f68aad1b41
commit fe938ea657
14 changed files with 643 additions and 380 deletions
--- a/image.c
+++ b/image.c
@@ -19,6 +19,9 @@

 #include <sys/types.h>

+#include <limits.h>
+#include <png.h>
+#include <resolv.h>
 #include <stdlib.h>
 #include <string.h>

@@ -27,6 +30,88 @@
 static struct images	images = RB_INITIALIZER(&images);
 static u_int		image_next_id;

+#define IMAGE_BACKEND_GRAPHICAL 0x1
+#define IMAGE_BACKEND_SCROLLS   0x2
+
+struct image_backend {
+	const char	*name;
+	int		 flags;
+	void		(*draw_rectangle)(struct tty *,
+		    const struct image_rectangle *, const struct tty_style_ctx *);
+	void		(*free)(struct tty *, int);
+	void		(*geometry_changed)(struct tty *);
+};
+
+static const struct image_backend image_backend_ascii = {
+	"ascii", IMAGE_BACKEND_SCROLLS, NULL, NULL, NULL
+};
+static const struct image_backend image_backend_kitty = {
+	"kitty", IMAGE_BACKEND_GRAPHICAL|IMAGE_BACKEND_SCROLLS,
+	kitty_draw_rectangle, kitty_free_output, kitty_geometry_changed
+};
+static const struct image_backend image_backend_sixel = {
+	"sixel", IMAGE_BACKEND_GRAPHICAL, sixel_draw_rectangle, NULL, NULL
+};
+
+static const struct image_backend *
+image_tty_find_backend(struct tty *tty)
+{
+	if (tty->term != NULL && tty->term->flags & TERM_KITTY)
+		return (&image_backend_kitty);
+	if (tty->term != NULL && tty->term->flags & TERM_SIXEL &&
+	    tty->xpixel != 0 && tty->ypixel != 0)
+		return (&image_backend_sixel);
+	return (&image_backend_ascii);
+}
+
+void
+image_tty_update(struct tty *tty)
+{
+	const struct image_backend	*backend;
+
+	backend = image_tty_find_backend(tty);
+	if (tty->image_backend == backend)
+		return;
+
+	if (tty->image_backend != NULL && tty->image_backend->free != NULL)
+		tty->image_backend->free(tty, !!(tty->flags & TTY_OPENED));
+	tty->image_data = NULL;
+	tty->image_backend = backend;
+	log_debug("%s: %s image backend is %s", __func__,
+	    tty->client->name, backend->name);
+}
+
+int
+image_tty_is_graphical(struct tty *tty)
+{
+	image_tty_update(tty);
+	return (!!(tty->image_backend->flags & IMAGE_BACKEND_GRAPHICAL));
+}
+
+int
+image_tty_scrolls(struct tty *tty)
+{
+	image_tty_update(tty);
+	return (!!(tty->image_backend->flags & IMAGE_BACKEND_SCROLLS));
+}
+
+void
+image_tty_geometry_changed(struct tty *tty)
+{
+	image_tty_update(tty);
+	if (tty->image_backend->geometry_changed != NULL)
+		tty->image_backend->geometry_changed(tty);
+}
+
+void
+image_tty_free(struct tty *tty, int send)
+{
+	if (tty->image_backend != NULL && tty->image_backend->free != NULL)
+		tty->image_backend->free(tty, send);
+	tty->image_backend = NULL;
+	tty->image_data = NULL;
+}
+
 static int
 image_cmp(struct image *a, struct image *b)
 {
@@ -38,6 +123,79 @@ image_cmp(struct image *a, struct image *b)
 }
 RB_GENERATE_STATIC(images, image, entry, image_cmp);

+static void
+image_sample(struct image *im, uint64_t sample_x, uint64_t sample_y,
+    uint64_t sample_columns, uint64_t sample_rows, struct image_sample *sample)
+{
+	const u_char	*pixel;
+	uint64_t	 red = 0, green = 0, blue = 0, alpha = 0;
+	uint64_t	 brightness = 0, count = 0;
+	u_int		 x, y, x0, x1, y0, y1;
+
+	x0 = sample_x * im->width / sample_columns;
+	x1 = (sample_x + 1) * im->width / sample_columns;
+	y0 = sample_y * im->height / sample_rows;
+	y1 = (sample_y + 1) * im->height / sample_rows;
+	if (x1 <= x0)
+		x1 = x0 + 1;
+	if (y1 <= y0)
+		y1 = y0 + 1;
+	if (x1 > im->width)
+		x1 = im->width;
+	if (y1 > im->height)
+		y1 = im->height;
+
+	for (y = y0; y < y1; y++) {
+		for (x = x0; x < x1; x++) {
+			pixel = im->pixels + y * im->stride + x * 4;
+			red += pixel[0] * pixel[3] / 255;
+			green += pixel[1] * pixel[3] / 255;
+			blue += pixel[2] * pixel[3] / 255;
+			alpha += pixel[3];
+			brightness += ((2126ULL * pixel[0] +
+			    7152ULL * pixel[1] + 722ULL * pixel[2]) / 10000) *
+			    pixel[3] / 255;
+			count++;
+		}
+	}
+	if (count == 0)
+		return;
+	sample->red = red / count;
+	sample->green = green / count;
+	sample->blue = blue / count;
+	sample->alpha = alpha / count;
+	sample->brightness = brightness / count;
+}
+
+static void
+image_make_cells(struct image *im)
+{
+	struct image_cell	*cell;
+	uint64_t		 columns, rows;
+	u_int			 x, y, sample_x, sample_y;
+
+	columns = (uint64_t)im->sx * IMAGE_SAMPLE_COLUMNS;
+	rows = (uint64_t)im->sy * IMAGE_SAMPLE_ROWS;
+	im->cells = xcalloc((size_t)im->sx * im->sy, sizeof *im->cells);
+	for (y = 0; y < im->sy; y++) {
+		for (x = 0; x < im->sx; x++) {
+			cell = &im->cells[(size_t)y * im->sx + x];
+			image_sample(im, x, y, im->sx, im->sy, &cell->whole);
+			for (sample_y = 0; sample_y < IMAGE_SAMPLE_ROWS;
+			    sample_y++) {
+				for (sample_x = 0;
+				    sample_x < IMAGE_SAMPLE_COLUMNS; sample_x++) {
+					image_sample(im,
+					    (uint64_t)x * IMAGE_SAMPLE_COLUMNS + sample_x,
+					    (uint64_t)y * IMAGE_SAMPLE_ROWS + sample_y,
+					    columns, rows,
+					    &cell->samples[sample_y][sample_x]);
+				}
+			}
+		}
+	}
+}
+
 struct image *
 image_find(u_int id)
 {
@@ -56,7 +214,7 @@ image_create(u_int width, u_int height, u_int sx, u_int sy, u_char *pixels)
 		return (NULL);
 	if ((uint64_t)width * height * 4 > SIZE_MAX)
 		return (NULL);
-	if ((uint64_t)sx * sy > SIZE_MAX)
+	if ((uint64_t)sx * sy > SIZE_MAX / sizeof *im->cells)
 		return (NULL);

 	im = xcalloc(1, sizeof *im);
@@ -74,7 +232,6 @@ image_create(u_int width, u_int height, u_int sx, u_int sy, u_char *pixels)
 	im->stride = (size_t)width * 4;
 	im->size = im->stride * height;
 	im->pixels = pixels;
-	image_make_ascii(im);

 	RB_INSERT(images, &images, im);
 	log_debug("%s: image %u is %ux%u pixels, %ux%u cells", __func__,
@@ -107,10 +264,20 @@ image_free(u_int id)
 	log_debug("%s: freeing image %u", __func__, id);
 	RB_REMOVE(images, &images, im);
 	free(im->pixels);
-	free(im->ascii);
+	free(im->cells);
 	free(im);
 }

+const struct image_cell *
+image_get_cell(struct image *im, u_int x, u_int y)
+{
+	if (im == NULL || x >= im->sx || y >= im->sy)
+		return (NULL);
+	if (im->cells == NULL)
+		image_make_cells(im);
+	return (&im->cells[(size_t)y * im->sx + x]);
+}
+
 void
 image_set_cell(struct grid_cell *gc, struct image *im, u_int x, u_int y)
 {
@@ -121,9 +288,118 @@ image_set_cell(struct grid_cell *gc, struct image *im, u_int x, u_int y)
 	gc->image_y = y;
 }

+/* Convert a cell-aligned image rectangle into source pixel coordinates. */
 void
-image_clear(struct screen *s, u_int id)
+image_get_pixel_rectangle(const struct image *im, u_int x, u_int y,
+    u_int width, u_int height, u_int *px, u_int *py, u_int *pwidth,
+    u_int *pheight)
 {
+	u_int	x1, y1;
+
+	*px = *py = *pwidth = *pheight = 0;
+	if (im == NULL || x >= im->sx || y >= im->sy || width == 0 ||
+	    height == 0)
+		return;
+	if (width > im->sx - x)
+		width = im->sx - x;
+	if (height > im->sy - y)
+		height = im->sy - y;
+
+	*px = (uint64_t)x * im->width / im->sx;
+	*py = (uint64_t)y * im->height / im->sy;
+	x1 = (uint64_t)(x + width) * im->width / im->sx;
+	y1 = (uint64_t)(y + height) * im->height / im->sy;
+	if (*px >= im->width)
+		*px = im->width - 1;
+	if (*py >= im->height)
+		*py = im->height - 1;
+	if (x1 <= *px)
+		x1 = *px + 1;
+	if (y1 <= *py)
+		y1 = *py + 1;
+	if (x1 > im->width)
+		x1 = im->width;
+	if (y1 > im->height)
+		y1 = im->height;
+	*pwidth = x1 - *px;
+	*pheight = y1 - *py;
+}
+
+void
+image_size_in_cells(u_int width, u_int height, u_int xpixel, u_int ypixel,
+    u_int *sx, u_int *sy)
+{
+	if (xpixel == 0)
+		xpixel = 8;
+	if (ypixel == 0)
+		ypixel = 16;
+	*sx = ((uint64_t)width + xpixel - 1) / xpixel;
+	*sy = ((uint64_t)height + ypixel - 1) / ypixel;
+}
+
+u_char *
+image_base64_decode(const char *data, size_t len, size_t limit, size_t *size)
+{
+	char	*copy;
+	u_char	*out;
+	size_t	 needed;
+	int	 result;
+
+	if (len > SIZE_MAX - 3)
+		return (NULL);
+	needed = (len + 3) / 4 * 3;
+	if (needed > limit || needed > INT_MAX)
+		return (NULL);
+
+	copy = xmalloc(len + 1);
+	memcpy(copy, data, len);
+	copy[len] = '\0';
+	out = xmalloc(needed == 0 ? 1 : needed);
+	result = b64_pton(copy, out, needed);
+	free(copy);
+	if (result < 0) {
+		free(out);
+		return (NULL);
+	}
+	*size = result;
+	return (out);
+}
+
+u_char *
+image_png_decode(const u_char *data, size_t size, size_t limit, u_int *width,
+    u_int *height)
+{
+	png_image	 pi;
+	u_char		*pixels;
+
+	if (size == 0 || size > limit)
+		return (NULL);
+	memset(&pi, 0, sizeof pi);
+	pi.version = PNG_IMAGE_VERSION;
+	if (!png_image_begin_read_from_memory(&pi, data, size))
+		return (NULL);
+	pi.format = PNG_FORMAT_RGBA;
+	if (pi.width == 0 || pi.height == 0 ||
+	    (uint64_t)pi.width * pi.height * 4 > limit) {
+		png_image_free(&pi);
+		return (NULL);
+	}
+	pixels = xmalloc(PNG_IMAGE_SIZE(pi));
+	if (!png_image_finish_read(&pi, NULL, pixels, 0, NULL)) {
+		free(pixels);
+		png_image_free(&pi);
+		return (NULL);
+	}
+	*width = pi.width;
+	*height = pi.height;
+	png_image_free(&pi);
+	return (pixels);
+}
+
+void
+image_clear(struct screen_write_ctx *ctx, u_int id)
+{
+	struct screen		*s = ctx->s;
 	struct grid		*gd = s->grid;
 	struct grid_cell	 gc;
 	u_int			 x, y;
@@ -132,24 +408,33 @@ image_clear(struct screen *s, u_int id)
 		for (x = 0; x < gd->sx; x++) {
 			grid_get_cell(gd, x, y, &gc);
 			if ((gc.flags & GRID_FLAG_IMAGE) &&
-			    (id == 0 || gc.image_id == id))
+			    (id == 0 || gc.image_id == id)) {
+				if (y >= gd->hsize)
+					image_damage_area(ctx, x, y - gd->hsize,
+					    1, 1);
 				grid_set_cell(gd, x, y, &grid_default_cell);
+			}
 		}
 	}
 }

-int
+static int
 image_check_area(struct screen *s, u_int px, u_int py, u_int nx, u_int ny)
 {
 	struct grid_cell	 gc;
 	u_int			 x, y, ex, ey;
+	u_int			 sx = screen_size_x(s), sy = screen_size_y(s);

-	ex = px + nx;
-	if (ex > screen_size_x(s))
-		ex = screen_size_x(s);
-	ey = py + ny;
-	if (ey > screen_size_y(s))
-		ey = screen_size_y(s);
+	if (px >= sx || py >= sy || nx == 0 || ny == 0)
+		return (0);
+	if (nx > sx - px)
+		ex = sx;
+	else
+		ex = px + nx;
+	if (ny > sy - py)
+		ey = sy;
+	else
+		ey = py + ny;
 	for (y = py; y < ey; y++) {
 		for (x = px; x < ex; x++) {
 			grid_view_get_cell(s->grid, x, y, &gc);
@@ -160,23 +445,72 @@ image_check_area(struct screen *s, u_int px, u_int py, u_int nx, u_int ny)
 	return (0);
 }

-int
-image_check_line(struct screen *s, u_int py, u_int ny)
+void
+image_damage_area(struct screen_write_ctx *ctx, u_int px, u_int py, u_int nx,
+    u_int ny)
 {
-	return (image_check_area(s, 0, py, screen_size_x(s), ny));
+	if (ctx->wp != NULL && image_check_area(ctx->s, px, py, nx, ny))
+		ctx->wp->flags |= PANE_REDRAW;
 }

-int
-image_free_all(struct screen *s)
+void
+image_damage_all(struct screen_write_ctx *ctx)
 {
-	return (image_check_area(s, 0, 0, screen_size_x(s),
-	    screen_size_y(s)));
+	image_damage_area(ctx, 0, 0, screen_size_x(ctx->s),
+	    screen_size_y(ctx->s));
 }

-int
-image_scroll_up(struct screen *s, __unused u_int lines)
+void
+image_damage_scroll(struct screen_write_ctx *ctx, __unused u_int lines)
 {
-	return (image_free_all(s));
+	image_damage_all(ctx);
+}
+
+/* Draw the graphical image marker runs in one visible scene span. */
+void
+image_draw_line(struct tty *tty, struct screen *s, u_int px, u_int py,
+    u_int nx, u_int atx, u_int aty, const struct tty_style_ctx *style_ctx)
+{
+	const struct image_backend	*backend;
+	struct image_rectangle		 rectangle;
+	struct grid_cell		 gc, next;
+	struct image			*im;
+	u_int				 i, run;
+
+	image_tty_update(tty);
+	backend = tty->image_backend;
+	if (~backend->flags & IMAGE_BACKEND_GRAPHICAL)
+		return;
+
+	for (i = 0; i < nx; i += run) {
+		grid_view_get_cell(s->grid, px + i, py, &gc);
+		if (~gc.flags & GRID_FLAG_IMAGE) {
+			run = 1;
+			continue;
+		}
+		im = image_find(gc.image_id);
+		if (im == NULL) {
+			run = 1;
+			continue;
+		}
+		for (run = 1; i + run < nx; run++) {
+			grid_view_get_cell(s->grid, px + i + run, py, &next);
+			if (~next.flags & GRID_FLAG_IMAGE ||
+			    next.image_id != gc.image_id ||
+			    next.image_y != gc.image_y ||
+			    next.image_x != gc.image_x + run)
+				break;
+		}
+		rectangle.image = im;
+		memcpy(&rectangle.cell, &gc, sizeof rectangle.cell);
+		rectangle.source_x = gc.image_x;
+		rectangle.source_y = gc.image_y;
+		rectangle.width = run;
+		rectangle.height = 1;
+		rectangle.destination_x = atx + i;
+		rectangle.destination_y = aty;
+		backend->draw_rectangle(tty, &rectangle, style_ctx);
+	}
 }

 /*
@@ -218,7 +552,6 @@ image_write(struct screen_write_ctx *ctx, struct image *im, u_int bg)
 			grid_view_set_cell(gd, cx + x, cy + y, &gc);
 		}
 	}
-	if (ctx->wp != NULL)
-		ctx->wp->flags |= PANE_REDRAW;
+	image_damage_area(ctx, cx, cy, sx, sy);
 	screen_write_cursormove(ctx, 0, cy + sy, 0);
 }