Initial attempt at supporting new backend for kitty images

Cargo.lock

@@ -1518,6 +1518,22 @@ dependencies = [
  "thiserror 2.0.12",
 ]
 
+[[package]]
+name = "kittage"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "crossterm",
+ "futures-core",
+ "image",
+ "memchr",
+ "memmap2",
+ "psx-shm",
+ "rustix 1.0.8",
+ "thiserror 2.0.12",
+ "tokio",
+]
+
 [[package]]
 name = "kqueue"
 version = "1.1.1"
@@ -1698,6 +1714,15 @@ version = "2.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
 
+[[package]]
+name = "memmap2"
+version = "0.9.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "memmem"
 version = "0.1.1"
@@ -2291,6 +2316,15 @@ dependencies = [
  "prost",
 ]
 
+[[package]]
+name = "psx-shm"
+version = "0.1.1"
+source = "git+https://github.com/itsjunetime/psx-shm.git#3fcbae91217cd50ea0e4c838276ef7500cccf024"
+dependencies = [
+ "memmap2",
+ "rustix 1.0.8",
+]
+
 [[package]]
 name = "quick-error"
 version = "2.0.1"
@@ -2907,6 +2941,8 @@ dependencies = [
  "futures-util",
  "image",
  "itertools 0.14.0",
+ "kittage",
+ "memmap2",
  "mimalloc",
  "mupdf",
  "nix 0.30.1",

Cargo.toml

@@ -40,6 +40,8 @@ mimalloc = "0.1.43"
 nix = { version = "0.30.0", features = ["signal"] }
 mupdf = { version = "0.5.0", default-features = false, features = ["svg", "system-fonts", "img"] }
 rayon = { version = "*", default-features = false }
+kittage = { path = "../kittage/", features = ["crossterm-tokio", "image-crate"] }
+memmap2 = "*"
 
 # for tracing with tokio-console
 console-subscriber = { version = "0.4.0", optional = true }

src/converter.rs

@@ -1,15 +1,42 @@
+use std::num::NonZeroU32;
+
 use flume::{Receiver, SendError, Sender, TryRecvError};
 use futures_util::stream::StreamExt;
 use image::DynamicImage;
 use itertools::Itertools;
+use kittage::NumberOrId;
 use ratatui::layout::Rect;
-use ratatui_image::{Resize, picker::Picker, protocol::Protocol};
+use ratatui_image::{
+	Resize,
+	picker::{Picker, ProtocolType},
+	protocol::Protocol
+};
 use rayon::iter::ParallelIterator;
 
 use crate::renderer::{PageInfo, RenderError, fill_default};
 
+#[derive(Debug)]
+pub enum MaybeTransferred {
+	NotYet(kittage::image::Image<'static>, memmap2::MmapMut),
+	Transferred(kittage::ImageId)
+}
+
+pub enum ConvertedImage {
+	Generic(Protocol),
+	Kitty { img: MaybeTransferred, area: Rect }
+}
+
+impl ConvertedImage {
+	pub fn area(&self) -> Rect {
+		match self {
+			Self::Generic(prot) => prot.area(),
+			Self::Kitty { img: _, area } => *area
+		}
+	}
+}
+
 pub struct ConvertedPage {
-	pub page: Protocol,
+	pub page: ConvertedImage,
 	pub num: usize,
 	pub num_results: usize
 }
@@ -28,13 +55,15 @@ pub async fn run_conversion_loop(
 ) -> Result<(), SendError<Result<ConvertedPage, RenderError>>> {
 	let mut images = vec![];
 	let mut page: usize = 0;
+	let pid = std::process::id();
 
 	fn next_page(
 		images: &mut [Option<PageInfo>],
 		picker: &mut Picker,
 		page: usize,
 		iteration: &mut usize,
-		prerender: usize
+		prerender: usize,
+		pid: u32
 	) -> Result<Option<ConvertedPage>, RenderError> {
 		if images.is_empty() || *iteration >= prerender {
 			return Ok(None);
@@ -85,13 +114,43 @@ pub async fn run_conversion_loop(
 		// verified (with the ImageSurface stuff) that the image is the correct
 		// size for the area given, so to save ratatui the work of having to
 		// resize it, we tell them to crop it to fit.
-		let txt_img = picker
+		let txt_img = match picker.protocol_type() {
+			ProtocolType::Kitty => {
+				let area = ratatui_image::protocol::ImageSource::round_pixel_size_to_cells(
+					dyn_img.width(),
+					dyn_img.height(),
+					picker.font_size()
+				);
+
+				match kittage::image::Image::shm_from(
+					dyn_img,
+					format!("__tdf_kittage_{pid}_page_{page}").into()
+				) {
+					Ok((mut img, map)) => {
+						img.num_or_id = NumberOrId::Id(NonZeroU32::new(page as u32 + 1).unwrap());
+						ConvertedImage::Kitty {
+							img: MaybeTransferred::NotYet(img, map),
+							area
+						}
+					}
+					// todo: fallback to non-shm image here without cloning dyn_img above
+					// Err(_) => ConvertedImage::Kitty(dyn_img.into())
+					Err(e) =>
+						return Err(RenderError::Converting(format!(
+							"Couldn't write to shm: {e}"
+						))),
+				}
+			}
+			_ => ConvertedImage::Generic(
+				picker
 					.new_protocol(dyn_img, img_area, Resize::None)
 					.map_err(|e| {
 						RenderError::Converting(format!(
 							"Couldn't convert DynamicImage to ratatui image: {e}"
 						))
-			})?;
+					})?
+			)
+		};
 
 		// update the iteration to the iteration that we stole this image from
 		*iteration = new_iter;
@@ -130,7 +189,14 @@ pub async fn run_conversion_loop(
 				Err(TryRecvError::Disconnected) => return Ok(())
 			}
 
-			match next_page(&mut images, &mut picker, page, &mut iteration, prerender) {
+			match next_page(
+				&mut images,
+				&mut picker,
+				page,
+				&mut iteration,
+				prerender,
+				pid
+			) {
 				Ok(None) => break,
 				Ok(Some(img)) => sender.send(Ok(img))?,
 				Err(e) => sender.send(Err(e))?

src/main.rs

@@ -2,8 +2,8 @@ use core::error::Error;
 use std::{
 	borrow::Cow,
 	ffi::OsString,
-	io::{BufReader, Read, Stdout, Write, stdout},
+	io::{BufReader, Read, StdoutLock, Write, stdout},
-	num::NonZeroUsize,
+	num::{NonZeroU32, NonZeroUsize},
 	path::PathBuf
 };
 
@@ -17,12 +17,20 @@ use crossterm::{
 };
 use flume::{Sender, r#async::RecvStream};
 use futures_util::{FutureExt, stream::StreamExt};
+use kittage::{
+	ImageDimensions, PixelFormat,
+	action::Action,
+	display::{DisplayConfig, DisplayLocation},
+	error::TransmitError,
+	image::Image as KImage,
+	medium::Medium
+};
 use notify::{Event, EventKind, RecursiveMode, Watcher};
 use ratatui::{Terminal, backend::CrosstermBackend};
 use ratatui_image::picker::Picker;
 use tdf::{
 	PrerenderLimit,
-	converter::{ConvertedPage, ConverterMsg, run_conversion_loop},
+	converter::{ConvertedPage, ConverterMsg, MaybeTransferred, run_conversion_loop},
 	renderer::{self, RenderError, RenderInfo, RenderNotif},
 	tui::{BottomMessage, InputAction, MessageSetting, Tui}
 };
@@ -373,10 +381,90 @@ async fn enter_redraw_loop(
 		}
 
 		if needs_redraw {
+			let mut to_display = vec![];
 			term.draw(|f| {
-				tui.render(f, &main_area);
+				to_display = tui.render(f, &main_area);
 			})?;
-			execute!(stdout(), EndSynchronizedUpdate)?;
+
+			let mut stdout = stdout().lock();
+			let mut maybe_err = Ok(());
+			for (img, area) in to_display {
+				let config = DisplayConfig {
+					location: DisplayLocation {
+						x: area.x.into(),
+						y: area.y.into(),
+						..DisplayLocation::default()
+					},
+					..DisplayConfig::default()
+				};
+
+				maybe_err = match img {
+					MaybeTransferred::NotYet(image, _map) => {
+						let mut fake_image = KImage {
+							num_or_id: image.num_or_id,
+							format: PixelFormat::Rgb24(
+								ImageDimensions {
+									width: 0,
+									height: 0
+								},
+								None
+							),
+							medium: Medium::Direct {
+								chunk_size: None,
+								data: (&[]).into()
+							}
+						};
+						std::mem::swap(image, &mut fake_image);
+
+						let res = Action::TransmitAndDisplay {
+							image: fake_image,
+							config,
+							placement_id: None
+						}
+						.execute_async(&mut stdout, &mut ev_stream)
+						.await;
+
+						match res {
+							Ok((_, img_id)) => {
+								// We need the `_map` to be dropped here, but can't explicitly carry it
+								// over to here. So we're just relying on the overwrite of `img` to
+								// drop `_map` (and thus unmap the memory) for us
+								*img = MaybeTransferred::Transferred(img_id);
+								Ok(())
+							}
+							Err(e) => Err(match e {
+								TransmitError::Writing(
+									Action::TransmitAndDisplay {
+										image: failed_img, ..
+									},
+									e
+								) => {
+									*image = failed_img;
+									e.to_string()
+								}
+								_ => e.to_string()
+							})
+						}
+					}
+					MaybeTransferred::Transferred(image_id) => Action::Display {
+						image_id: *image_id,
+						placement_id: NonZeroU32::new(1).unwrap(),
+						config
+					}
+					.execute_async(&mut stdout, &mut ev_stream)
+					.await
+					.map(|(_, _)| ())
+					.map_err(|e| e.to_string())
+				};
+			}
+
+			if let Err(e) = maybe_err {
+				tui.set_msg(MessageSetting::Some(BottomMessage::Error(format!(
+					"Couldn't transfer image to the terminal: {e}"
+				))));
+			}
+
+			execute!(&mut stdout, EndSynchronizedUpdate)?;
 		}
 	}
 }

src/tui.rs

@@ -20,9 +20,10 @@ use ratatui::{
 	text::{Span, Text},
 	widgets::{Block, Borders, Clear, Padding}
 };
-use ratatui_image::{Image, protocol::Protocol};
+use ratatui_image::Image;
 
 use crate::{
+	converter::{ConvertedImage, MaybeTransferred},
 	renderer::{RenderError, fill_default},
 	skip::Skip
 };
@@ -74,7 +75,7 @@ struct PageConstraints {
 #[derive(Default)]
 struct RenderedInfo {
 	// The image, if it has been rendered by `Converter` to that struct
-	img: Option<Protocol>,
+	img: Option<ConvertedImage>,
 	// The number of results for the current search term that have been found on this page. None if
 	// we haven't checked this page yet
 	// Also this isn't the most efficient representation of this value, but it's accurate, so like
@@ -127,10 +128,15 @@ impl Tui {
 	}
 
 	// TODO: Make a way to fill the width of the screen with one page and scroll down to view it
-	pub fn render(&mut self, frame: &mut Frame<'_>, full_layout: &RenderLayout) {
+	#[must_use]
+	pub fn render<'s>(
+		&'s mut self,
+		frame: &mut Frame<'_>,
+		full_layout: &RenderLayout
+	) -> Vec<(&'s mut MaybeTransferred, Rect)> {
 		if self.showing_help_msg {
 			self.render_help_msg(frame);
-			return;
+			return vec![];
 		}
 
 		if let Some((top_area, bottom_area)) = full_layout.top_and_bottom {
@@ -237,6 +243,7 @@ impl Tui {
 			// be written and set to skip it so that ratatui doesn't spend a lot of time diffing it
 			// each re-render
 			frame.render_widget(Skip::new(true), img_area);
+			vec![]
 		} else {
 			// here we calculate how many pages can fit in the available area.
 			let mut test_area_w = img_area.width;
@@ -254,7 +261,7 @@ impl Tui {
 					take
 				})
 				// and map it to their width (in cells on the terminal, not pixels)
-				.filter_map(|(_, page)| page.img.as_mut().map(|img| (img.rect().width, img)))
+				.filter_map(|(_, page)| page.img.as_mut().map(|img| (img.area().width, img)))
 				// and then take them as long as they won't overflow the available area.
 				.take_while(|(width, _)| match test_area_w.checked_sub(*width) {
 					Some(new_val) => {
@@ -272,6 +279,7 @@ impl Tui {
 			if page_widths.is_empty() {
 				// If none are ready to render, just show the loading thing
 				Self::render_loading_in(frame, img_area);
+				vec![]
 			} else {
 				execute!(stdout(), BeginSynchronizedUpdate).unwrap();
 
@@ -283,20 +291,42 @@ impl Tui {
 				self.last_render.unused_width = unused_width;
 				img_area.x += unused_width / 2;
 
-				for (width, img) in page_widths {
+				let to_display = page_widths
+					.into_iter()
+					.filter_map(|(width, img)| {
+						let maybe_img =
 							Self::render_single_page(frame, img, Rect { width, ..img_area });
 						img_area.x += width;
-				}
+						maybe_img
+					})
+					.collect::<Vec<_>>();
 
 				// we want to set this at the very end so it doesn't get set somewhere halfway through and
 				// then the whole diffing thing messes it up
 				self.last_render.rect = size;
+
+				to_display
 			}
 		}
 	}
 
-	fn render_single_page(frame: &mut Frame<'_>, page_img: &mut Protocol, img_area: Rect) {
+	fn render_single_page<'img>(
+		frame: &mut Frame<'_>,
+		page_img: &'img mut ConvertedImage,
+		img_area: Rect
+	) -> Option<(&'img mut MaybeTransferred, Rect)> {
+		match page_img {
+			ConvertedImage::Generic(page_img) => {
 				frame.render_widget(Image::new(page_img), img_area);
+				None
+			}
+			ConvertedImage::Kitty { img, area } => Some((img, Rect {
+				x: img_area.x,
+				y: img_area.y,
+				width: area.width,
+				height: area.height
+			}))
+		}
 	}
 
 	fn render_loading_in(frame: &mut Frame<'_>, area: Rect) {
@@ -344,7 +374,7 @@ impl Tui {
 		self.page = self.page.min(n_pages - 1);
 	}
 
-	pub fn page_ready(&mut self, img: Protocol, page_num: usize, num_results: usize) {
+	pub fn page_ready(&mut self, img: ConvertedImage, page_num: usize, num_results: usize) {
 		// If this new image woulda fit within the available space on the last render AND it's
 		// within the range where it might've been rendered with the last shown pages, then reset
 		// the last rect marker so that all images are forced to redraw on next render and this one
@@ -352,7 +382,7 @@ impl Tui {
 		if page_num >= self.page && page_num <= self.page + self.last_render.pages_shown {
 			self.last_render.rect = Rect::default();
 		} else {
-			let img_w = img.rect().width;
+			let img_w = img.area().width;
 			if img_w <= self.last_render.unused_width {
 				let num_fit = self.last_render.unused_width / img_w;
 				if page_num >= self.page && (self.page + num_fit as usize) >= page_num {