Add converter to prerender a subset of images while being more friendly towards memory usage

TODO.txt

@@ -0,0 +1,3 @@
+- Look into a way to tell ratatui to skip diffing and just overwrite the whole thing 'cause we know it won't save time
+- Look into more efficient 'skip'ing in ratatui so that you don't have to unicode_width::width a bunch of escape codes
+- Render to Box<dyn Protocol> aot but only in the vicinity of the current view to save space

src/converter.rs

@@ -0,0 +1,116 @@
+use std::{pin::Pin, task::{Context, Poll}};
+
+use futures_util::Stream;
+use image::ImageFormat;
+use itertools::Itertools;
+use ratatui_image::{picker::Picker, protocol::Protocol, Resize};
+
+use crate::renderer::{ImageData, RenderError};
+
+const MAX_ITER: usize = 20;
+
+pub struct Converter {
+	images: Vec<Option<ImageData>>,
+	picker: Picker,
+	page: usize,
+	// once it reaches 20, we're done rendering images
+	iteration: usize
+}
+
+impl Converter {
+	pub fn new(picker: Picker) -> Self {
+		Self {
+			images: vec![],
+			picker,
+			page: 0,
+			iteration: 0
+		}
+	}
+
+	pub fn add_img(&mut self, image: ImageData, idx: usize) {
+		self.images[idx] = Some(image);
+	}
+
+	pub fn set_n_pages(&mut self, pages: usize) {
+		self.images = Vec::with_capacity(pages);
+		for _ in 0..pages {
+			self.images.push(None);
+		}
+
+		self.page = self.page.min(pages - 1);
+	}
+
+	pub fn go_to_page(&mut self, page: usize) {
+		self.page = page;
+		self.iteration = 0;
+	}
+
+	pub fn change_page_by(&mut self, change: isize) {
+		self.page = (self.page as isize + change) as usize;
+		// We just reset iteration here. I think there's some heuristic we could do to place
+		// iteration exactly where it needs to be to render the next page, but trying to determine
+		// that caused me a lot of bugs, and only causes the slightest inefficiency (down below,
+		// when we skip `self.iteration` elements in an iterator), so it's like whatever
+		self.iteration = 0;
+	}
+
+	pub fn get_next_img(&mut self) -> Option<ConversionResult> {
+		// In this fn, we return Poll::Pending and don't store a Waker 'cause this will be called
+		// in a loop with tokio::select, and in no other context. The pending that we return on one
+		// iteration will just be dropped/cancelled as soon as some other action happens, and then
+		// next time select is called, this'll be checked again, and then we might be in the right
+		// circumstance to return a Ready
+		if self.iteration >= MAX_ITER || self.images.is_empty() {
+			return None;
+		}
+
+		// This kinda mimics the way the renderer alternates between going above and below the
+		// current page (within the bounds of how many pages there are) until we've done 20
+		let idx_start = self.page.saturating_sub(MAX_ITER / 2);
+		let idx_end = idx_start.saturating_add(MAX_ITER).min(self.images.len());
+
+		// then we go through all the indices available to us and find the first one that has an
+		// image available to steal
+		let (img_data, iteration, new_idx) = (idx_start..self.page)
+			.interleave(self.page..idx_end)
+			.enumerate()
+			.skip(self.iteration)
+			.find_map(|(i_idx, p_idx)|
+				self.images[p_idx].take().map(|p| (p, i_idx, p_idx))
+			)?;
+
+		let img_area = img_data.area;
+
+		let dyn_img = match image::load_from_memory_with_format(&img_data.data, ImageFormat::Png) {
+			Ok(dt) => dt,
+			Err(e) => return Some(Err(RenderError::Render(format!("Couldn't convert Vec<u8> to DynamicImage: {e}"))))
+		};
+
+		// We don't actually want to Crop this image, but we've already
+		// verified (with the ImageSurface stuff) that the image is the correct
+		// size for the area given, so to save ratatui the work of having to
+		// resize it, we tell them to crop it to fit.
+		let txt_img = match self.picker.new_protocol(dyn_img, img_area, Resize::Crop) {
+			Ok(img) => img,
+			Err(e) => return Some(Err(RenderError::Render(format!("Couldn't convert DynamicImage to ratatui image: {e}"))))
+		};
+
+		// update the iteration to the iteration that we stole this image from
+		self.iteration = iteration;
+
+		Some(Ok((txt_img, new_idx)))
+	}
+}
+
+type ConversionResult = Result<(Box<dyn Protocol>, usize), RenderError>;
+
+impl Stream for Converter {
+	type Item = ConversionResult;
+
+	fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		match self.get_next_img() {
+			Some(res) => Poll::Ready(Some(res)),
+			None => Poll::Pending
+		}
+	}
+}

src/main.rs

@@ -1,5 +1,6 @@
 use std::{path::PathBuf, str::FromStr};
 
+use converter::Converter;
 use crossterm::{execute, terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen}};
 use glib::{LogField, LogLevel, LogWriterOutput};
 use notify::{RecursiveMode, Watcher};
@@ -11,6 +12,7 @@ use renderer::{RenderInfo, RenderNotif};
 
 mod tui;
 mod renderer;
+mod converter;
 mod skip;
 
 #[tokio::main(flavor = "current_thread")]
@@ -55,7 +57,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 		.map(|n| n.to_string_lossy())
 		.unwrap_or_else(|| "Unknown file".into())
 		.to_string();
-	let mut tui = tui::Tui::new(file_name, picker);
+	let mut tui = tui::Tui::new(file_name);
 
 	let backend = CrosstermBackend::new(std::io::stdout());
 	let mut term = Terminal::new(backend)?;
@@ -65,6 +67,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 	// so we want to just ignore all logging since this is a tui app.
 	glib::log_set_writer_func(noop);
 
+	let mut converter = Converter::new(picker);
+
 	execute!(
 		term.backend_mut(),
 		EnterAlternateScreen,
@@ -79,6 +83,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 		let mut needs_redraw;
 
 		tokio::select! {
+			Some(img_res) = converter.next() => {
+				match img_res {
+					Ok((img, page)) => tui.page_ready(img, page),
+					Err(e) => tui.show_error(e),
+				}
+				needs_redraw = true;
+			},
 			// First we check if we have any keystrokes
 			Some(ev) = ev_stream.next() => {
 				// If we can't get user input, just crash.
@@ -88,22 +99,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 					None => false,
 					Some(InputAction::Redraw) => true,
 					Some(InputAction::QuitApp) => break,
-					Some(InputAction::JumpingToPage(usize)) => {
+					Some(InputAction::ChangePageBy(change)) => {
-						tui_tx.send(RenderNotif::JumpToPage(usize)).await?;
+						converter.change_page_by(change);
+						true
+					},
+					Some(InputAction::JumpingToPage(page)) => {
+						tui_tx.send(RenderNotif::JumpToPage(page)).await?;
+						converter.go_to_page(page);
 						true
 					}
 				};
 			},
 			Some(renderer_msg) = tui_rx.recv() => {
-				match renderer_msg {
+				needs_redraw = match renderer_msg {
-					Ok(RenderInfo::NumPages(num)) => tui.set_n_pages(num),
+					Ok(RenderInfo::NumPages(num)) => {
-					// TODO maybe somehow add more incremental creation of ImageData where it
+						tui.set_n_pages(num);
-					// renders them all to `Box<dyn Protocol>` only when you approach the page but
+						converter.set_n_pages(num);
-					// still renders them to Vec<u8> no matter what?
+						true
-					Ok(RenderInfo::Page(img, page_num)) => tui.page_ready(img, page_num),
+					},
-					Err(e) => tui.show_error(e)
+					Ok(RenderInfo::Page(img, page_num)) => {
-				}
+						converter.add_img(img, page_num);
-				needs_redraw = true;
+						false
+					},
+					Err(e) => {
+						tui.show_error(e);
+						true
+					}
+				};
 			}
 		}
 

src/renderer.rs

@@ -25,8 +25,7 @@ pub enum RenderInfo {
 
 pub struct ImageData {
 	pub data: Vec<u8>,
-	pub width: u16,
+	pub area: Rect
-	pub _height: u16
 }
 
 // this function has to be sync (non-async) because the poppler::Document needs to be held during
@@ -226,7 +225,10 @@ fn render_single_page(
 
 	Ok(ImageData {
 		data: img_data,
-		width: surface_width as u16 / col_w,
+		area: Rect {
-		_height: surface_height as u16 / col_h
+			width: surface_width as u16 / col_w,
+			height: surface_height as u16 / col_h,
+			..Rect::default()
+		}
 	})
 }

src/tui.rs

@@ -1,55 +1,42 @@
 use std::rc::Rc;
 
 use crossterm::event::{Event, KeyCode, MouseEventKind};
-use image::ImageFormat;
 use ratatui::{layout::{Constraint, Flex, Layout, Rect}, style::{Color, Style}, text::Span, widgets::{Block, Borders, Padding}, Frame};
-use ratatui_image::{picker::Picker, protocol::Protocol, Image, Resize};
+use ratatui_image::{protocol::Protocol, Image};
 
-use crate::{renderer::{ImageData, RenderError}, skip::Skip};
+use crate::{renderer::RenderError, skip::Skip};
 
 pub struct Tui {
 	name: String,
 	page: usize,
 	error: Option<String>,
 	input_state: Option<InputCommand>,
-	// Used as a way to track if we need to draw the images, to save ratatui from doing a lot of
-	// diffing work
 	last_render: LastRender,
-	// So we hold the `Picker` here and store the `RenderedImage` as a option between the
+	rendered: Vec<Option<Box<dyn Protocol>>>,
-	// `DynamicImage` and `Box<dyn Protocol>` because the Protocol thing is much less
-	// space-effecient (since it needs to store like a large string instead of just bytes of data)
-	// so we want to store them as `DynamicImage`s until they're shown, at which point we render
-	// them to the `Box<dyn Protocol>` and keep them like that
-	picker: Picker,
-	rendered: Vec<Option<RenderedImage>>,
 }
 
 #[derive(Default, Debug)]
 struct LastRender {
+	// Used as a way to track if we need to draw the images, to save ratatui from doing a lot of
+	// diffing work
 	rect: Rect,
 	pages_shown: usize,
 	unused_width: u16
 }
 
-enum RenderedImage {
-	Data(ImageData),
-	Image(Box<dyn Protocol>)
-}
-
 enum InputCommand {
 	GoToPage(usize)
 }
 
 impl Tui {
-	pub fn new(name: String, picker: Picker) -> Tui {
+	pub fn new(name: String) -> Tui {
 		Self {
 			name,
 			page: 0,
 			error: None,
 			input_state: None,
-			picker,
 			last_render: LastRender::default(),
-			rendered: vec![]
+			rendered: vec![],
 		}
 	}
 
@@ -172,10 +159,7 @@ impl Tui {
 				.flat_map(|(idx, page)|
 					page.as_ref().map(|img| (
 						idx,
-						match img {
+						img.rect().width,
-							RenderedImage::Data(data) => data.width,
-							RenderedImage::Image(img) => img.rect().width,
-						}
 					))
 				)
 				// and then take them as long as they won't overflow the available area.
@@ -192,7 +176,7 @@ impl Tui {
 
 			if page_widths.is_empty() {
 				// If none are ready to render, just show the loading thing
-				Self::render_loading_in(frame, img_area)
+				Self::render_loading_in(frame, img_area);
 			} else {
 				let total_width = page_widths
 					.iter()
@@ -226,36 +210,7 @@ impl Tui {
 
 	fn render_single_page(&mut self, frame: &mut Frame<'_>, page_idx: usize, img_area: Rect) {
 		match self.rendered[page_idx] {
-			Some(ref page_img) => {
+			Some(ref page_img) => frame.render_widget(Image::new(&**page_img), img_area),
-				let txt_img = match page_img {
-					RenderedImage::Data(img_data) => {
-						let dyn_img = image::load_from_memory_with_format(&img_data.data, ImageFormat::Png)
-							.expect("Cairo produced invalid png data; that really shouldn't happen");
-
-						// We don't actually want to Crop this image, but we've already
-						// verified (with the ImageSurface stuff) that the image is the correct
-						// size for the area given, so to save ratatui the work of having to
-						// resize it, we tell them to crop it to fit.
-						let txt_img = match self.picker.new_protocol(dyn_img, img_area, Resize::Crop) {
-							Ok(img) => img,
-							Err(e) => {
-								self.error = Some(format!("Couldn't convert DynamicImage to ratatui image: {e}"));
-								return;
-							}
-						};
-
-						self.rendered[page_idx] = Some(RenderedImage::Image(txt_img));
-						let Some(RenderedImage::Image(ref txt)) = self.rendered[page_idx] else {
-							unreachable!();
-						};
-
-						txt
-					}
-					RenderedImage::Image(ref img) => img,
-				};
-
-				frame.render_widget(Image::new(&**txt_img), img_area);
-			},
 			None => Self::render_loading_in(frame, img_area)
 		};
 	}
@@ -284,7 +239,10 @@ impl Tui {
 			PageChange::Prev => self.set_page(self.page.saturating_sub(diff)),
 		}
 
-		(old != self.page).then_some(InputAction::Redraw)
+		match self.page as isize - old as isize {
+			0 => None,
+			change => Some(InputAction::ChangePageBy(change))
+		}
 	}
 
 	pub fn set_n_pages(&mut self, n_pages: usize) {
@@ -292,25 +250,30 @@ impl Tui {
 		for _ in 0..n_pages {
 			self.rendered.push(None);
 		}
-		// mark that we need to re-render the images
+		self.page = self.page.min(n_pages - 1);
 	}
 
-	pub fn page_ready(&mut self, img: ImageData, page_num: usize) {
+	pub fn page_ready(&mut self, img: Box<dyn Protocol>, page_num: usize) {
 		// If this new image woulda fit within the available space on the last render AND it's
 		// within the range where it might've been rendered with the last shown pages, then reset
 		// the last rect marker so that all images are forced to redraw on next render and this one
 		// is drawn with them
-		if img.width <= self.last_render.unused_width {
+		if page_num == self.page {
-			let num_fit = self.last_render.unused_width / img.width;
+			self.last_render.rect = Rect::default();
-			if page_num >= self.page && (self.page + num_fit as usize) >= page_num {
+		} else {
-				self.last_render.rect = Rect::default();
+			let img_w = img.rect().width;
+			if img_w <= self.last_render.unused_width {
+				let num_fit = self.last_render.unused_width / img_w;
+				if page_num >= self.page && (self.page + num_fit as usize) >= page_num {
+					self.last_render.rect = Rect::default();
+				}
 			}
 		}
 
 		// We always just set this here because we handle reloading in the `set_n_pages` function.
 		// If the document was reloaded, then It'll have the `set_n_pages` called to set the new
 		// number of pages, so the vec will already be cleared
-		self.rendered[page_num] = Some(RenderedImage::Data(img));
+		self.rendered[page_num] = Some(img);
 	}
 
 	pub fn handle_event(&mut self, ev: Event) -> Option<InputAction> {
@@ -380,6 +343,7 @@ impl Tui {
 
 pub enum InputAction {
 	Redraw,
+	ChangePageBy(isize),
 	JumpingToPage(usize),
 	QuitApp
 }