From 5825849434ce6978fb96a2304ae000f282ab25c5 Mon Sep 17 00:00:00 2001 From: itsjunetime Date: Tue, 4 Jun 2024 15:46:25 -0600 Subject: [PATCH] - Significantly improved time to render full document - Added support for debugging with tokio-console through tracing feature - Added extra benchmark for checking time to render first page - Removed unwraps to just make background loops return and terminate if something goes wrong - Modularize rendering somewhat --- Cargo.lock | 602 ++++++++++++++++++++++++++++++++++++++- Cargo.toml | 7 + benches/for_profiling.rs | 3 + benches/rendering.rs | 53 ++-- benches/utils.rs | 137 ++++++--- src/converter.rs | 3 +- src/main.rs | 9 +- src/renderer.rs | 165 +++++++---- 8 files changed, 846 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2827128..3fe4632 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,12 +56,96 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.72" @@ -77,6 +161,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -113,6 +203,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + [[package]] name = "cairo-rs" version = "0.19.4" @@ -245,6 +341,43 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "console-api" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -426,6 +559,12 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -608,6 +747,25 @@ dependencies = [ "system-deps", ] +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.4.1" @@ -618,6 +776,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -628,6 +792,19 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.5.0" @@ -640,6 +817,82 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "icy_sixel" version = "0.1.2" @@ -661,6 +914,16 @@ dependencies = [ "zune-jpeg", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -668,7 +931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -764,6 +1027,12 @@ dependencies = [ "libc", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.155" @@ -798,15 +1067,42 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.3" @@ -829,6 +1125,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "notify" version = "6.1.1" @@ -917,6 +1223,32 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -1027,6 +1359,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.36" @@ -1090,7 +1454,7 @@ name = "ratatui-image" version = "1.0.0" source = "git+https://github.com/itsjunetime/ratatui-image.git?branch=vb64_on_personal#e5c13ed29c9decdff093c2be5d673d84fb3589a9" dependencies = [ - "base64", + "base64 0.22.1", "dyn-clone", "icy_sixel", "image", @@ -1146,8 +1510,17 @@ checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1158,9 +1531,15 @@ checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.3", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.3" @@ -1253,6 +1632,15 @@ dependencies = [ "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook" version = "0.3.17" @@ -1304,6 +1692,16 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "spin" version = "0.9.8" @@ -1362,6 +1760,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "system-deps" version = "6.2.2" @@ -1386,6 +1790,7 @@ name = "tdf" version = "0.1.0" dependencies = [ "cairo-rs", + "console-subscriber", "criterion", "crossterm", "flume", @@ -1420,6 +1825,16 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -1437,9 +1852,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", + "bytes", + "libc", + "mio", "num_cpus", "pin-project-lite", + "socket2", "tokio-macros", + "tracing", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", ] [[package]] @@ -1453,6 +1884,30 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.14" @@ -1480,7 +1935,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap", + "indexmap 2.2.6", "toml_datetime", "winnow 0.5.40", ] @@ -1491,13 +1946,125 @@ version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ - "indexmap", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", "winnow 0.6.11", ] +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "once_cell", + "regex", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unicode-ident" version = "1.0.12" @@ -1522,9 +2089,15 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "vb64" @@ -1553,6 +2126,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 6fa2be3..8f8f047 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,17 @@ glib = "0.19.6" itertools = "*" flume = { version = "0.11.0", default-features = false, features = ["async"] } +# for tracing with tokio-console +console-subscriber = { version = "0.2.0", optional = true } + [profile.production] inherits = "release" lto = "fat" +[features] +default = [] +tracing = ["tokio/tracing", "dep:console-subscriber"] + [dev-dependencies] criterion = "0.5.1" diff --git a/benches/for_profiling.rs b/benches/for_profiling.rs index 1879d7c..a9d4142 100644 --- a/benches/for_profiling.rs +++ b/benches/for_profiling.rs @@ -2,6 +2,9 @@ mod utils; #[tokio::main] async fn main() { + #[cfg(feature = "tracing")] + console_subscriber::init(); + let file = std::env::args() .nth(1) .expect("Please enter a file to profile"); diff --git a/benches/rendering.rs b/benches/rendering.rs index df1352a..0f4bd8e 100644 --- a/benches/rendering.rs +++ b/benches/rendering.rs @@ -1,34 +1,45 @@ mod utils; -use utils::render_doc; +use utils::{render_doc, render_first_page}; -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -fn render_dict(c: &mut Criterion) { - c.bench_function( - "example dictionary", - |b| b.iter(|| - tokio::runtime::Runtime::new() - .unwrap() - .block_on(render_doc("./benches/example_dictionary.pdf")) - ) - ); +const FILES: [&str; 2] = [ + "./benches/example_dictionary.pdf", + "./benches/adobe_example.pdf" +]; + +fn render_full(c: &mut Criterion) { + for file in FILES { + c.bench_with_input( + BenchmarkId::new("render_full", file), + &file, + |b, &file| b.iter(|| + tokio::runtime::Runtime::new() + .unwrap() + .block_on(render_doc(file)) + ) + ); + } } -fn render_example(c: &mut Criterion) { - c.bench_function( - "adobe-provided sample", - |b| b.iter(|| - tokio::runtime::Runtime::new() - .unwrap() - .block_on(render_doc("./benches/adobe_example.pdf")) - ) - ); +fn render_to_first_page(c: &mut Criterion) { + for file in FILES { + c.bench_with_input( + BenchmarkId::new("render_first_page", file), + &file, + |b, &file| b.iter(|| + tokio::runtime::Runtime::new() + .unwrap() + .block_on(render_first_page(file)) + ) + ); + } } criterion_group!( name = benches; config = Criterion::default().sample_size(10); - targets = render_dict, render_example + targets = render_full, render_to_first_page ); criterion_main!(benches); diff --git a/benches/utils.rs b/benches/utils.rs index 4ce6a19..ca43048 100644 --- a/benches/utils.rs +++ b/benches/utils.rs @@ -1,13 +1,55 @@ use std::{hint::black_box, path::Path}; use crossterm::terminal::WindowSize; -use flume::{unbounded, Sender}; +use flume::{r#async::RecvStream, unbounded, Sender}; use ratatui::layout::Rect; use ratatui_image::picker::{Picker, ProtocolType}; use tdf::{converter::{run_conversion_loop, ConvertedPage, ConverterMsg}, renderer::{fill_default, start_rendering, RenderError, RenderInfo, RenderNotif}}; use futures_util::stream::StreamExt as _; -pub async fn render_doc(path: impl AsRef) { +fn handle_renderer_msg( + msg: Result, + pages: &mut Vec>, + to_converter_tx: &mut Sender, +) { + match msg { + Ok(RenderInfo::NumPages(num)) => { + fill_default(pages, num); + to_converter_tx.send(ConverterMsg::NumPages(num)).unwrap(); + }, + Ok(RenderInfo::Page(info)) => to_converter_tx.send(ConverterMsg::AddImg(info)).unwrap(), + Err(e) => panic!("Got error from renderer: {e:?}") + } +} + +fn handle_converter_msg( + msg: Result, + pages: &mut [Option], + to_converter_tx: &mut Sender +) { + let page = msg.expect("Got error from converter"); + let num = page.num; + + pages[num] = Some(page); + + let num_got = pages.iter() + .filter(|p| p.is_some()) + .count(); + + // we have to tell it to jump to a certain page so that it will actually render it (since + // it only renders fanning out from the page that we currently have selected) + to_converter_tx.send(ConverterMsg::GoToPage(num_got)).unwrap(); +} + +struct RenderState { + from_render_rx: RecvStream<'static, Result>, + from_converter_rx: RecvStream<'static, Result>, + pages: Vec>, + to_converter_tx: Sender, + to_render_tx: Sender +} + +fn start_all_rendering(path: impl AsRef) -> RenderState { let pathbuf = path.as_ref().canonicalize().unwrap(); let str_path = format!("file://{}", pathbuf.into_os_string().to_string_lossy()); @@ -28,7 +70,7 @@ pub async fn render_doc(path: impl AsRef) { start_rendering(str_path, to_main_tx, from_main_rx, size) }); - let (mut to_converter_tx, from_main_rx) = unbounded(); + let (to_converter_tx, from_main_rx) = unbounded(); let (to_main_tx, from_converter_rx) = unbounded(); let mut picker = Picker::new(font_size); @@ -36,41 +78,7 @@ pub async fn render_doc(path: impl AsRef) { tokio::spawn(run_conversion_loop(to_main_tx, from_main_rx, picker)); - let mut pages: Vec> = Vec::new(); - - fn handle_renderer_msg( - msg: Result, - pages: &mut Vec>, - to_converter_tx: &mut Sender, - ) { - match msg { - Ok(RenderInfo::NumPages(num)) => { - fill_default(pages, num); - to_converter_tx.send(ConverterMsg::NumPages(num)).unwrap(); - }, - Ok(RenderInfo::Page(info)) => to_converter_tx.send(ConverterMsg::AddImg(info)).unwrap(), - Err(e) => panic!("Got error from renderer: {e:?}") - } - } - - fn handle_converter_msg( - msg: Result, - pages: &mut [Option], - to_converter_tx: &mut Sender - ) { - let page = msg.expect("Got error from converter"); - let num = page.num; - - pages[num] = Some(page); - - let num_got = pages.iter() - .filter(|p| p.is_some()) - .count(); - - // we have to tell it to jump to a certain page so that it will actually render it (since - // it only renders fanning out from the page that we currently have selected) - to_converter_tx.send(ConverterMsg::GoToPage(num_got)).unwrap(); - } + let pages: Vec> = Vec::new(); let main_area = Rect { x: 0, @@ -80,8 +88,26 @@ pub async fn render_doc(path: impl AsRef) { }; to_render_tx.send(RenderNotif::Area(main_area)).unwrap(); - let mut from_render_rx = from_render_rx.into_stream(); - let mut from_converter_rx = from_converter_rx.into_stream(); + let from_render_rx = from_render_rx.into_stream(); + let from_converter_rx = from_converter_rx.into_stream(); + + RenderState { + from_render_rx, + from_converter_rx, + pages, + to_converter_tx, + to_render_tx + } +} + +pub async fn render_doc(path: impl AsRef) { + let RenderState { + mut from_render_rx, + mut from_converter_rx, + mut pages, + mut to_converter_tx, + to_render_tx + } = start_all_rendering(path); while pages.is_empty() || pages.iter().any(|p| p.is_none()) { tokio::select! { @@ -95,4 +121,35 @@ pub async fn render_doc(path: impl AsRef) { } black_box(pages); + // we want to make sure this is kept around until the end of this function, or else the other + // thread will see that this is disconnected and think that we're done communicating with them + drop(to_render_tx); +} + +#[cfg(test)] +pub async fn render_first_page(path: impl AsRef) { + let RenderState { + mut from_render_rx, + mut from_converter_rx, + mut pages, + mut to_converter_tx, + to_render_tx + } = start_all_rendering(path); + + // we only want to render until the first page is ready to be printed + while pages.is_empty() { + tokio::select! { + Some(renderer_msg) = from_render_rx.next() => { + handle_renderer_msg(renderer_msg, &mut pages, &mut to_converter_tx); + }, + Some(converter_msg) = from_converter_rx.next() => { + handle_converter_msg(converter_msg, &mut pages, &mut to_converter_tx); + } + } + } + + black_box(pages); + // we want to make sure this is kept around until the end of this function, or else the other + // thread will see that this is disconnected and think that we're done communicating with them + drop(to_render_tx); } diff --git a/src/converter.rs b/src/converter.rs index 78a3832..55be9bb 100644 --- a/src/converter.rs +++ b/src/converter.rs @@ -107,7 +107,8 @@ pub async fn run_conversion_loop( continue 'outer; } Err(TryRecvError::Empty) => (), - Err(TryRecvError::Disconnected) => panic!("Disconnected :(") + // if it's disconnected, we're done. just return. + Err(TryRecvError::Disconnected) => return Ok(()) } match next_page(&mut images, &mut picker, page, &mut iteration) { diff --git a/src/main.rs b/src/main.rs index 3918513..e44e5e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,6 +41,9 @@ impl std::error::Error for BadTermSizeStdin {} #[tokio::main] async fn main() -> Result<(), Box> { + #[cfg(feature = "tracing")] + console_subscriber::init(); + let file = std::env::args().nth(1).ok_or("Program requires a file to process")?; let path = PathBuf::from_str(&file)?.canonicalize()?; @@ -53,9 +56,9 @@ async fn main() -> Result<(), Box> { // calling thread::spawn) and that will cause a panic let mut watcher = notify::recommended_watcher(move |_| { // This shouldn't fail to send unless the receiver gets disconnected. If that's happened, - // then like the main thread has panicked or something, so it doesn't matter if this panics - // as well - watch_tx.send(renderer::RenderNotif::Reload).unwrap(); + // then like the main thread has panicked or something, so it doesn't matter we don't + // handle the error here + _ = watch_tx.send(renderer::RenderNotif::Reload); })?; // We're making this nonrecursive 'cause we're just watching a single file, so there's nothing diff --git a/src/renderer.rs b/src/renderer.rs index 2c4ac2d..255de73 100644 --- a/src/renderer.rs +++ b/src/renderer.rs @@ -1,6 +1,8 @@ -use cairo::{Antialias, Format}; +use std::thread; + +use cairo::{Antialias, Context, Format, Surface}; use crossterm::terminal::WindowSize; -use flume::{Receiver, Sender, TryRecvError}; +use flume::{Receiver, SendError, Sender, TryRecvError}; use itertools::Itertools; use poppler::{Color, Document, FindFlags, Page, Rectangle, SelectionStyle}; use ratatui::layout::Rect; @@ -61,10 +63,10 @@ pub fn fill_default(vec: &mut Vec, size: usize) { // we're done. pub fn start_rendering( path: String, - sender: Sender>, + mut sender: Sender>, receiver: Receiver, size: WindowSize -) { +) -> Result<(), SendError>> { // first, wait 'til we get told what the current starting area is so that we can set it to // know what to render to let mut area; @@ -79,17 +81,19 @@ pub fn start_rendering( // set will still get highlighted in the reloaded doc let mut search_term = None; + // And although the font size could theoretically change, we aren't accounting for that right + // now, so we just keep this out of the loop. + let col_w = size.width / size.columns; + let col_h = size.height / size.rows; + 'reload: loop { let doc = match Document::from_file(&path, None) { - Err(e) => { - sender.send(Err(RenderError::Doc(e))).unwrap(); - return; - } + Err(e) => return sender.send(Err(RenderError::Doc(e))), Ok(d) => d }; let n_pages = doc.n_pages() as usize; - sender.send(Ok(RenderInfo::NumPages(n_pages))).unwrap(); + sender.send(Ok(RenderInfo::NumPages(n_pages)))?; // We're using this vec of bools to indicate which page numbers have already been rendered, // to support people jumping to specific pages and having quick rendering results. We @@ -168,6 +172,9 @@ pub fn start_rendering( .map(|(idx, p)| (start_point - (idx + 1), p)) ); + let area_w = area.width as f64 * col_w as f64; + let area_h = area.height as f64 * col_h as f64; + // we go through each page for (num, rendered) in page_iter { // we only want to continue if one of the following is met: @@ -182,19 +189,20 @@ pub fn start_rendering( // check if we've been told to change the area that we're rendering to, // or if we're told to rerender match receiver.try_recv() { - Err(TryRecvError::Disconnected) => panic!("disconnected :("), + // If it's disconnected, then the main loop is done, so we should just give up + Err(TryRecvError::Disconnected) => return Ok(()), Ok(notif) => handle_notif!(notif), Err(TryRecvError::Empty) => () }; - // We know this is in range 'cause we're iterating over it + // We know this is in range 'cause we're iterating over it but we still just want + // to be safe let Some(page) = doc.page(num as i32) else { sender .send(Err(RenderError::Render(format!( "Couldn't get page {num} ({}) of doc?", num as i32 - )))) - .unwrap(); + ))))?; continue; }; @@ -202,37 +210,51 @@ pub fn start_rendering( rendered.successful && rendered.contained_term == Some(false); // render the page - match render_single_page( + match render_single_page_to_ctx( page, - area, - num, &search_term, rendered_with_no_results, - &size + (area_w, area_h) ) { // If we've already rendered it just fine and we don't need to render it again, // just continue. We're all good Ok(None) => (), // If that fn returned Some, that means it needed to be re-rendered for some // reason or another, so we're sending it here - Ok(Some(img)) => { - // But we first need to store if we already rendered it correctly so that - // the next time we iterate through, it might see that we're already good - rendered.contained_term = Some(img.search_results > 0); + Ok(Some(ctx)) => { + // we make a potentially incorrect assumption here that writing the context + // to a png won't fail, and mark that it all rendered correctly here before + // spawning off the thread to do so and send it. + rendered.contained_term = Some(ctx.num_results > 0); rendered.successful = true; - sender.send(Ok(RenderInfo::Page(img))).unwrap() - } + + // if this is the page that the user is currently trying to look at, don't + // bother spawning off a thread to render it to a png - it'll only slow + // down the time til the user can see it (due to the overhead of creating a + // thread), but we still want to spawn threads to render the other pages + // since the effects of parallelizing that will be noticeable if the user + // tries to move through pages more quickly + if num == start_point { + render_ctx_to_png(ctx, &mut sender, (col_w, col_h), num)?; + } else { + let mut sender = sender.clone(); + thread::spawn(move || { + render_ctx_to_png(ctx, &mut sender, (col_w, col_h), num) + }); + } + }, // And if we got an error, then obviously we need to propagate that - Err(e) => sender.send(Err(RenderError::Render(e))).unwrap() + Err(e) => sender.send(Err(RenderError::Render(e)))? } } + // Then once we've rendered all these pages, wait until we get another notification // that this doc needs to be reloaded loop { // This once returned None despite the main thing being still connected (I think, at - // last), so I'm just being safe here + // least), so I'm just being safe here let Ok(msg) = receiver.recv() else { - return; + return Ok(()); }; handle_notif!(msg); } @@ -240,14 +262,31 @@ pub fn start_rendering( } } -fn render_single_page( +struct RenderedContext { + surface: Surface, + num_results: usize, + surface_width: f64, + surface_height: f64 +} + +/// SAFETY: I think this is safe because, although the backing struct for `Surface` does contain +/// pointers to like the cairo_backend_t struct that all the cairo stuff is using, that struct is +/// basically just a vtable, so accessing it from multiple threads *should* be safe since we're +/// just calling the same functions with different data. The only other thing it holds reference to +/// is a `cairo_device_t`, but that seems to be thread-safe because it's managed through ref counts +/// and a mutex. Also, as far as I can tell from reading the source code, write_to_png_stream (the +/// only function we call on this struct) doesn't access the device at all, so we should be fine +/// there. +/// We want this to be Send so that we can delegate the png writing to a separate thread (since +/// that's the thing that takes the most time, by far, in this app). +unsafe impl Send for RenderedContext {} + +fn render_single_page_to_ctx( page: Page, - area: Rect, - page_num: usize, search_term: &Option, already_rendered_no_results: bool, - size: &WindowSize -) -> Result, String> { + (area_w, area_h): (f64, f64), +) -> Result, String> { let mut result_rects = search_term .as_ref() .map(|term| page.find_text_with_options(term, FindFlags::DEFAULT | FindFlags::MULTILINE)) @@ -259,11 +298,6 @@ fn render_single_page( return Ok(None); } - // First, get the font size; the number of pixels (width x height) per font character (I - // think; it's at least something like that) on this terminal screen. - let col_h = size.height / size.rows; - let col_w = size.width / size.columns; - // then, get the size of the page let (p_width, p_height) = page.size(); @@ -272,9 +306,7 @@ fn render_single_page( // Then we get the full pixel dimensions of the area provided to us, and the aspect ratio // of that area - let area_full_h = (area.height * col_h) as f64; - let area_full_w = (area.width * col_w) as f64; - let area_aspect_ratio = area_full_w / area_full_h; + let area_aspect_ratio = area_w / area_h; // and get the ratio that this page would have to be scaled by to fit perfectly within the // area provided to us. @@ -284,9 +316,9 @@ fn render_single_page( // scale the height to fit perfectly. The dimension that _is not_ scaled to fit perfectly // is scaled by the same factor as the dimension that _is_ scaled perfectly. let scale_factor = if p_aspect_ratio > area_aspect_ratio { - area_full_w / p_width + area_w / p_width } else { - area_full_h / p_height + area_h / p_height }; let surface_width = p_width * scale_factor; @@ -308,7 +340,7 @@ fn render_single_page( .map_err(|e| format!("Couldn't create ImageSurface: {e}"))?; surface.set_device_scale(scale_factor, scale_factor); - let ctx = cairo::Context::new(surface).map_err(|e| format!("Couldn't create Context: {e}"))?; + let ctx = Context::new(surface).map_err(|e| format!("Couldn't create Context: {e}"))?; // The default background color of PDFs (at least, I think) is white, so we need to set // that as the background color, then paint, then render. @@ -344,21 +376,38 @@ fn render_single_page( } } - let mut img_data = Vec::with_capacity((surface_height * surface_width) as usize); - ctx.target() - .write_to_png(&mut img_data) - .map_err(|e| format!("Couldn't write surface to png: {e}"))?; - - Ok(Some(PageInfo { - img_data: ImageData { - data: img_data, - area: Rect { - width: surface_width as u16 / col_w, - height: surface_height as u16 / col_h, - ..Rect::default() - } - }, - page: page_num, - search_results: num_results + Ok(Some(RenderedContext { + surface: ctx.target(), + num_results, + surface_width, + surface_height })) } + +fn render_ctx_to_png( + ctx: RenderedContext, + sender: &mut Sender>, + (col_w, col_h): (u16, u16), + page: usize +) -> Result<(), SendError>> { + let mut img_data = Vec::with_capacity((ctx.surface_height * ctx.surface_width) as usize); + + match ctx.surface.write_to_png(&mut img_data) { + Err(e) => sender.send(Err(RenderError::Render( + format!("Couldn't write surface to png: {e}") + ))), + Ok(()) => sender.send(Ok(RenderInfo::Page(PageInfo { + img_data: ImageData { + data: img_data, + area: Rect { + width: ctx.surface_width as u16 / col_w, + height: ctx.surface_height as u16 / col_h, + x: 0, + y: 0 + } + }, + page, + search_results: ctx.num_results + }))) + } +}