IconForge

Start blending

Huge cleanup

Finish optimizing the thing

Finish the thing!!

Clean up a bit

Re-add 32-bit thing

Fix TOML sorting

Add dmsrc

Fix clippy suggestions

Clippy.. stop being mean

Cargo fmt + doc comments

Code cleanup

More cleanup, remove most unsafe unwrap()s, use Match syntax.

Remove unneccesarily verbose casting

Fix overlay blending

Cleanup with new DMI version

Cargo fmt

Leaf test, DynamicImage->RgbaImage, better Error handling, DashMap, and cleanup command

Fix

Further tree optimizations, hashing optimization, cache icostrings more effectively.

Optimize unique_icons insertion a little

Fix macro

Little more cleanup

Add to README

Update dmi, add caching logic.

Address reviews

Cleanup panic unwind

Fix lint failure

Cargo.toml

@@ -53,7 +53,7 @@ serde_json = { version = "1.0", optional = true }
 lazy_static = { version = "1.4", optional = true }
 once_cell = { version = "1.19", optional = true }
 mysql = { version = "24.0", default_features = false, optional = true }
-dashmap = { version = "5.5", optional = true }
+dashmap = { version = "5.5", optional = true, features = ["rayon", "serde"] }
 zip = { version = "0.6", optional = true }
 rand = { version = "0.8", optional = true }
 toml-dep = { version = "0.8.8", package = "toml", optional = true }
@@ -62,7 +62,8 @@ rayon = { version = "1.8", optional = true }
 dbpnoise = { version = "0.1.2", optional = true }
 pathfinding = { version = "4.4", optional = true }
 num-integer = { version = "0.1.45", optional = true }
-dmi = { version = "0.3.1", optional = true }
+dmi = { version = "0.3.4", optional = true }
+tracy_full = { version = "1.6.1", optional = true }
 
 [features]
 default = [
@@ -89,6 +90,7 @@ all = [
     "file",
     "git",
     "http",
+    "iconforge",
     "json",
     "log",
     "noise",
@@ -133,6 +135,20 @@ hash = [
     "serde",
     "serde_json",
 ]
+iconforge = [
+    "dashmap",
+    "dep:dmi",
+    "hash",
+    "image",
+    "jobs",
+    "once_cell",
+    "png",
+    "rayon",
+    "serde",
+    "serde_json",
+    "tracy_full",
+    "twox-hash",
+]
 pathfinder = ["num-integer", "pathfinding", "serde", "serde_json"]
 redis_pubsub = ["flume", "redis", "serde", "serde_json"]
 redis_reliablequeue = ["flume", "redis", "serde", "serde_json"]

README.md

@@ -103,6 +103,7 @@ The default features are:
 Additional features are:
 * batchnoise: Discrete Batched Perlin-like Noise, fast and multi-threaded - sent over once instead of having to query for every tile.
 * hash: Faster replacement for `md5`, support for SHA-1, SHA-256, and SHA-512. Requires OpenSSL on Linux.
+* iconforge: A much faster replacement for the spritesheet generation system used by [/tg/station].
 * pathfinder: An a* pathfinder used for finding the shortest path in a static node map. Not to be used for a non-static map.
 * redis_pubsub: Library for sending and receiving messages through Redis.
 * redis_reliablequeue: Library for using a reliable queue pattern through Redis.

dmsrc/iconforge.dm

@@ -0,0 +1,59 @@
+/// Generates a spritesheet at: [file_path][spritesheet_name]_[size_id].png
+/// The resulting spritesheet arranges icons in a random order, with the position being denoted in the "sprites" return value.
+/// All icons have the same y coordinate, and their x coordinate is equal to `icon_width * position`.
+///
+/// hash_icons is a boolean (0 or 1), and determines if the generator will spend time creating hashes for the output field dmi_hashes.
+/// These hashes can be heplful for 'smart' caching (see rustg_iconforge_cache_valid), but require extra computation.
+///
+/// Spritesheet will contain all sprites listed within "sprites".
+/// "sprites" format:
+/// list(
+///     "sprite_name" = list( // <--- this list is a [SPRITE_OBJECT]
+///         icon_file = 'icons/path_to/an_icon.dmi',
+///         icon_state = "some_icon_state",
+///         dir = SOUTH,
+///         frame = 1,
+///         transform = list([TRANSFORM_OBJECT], ...)
+///     ),
+///     ...,
+/// )
+/// TRANSFORM_OBJECT format:
+/// list("type" = "BlendColor", "color" = "#ff0000", "blend_mode" = ICON_MULTIPLY)
+/// list("type" = "BlendIcon", "icon" = [SPRITE_OBJECT], "blend_mode" = ICON_OVERLAY)
+/// list("type" = "Scale", "width" = 32, "height" = 32)
+/// list("type" = "Crop", "x1" = 0, "y1" = 0, "x2" = 32, "y2" = 32)
+///
+/// Returns a SpritesheetResult as JSON, containing fields:
+/// list(
+///     "sizes" = list("32x32", "64x64", ...),
+///     "sprites" = list("sprite_name" = list("size_id" = "32x32", "position" = 0), ...),
+///     "dmi_hashes" = list("icons/path_to/an_icon.dmi" = "d6325c5b4304fb03", ...),
+///     "sprites_hash" = "a2015e5ff403fb5c", // This is the xxh64 hash of the INPUT field "sprites".
+///     "error" = "[A string, empty if there were no errors.]"
+/// )
+/// In the case of an unrecoverable panic from within Rust, this function ONLY returns a string containing the error.
+#define rustg_iconforge_generate(file_path, spritesheet_name, sprites, hash_icons) RUSTG_CALL(RUST_G, "iconforge_generate")(file_path, spritesheet_name, sprites, "[hash_icons]")
+/// Returns a job_id for use with rustg_iconforge_check()
+#define rustg_iconforge_generate_async(file_path, spritesheet_name, sprites, hash_icons) RUSTG_CALL(RUST_G, "iconforge_generate_async")(file_path, spritesheet_name, sprites, "[hash_icons]")
+/// Returns the status of a job_id
+#define rustg_iconforge_check(job_id) RUSTG_CALL(RUST_G, "iconforge_check")("[job_id]")
+/// Clears all cached DMIs and images, freeing up memory.
+/// This should be used after spritesheets are done being generated.
+#define rustg_iconforge_cleanup RUSTG_CALL(RUST_G, "iconforge_cleanup")
+/// Takes in a set of hashes, generate inputs, and DMI filepaths, and compares them to determine cache validity.
+/// input_hash: xxh64 hash of "sprites" from the cache.
+/// dmi_hashes: xxh64 hashes of the DMIs in a spritesheet, given by `rustg_iconforge_generate` with `hash_icons` enabled. From the cache.
+/// sprites: The new input that will be passed to rustg_iconforge_generate().
+/// Returns a CacheResult with the following structure: list(
+///     "result": "1" (if cache is valid) or "0" (if cache is invalid)
+///     "fail_reason": "" (emtpy string if valid, otherwise a string containing the invalidation reason or an error with ERROR: prefixed.)
+/// )
+/// In the case of an unrecoverable panic from within Rust, this function ONLY returns a string containing the error.
+#define rustg_iconforge_cache_valid(input_hash, dmi_hashes, sprites) RUSTG_CALL(RUST_G, "iconforge_cache_valid")(input_hash, dmi_hashes, sprites)
+/// Returns a job_id for use with rustg_iconforge_check()
+#define rustg_iconforge_cache_valid_async(input_hash, dmi_hashes, sprites) RUSTG_CALL(RUST_G, "iconforge_cache_valid_async")(input_hash, dmi_hashes, sprites)
+
+#define RUSTG_ICONFORGE_BLEND_COLOR "BlendColor"
+#define RUSTG_ICONFORGE_BLEND_ICON "BlendIcon"
+#define RUSTG_ICONFORGE_CROP "Crop"
+#define RUSTG_ICONFORGE_SCALE "Scale"

src/byond.rs

@@ -1,11 +1,17 @@
+use crate::error::Error;
 use std::{
+    backtrace::Backtrace,
     borrow::Cow,
     cell::RefCell,
     ffi::{CStr, CString},
+    fs::OpenOptions,
+    io::Write,
     os::raw::{c_char, c_int},
     slice,
+    sync::Once,
 };
 
+static SET_HOOK: Once = Once::new();
 static EMPTY_STRING: c_char = 0;
 thread_local! {
     static RETURN_STRING: RefCell<CString> = RefCell::new(CString::default());
@@ -50,6 +56,7 @@ macro_rules! byond_fn {
         pub unsafe extern "C" fn $name(
             _argc: ::std::os::raw::c_int, _argv: *const *const ::std::os::raw::c_char
         ) -> *const ::std::os::raw::c_char {
+            $crate::byond::set_panic_hook();
             let closure = || ($body);
             $crate::byond::byond_return(closure().map(From::from))
         }
@@ -84,3 +91,53 @@ byond_fn!(
         Some(env!("CARGO_PKG_VERSION"))
     }
 );
+
+/// Print any panics before exiting.
+pub fn set_panic_hook() {
+    SET_HOOK.call_once(|| {
+        std::panic::set_hook(Box::new(|panic_info| {
+            let mut file = OpenOptions::new()
+                .write(true)
+                .append(true)
+                .create(true)
+                .open("rustg-panic.log")
+                .unwrap();
+            file.write_all(
+                panic_info
+                    .payload()
+                    .downcast_ref::<&'static str>()
+                    .map(|payload| payload.to_string())
+                    .or_else(|| panic_info.payload().downcast_ref::<String>().cloned())
+                    .unwrap()
+                    .as_bytes(),
+            )
+            .expect("Failed to extract error payload");
+            file.write_all(Backtrace::capture().to_string().as_bytes())
+                .expect("Failed to extract error backtrace");
+        }))
+    });
+}
+
+/// Utility for BYOND functions to catch panic unwinds safely and return a Result<String, Error>, as expected.
+/// Usage: catch_panic(|| internal_safe_function(arguments))
+pub fn catch_panic<F>(f: F) -> Result<String, Error>
+where
+    F: FnOnce() -> Result<String, Error> + std::panic::UnwindSafe,
+{
+    match std::panic::catch_unwind(f) {
+        Ok(o) => o,
+        Err(e) => {
+            let message: Option<String> = e
+                .downcast_ref::<&'static str>()
+                .map(|payload| payload.to_string())
+                .or_else(|| e.downcast_ref::<String>().cloned());
+            Err(Error::Panic(
+                message
+                    .unwrap_or(String::from(
+                        "Failed to stringify panic! Check rustg-panic.log!",
+                    ))
+                    .to_owned(),
+            ))
+        }
+    }
+}

src/error.rs

@@ -61,6 +61,11 @@ pub enum Error {
     #[cfg(feature = "hash")]
     #[error("Unable to decode hex value.")]
     HexDecode,
+    #[cfg(feature = "iconforge")]
+    #[error("IconForge error: {0}")]
+    IconForge(String),
+    #[error("Panic during function execution: {0}")]
+    Panic(String),
 }
 
 impl From<Utf8Error> for Error {

src/hash.rs

@@ -71,16 +71,21 @@ fn hash_algorithm<B: AsRef<[u8]>>(name: &str, bytes: B) -> Result<String> {
             hasher.write(bytes.as_ref());
             Ok(format!("{:x}", hasher.finish()))
         }
+        "xxh64_fixed" => {
+            let mut hasher = XxHash64::with_seed(17479268743136991876);
+            hasher.write(bytes.as_ref());
+            Ok(format!("{:x}", hasher.finish()))
+        }
         "base64" => Ok(base64::prelude::BASE64_STANDARD.encode(bytes.as_ref())),
         _ => Err(Error::InvalidAlgorithm),
     }
 }
 
-fn string_hash(algorithm: &str, string: &str) -> Result<String> {
+pub fn string_hash(algorithm: &str, string: &str) -> Result<String> {
     hash_algorithm(algorithm, string)
 }
 
-fn file_hash(algorithm: &str, path: &str) -> Result<String> {
+pub fn file_hash(algorithm: &str, path: &str) -> Result<String> {
     let mut bytes: Vec<u8> = Vec::new();
     let mut file = BufReader::new(File::open(path)?);
     file.read_to_end(&mut bytes)?;