diff --git a/pillow_jxl/JpegXLImagePlugin.py b/pillow_jxl/JpegXLImagePlugin.py
index 39fd485..7021572 100644
--- a/pillow_jxl/JpegXLImagePlugin.py
+++ b/pillow_jxl/JpegXLImagePlugin.py
@@ -121,9 +121,9 @@ def _save(im, fp, filename, save_all=False):
         if exif and exif.startswith(b"Exif\x00\x00"):
             exif = exif[6:]
         metadata = {
-            "exif": exif if exif is not None else b"",
-            "jumb": info.get("jumb", b""),
-            "xmp": info.get("xmp", b""),
+            "exif": exif,
+            "jumb": info.get("jumb"),
+            "xmp": info.get("xmp"),
         }
         data = enc(im.tobytes(), im.width, im.height, jpeg_encode=False, **metadata)
     fp.write(data)
diff --git a/src/encode.rs b/src/encode.rs
index 2f7777a..2635b4f 100644
--- a/src/encode.rs
+++ b/src/encode.rs
@@ -118,15 +118,21 @@ impl Encoder {
             true => encoder.encode_jpeg(&data).unwrap(),
             false => {
                 let frame = EncoderFrame::new(data).num_channels(self.num_channels);
-                encoder
-                    .add_metadata(&Metadata::Exif(exif.unwrap()), true)
-                    .unwrap();
-                encoder
-                    .add_metadata(&Metadata::Xmp(xmp.unwrap()), true)
-                    .unwrap();
-                encoder
-                    .add_metadata(&Metadata::Jumb(jumb.unwrap()), true)
-                    .unwrap();
+                if let Some(exif_data) = exif {
+                    encoder
+                        .add_metadata(&Metadata::Exif(exif_data), true)
+                        .unwrap();
+                }
+                if let Some(xmp_data) = xmp {
+                    encoder
+                        .add_metadata(&Metadata::Xmp(xmp_data), true)
+                        .unwrap();
+                }
+                if let Some(jumb_data) = jumb {
+                    encoder
+                        .add_metadata(&Metadata::Jumb(jumb_data), true)
+                        .unwrap();
+                }
                 encoder.encode_frame(&frame, width, height).unwrap()
             }
         };