diff --git a/tests/test_core.py b/tests/test_core.py
index 688c04b..e1425ff 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -31,6 +31,7 @@ def test_to_xarray_with_pystac_client_search_passes_kwargs_through(simple_search
 def test_to_xarray_with_different_stacking_library(simple_search, stacking_library):
     ds = to_xarray(simple_search, stacking_library=stacking_library)
     assert isinstance(ds, xr.Dataset)
+    assert "band" not in ds.dims
 
 
 def test_to_xarray_with_drop_variables_raises(simple_search):
diff --git a/xpystac/core.py b/xpystac/core.py
index bcc2808..8f22e2d 100644
--- a/xpystac/core.py
+++ b/xpystac/core.py
@@ -70,7 +70,17 @@ def _(
         return odc_stac.load(items, **{"chunks": {"x": 1024, "y": 1024}, **kwargs})
     elif stacking_library == "stackstac":
         stackstac = _import_optional_dependency("stackstac")
-        return stackstac.stack(obj, **kwargs).to_dataset(dim="band", promote_attrs=True)
+        da = stackstac.stack(obj, **kwargs)
+        bands = {}
+        for band in da.band.values:
+            b = da.sel(band=band)
+            scalar_coords = {k: v.item() for k, v in b.coords.items() if v.shape == ()}
+            b = b.assign_attrs(
+                **{k: v for k, v in scalar_coords.items() if v is not None}
+            )
+            b = b.drop_vars(scalar_coords)
+            bands[band] = b
+        return xarray.Dataset(bands, attrs=da.attrs)
 
 
 @to_xarray.register