diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py
index 9a53f24c67c..388fea92d51 100644
--- a/xarray/core/dataset.py
+++ b/xarray/core/dataset.py
@@ -4698,7 +4698,9 @@ def set_index(
             if len(var_names) == 1 and (not append or dim not in self._indexes):
                 var_name = var_names[0]
                 var = self._variables[var_name]
-                if var.dims != (dim,):
+                # an error with a better message will be raised for scalar variables
+                # when creating the PandasIndex
+                if var.ndim > 0 and var.dims != (dim,):
                     raise ValueError(
                         f"dimension mismatch: try setting an index for dimension {dim!r} with "
                         f"variable {var_name!r} that has dimensions {var.dims}"
diff --git a/xarray/core/indexes.py b/xarray/core/indexes.py
index 7de290f4e14..b5e396963a1 100644
--- a/xarray/core/indexes.py
+++ b/xarray/core/indexes.py
@@ -616,7 +616,14 @@ def from_variables(
 
         name, var = next(iter(variables.items()))
 
-        if var.ndim != 1:
+        if var.ndim == 0:
+            raise ValueError(
+                f"cannot set a PandasIndex from the scalar variable {name!r}, "
+                "only 1-dimensional variables are supported. "
+                f"Note: you might want to use `obj.expand_dims({name!r})` to create a "
+                f"new dimension and turn {name!r} as an indexed dimension coordinate."
+            )
+        elif var.ndim != 1:
             raise ValueError(
                 "PandasIndex only accepts a 1-dimensional variable, "
                 f"variable {name!r} has {var.ndim} dimensions"
diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py
index 3110c4e2882..a3e93414c0f 100644
--- a/xarray/tests/test_dataset.py
+++ b/xarray/tests/test_dataset.py
@@ -3397,6 +3397,12 @@ def test_set_index(self) -> None:
         with pytest.raises(ValueError, match=r"dimension mismatch.*"):
             ds.set_index(y="x_var")
 
+        ds = Dataset(coords={"x": 1})
+        with pytest.raises(
+            ValueError, match=r".*cannot set a PandasIndex.*scalar variable.*"
+        ):
+            ds.set_index(x="x")
+
     def test_set_index_deindexed_coords(self) -> None:
         # test de-indexed coordinates are converted to base variable
         # https://github.com/pydata/xarray/issues/6969
diff --git a/xarray/tests/test_indexes.py b/xarray/tests/test_indexes.py
index ebe9f3fb932..05d748541ed 100644
--- a/xarray/tests/test_indexes.py
+++ b/xarray/tests/test_indexes.py
@@ -145,6 +145,11 @@ def test_from_variables(self) -> None:
         with pytest.raises(ValueError, match=r".*only accepts one variable.*"):
             PandasIndex.from_variables({"x": var, "foo": var2}, options={})
 
+        with pytest.raises(
+            ValueError, match=r".*cannot set a PandasIndex.*scalar variable.*"
+        ):
+            PandasIndex.from_variables({"foo": xr.Variable((), 1)}, options={})
+
         with pytest.raises(
             ValueError, match=r".*only accepts a 1-dimensional variable.*"
         ):