diff --git a/paperqa/agents/search.py b/paperqa/agents/search.py
index f010a2a2..b14f9070 100644
--- a/paperqa/agents/search.py
+++ b/paperqa/agents/search.py
@@ -535,6 +535,10 @@ async def get_directory_index(  # noqa: PLR0912
     # NOTE: if the index was not previously built, its index_files will be empty.
     # Otherwise, the index_files will not be empty
     if not build:
+        if not await search_index.index_files:
+            raise RuntimeError(
+                f"Index {search_index.index_name} was empty, please rebuild it."
+            )
         return search_index
 
     if not sync_index_w_directory:
diff --git a/tests/test_agents.py b/tests/test_agents.py
index d561afb2..0bb60beb 100644
--- a/tests/test_agents.py
+++ b/tests/test_agents.py
@@ -94,6 +94,11 @@ async def test_get_directory_index(agent_test_settings: Settings) -> None:
         assert len(await index.index_files) == len(path_to_id) - 1
         mock_aadd.assert_not_awaited(), "Expected we didn't re-add files"
 
+        # Note let's delete files.zip, and confirm we can't load the index
+        await (await index.file_index_filename).unlink()
+        with pytest.raises(RuntimeError, match="please rebuild"):
+            await get_directory_index(settings=agent_test_settings, build=False)
+
 
 EXPECTED_STUB_DATA_FILES = {
     "bates.txt",