cleanup: ensure clustering doesnt run on small datasets

devflowinc · Aug 13, 2024 · 9960930 · 9960930
1 parent 6fa7e50
commit 9960930
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/docker/clustering-script/get_clusters.py b/docker/clustering-script/get_clusters.py
@@ -161,6 +161,10 @@ def insert_centroids(
             # Fetch data
             data = fetch_dataset_vectors(client, dataset_id[0], 3000)
 
+            if len(data) < 30:
+                print(f"Skipping dataset {dataset_id[0]} due to insufficient data")
+                continue
+
             # Perform spherical k-means clustering
             hdbscan = hdbscan_clustering(data)