Optimization for text column profile ksneab (#791)

* changed list to set on line 147 in text_column_profile script

* Added np array encoder function for json dump. Error would sometiems happen on dump for not jsonable types

dataprofiler/profilers/text_column_profile.py

@@ -144,7 +144,7 @@ def _update_vocab(
         :type subset_properties: dict
         :return: None
         """
-        data_flat = list(itertools.chain(*data))
+        data_flat = set(itertools.chain(*data))
         self.vocab = utils._combine_unique_sets(self.vocab, data_flat)
 
     def _update_helper(self, df_series_clean: pd.Series, profile: dict) -> None:

dataprofiler/profilers/utils.py

@@ -65,12 +65,12 @@ def __missing__(self, key: str) -> str:
         return key
 
 
-def _combine_unique_sets(a: list, b: list) -> list:
+def _combine_unique_sets(a: list | set, b: list | set) -> list:
     """
     Unify two lists.
 
-    :type a: list
-    :type b: list
+    :type a: Union [list, set]
+    :type b: Union[list, set]
     :rtype: list
     """
     combined_list: set = set()

dataprofiler/tests/space_time_analysis/dataset_generation.py

@@ -1,4 +1,5 @@
 import copy
+import json
 import string
 from typing import List, Optional
 
@@ -15,6 +16,17 @@
     import dataprofiler as dp
 
 
+class NumpyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return super().default(obj)
+
+
 def nan_injection(
     rng: Generator, df: pd.DataFrame, percent_to_nan: float = 0.0
 ) -> pd.DataFrame:

dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py

@@ -20,7 +20,7 @@
 except ImportError:
     import dataprofiler as dp
 
-from dataset_generation import generate_dataset_by_class, nan_injection
+from dataset_generation import NumpyEncoder, generate_dataset_by_class, nan_injection
 
 from dataprofiler import StructuredProfiler
 
@@ -236,7 +236,7 @@ def dp_space_time_analysis(
         if not os.path.exists(os.path.dirname(path)):
             os.makedirs(os.path.dirname(path))
         with open(path, "w") as fp:
-            json.dump(profile_times, fp, indent=4)
+            json.dump(profile_times, fp, indent=4, cls=NumpyEncoder)
         times_table.to_csv(path)