Update user_utils.py

Fixed dt error in clean_data
CIERA-Northwestern · Nov 8, 2023 · 44e7b28 · 44e7b28
1 parent 9ce547a
commit 44e7b28
Showing 1 changed file with 10 additions and 5 deletions.
diff --git a/press_dash_lib/user_utils.py b/press_dash_lib/user_utils.py
@@ -95,11 +95,16 @@ def clean_data(raw_df, config):
         config (dict): The (possibly altered) configuration dictionary.
     '''
 
-    # Drop drafts
-    cleaned_df = raw_df.drop(
-        raw_df.index[raw_df['Date'].dt.year == 1970],
-        axis='rows',
-    )
+    raw_df['Date'] = pd.to_datetime(raw_df['Date'], errors='coerce')
+
+    # Drop rows where 'Date' year is 1970
+    cleaned_df = raw_df[raw_df['Date'].dt.year != 1970]
+
+    # # Drop drafts
+    # cleaned_df = raw_df.drop(
+    #     raw_df.index[raw_df['Date'].dt.year == 1970],
+    #     axis='rows',
+    # )
 
     # Drop weird articles---ancient ones w/o a title or press type
     cleaned_df.dropna(