added endpoint field to estasks

Author: Joel Klinger

nesta/core/routines/arxiv/arxiv_lolvelty.py

@@ -86,6 +86,7 @@ def requires(self):
                                        test=test,
                                        index=index,
                                        dataset='arxiv',
+                                       endpoint='arxlive',
                                        entity_type='article',
                                        kwargs=kwargs,
                                        batchable=f3p("batchables/novelty"

nesta/core/routines/arxiv/arxiv_root_task.py

@@ -66,18 +66,14 @@ def requires(self):
         test = not self.production
         routine_id = f"ArxivLolveltyTask-{self.date}-{test}"
 
-        # Elasticsearch setup
-        _, es_config = setup_es(endpoint='arxlive',
-                                dataset='arxiv',
-                                production=self.production,
-                                drop_and_recreate=self.drop_and_recreate)
         yield ArxivElasticsearchTask(date=self.date,
                                      process_batch_size=1000,
                                      routine_id=routine_id,
                                      grid_task_kwargs=grid_task_kwargs,
                                      test=not self.production,
-                                     index=es_config['index'],
+                                     drop_and_recreate=self.drop_and_recreate,
                                      dataset='arxiv',
+                                     endpoint='arxlive',
                                      entity_type='article',
                                      kwargs=kwargs,
                                      batchable=f3p("batchables/novelty"

nesta/core/routines/crunchbase/crunchbase_lolvelty.py

@@ -37,7 +37,8 @@ def requires(self):
         return LazyElasticsearchTask(routine_id=routine_id,
                                      test=test,
                                      index=index,
-                                     dataset='crunchbase',
+                                     dataset='companies',
+                                     endpoint='health-scanner'
                                      entity_type='company',
                                      kwargs=kwargs,
                                      batchable=f3p("batchables/novelty/lolvelty"),

nesta/core/routines/health_data/nih_data/nih_lolvelty.py

@@ -24,6 +24,7 @@ def requires(self):
                                      test=test,
                                      index=index,
                                      dataset='nih',
+                                     endpoint='health-scanner',
                                      entity_type='paper',
                                      kwargs=kwargs,
                                      batchable=f3p("batchables/novelty/lolvelty"),

nesta/core/routines/meetup/health_tagging/meetup_lolvelty.py

@@ -35,6 +35,7 @@ def requires(self):
                                      test=test,
                                      index=index,
                                      dataset='meetup',
+                                     endpoint='health-scanner',
                                      entity_type='meetup',
                                      kwargs=kwargs,
                                      batchable=f3p("batchables/novelty/lolvelty"),