Adding key/value store and updating Dockerfile

ndscheduler/core/datastore/providers/base.py

@@ -14,7 +14,6 @@
 
 
 class DatastoreBase(sched_sqlalchemy.SQLAlchemyJobStore):
-
     instance = None
 
     @classmethod
@@ -213,3 +212,69 @@ def _build_audit_log(self, row):
             'created_time': self.get_time_isoformat_from_db(row.created_time),
             'description': row.description}
         return return_dict
+
+    def add_keyvalue(self, name, key, val, **kwargs):
+        """Insert a key/value pair.
+
+        :param str name: string for the name of the key/value pair.
+        :param str key: string for key.
+        :param int val: string for value.
+        """
+        kv = {
+            'name': name,
+            'key': key,
+            'val': val
+        }
+        kv.update(kwargs)
+
+        kv_insert = tables.KV_STORE.insert().values(**kv)
+        self.engine.execute(kv_insert)
+
+    def del_keyvalue(self, name, key, val, **kwargs):
+        """Deletes a key/value pair.
+
+        :param str name: string for the name of the key/value pair.
+        :param str key: string for key.
+        :param int val: string for value.
+        """
+        kv = {
+            'name': name,
+            'key': key,
+            'val': val
+        }
+        kv.update(kwargs)
+
+        kv_delete = tables.KV_STORE.delete().values(**kv)
+        self.engine.execute(kv_delete)
+
+    def get_keyvalue_by_name(self, name):
+        """Returns a list of key/value pairs.
+
+        :param str name: string for the name of the key/value pair.
+        :return: A dictionary of multiple key/value pairs. Sorted by created_time.
+        :rtype: dict
+        """
+        selectable = select('*').where(tables.KV_STORE.c.name == name). \
+            order_by(desc(tables.KV_STORE.c.created_time))
+        rows = self.engine.execute(selectable)
+
+        for row in rows:
+            return self._build_keyvalue(row)
+
+    def update_keyvalue(self, name, **kwargs):
+        """Update a key/value pair.
+
+        :param str name: string for the name of the key/value pair.
+        """
+        kv_update = tables.KV_STORE.update().where(tables.KV_STORE.c.name == name).values(**kwargs)
+        self.engine.execute(kv_update)
+
+    def _build_keyvalue(self, row):
+        return_dict = {
+            'name': row.name,
+            'key': row.key,
+            'val': row.val,
+            'created_time': self.get_time_isoformat_from_db(row.created_time)
+        }
+
+        return return_dict

ndscheduler/core/datastore/tables.py

@@ -42,3 +42,15 @@
     sqlalchemy.Column('created_time', sqlalchemy.DateTime(timezone=True), nullable=False,
                       default=utils.get_current_datetime),
     sqlalchemy.Column('description', sqlalchemy.Text, nullable=True))
+
+#
+# KVStore
+#
+KV_STORE = sqlalchemy.Table(
+    settings.KV_STORE_TABLENAME, METADATA,
+    sqlalchemy.Column('name', sqlalchemy.Integer, nullable=False),
+    sqlalchemy.Column('key', sqlalchemy.Text, nullable=True),
+    sqlalchemy.Column('val', sqlalchemy.Text, nullable=True),
+    sqlalchemy.Column('created_time', sqlalchemy.DateTime(timezone=True), nullable=False,
+                      default=utils.get_current_datetime)
+)

ndscheduler/core/scheduler_manager.py

@@ -1,19 +1,16 @@
 """Represents the core scheduler instance that actually schedules jobs."""
 
-
 import logging
 
 from apscheduler.executors import pool
 
 from ndscheduler import settings
 from ndscheduler import utils
 
-
 logger = logging.getLogger(__name__)
 
 
 class SchedulerManager:
-
     instance = None
 
     @classmethod

ndscheduler/default_settings.py

@@ -49,6 +49,7 @@
 JOBS_TABLENAME = 'scheduler_jobs'
 EXECUTIONS_TABLENAME = 'scheduler_execution'
 AUDIT_LOGS_TABLENAME = 'scheduler_jobauditlog'
+KV_STORE_TABLENAME = 'scheduler_keyvaluestore'
 
 # See different database providers in ndscheduler/core/datastore/providers/
 

simple_scheduler/docker/Dockerfile

@@ -1,19 +1,24 @@
-FROM ubuntu:14.04
+FROM python:3.6
 
-MAINTAINER Wenbin Fang <[email protected]>
+# set vars
+ENV LOCAL_APPDIR simple_scheduler
+ENV WORK_APPDIR app
+RUN mkdir -p /${WORK_APPDIR}
 
-RUN apt-get -qq update && \
-    apt-get -qq install python-virtualenv git && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+# install deps
+WORKDIR /
+RUN pip install --upgrade pip
 
-RUN virtualenv /mnt/scheduler && \
-    . /mnt/scheduler/bin/activate && \
-    pip install -e git+https://github.com/Nextdoor/ndscheduler.git#egg=ndscheduler && \
-    pip install -r /mnt/scheduler/src/ndscheduler/simple_scheduler/requirements.txt
+# for local development only!
+RUN mkdir -p /tmp/nds
+COPY . /tmp/nds
+RUN rm -rf /tmp/nds/simple_scheduler
+RUN pip install --no-cache-dir -e /tmp/nds
 
-ADD apns.pem /mnt/scheduler/
-ADD run_scheduler /mnt/scheduler/bin/run_scheduler
-RUN chmod 755 /mnt/scheduler/bin/run_scheduler
+ADD ${LOCAL_APPDIR}/requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
-CMD ["/mnt/scheduler/bin/run_scheduler"]
+COPY ${LOCAL_APPDIR} /${WORK_APPDIR}
+
+WORKDIR /${WORK_APPDIR}
+ENTRYPOINT ["/bin/bash", "docker/run_scheduler"]

simple_scheduler/docker/run_scheduler

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-# Run scheduler server inside docker container
-source /mnt/scheduler/bin/activate && \
-	NDSCHEDULER_SETTINGS_MODULE=simple_scheduler.settings \
-	python /mnt/scheduler/src/ndscheduler/simple_scheduler/scheduler.py
+cd /app
+export NDSCHEDULER_SETTINGS_MODULE=settings
+python scheduler.py

simple_scheduler/jobs/kvstore_job.py

@@ -0,0 +1,34 @@
+"""A sample job that prints string."""
+
+from ndscheduler import job
+from ndscheduler.utils import get_datastore_instance
+
+
+class KVStoreJob(job.JobBase):
+
+    @classmethod
+    def meta_info(cls):
+        return {
+            'job_class_string': '%s.%s' % (cls.__module__, cls.__name__),
+            'notes': 'This will print a string in your shell. Check it out! And it save the string in a key/value store.',
+            'arguments': [
+                # argument1
+                {'type': 'string', 'description': 'First argument'},
+
+                # argument2
+                {'type': 'string', 'description': 'Second argument'}
+            ],
+            'example_arguments': '["first argument AAA", "second argument BBB"]'
+        }
+
+    def run(self, argument1, argument2, *args, **kwargs):
+        print('Hello from AwesomeJob! Argument1: %s, Argument2: %s' % (argument1, argument2))
+        datastore = get_datastore_instance()
+        datastore.add_keyvalue("sample job with storage", argument1, argument2)
+        return [argument1, argument2]
+
+
+if __name__ == "__main__":
+    # You can easily test this job here
+    job = KVStoreJob.create_test_instance()
+    job.run(123, 456)

simple_scheduler/settings.py

@@ -15,4 +15,4 @@
 #
 logging.getLogger().setLevel(logging.DEBUG)
 
-JOB_CLASS_PACKAGES = ['simple_scheduler.jobs']
+JOB_CLASS_PACKAGES = ['jobs']