From ff206bf6ffd758e6816f8f6d2ffdf07553c06235 Mon Sep 17 00:00:00 2001 From: tbl Date: Wed, 24 Oct 2018 20:28:08 +0200 Subject: [PATCH 1/2] adding k/v store, updating Dockerfile --- ndscheduler/core/datastore/providers/base.py | 67 +++++++++++++++++++- ndscheduler/core/datastore/tables.py | 12 ++++ ndscheduler/core/scheduler_manager.py | 3 - ndscheduler/default_settings.py | 1 + simple_scheduler/docker/Dockerfile | 35 ++++++---- simple_scheduler/jobs/kvstore_job.py | 34 ++++++++++ 6 files changed, 134 insertions(+), 18 deletions(-) create mode 100644 simple_scheduler/jobs/kvstore_job.py diff --git a/ndscheduler/core/datastore/providers/base.py b/ndscheduler/core/datastore/providers/base.py index df1a03b..b6c8351 100644 --- a/ndscheduler/core/datastore/providers/base.py +++ b/ndscheduler/core/datastore/providers/base.py @@ -14,7 +14,6 @@ class DatastoreBase(sched_sqlalchemy.SQLAlchemyJobStore): - instance = None @classmethod @@ -213,3 +212,69 @@ def _build_audit_log(self, row): 'created_time': self.get_time_isoformat_from_db(row.created_time), 'description': row.description} return return_dict + + def add_keyvalue(self, name, key, val, **kwargs): + """Insert a key/value pair. + + :param str name: string for the name of the key/value pair. + :param str key: string for key. + :param int val: string for value. + """ + kv = { + 'name': name, + 'key': key, + 'val': val + } + kv.update(kwargs) + + kv_insert = tables.KV_STORE.insert().values(**kv) + self.engine.execute(kv_insert) + + def del_keyvalue(self, name, key, val, **kwargs): + """Deletes a key/value pair. + + :param str name: string for the name of the key/value pair. + :param str key: string for key. + :param int val: string for value. + """ + kv = { + 'name': name, + 'key': key, + 'val': val + } + kv.update(kwargs) + + kv_delete = tables.KV_STORE.delete().values(**kv) + self.engine.execute(kv_delete) + + def get_keyvalue_by_name(self, name): + """Returns a list of key/value pairs. + + :param str name: string for the name of the key/value pair. + :return: A dictionary of multiple key/value pairs. Sorted by created_time. + :rtype: dict + """ + selectable = select('*').where(tables.KV_STORE.c.name == name). \ + order_by(desc(tables.KV_STORE.c.created_time)) + rows = self.engine.execute(selectable) + + for row in rows: + return self._build_keyvalue(row) + + def update_keyvalue(self, name, **kwargs): + """Update a key/value pair. + + :param str name: string for the name of the key/value pair. + """ + kv_update = tables.KV_STORE.update().where(tables.KV_STORE.c.name == name).values(**kwargs) + self.engine.execute(kv_update) + + def _build_keyvalue(self, row): + return_dict = { + 'name': row.name, + 'key': row.key, + 'val': row.val, + 'created_time': self.get_time_isoformat_from_db(row.created_time) + } + + return return_dict diff --git a/ndscheduler/core/datastore/tables.py b/ndscheduler/core/datastore/tables.py index 9871617..8379523 100644 --- a/ndscheduler/core/datastore/tables.py +++ b/ndscheduler/core/datastore/tables.py @@ -42,3 +42,15 @@ sqlalchemy.Column('created_time', sqlalchemy.DateTime(timezone=True), nullable=False, default=utils.get_current_datetime), sqlalchemy.Column('description', sqlalchemy.Text, nullable=True)) + +# +# KVStore +# +KV_STORE = sqlalchemy.Table( + settings.KV_STORE_TABLENAME, METADATA, + sqlalchemy.Column('name', sqlalchemy.Integer, nullable=False), + sqlalchemy.Column('key', sqlalchemy.Text, nullable=True), + sqlalchemy.Column('val', sqlalchemy.Text, nullable=True), + sqlalchemy.Column('created_time', sqlalchemy.DateTime(timezone=True), nullable=False, + default=utils.get_current_datetime) +) diff --git a/ndscheduler/core/scheduler_manager.py b/ndscheduler/core/scheduler_manager.py index 58a37e1..b2f4b7f 100644 --- a/ndscheduler/core/scheduler_manager.py +++ b/ndscheduler/core/scheduler_manager.py @@ -1,6 +1,5 @@ """Represents the core scheduler instance that actually schedules jobs.""" - import logging from apscheduler.executors import pool @@ -8,12 +7,10 @@ from ndscheduler import settings from ndscheduler import utils - logger = logging.getLogger(__name__) class SchedulerManager: - instance = None @classmethod diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index f8251f7..5671e28 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -49,6 +49,7 @@ JOBS_TABLENAME = 'scheduler_jobs' EXECUTIONS_TABLENAME = 'scheduler_execution' AUDIT_LOGS_TABLENAME = 'scheduler_jobauditlog' +KV_STORE_TABLENAME = 'scheduler_keyvaluestore' # See different database providers in ndscheduler/core/datastore/providers/ diff --git a/simple_scheduler/docker/Dockerfile b/simple_scheduler/docker/Dockerfile index 8f1c106..affe85d 100644 --- a/simple_scheduler/docker/Dockerfile +++ b/simple_scheduler/docker/Dockerfile @@ -1,19 +1,26 @@ -FROM ubuntu:14.04 +FROM python:3.6 -MAINTAINER Wenbin Fang +# set vars +ENV LOCAL_APPDIR simple_scheduler +ENV WORK_APPDIR app +RUN mkdir -p /${WORK_APPDIR} -RUN apt-get -qq update && \ - apt-get -qq install python-virtualenv git && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# install deps +WORKDIR / +RUN pip install --upgrade pip -RUN virtualenv /mnt/scheduler && \ - . /mnt/scheduler/bin/activate && \ - pip install -e git+https://github.com/Nextdoor/ndscheduler.git#egg=ndscheduler && \ - pip install -r /mnt/scheduler/src/ndscheduler/simple_scheduler/requirements.txt +# for local development only! +RUN mkdir -p /tmp/nds +COPY . /tmp/nds +RUN pip install --no-cache-dir -e /tmp/nds -ADD apns.pem /mnt/scheduler/ -ADD run_scheduler /mnt/scheduler/bin/run_scheduler -RUN chmod 755 /mnt/scheduler/bin/run_scheduler +ADD ${LOCAL_APPDIR}/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt -CMD ["/mnt/scheduler/bin/run_scheduler"] +COPY ${LOCAL_APPDIR} /${WORK_APPDIR} + +RUN mkdir -p /root/.config/ +COPY docker/${LOCAL_APPDIR}/telegram-send.conf /root/.config/ + +WORKDIR /${WORK_APPDIR} +ENTRYPOINT ["/bin/bash", "docker/run_scheduler"] diff --git a/simple_scheduler/jobs/kvstore_job.py b/simple_scheduler/jobs/kvstore_job.py new file mode 100644 index 0000000..be63c8a --- /dev/null +++ b/simple_scheduler/jobs/kvstore_job.py @@ -0,0 +1,34 @@ +"""A sample job that prints string.""" + +from ndscheduler import job +from ndscheduler.utils import get_datastore_instance + + +class AwesomeJob(job.JobBase): + + @classmethod + def meta_info(cls): + return { + 'job_class_string': '%s.%s' % (cls.__module__, cls.__name__), + 'notes': 'This will print a string in your shell. Check it out! And it save the string in a key/value store.', + 'arguments': [ + # argument1 + {'type': 'string', 'description': 'First argument'}, + + # argument2 + {'type': 'string', 'description': 'Second argument'} + ], + 'example_arguments': '["first argument AAA", "second argument BBB"]' + } + + def run(self, argument1, argument2, *args, **kwargs): + print('Hello from AwesomeJob! Argument1: %s, Argument2: %s' % (argument1, argument2)) + datastore = get_datastore_instance() + datastore.add_keyvalue("sample job with storage", argument1, argument2) + return [argument1, argument2] + + +if __name__ == "__main__": + # You can easily test this job here + job = AwesomeJob.create_test_instance() + job.run(123, 456) From d23693296a5cf1d685f6a4381971e32f0e5da712 Mon Sep 17 00:00:00 2001 From: tbl Date: Wed, 24 Oct 2018 20:52:07 +0200 Subject: [PATCH 2/2] updating Dockerfile and settings --- simple_scheduler/docker/Dockerfile | 4 +--- simple_scheduler/docker/run_scheduler | 7 +++---- simple_scheduler/jobs/kvstore_job.py | 4 ++-- simple_scheduler/settings.py | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/simple_scheduler/docker/Dockerfile b/simple_scheduler/docker/Dockerfile index affe85d..5c0f50f 100644 --- a/simple_scheduler/docker/Dockerfile +++ b/simple_scheduler/docker/Dockerfile @@ -12,6 +12,7 @@ RUN pip install --upgrade pip # for local development only! RUN mkdir -p /tmp/nds COPY . /tmp/nds +RUN rm -rf /tmp/nds/simple_scheduler RUN pip install --no-cache-dir -e /tmp/nds ADD ${LOCAL_APPDIR}/requirements.txt /tmp/requirements.txt @@ -19,8 +20,5 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt COPY ${LOCAL_APPDIR} /${WORK_APPDIR} -RUN mkdir -p /root/.config/ -COPY docker/${LOCAL_APPDIR}/telegram-send.conf /root/.config/ - WORKDIR /${WORK_APPDIR} ENTRYPOINT ["/bin/bash", "docker/run_scheduler"] diff --git a/simple_scheduler/docker/run_scheduler b/simple_scheduler/docker/run_scheduler index 980ae80..41b477d 100755 --- a/simple_scheduler/docker/run_scheduler +++ b/simple_scheduler/docker/run_scheduler @@ -1,6 +1,5 @@ #!/bin/bash -# Run scheduler server inside docker container -source /mnt/scheduler/bin/activate && \ - NDSCHEDULER_SETTINGS_MODULE=simple_scheduler.settings \ - python /mnt/scheduler/src/ndscheduler/simple_scheduler/scheduler.py +cd /app +export NDSCHEDULER_SETTINGS_MODULE=settings +python scheduler.py diff --git a/simple_scheduler/jobs/kvstore_job.py b/simple_scheduler/jobs/kvstore_job.py index be63c8a..251c406 100644 --- a/simple_scheduler/jobs/kvstore_job.py +++ b/simple_scheduler/jobs/kvstore_job.py @@ -4,7 +4,7 @@ from ndscheduler.utils import get_datastore_instance -class AwesomeJob(job.JobBase): +class KVStoreJob(job.JobBase): @classmethod def meta_info(cls): @@ -30,5 +30,5 @@ def run(self, argument1, argument2, *args, **kwargs): if __name__ == "__main__": # You can easily test this job here - job = AwesomeJob.create_test_instance() + job = KVStoreJob.create_test_instance() job.run(123, 456) diff --git a/simple_scheduler/settings.py b/simple_scheduler/settings.py index 4bf9d40..4895da9 100644 --- a/simple_scheduler/settings.py +++ b/simple_scheduler/settings.py @@ -15,4 +15,4 @@ # logging.getLogger().setLevel(logging.DEBUG) -JOB_CLASS_PACKAGES = ['simple_scheduler.jobs'] +JOB_CLASS_PACKAGES = ['jobs']