From cd5730f9ea0ff81ec37c43b6a06f8a266e7225ab Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 23 Jan 2019 15:58:32 +0800 Subject: [PATCH 1/8] Add basic auth feature --- ndscheduler/default_settings.py | 4 ++++ ndscheduler/server/handlers/base.py | 5 ++++- setup.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index f8251f7..9ff2fdb 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -59,6 +59,10 @@ 'file_path': 'datastore.db' } + +# Basic Auth +BASIC_AUTH_CREDENTIALS = {'username': 'password'} + # Postgres # # DATABASE_CLASS = 'ndscheduler.core.datastore.providers.postgresql.DatastorePostgresql' diff --git a/ndscheduler/server/handlers/base.py b/ndscheduler/server/handlers/base.py index 6754e85..dfab532 100644 --- a/ndscheduler/server/handlers/base.py +++ b/ndscheduler/server/handlers/base.py @@ -13,11 +13,14 @@ from ndscheduler import settings +from tornado_http_auth import DigestAuthMixin, auth_required -class BaseHandler(tornado.web.RequestHandler): + +class BaseHandler(tornado.web.RequestHandler, DigestAuthMixin): executor = futures.ThreadPoolExecutor(max_workers=settings.TORNADO_MAX_WORKERS) + @auth_required(realm='Protected', auth_func=settings.BASIC_AUTH_CREDENTIALS.get) def prepare(self): """Preprocess requests.""" try: diff --git a/setup.py b/setup.py index 8b01c22..e825133 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ def maybe_rm(path): 'future == 0.15.2', 'tornado == 4.3.0', 'python-dateutil == 2.2', + 'tornado-http-auth= = 1.1.1', ], classifiers=classifiers, cmdclass={'clean': CleanHook}, From 02a7d8438b6aea74e38d0701671a66c6cbfdbb5f Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Sun, 27 Jan 2019 15:17:01 +0800 Subject: [PATCH 2/8] Typo fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e825133..a1c9f6e 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ def maybe_rm(path): 'future == 0.15.2', 'tornado == 4.3.0', 'python-dateutil == 2.2', - 'tornado-http-auth= = 1.1.1', + 'tornado-http-auth == 1.1.1', ], classifiers=classifiers, cmdclass={'clean': CleanHook}, From 70b58a931c776b6f497812b7a2e0e835a1db2f1d Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 28 Jan 2019 22:32:16 +0800 Subject: [PATCH 3/8] Refactor basic auth function Based on code and ideas from https://github.com/gvalkov/tornado-http-auth --- ndscheduler/default_settings.py | 8 +++---- ndscheduler/server/handlers/base.py | 33 +++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index 9ff2fdb..03029c0 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -59,10 +59,6 @@ 'file_path': 'datastore.db' } - -# Basic Auth -BASIC_AUTH_CREDENTIALS = {'username': 'password'} - # Postgres # # DATABASE_CLASS = 'ndscheduler.core.datastore.providers.postgresql.DatastorePostgresql' @@ -98,3 +94,7 @@ # Packages that contains job classes, e.g., simple_scheduler.jobs JOB_CLASS_PACKAGES = [] + +# Basic Auth +# +BASIC_AUTH_CREDENTIALS = {'username': 'password'} diff --git a/ndscheduler/server/handlers/base.py b/ndscheduler/server/handlers/base.py index dfab532..1d74032 100644 --- a/ndscheduler/server/handlers/base.py +++ b/ndscheduler/server/handlers/base.py @@ -5,6 +5,7 @@ """ import json +import base64 from concurrent import futures @@ -13,16 +14,40 @@ from ndscheduler import settings -from tornado_http_auth import DigestAuthMixin, auth_required - -class BaseHandler(tornado.web.RequestHandler, DigestAuthMixin): +class BaseHandler(tornado.web.RequestHandler): executor = futures.ThreadPoolExecutor(max_workers=settings.TORNADO_MAX_WORKERS) - @auth_required(realm='Protected', auth_func=settings.BASIC_AUTH_CREDENTIALS.get) + basic_auth_credentials_func = settings.BASIC_AUTH_CREDENTIALS.get + basic_auth_realm = 'Scheduler' + + def send_challenge(self): + """Send challenge response.""" + header = 'Basic realm="{}"'.format(self.basic_auth_realm) + self.set_status(401) + self.set_header('WWW-Authenticate', header) + self.finish() + + def get_basic_auth_result(self): + """Get HTTP basic access authentication result.""" + auth_header = self.request.headers.get('Authorization', '') + if not auth_header.startswith('Basic '): + self.send_challenge() + return + + auth_data = auth_header.split(' ')[-1] + auth_data = base64.b64decode(auth_data).decode('utf-8') + username, password = auth_data.split(':') + + challenge = self.basic_auth_credentials_func(username) + if challenge != password: + self.send_challenge() + def prepare(self): """Preprocess requests.""" + self.get_basic_auth_result() + try: if self.request.headers['Content-Type'].startswith('application/json'): self.json_args = json.loads(self.request.body.decode()) From 34b0337b0d803bdfadb0edb7b837e0da97e1bf62 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 28 Jan 2019 23:10:20 +0800 Subject: [PATCH 4/8] Make basic auth optional --- ndscheduler/default_settings.py | 5 ++++- ndscheduler/server/handlers/base.py | 7 ++++--- setup.py | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index 03029c0..1f5252f 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -97,4 +97,7 @@ # Basic Auth # -BASIC_AUTH_CREDENTIALS = {'username': 'password'} +# If needs basic auth, modify the dict below +# e.g. BASIC_AUTH_CREDENTIALS = {'username': 'password'} +# +BASIC_AUTH_CREDENTIALS = {} diff --git a/ndscheduler/server/handlers/base.py b/ndscheduler/server/handlers/base.py index 1d74032..539549b 100644 --- a/ndscheduler/server/handlers/base.py +++ b/ndscheduler/server/handlers/base.py @@ -19,7 +19,7 @@ class BaseHandler(tornado.web.RequestHandler): executor = futures.ThreadPoolExecutor(max_workers=settings.TORNADO_MAX_WORKERS) - basic_auth_credentials_func = settings.BASIC_AUTH_CREDENTIALS.get + basic_auth_credentials = settings.BASIC_AUTH_CREDENTIALS basic_auth_realm = 'Scheduler' def send_challenge(self): @@ -40,13 +40,14 @@ def get_basic_auth_result(self): auth_data = base64.b64decode(auth_data).decode('utf-8') username, password = auth_data.split(':') - challenge = self.basic_auth_credentials_func(username) + challenge = self.basic_auth_credentials.get(username) if challenge != password: self.send_challenge() def prepare(self): """Preprocess requests.""" - self.get_basic_auth_result() + if len(self.basic_auth_credentials) > 0: + self.get_basic_auth_result() try: if self.request.headers['Content-Type'].startswith('application/json'): diff --git a/setup.py b/setup.py index a1c9f6e..8b01c22 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,6 @@ def maybe_rm(path): 'future == 0.15.2', 'tornado == 4.3.0', 'python-dateutil == 2.2', - 'tornado-http-auth == 1.1.1', ], classifiers=classifiers, cmdclass={'clean': CleanHook}, From c47fe6380f04c2df9599f43a54c46ace4e082339 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 31 Jan 2019 17:26:23 +0800 Subject: [PATCH 5/8] Add website settings --- ndscheduler/default_settings.py | 1 + ndscheduler/server/handlers/index.py | 9 ++++++++- ndscheduler/static/index.html | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index 1f5252f..050f9f0 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -19,6 +19,7 @@ STATIC_DIR_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'static') TEMPLATE_DIR_PATH = STATIC_DIR_PATH APP_INDEX_PAGE = 'index.html' +WEBSITE_TITLE = 'Scheduler' # # Server setup diff --git a/ndscheduler/server/handlers/index.py b/ndscheduler/server/handlers/index.py index ff0130f..145e5bf 100644 --- a/ndscheduler/server/handlers/index.py +++ b/ndscheduler/server/handlers/index.py @@ -13,5 +13,12 @@ class Handler(base.BaseHandler): def get(self): """Serve up the single page app for scheduler dashboard.""" + website_info = { + 'title': settings.WEBSITE_TITLE, + } + meta_info = utils.get_all_available_jobs() - self.render(settings.APP_INDEX_PAGE, jobs_meta_info=json.dumps(meta_info)) + self.render(settings.APP_INDEX_PAGE, + jobs_meta_info=json.dumps(meta_info), + website_info=website_info, + ) diff --git a/ndscheduler/static/index.html b/ndscheduler/static/index.html index dfb97a6..07459fb 100644 --- a/ndscheduler/static/index.html +++ b/ndscheduler/static/index.html @@ -7,7 +7,7 @@ - Scheduler + {{ website_info['title'] }} @@ -33,7 +33,7 @@ - Nextdoor Scheduler + {{ website_info['title'] }}
    From c7174098438db84662742ed12ca66dfdc6333224 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 31 Jan 2019 17:26:23 +0800 Subject: [PATCH 6/8] Add website settings --- ndscheduler/default_settings.py | 1 + ndscheduler/server/handlers/index.py | 8 +++++++- ndscheduler/static/index.html | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index 1f5252f..050f9f0 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -19,6 +19,7 @@ STATIC_DIR_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'static') TEMPLATE_DIR_PATH = STATIC_DIR_PATH APP_INDEX_PAGE = 'index.html' +WEBSITE_TITLE = 'Scheduler' # # Server setup diff --git a/ndscheduler/server/handlers/index.py b/ndscheduler/server/handlers/index.py index ff0130f..ac5cbab 100644 --- a/ndscheduler/server/handlers/index.py +++ b/ndscheduler/server/handlers/index.py @@ -13,5 +13,11 @@ class Handler(base.BaseHandler): def get(self): """Serve up the single page app for scheduler dashboard.""" + website_info = { + 'title': settings.WEBSITE_TITLE, + } meta_info = utils.get_all_available_jobs() - self.render(settings.APP_INDEX_PAGE, jobs_meta_info=json.dumps(meta_info)) + self.render(settings.APP_INDEX_PAGE, + jobs_meta_info=json.dumps(meta_info), + website_info=website_info, + ) diff --git a/ndscheduler/static/index.html b/ndscheduler/static/index.html index dfb97a6..07459fb 100644 --- a/ndscheduler/static/index.html +++ b/ndscheduler/static/index.html @@ -7,7 +7,7 @@ - Scheduler + {{ website_info['title'] }} @@ -33,7 +33,7 @@ - Nextdoor Scheduler + {{ website_info['title'] }}
    From e5ec45a8941f13c78913e58f460d37873ee1973e Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 31 Jan 2019 17:38:46 +0800 Subject: [PATCH 7/8] Merge --- ndscheduler/server/handlers/index.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndscheduler/server/handlers/index.py b/ndscheduler/server/handlers/index.py index ac5cbab..273eb10 100644 --- a/ndscheduler/server/handlers/index.py +++ b/ndscheduler/server/handlers/index.py @@ -16,6 +16,7 @@ def get(self): website_info = { 'title': settings.WEBSITE_TITLE, } + meta_info = utils.get_all_available_jobs() self.render(settings.APP_INDEX_PAGE, jobs_meta_info=json.dumps(meta_info), From 5e80cf3fdc3175e2d961cfd71a1487bcc6e74231 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 31 Jan 2019 23:07:39 +0800 Subject: [PATCH 8/8] New job type: shell_output_job --- simple_scheduler/jobs/shell_output_job.py | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 simple_scheduler/jobs/shell_output_job.py diff --git a/simple_scheduler/jobs/shell_output_job.py b/simple_scheduler/jobs/shell_output_job.py new file mode 100644 index 0000000..d528abc --- /dev/null +++ b/simple_scheduler/jobs/shell_output_job.py @@ -0,0 +1,30 @@ +"""A job to run executable programs and get output.""" + +from subprocess import check_output + +from ndscheduler import job + + +class ShellJob(job.JobBase): + + @classmethod + def meta_info(cls): + return { + 'job_class_string': '%s.%s' % (cls.__module__, cls.__name__), + 'notes': ('This will run an executable program. You can specify as many ' + 'arguments as you want. This job will pass these arguments to the ' + 'program in order.'), + 'arguments': [ + {'type': 'string', 'description': 'Executable path'} + ], + 'example_arguments': '["/usr/local/my_program", "--file", "/tmp/abc", "--mode", "safe"]' + } + + def run(self, *args, **kwargs): + return {'result': check_output(args).decode('utf-8')} + + +if __name__ == "__main__": + # You can easily test this job here + job = ShellJob.create_test_instance() + job.run('ls', '-l')