From 3c27e8fa4840696c9035db25351dc76524bd4421 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 18 May 2019 00:39:30 -0700 Subject: [PATCH 01/11] Don't install a logger by default, because now I'm getting horribly cluttered logs. --- ndscheduler/__init__.py | 10 +++++----- ndscheduler/default_settings.py | 6 ------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/ndscheduler/__init__.py b/ndscheduler/__init__.py index 5136b73..f1c2880 100644 --- a/ndscheduler/__init__.py +++ b/ndscheduler/__init__.py @@ -17,11 +17,11 @@ from ndscheduler import default_settings -logger = logging.getLogger() -ch = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) +# logger = logging.getLogger() +# ch = logging.StreamHandler(sys.stdout) +# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# ch.setFormatter(formatter) +# logger.addHandler(ch) ENVIRONMENT_VARIABLE = 'NDSCHEDULER_SETTINGS_MODULE' diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index f8251f7..a524ec3 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -86,11 +86,5 @@ # Please see ndscheduler/core/scheduler/base.py SCHEDULER_CLASS = 'ndscheduler.core.scheduler.base.SingletonScheduler' -# -# Set logging level -# -logging.getLogger().setLevel(logging.INFO) - - # Packages that contains job classes, e.g., simple_scheduler.jobs JOB_CLASS_PACKAGES = [] From 7649f00c17371e38961bf9417b76fe5c02359323 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 18 May 2019 00:48:41 -0700 Subject: [PATCH 02/11] Support pypy postgresql. --- .../core/datastore/providers/postgresql.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/ndscheduler/core/datastore/providers/postgresql.py b/ndscheduler/core/datastore/providers/postgresql.py index 6bfd748..59e9cbe 100644 --- a/ndscheduler/core/datastore/providers/postgresql.py +++ b/ndscheduler/core/datastore/providers/postgresql.py @@ -1,5 +1,6 @@ """Represents PostgreSQL datastore.""" +import sys from ndscheduler import settings from ndscheduler.core.datastore.providers import base @@ -22,10 +23,19 @@ def get_db_url(cls): :rtype: str """ - return 'postgresql://%s:%s@%s:%d/%s?sslmode=%s' % ( - settings.DATABASE_CONFIG_DICT['user'], - settings.DATABASE_CONFIG_DICT['password'], - settings.DATABASE_CONFIG_DICT['hostname'], - settings.DATABASE_CONFIG_DICT['port'], - settings.DATABASE_CONFIG_DICT['database'], - settings.DATABASE_CONFIG_DICT['sslmode']) + # Work under Pypy, which doesn't have the default psycopg2 + if '__pypy__' in sys.builtin_module_names: + db_wrapper = 'postgresql+psycopg2cffi' + else: + db_wrapper = 'postgresql' + + + return '%s://%s:%s@%s:%d/%s?sslmode=%s' % ( + db_wrapper, + settings.DATABASE_CONFIG_DICT['user'], + settings.DATABASE_CONFIG_DICT['password'], + settings.DATABASE_CONFIG_DICT['hostname'], + settings.DATABASE_CONFIG_DICT['port'], + settings.DATABASE_CONFIG_DICT['database'], + settings.DATABASE_CONFIG_DICT['sslmode'] + ) From 4a2405e77b2f106b1ade45d339e4b0f4b01f0b10 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 18 May 2019 00:51:09 -0700 Subject: [PATCH 03/11] Fix derps. --- ndscheduler/__init__.py | 6 +++--- ndscheduler/default_settings.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ndscheduler/__init__.py b/ndscheduler/__init__.py index f1c2880..0343f24 100644 --- a/ndscheduler/__init__.py +++ b/ndscheduler/__init__.py @@ -11,13 +11,13 @@ """ import importlib -import logging import os -import sys +import logging from ndscheduler import default_settings -# logger = logging.getLogger() +# import sys +logger = logging.getLogger() # ch = logging.StreamHandler(sys.stdout) # formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # ch.setFormatter(formatter) diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index a524ec3..01ba5b4 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -1,9 +1,7 @@ """Default settings.""" -import logging import os - # # Development mode or production mode # If DEBUG is True, then auto-reload is enabled, i.e., when code is modified, server will be From b850973c7c2ce5746a39bff480367224e8b6dc44 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 18 May 2019 00:53:54 -0700 Subject: [PATCH 04/11] FFfuuuuu flake8 --- ndscheduler/core/datastore/providers/postgresql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ndscheduler/core/datastore/providers/postgresql.py b/ndscheduler/core/datastore/providers/postgresql.py index 59e9cbe..b2f6a28 100644 --- a/ndscheduler/core/datastore/providers/postgresql.py +++ b/ndscheduler/core/datastore/providers/postgresql.py @@ -29,7 +29,6 @@ def get_db_url(cls): else: db_wrapper = 'postgresql' - return '%s://%s:%s@%s:%d/%s?sslmode=%s' % ( db_wrapper, settings.DATABASE_CONFIG_DICT['user'], From 8c2cab6229943042e53a841cbac969f8f83138d9 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 18 May 2019 01:10:07 -0700 Subject: [PATCH 05/11] Unrelated: Show more items in the table views by default. --- ndscheduler/static/js/views/executions/table-view.js | 1 + ndscheduler/static/js/views/jobs/table-view.js | 3 ++- ndscheduler/static/js/views/logs/table-view.js | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ndscheduler/static/js/views/executions/table-view.js b/ndscheduler/static/js/views/executions/table-view.js index fe63cda..0edd939 100644 --- a/ndscheduler/static/js/views/executions/table-view.js +++ b/ndscheduler/static/js/views/executions/table-view.js @@ -52,6 +52,7 @@ define(['utils', this.table = $('#executions-table').dataTable({ // Sorted by last updated time 'order': [[3, 'desc']], + "iDisplayLength": 50, // Disable sorting on result column "columnDefs": [ { "orderable": false, "className": "table-result-column", "targets": 5 } diff --git a/ndscheduler/static/js/views/jobs/table-view.js b/ndscheduler/static/js/views/jobs/table-view.js index 9b11d56..649a1f8 100644 --- a/ndscheduler/static/js/views/jobs/table-view.js +++ b/ndscheduler/static/js/views/jobs/table-view.js @@ -66,7 +66,8 @@ define(['utils', // Initialize data table this.table = $('#jobs-table').dataTable({ // Sorted by job name - 'order': [[0, 'asc']] + 'order': [[0, 'asc']], + "iDisplayLength": 50 }); }, diff --git a/ndscheduler/static/js/views/logs/table-view.js b/ndscheduler/static/js/views/logs/table-view.js index 21fe298..03c0816 100644 --- a/ndscheduler/static/js/views/logs/table-view.js +++ b/ndscheduler/static/js/views/logs/table-view.js @@ -48,6 +48,7 @@ define(['utils', // Initialize data table this.table = $('#logs-table').dataTable({ + "iDisplayLength": 50, // Sorted by job name 'order': [[3, 'desc']] }); From 7a4a95ffd4195d6ec4758199c358c76523174aef Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 29 Jun 2019 10:11:16 -0700 Subject: [PATCH 06/11] Support at least viewing non-cron-based jobs in the web-ui. Holy potatoes jerberscript is MISERABLE. So much magic and action at a distance. Moar for https://github.com/Nextdoor/ndscheduler/issues/64 --- ndscheduler/core/scheduler/base.py | 40 ++ ndscheduler/core/scheduler_manager.py | 22 + ndscheduler/server/handlers/jobs.py | 13 +- ndscheduler/static/js/models/job.js | 35 +- ...b-row-action.html => job-row-actions.html} | 0 ...b-row-name.html => job-row-name-cron.html} | 1 + .../js/templates/job-row-name-interval.html | 14 + .../js/templates/job-row-name-unknown.html | 13 + .../static/js/views/jobs/table-view.js | 382 +++++++++++------- 9 files changed, 360 insertions(+), 160 deletions(-) rename ndscheduler/static/js/templates/{job-row-action.html => job-row-actions.html} (100%) rename ndscheduler/static/js/templates/{job-row-name.html => job-row-name-cron.html} (95%) create mode 100644 ndscheduler/static/js/templates/job-row-name-interval.html create mode 100644 ndscheduler/static/js/templates/job-row-name-unknown.html diff --git a/ndscheduler/core/scheduler/base.py b/ndscheduler/core/scheduler/base.py index 44d572e..3d3c148 100644 --- a/ndscheduler/core/scheduler/base.py +++ b/ndscheduler/core/scheduler/base.py @@ -103,6 +103,46 @@ def add_scheduler_job(self, job_class_string, name, pub_args=None, minute=minute, args=arguments, kwargs=kwargs, name=name, id=job_id) return job_id + def add_trigger_scheduler_job(self, job_class_string, name, pub_args, trigger, + **kwargs): + """Add a job. Job infomation will be persistent in postgres. + + This is a NON-BLOCKING operation, as internally, apscheduler calls wakeup() + that is async. + + :param str job_class_string: String for job class, e.g., myscheduler.jobs.a_job.NiceJob + :param str name: String for job name, e.g., Check Melissa job. + :param str pub_args: List for arguments passed to publish method of a task. + :param str month: String for month cron string, e.g., */10 + :param str day_of_week: String for day of week cron string, e.g., 1-6 + :param str day: String for day cron string, e.g., */1 + :param str hour: String for hour cron string, e.g., */2 + :param str minute: String for minute cron string, e.g., */3 + :param dict kwargs: Other keyword arguments passed to run_job function. + :return: String of job id, e.g., 6bca19736d374ef2b3df23eb278b512e + :rtype: str + + Returns: + String of job id, e.g., 6bca19736d374ef2b3df23eb278b512e + """ + if not pub_args: + pub_args = [] + + job_id = utils.generate_uuid() + + arguments = [job_class_string, job_id] + arguments.extend(pub_args) + + scheduler_class = utils.import_from_path(settings.SCHEDULER_CLASS) + self.add_job(scheduler_class.run_job, + trigger = trigger, + args = arguments, + kwargs = kwargs, + name = name, + id = job_id + ) + return job_id + def modify_scheduler_job(self, job_id, **kwargs): """Modifies a job. diff --git a/ndscheduler/core/scheduler_manager.py b/ndscheduler/core/scheduler_manager.py index 58a37e1..4be5d4c 100644 --- a/ndscheduler/core/scheduler_manager.py +++ b/ndscheduler/core/scheduler_manager.py @@ -91,6 +91,28 @@ def add_job(self, job_class_string, name, pub_args=None, return self.sched.add_scheduler_job(job_class_string, name, pub_args, month, day_of_week, day, hour, minute, **kwargs) + def add_trigger_job(self, job_class_string, name, pub_args=None, + trigger=None, + **kwargs): + """Add a job. Job infomation will be persistent in postgres. + + This is a NON-BLOCKING operation, as internally, apscheduler calls wakeup() + that is async. + + :param str job_class_string: String for job class, e.g., myscheduler.jobs.a_job.NiceJob + :param str name: String for job name, e.g., Check Melissa job. + :param str pub_args: List for arguments passed to publish method of a task. + :param str month: String for month cron string, e.g., */10 + :param str day_of_week: String for day of week cron string, e.g., 1-6 + :param str day: String for day cron string, e.g., */1 + :param str hour: String for hour cron string, e.g., */2 + :param str minute: String for minute cron string, e.g., */3 + :param dict kwargs: Other keyword arguments passed to run_job function. + :return: String of job id, e.g., 6bca19736d374ef2b3df23eb278b512e + :rtype: str + """ + return self.sched.add_trigger_scheduler_job(job_class_string, name, pub_args, trigger, **kwargs) + def pause_job(self, job_id): """Pauses the schedule of a job. diff --git a/ndscheduler/server/handlers/jobs.py b/ndscheduler/server/handlers/jobs.py index cab8e2e..31575b5 100644 --- a/ndscheduler/server/handlers/jobs.py +++ b/ndscheduler/server/handlers/jobs.py @@ -6,6 +6,9 @@ import tornado.gen import tornado.web +import apscheduler.triggers.cron +import apscheduler.triggers.interval + from ndscheduler import constants from ndscheduler import utils from ndscheduler.server.handlers import base @@ -42,7 +45,15 @@ def _build_job_dict(self, job): 'job_class_string': utils.get_job_name(job), 'pub_args': utils.get_job_args(job)} - return_dict.update(utils.get_cron_strings(job)) + if isinstance(job.trigger, apscheduler.triggers.cron.CronTrigger): + return_dict.update(utils.get_cron_strings(job)) + return_dict["trigger_type"] = "cron" + elif isinstance(job.trigger, apscheduler.triggers.interval.IntervalTrigger): + return_dict["interval"] = job.trigger.interval.total_seconds() + return_dict["trigger_type"] = "interval" + else: + return_dict["trigger_type"] = "unknown" + return return_dict @tornado.concurrent.run_on_executor diff --git a/ndscheduler/static/js/models/job.js b/ndscheduler/static/js/models/job.js index 5f95a70..0987a5e 100644 --- a/ndscheduler/static/js/models/job.js +++ b/ndscheduler/static/js/models/job.js @@ -20,6 +20,26 @@ require.config({ } }); +function secondsToStr( seconds_in ) { + let temp = seconds_in; + const years = Math.floor( temp / 31536000 ), + days = Math.floor( ( temp %= 31536000 ) / 86400 ), + hours = Math.floor( ( temp %= 86400 ) / 3600 ), + minutes = Math.floor( ( temp %= 3600 ) / 60 ), + seconds = temp % 60; + + if ( days || hours || seconds || minutes ) { + return ( years ? years + "y " : "" ) + + ( days ? days + "d " : "" ) + + ( hours ? hours + "h " : "" ) + + ( minutes ? minutes + "m " : "" ) + + Number.parseFloat( seconds ).toFixed( 2 ) + "s"; + } + + return "< 1s"; +} + + define(['backbone', 'vendor/moment-timezone-with-data'], function(backbone, moment) { 'use strict'; @@ -31,9 +51,18 @@ define(['backbone', 'vendor/moment-timezone-with-data'], function(backbone, mome * @return {string} schedule string for this job. */ getScheduleString: function() { - return 'minute: ' + this.get('minute') + ', hour: ' + this.get('hour') + - ', day: ' + this.get('day') + ', month: ' + this.get('month') + - ', day of week: ' + this.get('day_of_week'); + var trig = this.get('trigger_type'); + + if (trig == 'cron') + return 'Cron: minute: ' + this.get('minute') + ', hour: ' + this.get('hour') + + ', day: ' + this.get('day') + ', month: ' + this.get('month') + + ', day of week: ' + this.get('day_of_week'); + else if (trig == 'interval') + return 'Interval: ' + secondsToStr(this.get('interval')); + else + return 'Unknown trigger type!'; + + }, /** diff --git a/ndscheduler/static/js/templates/job-row-action.html b/ndscheduler/static/js/templates/job-row-actions.html similarity index 100% rename from ndscheduler/static/js/templates/job-row-action.html rename to ndscheduler/static/js/templates/job-row-actions.html diff --git a/ndscheduler/static/js/templates/job-row-name.html b/ndscheduler/static/js/templates/job-row-name-cron.html similarity index 95% rename from ndscheduler/static/js/templates/job-row-name.html rename to ndscheduler/static/js/templates/job-row-name-cron.html index 258cf89..b204d43 100644 --- a/ndscheduler/static/js/templates/job-row-name.html +++ b/ndscheduler/static/js/templates/job-row-name-cron.html @@ -5,6 +5,7 @@ data-target="#edit-job-modal" data-id="<%= job_id %>" data-job-name="<%= job_name %>" + data-job-trigtype="cron" data-job-month="<%= job_month %>" data-job-day-of-week="<%= job_day_of_week %>" data-job-day="<%= job_day %>" diff --git a/ndscheduler/static/js/templates/job-row-name-interval.html b/ndscheduler/static/js/templates/job-row-name-interval.html new file mode 100644 index 0000000..76b1afa --- /dev/null +++ b/ndscheduler/static/js/templates/job-row-name-interval.html @@ -0,0 +1,14 @@ + + + + <%= job_name %> + diff --git a/ndscheduler/static/js/templates/job-row-name-unknown.html b/ndscheduler/static/js/templates/job-row-name-unknown.html new file mode 100644 index 0000000..5d8df5d --- /dev/null +++ b/ndscheduler/static/js/templates/job-row-name-unknown.html @@ -0,0 +1,13 @@ + + + + <%= job_name %> + diff --git a/ndscheduler/static/js/views/jobs/table-view.js b/ndscheduler/static/js/views/jobs/table-view.js index 649a1f8..42aea32 100644 --- a/ndscheduler/static/js/views/jobs/table-view.js +++ b/ndscheduler/static/js/views/jobs/table-view.js @@ -5,162 +5,232 @@ */ require.config({ - paths: { - 'jquery': 'vendor/jquery', - 'underscore': 'vendor/underscore', - 'backbone': 'vendor/backbone', - 'bootstrap': 'vendor/bootstrap', - 'datatables': 'vendor/jquery.dataTables', - - 'utils': 'utils', - 'run-job-view': 'views/jobs/run-job-view', - 'edit-job-view': 'views/jobs/edit-job-view', - - 'text': 'vendor/text', - 'job-row-name': 'templates/job-row-name.html', - 'job-row-action': 'templates/job-row-action.html' - }, - - shim: { - 'bootstrap': { - deps: ['jquery'] - }, - - 'backbone': { - deps: ['underscore', 'jquery'], - exports: 'Backbone' - }, - - 'datatables': { - deps: ['jquery'], - exports: '$.fn.dataTable' - } - } + paths: { + 'jquery': 'vendor/jquery', + 'underscore': 'vendor/underscore', + 'backbone': 'vendor/backbone', + 'bootstrap': 'vendor/bootstrap', + 'datatables': 'vendor/jquery.dataTables', + + 'utils': 'utils', + 'run-job-view': 'views/jobs/run-job-view', + 'edit-job-view': 'views/jobs/edit-job-view', + + 'text': 'vendor/text', + 'job-row-name-cron' : 'templates/job-row-name-cron.html', + 'job-row-name-interval' : 'templates/job-row-name-interval.html', + 'job-row-name-unknown' : 'templates/job-row-name-unknown.html', + 'job-row-actions' : 'templates/job-row-actions.html' + }, + + shim: { + 'bootstrap': { + deps: ['jquery'] + }, + + 'backbone': { + deps: ['underscore', 'jquery'], + exports: 'Backbone' + }, + + 'datatables': { + deps: ['jquery'], + exports: '$.fn.dataTable' + } + } }); define(['utils', - 'run-job-view', - 'edit-job-view', - 'text!job-row-name', - 'text!job-row-action', - 'backbone', - 'bootstrap', - 'datatables'], function(utils, - RunJobView, - EditJobView, - JobRowNameHtml, - JobRowActionHtml) { - 'use strict'; - - return Backbone.View.extend({ - - initialize: function() { - this.listenTo(this.collection, 'sync', this.render); - this.listenTo(this.collection, 'request', this.requestRender); - this.listenTo(this.collection, 'reset', this.resetRender); - this.listenTo(this.collection, 'error', this.requestError); - - $('#jobs-refresh-button').on('click', _.bind(this.resetRender, this)); - $('#display-tz').on('change', _.bind(this.resetRender, this)); - - // Initialize data table - this.table = $('#jobs-table').dataTable({ - // Sorted by job name - 'order': [[0, 'asc']], - "iDisplayLength": 50 - }); - }, - - /** - * Request error handler. - * - * @param {object} model - * @param {object} response - * @param {object} options - */ - requestError: function(model, response, options) { - this.spinner.stop(); - utils.alertError('Request failed: ' + response.responseText); - }, - - /** - * Event handler for starting to make network request. - */ - requestRender: function() { - this.table.fnClearTable(); - this.spinner = utils.startSpinner('jobs-spinner'); - }, - - /** - * Event handler for resetting jobs data. - * - * This is registered with click handlers for the #jobs-refresh-button, the #display-tz - * dropdown, as well as this collection. When called from the collection, no parameter - * is given. - */ - resetRender: function(e) { - // It'll trigger sync event - if (e) { - e.preventDefault(); - } - this.collection.getJobs(); - }, - - /** - * Event handler for finishing fetching jobs data. - */ - render: function() { - var jobs = this.collection.jobs; - - var data = []; - - // Build up data to pass to data tables - _.each(jobs, function(job) { - var jobObj = job.toJSON(); - data.push([ - _.template(JobRowNameHtml)({ - 'job_name': _.escape(jobObj.name), - 'job_schedule': job.getScheduleString(), - 'next_run_at': job.getNextRunTimeHTMLString(), - 'job_id': jobObj.job_id, - 'job_class': _.escape(jobObj.job_class_string), - 'job_month': _.escape(jobObj.month), - 'job_day_of_week': _.escape(jobObj.day_of_week), - 'job_day': _.escape(jobObj.day), - 'job_hour': _.escape(jobObj.hour), - 'job_minute': _.escape(jobObj.minute), - 'job_active': job.getActiveString(), - 'job_pubargs': _.escape(job.getPubArgsString()) - }), - job.getScheduleString(), - job.getNextRunTimeHTMLString(), - _.template(JobRowActionHtml)({ - 'job_name': _.escape(jobObj.name), - 'job_id': jobObj.job_id, - 'job_class': _.escape(jobObj.job_class_string), - 'job_pubargs': _.escape(job.getPubArgsString()) - }) - ]); - }); - - if (data.length) { - this.table.fnClearTable(); - this.table.fnAddData(data); - } - - // Stop the spinner - this.spinner.stop(); - - // Set up the RunJob thing - new RunJobView({ - collection: this.collection - }); - - // Set up EditJob thing - new EditJobView({ - collection: this.collection - }); - - } - }); -}); + 'run-job-view', + 'edit-job-view', + 'text!job-row-name-cron', + 'text!job-row-name-interval', + 'text!job-row-name-unknown', + 'text!job-row-actions', + 'backbone', + 'bootstrap', + 'datatables'], + function(utils, + RunJobView, + EditJobView, + JobRowNameHtmlCron, + JobRowNameHtmlInterval, + JobRowNameHtmlUnknown, + JobRowActionsHtml + ) + { + 'use strict'; + + return Backbone.View.extend({ + + initialize: function() { + this.listenTo(this.collection, 'sync', this.render); + this.listenTo(this.collection, 'request', this.requestRender); + this.listenTo(this.collection, 'reset', this.resetRender); + this.listenTo(this.collection, 'error', this.requestError); + + $('#jobs-refresh-button').on('click', _.bind(this.resetRender, this)); + $('#display-tz').on('change', _.bind(this.resetRender, this)); + + // Initialize data table + this.table = $('#jobs-table').dataTable({ + // Sorted by job name + 'order': [[0, 'asc']], + "iDisplayLength": 50 + }); + }, + + /** + * Request error handler. + * + * @param {object} model + * @param {object} response + * @param {object} options + */ + requestError: function(model, response, options) { + this.spinner.stop(); + utils.alertError('Request failed: ' + response.responseText); + }, + + /** + * Event handler for starting to make network request. + */ + requestRender: function() { + this.table.fnClearTable(); + this.spinner = utils.startSpinner('jobs-spinner'); + }, + + /** + * Event handler for resetting jobs data. + * + * This is registered with click handlers for the #jobs-refresh-button, the #display-tz + * dropdown, as well as this collection. When called from the collection, no parameter + * is given. + */ + resetRender: function(e) { + // It'll trigger sync event + if (e) { + e.preventDefault(); + } + this.collection.getJobs(); + }, + + /** + * Event handler for finishing fetching jobs data. + */ + render: function() { + var jobs = this.collection.jobs; + + var data = []; + + // Build up data to pass to data tables + _.each(jobs, function(job) { + var jobObj = job.toJSON(); + + console.log("Job:") + console.log(jobObj) + + if (jobObj.trigger_type == 'cron') + { + data.push([ + _.template(JobRowNameHtmlCron)({ + 'job_name': _.escape(jobObj.name), + 'job_schedule': job.getScheduleString(), + 'next_run_at': job.getNextRunTimeHTMLString(), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_sched_type': _.escape("Cron"), + 'job_month': _.escape(jobObj.month), + 'job_day_of_week': _.escape(jobObj.day_of_week), + 'job_day': _.escape(jobObj.day), + 'job_hour': _.escape(jobObj.hour), + 'job_minute': _.escape(jobObj.minute), + 'job_active': job.getActiveString(), + 'job_pubargs': _.escape(job.getPubArgsString()) + }), + job.getScheduleString(), + job.getNextRunTimeHTMLString(), + _.template(JobRowActionsHtml)({ + 'job_name': _.escape(jobObj.name), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_pubargs': _.escape(job.getPubArgsString()) + }) + ]); + } + + + else if (jobObj.trigger_type == 'interval') + { + data.push([ + _.template(JobRowNameHtmlInterval)({ + 'job_name': _.escape(jobObj.name), + 'job_schedule': job.getScheduleString(), + 'next_run_at': job.getNextRunTimeHTMLString(), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_sched_type': _.escape("Interval"), + 'job_interval': _.escape(jobObj.interval), + 'job_active': job.getActiveString(), + 'job_pubargs': _.escape(job.getPubArgsString()) + }), + job.getScheduleString(), + job.getNextRunTimeHTMLString(), + _.template(JobRowActionsHtml)({ + 'job_name': _.escape(jobObj.name), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_pubargs': _.escape(job.getPubArgsString()) + }) + ]); + } + else + { + data.push([ + _.template(JobRowNameHtmlUnknown)({ + 'job_name': _.escape(jobObj.name), + 'job_schedule': job.getScheduleString(), + 'next_run_at': job.getNextRunTimeHTMLString(), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_sched_type': _.escape("Unknown"), + 'job_active': job.getActiveString(), + 'job_pubargs': _.escape(job.getPubArgsString()) + }), + job.getScheduleString(), + job.getNextRunTimeHTMLString(), + _.template(JobRowActionsHtml)({ + 'job_name': _.escape(jobObj.name), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_pubargs': _.escape(job.getPubArgsString()) + }) + ]); + } + + + + }); + + if (data.length) { + this.table.fnClearTable(); + this.table.fnAddData(data); + } + + // Stop the spinner + this.spinner.stop(); + + // Set up the RunJob thing + new RunJobView({ + collection: this.collection + }); + + // Set up EditJob thing + new EditJobView({ + collection: this.collection + }); + + } + }); + } +); From 2c3150beeec2ff73b96e93def7ce85775dbf5da6 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Wed, 10 Jul 2019 00:08:17 -0700 Subject: [PATCH 07/11] Handle interval scheduler. --- ndscheduler/core/datastore/providers/base.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ndscheduler/core/datastore/providers/base.py b/ndscheduler/core/datastore/providers/base.py index df1a03b..37ea914 100644 --- a/ndscheduler/core/datastore/providers/base.py +++ b/ndscheduler/core/datastore/providers/base.py @@ -3,6 +3,9 @@ import dateutil.parser import dateutil.tz +import apscheduler.triggers.cron +import apscheduler.triggers.interval + from apscheduler.jobstores import sqlalchemy as sched_sqlalchemy from sqlalchemy import select from sqlalchemy import desc @@ -104,7 +107,17 @@ def _build_execution(self, row): 'name': job.name, 'task_name': utils.get_job_name(job), 'pub_args': utils.get_job_args(job)} - return_json['job'].update(utils.get_cron_strings(job)) + + + if isinstance(job.trigger, apscheduler.triggers.cron.CronTrigger): + return_json.update(utils.get_cron_strings(job)) + return_json["trigger_type"] = "cron" + elif isinstance(job.trigger, apscheduler.triggers.interval.IntervalTrigger): + return_json["interval"] = job.trigger.interval.total_seconds() + return_json["trigger_type"] = "interval" + else: + return_json["trigger_type"] = "unknown" + return return_json def get_time_isoformat_from_db(self, time_object): From 6bcc85649f60a31803af0d9a067588bb8fd4d7b3 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sun, 29 Dec 2019 23:16:09 -0800 Subject: [PATCH 08/11] Handle stupid flake8 errors where the "correct" way to do things actively makes the code less readable. --- ndscheduler/core/scheduler/base.py | 12 ++++++------ ndscheduler/core/scheduler_manager.py | 10 +++++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ndscheduler/core/scheduler/base.py b/ndscheduler/core/scheduler/base.py index 3d3c148..e21051c 100644 --- a/ndscheduler/core/scheduler/base.py +++ b/ndscheduler/core/scheduler/base.py @@ -134,12 +134,12 @@ def add_trigger_scheduler_job(self, job_class_string, name, pub_args, trigger, arguments.extend(pub_args) scheduler_class = utils.import_from_path(settings.SCHEDULER_CLASS) - self.add_job(scheduler_class.run_job, - trigger = trigger, - args = arguments, - kwargs = kwargs, - name = name, - id = job_id + self.add_job(scheduler_class.run_job, # noqa + trigger = trigger, # noqa + args = arguments, # noqa + kwargs = kwargs, # noqa + name = name, # noqa + id = job_id # noqa ) return job_id diff --git a/ndscheduler/core/scheduler_manager.py b/ndscheduler/core/scheduler_manager.py index 4be5d4c..1b63928 100644 --- a/ndscheduler/core/scheduler_manager.py +++ b/ndscheduler/core/scheduler_manager.py @@ -92,8 +92,8 @@ def add_job(self, job_class_string, name, pub_args=None, day, hour, minute, **kwargs) def add_trigger_job(self, job_class_string, name, pub_args=None, - trigger=None, - **kwargs): + trigger=None, + **kwargs): """Add a job. Job infomation will be persistent in postgres. This is a NON-BLOCKING operation, as internally, apscheduler calls wakeup() @@ -111,7 +111,11 @@ def add_trigger_job(self, job_class_string, name, pub_args=None, :return: String of job id, e.g., 6bca19736d374ef2b3df23eb278b512e :rtype: str """ - return self.sched.add_trigger_scheduler_job(job_class_string, name, pub_args, trigger, **kwargs) + return self.sched.add_trigger_scheduler_job(job_class_string, + name, + pub_args, + trigger, + **kwargs) def pause_job(self, job_id): """Pauses the schedule of a job. From cab435a5c44778b72fb61a655741e1208b0a1d80 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sun, 29 Dec 2019 23:29:18 -0800 Subject: [PATCH 09/11] Flake8. --- ndscheduler/corescheduler/core/base.py | 3 ++- ndscheduler/corescheduler/datastore/base.py | 1 - ndscheduler/corescheduler/datastore/providers/postgres.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ndscheduler/corescheduler/core/base.py b/ndscheduler/corescheduler/core/base.py index e583f3e..810fa7a 100644 --- a/ndscheduler/corescheduler/core/base.py +++ b/ndscheduler/corescheduler/core/base.py @@ -2,6 +2,7 @@ import json +from ndscheduler import settings from apscheduler.schedulers import tornado as apscheduler_tornado from ndscheduler.corescheduler import constants @@ -150,7 +151,7 @@ def add_scheduler_job(self, job_class_string, name, pub_args=None, return job_id def add_trigger_scheduler_job(self, job_class_string, name, pub_args, trigger, - **kwargs): + **kwargs): """Add a job. Job infomation will be persistent in postgres. This is a NON-BLOCKING operation, as internally, apscheduler calls wakeup() diff --git a/ndscheduler/corescheduler/datastore/base.py b/ndscheduler/corescheduler/datastore/base.py index d63bc15..291cda3 100644 --- a/ndscheduler/corescheduler/datastore/base.py +++ b/ndscheduler/corescheduler/datastore/base.py @@ -129,7 +129,6 @@ def _build_execution(self, row): 'task_name': utils.get_job_name(job), 'pub_args': utils.get_job_args(job)} - if isinstance(job.trigger, apscheduler.triggers.cron.CronTrigger): return_json.update(utils.get_cron_strings(job)) return_json["trigger_type"] = "cron" diff --git a/ndscheduler/corescheduler/datastore/providers/postgres.py b/ndscheduler/corescheduler/datastore/providers/postgres.py index b4d2d4b..5d9a89b 100644 --- a/ndscheduler/corescheduler/datastore/providers/postgres.py +++ b/ndscheduler/corescheduler/datastore/providers/postgres.py @@ -1,5 +1,5 @@ """Represents Postgres datastore.""" - +import sys from ndscheduler.corescheduler.datastore import base From b09373b70637ba5dd528c99511ba78265826d664 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sun, 29 Dec 2019 23:33:38 -0800 Subject: [PATCH 10/11] More stupid. --- ndscheduler/server/handlers/jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndscheduler/server/handlers/jobs.py b/ndscheduler/server/handlers/jobs.py index 055a3a9..753b9d9 100644 --- a/ndscheduler/server/handlers/jobs.py +++ b/ndscheduler/server/handlers/jobs.py @@ -9,9 +9,9 @@ import apscheduler.triggers.cron import apscheduler.triggers.interval -from ndscheduler import constants -from ndscheduler import utils from ndscheduler.server.handlers import base +from ndscheduler.corescheduler import utils +from ndscheduler.corescheduler import constants class Handler(base.BaseHandler): From 8b1ec1ec8109ecc2c7c385e67e4e53863721b26d Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Mon, 30 Dec 2019 23:15:59 -0800 Subject: [PATCH 11/11] Fix trigger jobs being broken. --- ndscheduler/corescheduler/core/base.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ndscheduler/corescheduler/core/base.py b/ndscheduler/corescheduler/core/base.py index 810fa7a..73eee8a 100644 --- a/ndscheduler/corescheduler/core/base.py +++ b/ndscheduler/corescheduler/core/base.py @@ -145,9 +145,19 @@ def add_scheduler_job(self, job_class_string, name, pub_args=None, datastore.db_config, datastore.table_names] arguments.extend(pub_args) - self.add_job(self.run_job, - 'cron', month=month, day=day, day_of_week=day_of_week, hour=hour, - minute=minute, args=arguments, kwargs=kwargs, name=name, id=job_id) + self.add_job( + func = self.run_job, # noqa + trigger = 'cron', # noqa + month = month, # noqa + day = day, # noqa + day_of_week = day_of_week, # noqa + hour = hour, # noqa + minute = minute, # noqa + args = arguments, # noqa + kwargs = kwargs, # noqa + name = name, # noqa + id = job_id # noqa + ) return job_id def add_trigger_scheduler_job(self, job_class_string, name, pub_args, trigger, @@ -177,11 +187,13 @@ def add_trigger_scheduler_job(self, job_class_string, name, pub_args, trigger, job_id = utils.generate_uuid() - arguments = [job_class_string, job_id] + datastore = self._lookup_jobstore('default') + arguments = [job_class_string, job_id, self.datastore_class_path, + datastore.db_config, datastore.table_names] arguments.extend(pub_args) - scheduler_class = utils.import_from_path(settings.SCHEDULER_CLASS) - self.add_job(scheduler_class.run_job, # noqa + self.add_job( + func = self.run_job, # noqa trigger = trigger, # noqa args = arguments, # noqa kwargs = kwargs, # noqa