diff --git a/ndscheduler/__init__.py b/ndscheduler/__init__.py index 5136b73..0343f24 100644 --- a/ndscheduler/__init__.py +++ b/ndscheduler/__init__.py @@ -11,17 +11,17 @@ """ import importlib -import logging import os -import sys +import logging from ndscheduler import default_settings +# import sys logger = logging.getLogger() -ch = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) +# ch = logging.StreamHandler(sys.stdout) +# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# ch.setFormatter(formatter) +# logger.addHandler(ch) ENVIRONMENT_VARIABLE = 'NDSCHEDULER_SETTINGS_MODULE' diff --git a/ndscheduler/corescheduler/core/base.py b/ndscheduler/corescheduler/core/base.py index d01eaf0..73eee8a 100644 --- a/ndscheduler/corescheduler/core/base.py +++ b/ndscheduler/corescheduler/core/base.py @@ -2,6 +2,7 @@ import json +from ndscheduler import settings from apscheduler.schedulers import tornado as apscheduler_tornado from ndscheduler.corescheduler import constants @@ -144,9 +145,61 @@ def add_scheduler_job(self, job_class_string, name, pub_args=None, datastore.db_config, datastore.table_names] arguments.extend(pub_args) - self.add_job(self.run_job, - 'cron', month=month, day=day, day_of_week=day_of_week, hour=hour, - minute=minute, args=arguments, kwargs=kwargs, name=name, id=job_id) + self.add_job( + func = self.run_job, # noqa + trigger = 'cron', # noqa + month = month, # noqa + day = day, # noqa + day_of_week = day_of_week, # noqa + hour = hour, # noqa + minute = minute, # noqa + args = arguments, # noqa + kwargs = kwargs, # noqa + name = name, # noqa + id = job_id # noqa + ) + return job_id + + def add_trigger_scheduler_job(self, job_class_string, name, pub_args, trigger, + **kwargs): + """Add a job. Job infomation will be persistent in postgres. + + This is a NON-BLOCKING operation, as internally, apscheduler calls wakeup() + that is async. + + :param str job_class_string: String for job class, e.g., myscheduler.jobs.a_job.NiceJob + :param str name: String for job name, e.g., Check Melissa job. + :param str pub_args: List for arguments passed to publish method of a task. + :param str month: String for month cron string, e.g., */10 + :param str day_of_week: String for day of week cron string, e.g., 1-6 + :param str day: String for day cron string, e.g., */1 + :param str hour: String for hour cron string, e.g., */2 + :param str minute: String for minute cron string, e.g., */3 + :param dict kwargs: Other keyword arguments passed to run_job function. + :return: String of job id, e.g., 6bca19736d374ef2b3df23eb278b512e + :rtype: str + + Returns: + String of job id, e.g., 6bca19736d374ef2b3df23eb278b512e + """ + if not pub_args: + pub_args = [] + + job_id = utils.generate_uuid() + + datastore = self._lookup_jobstore('default') + arguments = [job_class_string, job_id, self.datastore_class_path, + datastore.db_config, datastore.table_names] + arguments.extend(pub_args) + + self.add_job( + func = self.run_job, # noqa + trigger = trigger, # noqa + args = arguments, # noqa + kwargs = kwargs, # noqa + name = name, # noqa + id = job_id # noqa + ) return job_id def modify_scheduler_job(self, job_id, **kwargs): diff --git a/ndscheduler/corescheduler/datastore/base.py b/ndscheduler/corescheduler/datastore/base.py index be10772..291cda3 100644 --- a/ndscheduler/corescheduler/datastore/base.py +++ b/ndscheduler/corescheduler/datastore/base.py @@ -2,6 +2,9 @@ import dateutil.tz import dateutil.parser +import apscheduler.triggers.cron +import apscheduler.triggers.interval + from apscheduler.jobstores import sqlalchemy as sched_sqlalchemy from sqlalchemy import desc, select, MetaData @@ -125,7 +128,16 @@ def _build_execution(self, row): 'name': job.name, 'task_name': utils.get_job_name(job), 'pub_args': utils.get_job_args(job)} - return_json['job'].update(utils.get_cron_strings(job)) + + if isinstance(job.trigger, apscheduler.triggers.cron.CronTrigger): + return_json.update(utils.get_cron_strings(job)) + return_json["trigger_type"] = "cron" + elif isinstance(job.trigger, apscheduler.triggers.interval.IntervalTrigger): + return_json["interval"] = job.trigger.interval.total_seconds() + return_json["trigger_type"] = "interval" + else: + return_json["trigger_type"] = "unknown" + return return_json def get_time_isoformat_from_db(self, time_object): diff --git a/ndscheduler/corescheduler/datastore/providers/postgres.py b/ndscheduler/corescheduler/datastore/providers/postgres.py index 20ee0d7..5d9a89b 100644 --- a/ndscheduler/corescheduler/datastore/providers/postgres.py +++ b/ndscheduler/corescheduler/datastore/providers/postgres.py @@ -1,5 +1,5 @@ """Represents Postgres datastore.""" - +import sys from ndscheduler.corescheduler.datastore import base @@ -18,7 +18,15 @@ def get_db_url(self): } :return: string db url """ - return 'postgresql://%s:%s@%s:%d/%s?sslmode=%s' % ( + + # Work under Pypy, which doesn't have the default psycopg2 + if '__pypy__' in sys.builtin_module_names: + db_wrapper = 'postgresql+psycopg2cffi' + else: + db_wrapper = 'postgresql' + + return '%s://%s:%s@%s:%d/%s?sslmode=%s' % ( + db_wrapper, self.db_config['user'], self.db_config['password'], self.db_config['hostname'], diff --git a/ndscheduler/corescheduler/scheduler_manager.py b/ndscheduler/corescheduler/scheduler_manager.py index eb44247..0820738 100644 --- a/ndscheduler/corescheduler/scheduler_manager.py +++ b/ndscheduler/corescheduler/scheduler_manager.py @@ -99,6 +99,32 @@ def add_job(self, job_class_string, name, pub_args=None, month=None, return self.sched.add_scheduler_job(job_class_string, name, pub_args, month, day_of_week, day, hour, minute, **kwargs) + def add_trigger_job(self, job_class_string, name, pub_args=None, + trigger=None, + **kwargs): + """Add a job. Job infomation will be persistent in postgres. + + This is a NON-BLOCKING operation, as internally, apscheduler calls wakeup() + that is async. + + :param str job_class_string: String for job class, e.g., myscheduler.jobs.a_job.NiceJob + :param str name: String for job name, e.g., Check Melissa job. + :param str pub_args: List for arguments passed to publish method of a task. + :param str month: String for month cron string, e.g., */10 + :param str day_of_week: String for day of week cron string, e.g., 1-6 + :param str day: String for day cron string, e.g., */1 + :param str hour: String for hour cron string, e.g., */2 + :param str minute: String for minute cron string, e.g., */3 + :param dict kwargs: Other keyword arguments passed to run_job function. + :return: String of job id, e.g., 6bca19736d374ef2b3df23eb278b512e + :rtype: str + """ + return self.sched.add_trigger_scheduler_job(job_class_string, + name, + pub_args, + trigger, + **kwargs) + def pause_job(self, job_id): """Pauses the schedule of a job. This is a NON-BLOCKING operation, as internally, apscheduler calls wakeup() diff --git a/ndscheduler/default_settings.py b/ndscheduler/default_settings.py index 00d1626..fb8694f 100644 --- a/ndscheduler/default_settings.py +++ b/ndscheduler/default_settings.py @@ -1,9 +1,7 @@ """Default settings.""" -import logging import os - # # Development mode or production mode # If DEBUG is True, then auto-reload is enabled, i.e., when code is modified, server will be @@ -92,11 +90,5 @@ # Please see ndscheduler/core/scheduler/base.py SCHEDULER_CLASS = 'ndscheduler.corescheduler.core.base.BaseScheduler' -# -# Set logging level -# -logging.getLogger().setLevel(logging.INFO) - - # Packages that contains job classes, e.g., simple_scheduler.jobs JOB_CLASS_PACKAGES = [] diff --git a/ndscheduler/server/handlers/jobs.py b/ndscheduler/server/handlers/jobs.py index 2d0f3e0..753b9d9 100644 --- a/ndscheduler/server/handlers/jobs.py +++ b/ndscheduler/server/handlers/jobs.py @@ -6,9 +6,12 @@ import tornado.gen import tornado.web -from ndscheduler.corescheduler import constants -from ndscheduler.corescheduler import utils +import apscheduler.triggers.cron +import apscheduler.triggers.interval + from ndscheduler.server.handlers import base +from ndscheduler.corescheduler import utils +from ndscheduler.corescheduler import constants class Handler(base.BaseHandler): @@ -42,7 +45,15 @@ def _build_job_dict(self, job): 'job_class_string': utils.get_job_name(job), 'pub_args': utils.get_job_args(job)} - return_dict.update(utils.get_cron_strings(job)) + if isinstance(job.trigger, apscheduler.triggers.cron.CronTrigger): + return_dict.update(utils.get_cron_strings(job)) + return_dict["trigger_type"] = "cron" + elif isinstance(job.trigger, apscheduler.triggers.interval.IntervalTrigger): + return_dict["interval"] = job.trigger.interval.total_seconds() + return_dict["trigger_type"] = "interval" + else: + return_dict["trigger_type"] = "unknown" + return return_dict @tornado.concurrent.run_on_executor diff --git a/ndscheduler/static/js/models/job.js b/ndscheduler/static/js/models/job.js index 5f95a70..0987a5e 100644 --- a/ndscheduler/static/js/models/job.js +++ b/ndscheduler/static/js/models/job.js @@ -20,6 +20,26 @@ require.config({ } }); +function secondsToStr( seconds_in ) { + let temp = seconds_in; + const years = Math.floor( temp / 31536000 ), + days = Math.floor( ( temp %= 31536000 ) / 86400 ), + hours = Math.floor( ( temp %= 86400 ) / 3600 ), + minutes = Math.floor( ( temp %= 3600 ) / 60 ), + seconds = temp % 60; + + if ( days || hours || seconds || minutes ) { + return ( years ? years + "y " : "" ) + + ( days ? days + "d " : "" ) + + ( hours ? hours + "h " : "" ) + + ( minutes ? minutes + "m " : "" ) + + Number.parseFloat( seconds ).toFixed( 2 ) + "s"; + } + + return "< 1s"; +} + + define(['backbone', 'vendor/moment-timezone-with-data'], function(backbone, moment) { 'use strict'; @@ -31,9 +51,18 @@ define(['backbone', 'vendor/moment-timezone-with-data'], function(backbone, mome * @return {string} schedule string for this job. */ getScheduleString: function() { - return 'minute: ' + this.get('minute') + ', hour: ' + this.get('hour') + - ', day: ' + this.get('day') + ', month: ' + this.get('month') + - ', day of week: ' + this.get('day_of_week'); + var trig = this.get('trigger_type'); + + if (trig == 'cron') + return 'Cron: minute: ' + this.get('minute') + ', hour: ' + this.get('hour') + + ', day: ' + this.get('day') + ', month: ' + this.get('month') + + ', day of week: ' + this.get('day_of_week'); + else if (trig == 'interval') + return 'Interval: ' + secondsToStr(this.get('interval')); + else + return 'Unknown trigger type!'; + + }, /** diff --git a/ndscheduler/static/js/templates/job-row-action.html b/ndscheduler/static/js/templates/job-row-actions.html similarity index 100% rename from ndscheduler/static/js/templates/job-row-action.html rename to ndscheduler/static/js/templates/job-row-actions.html diff --git a/ndscheduler/static/js/templates/job-row-name.html b/ndscheduler/static/js/templates/job-row-name-cron.html similarity index 95% rename from ndscheduler/static/js/templates/job-row-name.html rename to ndscheduler/static/js/templates/job-row-name-cron.html index 258cf89..b204d43 100644 --- a/ndscheduler/static/js/templates/job-row-name.html +++ b/ndscheduler/static/js/templates/job-row-name-cron.html @@ -5,6 +5,7 @@ data-target="#edit-job-modal" data-id="<%= job_id %>" data-job-name="<%= job_name %>" + data-job-trigtype="cron" data-job-month="<%= job_month %>" data-job-day-of-week="<%= job_day_of_week %>" data-job-day="<%= job_day %>" diff --git a/ndscheduler/static/js/templates/job-row-name-interval.html b/ndscheduler/static/js/templates/job-row-name-interval.html new file mode 100644 index 0000000..76b1afa --- /dev/null +++ b/ndscheduler/static/js/templates/job-row-name-interval.html @@ -0,0 +1,14 @@ + + + + <%= job_name %> + diff --git a/ndscheduler/static/js/templates/job-row-name-unknown.html b/ndscheduler/static/js/templates/job-row-name-unknown.html new file mode 100644 index 0000000..5d8df5d --- /dev/null +++ b/ndscheduler/static/js/templates/job-row-name-unknown.html @@ -0,0 +1,13 @@ + + + + <%= job_name %> + diff --git a/ndscheduler/static/js/views/executions/table-view.js b/ndscheduler/static/js/views/executions/table-view.js index d667533..99fb2b8 100644 --- a/ndscheduler/static/js/views/executions/table-view.js +++ b/ndscheduler/static/js/views/executions/table-view.js @@ -52,6 +52,7 @@ define(['utils', this.table = $('#executions-table').dataTable({ // Sorted by last updated time 'order': [[3, 'desc']], + "iDisplayLength": 50, // Disable sorting on result column "columnDefs": [ { "orderable": false, "className": "table-result-column", "targets": 5 } diff --git a/ndscheduler/static/js/views/jobs/table-view.js b/ndscheduler/static/js/views/jobs/table-view.js index 9b11d56..42aea32 100644 --- a/ndscheduler/static/js/views/jobs/table-view.js +++ b/ndscheduler/static/js/views/jobs/table-view.js @@ -5,161 +5,232 @@ */ require.config({ - paths: { - 'jquery': 'vendor/jquery', - 'underscore': 'vendor/underscore', - 'backbone': 'vendor/backbone', - 'bootstrap': 'vendor/bootstrap', - 'datatables': 'vendor/jquery.dataTables', - - 'utils': 'utils', - 'run-job-view': 'views/jobs/run-job-view', - 'edit-job-view': 'views/jobs/edit-job-view', - - 'text': 'vendor/text', - 'job-row-name': 'templates/job-row-name.html', - 'job-row-action': 'templates/job-row-action.html' - }, - - shim: { - 'bootstrap': { - deps: ['jquery'] - }, - - 'backbone': { - deps: ['underscore', 'jquery'], - exports: 'Backbone' - }, - - 'datatables': { - deps: ['jquery'], - exports: '$.fn.dataTable' - } - } + paths: { + 'jquery': 'vendor/jquery', + 'underscore': 'vendor/underscore', + 'backbone': 'vendor/backbone', + 'bootstrap': 'vendor/bootstrap', + 'datatables': 'vendor/jquery.dataTables', + + 'utils': 'utils', + 'run-job-view': 'views/jobs/run-job-view', + 'edit-job-view': 'views/jobs/edit-job-view', + + 'text': 'vendor/text', + 'job-row-name-cron' : 'templates/job-row-name-cron.html', + 'job-row-name-interval' : 'templates/job-row-name-interval.html', + 'job-row-name-unknown' : 'templates/job-row-name-unknown.html', + 'job-row-actions' : 'templates/job-row-actions.html' + }, + + shim: { + 'bootstrap': { + deps: ['jquery'] + }, + + 'backbone': { + deps: ['underscore', 'jquery'], + exports: 'Backbone' + }, + + 'datatables': { + deps: ['jquery'], + exports: '$.fn.dataTable' + } + } }); define(['utils', - 'run-job-view', - 'edit-job-view', - 'text!job-row-name', - 'text!job-row-action', - 'backbone', - 'bootstrap', - 'datatables'], function(utils, - RunJobView, - EditJobView, - JobRowNameHtml, - JobRowActionHtml) { - 'use strict'; - - return Backbone.View.extend({ - - initialize: function() { - this.listenTo(this.collection, 'sync', this.render); - this.listenTo(this.collection, 'request', this.requestRender); - this.listenTo(this.collection, 'reset', this.resetRender); - this.listenTo(this.collection, 'error', this.requestError); - - $('#jobs-refresh-button').on('click', _.bind(this.resetRender, this)); - $('#display-tz').on('change', _.bind(this.resetRender, this)); - - // Initialize data table - this.table = $('#jobs-table').dataTable({ - // Sorted by job name - 'order': [[0, 'asc']] - }); - }, - - /** - * Request error handler. - * - * @param {object} model - * @param {object} response - * @param {object} options - */ - requestError: function(model, response, options) { - this.spinner.stop(); - utils.alertError('Request failed: ' + response.responseText); - }, - - /** - * Event handler for starting to make network request. - */ - requestRender: function() { - this.table.fnClearTable(); - this.spinner = utils.startSpinner('jobs-spinner'); - }, - - /** - * Event handler for resetting jobs data. - * - * This is registered with click handlers for the #jobs-refresh-button, the #display-tz - * dropdown, as well as this collection. When called from the collection, no parameter - * is given. - */ - resetRender: function(e) { - // It'll trigger sync event - if (e) { - e.preventDefault(); - } - this.collection.getJobs(); - }, - - /** - * Event handler for finishing fetching jobs data. - */ - render: function() { - var jobs = this.collection.jobs; - - var data = []; - - // Build up data to pass to data tables - _.each(jobs, function(job) { - var jobObj = job.toJSON(); - data.push([ - _.template(JobRowNameHtml)({ - 'job_name': _.escape(jobObj.name), - 'job_schedule': job.getScheduleString(), - 'next_run_at': job.getNextRunTimeHTMLString(), - 'job_id': jobObj.job_id, - 'job_class': _.escape(jobObj.job_class_string), - 'job_month': _.escape(jobObj.month), - 'job_day_of_week': _.escape(jobObj.day_of_week), - 'job_day': _.escape(jobObj.day), - 'job_hour': _.escape(jobObj.hour), - 'job_minute': _.escape(jobObj.minute), - 'job_active': job.getActiveString(), - 'job_pubargs': _.escape(job.getPubArgsString()) - }), - job.getScheduleString(), - job.getNextRunTimeHTMLString(), - _.template(JobRowActionHtml)({ - 'job_name': _.escape(jobObj.name), - 'job_id': jobObj.job_id, - 'job_class': _.escape(jobObj.job_class_string), - 'job_pubargs': _.escape(job.getPubArgsString()) - }) - ]); - }); - - if (data.length) { - this.table.fnClearTable(); - this.table.fnAddData(data); - } - - // Stop the spinner - this.spinner.stop(); - - // Set up the RunJob thing - new RunJobView({ - collection: this.collection - }); - - // Set up EditJob thing - new EditJobView({ - collection: this.collection - }); - - } - }); -}); + 'run-job-view', + 'edit-job-view', + 'text!job-row-name-cron', + 'text!job-row-name-interval', + 'text!job-row-name-unknown', + 'text!job-row-actions', + 'backbone', + 'bootstrap', + 'datatables'], + function(utils, + RunJobView, + EditJobView, + JobRowNameHtmlCron, + JobRowNameHtmlInterval, + JobRowNameHtmlUnknown, + JobRowActionsHtml + ) + { + 'use strict'; + + return Backbone.View.extend({ + + initialize: function() { + this.listenTo(this.collection, 'sync', this.render); + this.listenTo(this.collection, 'request', this.requestRender); + this.listenTo(this.collection, 'reset', this.resetRender); + this.listenTo(this.collection, 'error', this.requestError); + + $('#jobs-refresh-button').on('click', _.bind(this.resetRender, this)); + $('#display-tz').on('change', _.bind(this.resetRender, this)); + + // Initialize data table + this.table = $('#jobs-table').dataTable({ + // Sorted by job name + 'order': [[0, 'asc']], + "iDisplayLength": 50 + }); + }, + + /** + * Request error handler. + * + * @param {object} model + * @param {object} response + * @param {object} options + */ + requestError: function(model, response, options) { + this.spinner.stop(); + utils.alertError('Request failed: ' + response.responseText); + }, + + /** + * Event handler for starting to make network request. + */ + requestRender: function() { + this.table.fnClearTable(); + this.spinner = utils.startSpinner('jobs-spinner'); + }, + + /** + * Event handler for resetting jobs data. + * + * This is registered with click handlers for the #jobs-refresh-button, the #display-tz + * dropdown, as well as this collection. When called from the collection, no parameter + * is given. + */ + resetRender: function(e) { + // It'll trigger sync event + if (e) { + e.preventDefault(); + } + this.collection.getJobs(); + }, + + /** + * Event handler for finishing fetching jobs data. + */ + render: function() { + var jobs = this.collection.jobs; + + var data = []; + + // Build up data to pass to data tables + _.each(jobs, function(job) { + var jobObj = job.toJSON(); + + console.log("Job:") + console.log(jobObj) + + if (jobObj.trigger_type == 'cron') + { + data.push([ + _.template(JobRowNameHtmlCron)({ + 'job_name': _.escape(jobObj.name), + 'job_schedule': job.getScheduleString(), + 'next_run_at': job.getNextRunTimeHTMLString(), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_sched_type': _.escape("Cron"), + 'job_month': _.escape(jobObj.month), + 'job_day_of_week': _.escape(jobObj.day_of_week), + 'job_day': _.escape(jobObj.day), + 'job_hour': _.escape(jobObj.hour), + 'job_minute': _.escape(jobObj.minute), + 'job_active': job.getActiveString(), + 'job_pubargs': _.escape(job.getPubArgsString()) + }), + job.getScheduleString(), + job.getNextRunTimeHTMLString(), + _.template(JobRowActionsHtml)({ + 'job_name': _.escape(jobObj.name), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_pubargs': _.escape(job.getPubArgsString()) + }) + ]); + } + + + else if (jobObj.trigger_type == 'interval') + { + data.push([ + _.template(JobRowNameHtmlInterval)({ + 'job_name': _.escape(jobObj.name), + 'job_schedule': job.getScheduleString(), + 'next_run_at': job.getNextRunTimeHTMLString(), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_sched_type': _.escape("Interval"), + 'job_interval': _.escape(jobObj.interval), + 'job_active': job.getActiveString(), + 'job_pubargs': _.escape(job.getPubArgsString()) + }), + job.getScheduleString(), + job.getNextRunTimeHTMLString(), + _.template(JobRowActionsHtml)({ + 'job_name': _.escape(jobObj.name), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_pubargs': _.escape(job.getPubArgsString()) + }) + ]); + } + else + { + data.push([ + _.template(JobRowNameHtmlUnknown)({ + 'job_name': _.escape(jobObj.name), + 'job_schedule': job.getScheduleString(), + 'next_run_at': job.getNextRunTimeHTMLString(), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_sched_type': _.escape("Unknown"), + 'job_active': job.getActiveString(), + 'job_pubargs': _.escape(job.getPubArgsString()) + }), + job.getScheduleString(), + job.getNextRunTimeHTMLString(), + _.template(JobRowActionsHtml)({ + 'job_name': _.escape(jobObj.name), + 'job_id': jobObj.job_id, + 'job_class': _.escape(jobObj.job_class_string), + 'job_pubargs': _.escape(job.getPubArgsString()) + }) + ]); + } + + + + }); + + if (data.length) { + this.table.fnClearTable(); + this.table.fnAddData(data); + } + + // Stop the spinner + this.spinner.stop(); + + // Set up the RunJob thing + new RunJobView({ + collection: this.collection + }); + + // Set up EditJob thing + new EditJobView({ + collection: this.collection + }); + + } + }); + } +); diff --git a/ndscheduler/static/js/views/logs/table-view.js b/ndscheduler/static/js/views/logs/table-view.js index 21fe298..03c0816 100644 --- a/ndscheduler/static/js/views/logs/table-view.js +++ b/ndscheduler/static/js/views/logs/table-view.js @@ -48,6 +48,7 @@ define(['utils', // Initialize data table this.table = $('#logs-table').dataTable({ + "iDisplayLength": 50, // Sorted by job name 'order': [[3, 'desc']] });