From e2e9c3024c72a0f229dd9d23cb70f70ee0af1aad Mon Sep 17 00:00:00 2001 From: Jeremiah Millay Date: Fri, 9 Sep 2022 12:49:49 -0400 Subject: [PATCH 1/2] Adjust sensor to better support large jira projects --- CHANGES.md | 5 +++++ pack.yaml | 2 +- sensors/jira_sensor.py | 24 ++++++++++++++++-------- sensors/jira_sensor_for_apiv2.py | 24 ++++++++++++++++-------- 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2699909..6bc9313 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ # Change Log +## 2.2.0 + +- Adjust jql in sensor to better support large JIRA projects +- Detect new issues by id vs comparing to an in-memory list + ## 2.1.0 - Add new ``jira.bulk_link_issues`` action (PR #50) diff --git a/pack.yaml b/pack.yaml index 0627a71..8ee7255 100644 --- a/pack.yaml +++ b/pack.yaml @@ -6,7 +6,7 @@ keywords: - issues - ticket management - project management -version: 2.1.0 +version: 2.2.0 python_versions: - "3" author : StackStorm, Inc. diff --git a/sensors/jira_sensor.py b/sensors/jira_sensor.py index c168d40..120074a 100644 --- a/sensors/jira_sensor.py +++ b/sensors/jira_sensor.py @@ -27,7 +27,7 @@ def __init__(self, sensor_service, config=None, poll_interval=5): self._projects_available = None self._poll_interval = 30 self._project = None - self._issues_in_project = None + self._latest_id = None self._jql_query = None self._trigger_name = 'issues_tracker' self._trigger_pack = 'jira' @@ -73,9 +73,12 @@ def setup(self): self._project = self._config.get('project', None) if not self._project or self._project not in self._projects_available: raise Exception('Invalid project (%s) to track.' % self._project) - self._jql_query = 'project=%s' % self._project - all_issues = self._jira_client.search_issues(self._jql_query, maxResults=None) - self._issues_in_project = {issue.key: issue for issue in all_issues} + + self._jql_query = 'project={} ORDER BY id DESC'.format(self._project) + latest_issue = self._jira_client.search_issues(self._jql_query, maxResults=1) + if latest_issue: + self._latest_id = int(latest_issue[0].id) + self._update_jql(self._latest_id) def poll(self): self._detect_new_issues() @@ -92,13 +95,18 @@ def update_trigger(self, trigger): def remove_trigger(self, trigger): pass + def _update_jql(self, latest_id=None): + jql = 'project={}'.format(self._project) + if latest_id: + jql = '{} AND id > {}'.format(jql, latest_id) + self._jql_query = '{} ORDER BY id ASC'.format(jql) + def _detect_new_issues(self): new_issues = self._jira_client.search_issues(self._jql_query, maxResults=50, startAt=0) - for issue in new_issues: - if issue.key not in self._issues_in_project: - self._dispatch_issues_trigger(issue) - self._issues_in_project[issue.key] = issue + self._latest_id = int(issue.id) + self._dispatch_issues_trigger(issue) + self._update_jql(self._latest_id) def _dispatch_issues_trigger(self, issue): trigger = self._trigger_ref diff --git a/sensors/jira_sensor_for_apiv2.py b/sensors/jira_sensor_for_apiv2.py index c676225..4b488e4 100644 --- a/sensors/jira_sensor_for_apiv2.py +++ b/sensors/jira_sensor_for_apiv2.py @@ -27,7 +27,7 @@ def __init__(self, sensor_service, config=None, poll_interval=5): self._projects_available = None self._poll_interval = 30 self._project = None - self._issues_in_project = None + self._latest_id = None self._jql_query = None self._trigger_name = 'issues_tracker_for_apiv2' self._trigger_pack = 'jira' @@ -73,9 +73,12 @@ def setup(self): self._project = self._config.get('project', None) if not self._project or self._project not in self._projects_available: raise Exception('Invalid project (%s) to track.' % self._project) - self._jql_query = 'project=%s' % self._project - all_issues = self._jira_client.search_issues(self._jql_query, maxResults=None) - self._issues_in_project = {issue.key: issue for issue in all_issues} + + self._jql_query = 'project={} ORDER BY id DESC'.format(self._project) + latest_issue = self._jira_client.search_issues(self._jql_query, maxResults=1) + if latest_issue: + self._latest_id = int(latest_issue[0].id) + self._update_jql(self._latest_id) def poll(self): self._detect_new_issues() @@ -92,13 +95,18 @@ def update_trigger(self, trigger): def remove_trigger(self, trigger): pass + def _update_jql(self, latest_id=None): + jql = 'project={}'.format(self._project) + if latest_id: + jql = '{} AND id > {}'.format(jql, latest_id) + self._jql_query = '{} ORDER BY id ASC'.format(jql) + def _detect_new_issues(self): new_issues = self._jira_client.search_issues(self._jql_query, maxResults=50, startAt=0) - for issue in new_issues: - if issue.key not in self._issues_in_project: - self._dispatch_issues_trigger(issue) - self._issues_in_project[issue.key] = issue + self._latest_id = int(issue.id) + self._dispatch_issues_trigger(issue) + self._update_jql(self._latest_id) def _dispatch_issues_trigger(self, issue): trigger = self._trigger_ref From 9fa11ad3d2ae9ba62a4b7b4aa63ccef9dc5515d8 Mon Sep 17 00:00:00 2001 From: Jeremiah Millay Date: Fri, 10 Feb 2023 11:31:43 -0500 Subject: [PATCH 2/2] add issue id to sensor test case --- tests/test_sensor_jira_sensor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_sensor_jira_sensor.py b/tests/test_sensor_jira_sensor.py index 32fa547..d921d15 100644 --- a/tests/test_sensor_jira_sensor.py +++ b/tests/test_sensor_jira_sensor.py @@ -24,6 +24,7 @@ } MOCK_ISSUE_RAW = { + "id": '1', "fields": { "created": MOCK_PAYLOAD["created"], "assignee": MOCK_PAYLOAD["assignee"], @@ -52,6 +53,7 @@ def test_poll(self): issue = mock.Mock() issue.raw = MOCK_ISSUE_RAW + issue.id = MOCK_ISSUE_RAW["id"] issue.key = MOCK_PAYLOAD["issue_name"] issue.self = MOCK_PAYLOAD["issue_url"] sensor._project = MOCK_PAYLOAD["project"]