Merge pull request #46 from floatingstatic/sensor-improvements

Adjust sensor to better support large jira projects
StackStorm-Exchange · Feb 13, 2023 · 283cc7f · 283cc7f
2 parents 4879411 + 9fa11ad
commit 283cc7f
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 17 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,10 @@
 # Change Log
 
+## 2.2.0
+
+- Adjust jql in sensor to better support large JIRA projects
+- Detect new issues by id vs comparing to an in-memory list
+
 ## 2.1.0
 
 - Add new ``jira.bulk_link_issues`` action (PR #50)

diff --git a/pack.yaml b/pack.yaml
@@ -6,7 +6,7 @@ keywords:
   - issues
   - ticket management
   - project management
-version: 2.1.0
+version: 2.2.0
 python_versions:
   - "3"
 author : StackStorm, Inc.

diff --git a/sensors/jira_sensor.py b/sensors/jira_sensor.py
@@ -27,7 +27,7 @@ def __init__(self, sensor_service, config=None, poll_interval=5):
         self._projects_available = None
         self._poll_interval = 30
         self._project = None
-        self._issues_in_project = None
+        self._latest_id = None
         self._jql_query = None
         self._trigger_name = 'issues_tracker'
         self._trigger_pack = 'jira'
@@ -73,9 +73,12 @@ def setup(self):
         self._project = self._config.get('project', None)
         if not self._project or self._project not in self._projects_available:
             raise Exception('Invalid project (%s) to track.' % self._project)
-        self._jql_query = 'project=%s' % self._project
-        all_issues = self._jira_client.search_issues(self._jql_query, maxResults=None)
-        self._issues_in_project = {issue.key: issue for issue in all_issues}
+
+        self._jql_query = 'project={} ORDER BY id DESC'.format(self._project)
+        latest_issue = self._jira_client.search_issues(self._jql_query, maxResults=1)
+        if latest_issue:
+            self._latest_id = int(latest_issue[0].id)
+        self._update_jql(self._latest_id)
 
     def poll(self):
         self._detect_new_issues()
@@ -92,13 +95,18 @@ def update_trigger(self, trigger):
     def remove_trigger(self, trigger):
         pass
 
+    def _update_jql(self, latest_id=None):
+        jql = 'project={}'.format(self._project)
+        if latest_id:
+            jql = '{} AND id > {}'.format(jql, latest_id)
+        self._jql_query = '{} ORDER BY id ASC'.format(jql)
+
     def _detect_new_issues(self):
         new_issues = self._jira_client.search_issues(self._jql_query, maxResults=50, startAt=0)
-
         for issue in new_issues:
-            if issue.key not in self._issues_in_project:
-                self._dispatch_issues_trigger(issue)
-                self._issues_in_project[issue.key] = issue
+            self._latest_id = int(issue.id)
+            self._dispatch_issues_trigger(issue)
+        self._update_jql(self._latest_id)
 
     def _dispatch_issues_trigger(self, issue):
         trigger = self._trigger_ref

diff --git a/sensors/jira_sensor_for_apiv2.py b/sensors/jira_sensor_for_apiv2.py
@@ -27,7 +27,7 @@ def __init__(self, sensor_service, config=None, poll_interval=5):
         self._projects_available = None
         self._poll_interval = 30
         self._project = None
-        self._issues_in_project = None
+        self._latest_id = None
         self._jql_query = None
         self._trigger_name = 'issues_tracker_for_apiv2'
         self._trigger_pack = 'jira'
@@ -73,9 +73,12 @@ def setup(self):
         self._project = self._config.get('project', None)
         if not self._project or self._project not in self._projects_available:
             raise Exception('Invalid project (%s) to track.' % self._project)
-        self._jql_query = 'project=%s' % self._project
-        all_issues = self._jira_client.search_issues(self._jql_query, maxResults=None)
-        self._issues_in_project = {issue.key: issue for issue in all_issues}
+
+        self._jql_query = 'project={} ORDER BY id DESC'.format(self._project)
+        latest_issue = self._jira_client.search_issues(self._jql_query, maxResults=1)
+        if latest_issue:
+            self._latest_id = int(latest_issue[0].id)
+        self._update_jql(self._latest_id)
 
     def poll(self):
         self._detect_new_issues()
@@ -92,13 +95,18 @@ def update_trigger(self, trigger):
     def remove_trigger(self, trigger):
         pass
 
+    def _update_jql(self, latest_id=None):
+        jql = 'project={}'.format(self._project)
+        if latest_id:
+            jql = '{} AND id > {}'.format(jql, latest_id)
+        self._jql_query = '{} ORDER BY id ASC'.format(jql)
+
     def _detect_new_issues(self):
         new_issues = self._jira_client.search_issues(self._jql_query, maxResults=50, startAt=0)
-
         for issue in new_issues:
-            if issue.key not in self._issues_in_project:
-                self._dispatch_issues_trigger(issue)
-                self._issues_in_project[issue.key] = issue
+            self._latest_id = int(issue.id)
+            self._dispatch_issues_trigger(issue)
+        self._update_jql(self._latest_id)
 
     def _dispatch_issues_trigger(self, issue):
         trigger = self._trigger_ref

diff --git a/tests/test_sensor_jira_sensor.py b/tests/test_sensor_jira_sensor.py
@@ -24,6 +24,7 @@
 }
 
 MOCK_ISSUE_RAW = {
+    "id": '1',
     "fields": {
         "created": MOCK_PAYLOAD["created"],
         "assignee": MOCK_PAYLOAD["assignee"],
@@ -52,6 +53,7 @@ def test_poll(self):
         issue = mock.Mock()
         issue.raw = MOCK_ISSUE_RAW
 
+        issue.id = MOCK_ISSUE_RAW["id"]
         issue.key = MOCK_PAYLOAD["issue_name"]
         issue.self = MOCK_PAYLOAD["issue_url"]
         sensor._project = MOCK_PAYLOAD["project"]