diff --git a/.github/workflows/test-and-push.yml b/.github/workflows/test-and-push.yml index 6a9f034..043182d 100644 --- a/.github/workflows/test-and-push.yml +++ b/.github/workflows/test-and-push.yml @@ -9,13 +9,13 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - name: Install dependencies run: | - pip3 install pytest-cov pycodestyle codecov + pip3 install pytest-cov pycodestyle codecov hatch pip3 install -r requirements.txt - name: Run dependencies diff --git a/README.md b/README.md index eb36d10..c664c2f 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,13 @@ execute tasks, coordinated by Celery. Celery supports various message brokers in Clone the repo anywhere and install dependencies with (from the repo root): ``` -pip3 install --user -r requirements.txt +hatch shell +pip3 install -r requirements.txt ``` ## Development -Run dependencies (Montagu API and DB, OrderlyWeb and a local Redis message queue in docker) with `scripts/run-dependencies.sh` +Run dependencies (Montagu API and DB, Proxy, Packit and a local Redis message queue in docker) with `scripts/run-dev-dependencies.sh` Dependencies also include a fake smtp server run from a [docker image](https://hub.docker.com/r/reachfive/fake-smtp-server) to enable development and testing of email functionality. You can see a web front end for the emails 'sent' via this server @@ -54,7 +55,7 @@ The worker expects to find a config file at `config/config.yml`. The Dockerfile copies `config/docker_config.yml` to `config/config.yml`. This allows the worker running on metal to use a broker on `localhost` while the worker in docker needs to use -`montagu_mq`, the container name of the broker, to access its port. +`montagu-mq`, the container name of the broker, to access its port. Note that if a YouTrack token is not provided in the config the app will look for an environment variable called `YOUTRACK_TOKEN`. This makes local and automated testing of the YouTrack integration possible. diff --git a/config/config.yml b/config/config.yml index eb769fd..47eef76 100644 --- a/config/config.yml +++ b/config/config.yml @@ -4,8 +4,9 @@ servers: url: http://localhost:8080 user: test.user@example.com password: password - orderlyweb: - url: http://localhost:8888 + packit: + url: https://localhost/packit + disable_certificate_verify: true youtrack: token: None smtp: @@ -27,17 +28,22 @@ tasks: - minimal_modeller@example.com - science@example.com subject: "VIMC diagnostic report: {touchstone} - {group} - {disease}" - timeout: 300 assignee: a.hill + publish_roles: + - minimal.modeller + - Funders - report_name: diagnostic-param parameters: - nmin: 0 + a: 1 + b: 2 + c: 3 success_email: recipients: - other_modeller@example.com - science@example.com subject: "New version of another Orderly report" - timeout: 1200 assignee: e.russell + publish_roles: + - other.modeller archive_folder_contents: min_file_age_seconds: 0 diff --git a/config/docker_config.yml b/config/docker_config.yml index 0de73b9..7d675f2 100644 --- a/config/docker_config.yml +++ b/config/docker_config.yml @@ -1,15 +1,16 @@ -host: montagu_mq_1 +host: montagu-mq-1 servers: montagu: - url: http://montagu_api_1:8080 + url: http://montagu-api-1:8080 user: test.user@example.com password: password - orderlyweb: - url: http://montagu_orderly_web:8888 + packit: + url: https://reverse-proxy:443/packit + disable_certificate_verify: true youtrack: token: smtp: - host: montagu_smtp_server_1 + host: montagu-smtp_server-1 port: 1025 from: noreply@example.com tasks: @@ -26,14 +27,21 @@ tasks: - science@example.com subject: "VIMC diagnostic report: {touchstone} - {group} - {disease}" assignee: a.hill + publish_roles: + - minimal.modeller + - Funders - report_name: diagnostic-param parameters: - nmin: 0 + a: 1 + b: 2 + c: 3 success_email: recipients: - other_modeller@example.com - science@example.com subject: "New version of another Orderly report" assignee: e.russell + publish_roles: + - other.modeller archive_folder_contents: min_file_age_seconds: 0 diff --git a/docker-compose.yml b/docker-compose.yml index 66087e6..6f688c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: mq: image: redis @@ -9,8 +8,8 @@ services: ports: - "5555:5555" environment: - - CELERY_BROKER_URL=redis://montagu_mq_1// - - CELERY_RESULT_BACKEND=redis://montagu_mq_1/0 + - CELERY_BROKER_URL=redis://montagu-mq-1// + - CELERY_RESULT_BACKEND=redis://montagu-mq-1/0 - FLOWER_PORT=5555 api: image: ${REGISTRY}/montagu-api:master @@ -23,8 +22,16 @@ services: ports: - "5432:5432" command: /etc/montagu/postgresql.test.conf + contrib: + image: ${REGISTRY}/montagu-contrib-portal:master + depends_on: + - api + admin: + image: ${REGISTRY}/montagu-admin-portal:master + depends_on: + - api smtp_server: image: reachfive/fake-smtp-server ports: - "1025:1025" - - "1080:1080" \ No newline at end of file + - "1080:1080" diff --git a/requirements.txt b/requirements.txt index db26768..ccb6a43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ celery[redis] future pyyaml montagu>=0.0.2 -orderlyweb-api>=1.0.0 git+https://github.com/reside-ic/youtrack-rest-python-library constellation +requests-mock diff --git a/scripts/clear-docker.sh b/scripts/clear-docker.sh index a210d3e..811b996 100755 --- a/scripts/clear-docker.sh +++ b/scripts/clear-docker.sh @@ -1,3 +1,5 @@ +set -x + docker stop $(docker ps -aq) docker rm $(docker ps -aq) docker network prune --force diff --git a/scripts/orderly-web.yml b/scripts/orderly-web.yml deleted file mode 100644 index 13a51bf..0000000 --- a/scripts/orderly-web.yml +++ /dev/null @@ -1,57 +0,0 @@ -container_prefix: montagu_orderly - -network: montagu_default - -volumes: - orderly: orderly_volume - proxy_logs: orderly_web_proxy_logs - redis: orderly_web_redis_data -## Redis configuration -redis: - image: - name: redis - tag: "5.0" - volume: orderly_web_redis_data -## Orderly configuration -orderly: - image: - repo: vimc - name: orderly.server - tag: master - worker_name: orderly.server - initial: - source: clone - url: https://github.com/vimc/montagu-task-queue-orderly - -web: - image: - repo: vimc - name: orderly-web - tag: master - migrate: orderlyweb-migrate - admin: orderly-web-user-cli - url: https://localhost - dev_mode: true - port: 8888 - name: OrderlyWeb - email: admin@example.com - auth: - github_org: vimc - github_team: "" - github_oauth: - id: "notarealid" - secret: "notarealsecret" - fine_grained: true - montagu: true - montagu_url: http://montagu_api_1:8080 - montagu_api_url: http://montagu_api_1:8080/v1 - -proxy: - enabled: false - hostname: localhost - port_http: 80 - port_https: 443 - image: - repo: vimc - name: orderly-web-proxy - tag: master diff --git a/scripts/orderlyweb_cli.sh b/scripts/orderlyweb_cli.sh deleted file mode 100755 index 4117c92..0000000 --- a/scripts/orderlyweb_cli.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -image=${REGISTRY}/orderly-web-user-cli:master -docker pull $image -docker run --rm -v orderly_volume:/orderly --network ${NETWORK} $image $@ \ No newline at end of file diff --git a/scripts/packit.yml b/scripts/packit.yml new file mode 100644 index 0000000..0e01a14 --- /dev/null +++ b/scripts/packit.yml @@ -0,0 +1,50 @@ +container_prefix: montagu + +protect_data: false + +repo: ghcr.io/mrc-ide + +network: montagu_default + +volumes: + outpack: montagu_outpack_volume + packit_db: montagu_packit_db + orderly_library: montagu_orderly_library + orderly_logs: montagu_orderly_logs + +outpack: + server: + name: outpack_server + tag: main + migrate: + name: outpack.orderly + tag: main + +packit: + base_url: https://localhost + api: + name: packit-api + tag: main + app: + name: montagu-packit + tag: main + db: + name: packit-db + tag: main + user: packituser + password: changeme + auth: + enabled: true + auth_method: preauth + # We'll get this from the vault on production + jwt: + secret: "0b4g4f8z4mdsrhoxfde2mam8f00vmt0f" + expiry_days: 1 + +orderly-runner: + image: + name: orderly.runner + tag: main + git: + url: https://github.com/reside-ic/orderly2-example.git + workers: 1 diff --git a/scripts/run-dependencies.sh b/scripts/run-dependencies.sh index 2f55700..717eb98 100755 --- a/scripts/run-dependencies.sh +++ b/scripts/run-dependencies.sh @@ -1,32 +1,27 @@ #!/usr/bin/env bash set -ex -here=$(dirname $0) +export REGISTRY=vimc +here=$(dirname $0) ./scripts/clear-docker.sh docker network prune -f -export REGISTRY=vimc export NETWORK=montagu_default # Run the API and database -docker-compose pull -docker-compose --project-name montagu up -d +docker compose pull +docker compose --project-name montagu up -d # Clear redis -docker exec montagu_mq_1 redis-cli FLUSHALL - -# Install orderly-web -pip3 install constellation -pip3 install orderly-web -orderly-web start $here +docker exec montagu-mq-1 redis-cli FLUSHALL # Start the APIs -docker exec montagu_api_1 mkdir -p /etc/montagu/api/ -docker exec montagu_api_1 touch /etc/montagu/api/go_signal +docker exec montagu-api-1 mkdir -p /etc/montagu/api/ +docker exec montagu-api-1 touch /etc/montagu/api/go_signal # Wait for the database -docker exec montagu_db_1 montagu-wait.sh +docker exec montagu-db-1 montagu-wait.sh # migrate the database migrate_image=${REGISTRY}/montagu-migrate:master @@ -40,6 +35,41 @@ $here/montagu_cli.sh add "Test User" test.user \ $here/montagu_cli.sh addRole test.user user -# Add user to orderlyweb -$here/orderlyweb_cli.sh add-users test.user@example.com -$here/orderlyweb_cli.sh grant test.user@example.com */reports.read */reports.run */reports.review +# Run packit +hatch env run pip3 install constellation +hatch env run pip3 install packit-deploy +# For some reason packit is emitting exit code 1 despite apparently succeeding. Allow this for now... +set +e +hatch env run -- packit start --pull $here +echo Packit deployed with exit code $? +set -e + +docker exec montagu-packit-db wait-for-db + +# Run the proxy here, not through docker compose - it needs packit to be running before it will start up +MONTAGU_PROXY_TAG=vimc/montagu-reverse-proxy:master +docker pull $MONTAGU_PROXY_TAG +docker run -d \ + -p "443:443" -p "80:80" \ + --name reverse-proxy \ + --network montagu_default\ + $MONTAGU_PROXY_TAG 443 localhost + +# give packit api some time to migrate the db... +sleep 5 + +# create roles to publish to... +docker exec -i montagu-packit-db psql -U packituser -d packit --single-transaction < 0 else None + + @staticmethod + def serialize(data): + return None if data is None else json.dumps(data) + + def __get(self, relative_url, headers=None): + if headers is None: + headers = self.__default_headers + response = requests.get(self.__url(relative_url), + headers=headers, + verify=self.__verify) + return PackitClient.handle_response(response) + + def __post(self, relative_url, data): + response = requests.post(self.__url(relative_url), + data=PackitClient.serialize(data), + headers=self.__default_headers, + verify=self.__verify) + return PackitClient.handle_response(response) + + def __put(self, relative_url, data): + response = requests.put(self.__url(relative_url), + data=PackitClient.serialize(data), + headers=self.__default_headers, + verify=self.__verify) + return PackitClient.handle_response(response) + + def __authenticate(self): + try: + monty = montagu.MontaguAPI(self.__config.montagu_url, + self.__config.montagu_user, + self.__config.montagu_password) + logging.info(f"MONTAGU TOKEN IS {monty.token}") + packit_login_response = self.__get( + "/auth/login/montagu", + {"Authorization": f"Bearer {monty.token}"}) + logging.info(f"AUTH OK!!") + self.auth_success = True + self.token = packit_login_response["token"] + self.__default_headers = { + "Authorization": f"Bearer {self.token}", + "Content-Type": f"application/json" + } + except Exception as ex: + self.auth_success = False + logging.exception(ex) + + def __execute(self, func, *args): + # retry an operation if it fails auth (probably because of an expired + # packit token) + try: + return func(*args) + except PackitClientException as ex: + if ex.response.status_code == 401: + self.__authenticate() + return func(*args) + else: + raise ex + + def __get_latest_commit_for_branch(self, branch): + branches_response = self.__get("/runner/git/branches") + branches = branches_response["branches"] + branch_details = next( + filter(lambda b: b["name"] == branch, branches), None) + if branch_details is None: + raise Exception(f"Git details not found for branch {branch}") + return branch_details["commitHash"] + + def __wait_for_packet_to_be_imported(self, packet_id): + # packet api imports new packets every 10s - poll it for a generous + # 30s to find a new packet before attempting to publish it + poll_seconds = 2 + seconds_max = 30 + poll_max = seconds_max / poll_seconds + poll_counter = 0 + while poll_counter <= poll_max: + try: + self.__get(f"/packets/{packet_id}") + return + except PackitClientException as ex: + logging.info(f"Waiting for packet {packet_id}...") + poll_counter = poll_counter + 1 + time.sleep(poll_seconds) + raise Exception( + f"Packet {packet_id} was not imported into Packit after " + + f"{seconds_max}s") + + def refresh_git(self): + def do_refresh_git(): + self.__post("/runner/git/fetch", None) + self.__execute(do_refresh_git) + + def run(self, packet_group, parameters): + def do_run(): + branch = "main" + commit = self.__get_latest_commit_for_branch(branch) + data = { + "name": packet_group, + "parameters": parameters, + "branch": branch, + "hash": commit + } + response = self.__post("/runner/run", data) + return response["taskId"] + + return self.__execute(do_run) + + def poll(self, task_id): + def do_poll(): + return self.__get(f"/runner/status/{task_id}") + return self.__execute(do_poll) + + def kill_task(self, task_id): + def do_kill_task(): + return self.__post(f"/runner/cancel/{task_id}", None) + return self.__execute(do_kill_task) + + def publish(self, name, packet_id, roles): + # mimic OW publishing by setting packet-level permission for a new + # report packet permission on a list of configured roles. + # NB: These role can either be user # roles or groups. + # If users, these need to be user names not email addresses. + def do_publish_to_role(role): + data = { + "addPermissions": [{ + "permission": "packet.read", + "packetId": packet_id + }], + "removePermissions": [] + } + self.__put(f"/roles/{role}/permissions", data) + + self.__execute( + lambda: self.__wait_for_packet_to_be_imported(packet_id)) + + logging.info(f"Publishing packet {name}({packet_id})") + success = True + for role in roles: + try: + logging.info(f"...to role {role}") + self.__execute(lambda: do_publish_to_role(role)) + except Exception as ex: + logging.exception(ex) + success = False + return success diff --git a/src/packit_client_exception.py b/src/packit_client_exception.py new file mode 100644 index 0000000..34336d8 --- /dev/null +++ b/src/packit_client_exception.py @@ -0,0 +1,10 @@ +class PackitClientException(Exception): + def __init__(self, response): + self.response = response + json = response.json() + msg = "Unexpected response status from Packit API: " + \ + f"{response.status_code}." + if "error" in json and "detail" in json["error"]: + detail = json["error"]["detail"] + msg = f"{msg} Detail: {detail}" + super().__init__(msg) diff --git a/src/task_run_diagnostic_reports.py b/src/task_run_diagnostic_reports.py index 6fd526e..a8b7346 100644 --- a/src/task_run_diagnostic_reports.py +++ b/src/task_run_diagnostic_reports.py @@ -12,7 +12,7 @@ from src.utils.email import send_email, Emailer from urllib.parse import quote as urlencode import logging -from src.orderlyweb_client_wrapper import OrderlyWebClientWrapper +from src.packit_client import PackitClient from src.utils.running_reports_repository import RunningReportsRepository from YTClient.YTClient import YTClient, YTException @@ -29,7 +29,7 @@ def run_diagnostic_reports(group, config = Config() reports = config.diagnostic_reports(group, disease) if len(reports) > 0: - wrapper = OrderlyWebClientWrapper(config) + packit = PackitClient(config) emailer = Emailer(config.smtp_host, config.smtp_port, config.smtp_user, config.smtp_password) yt_token = config.youtrack_token @@ -40,10 +40,10 @@ def run_diagnostic_reports(group, yt = YTClient('https://mrc-ide.myjetbrains.com/youtrack/', token=yt_token) - def success_callback(report, version): + def success_callback(report, packet_id): send_diagnostic_report_email(emailer, report, - version, + packet_id, group, disease, touchstone, @@ -52,7 +52,7 @@ def success_callback(report, version): config, *additional_recipients) create_ticket(group, disease, touchstone, scenario, - report, version, None, yt, config) + report, packet_id, None, yt, config) def error_callback(report, error): create_ticket(group, disease, touchstone, scenario, @@ -60,7 +60,7 @@ def error_callback(report, error): running_reports_repo = RunningReportsRepository(host=config.host) - return run_reports(wrapper, + return run_reports(packit, group, disease, touchstone, @@ -76,16 +76,16 @@ def error_callback(report, error): def create_ticket(group, disease, touchstone, scenario, - report: ReportConfig, version, + report: ReportConfig, packet_id, error, yt: YTClient, config: Config): try: - report_success = version is not None + report_success = packet_id is not None summary = "Check & share diag report with {} ({}) {}" if \ report_success else \ "Run, check & share diag report with {} ({}) {}" - result = get_version_url(report, version, config) if \ + result = get_packet_url(report, packet_id, config) if \ report_success else \ "Auto-run failed with error: {}".format(error) description = "Report run triggered by upload to scenario: {}. {}"\ @@ -116,21 +116,28 @@ def create_ticket(group, disease, touchstone, scenario, logging.exception(ex) +def create_tag(yt, tag_name): + try: + yt.create_tag(tag_name) + except YTException: + logging.error(f"Failed to create YouTrack tag {tag_name}") + + def create_tags(yt, group, disease, touchstone, report): tags = yt.get_tags(fields=["name"]) if len([t for t in tags if t["name"] == disease]) == 0: - yt.create_tag(disease) + create_tag(yt, disease) if len([t for t in tags if t["name"] == group]) == 0: - yt.create_tag(group) + create_tag(yt, group) if len([t for t in tags if t["name"] == touchstone]) == 0: - yt.create_tag(touchstone) + create_tag(yt, touchstone) if len([t for t in tags if t["name"] == report.name]) == 0: - yt.create_tag(report.name) + create_tag(yt, report.name) def send_diagnostic_report_email(emailer, report, - version, + packet_id, group, disease, touchstone, @@ -139,7 +146,7 @@ def send_diagnostic_report_email(emailer, config, *additional_recipients): template_values = { - "report_version_url": get_version_url(report, version, config), + "report_version_url": get_packet_url(report, packet_id, config), "disease": disease, "group": group, "touchstone": touchstone, @@ -171,7 +178,7 @@ def get_time_strings(utc_time): } -def get_version_url(report, version, config): +def get_packet_url(report, packet_id, config): r_enc = urlencode(report.name) - v_enc = urlencode(version) - return "{}/report/{}/{}/".format(config.orderlyweb_url, r_enc, v_enc) + p_enc = urlencode(packet_id) + return "{}/{}/{}/".format(config.packit_url, r_enc, p_enc) diff --git a/src/utils/run_reports.py b/src/utils/run_reports.py index d94b45b..7c47170 100644 --- a/src/utils/run_reports.py +++ b/src/utils/run_reports.py @@ -1,11 +1,23 @@ import logging import time +TASK_STATUS_PENDING = "PENDING" +TASK_STATUS_RUNNING = "RUNNING" +TASK_STATUS_COMPLETE = "COMPLETE" +TASK_STATUS_CANCELLED = "CANCELLED" -def publish_report(wrapper, name, version): + +def task_is_finished(poll_response): + status = poll_response["status"] + return status not in [ + TASK_STATUS_PENDING, + TASK_STATUS_RUNNING + ] + + +def publish_report(packit, name, packet_id, roles): try: - logging.info("Publishing report version {}-{}".format(name, version)) - return wrapper.execute(wrapper.ow.publish_report, name, version) + return packit.publish(name, packet_id, roles) except Exception as ex: logging.exception(ex) return False @@ -15,27 +27,36 @@ def params_to_string(params): return ", ".join([f"{key}={value}" for key, value in params.items()]) -def run_reports(wrapper, group, disease, touchstone, config, reports, +def run_reports(packit, group, disease, touchstone, config, reports, success_callback, error_callback, running_reports_repo): running_reports = {} - new_versions = {} + new_packets = {} - if wrapper.ow is None: - error = "Orderlyweb authentication failed; could not begin task" + if not packit.auth_success: + error = "Packit authentication failed; could not begin task" for report in reports: error_callback(report, error) logging.error(error) - return new_versions + return new_packets + + try: + packit.refresh_git() + except Exception as ex: + error = "Failed to refresh git; could not begin task" + for report in reports: + error_callback(report, error) + logging.error(error) + return new_packets # Start configured reports for report in reports: - # Kill any currently running report for this group/disease/report + # Kill any currently running task for this group/disease/report already_running = running_reports_repo.get(group, disease, report.name) if already_running is not None: try: - logging.info("Killing already running report: {}. Key is {}" + logging.info("Killing already running task: {}. Key is {}." .format(report.name, already_running)) - wrapper.execute(wrapper.ow.kill_report, already_running) + packit.kill_task(already_running) except Exception as ex: logging.exception(ex) @@ -45,23 +66,22 @@ def run_reports(wrapper, group, disease, touchstone, config, reports, parameters["touchstone_name"] = touchstone.rsplit('-', 1)[0] try: - key = wrapper.execute(wrapper.ow.run_report, - report.name, - parameters, - report.timeout) + key = packit.run( + report.name, + parameters + ) running_reports[key] = report # Save key to shared data - may be killed by subsequent task running_reports_repo.set(group, disease, report.name, key) - logging.info("Running report: {} with parameters {}. Key is {}. " - "Timeout is {}s." + logging.info("Running report: {} with parameters {}. Key is {}." .format(report.name, params_to_string(parameters), - key, report.timeout)) + key)) except Exception as ex: error_callback(report, str(ex)) logging.exception(ex) - # Poll running reports until they complete + # Poll running tasks until they complete report_poll_seconds = config.report_poll_seconds while len(running_reports.items()) > 0: finished = [] @@ -69,36 +89,48 @@ def run_reports(wrapper, group, disease, touchstone, config, reports, for key in keys: report = running_reports[key] try: - result = wrapper.execute(wrapper.ow.report_status, key) - if result.finished: + result = packit.poll(key) + if task_is_finished(result): finished.append(key) - if result.success: - logging.info("Success for key {}. New version is {}" - .format(key, result.version)) + if result["status"] == TASK_STATUS_COMPLETE: + logging.info("Success for key {}. New packet id is {}" + .format(key, result["packetId"])) - version = result.version + packet_id = result["packetId"] name = report.name - published = publish_report(wrapper, name, version) - if published: + + report_config = next( + filter(lambda report: report.name == name, + reports), + None) + if report_config is not None: logging.info( - "Successfully published report version {}-{}" - .format(name, version)) - success_callback(report, version) - else: - error = "Failed to publish report version {}-{}"\ - .format(name, version) - logging.error(error) - error_callback(report, error) - new_versions[version] = { + "Publishing report packet {} ({})" + .format(name, packet_id)) + published = publish_report( + packit, name, packet_id, + report_config.publish_roles) + if published: + logging.info( + "Successfully published report packet" + + f" {name} ({packet_id})" + ) + success_callback(report, packet_id) + else: + error = "Failed to publish report packet" + \ + f" {name} ({packet_id})" + logging.error(error) + error_callback(report, error) + new_packets[packet_id] = { "published": published, "report": name } else: error = "Failure for key {}. Status: {}"\ - .format(key, result.status) + .format(key, result["status"]) logging.error(error) # don't invoke error callback for cancelled runs - if result.status != "interrupted": + if result["status"] != TASK_STATUS_CANCELLED: error_callback(report, error) except Exception as ex: @@ -115,4 +147,4 @@ def run_reports(wrapper, group, disease, touchstone, config, reports, key) time.sleep(report_poll_seconds) - return new_versions + return new_packets diff --git a/test/integration/test_config.py b/test/integration/test_config.py index 6efaeb9..b52c230 100644 --- a/test/integration/test_config.py +++ b/test/integration/test_config.py @@ -19,8 +19,12 @@ def test_montagu_password(): assert config.montagu_password == "password" -def test_orderlyweb_url(): - assert config.orderlyweb_url == "http://localhost:8888" +def test_packit_url(): + assert config.packit_url == "https://localhost/packit" + + +def test_packit_disable_certificate_verify(): + assert config.packit_disable_certificate_verify def test_youtrack_token(): @@ -58,19 +62,18 @@ def test_diagnostic_reports(): assert reports[0].name == "diagnostic" assert len(reports[0].parameters.keys()) == 0 assert reports[0].success_email_recipients == \ - ["minimal_modeller@example.com", "science@example.com"] + ["minimal_modeller@example.com", "science@example.com"] assert reports[0].success_email_subject == \ "VIMC diagnostic report: {touchstone} - {group} - {disease}" - assert reports[0].timeout == 300 + assert reports[0].publish_roles == ["minimal.modeller", "Funders"] assert reports[1].name == "diagnostic-param" - assert len(reports[1].parameters.keys()) == 1 - assert reports[1].parameters["nmin"] == 0 + assert reports[1].parameters == {"a": 1, "b": 2, "c": 3} assert reports[1].success_email_recipients == \ ["other_modeller@example.com", "science@example.com"] assert reports[1].success_email_subject == \ "New version of another Orderly report" - assert reports[1].timeout == 1200 + assert reports[1].publish_roles == ["other.modeller"] def test_diagnostic_reports_nonexistent(): diff --git a/test/integration/test_run_reports.py b/test/integration/test_run_reports.py index 024dda2..0a714a9 100644 --- a/test/integration/test_run_reports.py +++ b/test/integration/test_run_reports.py @@ -1,5 +1,5 @@ from src.config import Config, ReportConfig -from src.orderlyweb_client_wrapper import OrderlyWebClientWrapper +from src.packit_client import PackitClient from src.utils.run_reports import run_reports from src.utils.running_reports_repository import RunningReportsRepository @@ -7,11 +7,11 @@ def test_run_reports_handles_error(): reports = [ ReportConfig("nonexistent", None, ["test1@test.com"], "subject1", - 600, "a.ssignee"), - ReportConfig("diagnostic", {}, ["test2@test.com"], "subject2", 600, - "a.ssignee")] + "a.ssignee", ["Funders"]), + ReportConfig("diagnostic", {}, ["test2@test.com"], "subject2", + "a.ssignee", ["Funders"])] config = Config() - wrapper = OrderlyWebClientWrapper(config) + packit = PackitClient(config) success = {} error = {} @@ -23,7 +23,7 @@ def error_callback(report, message): running_reports_repository = RunningReportsRepository() - versions = run_reports(wrapper, "testGroup", "testDisease", + versions = run_reports(packit, "testGroup", "testDisease", "testTouchstone", config, reports, success_callback, error_callback, running_reports_repository) diff --git a/test/integration/test_task_run_diagnostic_reports.py b/test/integration/test_task_run_diagnostic_reports.py index f033759..3509425 100644 --- a/test/integration/test_task_run_diagnostic_reports.py +++ b/test/integration/test_task_run_diagnostic_reports.py @@ -111,7 +111,7 @@ def test_run_diagnostic_reports(): report_2 = "diagnostic" email_props = [diagnostic_param_email_props, diagnostic_email_props] - url_template = "http://localhost:8888/report/{}/{}/" + url_template = "https://localhost/packit/{}/{}/" url_1 = url_template.format(report_1, versions[0]) url_2 = url_template.format(report_2, versions[1]) @@ -178,7 +178,7 @@ def test_ticket_created_on_success(): "Check & share diag report with testGroup (testDisease) {}" \ .format(yt.test_touchstone) expected_desc1 = "Report run triggered by upload to scenario: s1. " \ - "http://localhost:8888/report/{}/{}/".format(r1, v1) + "https://localhost/packit/{}/{}/".format(r1, v1) assert i1["summary"] == expected_summary assert i1["description"] == expected_desc1 assignee = get_field(i1, "Assignee") @@ -194,7 +194,7 @@ def test_ticket_created_on_success(): assert yt.test_touchstone in tags expected_desc2 = "Report run triggered by upload to scenario: s1. " \ - "http://localhost:8888/report/{}/{}/".format(r2, v2) + "https://localhost/packit/{}/{}/".format(r2, v2) assert i2["summary"] == expected_summary assert i2["description"] == expected_desc2 assignee = get_field(i2, "Assignee") @@ -210,9 +210,9 @@ def test_ticket_created_on_success(): assert yt.test_touchstone in tags -@mock.patch('src.config.Config.orderlyweb_url', new_callable=PropertyMock) -def test_ticket_created_on_error(mock_orderlyweb_url): - mock_orderlyweb_url.return_value = "http://bad-url" +@mock.patch('src.config.Config.packit_url', new_callable=PropertyMock) +def test_ticket_created_on_error(mock_packit_url): + mock_packit_url.return_value = "http://bad-url" result = run_diagnostic_reports("testGroup", "testDisease", yt.test_touchstone, @@ -235,7 +235,7 @@ def test_ticket_created_on_error(mock_orderlyweb_url): expected_err = "Report run triggered by upload to scenario: s1. " \ "Auto-run failed with error: " + \ - "Orderlyweb authentication failed; could not begin task" + "Packit authentication failed; could not begin task" i1 = issues[0] i2 = issues[1] @@ -288,12 +288,12 @@ def test_ticket_update_on_success(): "Check & share diag report with testGroup (testDisease) {}" \ .format(yt.test_touchstone) expected_desc1 = "Report run triggered by upload to scenario: s1. " \ - "http://localhost:8888/report/{}/{}/".format(r1, v1) + "https://localhost/packit/{}/{}/".format(r1, v1) assert i1["summary"] == expected_summary assert i1["description"] == expected_desc1 expected_desc2 = "Report run triggered by upload to scenario: s1. " \ - "http://localhost:8888/report/{}/{}/".format(r2, v2) + "https://localhost/packit/{}/{}/".format(r2, v2) assert i2["summary"] == expected_summary assert i2["description"] == expected_desc2 @@ -325,11 +325,11 @@ def test_ticket_update_on_success(): "Check & share diag report with testGroup (testDisease) {}" \ .format(yt.test_touchstone) expected_desc1 = "Report run triggered by upload to scenario: s1. " \ - "http://localhost:8888/report/{}/{}/".format(r1, v1) + "https://localhost/packit/{}/{}/".format(r1, v1) assert i1["summary"] == expected_summary assert i1["description"] == expected_desc1 expected_desc2 = "Report run triggered by upload to scenario: s1. " \ - "http://localhost:8888/report/{}/{}/".format(r2, v2) + "https://localhost/packit/{}/{}/".format(r2, v2) assert i2["summary"] == expected_summary assert i2["description"] == expected_desc2 diff --git a/test/integration/test_worker.py b/test/integration/test_worker.py index d382d80..4811f11 100644 --- a/test/integration/test_worker.py +++ b/test/integration/test_worker.py @@ -6,7 +6,7 @@ from src.config import Config from src.utils.running_reports_repository import RunningReportsRepository -from src.orderlyweb_client_wrapper import OrderlyWebClientWrapper +from src.packit_client import PackitClient from test.integration.yt_utils import YouTrackUtils from test.integration.file_utils import write_text_file @@ -68,12 +68,11 @@ def test_later_task_kills_earlier_task_report(): assert len(versions) == 2 - # Check first report key's status with OrderlyWeb - should have been killed + # Check first report key's status with Packit - should have been killed config = Config() - wrapper = OrderlyWebClientWrapper(config) - result = wrapper.execute(wrapper.ow.report_status, first_report_key) - assert result.status == "interrupted" - assert result.finished + packit = PackitClient(config) + result = packit.poll(first_report_key) + assert result["status"] == "CANCELLED" # Check redis key has been tidied up assert running_repo.get("testGroup", "testDisease", "diagnostic") is None diff --git a/test/unit/test_create_ticket.py b/test/unit/test_create_ticket.py index 7485bba..509d6cf 100644 --- a/test/unit/test_create_ticket.py +++ b/test/unit/test_create_ticket.py @@ -14,7 +14,7 @@ def test_tags_created(): report = ReportConfig("TEST", {}, ["to@example.com"], - "Hi", 100, "a.ssignee") + "Hi", "a.ssignee", ["Funders"]) mock_config: Config = MockConfig() mock_client = Mock(spec=YTClient("", "")) mock_client.create_issue = Mock(return_value="ISSUE") @@ -28,7 +28,7 @@ def test_tags_created(): def test_tags_not_created_if_exists(): report = ReportConfig("TEST", {}, ["to@example.com"], - "Hi", 100, "a.ssignee") + "Hi", "a.ssignee", ["Funders"]) mock_config: Config = MockConfig() mock_client = Mock(spec=YTClient("", "")) mock_client.create_issue = Mock(return_value="ISSUE") @@ -41,7 +41,7 @@ def test_tags_not_created_if_exists(): def test_create_ticket_with_version(): report = ReportConfig("TEST", {}, ["to@example.com"], - "Hi", 100, "a.ssignee") + "Hi", "a.ssignee", ["Funders"]) mock_config: Config = MockConfig() mock_client = Mock(spec=YTClient("", "")) mock_client.create_issue = Mock(return_value="ISSUE") @@ -52,7 +52,7 @@ def test_create_ticket_with_version(): expected_create = call(Project(id="78-0"), "Check & share diag report with g1 (d1) t1", "Report run triggered by upload to scenario: s1. " - "http://orderly-web/report/TEST/1234/") + "http://test-packit/TEST/1234/") mock_client.create_issue.assert_has_calls([expected_create]) expected_command_query = \ "for a.ssignee implementer a.ssignee tag g1 tag d1 tag t1 tag TEST" @@ -63,7 +63,7 @@ def test_create_ticket_with_version(): def test_create_ticket_without_version(): report = ReportConfig("TEST", {}, ["to@example.com"], - "Hi", 100, "a.ssignee") + "Hi", "a.ssignee", ["Funders"]) mock_config: Config = MockConfig() mock_client = Mock(spec=YTClient("", "")) mock_client.create_issue = Mock(return_value="ISSUE") @@ -87,7 +87,7 @@ def test_create_ticket_without_version(): @patch("src.task_run_diagnostic_reports.logging") def test_create_ticket_logs_errors(logging): report = ReportConfig("TEST", {}, ["to@example.com"], - "Hi", 100, "a.ssignee") + "Hi", "a.ssignee", ["Funders"]) mock_config: Config = MockConfig() mock_client = Mock(spec=YTClient("", "")) mock_client.get_issues = Mock(return_value=[]) @@ -101,7 +101,7 @@ def test_create_ticket_logs_errors(logging): def test_update_ticket(): report = ReportConfig("TEST", {}, ["to@example.com"], - "Hi", 100, "a.ssignee") + "Hi", "a.ssignee", ["Funders"]) mock_config: Config = MockConfig() mock_client = Mock(spec=YTClient("", "")) mock_client.get_issues = Mock(return_value=["ISSUE"]) @@ -111,5 +111,5 @@ def test_update_ticket(): expected_command = call("ISSUE", "Check & share diag report with g1 (d1) t1", "Report run triggered by upload to scenario: s1. " - "http://orderly-web/report/TEST/1234/") + "http://test-packit/TEST/1234/") mock_client.update_issue.assert_has_calls([expected_command]) diff --git a/test/unit/test_orderlyweb_client_wrapper.py b/test/unit/test_orderlyweb_client_wrapper.py deleted file mode 100644 index f20f2ac..0000000 --- a/test/unit/test_orderlyweb_client_wrapper.py +++ /dev/null @@ -1,121 +0,0 @@ -from src.utils.run_reports import run_reports -from src.config import ReportConfig -from orderlyweb_api import ReportStatusResult -from unittest.mock import patch, call -from src.orderlyweb_client_wrapper import OrderlyWebClientWrapper -from test import ExceptionMatching -from test.unit.test_run_reports import MockConfig, MockRunningReportRepository - -reports = [ReportConfig("r1", None, ["r1@example.com"], - "Subj: r1", 1000, "a.ssignee")] - -report_response = ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None}) - -group = "test_group" -disease = "test_disease" -touchstone = "2021test-1" - - -class MockOrderlyWebAPIWithExpiredToken: - - def run_report(self, name, params, timeout): - raise Exception("Token expired") - - def report_status(self, key): - raise Exception("Token expired") - - def publish_report(self, name, version): - raise Exception("Token expired") - - -class MockOrderlyWebAPIWithValidToken: - - def run_report(self, name, params, timeout): - return "r1-key" - - def report_status(self, key): - return report_response - - def publish_report(self, name, version): - return True - - -class MockReturnAuthorisedClient: - def __init__(self): - self.callCount = 0 - - def auth(self, config): - if self.callCount == 0: - self.callCount = 1 - # if this is the first call, return an expired token error - return MockOrderlyWebAPIWithExpiredToken() - else: - # on the second call, return success reponses - return MockOrderlyWebAPIWithValidToken() - - -@patch("src.utils.run_reports.logging") -def test_retries_when_token_expired(logging): - auth = MockReturnAuthorisedClient().auth - wrapper = OrderlyWebClientWrapper(None, auth) - success = {} - error = {} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, version=None): - error["called"] = True - - mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), - reports, success_callback, error_callback, - mock_running_reports) - - assert versions == {"r1-version": {"published": True, "report": "r1"}} - logging.info.assert_has_calls([ - call("Running report: r1 with parameters touchstone=2021test-1," - " touchstone_name=2021test. " - "Key is r1-key. Timeout is 1000s."), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version") - ], any_order=False) - - assert success["called"] is True - assert len(error) == 0 - - -@patch("src.utils.run_reports.logging") -@patch("src.orderlyweb_client_wrapper.logging") -def test_handles_auth_errors(logging_ow, logging_reports): - wrapper = OrderlyWebClientWrapper({}) - success = {} - error = {} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, version=None): - error["called"] = True - - mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), - reports, success_callback, error_callback, - mock_running_reports) - - # the wrapper will have an auth failure because no auth config - # supplied - expected_error = AttributeError( - "'dict' object has no attribute 'montagu_url'") - logging_ow.exception.assert_called_once_with( - ExceptionMatching(expected_error)) - - logging_reports.error.assert_called_once_with( - "Orderlyweb authentication failed; could not begin task") - - assert len(success) == 0 - assert len(versions) == 0 - assert error["called"] is True diff --git a/test/unit/test_packit_client.py b/test/unit/test_packit_client.py new file mode 100644 index 0000000..fddeccd --- /dev/null +++ b/test/unit/test_packit_client.py @@ -0,0 +1,221 @@ +import json +import montagu +import pytest +import requests_mock +from src.packit_client import PackitClient +from src.packit_client_exception import PackitClientException +from test.unit.test_run_reports import MockConfig, PACKIT_URL +from unittest.mock import patch, MagicMock + +config = MockConfig() + + +def mock_auth(mock_montagu_api_class, requests_mock): + mock_montagu_api = mock_montagu_api_class.return_value + mock_montagu_api.token = "test-montagu-token" + requests_mock.get(f"{PACKIT_URL}/api/auth/login/montagu", + text=json.dumps({"token": "test-packit-token"})) + + +def assert_expected_packit_api_request( + requests_mock, + index, + method, + url, + text=None): + req = requests_mock.request_history[index] + assert req.headers["Authorization"] == "Bearer test-packit-token" + assert req.method == method + assert req.url == url + assert req.text == text + + +def assert_expected_packit_auth_request(requests_mock, index): + auth_req = requests_mock.request_history[index] + assert auth_req.method == "GET" + assert auth_req.url == f"{PACKIT_URL}/api/auth/login/montagu" + assert auth_req.headers["Authorization"] == "Bearer test-montagu-token" + + +@patch("montagu.MontaguAPI") +def test_authenticates_on_init(MockMontaguAPI, requests_mock): + mock_auth(MockMontaguAPI, requests_mock) + + sut = PackitClient(config) + + MockMontaguAPI.assert_called_with( + "http://test-montagu", "test.montagu.user", "montagu_password") + auth_call = requests_mock.request_history[0] + assert_expected_packit_auth_request(requests_mock, 0) + assert sut.auth_success + assert sut.token == "test-packit-token" + + +@patch("montagu.MontaguAPI") +def test_refresh_git(MockMontaguAPI, requests_mock): + mock_auth(MockMontaguAPI, requests_mock) + requests_mock.post(f"{PACKIT_URL}/api/runner/git/fetch", text=None) + + sut = PackitClient(config) + sut.refresh_git() + + assert_expected_packit_api_request( + requests_mock, 1, "POST", f"{PACKIT_URL}/api/runner/git/fetch") + + +@patch("montagu.MontaguAPI") +def test_run(MockMontaguAPI, requests_mock): + mock_auth(MockMontaguAPI, requests_mock) + mock_branches_response = {"branches": [ + {"name": "some_other_branch", "commitHash": "xyz987"}, + {"name": "main", "commitHash": "abc123"} + ]} + requests_mock.get(f"{PACKIT_URL}/api/runner/git/branches", + text=json.dumps(mock_branches_response)) + requests_mock.post(f"{PACKIT_URL}/api/runner/run", + text=json.dumps({"taskId": "test-task-id"})) + + sut = PackitClient(config) + task_id = sut.run("test-packet-group", {"a": 1, "b": 2}) + + assert task_id == "test-task-id" + assert_expected_packit_api_request( + requests_mock, 1, "GET", f"{PACKIT_URL}/api/runner/git/branches") + expected_run_payload = json.dumps({ + "name": "test-packet-group", + "parameters": {"a": 1, "b": 2}, + "branch": "main", + "hash": "abc123" + }) + assert_expected_packit_api_request( + requests_mock, 2, "POST", + f"{PACKIT_URL}/api/runner/run", expected_run_payload) + + +@patch("montagu.MontaguAPI") +def test_poll_status(MockMontaguAPI, requests_mock): + mock_auth(MockMontaguAPI, requests_mock) + mock_poll_response = {"status": "RUNNING"} + requests_mock.get(f"{PACKIT_URL}/api/runner/status/test-task-id", + text=json.dumps(mock_poll_response)) + + sut = PackitClient(config) + resp = sut.poll("test-task-id") + + assert resp == mock_poll_response + + +@patch("montagu.MontaguAPI") +def test_kill_task(MockMontaguAPI, requests_mock): + mock_auth(MockMontaguAPI, requests_mock) + mock_kill_response = {"status": "dead"} + requests_mock.post(f"{PACKIT_URL}/api/runner/cancel/test-task-id", + text=json.dumps(mock_kill_response)) + + sut = PackitClient(config) + resp = sut.kill_task("test-task-id") + + assert resp == mock_kill_response + assert_expected_packit_api_request( + requests_mock, 1, "POST", + f"{PACKIT_URL}/api/runner/cancel/test-task-id") + + +@patch("montagu.MontaguAPI") +def test_publish(MockMontaguAPI, requests_mock): + mock_auth(MockMontaguAPI, requests_mock) + # publish polls for the packit - return 404 on first poll, + # 200 on second poll + requests_mock.get(f"{PACKIT_URL}/api/packets/test-packet-id", [ + {"text": json.dumps({"error": {"detail": "not found"}}), + "status_code": 404}, + {"text": json.dumps( + {"id": "test-packet-id", "name": "A packet"}), "status_code": 200} + ]) + expected_permissions_payload = json.dumps({ + "addPermissions": [ + {"permission": "packet.read", "packetId": "test-packet-id"} + ], + "removePermissions": [] + }) + requests_mock.put( + f"{PACKIT_URL}/api/roles/test-role-1/permissions", json="null") + requests_mock.put( + f"{PACKIT_URL}/api/roles/test-role-2/permissions", text="null") + + sut = PackitClient(config) + result = sut.publish("A packet", "test-packet-id", + ["test-role-1", "test-role-2"]) + assert_expected_packit_api_request( + requests_mock, 1, "GET", + f"{PACKIT_URL}/api/packets/test-packet-id", + None) + assert_expected_packit_api_request( + requests_mock, 2, "GET", + f"{PACKIT_URL}/api/packets/test-packet-id", + None) + + assert_expected_packit_api_request( + requests_mock, 3, "PUT", + f"{PACKIT_URL}/api/roles/test-role-1/permissions", + expected_permissions_payload) + assert_expected_packit_api_request( + requests_mock, 4, "PUT", + f"{PACKIT_URL}/api/roles/test-role-2/permissions", + expected_permissions_payload) + assert result + + +@patch("montagu.MontaguAPI") +def test_sets_auth_success_to_false_when_auth_fails( + MockMontaguAPI, + requests_mock): + requests_mock.get(f"{PACKIT_URL}/api/auth/login/montagu", + status_code=401, + text=json.dumps({"error": "Unauthorized"})) + sut = PackitClient(config) + assert not sut.auth_success + + +@patch("montagu.MontaguAPI") +def test_reauthenticates_on_401(MockMontaguAPI, requests_mock): + # Reauthentication should take place as part of the __execute wrapper used + # with all methods which require authentication - here we just test a + # sample method to check the pattern works. + mock_auth(MockMontaguAPI, requests_mock) + mock_successful_kill_response = {"status": "dead"} + requests_mock.post(f"{PACKIT_URL}/api/runner/cancel/test-task-id", [ + {"status_code": 401, "text": json.dumps({"error": "Unauthorized"})}, + {"status_code": 200, "text": json.dumps(mock_successful_kill_response)} + ]) + + sut = PackitClient(config) + resp = sut.kill_task("test-task-id") + + assert resp == mock_successful_kill_response + + assert_expected_packit_api_request( + requests_mock, 1, "POST", + f"{PACKIT_URL}/api/runner/cancel/test-task-id") + assert_expected_packit_auth_request(requests_mock, 2) + assert_expected_packit_api_request( + requests_mock, 3, "POST", + f"{PACKIT_URL}/api/runner/cancel/test-task-id") + + +@patch("montagu.MontaguAPI") +def test_raises_exception_on_unexpected_status( + MockMontaguAPI, + requests_mock): + mock_auth(MockMontaguAPI, requests_mock) + # execute does not tolerate status codes other than 401 + # - should get an exception + bad_response = {"error": "Bad request"} + requests_mock.get(f"{PACKIT_URL}/api/runner/status/test-task-id", + status_code=400, text=json.dumps(bad_response)) + + sut = PackitClient(config) + with pytest.raises(PackitClientException) as exc_info: + sut.poll("test-task-id") + assert exc_info.value.response.status_code == 400 + assert exc_info.value.response.json() == bad_response diff --git a/test/unit/test_run_reports.py b/test/unit/test_run_reports.py index 747563b..8a07cda 100644 --- a/test/unit/test_run_reports.py +++ b/test/unit/test_run_reports.py @@ -1,13 +1,11 @@ from src.utils.run_reports import run_reports from src.config import ReportConfig -from orderlyweb_api import ReportStatusResult from unittest.mock import patch, call, Mock -from src.orderlyweb_client_wrapper import OrderlyWebClientWrapper reports = [ReportConfig("r1", None, ["r1@example.com"], "Subj: r1", - 1000, "a.ssignee"), + "a.ssignee", ["Funders"]), ReportConfig("r2", {"p1": "v1"}, ["r2@example.com"], "Subj: r2", - 2000, "a.ssignee")] + "a.ssignee", ["Tech"])] expected_params = { "r1": {"touchstone": "2021test-1", "touchstone_name": "2021test"}, @@ -15,54 +13,42 @@ "touchstone_name": "2021test"} } -expected_timeouts = { - "r1": 1000, - "r2": 2000 -} - group = "test_group" disease = "test_disease" touchstone = "2021test-1" expected_run_rpt_1_log = "Running report: r1 with parameters " \ "touchstone=2021test-1, touchstone_name=2021test. " \ - "Key is r1-key. Timeout is 1000s." + "Key is r1-key." expected_run_rpt_2_log = "Running report: r2 with parameters p1=v1, " \ "touchstone=2021test-1, touchstone_name=2021test. " \ - "Key is r2-key. Timeout is 2000s." + "Key is r2-key." @patch("src.utils.run_reports.logging") def test_run_reports(logging): run_successfully = ["r1", "r2"] report_responses = { - "r1-key": [ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None})], - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] + "r1-key": [{"status": "COMPLETE", + "packetId": "r1-version"}], + "r2-key": [{"status": "COMPLETE", + "packetId": "r2-version"}] } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient( + expected_params, run_successfully, report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) + packit.refresh_git.assert_called_with() + assert versions == { "r1-version": {"published": True, "report": "r1"}, "r2-version": {"published": True, "report": "r2"} @@ -71,30 +57,31 @@ def error_callback(report, message): logging.info.assert_has_calls([ call(expected_run_rpt_1_log), call(expected_run_rpt_2_log), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version"), - call("Success for key r2-key. New version is r2-version"), - call("Publishing report version r2-r2-version"), - call("Successfully published report version r2-r2-version") + call("Success for key r1-key. New packet id is r1-version"), + call("Publishing report packet r1 (r1-version)"), + call("Successfully published report packet r1 (r1-version)"), + call("Success for key r2-key. New packet id is r2-version"), + call("Publishing report packet r2 (r2-version)"), + call("Successfully published report packet r2 (r2-version)") ], any_order=False) mock_running_reports.assert_expected_calls() - ow.kill_report.assert_not_called() + packit.kill_task.assert_not_called() - assert success["called"] is True - assert error["called"] is False + success_callback.assert_has_calls([ + call(reports[0], "r1-version"), + call(reports[1], "r2-version") + ]) + error_callback.assert_not_called() def test_run_reports_with_multi_hyphen_touchstone(): run_successfully = ["r1", "r2"] report_responses = { - "r1-key": [ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None})], - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] + "r1-key": [{"status": "COMPLETE", + "packetId": "r1-version"}], + "r2-key": [{"status": "COMPLETE", + "packetId": "r2-version"}] } multi_touchstone = "2021test-extra-1" @@ -104,22 +91,14 @@ def test_run_reports_with_multi_hyphen_touchstone(): "r2": {"p1": "v1", "touchstone": "2021test-extra-1", "touchstone_name": "2021test-extra"} } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_multi_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient(expected_multi_params, + run_successfully, report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, multi_touchstone, + versions = run_reports(packit, group, disease, multi_touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -128,38 +107,32 @@ def error_callback(report, message): "r1-version": {"published": True, "report": "r1"}, "r2-version": {"published": True, "report": "r2"} } - assert success["called"] is True - assert error["called"] is False + success_callback.assert_has_calls([ + call(reports[0], "r1-version"), + call(reports[1], "r2-version") + ]) + error_callback.assert_not_called() @patch("src.utils.run_reports.logging") def test_run_reports_kills_currently_running(logging): run_successfully = ["r1", "r2"] report_responses = { - "r1-key": [ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None})], - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] + "r1-key": [{"status": "COMPLETE", + "packetId": "r1-version"}], + "r2-key": [{"status": "COMPLETE", + "packetId": "r2-version"}] } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient( + expected_params, run_successfully, report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = \ MockRunningReportRepository(["r1-old-key", "r2-old-key"]) - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -169,115 +142,50 @@ def error_callback(report, message): } logging.info.assert_has_calls([ - call("Killing already running report: r1. Key is r1-old-key"), + call("Killing already running task: r1. Key is r1-old-key."), call(expected_run_rpt_1_log), - call("Killing already running report: r2. Key is r2-old-key"), + call("Killing already running task: r2. Key is r2-old-key."), call(expected_run_rpt_2_log), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version"), - call("Success for key r2-key. New version is r2-version"), - call("Publishing report version r2-r2-version"), - call("Successfully published report version r2-r2-version") + call("Success for key r1-key. New packet id is r1-version"), + call("Publishing report packet r1 (r1-version)"), + call("Successfully published report packet r1 (r1-version)"), + call("Success for key r2-key. New packet id is r2-version"), + call("Publishing report packet r2 (r2-version)"), + call("Successfully published report packet r2 (r2-version)") ], any_order=False) mock_running_reports.assert_expected_calls() - ow.kill_report.assert_has_calls([ + packit.kill_task.assert_has_calls([ call("r1-old-key"), call("r2-old-key") ], any_order=False) - assert success["called"] is True - assert error["called"] is False - - -@patch("src.utils.run_reports.logging") -def test_run_reports_with_additional_recipients(logging): - run_successfully = ["r1", "r2"] - report_responses = { - "r1-key": [ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None})], - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] - } - - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, message): - error["called"] = message - - mock_running_reports = MockRunningReportRepository() - - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), - reports, success_callback, error_callback, - mock_running_reports) - - assert versions == { - "r1-version": {"published": True, "report": "r1"}, - "r2-version": {"published": True, "report": "r2"} - } - - logging.info.assert_has_calls([ - call(expected_run_rpt_1_log), - call(expected_run_rpt_2_log), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version"), - call("Success for key r2-key. New version is r2-version"), - call("Publishing report version r2-r2-version"), - call("Successfully published report version r2-r2-version") - ], any_order=False) - - mock_running_reports.assert_expected_calls() - ow.kill_report.assert_not_called() - - assert success["called"] is True - assert error["called"] is False + success_callback.assert_has_calls([ + call(reports[0], "r1-version"), + call(reports[1], "r2-version") + ]) + error_callback.assert_not_called() @patch("src.utils.run_reports.logging") def test_run_reports_finish_on_different_poll_cycles(logging): run_successfully = ["r1", "r2"] report_responses = { - "r1-key": [ReportStatusResult({"status": "running", - "version": None, - "output": None}), - ReportStatusResult({"status": "running", - "version": None, - "output": None}), - ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None}) + "r1-key": [{"status": "RUNNING", "packetId": None}, + {"status": "RUNNING", "packetId": None}, + {"status": "COMPLETE", "packetId": "r1-version"} ], - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] + "r2-key": [{"status": "COMPLETE", "packetId": "r2-version"}] } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient(expected_params, run_successfully, + report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -289,44 +197,39 @@ def error_callback(report, message): logging.info.assert_has_calls([ call(expected_run_rpt_1_log), call(expected_run_rpt_2_log), - call("Success for key r2-key. New version is r2-version"), - call("Publishing report version r2-r2-version"), - call("Successfully published report version r2-r2-version"), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version") + call("Success for key r2-key. New packet id is r2-version"), + call("Publishing report packet r2 (r2-version)"), + call("Successfully published report packet r2 (r2-version)"), + call("Success for key r1-key. New packet id is r1-version"), + call("Publishing report packet r1 (r1-version)"), + call("Successfully published report packet r1 (r1-version)") ], any_order=False) mock_running_reports.assert_expected_calls() - ow.kill_report.assert_not_called() + packit.kill_task.assert_not_called() - assert success["called"] is True - assert error["called"] is False + success_callback.assert_has_calls([ + call(reports[1], "r2-version"), + call(reports[0], "r1-version") + ]) + error_callback.assert_not_called() @patch("src.utils.run_reports.logging") def test_run_reports_with_run_error(logging): run_successfully = ["r2"] report_responses = { - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] + "r2-key": [{"status": "COMPLETE", + "packetId": "r2-version"}] } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = True - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient(expected_params, run_successfully, + report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -337,9 +240,9 @@ def error_callback(report, message): expected_err = "test-run-error: r1" logging.info.assert_has_calls([ call(expected_run_rpt_2_log), - call("Success for key r2-key. New version is r2-version"), - call("Publishing report version r2-r2-version"), - call("Successfully published report version r2-r2-version") + call("Success for key r2-key. New packet id is r2-version"), + call("Publishing report packet r2 (r2-version)"), + call("Successfully published report packet r2 (r2-version)") ], any_order=False) args, kwargs = logging.exception.call_args assert str(args[0]) == expected_err @@ -358,36 +261,28 @@ def error_callback(report, message): call(group, disease, "r2", "r2-key") ], any_order=False) - ow.kill_report.assert_not_called() + packit.kill_task.assert_not_called() - assert success["called"] is True - assert error["called"] == expected_err + success_callback.assert_called_with(reports[1], "r2-version") + error_callback.assert_called_with(reports[0], expected_err) @patch("src.utils.run_reports.logging") def test_run_reports_with_status_error(logging): run_successfully = ["r1", "r2"] report_responses = { - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] + "r2-key": [{"status": "COMPLETE", + "packetId": "r2-version"}] } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = version - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient(expected_params, run_successfully, + report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -398,47 +293,38 @@ def error_callback(report, message): logging.info.assert_has_calls([ call(expected_run_rpt_1_log), call(expected_run_rpt_2_log), - call("Success for key r2-key. New version is r2-version"), - call("Publishing report version r2-r2-version"), - call("Successfully published report version r2-r2-version") + call("Success for key r2-key. New packet id is r2-version"), + call("Publishing report packet r2 (r2-version)"), + call("Successfully published report packet r2 (r2-version)") ], any_order=False) args, kwargs = logging.exception.call_args assert str(args[0]) == "test-status-error: r1-key" mock_running_reports.assert_expected_calls() - ow.kill_report.assert_not_called() + packit.kill_task.assert_not_called() expected_err = "test-status-error: r1-key" - assert success["called"] == "r2-version" - assert error["called"] == expected_err + success_callback.assert_called_with(reports[1], "r2-version") + error_callback.assert_called_with(reports[0], expected_err) @patch("src.utils.run_reports.logging") def test_run_reports_with_status_failure(logging): run_successfully = ["r1", "r2"] report_responses = { - "r1-key": [ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None})], - "r2-key": [ReportStatusResult({"status": "error", - "version": None, - "output": None})] + "r1-key": [{"status": "COMPLETE", + "packetId": "r1-version"}], + "r2-key": [{"status": "ERROR", + "packetId": None}] } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = version - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient(expected_params, run_successfully, + report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -449,47 +335,39 @@ def error_callback(report, message): logging.info.assert_has_calls([ call(expected_run_rpt_1_log), call(expected_run_rpt_2_log), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version") + call("Success for key r1-key. New packet id is r1-version"), + call("Publishing report packet r1 (r1-version)"), + call("Successfully published report packet r1 (r1-version)") ], any_order=False) logging.error.assert_has_calls([ - call("Failure for key r2-key. Status: error") + call("Failure for key r2-key. Status: ERROR") ], any_order=False) mock_running_reports.assert_expected_calls() - ow.kill_report.assert_not_called() - assert success["called"] == "r1-version" - assert error["called"] == "Failure for key r2-key. Status: error" + packit.kill_task.assert_not_called() + success_callback.assert_called_with(reports[0], "r1-version") + error_callback.assert_called_with(reports[1], + "Failure for key r2-key. Status: ERROR") @patch("src.utils.run_reports.logging") def test_run_reports_with_run_cancelled(logging): run_successfully = ["r1", "r2"] report_responses = { - "r1-key": [ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None})], - "r2-key": [ReportStatusResult({"status": "interrupted", - "version": None, - "output": None})] + "r1-key": [{"status": "COMPLETE", + "packetId": "r1-version"}], + "r2-key": [{"status": "CANCELLED", + "packetId": None}] } - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {"called": False} - - def success_callback(report, version): - success["called"] = version - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient(expected_params, run_successfully, + report_responses) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -500,48 +378,39 @@ def error_callback(report, message): logging.info.assert_has_calls([ call(expected_run_rpt_1_log), call(expected_run_rpt_2_log), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version") + call("Success for key r1-key. New packet id is r1-version"), + call("Publishing report packet r1 (r1-version)"), + call("Successfully published report packet r1 (r1-version)") ], any_order=False) logging.error.assert_has_calls([ - call("Failure for key r2-key. Status: interrupted") + call("Failure for key r2-key. Status: CANCELLED") ], any_order=False) mock_running_reports.assert_expected_calls() - ow.kill_report.assert_not_called() - assert success["called"] == "r1-version" + packit.kill_task.assert_not_called() + success_callback.assert_called_with(reports[0], "r1-version") # expect error callback not called for cancelled runs - assert error["called"] is False + error_callback.assert_not_called() @patch("src.utils.run_reports.logging") def test_run_reports_with_publish_failure(logging): run_successfully = ["r1", "r2"] report_responses = { - "r1-key": [ReportStatusResult({"status": "success", - "version": "r1-version", - "output": None})], - "r2-key": [ReportStatusResult({"status": "success", - "version": "r2-version", - "output": None})] + "r1-key": [{"status": "COMPLETE", + "packetId": "r1-version"}], + "r2-key": [{"status": "COMPLETE", + "packetId": "r2-version"}] } fail_publish = ["r2"] - ow = MockOrderlyWebAPI(run_successfully, report_responses, - expected_params, expected_timeouts, fail_publish) - wrapper = OrderlyWebClientWrapper(None, lambda x: ow) - success = {} - error = {} - - def success_callback(report, version): - success["called"] = version - - def error_callback(report, message): - error["called"] = message + packit = MockPackitClient(expected_params, run_successfully, + report_responses, fail_publish) + success_callback = Mock() + error_callback = Mock() mock_running_reports = MockRunningReportRepository() - versions = run_reports(wrapper, group, disease, touchstone, MockConfig(), + versions = run_reports(packit, group, disease, touchstone, MockConfig(), reports, success_callback, error_callback, mock_running_reports) @@ -550,24 +419,24 @@ def error_callback(report, message): "r2-version": {"published": False, "report": "r2"} } - expected_err = "Failed to publish report version r2-r2-version" + expected_err = "Failed to publish report packet r2 (r2-version)" logging.info.assert_has_calls([ call(expected_run_rpt_1_log), call(expected_run_rpt_2_log), - call("Success for key r1-key. New version is r1-version"), - call("Publishing report version r1-r1-version"), - call("Successfully published report version r1-r1-version"), - call("Success for key r2-key. New version is r2-version"), - call("Publishing report version r2-r2-version") + call("Success for key r1-key. New packet id is r1-version"), + call("Publishing report packet r1 (r1-version)"), + call("Successfully published report packet r1 (r1-version)"), + call("Success for key r2-key. New packet id is r2-version"), + call("Publishing report packet r2 (r2-version)") ], any_order=False) logging.error.assert_has_calls([ call(expected_err) ], any_order=False) mock_running_reports.assert_expected_calls() - ow.kill_report.assert_not_called() - assert success["called"] == "r1-version" - assert error["called"] == expected_err + packit.kill_task.assert_not_called() + success_callback.assert_called_with(reports[0], "r1-version") + error_callback.assert_called_with(reports[1], expected_err) class MockRunningReportRepository: @@ -593,40 +462,41 @@ def assert_expected_calls(self): ], any_order=False) -class MockOrderlyWebAPI: - def __init__(self, run_successfully, report_responses, expected_params, - expected_timeouts, fail_publish=None): - if fail_publish is None: - fail_publish = [] +class MockPackitClient: + def __init__(self, expected_params, run_successfully, report_responses, + fail_publish=[]): + self.auth_success = True + self.expected_params = expected_params self.run_successfully = run_successfully self.report_responses = report_responses - self.expected_params = expected_params - self.expected_timeouts = expected_timeouts self.fail_publish = fail_publish - self.kill_report = Mock() + self.kill_task = Mock() + self.refresh_git = Mock() - def run_report(self, name, params, timeout): + def run(self, name, params): assert params == self.expected_params[name] - assert timeout == self.expected_timeouts[name] if name in self.run_successfully: return name + "-key" else: raise Exception("test-run-error: " + name) - def report_status(self, key): + def poll(self, key): if key in self.report_responses and \ len(self.report_responses[key]) > 0: return self.report_responses[key].pop(0) else: raise Exception("test-status-error: " + key) - def publish_report(self, name, version): + def publish(self, name, packit_id, roles): if name in self.fail_publish: raise Exception("Publish failed") else: return True +PACKIT_URL = "http://test-packit" + + class MockConfig: def __init__(self, use_additional_recipients=True): @@ -656,10 +526,26 @@ def smtp_user(self): def smtp_password(self): return None - @property - def orderlyweb_url(self): - return "http://orderly-web" - @property def youtrack_token(self): return "12345" + + @property + def packit_url(self): + return PACKIT_URL + + @property + def packit_disable_certificate_verify(self): + return False + + @property + def montagu_url(self): + return "http://test-montagu" + + @property + def montagu_user(self): + return "test.montagu.user" + + @property + def montagu_password(self): + return "montagu_password" diff --git a/test/unit/test_send_diagnostic_report_email.py b/test/unit/test_send_diagnostic_report_email.py index 1e0b250..cb04d08 100644 --- a/test/unit/test_send_diagnostic_report_email.py +++ b/test/unit/test_send_diagnostic_report_email.py @@ -38,7 +38,7 @@ def test_url_encodes_url_in_email(send_email): "2020-11-04T12:22:53", "no_vaccination", mock_config) - url = "http://orderly-web/report/{}/1234-abcd/".format(encoded, encoded) + url = "http://test-packit/{}/1234-abcd/".format(encoded) send_email.assert_has_calls([ call(fake_emailer, report, @@ -64,7 +64,7 @@ def test_additional_recipients_used(send_email): "no_vaccination", mock_config, "someone@example.com") - url = "http://orderly-web/report/{}/1234-abcd/".format(name) + url = "http://test-packit/{}/1234-abcd/".format(name) send_email.assert_has_calls([ call(fake_emailer, report, @@ -90,7 +90,7 @@ def test_additional_recipients_not_used(send_email): "no_vaccination", mock_config, "someone@example.com") - url = "http://orderly-web/report/{}/1234-abcd/".format(name) + url = "http://test-packit/{}/1234-abcd/".format(name) send_email.assert_has_calls([ call(fake_emailer, report,