feat: Add data support for Change Requests (#92)

This exposes some APIs and schemas for change requests. It does not expose any UI-related components.

.vscode/extensions.json

@@ -7,6 +7,7 @@
         "donjayamanne.python",
         "dbaeumer.vscode-eslint",
         "lextudio.restructuredtext",
-        "PeterJausovec.vscode-docker"
+        "PeterJausovec.vscode-docker",
+        "DavidAnson.vscode-markdownlint"
     ]
 }

README.md

@@ -186,6 +186,8 @@ zeus
 |   |       └── TestCase
 |   |           ├── Artifact
 |   |           └── ItemStat
+|   ├── ChangeRequest
+|   |   └── Revision
 |   └── Source
 |       ├── Author
 |       ├── Patch
@@ -229,9 +231,16 @@ This will look for a Build object with the following characteristics:
 - `external_id=abc`
 - `repository_id={Hook.repository_id}`
 
-If a match is found, it will be updated with the given API parameters. If it isn't found, it will be created.
+If a match is found, it will be updated with the given API parameters. If it isn't found, it will be created. All of these operations are treated like a standard UPSERT (UPDATE IF EXISTS or INSERT).
 
-For example, if you were to publish job details from Travis without using the native webhooks:
+The process for publishing data generally looks like this:
+
+1. if applicable, upsert a change request and its source association
+2. upsert the build's basic parameters
+3. upsert the detailed job parameters
+4. publish artifacts
+
+For example, to publish job details from Travis without using the native webhooks:
 
 ```shell
 #!/bin/bash -eu

tests/fixtures/travis-build-pull-request.json

@@ -0,0 +1,169 @@
+{
+    "id": 288639281,
+    "number": "2843",
+    "config": {
+      "sudo": false,
+      "dist": "trusty",
+      "language": "python",
+      "python": [
+        "3.5.2"
+      ],
+      "branches": {
+        "only": [
+          "master"
+        ]
+      },
+      "cache": {
+        "pip": true,
+        "directories": [
+          "vendor/bundle",
+          "node_modules"
+        ]
+      },
+      "deploy": {
+        "provider": "heroku",
+        "api_key": {
+          "secure": "hylw2GIHMvZKOKX3uPSaLEzVrUGEA9mzGEA0s4zK37W9HJCTnvAcmgRCwOkRuC4L7R4Zshdh/CGORNnBBgh1xx5JGYwkdnqtjHuUQmWEXCusrIURu/iEBNSsZZEPK7zBuwqMHj2yRm64JfbTDJsku3xdoA5Z8XJG5AMJGKLFgUQ="
+        },
+        "app": "docs-travis-ci-com",
+        "skip_cleanup": true,
+        "true": {
+          "branch": [
+            "master"
+          ]
+        }
+      },
+      "notifications": {
+        "slack": {
+          "rooms": {
+            "secure": "LPNgf0Ra6Vu6I7XuK7tcnyFWJg+becx1RfAR35feWK81sru8TyuldQIt7uAKMA8tqFTP8j1Af7iz7UDokbCCfDNCX1GxdAWgXs+UKpwhO89nsidHAsCkW2lWSEM0E3xtOJDyNFoauiHxBKGKUsApJTnf39H+EW9tWrqN5W2sZg8="
+          },
+          "on_success": "never"
+        },
+        "webhooks": "https://docs.travis-ci.com/update_webhook_payload_doc"
+      },
+      "install": [
+        "rvm use 2.3.1 --install",
+        "bundle install --deployment"
+      ],
+      "script": [
+        "bundle exec rake test"
+      ],
+      ".result": "configured",
+      "global_env": [
+        "PATH=$HOME/.local/user/bin:$PATH"
+      ],
+      "group": "stable"
+    },
+    "type": "cron",
+    "state": "passed",
+    "status": 0,
+    "result": 0,
+    "status_message": "Passed",
+    "result_message": "Passed",
+    "started_at": "2017-10-16T16:08:56Z",
+    "finished_at": "2017-10-16T16:12:35Z",
+    "duration": 219,
+    "build_url": "https://travis-ci.org/travis-ci/docs-travis-ci-com/builds/288639281",
+    "commit_id": 84531696,
+    "commit": "d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+    "base_commit": "d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+    "head_commit": "d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+    "branch": "master",
+    "message": "Merge pull request #1389 from christopher-dG/patch-1\\n\\nJulia: Refer to PkgDev's generate instead of Pkg's",
+    "compare_url": "https://github.com/travis-ci/docs-travis-ci-com/compare/eb58bd2fac2e339d0339689d9eb7290246805a1d...d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+    "committed_at": "2017-10-16T14:56:34Z",
+    "author_name": "Plaindocs",
+    "author_email": "[email protected]",
+    "committer_name": "GitHub",
+    "committer_email": "[email protected]",
+    "pull_request": true,
+    "pull_request_number": 123,
+    "pull_request_title": "The title of the pull request",
+    "tag": null,
+    "repository": {
+      "id": 1771959,
+      "name": "docs-travis-ci-com",
+      "owner_name": "travis-ci",
+      "url": "http://docs.travis-ci.com"
+    },
+    "matrix": [
+      {
+        "id": 288639284,
+        "repository_id": 1771959,
+        "parent_id": 288639281,
+        "number": "2843.1",
+        "state": "passed",
+        "config": {
+          "sudo": false,
+          "dist": "trusty",
+          "language": "python",
+          "python": "3.5.2",
+          "branches": {
+            "only": [
+              "master"
+            ]
+          },
+          "env": ["TEST_SUITE=integration"],
+          "cache": {
+            "pip": true,
+            "directories": [
+              "vendor/bundle",
+              "node_modules"
+            ]
+          },
+          "notifications": {
+            "slack": {
+              "rooms": {
+                "secure": "LPNgf0Ra6Vu6I7XuK7tcnyFWJg+becx1RfAR35feWK81sru8TyuldQIt7uAKMA8tqFTP8j1Af7iz7UDokbCCfDNCX1GxdAWgXs+UKpwhO89nsidHAsCkW2lWSEM0E3xtOJDyNFoauiHxBKGKUsApJTnf39H+EW9tWrqN5W2sZg8="
+              },
+              "on_success": "never"
+            },
+            "webhooks": "https://docs.travis-ci.com/update_webhook_payload_doc"
+          },
+          "install": [
+            "rvm use 2.3.1 --install",
+            "bundle install --deployment"
+          ],
+          "script": [
+            "bundle exec rake test"
+          ],
+          ".result": "configured",
+          "global_env": [
+            "PATH=$HOME/.local/user/bin:$PATH"
+          ],
+          "group": "stable",
+          "os": "linux",
+          "addons": {
+            "deploy": {
+              "provider": "heroku",
+              "api_key": {
+                "secure": "hylw2GIHMvZKOKX3uPSaLEzVrUGEA9mzGEA0s4zK37W9HJCTnvAcmgRCwOkRuC4L7R4Zshdh/CGORNnBBgh1xx5JGYwkdnqtjHuUQmWEXCusrIURu/iEBNSsZZEPK7zBuwqMHj2yRm64JfbTDJsku3xdoA5Z8XJG5AMJGKLFgUQ="
+              },
+              "app": "docs-travis-ci-com",
+              "skip_cleanup": true,
+              "true": {
+                "branch": [
+                  "master"
+                ]
+              }
+            }
+          }
+        },
+        "status": 0,
+        "result": 0,
+        "commit": "d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+        "branch": "master",
+        "message": "Merge pull request #1389 from christopher-dG/patch-1\\n\\nJulia: Refer to PkgDev's generate instead of Pkg's",
+        "compare_url": "https://github.com/travis-ci/docs-travis-ci-com/compare/eb58bd2fac2e339d0339689d9eb7290246805a1d...d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+        "started_at": null,
+        "finished_at": null,
+        "committed_at": "2017-10-16T14:56:34Z",
+        "author_name": "Plaindocs",
+        "author_email": "[email protected]",
+        "committer_name": "GitHub",
+        "committer_email": "[email protected]",
+        "allow_failure": true
+      }
+    ]
+  }

tests/zeus/api/resources/test_repository_builds.py

@@ -65,8 +65,7 @@ def test_repo_build_create_missing_revision(
     client, default_login, default_revision, default_source, default_repo, default_repo_access, mocker
 ):
     mock_identify_revision = mocker.patch(
-        'zeus.api.resources.repository_builds.identify_revision')
-
+        'zeus.utils.revisions.identify_revision')
     mock_identify_revision.return_value = default_revision
 
     resp = client.post(

tests/zeus/api/resources/test_repository_change_requests.py

@@ -0,0 +1,67 @@
+from zeus import factories
+from zeus.models import ChangeRequest
+
+
+def test_repo_change_request_list(
+        client, default_login, default_change_request, default_repo, default_repo_access):
+    resp = client.get(
+        '/api/repos/{}/change-requests?show=all'.format(
+            default_repo.get_full_name())
+    )
+    assert resp.status_code == 200
+    data = resp.json()
+    assert len(data) == 1
+    assert data[0]['id'] == str(default_change_request.id)
+
+
+def test_repo_change_request_list_without_access(
+        client, default_login, default_change_request, default_repo):
+    resp = client.get(
+        '/api/repos/{}/change-requests'.format(default_repo.get_full_name()))
+    assert resp.status_code == 404
+
+
+def test_repo_change_request_list_mine_with_match(
+    client, default_login, default_change_request, default_repo, default_repo_access
+):
+    resp = client.get(
+        '/api/repos/{}/change-requests?show=mine'.format(default_repo.get_full_name()))
+    assert resp.status_code == 200
+    data = resp.json()
+    assert len(data) == 1
+
+
+def test_repo_change_request_list_mine_without_match(
+    client, default_login, default_repo, default_repo_access
+):
+    revision = factories.RevisionFactory(repository=default_repo)
+    source = factories.SourceFactory(revision=revision)
+    factories.BuildFactory(source=source)
+    resp = client.get(
+        '/api/repos/{}/change-requests?show=mine'.format(default_repo.get_full_name()))
+    assert resp.status_code == 200
+    data = resp.json()
+    assert len(data) == 0
+
+
+def test_create_change_request(
+        client, default_login, default_repo, default_repo_access, default_source):
+    resp = client.post(
+        '/api/repos/{}/change-requests'.format(
+            default_repo.get_full_name(),
+        ),
+        json={
+            'message': 'Hello world!',
+            'provider': 'github',
+            'external_id': '123',
+            'parent_revision_sha': default_source.revision_sha,
+        }
+    )
+    assert resp.status_code == 201
+    data = resp.json()
+    assert data['id']
+
+    cr = ChangeRequest.query.unrestricted_unsafe().get(data['id'])
+    assert cr.message == 'Hello world!'
+    assert cr.external_id == '123'
+    assert cr.provider == 'github'

tests/zeus/providers/travis/test_webhook.py

@@ -5,7 +5,7 @@
 
 from zeus import factories
 from zeus.constants import Result, Status
-from zeus.models import Build, Job
+from zeus.models import Build, ChangeRequest, Job
 
 CONFIG_RESPONSE = b"""
 {
@@ -40,6 +40,12 @@ def sample_travis_build_commit(fixture_path):
         return fp.read()
 
 
+@pytest.fixture(scope='session')
+def sample_travis_build_pr(fixture_path):
+    with open(os.path.join(fixture_path, 'travis-build-pull-request.json'), 'rb') as fp:
+        return fp.read()
+
+
 def make_signature(payload, private_key) -> bytes:
     from cryptography.hazmat.primitives import hashes
     from cryptography.hazmat.primitives.asymmetric import padding
@@ -93,8 +99,7 @@ def test_queued_build(client, default_repo, default_hook, default_revision,
     source = factories.SourceFactory.create(revision=default_revision)
 
     mock_identify_revision = mocker.patch(
-        'zeus.api.resources.repository_builds.identify_revision')
-
+        'zeus.utils.revisions.identify_revision')
     mock_identify_revision.return_value = default_revision
 
     resp = post_request(client, default_hook, sample_travis_build_commit,
@@ -123,3 +128,51 @@ def test_queued_build(client, default_repo, default_hook, default_revision,
     assert job.allow_failure
     assert job.label == 'python: 3.5.2 - TEST_SUITE=integration'
     assert job.url == 'https://travis-ci.org/travis-ci/docs-travis-ci-com/jobs/288639284'
+
+
+def test_pull_request(client, default_repo, default_hook, default_revision,
+                      sample_travis_build_pr, private_key, public_key_bytes, mocker, responses):
+    responses.add(responses.GET, 'https://api.travis-ci.com/config',
+                  get_config_response(public_key_bytes))
+
+    source = factories.SourceFactory.create(revision=default_revision)
+
+    mock_identify_revision = mocker.patch(
+        'zeus.utils.revisions.identify_revision')
+    mock_identify_revision.return_value = default_revision
+
+    resp = post_request(client, default_hook, sample_travis_build_pr,
+                        public_key_bytes, make_signature(sample_travis_build_pr, private_key))
+    assert resp.status_code == 200, repr(resp.data)
+
+    cr = ChangeRequest.query.unrestricted_unsafe().filter(
+        ChangeRequest.provider == 'github',
+        ChangeRequest.external_id == '123',
+    ).first()
+    assert cr
+    assert cr.message == 'The title of the pull request'
+    assert cr.parent_revision_sha == source.revision_sha
+    assert cr.head_revision_sha == source.revision_sha
+
+    build = Build.query.unrestricted_unsafe().filter(
+        Build.provider == 'travis',
+        Build.external_id == '288639281',
+    ).first()
+    assert build
+    assert build.repository_id == default_repo.id
+    assert build.source_id == source.id
+    assert build.label == 'PR #123 - The title of the pull request'
+    assert build.url == 'https://travis-ci.org/travis-ci/docs-travis-ci-com/builds/288639281'
+
+    job = Job.query.unrestricted_unsafe().filter(
+        Job.provider == 'travis',
+        Job.external_id == '288639284',
+    ).first()
+    assert job
+    assert job.build_id == build.id
+    assert job.repository_id == default_repo.id
+    assert job.status == Status.finished
+    assert job.result == Result.passed
+    assert job.allow_failure
+    assert job.label == 'python: 3.5.2 - TEST_SUITE=integration'
+    assert job.url == 'https://travis-ci.org/travis-ci/docs-travis-ci-com/jobs/288639284'

zeus/api/__init__.py

@@ -19,6 +19,9 @@
 app.add_resource(
     '/repos/<provider>/<owner_name>/<repo_name>/file-coverage-tree',
     r.RepositoryFileCoverageTreeResource)
+app.add_resource(
+    '/repos/<provider>/<owner_name>/<repo_name>/change-requests',
+    r.RepositoryChangeRequestsResource)
 app.add_resource(
     '/repos/<provider>/<owner_name>/<repo_name>/hooks', r.RepositoryHooksResource)
 app.add_resource(

zeus/api/resources/base_build.py

@@ -1,6 +1,6 @@
 from flask import Response
-
 from sqlalchemy.orm import contains_eager
+
 from zeus.models import Build, Repository, RepositoryProvider
 
 from .base import Resource