add pytest integration tests

.github/workflows/integration-test.yml

@@ -94,7 +94,7 @@ jobs:
           poetry run skyplane init -y --disable-config-azure
           poetry run skyplane config set usage_stats false
       - name: Deprovision
-        run: poetry run skyplane deprovision
+        run: poetry run skyplane deprovision --all
       - name: Delete matching S3 buckets
         run: |
           for pattern in "test-skyplane-" "skyplane-integration-" "integrationus-east-1-"; do

.github/workflows/pytest_integration.yml

@@ -0,0 +1,200 @@
+name: pytest-integration
+on:
+  push:
+    branches: [ main ]
+    # Publish semver tags as releases.
+    tags: [ '*.*.*' ]
+  pull_request:
+  pull_request_target:
+    types: [labeled]
+env:
+  SKYPLANE_USAGE_STATS_ENABLED: 0
+jobs:
+  test-integration:
+    # do not run on pull_request_target since it should be triggered by pull_request
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request')
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+      - name: Set Poetry config
+        run: |
+          poetry config virtualenvs.in-project false
+          poetry config virtualenvs.path ~/.virtualenvs
+      - name: Install Dependencies
+        run: |
+          poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm
+          poetry run pip install -r requirements-dev.txt
+        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Log into Azure
+        uses: azure/login@v1
+        with:
+          creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+        with:
+          project_id: '${{ secrets.GCP_PROJECT_ID }}'
+          export_default_credentials: true
+      - name: Run cloud tests
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          poetry run skyplane init -y --disable-config-azure --disable-config-gcp --disable-config-ibm --disable-config-cloudflare
+          poetry run skyplane config set usage_stats false
+          poetry run pytest -s tests/integration/test_cp
+  deprovision:
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    needs: [integration]
+    env:
+      STRATEGY_UUID: itest-d-${{ github.run_id }}-${{ github.run_attempt }}
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install skyplane from pypi
+        run: pip install skyplane[aws,azure,gcp]
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
+      - name: Skyplane init
+        run: |
+          poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
+          poetry run skyplane init -y --disable-config-azure
+          poetry run skyplane config set usage_stats false
+      - name: Deprovision
+        run: poetry run skyplane deprovision
+      - name: Delete matching S3 buckets
+        run: |
+          for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do
+            aws s3api list-buckets --query "Buckets[?starts_with(Name, \`${pattern}\`) == \`true\`].Name" --output text | tr '\t' '\n' | while read bucket; do aws s3 rb "s3://$bucket" --force; done
+          done
+      - name: Cleanup GCP service account
+        if: always()
+        run: gcloud iam service-accounts delete --quiet ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
+
+  test-aws:
+    # run if push to main or pull request by contributor or pull request target with label 'safe to test' (to avoid running on forks)
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test'))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+      - name: Set Poetry config
+        run: |
+          poetry config virtualenvs.in-project false
+          poetry config virtualenvs.path ~/.virtualenvs
+      - name: Install Dependencies
+        run: |
+          poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm
+          poetry run pip install -r requirements-dev.txt
+        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Run cloud tests
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          poetry run skyplane init -y --disable-config-azure --disable-config-gcp --disable-config-ibm --disable-config-cloudflare
+          poetry run skyplane config set usage_stats false
+          poetry run pytest -s tests/unit_aws
+  test-azure:
+    # run if push to main or pull request by contributor or pull request target with label 'safe to test' (to avoid running on forks)
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test'))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+      - name: Set Poetry config
+        run: |
+          poetry config virtualenvs.in-project false
+          poetry config virtualenvs.path ~/.virtualenvs
+      - name: Log into Azure
+        uses: azure/login@v1
+        with:
+          creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
+      - name: Install Dependencies
+        run: |
+          poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm
+          poetry run pip install -r requirements-dev.txt
+        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Run cloud tests
+        run: |
+          poetry run pip freeze
+          poetry run skyplane init -y --disable-config-aws --disable-config-gcp --disable-config-ibm --disable-config-cloudflare
+          poetry run skyplane config set usage_stats false
+          poetry run pytest -s tests/unit_azure
+  test-gcp:
+    # run if push to main or pull request or pull request target with label 'safe to test' (to avoid running on forks)
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test'))
+    runs-on: ubuntu-latest
+    env:
+      STRATEGY_UUID: pytest-${{ github.run_id }}-${{ github.run_attempt }}
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+      - name: Set Poetry config
+        run: |
+          poetry config virtualenvs.in-project false
+          poetry config virtualenvs.path ~/.virtualenvs
+          
+      - name: Install Dependencies
+        run: |
+          poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm
+          poetry run pip install -r requirements-dev.txt
+          poetry run sudo apt install default-jdk
+        if: steps.cache.outputs.cache-hit != 'true'
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+        with:
+          project_id: '${{ secrets.GCP_PROJECT_ID }}'
+          export_default_credentials: true
+      - name: Run cloud tests
+        run: |
+          poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
+          poetry run skyplane init -y --disable-config-aws --disable-config-azure --disable-config-ibm --disable-config-cloudflare
+          poetry run skyplane config set usage_stats false
+          poetry run pytest -s tests/unit_gcs
+      - name: Cleanup GCP service account
+        if: always()
+        run: gcloud iam service-accounts delete ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com

skyplane/api/pipeline.py

@@ -70,19 +70,11 @@ def __init__(
         # planner
         self.planning_algorithm = planning_algorithm
         if self.planning_algorithm == "direct":
-<<<<<<< HEAD
             self.planner = MulticastDirectPlanner(self.max_instances, self.n_connections, self.transfer_config)
         elif self.planning_algorithm == "src_one_sided":
             self.planner = DirectPlannerSourceOneSided(self.max_instances, self.n_connections, self.transfer_config)
         elif self.planning_algorithm == "dst_one_sided":
             self.planner = DirectPlannerDestOneSided(self.max_instances, self.n_connections, self.transfer_config)
-=======
-            self.planner = MulticastDirectPlanner(self.max_instances, 64, self.transfer_config)
-        elif self.planning_algorithm == "src_one_sided":
-            self.planner = DirectPlannerSourceOneSided(self.max_instances, 64, self.transfer_config)
-        elif self.planning_algorithm == "dst_one_sided":
-            self.planner = DirectPlannerDestOneSided(self.max_instances, 64, self.transfer_config)
->>>>>>> b265310 (Fix inconsistencies between VCPU and R2 merge  (#857))
         else:
             raise ValueError(f"No such planning algorithm {planning_algorithm}")
 

skyplane/api/tracker.py

@@ -262,6 +262,7 @@ def monitor_single_dst_helper(dst_region):
                 self.dataplane.topology.dest_region_tags,
                 session_start_timestamp_ms,
             )
+            print("transfer stats", transfer_stats)
             print_stats_completed(total_runtime_s=transfer_stats["total_runtime_s"], throughput_gbits=transfer_stats["throughput_gbits"])
 
     @imports.inject("pandas")

skyplane/compute/gcp/gcp_network.py

@@ -42,6 +42,11 @@ def delete_network(errors, self):
         Delete VPC. This might error our in some cases.
         """
         compute = self.auth.get_gcp_client()
+
+        # delete firewall rules
+        self.delete_firewall_rules()
+
+        # delete network
         request = compute.networks().delete(project=self.auth.project_id, network=self.vpc_name)
         try:
             delete_vpc_response = request.execute()
@@ -52,6 +57,24 @@ def delete_network(errors, self):
             )
             logger.fs.error(e)
 
+    @imports.inject("googleapiclient.errors", pip_extra="gcp")
+    def delete_firewall_rules(errors, self):
+        # Get the firewall rules in the network
+        compute = self.auth.get_gcp_client()
+        request = compute.firewalls().list(project=self.auth.project_id)
+        response = request.execute()
+        if "items" not in response: 
+            # no firewall rules found
+            return 
+
+        # Delete each firewall rule
+        for rule in response['items']:
+            if rule["network"] == f"https://www.googleapis.com/compute/v1/projects/{self.auth.project_id}/global/networks/{self.vpc_name}": 
+                rule_name = rule['name']
+                print(f'Deleting firewall rule: {rule_name}')
+                compute.firewalls().delete(project=self.auth.project_id, firewall=rule_name).execute()
+
+
     @imports.inject("googleapiclient.errors", pip_extra="gcp")
     def get_firewall_rule(errors, self, firewall_rule_name):
         compute = self.auth.get_gcp_client()