Integration test refactoring

.github/actions/test-control-plane/action.yaml

@@ -0,0 +1,58 @@
+name: 'Test Control Plane'
+description: 'Runs tests on the Pinecone control plane'
+
+inputs:
+  test_suite:
+    description: 'The type of the index (serverless, pod)'
+    required: true
+    default: 'serverless'
+  python_version:
+    description: 'The version of Python to use'
+    required: false
+    default: '3.9'
+  metric:
+    description: 'The metric of the index'
+    required: true
+  dimension:
+    description: 'The dimension of the index'
+    required: true
+  spec:
+    description: 'The deploy spec of the index'
+    required: true
+  use_grpc:
+    description: 'Whether to use gRPC or REST'
+    required: true
+  freshness_timeout_seconds:
+    description: 'The number of seconds to wait for the index to become fresh'
+    required: false
+    default: '60'
+  PINECONE_API_KEY:
+    description: 'The Pinecone API key'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Setup Poetry
+      uses: ./.github/actions/setup-poetry
+      with:
+        include_grpc: ${{ inputs.use_grpc }}
+        include_dev: 'true'
+
+    - name: Run tests
+      id: data-plane-tests
+      shell: bash
+      run: poetry run pytest tests/integration/control/${{ inputs.test_suite }}
+      env:
+        PINECONE_API_KEY: ${{ inputs.PINECONE_API_KEY }}
+        USE_GRPC: ${{ inputs.use_grpc }}
+        METRIC: ${{ inputs.metric }}
+        DIMENSION: ${{ inputs.dimension }}
+        SPEC: ${{ inputs.spec }}
+        FRESHNESS_TIMEOUT_SECONDS: ${{ inputs.freshness_timeout_seconds }}
+        GITHUB_BUILD_NUMBER: '${{ github.run_number }}-p-${{ matrix.testConfig.python-version}}'

.github/actions/test-data-plane/action.yaml

@@ -30,11 +30,6 @@ inputs:
     required: false
     default: '3.9'
 
-outputs:
-  index_name:
-    description: 'The name of the index, including randomized suffix'
-    value: ${{ steps.create-index.outputs.index_name }}
-
 runs:
   using: 'composite'
   steps:

.github/workflows/optimistic-updates.yaml

@@ -0,0 +1,93 @@
+name: Update Spec
+
+on:
+  workflow_dispatch: {}
+#   schedule:
+#     - cron: '0 0 * * *'
+
+jobs:
+  update-and-build:
+    name: Build docs with pdoc
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up git
+        run: |
+          git config --global user.name "Pinecone CI"
+          git config --global user.email "[email protected]"
+
+      - name: Update spec
+        run: |
+          ./scripts/build-oas.sh
+          git add codegen
+          git add pinecone/core
+
+      - name: Commit changes
+        id: commit
+        run:
+          if git diff --cached --quiet; then
+            echo "No changes are staged."
+            echo "new_changes=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "There are staged changes."
+            git commit -m'Regenerate core with latest specs'
+            echo "new_changes=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Abort if no recent changes
+        if: steps.commit.outputs.new_changes == 'false'
+        uses: andymckay/[email protected]
+
+      - name: Test data plane (rest)
+        uses: ./.github/actions/test-data-plane
+        with:
+            metric: 'cosine'
+            use_grpc: false
+            spec: '{ "serverless": { "cloud": "aws", "region": "us-west-2" }}'
+            PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+            freshness_timeout_seconds: 600
+
+      - name: Test control plane (serverless, rest)
+        uses: ./.github/actions/test-control-plane
+        with:
+            test_suite: 'serverless'
+            use_grpc: false
+            metric: 'cosine'
+            spec: '{ "serverless": { "cloud": "aws", "region": "us-west-2" }}'
+            PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+            freshness_timeout_seconds: 600
+
+      - name: Test control plane (serverless, grpc)
+        uses: ./.github/actions/test-control-plane
+        with:
+            test_suite: 'serverless'
+            use_grpc: true
+            metric: 'cosine'
+            spec: '{ "serverless": { "cloud": "aws", "region": "us-west-2" }}'
+            PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+            freshness_timeout_seconds: 600
+
+      - name: Test control plane (pod, rest)
+        uses: ./.github/actions/test-control-plane
+        with:
+            test_suite: 'pod'
+            use_grpc: false
+            metric: 'cosine'
+            spec: '{ "serverless": { "cloud": "aws", "region": "us-west-2" }}'
+            PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+            freshness_timeout_seconds: 600
+
+      - name: Test control plane (pod, grpc)
+        uses: ./.github/actions/test-control-plane
+        with:
+            test_suite: 'pod'
+            use_grpc: true
+            metric: 'cosine'
+            spec: '{ "serverless": { "cloud": "aws", "region": "us-west-2" }}'
+            PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+            freshness_timeout_seconds: 600
+
+      - name: Push changes
+        run: git push

.github/workflows/testing-integration.yaml

@@ -26,130 +26,66 @@ jobs:
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
 
   data-plane-serverless:
-    name: Data plane serverless integration tests
+    name: data serverless
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         python_version: [3.8, 3.12]
         use_grpc: [true, false]
-        metric: 
-          - cosine
-          # - euclidean
-          # - dotproduct
-        spec:
-          - '{ "serverless": { "region": "us-west-2", "cloud": "aws" }}'
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test-data-plane
         with:
           python_version: '${{ matrix.python_version }}'
           use_grpc: '${{ matrix.use_grpc }}'
-          metric: '${{ matrix.metric }}'
-          spec: '${{ matrix.spec }}'
+          metric: 'cosine'
+          spec: '{ "serverless": { "region": "us-west-2", "cloud": "aws" }}'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
           freshness_timeout_seconds: 600
-#   data-plane-pod:
-#     name: Data plane pod integration tests
-#     runs-on: ubuntu-latest
-#     strategy:
-#       max-parallel: 1
-#       matrix:
-#         use_grpc: [true, false]
-#         metric: [cosine, dotproduct]
-#         spec:
-#           - '{ "pod": { "environment": "us-east1-gcp", "pod_type": "p1" }}'
-#     steps:
-#       - uses: actions/checkout@v4
-#       - uses: ./.github/actions/test-data-plane
-#         with:
-#           use_grpc: '${{ matrix.use_grpc }}'
-#           metric: '${{ matrix.metric }}'
-#           spec: '${{ matrix.spec }}'
-#           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
 
-  control-rest-pod:
-    name: control plane pod/collection tests
+  control-plane-pod:
+    name: control pod
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pineconeEnv:
-          - prod
-        testConfig:
-          - python-version: 3.8
-            pod: { environment: 'us-east1-gcp'}
-          - python-version: 3.12
-            pod: { environment: 'us-east4-gcp'}
+        # python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.10']
+        use_grpc: [true, false]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
-      - name: 'Set up Python ${{ matrix.testConfig.python-version }}'
-        uses: actions/setup-python@v5
+      - name: Test control plane (pod, rest)
+        uses: ./.github/actions/test-control-plane
         with:
-          python-version: '${{ matrix.testConfig.python-version }}'
-      - name: Setup Poetry
-        uses: ./.github/actions/setup-poetry
-      - name: 'Run integration tests (REST, prod)'
-        if: matrix.pineconeEnv == 'prod'
-        run: poetry run pytest tests/integration/control/pod -s -v
-        env:
-          PINECONE_DEBUG_CURL: 'true'
-          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
-          PINECONE_ENVIRONMENT: '${{ matrix.testConfig.pod.environment }}'
-          GITHUB_BUILD_NUMBER: '${{ github.run_number }}-s-${{ matrix.testConfig.python-version}}'
-          DIMENSION: 2500
-          METRIC: 'cosine'
-      - name: 'Run integration tests (REST, staging)'
-        if: matrix.pineconeEnv == 'staging'
-        run: poetry run pytest tests/integration/control/pod -s -v
-        env:
-          PINECONE_DEBUG_CURL: 'true'
-          PINECONE_CONTROLLER_HOST: 'https://api-staging.pinecone.io'
-          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY_STAGING }}'
-          PINECONE_ENVIRONMENT: '${{ matrix.testConfig.pod.environment }}'
-          GITHUB_BUILD_NUMBER: '${{ github.run_number }}-p-${{ matrix.testConfig.python-version}}'
-          DIMENSION: 1536
-          METRIC: 'cosine'
+          python_version: '${{ matrix.python-version }}'
+          test_suite: 'pod'
+          use_grpc: ${{ matrix.use_grpc }}
+          metric: 'cosine'
+          dimension: 10
+          spec: '{ "pod": { "environment": "us-east4-gcp", "pod_type": "p1.x1" }}'
+          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+          freshness_timeout_seconds: 600
 
-  control-rest-serverless:
-    name: control plane serverless
+  control-plane-serverless:
+    name: control serverless
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pineconeEnv:
-          - prod
-        testConfig:
-          - python-version: 3.8 # Do one test run with 3.8 for sanity check
-            pod: { environment: 'us-east1-gcp'}
-            serverless: { cloud: 'aws', region: 'us-west-2'}
-          - python-version: 3.12
-            pod: { environment: 'us-east1-gcp'}
-            serverless: { cloud: 'aws', region: 'us-west-2'}
+        # python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.10']
+        use_grpc: [true, false]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
-      - name: 'Set up Python ${{ matrix.testConfig.python-version }}'
-        uses: actions/setup-python@v5
+      - name: Test control plane (serverless, rest)
+        uses: ./.github/actions/test-control-plane
         with:
-          python-version: '${{ matrix.testConfig.python-version }}'
-      - name: Setup Poetry
-        uses: ./.github/actions/setup-poetry
-      - name: 'Run integration tests (REST, prod)'
-        if: matrix.pineconeEnv == 'prod'
-        run: poetry run pytest tests/integration/control/serverless -s -vv
-        env:
-          PINECONE_DEBUG_CURL: 'true'
-          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
-          GITHUB_BUILD_NUMBER: '${{ github.run_number }}-p-${{ matrix.testConfig.python-version}}'
-          SERVERLESS_CLOUD: '${{ matrix.testConfig.serverless.cloud }}'
-          SERVERLESS_REGION: '${{ matrix.testConfig.serverless.region }}'
-      - name: 'Run integration tests (REST, staging)'
-        if: matrix.pineconeEnv == 'staging'
-        run: poetry run pytest tests/integration/control/serverless -s -vv
-        env:
-          PINECONE_DEBUG_CURL: 'true'
-          PINECONE_CONTROLLER_HOST: 'https://api-staging.pinecone.io'
-          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY_STAGING }}'
-          GITHUB_BUILD_NUMBER: '${{ github.run_number }}-s-${{ matrix.testConfig.python-version}}'
-          SERVERLESS_CLOUD: '${{ matrix.testConfig.serverless.cloud }}'
-          SERVERLESS_REGION: '${{ matrix.testConfig.serverless.region }}'
+          python_version: '${{ matrix.python-version }}'
+          test_suite: 'serverless'
+          use_grpc: ${{ matrix.use_grpc }}
+          metric: 'cosine'
+          dimension: 10
+          spec: '{ "serverless": { "cloud": "aws", "region": "us-west-2" }}'
+          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+          freshness_timeout_seconds: 600

tests/dependency/grpc/test_sanity.py

@@ -31,7 +31,7 @@ def test_sanity(self, index_name, client):
         idx.upsert(vectors=[("1", [1.0, 2.0]), ("2", [3.0, 4.0]), ("3", [5.0, 6.0])])
 
         # Wait for index freshness
-        time.sleep(30)
+        time.sleep(60)
 
         # Check the vector count reflects some data has been upserted
         description = idx.describe_index_stats()

tests/dependency/rest/test_sanity.py

@@ -31,7 +31,7 @@ def test_sanity(self, index_name, client):
         idx.upsert(vectors=[("1", [1.0, 2.0]), ("2", [3.0, 4.0]), ("3", [5.0, 6.0])])
 
         # Wait for index freshness
-        time.sleep(30)
+        time.sleep(60)
 
         # Check the vector count reflects some data has been upserted
         description = idx.describe_index_stats()