Skip to content

Metadata Extract

Metadata Extract #107

name: Metadata Extract
on:
workflow_dispatch:
inputs:
environment:
description: 'The environment to run the workflow in'
required: true
type: choice
default: 'preview'
options:
- 'preview'
- 'production'
schedule:
- cron: '0 9 * * 1' # Run at midnight UTC on Mondays
jobs:
get_variants_list:
runs-on: ubuntu-latest
outputs:
sdk_ex_matrix: ${{ steps.setmatrix_sdk_ex.outputs.sdk_ex_matrix }}
sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }}
airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }}
steps:
- uses: actions/[email protected]
- name: Install hub-utils
run: pipx install git+https://github.com/meltano/hub-utils.git
- name: Get Variants List - SDK Extractors
id: get-variants-list-sdk-ex
run: echo "VARIANTS_SDK_EX=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk)" >> $GITHUB_OUTPUT
- name: Set Dynamic Matrix - SDK Extractors
id: setmatrix_sdk_ex
run: |
matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex.outputs.VARIANTS_SDK_EX }}}"
echo "::set-output name=sdk_ex_matrix::$matrixStringifiedObject"
- name: Get Variants List - SDK Loaders
id: get-variants-list-sdk-loaders
run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT
- name: Set Dynamic Matrix - SDK Loaders
id: setmatrix_sdk_loaders
run: |
matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}"
echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject"
- name: Get Variants List - Airbyte
id: get-variants-list-airbyte
run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT
- name: Set Dynamic Airbyte Matrix
id: setmatrix_airbyte
run: |
matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}"
echo "::set-output name=airbyte_matrix::$matrixStringifiedObject"
metadata_extract_sdk_ex:
name: SDK Extractors - Metadata Extract
needs: get_variants_list
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
env:
AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_matrix) }}
permissions:
id-token: write # This is required for requesting the JWT
steps:
- uses: actions/[email protected]
- name: Configure AWS credentials
uses: aws-actions/[email protected]
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: us-west-2
role-session-name: "GitHubActions"
- name: Install hub-utils
run: pipx install git+https://github.com/meltano/hub-utils.git
- name: Install Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Extract Metadata (3.10)
id: extract_310
run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
continue-on-error: true
- name: Install Python 3.9 (Only if 3.10 failed)
if: always() && (steps.extract_310.outcome == 'failure')
uses: actions/setup-python@v5
with:
python-version: '3.9'
- name: Extract Metadata (3.9)
if: always() && (steps.extract_310.outcome == 'failure')
id: extract_39
run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
continue-on-error: true
- name: Install Python 3.8 (Only if 3.9 failed)
if: always() && (steps.extract_39.outcome == 'failure')
uses: actions/setup-python@v5
with:
python-version: '3.8'
- name: Extract Metadata (3.8)
if: always() && (steps.extract_39.outcome == 'failure')
id: extract_38
run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
metadata_extract_sdk_loaders:
name: SDK Loaders - Metadata Extract
needs: get_variants_list
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
env:
AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }}
permissions:
id-token: write # This is required for requesting the JWT
steps:
- uses: actions/[email protected]
- name: Configure AWS credentials
uses: aws-actions/[email protected]
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: us-west-2
role-session-name: "GitHubActions"
- name: Install hub-utils
run: pipx install git+https://github.com/meltano/hub-utils.git
- name: Install Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Extract Metadata (3.10)
id: extract_310
run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
continue-on-error: true
- name: Install Python 3.9 (Only if 3.10 failed)
if: always() && (steps.extract_310.outcome == 'failure')
uses: actions/setup-python@v5
with:
python-version: '3.9'
- name: Extract Metadata (3.9)
if: always() && (steps.extract_310.outcome == 'failure')
id: extract_39
run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
continue-on-error: true
- name: Install Python 3.8 (Only if 3.9 failed)
if: always() && (steps.extract_39.outcome == 'failure')
uses: actions/setup-python@v5
with:
python-version: '3.8'
- name: Extract Metadata (3.8)
if: always() && (steps.extract_39.outcome == 'failure')
id: extract_38
run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
metadata_extract_airbyte_p1:
name: Airbyte - Part 1 Metadata Extract
needs: get_variants_list
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
container:
image: airbyte/${{ matrix.source-name }}:latest
steps:
- name: Get Airbyte Spec
run: |
$AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json
id: get-airbyte-stdout
- uses: actions/upload-artifact@v3
with:
name: extractor--${{ matrix.source-name }}--airbyte.json
path: extractor--${{ matrix.source-name }}--airbyte.json
metadata_extract_airbyte_p2:
name: Airbyte - Part 2 Metadata Extract
if: ${{ always() }}
needs: [metadata_extract_airbyte_p1, get_variants_list]
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
env:
AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
permissions:
id-token: write # This is required for requesting the JWT
steps:
- uses: actions/[email protected]
- name: Configure AWS credentials
uses: aws-actions/[email protected]
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: us-west-2
role-session-name: "GitHubActions"
- uses: actions/download-artifact@v3
with:
name: extractor--${{ matrix.source-name }}--airbyte.json
- name: Install hub-utils
run: pipx install git+https://github.com/meltano/hub-utils.git
- name: Upload Metadata
run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json