fix(csv_export): use custom CSV_EXPORT parameters in pd.read_csv for pivot table #45245
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Ephemeral env workflow | |
| # Example manual trigger: gh workflow run ephemeral-env.yml --ref fix_ephemerals --field comment_body="/testenv up" --field issue_number=666 | |
| on: | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| comment_body: | |
| description: 'Comment body to simulate /testenv command' | |
| required: true | |
| default: '/testenv up' | |
| issue_number: | |
| description: 'Issue or PR number' | |
| required: true | |
| jobs: | |
| ephemeral-env-comment: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment | |
| cancel-in-progress: true | |
| name: Evaluate ephemeral env comment trigger (/testenv) | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| outputs: | |
| slash-command: ${{ steps.eval-body.outputs.result }} | |
| feature-flags: ${{ steps.eval-feature-flags.outputs.result }} | |
| steps: | |
| - name: Debug | |
| run: | | |
| echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}" | |
| - name: Eval comment body for /testenv slash command | |
| uses: actions/github-script@v7 | |
| id: eval-body | |
| with: | |
| result-encoding: string | |
| script: | | |
| const pattern = /^\/testenv (up|down)/; | |
| const result = pattern.exec('${{ github.event.inputs.comment_body || github.event.comment.body }}'); | |
| return result === null ? 'noop' : result[1]; | |
| - name: Looking for feature flags | |
| uses: actions/github-script@v7 | |
| id: eval-feature-flags | |
| with: | |
| script: | | |
| const pattern = /FEATURE_(\w+)=(\w+)/g; | |
| let results = []; | |
| [...'${{ github.event.inputs.comment_body || github.event.comment.body }}'.matchAll(pattern)].forEach(match => { | |
| const config = { | |
| name: `SUPERSET_FEATURE_${match[1]}`, | |
| value: match[2], | |
| }; | |
| results.push(config); | |
| }); | |
| return results; | |
| - name: Limit to committers | |
| if: > | |
| steps.eval-body.outputs.result != 'noop' && | |
| github.event_name == 'issue_comment' && | |
| github.event.comment.author_association != 'MEMBER' && | |
| github.event.comment.author_association != 'OWNER' | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'; | |
| github.rest.issues.createComment({ | |
| issue_number: ${{ github.event.issue.number }}, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: errMsg | |
| }); | |
| core.setFailed(errMsg); | |
| - name: Reply with confirmation comment | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const issueNumber = ${{ github.event.inputs.issue_number || github.event.issue.number }}; | |
| const user = '${{ github.event.comment.user.login || github.actor }}'; | |
| const action = '${{ steps.eval-body.outputs.result }}'; | |
| const runId = context.runId; | |
| const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; | |
| const body = action === 'noop' | |
| ? `@${user} No ephemeral environment action detected. Please use '/testenv up' or '/testenv down'. [View workflow run](${workflowUrl}).` | |
| : `@${user} Processing your ephemeral environment request [here](${workflowUrl}).`; | |
| if (action !== 'noop') { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issueNumber, | |
| body, | |
| }); | |
| } | |
| ephemeral-docker-build: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build | |
| cancel-in-progress: true | |
| needs: ephemeral-env-comment | |
| name: ephemeral-docker-build | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Get Info from comment | |
| uses: actions/github-script@v7 | |
| id: get-pr-info | |
| with: | |
| script: | | |
| const request = { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, | |
| }; | |
| core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`); | |
| const pr = await github.rest.pulls.get(request); | |
| return pr.data; | |
| - name: Debug | |
| id: get-sha | |
| run: | | |
| echo "sha=${{ fromJSON(steps.get-pr-info.outputs.result).head.sha }}" >> $GITHUB_OUTPUT | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : ${{steps.get-sha.outputs.sha}} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.get-sha.outputs.sha }} | |
| persist-credentials: false | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Setup supersetbot | |
| uses: ./.github/actions/setup-supersetbot/ | |
| - name: Build ephemeral env image | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| supersetbot docker \ | |
| --preset ci \ | |
| --platform linux/amd64 \ | |
| --context-ref "$RELEASE" | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Load, tag and push image to ECR | |
| id: push-image | |
| env: | |
| ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| ECR_REPOSITORY: superset-ci | |
| IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci | |
| run: | | |
| docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci | |
| docker push -a $ECR_REGISTRY/$ECR_REPOSITORY | |
| ephemeral-env-up: | |
| needs: [ephemeral-env-comment, ephemeral-docker-build] | |
| if: needs.ephemeral-env-comment.outputs.slash-command == 'up' | |
| name: Spin up an ephemeral environment | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Check target image exists in ECR | |
| id: check-image | |
| continue-on-error: true | |
| run: | | |
| aws ecr describe-images \ | |
| --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ | |
| --repository-name superset-ci \ | |
| --image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci | |
| - name: Fail on missing container image | |
| if: steps.check-image.outcome == 'failure' | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'; | |
| github.rest.issues.createComment({ | |
| issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: errMsg | |
| }); | |
| core.setFailed(errMsg); | |
| - name: Fill in the new image ID in the Amazon ECS task definition | |
| id: task-def | |
| uses: aws-actions/amazon-ecs-render-task-definition@v1 | |
| with: | |
| task-definition: .github/workflows/ecs-task-definition.json | |
| container-name: superset-ci | |
| image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci | |
| - name: Update env vars in the Amazon ECS task definition | |
| run: | | |
| cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }} | |
| - name: Describe ECS service | |
| id: describe-services | |
| run: | | |
| echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT | |
| - name: Create ECS service | |
| id: create-service | |
| if: steps.describe-services.outputs.active != 'true' | |
| env: | |
| ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974 | |
| ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91 | |
| run: | | |
| aws ecs create-service \ | |
| --cluster superset-ci \ | |
| --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service \ | |
| --task-definition superset-ci \ | |
| --launch-type FARGATE \ | |
| --desired-count 1 \ | |
| --platform-version LATEST \ | |
| --network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \ | |
| --tags key=pr,value=${{ github.event.inputs.issue_number || github.event.issue.number }} key=github_user,value=${{ github.actor }} | |
| - name: Deploy Amazon ECS task definition | |
| id: deploy-task | |
| uses: aws-actions/amazon-ecs-deploy-task-definition@v2 | |
| with: | |
| task-definition: ${{ steps.task-def.outputs.task-definition }} | |
| service: pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | |
| cluster: superset-ci | |
| wait-for-service-stability: true | |
| wait-for-minutes: 10 | |
| - name: List tasks | |
| id: list-tasks | |
| run: | | |
| echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT | |
| - name: Get network interface | |
| id: get-eni | |
| run: | | |
| echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT | |
| - name: Get public IP | |
| id: get-ip | |
| run: | | |
| echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT | |
| - name: Comment (success) | |
| if: ${{ success() }} | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{github.token}} | |
| script: | | |
| github.rest.issues.createComment({ | |
| issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.' | |
| }) | |
| - name: Comment (failure) | |
| if: ${{ failure() }} | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{github.token}} | |
| script: | | |
| github.rest.issues.createComment({ | |
| issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.' | |
| }) |