pi-argo-workflows.yaml

apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  name: spark-operator
  namespace: default
spec:
  arguments: {}
  entrypoint: demo-workflow
  serviceAccountName: argo-workflows
  templates:
    - name: demo-workflow
      steps:
        - - name: sleep
            template: sleep
        - - name: spark-operator
            template: sparkapp
    - name: sleep
      container:
        image: docker/whalesay
        command: [ sleep ]
        args: [ "60" ]
    - name: sparkapp
      resource:
        action: create
        setOwnerReference: true
        successCondition: status.applicationState.state == COMPLETED
        failureCondition: status.applicationState.state in (FAILED, ERROR)
        manifest: |
          apiVersion: "sparkoperator.k8s.io/v1beta2"
          kind: SparkApplication
          metadata:
            generateName: pyspark-pi-
            namespace: default
          spec:
            type: Python
            pythonVersion: "3"
            mode: cluster
            image: "public.ecr.aws/r1l5w1y9/spark-operator:3.2.1-hadoop-3.3.1-java-11-scala-2.12-python-3.8-latest"
            mainApplicationFile: "local:///opt/spark/examples/src/main/python/pi.py"
            sparkVersion: "3.1.1"
            restartPolicy:
              type: OnFailure
              onFailureRetries: 1
              onFailureRetryInterval: 10
              onSubmissionFailureRetries: 1
              onSubmissionFailureRetryInterval: 20
            driver:
              cores: 1
              coreLimit: "1200m"
              memory: "512m"
              labels:
                version: 3.1.1
              serviceAccount: spark
            executor:
              cores: 1
              instances: 2
              memory: "512m"
              serviceAccount: spark
              labels:
                version: 3.1.1