EMR

Add-step Example

aws emr add-steps \
    --cluster-id $CLUSTER_ID \
    --steps '[{
        "Name": "Example PySpark Job",
        "Type": "Spark",
        "ActionOnFailure": "CONTINUE",
        "Args": [
            "--deploy-mode", "cluster",
            "--master", "yarn",
            "--jars", "s3://path/to/definity-spark-agent-X-X.jar",
            "--conf", "spark.extraListeners=ai.definity.spark.AppListener",
            "--conf", "spark.definity.server=https://app.definity.run",
            "--conf", "spark.definity.api.token='$DEFINITY_API_TOKEN'",
            "--conf", "spark.definity.env.name=demo",
            "--conf", "spark.definity.pipeline.name=example_pipeline",
            "--conf", "spark.definity.pipeline.pit=2024-12-10",
            "--conf", "spark.definity.task.name=example_task",
            "--conf", "spark.executor.memory=3g",
            "--conf", "spark.executor.cores=3",
            "s3://path/to/task.py"
        ]
    }]'