I have been unable to determine where the issue with my state definition is. I am able to successfully create the machine in AWS however the validator keeps giving a unspecified error.
{
"Comment": "A step function for Decision Engine flow",
"StartAt": "Pass Component Name",
"States": {
"Pass Component Name": {
"Type": "Pass",
"Result": "emr-handler-steps-dev-east",
"ResultPath": "$.message.componentName",
"Next": "Create EMR"
},
"Create EMR": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:123456789012:function:emr-handler-lambda-dev-east",
"ResultPath": "$.jobFlowId",
"Next": "Pass Job Flow ID",
"Parameters": {
"action": "create-emr",
"emrConfig.$": "$.emrConfig"
},
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 300,
"MaxAttempts": 4,
"BackoffRate": 1
}
],
"Catch": [
{
"ErrorEquals": [
"States.ALL"
],
"ResultPath": "$.message.message.error-info",
"Next": "EMR Failure"
}
]
},
"Pass Job Flow ID": {
"Type": "Pass",
"InputPath": "$.jobFlowId",
"ResultPath": "$.message.message.jobFlowId",
"Next": "Wait 4 Minutes"
},
"Wait 4 Minutes": {
"Type": "Wait",
"Seconds": 240,
"Next": "Get Cluster Status"
},
"Get Cluster Status": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:123456789012:function:emr-handler-lambda-dev-east",
"Parameters": {
"action": "is-emr-ready",
"jobFlowId.$": "$.jobFlowId.jobFlowId"
},
"ResultPath": "$.isClusterReady",
"Next": "Is EMR Ready",
"Retry": [
{
"ErrorEquals": [
"Lambda.ServiceException",
"Lambda.SdkClientException",
"Lambda.EC2ThrottledException",
"Lambda.EC2UnexpectedException",
"Lambda.TooManyRequestsException",
"Lambda.Unknown",
"ClientError"
],
"IntervalSeconds": 15,
"MaxAttempts": 6,
"BackoffRate": 2
},
{
"ErrorEquals": [
"EMRNotReadyException"
],
"IntervalSeconds": 300,
"BackoffRate": 1,
"MaxAttempts": 12
}
],
"Catch": [
{
"ErrorEquals": [
"EMRNotReadyException",
"EMRFailedException",
"ClientError"
],
"ResultPath": "$.message.message.error-info",
"Next": "EMR Failure"
}
]
},
"Is EMR Ready": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.isClusterReady.emr-status",
"StringEquals": "READY",
"Next": "Submit Steps"
},
{
"Variable": "$.isClusterReady.emr-status",
"StringEquals": "FAILED",
"Next": "EMR Failure"
}
]
},
"Submit Steps": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:123456789012:function:emr-handler-lambda-dev-east",
"Next": "Pass Step IDs",
"Parameters": {
"action": "submit-steps",
"jobFlowId.$": "$.jobFlowId.jobFlowId",
"steps.$": "$.steps"
},
"ResultPath": "$.stepIds",
"Retry": [
{
"ErrorEquals": [
"Lambda.ServiceException",
"Lambda.SdkClientException",
"Lambda.EC2ThrottledException",
"Lambda.EC2UnexpectedException",
"Lambda.Unknown",
"ClientError"
],
"IntervalSeconds": 15,
"MaxAttempts": 6,
"BackoffRate": 2
}
],
"Catch": [
{
"ErrorEquals": [
"States.ALL"
],
"ResultPath": "$.message.message.error-info",
"Next": "EMR Failure"
}
]
},
"Pass Step IDs": {
"Type": "Pass",
"InputPath": "$.stepIds",
"ResultPath": "$.message.message.stepIds",
"Next": "Step Status Check"
},
"Step Status Check": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:123456789012:function:emr-handler-lambda-dev-east",
"Parameters": {
"action": "are-steps-done",
"jobFlowId.$": "$.jobFlowId.jobFlowId",
"stepIds.$": "$.stepIds.stepIds"
},
"Retry": [
{
"ErrorEquals": [
"StepsIncompleteException"
],
"IntervalSeconds": 300,
"BackoffRate": 1,
"MaxAttempts": 12
},
{
"ErrorEquals": [
"Lambda.ServiceException",
"Lambda.SdkClientException",
"Lambda.EC2ThrottledException",
"Lambda.EC2UnexpectedException",
"Lambda.TooManyRequestsException",
"Lambda.Unknown",
"ClientError"
],
"IntervalSeconds": 15,
"MaxAttempts": 6,
"BackoffRate": 2
}
],
"Catch": [
{
"ErrorEquals": [
"StepsIncompleteException",
"StepsFailedException",
"ClientError"
],
"ResultPath": "$.message.message.error-info",
"Next": "EMR Failure"
}
],
"ResultPath": "$.areStepsDone",
"Next": "Are Steps Done"
},
"Are Steps Done": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.areStepsDone.are-steps-done",
"StringEquals": "YES",
"Next": "EMR Success"
},
{
"Variable": "$.areStepsDone.are-steps-done",
"StringEquals": "FAILED",
"Next": "EMR Failure"
}
]
},
"EMR Failure": {
"Type": "Pass",
"Result": "failure",
"ResultPath": "$.state",
"Next": "Notify Failure"
},
"Destroy EMR": {
"Type": "Task",
"Resource": "arn:aws:lambda:us-east-1:123456789012:function:emr-handler-lambda-dev-east",
"Parameters": {
"action": "terminate-emr",
"jobFlowId.$": "$.jobFlowId.jobFlowId"
},
"ResultPath": "$.destroy-status",
"Next": "Check failure",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 60,
"MaxAttempts": 5,
"BackoffRate": 1
}
]
},
"EMR Success": {
"Type": "Pass",
"Result": "success",
"ResultPath": "$.state",
"Next": "Notify Success"
},
"Check failure": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.destroy-status.emr-status",
"StringEquals": "TERMINATED",
"Next": "Pipeline successful"
}
],
"Default": "Pipeline failed"
},
"Pipeline failed": {
"Type": "Fail",
"Cause": "Pipeline failure",
"Error": "FailedException"
},
"Pipeline successful": {
"Type": "Succeed"
},
"Notify Success": {
"Type": "Task",
"Resource": "arn:aws:states:::sns:publish",
"Parameters": {
"TopicArn": "arn:aws:sns:us-east-1:123456789012:emr-handler-result-dev-east",
"Subject": " dev-east reconciliation success. ✅",
"Message.$": "$.message",
"MessageAttributes": {
"severity": {
"DataType": "String",
"StringValue": "INFO"
}
},
"MessageStructure": "String"
},
"ResultPath": "$.snsresult",
"Next": "Destroy EMR"
},
"Notify Failure": {
"Type": "Task",
"Resource": "arn:aws:states:::sns:publish",
"Parameters": {
"TopicArn": "arn:aws:sns:us-east-1:123456789012:emr-handler-result-dev-east",
"Subject": " dev-east reconciliation failure. ❌",
"Message.$": "$.message",
"MessageAttributes": {
"severity": {
"DataType": "String",
"StringValue": "ERROR"
}
},
"MessageStructure": "String"
},
"ResultPath": "$.snsresult",
"Next": "Destroy EMR"
}
}
}