Merge pull request #3250 from mboersma/ec2-provision-polling

feat(contrib/ec2): loop until instances have passed health checks

Author: mboersma

contrib/ec2/provision-ec2-cluster.sh

@@ -5,9 +5,9 @@
 
 if [ -z "$1" ]
   then
-    NAME=deis
+    STACK_NAME=deis
   else
-    NAME=$1
+    STACK_NAME=$1
 fi
 
 set -e
@@ -41,8 +41,60 @@ $CONTRIB_DIR/util/check-user-data.sh
 # create an EC2 cloudformation stack based on CoreOS's default template
 aws cloudformation create-stack \
     --template-body "$($THIS_DIR/gen-json.py)" \
-    --stack-name $NAME \
+    --stack-name $STACK_NAME \
     --parameters "$(<$THIS_DIR/cloudformation.json)"
 
-echo_green "Your Deis cluster has been successfully deployed to AWS CloudFormation and is currently starting."
+# loop until the instances are created
+ATTEMPTS=45
+SLEEPTIME=10
+COUNTER=1
+INSTANCE_IDS=""
+until [ `wc -w <<< $INSTANCE_IDS` -eq $DEIS_NUM_INSTANCES ]; do
+    if [ $COUNTER -gt $ATTEMPTS ]; then exit 1; fi  # timeout after 7 1/2 minutes
+    if [ $COUNTER -ne 1 ]; then sleep $SLEEPTIME; fi
+    echo "Waiting for instances to be created..."
+    INSTANCE_IDS=$(aws ec2 describe-instances \
+        --filters Name=tag:aws:cloudformation:stack-name,Values=$STACK_NAME Name=instance-state-name,Values=running \
+        --query 'Reservations[].Instances[].[ InstanceId ]' \
+        --output text)
+    let COUNTER=COUNTER+1
+done
+
+# loop until the instances pass health checks
+COUNTER=1
+INSTANCE_STATUSES=""
+until [ `wc -w <<< $INSTANCE_STATUSES` -eq $DEIS_NUM_INSTANCES ]; do
+    if [ $COUNTER -gt $ATTEMPTS ]; then exit 1; fi  # timeout after 7 1/2 minutes
+    if [ $COUNTER -ne 1 ]; then sleep $SLEEPTIME; fi
+    echo "Waiting for instances to pass initial health checks..."
+    INSTANCE_STATUSES=$(aws ec2 describe-instance-status \
+        --filters Name=instance-status.reachability,Values=passed \
+        --instance-ids $INSTANCE_IDS \
+        --query 'InstanceStatuses[].[ InstanceId ]' \
+        --output text)
+    let COUNTER=COUNTER+1
+done
+
+# print instance info
+echo "Instances are available:"
+aws ec2 describe-instances \
+    --filters Name=tag:aws:cloudformation:stack-name,Values=$STACK_NAME Name=instance-state-name,Values=running \
+    --query 'Reservations[].Instances[].[InstanceId,PublicIpAddress,InstanceType,Placement.AvailabilityZone,State.Name]' \
+    --output text
+
+# get ELB public DNS name through cloudformation
+# TODO: is "first output value" going to be reliable enough?
+export ELB_DNS_NAME=$(aws cloudformation describe-stacks \
+    --stack-name $STACK_NAME \
+    --max-items 1 \
+    --query 'Stacks[].[ Outputs[0].[ OutputValue ] ]' \
+    --output=text)
+
+# get ELB friendly name through aws elb
+ELB_NAME=$(aws elb describe-load-balancers \
+    --query 'LoadBalancerDescriptions[].[ DNSName,LoadBalancerName ]' \
+    --output=text | grep -F $ELB_DNS_NAME | head -n1 | cut -f2)
+echo "Using ELB $ELB_NAME at $ELB_DNS_NAME"
+
+echo_green "Your Deis cluster has been successfully deployed to AWS CloudFormation and is started."
 echo_green "Please continue to follow the instructions in the documentation."