add timeout/attempts to util.connect_ssh, use 120s by default

Author: Gabriel Monroy

celerytasks/ec2.py

@@ -103,11 +103,9 @@ def launch_node(node_id, creds, params, init, ssh_username, ssh_private_key):
     node.fqdn = boto.public_dns_name
     node.metadata = format_metadata(boto)
     node.save()
-    # wait 10 seconds for ssh daemon to come up
-    time.sleep(10)
     # loop until cloud-init is finished
     ssh = util.connect_ssh(ssh_username, boto.public_dns_name, 22,
-                           ssh_private_key)
+                           ssh_private_key, timeout=120)
     initializing = True
     while initializing:
         time.sleep(10)

celerytasks/util.py

@@ -7,19 +7,21 @@
 import paramiko
 
 
-def connect_ssh(username, hostname, port, key):
+def connect_ssh(username, hostname, port, key,
+                timeout=120, attempts=10):
     key_f = StringIO.StringIO(key)
     pkey = paramiko.RSAKey.from_private_key(key_f)
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    for _ in range(10):
+    interval = int(timeout / attempts)
+    for _ in range(attempts):
         try:
             ssh.connect(hostname, username=username, pkey=pkey)
             break
         except paramiko.AuthenticationException as e:
             raise paramiko.AuthenticationException(e)
         except socket.error:
-            time.sleep(3)
+            time.sleep(interval)
     else:
         raise RuntimeError('SSH Connection Error')
     return ssh