Merge pull request #4090 from mboersma/etcd2-min

chore(*): use etcd2 inside a container, extended dance remix

Author: mboersma

Makefile

@@ -30,7 +30,7 @@ dev-cluster: discovery-url
 	deisctl install platform
 
 discovery-url:
-	sed -e "s,# discovery:,discovery:," -e "s,discovery: https://discovery.etcd.io/.*,discovery: $$(curl -s -w '\n' https://discovery.etcd.io/new)," contrib/coreos/user-data.example > contrib/coreos/user-data
+	sed -e "s,discovery #DISCOVERY_URL,discovery $$(curl -s -w '\n' https://discovery.etcd.io/new?size=$$DEIS_NUM_INSTANCES)," contrib/coreos/user-data.example > contrib/coreos/user-data
 
 build: check-docker
 	@$(foreach C, $(COMPONENTS), $(MAKE) -C $(C) build &&) echo done

Vagrantfile

@@ -90,7 +90,7 @@ Vagrant.configure("2") do |config|
   end
 
   config.trigger.before :up do
-    if !File.exists?(CLOUD_CONFIG_PATH) || File.readlines(CLOUD_CONFIG_PATH).grep(/#\s*discovery:/).any?
+    if !File.exists?(CLOUD_CONFIG_PATH) || File.readlines(CLOUD_CONFIG_PATH).grep(/\s*discovery #DISCOVERY_URL/).any?
       raise Vagrant::Errors::VagrantError.new, "Run 'make discovery-url' first to create user-data."
     end
   end

contrib/aws/gen-json.py

@@ -101,12 +101,13 @@
 # are started
 data['coreos']['units'] = new_units + data['coreos']['units']
 
-# configure etcd to use its EBS volume
-data['coreos']['etcd']['data-dir'] = '/media/etcd'
-
 header = ["#cloud-config", "---"]
 dump = yaml.dump(data, default_flow_style=False)
 
+# configure etcd to use its EBS volume
+dump = dump.replace('-data-dir /var/lib/etcd2', '-data-dir /media/etcd')
+dump = dump.replace('--volume=/var/lib/etcd2', '--volume=/media/etcd')
+
 template = json.load(open(os.path.join(CURR_DIR, 'deis.template.json'), 'r'))
 
 template['Resources']['CoreOSServerLaunchConfig']['Properties']['UserData']['Fn::Base64']['Fn::Join'] = ["\n", header + dump.split("\n")]

contrib/aws/provision-aws-cluster.sh

@@ -136,17 +136,29 @@ ELB_NAME=$(aws elb describe-load-balancers \
     $EXTRA_AWS_CLI_ARGS | grep -F $ELB_DNS_NAME | head -n1 | cut -f2)
 echo "Using ELB $ELB_NAME at $ELB_DNS_NAME"
 
-echo_green "Your Deis cluster has been successfully deployed to AWS CloudFormation and is started."
-echo_green "Please continue to follow the instructions in the documentation."
-
 FIRST_INSTANCE=$(aws ec2 describe-instances \
     --filters Name=tag:aws:cloudformation:stack-name,Values=$STACK_NAME Name=instance-state-name,Values=running \
     --query 'Reservations[].Instances[].[PublicIpAddress]' \
     --output text \
     $EXTRA_AWS_CLI_ARGS | head -1)
 export DEISCTL_TUNNEL=$FIRST_INSTANCE
-echo_green "Enabling proxy protocol"
 
+# loop until etcd2 / fleet are up and running
+COUNTER=1
+until deisctl list >/dev/null; do
+    if [ $COUNTER -gt $ATTEMPTS ];
+        then echo_red "Timed out waiting for fleet, giving up"
+        break
+    fi
+    echo "Waiting until fleet is up and running ..."
+    sleep 5
+    let COUNTER=COUNTER+1
+done
+
+echo_green "Your Deis cluster has been successfully deployed to AWS CloudFormation and is started."
+echo_green "Please continue to follow the instructions in the documentation."
+
+echo_green "Enabling proxy protocol"
 if ! deisctl config router set proxyProtocol=1; then
     echo_red "#"
     echo_red "# Enabling proxy protocol failed, please enable proxy protocol "

contrib/azure/create-azure-user-data

@@ -62,11 +62,12 @@ def main():
     configuration = combine_dicts(configuration_coreos_template,
                                   configuration_azure_template)
 
-    configuration["coreos"]["etcd"]["discovery"] = url
+    dump = yaml.dump(configuration, default_flow_style=False)
+    dump = dump.replace('#DISCOVERY_URL', url)
 
     with azure_user_data as outfile:
         try:
-            outfile.write("#cloud-config\n\n" + yaml.dump(configuration, default_flow_style=False))
+            outfile.write("#cloud-config\n\n" + dump)
         except (IOError, ValueError):
             print "There was an issue writing to file " + azure_user_data.name
             return 1

contrib/coreos/user-data.example

@@ -1,16 +1,6 @@
 #cloud-config
 ---
 coreos:
-  etcd:
-    # generate a new token for each unique cluster from https://discovery.etcd.io/new
-    # uncomment the following line and replace it with your discovery URL
-    # discovery: https://discovery.etcd.io/12345693838asdfasfadf13939923
-    addr: $private_ipv4:4001
-    peer-addr: $private_ipv4:7001
-    # give etcd more time if it's under heavy load - prevent leader election thrashing
-    peer-election-timeout: 2000
-    # heartbeat interval should ideally be 1/4 or 1/5 of peer election timeout
-    peer-heartbeat-interval: 500
   fleet:
     # We have to set the public_ip here so this works on Vagrant -- otherwise, Vagrant VMs
     # will all publish the same private IP. This is harmless for cloud providers.
@@ -20,6 +10,46 @@ coreos:
   units:
   - name: etcd.service
     command: start
+    content: |
+      [Unit]
+      Description=etcd2 container
+      Requires=early-docker.service
+      After=early-docker.service
+      Before=early-docker.target
+
+      [Service]
+      User=etcd
+      PermissionsStartOnly=true
+      Restart=always
+      RestartSec=10s
+      LimitNOFILE=40000
+      EnvironmentFile=/etc/environment
+      Environment="ETCD_IMAGE=quay.io/coreos/etcd:v2.0.13"
+      Environment="ETCD_ELECTION_TIMEOUT=2000"
+      Environment="ETCD_HEARTBEAT_INTERVAL=400"
+      Environment="ETCD_DATA_DIR=/var/lib/etcd2"
+      Environment="ETCD_NAME=%m"
+      Environment="DOCKER_HOST=unix:///var/run/early-docker.sock"
+      ExecStartPre=/bin/sh -c "docker history $ETCD_IMAGE >/dev/null 2>&1 || docker pull $ETCD_IMAGE"
+      ExecStartPre=/bin/sh -c "docker inspect $ETCD_NAME >/dev/null 2>&1 && docker rm -f $ETCD_NAME || true"
+      ExecStart=/usr/bin/docker run --net=host --rm \
+        --volume=${ETCD_DATA_DIR}:/var/lib/etcd2 \
+        --volume=/usr/share/ca-certificates:/etc/ssl/certs:ro \
+        -p 4001:4001 -p 2380:2380 -p 2379:2379 -p 7001:7001 \
+        --name ${ETCD_NAME} \
+        ${ETCD_IMAGE} \
+        -name ${ETCD_NAME} \
+        -data-dir /var/lib/etcd2 \
+        -advertise-client-urls http://${COREOS_PRIVATE_IPV4}:2379,http://${COREOS_PRIVATE_IPV4}:4001 \
+        -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 \
+        -initial-advertise-peer-urls http://${COREOS_PRIVATE_IPV4}:2380,http://${COREOS_PRIVATE_IPV4}:7001 \
+        -listen-peer-urls http://0.0.0.0:2380,http://0.0.0.0:7001 \
+        --heartbeat-interval ${ETCD_HEARTBEAT_INTERVAL} \
+        --election-timeout ${ETCD_ELECTION_TIMEOUT} \
+        --discovery #DISCOVERY_URL
+      ExecStop=-/usr/bin/docker stop $ETCD_NAME
+  - name: etcd2.service
+    mask: true
   - name: docker-tcp.socket
     command: start
     enable: true
@@ -244,9 +274,16 @@ write_files:
         /usr/bin/docker exec deis-store-admin ceph mon remove `hostname -f` # fixme
         /usr/bin/docker stop deis-store-metadata || true
       fi
-      NODE=$(curl -L http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`)
-      # remove from etcd cluster
-      if [ $NODE != 'null' ]; then
-        /usr/bin/curl -L -XDELETE http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`
+
+      # TODO: remove the next check once etcdctl is using etcd2
+      ETCDCTL=/usr/bin/etcdctl
+      if ! $ETCDCTL --version | grep -q "etcdctl version 2.0."; then
+        ETCD_VERSION=2.0.13
+        curl -sSL https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz | \
+          tar zxv -C /opt/bin etcd-v${ETCD_VERSION}-linux-amd64/etcdctl --strip=1
+        ETCDCTL="/opt/bin/etcdctl"
       fi
+      # removing the node from etcd
+      NODE=$($ETCDCTL member list | grep `cat /etc/machine-id` | cut -d ':' -f 1)
+      $ETCDCTL member remove $NODE
 manage_etc_hosts: localhost

contrib/gce/create-gce-user-data

@@ -62,14 +62,13 @@ def main():
     configuration = combine_dicts(configuration_coreos_template,
                                   configuration_gce_template)
 
-    configuration["coreos"]["etcd"]["discovery"] = url
+    dump = yaml.dump(configuration, default_flow_style=False,
+                     default_style='|')
+    dump = dump.replace('#DISCOVERY_URL', url)
 
     with gce_user_data as outfile:
         try:
-            outfile.write("#cloud-config\n\n" +
-                          yaml.safe_dump(configuration,
-                                         default_flow_style=False,
-                                         default_style='|'))
+            outfile.write("#cloud-config\n\n" + dump)
         except (IOError, ValueError):
             print "There was an issue writing to file " + gce_user_data.name
             return 1

contrib/linode/apply-firewall.py

@@ -46,8 +46,7 @@ def get_discovery_url_from_user_data():
     try:
         current_dir = os.path.dirname(__file__)
         user_data_file = file(os.path.abspath(os.path.join(current_dir, name)), 'r')
-        user_data = yaml.safe_load(user_data_file)
-        return user_data['coreos']['etcd']['discovery']
+        return re.search('--discovery (http\S+)', user_data_file.read()).group(1)
     except:
         raise IOError('Could not load discovery url from ' + name)
 

contrib/linode/create-linode-user-data.py

@@ -71,11 +71,13 @@ def main():
     configuration_coreos_template = yaml.safe_load(coreos_template)
 
     configuration = combine_dicts(configuration_coreos_template, configuration_linode_template)
-    configuration['coreos']['etcd']['discovery'] = 'https://discovery.etcd.io/' + str(etcd_token)
     configuration['ssh_authorized_keys'] = public_keys
 
+    dump = yaml.dump(configuration, default_flow_style=False, default_style='|')
+    dump = dump.replace('#DISCOVERY_URL', 'https://discovery.etcd.io/' + str(etcd_token))
+
     with linode_user_data as outfile:
-        outfile.write("#cloud-config\n\n" + yaml.safe_dump(configuration, default_flow_style=False, default_style='|'))
+        outfile.write("#cloud-config\n\n" + dump)
         log_success('Wrote Linode user data to ' + linode_user_data.name)
 
 

contrib/util/check-user-data.sh

@@ -26,8 +26,8 @@ function parse_yaml {
 }
 
 if [[ $NUM_INSTANCES -ne 1 ]] ; then
-    parse_yaml $USER_DATA | grep -q coreos_etcd_discovery
-    if [[ $? -ne 0 ]]; then
+    parse_yaml $USER_DATA | grep -q "#DISCOVERY_URL"
+    if [[ $? -ne 1 ]]; then
         echo "No etcd discovery URL set in $USER_DATA"
         exit 1
     fi