chore(*): use etcd2 inside a container

Author: aledbf

Makefile

@@ -30,7 +30,7 @@ dev-cluster: discovery-url
 	deisctl install platform
 
 discovery-url:
-	sed -e "s,# discovery:,discovery:," -e "s,discovery: https://discovery.etcd.io/.*,discovery: $$(curl -s -w '\n' https://discovery.etcd.io/new)," contrib/coreos/user-data.example > contrib/coreos/user-data
+	sed -e "s,discovery #DISCOVERY_URL,discovery $$(curl -s -w '\n' https://discovery.etcd.io/new?size=$$DEIS_NUM_INSTANCES)," contrib/coreos/user-data.example > contrib/coreos/user-data
 
 build: check-docker
 	@$(foreach C, $(COMPONENTS), $(MAKE) -C $(C) build &&) echo done

Vagrantfile

@@ -90,7 +90,7 @@ Vagrant.configure("2") do |config|
   end
 
   config.trigger.before :up do
-    if !File.exists?(CLOUD_CONFIG_PATH) || File.readlines(CLOUD_CONFIG_PATH).grep(/#\s*discovery:/).any?
+    if !File.exists?(CLOUD_CONFIG_PATH) || File.readlines(CLOUD_CONFIG_PATH).grep(/\s*discovery #DISCOVERY_URL/).any?
       raise Vagrant::Errors::VagrantError.new, "Run 'make discovery-url' first to create user-data."
     end
   end

contrib/aws/gen-json.py

@@ -73,7 +73,7 @@
   Description=Prepares the etcd data directory
   Requires=media-etcd.mount
   After=media-etcd.mount
-  Before=etcd.service
+  Before=etcd2.service
   [Service]
   Type=oneshot
   RemainAfterExit=yes
@@ -101,12 +101,13 @@
 # are started
 data['coreos']['units'] = new_units + data['coreos']['units']
 
-# configure etcd to use its EBS volume
-data['coreos']['etcd']['data-dir'] = '/media/etcd'
-
 header = ["#cloud-config", "---"]
 dump = yaml.dump(data, default_flow_style=False)
 
+# configure etcd to use its EBS volume
+dump = dump.replace('-data-dir /var/lib/etcd2', '-data-dir /media/etcd')
+dump = dump.replace('--volume=/var/lib/etcd2', '--volume=/media/etcd')
+
 template = json.load(open(os.path.join(CURR_DIR, 'deis.template.json'), 'r'))
 
 template['Resources']['CoreOSServerLaunchConfig']['Properties']['UserData']['Fn::Base64']['Fn::Join'] = ["\n", header + dump.split("\n")]

contrib/aws/provision-aws-cluster.sh

@@ -136,17 +136,29 @@ ELB_NAME=$(aws elb describe-load-balancers \
     $EXTRA_AWS_CLI_ARGS | grep -F $ELB_DNS_NAME | head -n1 | cut -f2)
 echo "Using ELB $ELB_NAME at $ELB_DNS_NAME"
 
-echo_green "Your Deis cluster has been successfully deployed to AWS CloudFormation and is started."
-echo_green "Please continue to follow the instructions in the documentation."
-
 FIRST_INSTANCE=$(aws ec2 describe-instances \
     --filters Name=tag:aws:cloudformation:stack-name,Values=$STACK_NAME Name=instance-state-name,Values=running \
     --query 'Reservations[].Instances[].[PublicIpAddress]' \
     --output text \
     $EXTRA_AWS_CLI_ARGS | head -1)
 export DEISCTL_TUNNEL=$FIRST_INSTANCE
-echo_green "Enabling proxy protocol"
 
+# loop until etcd2 / fleet are up and running
+COUNTER=1
+until deisctl list >/dev/null; do
+    if [ $COUNTER -gt $ATTEMPTS ];
+        then echo_red "Timed out waiting for fleet, giving up"
+        break
+    fi
+    echo "Waiting until fleet is up and running ..."
+    sleep 5
+    let COUNTER=COUNTER+1
+done
+
+echo_green "Your Deis cluster has been successfully deployed to AWS CloudFormation and is started."
+echo_green "Please continue to follow the instructions in the documentation."
+
+echo_green "Enabling proxy protocol"
 if ! deisctl config router set proxyProtocol=1; then
     echo_red "#"
     echo_red "# Enabling proxy protocol failed, please enable proxy protocol "

contrib/azure/azure-coreos-cluster

@@ -68,7 +68,7 @@ coreos:
     addr: $private_ipv4:4001
     peer-addr: $private_ipv4:7001
   units:
-    - name: etcd.service
+    - name: etcd2.service
       command: start
     - name: fleet.service
       command: start

contrib/azure/create-azure-user-data

@@ -62,7 +62,7 @@ def main():
     configuration = combine_dicts(configuration_coreos_template,
                                   configuration_azure_template)
 
-    configuration["coreos"]["etcd"]["discovery"] = url
+    configuration["coreos"]["etcd2"]["discovery"] = url
 
     with azure_user_data as outfile:
         try:

contrib/coreos/user-data.example

@@ -1,16 +1,6 @@
 #cloud-config
 ---
 coreos:
-  etcd:
-    # generate a new token for each unique cluster from https://discovery.etcd.io/new
-    # uncomment the following line and replace it with your discovery URL
-    # discovery: https://discovery.etcd.io/12345693838asdfasfadf13939923
-    addr: $private_ipv4:4001
-    peer-addr: $private_ipv4:7001
-    # give etcd more time if it's under heavy load - prevent leader election thrashing
-    peer-election-timeout: 2000
-    # heartbeat interval should ideally be 1/4 or 1/5 of peer election timeout
-    peer-heartbeat-interval: 500
   fleet:
     # We have to set the public_ip here so this works on Vagrant -- otherwise, Vagrant VMs
     # will all publish the same private IP. This is harmless for cloud providers.
@@ -19,7 +9,41 @@ coreos:
     etcd_request_timeout: 3.0
   units:
   - name: etcd.service
+    mask: true
+  - name: etcd2.service
     command: start
+    content: |
+      [Unit]
+      Description=etcd2 container
+      Requires=early-docker.service
+      After=early-docker.service
+      Before=early-docker.target
+
+      [Service]
+      User=etcd
+      PermissionsStartOnly=true
+      Restart=always
+      RestartSec=10s
+      EnvironmentFile=/etc/environment
+      Environment="ETCD_IMAGE=quay.io/coreos/etcd:v2.0.13"
+      Environment="DOCKER_HOST=unix:///var/run/early-docker.sock"
+      LimitNOFILE=40000
+      ExecStartPre=/bin/sh -c "docker history $ETCD_IMAGE >/dev/null 2>&1 || docker pull $ETCD_IMAGE"
+      ExecStartPre=/bin/sh -c "docker inspect etcd2 >/dev/null 2>&1 && docker rm -f etcd2 || true"
+      ExecStart=/usr/bin/docker run --net=host --rm \
+        --volume=/var/lib/etcd2:/var/lib/etcd2 \
+        --volume=/usr/share/ca-certificates:/etc/ssl/certs:ro \
+        -p 4001:4001 -p 2380:2380 -p 2379:2379 -p 7001:7001 \
+        --name etcd2 \
+        ${ETCD_IMAGE} \
+        -name %m \
+        -data-dir /var/lib/etcd2 \
+        -advertise-client-urls http://${COREOS_PRIVATE_IPV4}:2379,http://${COREOS_PRIVATE_IPV4}:4001 \
+        -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 \
+        -initial-advertise-peer-urls http://${COREOS_PRIVATE_IPV4}:2380,http://${COREOS_PRIVATE_IPV4}:7001 \
+        -listen-peer-urls http://0.0.0.0:2380,http://0.0.0.0:7001 \
+        --discovery #DISCOVERY_URL
+      ExecStop=-/usr/bin/docker stop etcd2
   - name: docker-tcp.socket
     command: start
     enable: true
@@ -48,8 +72,8 @@ coreos:
       [Unit]
       Description=Clean up
       DefaultDependencies=no
-      After=fleet.service etcd.service docker.service docker.socket deis-store-admin.service deis-store-daemon.service deis-store-volume.service deis-store-monitor.service
-      Requires=fleet.service etcd.service deis-store-admin.service deis-store-daemon.service deis-store-volume.service docker.service docker.socket deis-store-monitor.service
+      After=fleet.service etcd2.service docker.service docker.socket deis-store-admin.service deis-store-daemon.service deis-store-volume.service deis-store-monitor.service
+      Requires=fleet.service etcd2.service deis-store-admin.service deis-store-daemon.service deis-store-volume.service docker.service docker.socket deis-store-monitor.service
 
       [Install]
       WantedBy=shutdown.target halt.target reboot.target

contrib/gce/create-gce-user-data

@@ -62,7 +62,7 @@ def main():
     configuration = combine_dicts(configuration_coreos_template,
                                   configuration_gce_template)
 
-    configuration["coreos"]["etcd"]["discovery"] = url
+    configuration["coreos"]["etcd2"]["discovery"] = url
 
     with gce_user_data as outfile:
         try:

contrib/linode/apply-firewall.py

@@ -47,7 +47,7 @@ def get_discovery_url_from_user_data():
         current_dir = os.path.dirname(__file__)
         user_data_file = file(os.path.abspath(os.path.join(current_dir, name)), 'r')
         user_data = yaml.safe_load(user_data_file)
-        return user_data['coreos']['etcd']['discovery']
+        return user_data['coreos']['etcd2']['discovery']
     except:
         raise IOError('Could not load discovery url from ' + name)
 

contrib/util/check-user-data.sh

@@ -26,8 +26,8 @@ function parse_yaml {
 }
 
 if [[ $NUM_INSTANCES -ne 1 ]] ; then
-    parse_yaml $USER_DATA | grep -q coreos_etcd_discovery
-    if [[ $? -ne 0 ]]; then
+    parse_yaml $USER_DATA | grep -q "#DISCOVERY_URL"
+    if [[ $? -ne 1 ]]; then
         echo "No etcd discovery URL set in $USER_DATA"
         exit 1
     fi