Skip to content

Commit 757a4e1

Browse files
Sivaram Mothikimboersma
authored andcommitted
feat(swarm): initial installation
feat(swarm): add init scripts and fleet templates for swarm node and manager feat(swarm): fix dockerfile and fleet templates feat(swarm): modify variable names feat(swarm): add swarm.py controller module fix(swarm): fix swarm init() fix(swarm): change error1 str to int feat(swarm): add docker tcp socket service rectify errors fix(builder): hardcode overlay storage driver feat(swarm): fix errors and fleet swarm template files
1 parent f4c2a24 commit 757a4e1

10 files changed

Lines changed: 278 additions & 2 deletions

File tree

builder/image/bin/boot

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ DOCKER_PID=$!
6666

6767
# wait for docker to start
6868
while [[ ! -e /var/run/docker.sock ]]; do
69-
sleep 1
69+
sleep 1
7070
done
7171

7272
# build required images

contrib/coreos/user-data.example

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,21 @@ coreos:
3232
ExecStart=/usr/bin/bash -c 'if fleetd --version | grep -q 0.9.1; then curl -sSL --retry 5 --retry-delay 2 -o /opt/bin/fleetd-0.9.2 https://s3-us-west-2.amazonaws.com/opdemand/fleetd-v0.9.2 && chmod +x /opt/bin/fleetd-0.9.2 && mkdir -p /etc/systemd/system/fleet.service.d/ && ln -sf /opt/conf/fleetd-092-custom-binary.conf /etc/systemd/system/fleet.service.d/99-upgrade-fleet-091.conf; else rm -f /etc/systemd/system/fleet.service.d/99-upgrade-fleet-091.conf; fi'
3333
RemainAfterExit=yes
3434
Type=oneshot
35+
- name: docker.service
36+
command: restart
37+
content: |
38+
[Unit]
39+
Description=Docker Application Container Engine
40+
Documentation=http://docs.docker.io
41+
After=network.target
42+
[Service]
43+
ExecStartPre=/bin/mount --make-rprivate /
44+
# Run docker but don't have docker automatically restart
45+
# containers. This is a job for systemd and unit files.
46+
ExecStart=/usr/bin/docker -d -H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock -D
47+
48+
[Install]
49+
WantedBy=multi-user.target
3550
- name: stop-update-engine.service
3651
command: start
3752
content: |
@@ -113,7 +128,7 @@ write_files:
113128
content: |
114129
[Service]
115130
EnvironmentFile=/etc/environment_proxy
116-
Environment="DOCKER_OPTS=--insecure-registry 10.0.0.0/8 --insecure-registry 172.16.0.0/12 --insecure-registry 192.168.0.0/16 --insecure-registry 100.64.0.0/10"
131+
Environment="DOCKER_OPTS=--insecure-registry 10.0.0.0/8 --insecure-registry 172.16.0.0/12 --insecure-registry 192.168.0.0/16 --insecure-registry 100.64.0.0/10 --insecure-registry registry.smothiki.io:5000"
117132
- path: /run/deis/bin/get_image
118133
permissions: '0755'
119134
content: |

controller/conf.d/confd_settings.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@ keys = [
1010
"/deis/registry",
1111
"/deis/domains",
1212
"/deis/platform",
13+
"/deis/scheduler/swarm/host",
1314
]
1415
reload_cmd = "/app/bin/reload"

controller/scheduler/swarm.py

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import logging
2+
from docker import Client
3+
4+
from django.conf import settings
5+
6+
7+
8+
class SwarmClient(object):
9+
def __init__(self,target, auth, options, pkey):
10+
self.target = settings.SWARM_HOST
11+
# single global connection
12+
self.registry = settings.REGISTRY_HOST+":"+settings.REGISTRY_PORT
13+
self.docker_cli = Client(base_url='tcp://'+self.target+':'+"2395",timeout=1200)
14+
15+
def create(self, name, image, command='', template=None, **kwargs):
16+
"""Create a container"""
17+
cimage=self.registry+"/"+image
18+
cname=name
19+
ccommand=command
20+
# self.docker_cli.pull(cimage, stream=False,insecure_registry=True)
21+
self.docker_cli.create_container(image=cimage,name=cname,command=ccommand)#,hostname=self._get_hostname(cname),ports=self._get_ports(cimage))
22+
self.docker_cli.start(cname, port_bindings=self._get_portbindings(cimage),publish_all_ports=True)
23+
24+
def start(self, name):
25+
"""
26+
Start a container
27+
"""
28+
self.docker_cli.start(name)
29+
return
30+
31+
def stop(self, name):
32+
"""
33+
Stop a container
34+
"""
35+
self.docker_cli.stop(name)
36+
return
37+
def destroy(self, name):
38+
"""
39+
Destroy a container
40+
"""
41+
self.docker_cli.stop(name)
42+
self.docker_cli.remove_container(name)
43+
return
44+
45+
def run(self, name, image, entrypoint, command):
46+
"""
47+
Run a one-off command
48+
"""
49+
# dump input into a json object for testing purposes
50+
return 0, json.dumps({'name': name,
51+
'image': image,
52+
'entrypoint': entrypoint,
53+
'command': command})
54+
55+
def attach(self, name):
56+
"""
57+
Attach to a job's stdin, stdout and stderr
58+
"""
59+
return StringIO(), StringIO(), StringIO()
60+
61+
def _get_hostname(self, application_name):
62+
hostname = settings.UNIT_HOSTNAME
63+
if hostname == "default":
64+
return ''
65+
elif hostname == "application":
66+
# replace underscore with dots, since underscore is not valid in DNS hostnames
67+
dns_name = application_name.replace("_", ".")
68+
return dns_name
69+
elif hostname == "server":
70+
raise NotImplementedError
71+
else:
72+
raise RuntimeError('Unsupported hostname: ' + hostname)
73+
74+
def _get_portbindings(self,image):
75+
dictports=self.docker_cli.inspect_image(image)["ContainerConfig"]["ExposedPorts"]
76+
for port,mapping in dictports.items():
77+
dictports[port]=None
78+
return dictports
79+
80+
def _get_ports(self,image):
81+
ports=[]
82+
dictports=self.docker_cli.inspect_image(image)["ContainerConfig"]["ExposedPorts"]
83+
for port,mapping in dictports.items():
84+
ports.append(int(port.split('/')[0]))
85+
return ports
86+
87+
SchedulerClient = SwarmClient

controller/templates/confd_settings.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
except:
1111
SCHEDULER_OPTIONS = {}
1212

13+
# scheduler swarm manager host
14+
SWARM_HOST = """{{ or (.deis_scheduler_swarm_host) "" }}"""
15+
1316
# base64-encoded SSH private key to facilitate current version of "deis run"
1417
SSH_PRIVATE_KEY = """{{ if exists "/deis/platform/sshPrivateKey" }}{{ getv "/deis/platform/sshPrivateKey" }}{{ else }}""{{end}}"""
1518

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[Unit]
2+
Description=deis-cache
3+
4+
[Service]
5+
EnvironmentFile=/etc/environment
6+
TimeoutStartSec=20m
7+
ExecStartPre=/bin/sh -c "docker history smothiki/swarm >/dev/null || docker pull smothiki/swarm"
8+
ExecStartPre=/bin/sh -c "docker inspect deis-swarm-manager >/dev/null && docker rm -f deis-swarm-manager || true"
9+
ExecStart=/bin/sh -c "docker run -p 2395:2375 --name deis-swarm-manager -e HOST=$COREOS_PRIVATE_IPV4 smothiki/swarm manage"
10+
ExecStopPost=-/usr/bin/docker rm -f deis-swarm-manager
11+
Restart=on-failure
12+
RestartSec=5
13+
14+
[Install]
15+
WantedBy=multi-user.target
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[Unit]
2+
Description=deis-cache
3+
4+
[Service]
5+
EnvironmentFile=/etc/environment
6+
TimeoutStartSec=20m
7+
ExecStartPre=/bin/sh -c "docker history smothiki/swarm >/dev/null || docker pull smothiki/swarm "
8+
ExecStartPre=/bin/sh -c "docker inspect deis-swarm-node >/dev/null && docker rm -f deis-swarm-node || true"
9+
ExecStart=/bin/sh -c "docker run --name deis-swarm-node -e HOST=$COREOS_PRIVATE_IPV4 smothiki/swarm join"
10+
ExecStopPost=-/usr/bin/docker rm -f deis-swarm-node
11+
Restart=on-failure
12+
RestartSec=5
13+
14+
[Install]
15+
WantedBy=multi-user.target
16+
17+
[X-Fleet]
18+
Global=true

swarm/Dockerfile

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM golang:1.3
2+
WORKDIR /go/src/github.com/deis
3+
RUN git clone https://github.com/smothiki/deis
4+
WORKDIR /go/src/github.com/deis/deis
5+
RUN git fetch origin swarm
6+
RUN git checkout swarm
7+
WORKDIR /go/src/github.com/deis/deis/swarm
8+
ENV GOPATH /go/src/github.com/deis/deis/Godeps/_workspace:$GOPATH
9+
RUN CGO_ENABLED=0 go install -v -a "github.com/deis/deis/swarm"
10+
ENV SWARM_HOST :2375
11+
EXPOSE 2375
12+
RUN wget https://github.com/docker/swarm/releases/download/v0.1.0-rc2/docker-swarm_linux-amd64
13+
RUN mv docker-swarm_linux-amd64 deis-swarm
14+
RUN chmod 777 deis-swarm
15+
VOLUME $HOME/.deis-swarm
16+
ENTRYPOINT ["swarm"]
17+
CMD ["--help"]

swarm/swarm.go

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"io/ioutil"
8+
"log"
9+
"net/http"
10+
"os"
11+
"os/exec"
12+
"strings"
13+
"text/template"
14+
"time"
15+
16+
"github.com/coreos/go-etcd/etcd"
17+
"github.com/deis/deis/tests/utils"
18+
)
19+
20+
//swarm join --addr=<node_ip:2375> etcd://<etcd_ip>/<path>
21+
type Response struct {
22+
ClientURL string `json:"clientURL"`
23+
Name string `json:"name"`
24+
PeerURL string `json:"peerURL"`
25+
State string `json:"state"`
26+
}
27+
28+
const (
29+
swarmpath = "/deis/scheduler/swarm/node"
30+
swarmetcd = "/deis/scheduler/swarm/host"
31+
etcdport = "4001"
32+
timeout time.Duration = 3 * time.Second
33+
ttl time.Duration = timeout * 2
34+
)
35+
36+
func run(cmd string) {
37+
var cmdBuf bytes.Buffer
38+
tmpl := template.Must(template.New("cmd").Parse(cmd))
39+
if err := tmpl.Execute(&cmdBuf, nil); err != nil {
40+
log.Fatal(err)
41+
}
42+
cmdString := cmdBuf.String()
43+
fmt.Println(cmdString)
44+
var cmdl *exec.Cmd
45+
cmdl = exec.Command("sh", "-c", cmdString)
46+
if _, _, err := utils.RunCommandWithStdoutStderr(cmdl); err != nil {
47+
log.Fatal(err)
48+
} else {
49+
fmt.Println("ok")
50+
}
51+
}
52+
53+
func getleaderHost() string {
54+
var response []Response
55+
var host string
56+
client := &http.Client{}
57+
resp, _ := client.Get("http://" + os.Getenv("HOST") + ":7001/v2/admin/machines")
58+
defer resp.Body.Close()
59+
body, _ := ioutil.ReadAll(resp.Body)
60+
// body, _ := ioutil.ReadFile("test.json")
61+
// fmt.Println(body)
62+
json.Unmarshal(body, &response)
63+
// fmt.Println(response)
64+
for _, node := range response {
65+
if node.State == "leader" {
66+
host = strings.Split(node.ClientURL, "//")[1]
67+
}
68+
}
69+
return host
70+
}
71+
72+
func publishService(client *etcd.Client, host string, ttl uint64) {
73+
for {
74+
setEtcd(client, swarmetcd, host, ttl)
75+
time.Sleep(timeout)
76+
}
77+
}
78+
79+
func setEtcd(client *etcd.Client, key, value string, ttl uint64) {
80+
_, err := client.Set(key, value, ttl)
81+
if err != nil {
82+
log.Println(err)
83+
}
84+
}
85+
func main() {
86+
etcdproto := "etcd://" + getleaderHost() + swarmpath
87+
etcdhost := os.Getenv("HOST")
88+
addr := "--addr=" + etcdhost + ":2375"
89+
client := etcd.NewClient([]string{"http://" + etcdhost + ":" + etcdport})
90+
var wd, _ = os.Getwd()
91+
switch os.Args[1] {
92+
case "join":
93+
// fmt.Println("join")
94+
run(wd + "/deis-swarm join " + addr + " " + etcdproto)
95+
case "manage":
96+
// fmt.Println("manage")
97+
go publishService(client, etcdhost, uint64(ttl.Seconds()))
98+
run(wd + "/deis-swarm manage " + etcdproto)
99+
}
100+
}

swarm/test.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
[
2+
{
3+
"clientURL": "http://127.0.0.1:4001",
4+
"name": "peer1",
5+
"peerURL": "http://127.0.0.1:7001",
6+
"state": "leader"
7+
},
8+
{
9+
"clientURL": "http://127.0.0.1:4002",
10+
"name": "peer2",
11+
"peerURL": "http://127.0.0.1:7002",
12+
"state": "follower"
13+
},
14+
{
15+
"clientURL": "http://127.0.0.1:4003",
16+
"name": "peer3",
17+
"peerURL": "http://127.0.0.1:7003",
18+
"state": "follower"
19+
}
20+
]

0 commit comments

Comments
 (0)