Skip to content

Commit 9753211

Browse files
arschlesAaron Schlesinger
arschles
authored and
Aaron Schlesinger
committed
feat(build.go,config.go,storage.go): continue refactoring from shell to Go
1 parent 90f089a commit 9753211

5 files changed

Lines changed: 257 additions & 30 deletions

File tree

‎pkg/gitreceive/build.go‎

Lines changed: 146 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"fmt"
55
"path/filepath"
66
"strings"
7+
8+
"code.google.com/p/go-uuid"
79
)
810

911
const (
@@ -19,6 +21,37 @@ func (e errGitShaTooShort) Error() string {
1921
}
2022

2123
func build(conf *Config, newRev string) error {
24+
// HTTP_PREFIX="http"
25+
// REMOTE_STORAGE="0"
26+
// # if minio is in the cluster, use it. otherwise use fetcher
27+
// # TODO: figure out something for using S3 also
28+
// if [[ -n "$DEIS_MINIO_SERVICE_HOST" && -n "$DEIS_MINIO_SERVICE_PORT" ]]; then
29+
// S3EP=${DEIS_MINIO_SERVICE_HOST}:${DEIS_MINIO_SERVICE_PORT}
30+
// REMOTE_STORAGE="1"
31+
// elif [[ -n "$DEIS_OUTSIDE_STORAGE_HOST" && -n "$DEIS_OUTSIDE_STORAGE_PORT" ]]; then
32+
// HTTP_PREFIX="https"
33+
// S3EP=${DEIS_OUTSIDE_STORAGE_HOST}:${DEIS_OUTSIDE_STORAGE_PORT}
34+
// REMOTE_STORAGE="1"
35+
// elif [ -z "$S3EP" ]; then
36+
// S3EP=${HOST}:3000
37+
// fi
38+
//
39+
// TAR_URL=$HTTP_PREFIX://$S3EP/git/home/${SLUG_NAME}/tar
40+
// PUSH_URL=$HTTP_PREFIX://$S3EP/git/home/${SLUG_NAME}/push
41+
storage, err := getStorageConfig()
42+
if err != nil {
43+
log.Err(err.Error())
44+
os.Exit(1)
45+
}
46+
creds, err := getStorageCreds()
47+
if err == errMissingKey || err == errMissingSecret {
48+
log.Err(err.Error())
49+
os.Exit(1)
50+
}
51+
52+
tarURL := fmt.Sprintf("%s://%s:%s/git/home/%s/tar", storage.schema(), storage.host(), storage.port(), slugName)
53+
pushURL := fmt.Sprintf("%s://%s:%s/git/hom/%s/push", storage.schema(), storage.host(), storage.port(), slugName)
54+
2255
// #!/usr/bin/env bash
2356
// #
2457
// # builder hook called on every git receive-pack
@@ -111,7 +144,7 @@ func build(conf *Config, newRev string) error {
111144
// # use Procfile if provided, otherwise try default process types from ./release
112145
// git archive --format=tar.gz ${GIT_SHA} > ${APP_NAME}.tar.gz
113146
cmd := exec.Command("git", "archive", "--format=tar.gz", fmt.Sprintf("%s > %s.tar.gz", gitSha, appName))
114-
cmd.Path = repoDir
147+
cmd.Dir = repoDir
115148
cmd.Stdout = os.Stdout
116149
cmd.Stderr = os.Stderr
117150
if err := cmd.Run(); err != nil {
@@ -120,7 +153,7 @@ func build(conf *Config, newRev string) error {
120153
}
121154
// tar -xzf ${APP_NAME}.tar.gz -C $TMP_DIR/
122155
cmd := exec.Command("tar", "-xzf", fmt.Sprintf("%s.tar.gz", appName), "-C", fmt.Sprintf("%s/", tmpDir))
123-
cmd.Path = repoDir
156+
cmd.Dir = repoDir
124157
cmd.Stdout = os.Stdout
125158
cmd.Stderr = os.Stderr
126159
if err := cmd.Run(); err != nil {
@@ -167,42 +200,65 @@ func build(conf *Config, newRev string) error {
167200
// cp /etc/deis-slugbuilder.yaml /etc/${SLUG_NAME}.yaml
168201
// fi
169202
// fi
170-
creds, err := getStorageCreds()
171-
if err == errMissingKey || err == errMissingSecret {
172-
log.Err(err.Error())
173-
os.Exit(1)
174-
}
175203

176-
// both key and secret are missing, so proceed as if using fetcher
204+
var srcManifest string
177205
if err == os.ErrNotExist {
206+
// both key and secret are missing, proceed with no credentials
207+
if usingDockerfile {
208+
srcManifest = "/etc/deis-dockerbuilder-no-creds.yaml"
209+
} else {
210+
srcManifest = "/etc/deis-slugbuilder-no-creds.yaml"
211+
}
212+
} else if err == nil {
213+
// both key and secret are in place, so proceed with credentials
214+
if usingDockerfile {
215+
srcManifest = "/etc/deis-dockerbuilder.yaml"
216+
} else {
217+
srcManifest = "/etc/deis-slugbuilder.yaml"
218+
}
219+
} else if err != nil {
220+
// unexpected error, fail
221+
log.Err("unexpected error (%s)", err)
222+
os.Exit(1)
223+
}
178224

225+
fileBytes, err := ioutil.ReadFile(srcManifest)
226+
if err != nil {
227+
log.Err("reading kubernetes manifest %s (%s)", srcManifest, err)
228+
os.Exit(1)
179229
}
180230

181-
//
182-
//
183-
// git archive --format=tar.gz ${GIT_SHA} > ${APP_NAME}.tar.gz
184-
//
185-
// HTTP_PREFIX="http"
186-
// REMOTE_STORAGE="0"
187-
// # if minio is in the cluster, use it. otherwise use fetcher
188-
// # TODO: figure out something for using S3 also
189-
// if [[ -n "$DEIS_MINIO_SERVICE_HOST" && -n "$DEIS_MINIO_SERVICE_PORT" ]]; then
190-
// S3EP=${DEIS_MINIO_SERVICE_HOST}:${DEIS_MINIO_SERVICE_PORT}
191-
// REMOTE_STORAGE="1"
192-
// elif [[ -n "$DEIS_OUTSIDE_STORAGE_HOST" && -n "$DEIS_OUTSIDE_STORAGE_PORT" ]]; then
193-
// HTTP_PREFIX="https"
194-
// S3EP=${DEIS_OUTSIDE_STORAGE_HOST}:${DEIS_OUTSIDE_STORAGE_PORT}
195-
// REMOTE_STORAGE="1"
196-
// elif [ -z "$S3EP" ]; then
197-
// S3EP=${HOST}:3000
198-
// fi
199-
//
200-
// TAR_URL=$HTTP_PREFIX://$S3EP/git/home/${SLUG_NAME}/tar
201-
// PUSH_URL=$HTTP_PREFIX://$S3EP/git/home/${SLUG_NAME}/push
202-
//
203231
// sed -i -- "s#repo_name#$META_NAME#g" /etc/${SLUG_NAME}.yaml
204232
// sed -i -- "s#puturl#$PUSH_URL#g" /etc/${SLUG_NAME}.yaml
205233
// sed -i -- "s#tar-url#$TAR_URL#g" /etc/${SLUG_NAME}.yaml
234+
finalManifestFileName := fmt.Sprintf("/etc/%s", slugName)
235+
var finalManifest string
236+
if usingDockerfile {
237+
finalManifest = strings.Replace(string(fileBytes), "repo_name", fmt.Sprintf("%s-%s", tmpImage, uuid.New()))
238+
finalManifest = strings.Replace(finalManifest, "puturl", pushURL)
239+
finalManifest = strings.Replace(finalManifest, "tar-url", tarURL)
240+
} else {
241+
finalManifest = strings.Replace(string(fileBytes), "repo_name", fmt.Sprintf("%s-%s", slugName, uuid.New()))
242+
finalManifest = strings.Replace(finalManifest, "puturl", pushURL)
243+
finalManifest = strings.Replace(finalManifest, "tar-url", tarURL)
244+
}
245+
246+
if err := ioutil.WriteFile(finalManifestFileName, []byte(finalManifest), os.ModePerm); err != nil {
247+
log.Err("writing final manifest %s (%s)", finalManifestFileName, err)
248+
os.Exit(1)
249+
}
250+
//
251+
// git archive --format=tar.gz ${GIT_SHA} > ${APP_NAME}.tar.gz
252+
253+
cmd := exec.Command("git", "archive", "--format=tar.gz", fmt.Sprintf("%s > %s.tar.gz", gitSha, appName))
254+
cmd.Dir = repoDir
255+
cmd.Stdout = os.Stdout
256+
cmd.Stderr = os.Stderr
257+
if err := cmd.Run(); err != nil {
258+
log.Err("running %s", strings.Join(cmd.Args, " "))
259+
os.Exit(1)
260+
}
261+
206262
//
207263
// ACCESS_KEY=`cat /var/run/secrets/object/store/access-key-id`
208264
// ACCESS_SECRET=`cat /var/run/secrets/object/store/access-secret-key`
@@ -214,12 +270,72 @@ func build(conf *Config, newRev string) error {
214270
// mkdir -p /var/minio-conf
215271
// CONFIG_DIR=/var/minio-conf
216272
// MC_PREFIX="mc -C $CONFIG_DIR --quiet"
273+
configDir = "/var/minio-conf"
274+
if err := os.MkdirAll(configDir, os.ModePerm); err != nil {
275+
log.Err("creating minio config file (%s)", err)
276+
os.Exit(1)
277+
}
278+
baseMinioCmd := exec.Command("mc", "-C", configDir, "--quiet")
279+
baseMinioCmd.Stderr = os.Stderr
280+
217281
// $MC_PREFIX config host add "$HTTP_PREFIX://$S3EP" $ACCESS_KEY $ACCESS_SECRET &>/dev/null
282+
configCmd := baseMinioCmd
283+
configCmd.Args = append(
284+
configCmd.Args,
285+
"config",
286+
"host",
287+
"add",
288+
fmt.Sprintf("%s://%s:%s", storage.schema(), storage.host(), storage.port()),
289+
storageCreds.key,
290+
storageCreds.secret,
291+
)
292+
if err := configCmd.Run(); err != nil {
293+
log.Err("configuring the minio client (%s)", err)
294+
os.Exit(1)
295+
}
296+
218297
// $MC_PREFIX mb "$HTTP_PREFIX://${S3EP}/git" &>/dev/null
298+
makeBucketCmd := baseMinioCmd
299+
makeBucketCmd.Args = append(
300+
makeBucketCmd.Args,
301+
"mb",
302+
fmt.Sprintf("%s://%s:%s/git", storage.schema(), storage.host(), storage.port()),
303+
)
304+
// Don't look for errors here. Buckets may already exist
305+
makeBucketCmd.Run()
306+
219307
// $MC_PREFIX cp ${APP_NAME}.tar.gz $TAR_URL &>/dev/null
308+
cpCmd := baseMinioCmd
309+
cpCmd.Args = append(
310+
cpCmd.Args,
311+
"cp",
312+
fmt.Sprintf("%s.tar.gz", appName),
313+
tarURL,
314+
)
315+
cpCmd.Dir = repoDir
316+
if err := cpCmd.Run(); err != nil {
317+
log.Err("copying %s.tar.gz to %s (%s)", apName, tarURL, err)
318+
os.Exit(1)
319+
}
320+
220321
//
221322
// puts-step "Starting build"
222323
// kubectl --namespace=${POD_NAMESPACE} create -f /etc/${SLUG_NAME}.yaml >/dev/null
324+
325+
log.Info("Starting build")
326+
kubectlCmd := exec.Command(
327+
"kubectl",
328+
fmt.Sprintf("--namespace=%s", conf.PodNamespace),
329+
"create",
330+
"-f",
331+
fmt.Sprintf("/etc/%s.yaml", slugName),
332+
)
333+
kubectlCmd.Stderr = os.Stderr
334+
if err := kubectlCmd.Run(); err != nil {
335+
log.Err("creating builder pod (%s)", err)
336+
os.Exit(1)
337+
}
338+
223339
//
224340
// # wait for pod to be running and then pull its logs
225341
// until [ "`kubectl --namespace=${POD_NAMESPACE} get pods -o yaml ${META_NAME} | grep "phase: " | awk {'print $2'}`" == "Running" ]; do

‎pkg/gitreceive/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@ type Config struct {
1616
App string `envconfing:"app"`
1717
Fingerprint string `envconfing:"fingerprint"`
1818
ImageName string `envconfig:"image_name"`
19+
PodNamespace string `envconfig:"POD_NAMESPACE"`
1920
}

‎pkg/gitreceive/storage.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package gitreceive
2+
3+
import (
4+
"fmt"
5+
"os"
6+
)
7+
8+
const (
9+
minioHostEnvVar = "DEIS_MINIO_SERVICE_HOST"
10+
minioPortEnvVar = "DEIS_MINIO_SERVICE_PORT"
11+
outsideStorageHostEnvVar = "DEIS_OUTSIDE_STORAGE_HOST"
12+
outsideStoragePortEnvVar = "DEIS_OUTSIDE_STORAGE_PORT"
13+
)
14+
15+
var (
16+
errNoStorageConfig = fmt.Errorf(
17+
"no storage config variables found (%s:%s or %s:%s)",
18+
minioHostEnvVar,
19+
minioPortEnvVar,
20+
outsideStorageHostEnvVar,
21+
outsideStoragePortEnvVar,
22+
)
23+
)
24+
25+
type storageConfig interface {
26+
schema() string
27+
host() string
28+
port() string
29+
}
30+
31+
func getStorageConfig() (storageConfig, error) {
32+
mHost := os.Getenv(minioHostEnvVar)
33+
mPort := os.Getenv(minioPortEnvVar)
34+
oHost := os.Getenv(outsideStorageHostEnvVar)
35+
oPost := os.Getenv(outsideStoragePortEnvVar)
36+
if mHost != "" && mPost != "" {
37+
return minioConfig{host: mHost, port: mPort}, nil
38+
} else if oHost != "" && oPort != "" {
39+
return outsideConfig{host: oHost, port: oPort}, nil
40+
} else {
41+
return nil, errNoStorageConfig
42+
}
43+
}
44+
45+
type minioConfig struct {
46+
host string
47+
port string
48+
}
49+
50+
func (m minioConfig) schema() string { return "http" }
51+
func (m minioConfig) host() string { return m.host }
52+
func (m minioConfig) port() string { return m.port }
53+
54+
type outsideConfig struct {
55+
host string
56+
port string
57+
}
58+
59+
func (o outsideConfig) schema() string { return "https" }
60+
func (o outsideConfig) host() string { return o.host }
61+
func (o outsideConfig) port() string { return o.port }

‎rootfs/etc/deis-dockerbuilder-no-creds.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: deis-dockerbuilder
5+
labels:
6+
heritage: deis
7+
version: 2.0.0-beta
8+
spec:
9+
restartPolicy: Never
10+
containers:
11+
- name: deis-dockerbuilder
12+
imagePullPolicy: Always
13+
image: quay.io/deisci/dockerbuilder:v2-beta
14+
env:
15+
- name: DEBUG
16+
value: "1"
17+
- name: TAR_URL
18+
value: tar-url
19+
- name: IMG_NAME
20+
value: imagename
21+
- name: ACCESS_KEY_FILE
22+
value: /var/run/secrets/object/store/access_key
23+
- name: ACCESS_SECRET_FILE
24+
value: /var/run/secrets/object/store/access_secret
25+
volumeMounts:
26+
- mountPath: /var/run/docker.sock
27+
name: docker-socket
28+
volumes:
29+
- name: docker-socket
30+
hostPath:
31+
path: /var/run/docker.sock

‎rootfs/etc/deis-slugbuilder-no-creds.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: repo_name
5+
labels:
6+
heritage: deis
7+
version: 2.0.0-beta
8+
spec:
9+
restartPolicy: Never
10+
containers:
11+
- name: deis-slugbuilder
12+
imagePullPolicy: Always
13+
image: quay.io/deisci/slugbuilder:v2-beta
14+
env:
15+
- name: TAR_URL
16+
value: tar-url
17+
- name : put_url
18+
value: puturl

0 commit comments

Comments
 (0)