feat(cache): allow cache to be persisted

We'll now persist the cache so that expensive operations can be cached
to speed up building the slug.

Author: jeroenvisser101

Makefile

@@ -10,7 +10,7 @@ BINDIR := ./rootfs/bin
 
 include versioning.mk
 
-SHELL_SCRIPTS = $(wildcard _scripts/*.sh) $(wildcard rootfs/bin/*_object) $(wildcard rootfs/builder/*)
+SHELL_SCRIPTS = $(wildcard _scripts/*.sh) $(wildcard rootfs/bin/*_object) $(wildcard rootfs/bin/*_cache) rootfs/bin/normalize_storage $(wildcard rootfs/builder/*)
 
 # The following variables describe the containerized development environment
 # and other build options

rootfs/Dockerfile

@@ -4,15 +4,26 @@ RUN mkdir /app
 RUN addgroup --quiet --gid 2000 slug && \
   useradd slug --uid=2000 --gid=2000 --home-dir /app --no-create-home
 
+# disable source repos (speeds up apt-get update)
+RUN sed -i -e 's/^deb-src/#deb-src/' /etc/apt/sources.list && \
+apt-get update && \
+apt-get install -y md5deep && \
+	apt-get clean && \
+	rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc && \
+bash -c "mkdir -p /usr/share/man/man{1..8}"
+
 ADD . /
 ENV PYTHONPATH $PYTHONPATH:/usr/local/lib/python3/site-packages
 ADD https://storage.googleapis.com/object-storage-cli/bb8e054/objstorage-bb8e054-linux-amd64 /bin/objstorage
 RUN chmod +x /bin/objstorage
 RUN chown -R slug:slug /app
 RUN chown slug:slug /bin/get_object
+RUN chown slug:slug /bin/normalize_storage
 RUN chown slug:slug /bin/put_object
 RUN chown slug:slug /bin/objstorage
 RUN chown slug:slug /bin/read_procfile_keys
+RUN chown slug:slug /bin/restore_cache
+RUN chown slug:slug /bin/store_cache
 
 USER slug
 ENV HOME /app

rootfs/bin/get_object

@@ -1,13 +1,5 @@
 #!/usr/bin/env bash
-
 GET_PATH=/tmp/slug.tgz
-
-export BUCKET_FILE=/var/run/secrets/deis/objectstore/creds/builder-bucket
-if [ "$BUILDER_STORAGE" == "minio" ]; then
-  mkdir -p /app/objectstore/minio/
-  echo "git" > /app/objectstore/minio/builder-bucket
-  export BUCKET_FILE=/app/objectstore/minio/builder-bucket
-elif [ "$BUILDER_STORAGE" == "azure" ] || [ "$BUILDER_STORAGE" == "swift" ] ; then
-  export CONTAINER_FILE=/var/run/secrets/deis/objectstore/creds/builder-container
-fi
+# shellcheck disable=SC1091
+source /bin/normalize_storage
 objstorage --storage-type="$BUILDER_STORAGE" download "$TAR_PATH" "$GET_PATH"

rootfs/bin/normalize_storage

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+export BUCKET_FILE=/var/run/secrets/deis/objectstore/creds/builder-bucket
+if [ "$BUILDER_STORAGE" == "minio" ]; then
+  mkdir -p /app/objectstore/minio/
+  echo "git" > /app/objectstore/minio/builder-bucket
+  export BUCKET_FILE=/app/objectstore/minio/builder-bucket
+elif [ "$BUILDER_STORAGE" == "azure" ] || [ "$BUILDER_STORAGE" == "swift" ] ; then
+  export CONTAINER_FILE=/var/run/secrets/deis/objectstore/creds/builder-container
+fi

rootfs/bin/put_object

@@ -3,11 +3,7 @@
 SLUG_PATH=/tmp/slug.tgz
 PROC_PATH=/tmp/build/Procfile
 
-export BUCKET_FILE=/var/run/secrets/deis/objectstore/creds/builder-bucket
-if [ "$BUILDER_STORAGE" == "minio" ]; then
-  export BUCKET_FILE=/app/objectstore/minio/builder-bucket
-elif [ "$BUILDER_STORAGE" == "azure" ] || [ "$BUILDER_STORAGE" == "swift" ] ; then
-  export CONTAINER_FILE=/var/run/secrets/deis/objectstore/creds/builder-container
-fi
+# shellcheck disable=SC1091
+source /bin/normalize_storage
 objstorage --storage-type="$BUILDER_STORAGE" upload "$SLUG_PATH" "${PUT_PATH}/slug.tgz"
 objstorage --storage-type="$BUILDER_STORAGE" upload "$PROC_PATH" "${PUT_PATH}/Procfile"

rootfs/bin/restore_cache

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+cache_file=/tmp/cache.tgz
+# shellcheck disable=SC1091
+source /bin/normalize_storage
+
+# Download the cache (which might not exist, hence || true to return zero exit code)
+objstorage --storage-type="$BUILDER_STORAGE" download "${CACHE_PATH}/cache.tgz" "$cache_file" &>/tmp/objstorage.log || true

rootfs/bin/store_cache

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+cache_file=/tmp/cache.tgz
+# shellcheck disable=SC1091
+source /bin/normalize_storage
+objstorage --storage-type="$BUILDER_STORAGE" upload "$cache_file" "${CACHE_PATH}/cache.tgz"

rootfs/builder/build.sh

@@ -13,6 +13,7 @@ unset DEIS_DEBUG
 app_dir=/app
 build_root=/tmp/build
 cache_root=/tmp/cache
+cache_file=/tmp/cache.tgz
 env_root=/tmp/env
 buildpack_root=/tmp/buildpacks
 
@@ -60,6 +61,24 @@ function ensure_indent() {
     done
 }
 
+function cache_fingerprint() {
+  md5deep -r ${cache_root} | sort | uniq | md5sum
+}
+
+# Restore cache when a $CACHE_PATH was supplied
+if ! [[ -z "${CACHE_PATH}" ]]; then
+  echo_title "Restoring cache..."
+  restore_cache
+  if [[ -f ${cache_file} ]]; then
+    tar -xzf ${cache_file} -C ${cache_root}
+    echo_normal "Done!"
+  else
+    echo_normal "No cache file could be found. If you're deploying for the first time, it'll be created now."
+  fi
+
+  original_cache_fingerprint=$(cache_fingerprint)
+fi
+
 ## Copy application code over
 if [ -d "/tmp/app" ]; then
     cp -rf /tmp/app/. $app_dir
@@ -191,6 +210,31 @@ if [[ ! -f "$build_root/Procfile" ]]; then
 	fi
 fi
 
+# Compress and save cache
+if ! [[ -z "${CACHE_PATH}" ]]; then
+  echo_title "Checking for changes inside the cache directory..."
+  # If there's any files in the cache_root folder, we'll create a tar and upload
+  # it for future use
+  if [ "$(ls -A ${cache_root})" ]; then
+    # Let's check if the fingerprint changed, if it did, we'll be updating
+    # the cache
+    if [[ "$original_cache_fingerprint" != "$(cache_fingerprint)" ]]; then
+      echo_normal "Files inside cache folder changed, uploading new cache..."
+
+      # Create a new cache file and check if it requires to be updated
+      tar -z -C ${cache_root} -cf ${cache_file} .
+      cache_size=$(du -Sh "$cache_file" | cut -f1)
+
+      store_cache
+      echo_normal "Done: Uploaded cache (${cache_size})"
+    else
+      echo_normal "Cache unchanged, not updating"
+    fi
+  else
+    echo_normal "No files were added to the cache folder, cache wasn't updated"
+  fi
+fi
+
 if [[ "$slug_file" != "-" ]]; then
     slug_size=$(du -Sh "$slug_file" | cut -f1)
     echo_title "Compiled slug size is $slug_size"