Commit 6a393dc6 authored by Thong Kuah's avatar Thong Kuah
Browse files

Use a single auto-deploy binary 

Need bash so that I can use variable expansion "${@:2}" for all
arguments except first. We also already use quite a few bashims, and we
already install bash, so might as well use it
parent 5201b1b6

.gitlab-ci.yml

+12 −18
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ test-kube-domain: 
    GIT_STRATEGY: none

    KUBE_INGRESS_BASE_DOMAIN: example.com

  script:

    - check_kube_domain

    - auto-deploy check_kube_domain



test-kube-domain-legacy:

  <<: *test-job
@@ -50,21 +50,21 @@ test-kube-domain-legacy: 
    GIT_STRATEGY: none

    AUTO_DEVOPS_DOMAIN: example.com

  script:

    - check_kube_domain && expected_error || failed_as_expected

    - auto-deploy check_kube_domain && expected_error || failed_as_expected



test-kube-domain_error:

  <<: *test-job

  variables:

    GIT_STRATEGY: none

  script:

    - check_kube_domain && expected_error || failed_as_expected

    - auto-deploy check_kube_domain && expected_error || failed_as_expected



test-download-chart:

  <<: *test-job

  variables:

    GIT_STRATEGY: none

  script:

    - download_chart

    - auto-deploy download_chart



test-deploy-name:

  <<: *test-job
@@ -108,8 +108,7 @@ test-ensure-namespace: 
  script:

    - download_k3s

    - start_k3s

    - source /build/deploy-helpers.sh

    - ensure_namespace

    - auto-deploy ensure_namespace



test-initialize-tiller:

  <<: *test-job
@@ -119,8 +118,7 @@ test-initialize-tiller: 
  script:

    - download_k3s

    - start_k3s

    - source /build/deploy-helpers.sh

    - initialize_tiller

    - auto-deploy initialize_tiller



# disabled, doesn't look like k3s supports docker-registry secret

.test-create-secret:
@@ -135,8 +133,7 @@ test-initialize-tiller: 
  script:

    - download_k3s

    - start_k3s

    - source /build/deploy-helpers.sh

    - create_secret

    - auto-deploy create_secret

    - kubectl get secret gitlab-registry -n $KUBE_NAMESPACE



test-create-secret-public-project:
@@ -148,8 +145,7 @@ test-create-secret-public-project: 
  script:

    - download_k3s

    - start_k3s

    - source /build/deploy-helpers.sh

    - create_secret

    - auto-deploy create_secret

    - kubectl get secret gitlab-registry -n $KUBE_NAMESPACE && expected_error || failed_as_expected



test-persist-environment-url:
@@ -158,8 +154,7 @@ test-persist-environment-url: 
    GIT_STRATEGY: none

    CI_ENVIRONMENT_URL: review-app.example.com

  script:

    - source /build/deploy-helpers.sh

    - persist_environment_url

    - auto-deploy persist_environment_url

    - grep review-app.example.com environment_url.txt



# disabled, k3s does not support helm charts
@@ -184,10 +179,9 @@ test-persist-environment-url: 
  script:

    - download_k3s

    - start_k3s

    - source /build/deploy-helpers.sh

    - initialize_tiller

    - download_chart

    - deploy

    - auto-deploy initialize_tiller

    - auto-deploy download_chart

    - auto-deploy deploy

    - helm ls



test-create-application-secret:

src/bin/auto-deploy

0 → 100755
+245 −0
Original line number Diff line number Diff line 
#! /bin/bash



[[ "$TRACE" ]] && set -x



auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}

export DATABASE_URL=${DATABASE_URL-$auto_database_url}

export TILLER_NAMESPACE=$KUBE_NAMESPACE



function check_kube_domain() {

  if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then

    echo "In order to deploy or use Review Apps,"

    echo "KUBE_INGRESS_BASE_DOMAIN variables must be set"

    echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings"

    echo "or by defining a variable at group or project level."

    echo "You can also manually add it in .gitlab-ci.yml"

    false

  else

    true

  fi

}



function download_chart() {

  if [[ ! -d chart ]]; then

    auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}

    auto_chart_name=$(basename $auto_chart)

    auto_chart_name=${auto_chart_name%.tgz}

    auto_chart_name=${auto_chart_name%.tar.gz}

  else

    auto_chart="chart"

    auto_chart_name="chart"

  fi



  helm init --client-only

  helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"}

  if [[ ! -d "$auto_chart" ]]; then

    helm fetch ${auto_chart} --untar

  fi

  if [ "$auto_chart_name" != "chart" ]; then

    mv ${auto_chart_name} chart

  fi



  helm dependency update chart/

  helm dependency build chart/

}



function ensure_namespace() {

  kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"

}



function initialize_tiller() {

  echo "Checking Tiller..."



  export HELM_HOST="localhost:44134"

  tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 &

  echo "Tiller is listening on ${HELM_HOST}"



  if ! helm version --debug; then

    echo "Failed to init Tiller."

    return 1

  fi

  echo ""

}



function create_secret() {

  echo "Create secret..."

  if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then

    return

  fi



  kubectl create secret -n "$KUBE_NAMESPACE" \

    docker-registry gitlab-registry \

    --docker-server="$CI_REGISTRY" \

    --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \

    --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \

    --docker-email="$GITLAB_USER_EMAIL" \

    -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -

}



function persist_environment_url() {

  echo $CI_ENVIRONMENT_URL > environment_url.txt

}



function deploy() {

  track="${1-stable}"

  percentage="${2:-100}"

  name=$(deploy_name "$track")



  if [[ -z "$CI_COMMIT_TAG" ]]; then

    image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG}

    image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA}

  else

    image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE}

    image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}

  fi



  service_enabled="true"

  postgres_enabled="$POSTGRES_ENABLED"



  # if track is different than stable,

  # re-use all attached resources

  if [[ "$track" != "stable" ]]; then

    service_enabled="false"

    postgres_enabled="false"

  fi



  replicas=$(get_replicas "$track" "$percentage")



  if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then

    secret_name='gitlab-registry'

  else

    secret_name=''

  fi



  create_application_secret "$track"



  env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]')

  eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS

  if [ -n "$env_ADDITIONAL_HOSTS" ]; then

    additional_hosts="{$env_ADDITIONAL_HOSTS}"

  elif [ -n "$ADDITIONAL_HOSTS" ]; then

    additional_hosts="{$ADDITIONAL_HOSTS}"

  fi



  if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then

    echo "Deploying first release with database initialization..."

    helm upgrade --install \

      --wait \

      --set service.enabled="$service_enabled" \

      --set gitlab.app="$CI_PROJECT_PATH_SLUG" \

      --set gitlab.env="$CI_ENVIRONMENT_SLUG" \

      --set releaseOverride="$CI_ENVIRONMENT_SLUG" \

      --set image.repository="$image_repository" \

      --set image.tag="$image_tag" \

      --set image.pullPolicy=IfNotPresent \

      --set image.secrets[0].name="$secret_name" \

      --set application.track="$track" \

      --set application.database_url="$DATABASE_URL" \

      --set application.secretName="$APPLICATION_SECRET_NAME" \

      --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \

      --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \

      --set service.url="$CI_ENVIRONMENT_URL" \

      --set service.additionalHosts="$additional_hosts" \

      --set replicaCount="$replicas" \

      --set postgresql.enabled="$postgres_enabled" \

      --set postgresql.nameOverride="postgres" \

      --set postgresql.postgresUser="$POSTGRES_USER" \

      --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \

      --set postgresql.postgresDatabase="$POSTGRES_DB" \

      --set postgresql.imageTag="$POSTGRES_VERSION" \

      --set application.initializeCommand="$DB_INITIALIZE" \

      $HELM_UPGRADE_EXTRA_ARGS \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/



    echo "Deploying second release..."

    helm upgrade --reuse-values \

      --wait \

      --set application.initializeCommand="" \

      --set application.migrateCommand="$DB_MIGRATE" \

      $HELM_UPGRADE_EXTRA_ARGS \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/

  else

    echo "Deploying new release..."

    helm upgrade --install \

      --wait \

      --set service.enabled="$service_enabled" \

      --set gitlab.app="$CI_PROJECT_PATH_SLUG" \

      --set gitlab.env="$CI_ENVIRONMENT_SLUG" \

      --set releaseOverride="$CI_ENVIRONMENT_SLUG" \

      --set image.repository="$image_repository" \

      --set image.tag="$image_tag" \

      --set image.pullPolicy=IfNotPresent \

      --set image.secrets[0].name="$secret_name" \

      --set application.track="$track" \

      --set application.database_url="$DATABASE_URL" \

      --set application.secretName="$APPLICATION_SECRET_NAME" \

      --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \

      --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \

      --set service.url="$CI_ENVIRONMENT_URL" \

      --set service.additionalHosts="$additional_hosts" \

      --set replicaCount="$replicas" \

      --set postgresql.enabled="$postgres_enabled" \

      --set postgresql.nameOverride="postgres" \

      --set postgresql.postgresUser="$POSTGRES_USER" \

      --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \

      --set postgresql.postgresDatabase="$POSTGRES_DB" \

      --set postgresql.imageTag="$POSTGRES_VERSION" \

      --set application.migrateCommand="$DB_MIGRATE" \

      $HELM_UPGRADE_EXTRA_ARGS \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/

  fi



  if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then

    kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name"

  fi

}



function scale() {

  track="${1-stable}"

  percentage="${2-100}"

  name=$(deploy_name "$track")



  replicas=$(get_replicas "$track" "$percentage")



  if [[ -n "$(helm ls -q "^$name$")" ]]; then

    helm upgrade --reuse-values \

      --wait \

      --set replicaCount="$replicas" \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/

  fi

}



function delete() {

  track="${1-stable}"

  name=$(deploy_name "$track")



  if [[ -n "$(helm ls -q "^$name$")" ]]; then

    helm delete --purge "$name"

  fi



  secret_name=$(application_secret_name "$track")

  kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name"

}



option=$1

case $option

in

  check_kube_domain) check_kube_domain ;;

  download_chart) download_chart ;;

  ensure_namespace) ensure_namespace ;;

  initialize_tiller) initialize_tiller ;;

  create_secret) create_secret ;;

  persist_environment_url) persist_environment_url ;;

  deploy) deploy "${@:2}" ;;

  scale) scale "${@:2}" ;;

  delete) delete "${@:2}" ;;

  *) exit 1 ;;

esac

src/bin/check_kube_domain

deleted100755 → 0
+0 −12
Original line number Diff line number Diff line 
#! /bin/sh



if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then

  echo "In order to deploy or use Review Apps,"

  echo "KUBE_INGRESS_BASE_DOMAIN variables must be set"

  echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings"

  echo "or by defining a variable at group or project level."

  echo "You can also manually add it in .gitlab-ci.yml"

  false

else

  true

fi

src/bin/download_chart

deleted100755 → 0
+0 −23
Original line number Diff line number Diff line 
#! /bin/sh



if [[ ! -d chart ]]; then

  auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}

  auto_chart_name=$(basename $auto_chart)

  auto_chart_name=${auto_chart_name%.tgz}

  auto_chart_name=${auto_chart_name%.tar.gz}

else

  auto_chart="chart"

  auto_chart_name="chart"

fi



helm init --client-only

helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"}

if [[ ! -d "$auto_chart" ]]; then

  helm fetch ${auto_chart} --untar

fi

if [ "$auto_chart_name" != "chart" ]; then

  mv ${auto_chart_name} chart

fi



helm dependency update chart/

helm dependency build chart/

src/deploy-helpers.sh

+0 −193
Original line number Diff line number Diff line 
#! /bin/sh



[[ "$TRACE" ]] && set -x



auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}

export DATABASE_URL=${DATABASE_URL-$auto_database_url}

export TILLER_NAMESPACE=$KUBE_NAMESPACE



function ensure_namespace() {

  kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"

}



function initialize_tiller() {

  echo "Checking Tiller..."



  export HELM_HOST="localhost:44134"

  tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 &

  echo "Tiller is listening on ${HELM_HOST}"



  if ! helm version --debug; then

    echo "Failed to init Tiller."

    return 1

  fi

  echo ""

}



function create_secret() {

  echo "Create secret..."

  if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then

    return

  fi



  kubectl create secret -n "$KUBE_NAMESPACE" \

    docker-registry gitlab-registry \

    --docker-server="$CI_REGISTRY" \

    --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \

    --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \

    --docker-email="$GITLAB_USER_EMAIL" \

    -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -

}



function persist_environment_url() {

  echo $CI_ENVIRONMENT_URL > environment_url.txt

}



function deploy() {

  track="${1-stable}"

  percentage="${2:-100}"

  name=$(deploy_name "$track")



  if [[ -z "$CI_COMMIT_TAG" ]]; then

    image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG}

    image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA}

  else

    image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE}

    image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}

  fi



  service_enabled="true"

  postgres_enabled="$POSTGRES_ENABLED"



  # if track is different than stable,

  # re-use all attached resources

  if [[ "$track" != "stable" ]]; then

    service_enabled="false"

    postgres_enabled="false"

  fi



  replicas=$(get_replicas "$track" "$percentage")



  if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then

    secret_name='gitlab-registry'

  else

    secret_name=''

  fi



  create_application_secret "$track"



  env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]')

  eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS

  if [ -n "$env_ADDITIONAL_HOSTS" ]; then

    additional_hosts="{$env_ADDITIONAL_HOSTS}"

  elif [ -n "$ADDITIONAL_HOSTS" ]; then

    additional_hosts="{$ADDITIONAL_HOSTS}"

  fi



  if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then

    echo "Deploying first release with database initialization..."

    helm upgrade --install \

      --wait \

      --set service.enabled="$service_enabled" \

      --set gitlab.app="$CI_PROJECT_PATH_SLUG" \

      --set gitlab.env="$CI_ENVIRONMENT_SLUG" \

      --set releaseOverride="$CI_ENVIRONMENT_SLUG" \

      --set image.repository="$image_repository" \

      --set image.tag="$image_tag" \

      --set image.pullPolicy=IfNotPresent \

      --set image.secrets[0].name="$secret_name" \

      --set application.track="$track" \

      --set application.database_url="$DATABASE_URL" \

      --set application.secretName="$APPLICATION_SECRET_NAME" \

      --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \

      --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \

      --set service.url="$CI_ENVIRONMENT_URL" \

      --set service.additionalHosts="$additional_hosts" \

      --set replicaCount="$replicas" \

      --set postgresql.enabled="$postgres_enabled" \

      --set postgresql.nameOverride="postgres" \

      --set postgresql.postgresUser="$POSTGRES_USER" \

      --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \

      --set postgresql.postgresDatabase="$POSTGRES_DB" \

      --set postgresql.imageTag="$POSTGRES_VERSION" \

      --set application.initializeCommand="$DB_INITIALIZE" \

      $HELM_UPGRADE_EXTRA_ARGS \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/



    echo "Deploying second release..."

    helm upgrade --reuse-values \

      --wait \

      --set application.initializeCommand="" \

      --set application.migrateCommand="$DB_MIGRATE" \

      $HELM_UPGRADE_EXTRA_ARGS \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/

  else

    echo "Deploying new release..."

    helm upgrade --install \

      --wait \

      --set service.enabled="$service_enabled" \

      --set gitlab.app="$CI_PROJECT_PATH_SLUG" \

      --set gitlab.env="$CI_ENVIRONMENT_SLUG" \

      --set releaseOverride="$CI_ENVIRONMENT_SLUG" \

      --set image.repository="$image_repository" \

      --set image.tag="$image_tag" \

      --set image.pullPolicy=IfNotPresent \

      --set image.secrets[0].name="$secret_name" \

      --set application.track="$track" \

      --set application.database_url="$DATABASE_URL" \

      --set application.secretName="$APPLICATION_SECRET_NAME" \

      --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \

      --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \

      --set service.url="$CI_ENVIRONMENT_URL" \

      --set service.additionalHosts="$additional_hosts" \

      --set replicaCount="$replicas" \

      --set postgresql.enabled="$postgres_enabled" \

      --set postgresql.nameOverride="postgres" \

      --set postgresql.postgresUser="$POSTGRES_USER" \

      --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \

      --set postgresql.postgresDatabase="$POSTGRES_DB" \

      --set postgresql.imageTag="$POSTGRES_VERSION" \

      --set application.migrateCommand="$DB_MIGRATE" \

      $HELM_UPGRADE_EXTRA_ARGS \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/

  fi



  if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then

    kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name"

  fi

}



function scale() {

  track="${1-stable}"

  percentage="${2-100}"

  name=$(deploy_name "$track")



  replicas=$(get_replicas "$track" "$percentage")



  if [[ -n "$(helm ls -q "^$name$")" ]]; then

    helm upgrade --reuse-values \

      --wait \

      --set replicaCount="$replicas" \

      --namespace="$KUBE_NAMESPACE" \

      "$name" \

      chart/

  fi

}



function delete() {

  track="${1-stable}"

  name=$(deploy_name "$track")



  if [[ -n "$(helm ls -q "^$name$")" ]]; then

    helm delete --purge "$name"

  fi



  secret_name=$(application_secret_name "$track")

  kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name"

}



## Helper functions



# Extracts variables prefixed with K8S_SECRET_