diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 2d8838f1ddcb36b3637c365e7a98280d298cf738..cf6c2657c7ca45c2460afe28f6084ed0ba2830af 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -132,41 +132,46 @@ test-get-replicas: variables: GIT_STRATEGY: none CI_ENVIRONMENT_SLUG: production - script: - - replicas=$(auto-deploy get_replicas "stable" "100") - - | - if [[ $replicas != 1 ]]; then - echo "$replicas should equal 1" - exit 1 - fi - -test-get-replicas-multiple: - <<: *test-job - variables: - GIT_STRATEGY: none - CI_ENVIRONMENT_SLUG: production - REPLICAS: "2" - script: - - replicas=$(auto-deploy get_replicas "stable" "100") - - | - if [[ $replicas != 2 ]]; then - echo "$replicas should equal 2" - exit 1 - fi - -test-get-replicas-fraction: + TRACK: stable + script: + # When `REPLICAS` variable is not specified + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 1 ]]; then echo "Unexpected replicas"; exit 1; fi + # When `REPLICAS` variable is specified + - export REPLICAS="2" + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 2 ]]; then echo "Unexpected replicas"; exit 1; fi + # When `_REPLICAS` variable is specified + - export PRODUCTION_REPLICAS="3" + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 3 ]]; then echo "Unexpected replicas"; exit 1; fi + # When `__REPLICAS` variable is specified + - export STABLE_PRODUCTION_REPLICAS="4" + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 4 ]]; then echo "Unexpected replicas"; exit 1; fi + +test-get-replicas-canary: <<: *test-job variables: GIT_STRATEGY: none CI_ENVIRONMENT_SLUG: production - REPLICAS: "2" - script: - - replicas=$(auto-deploy get_replicas "stable" "25") - - | - if [[ $replicas != 1 ]]; then - echo "$replicas should 1, (25% of 2 is 0.5, so set a floor of 1)" - exit 1 - fi + TRACK: canary + script: + # When `REPLICAS` variable is not specified + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 1 ]]; then echo "Unexpected replicas"; exit 1; fi + # When `REPLICAS` variable is specified + - export REPLICAS="2" + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 2 ]]; then echo "Unexpected replicas"; exit 1; fi + # When `_REPLICAS` variable is specified + - export PRODUCTION_REPLICAS="3" + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 3 ]]; then echo "Unexpected replicas"; exit 1; fi + # When `__REPLICAS` variable is specified + - export CANARY_PRODUCTION_REPLICAS="4" + - replicas=$(auto-deploy get_replicas ${TRACK}) + - if [[ $replicas != 4 ]]; then echo "Unexpected replicas"; exit 1; fi test-get-replicas-zero: <<: *test-job @@ -175,7 +180,7 @@ test-get-replicas-zero: CI_ENVIRONMENT_SLUG: production REPLICAS: "0" script: - - replicas=$(auto-deploy get_replicas "stable" "100") + - replicas=$(auto-deploy get_replicas "stable") - | if [[ $replicas != 0 ]]; then echo "$replicas should equal 0, as requested" @@ -375,6 +380,9 @@ test-deploy-canary: - auto-deploy download_chart - auto-deploy deploy canary - helm get all production-canary + # It should have Canary Ingress + - kubectl describe ingress production-canary-auto-deploy -n $KUBE_NAMESPACE > ingress.spec + - grep -q 'nginx.ingress.kubernetes.io/canary:.*true' ingress.spec || exit 1 test-deploy-modsecurity: extends: test-deploy diff --git a/assets/auto-deploy-app/Chart.yaml b/assets/auto-deploy-app/Chart.yaml index 1fab094b5158b0fc346bdf84d2245ce3e2e1edaa..31ee68fcaf4ef985866c004af4b404ca17df29ef 100644 --- a/assets/auto-deploy-app/Chart.yaml +++ b/assets/auto-deploy-app/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: GitLab's Auto-deploy Helm Chart name: auto-deploy-app -version: 2.0.0-beta.1 +version: 2.0.0-beta.2 icon: https://gitlab.com/gitlab-com/gitlab-artwork/raw/master/logo/logo-square.png diff --git a/assets/auto-deploy-app/templates/ingress.yaml b/assets/auto-deploy-app/templates/ingress.yaml index 6f39db2cecfe8d61a2e31bf2fae929966cf0d51c..81b2dbf9bfa4f2560da02eded4533795086a1ee7 100644 --- a/assets/auto-deploy-app/templates/ingress.yaml +++ b/assets/auto-deploy-app/templates/ingress.yaml @@ -1,4 +1,4 @@ -{{- if and (.Values.service.enabled) (eq .Values.application.track "stable") (or (.Values.ingress.enabled) (not (hasKey .Values.ingress "enabled"))) -}} +{{- if and (.Values.service.enabled) (or (.Values.ingress.enabled) (not (hasKey .Values.ingress "enabled"))) -}} apiVersion: extensions/v1beta1 kind: Ingress metadata: @@ -12,6 +12,13 @@ metadata: {{- if .Values.ingress.annotations }} {{ toYaml .Values.ingress.annotations | indent 4 }} {{- end }} +{{- if eq .Values.application.track "canary" }} + nginx.ingress.kubernetes.io/canary: "true" + nginx.ingress.kubernetes.io/canary-by-header: "canary" +{{- if .Values.ingress.canary.weight }} + nginx.ingress.kubernetes.io/canary-weight: {{ .Values.ingress.canary.weight | quote }} +{{- end }} +{{- end }} {{- with .Values.ingress.modSecurity }} {{- if .enabled }} nginx.ingress.kubernetes.io/modsecurity-transaction-id: "$server_name-$request_id" diff --git a/assets/auto-deploy-app/templates/service.yaml b/assets/auto-deploy-app/templates/service.yaml index 676406be11de5feedd801876627055405ac51670..1a67ce11f82aa47347d2d28b7699162288d61d11 100644 --- a/assets/auto-deploy-app/templates/service.yaml +++ b/assets/auto-deploy-app/templates/service.yaml @@ -1,4 +1,4 @@ -{{- if and (.Values.service.enabled) (eq .Values.application.track "stable") -}} +{{- if .Values.service.enabled -}} apiVersion: v1 kind: Service metadata: @@ -13,6 +13,7 @@ metadata: {{- end }} labels: app: {{ template "appname" . }} + track: "{{ .Values.application.track }}" chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: {{ .Release.Name }} heritage: {{ .Release.Service }} @@ -26,4 +27,5 @@ spec: selector: app: {{ template "appname" . }} tier: "{{ .Values.application.tier }}" + track: "{{ .Values.application.track }}" {{- end -}} diff --git a/assets/auto-deploy-app/test/template_test.go b/assets/auto-deploy-app/test/template_test.go index c7066d0342b82b1908b81b7042b2e5e05d34979a..8b862a3fb99e196cd4b60cef9ebd32f936cbec8a 100644 --- a/assets/auto-deploy-app/test/template_test.go +++ b/assets/auto-deploy-app/test/template_test.go @@ -17,7 +17,7 @@ import ( ) const ( - chartName = "auto-deploy-app-2.0.0-beta.1" + chartName = "auto-deploy-app-2.0.0-beta.2" helmChartPath = ".." ) @@ -719,6 +719,65 @@ SecRule REQUEST_HEADERS:Content-Type \"text/plain\" \"log,deny,id:\'20010\',stat } } +func TestIngressTemplate_DifferentTracks(t *testing.T) { + templates := []string{"templates/ingress.yaml"} + tcs := []struct { + name string + releaseName string + values map[string]string + + expectedName string + expectedLabels map[string]string + expectedSelector map[string]string + expectedAnnotations map[string]string + expectedInexistentAnnotationKeys []string + expectedErrorRegexp *regexp.Regexp + }{ + { + name: "defaults", + releaseName: "production", + expectedName: "production-auto-deploy", + expectedAnnotations: map[string]string{"kubernetes.io/ingress.class": "nginx"}, + expectedInexistentAnnotationKeys: []string{"nginx.ingress.kubernetes.io/canary"}, + }, + { + name: "with canary track", + releaseName: "production-canary", + values: map[string]string{"application.track": "canary"}, + expectedName: "production-canary-auto-deploy", + expectedAnnotations: map[string]string{"nginx.ingress.kubernetes.io/canary": "true", "nginx.ingress.kubernetes.io/canary-by-header": "canary", "kubernetes.io/ingress.class": "nginx"}, + expectedInexistentAnnotationKeys: []string{"nginx.ingress.kubernetes.io/canary-weight"}, + }, + { + name: "with canary weight", + releaseName: "production-canary", + values: map[string]string{"application.track": "canary", "ingress.canary.weight": "25"}, + expectedName: "production-canary-auto-deploy", + expectedAnnotations: map[string]string{"nginx.ingress.kubernetes.io/canary-weight": "25"}, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + output, ret := renderTemplate(t, tc.values, tc.releaseName, templates, tc.expectedErrorRegexp) + + if ret == false { + return + } + + ingress := new(extensions.Ingress) + helm.UnmarshalK8SYaml(t, output, ingress) + require.Equal(t, tc.expectedName, ingress.ObjectMeta.Name) + for key, value := range tc.expectedAnnotations { + require.Equal(t, ingress.ObjectMeta.Annotations[key], value) + } + for _, key := range tc.expectedInexistentAnnotationKeys { + require.Empty(t, ingress.ObjectMeta.Annotations[key]) + } + }) + } +} + func TestIngressTemplate_Disable(t *testing.T) { templates := []string{"templates/ingress.yaml"} releaseName := "ingress-disable-test" @@ -738,11 +797,6 @@ func TestIngressTemplate_Disable(t *testing.T) { values: map[string]string{"ingress.enabled": "null", "service.enabled": "true"}, expectedName: releaseName + "-auto-deploy", }, - { - name: "with service enabled and track non-stable", - values: map[string]string{"service.enabled": "true", "application.track": "non-stable"}, - expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/ingress.yaml in chart"), - }, { name: "with service disabled and track stable", values: map[string]string{"service.enabled": "false", "application.track": "stable"}, @@ -758,11 +812,6 @@ func TestIngressTemplate_Disable(t *testing.T) { values: map[string]string{"ingress.enabled": "false"}, expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/ingress.yaml in chart"), }, - { - name: "with ingress enabled and track non-stable", - values: map[string]string{"ingress.enabled": "true", "application.track": "non-stable"}, - expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/ingress.yaml in chart"), - }, { name: "with ingress enabled and service disabled", values: map[string]string{"ingress.enabled": "true", "service.enabled": "false"}, @@ -798,6 +847,56 @@ func TestIngressTemplate_Disable(t *testing.T) { } } +func TestServiceTemplate_DifferentTracks(t *testing.T) { + templates := []string{"templates/service.yaml"} + tcs := []struct { + name string + releaseName string + values map[string]string + + expectedName string + expectedLabels map[string]string + expectedSelector map[string]string + expectedErrorRegexp *regexp.Regexp + }{ + { + name: "defaults", + releaseName: "production", + expectedName: "production-auto-deploy", + expectedLabels: map[string]string{"app": "production", "release": "production", "track": "stable"}, + expectedSelector: map[string]string{"app": "production", "tier": "web", "track": "stable"}, + }, + { + name: "with canary track", + releaseName: "production-canary", + values: map[string]string{"application.track": "canary"}, + expectedName: "production-canary-auto-deploy", + expectedLabels: map[string]string{"app": "production-canary", "release": "production-canary", "track": "canary"}, + expectedSelector: map[string]string{"app": "production-canary", "tier": "web", "track": "canary"}, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + output, ret := renderTemplate(t, tc.values, tc.releaseName, templates, tc.expectedErrorRegexp) + + if ret == false { + return + } + + service := new(coreV1.Service) + helm.UnmarshalK8SYaml(t, output, service) + require.Equal(t, tc.expectedName, service.ObjectMeta.Name) + for key, value := range tc.expectedLabels { + require.Equal(t, service.ObjectMeta.Labels[key], value) + } + for key, value := range tc.expectedSelector { + require.Equal(t, service.Spec.Selector[key], value) + } + }) + } +} + func TestServiceTemplate_Disable(t *testing.T) { templates := []string{"templates/service.yaml"} releaseName := "service-disable-test" @@ -812,11 +911,6 @@ func TestServiceTemplate_Disable(t *testing.T) { name: "defaults", expectedName: releaseName + "-auto-deploy", }, - { - name: "with service enabled and track non-stable", - values: map[string]string{"service.enabled": "true", "application.track": "non-stable"}, - expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/service.yaml in chart"), - }, { name: "with service disabled and track stable", values: map[string]string{"service.enabled": "false", "application.track": "stable"}, @@ -831,17 +925,9 @@ func TestServiceTemplate_Disable(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - opts := &helm.Options{ - SetValues: tc.values, - } - output, err := helm.RenderTemplateE(t, opts, helmChartPath, releaseName, templates) + output, ret := renderTemplate(t, tc.values, releaseName, templates, tc.expectedErrorRegexp) - if tc.expectedErrorRegexp != nil { - require.Regexp(t, tc.expectedErrorRegexp, err.Error()) - return - } - if err != nil { - t.Error(err) + if ret == false { return } @@ -852,6 +938,28 @@ func TestServiceTemplate_Disable(t *testing.T) { } } +func renderTemplate(t *testing.T, values map[string]string, releaseName string, templates []string, expectedErrorRegexp *regexp.Regexp) (string, bool) { + opts := &helm.Options{ + SetValues: values, + } + + output, err := helm.RenderTemplateE(t, opts, helmChartPath, releaseName, templates) + if expectedErrorRegexp != nil { + if err == nil { + t.Error("Expected error but didn't happen") + } else { + require.Regexp(t, expectedErrorRegexp, err.Error()) + } + return "", false + } + if err != nil { + t.Error(err) + return "", false + } + + return output, true +} + type workerDeploymentTestCase struct { ExpectedName string ExpectedCmd []string diff --git a/assets/auto-deploy-app/values.yaml b/assets/auto-deploy-app/values.yaml index b7b44df77f6c48c420767fd2cd446cfe6aea0e44..9ea687143d872898a9b4a4aa0c10382313f37186 100644 --- a/assets/auto-deploy-app/values.yaml +++ b/assets/auto-deploy-app/values.yaml @@ -54,6 +54,8 @@ ingress: # - variable: "" # operator: "" # action: "" + canary: + weight: prometheus: metrics: false livenessProbe: diff --git a/src/bin/auto-deploy b/src/bin/auto-deploy index a019c1af13701160a7bec1c04a1ebf24c965fad6..61fd108fb2fc2c7c67ebcfdd26179c205647b34a 100755 --- a/src/bin/auto-deploy +++ b/src/bin/auto-deploy @@ -13,7 +13,6 @@ if [[ "$AUTO_DEVOPS_POSTGRES_CHANNEL" == "2" ]]; then elif [[ "$AUTO_DEVOPS_POSTGRES_CHANNEL" == "1" ]]; then export POSTGRES_VERSION="${POSTGRES_VERSION:-"9.6.2"}" fi -export BIN_DIR="/build/bin" export ASSETS_DIR='/assets' export ASSETS_CHART_DIR="${ASSETS_DIR}/auto-deploy-app" @@ -203,7 +202,7 @@ channel 1 database.' old_postgres_enabled="$POSTGRES_ENABLED" fi - ${BIN_DIR}/validate-chart-version "$(helm list --namespace "$KUBE_NAMESPACE" --output json)" "chart" "$name" + validate-chart-version "$(helm list --namespace "$KUBE_NAMESPACE" --output json)" "chart" "$name" local database_url database_url=$(auto_database_url) @@ -230,7 +229,7 @@ channel 1 database.' fi local replicas - replicas=$(get_replicas "$track" "$percentage") + replicas=$(get_replicas "$track") local modsecurity_set_args=() if [[ -n "$AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE" ]]; then @@ -294,6 +293,7 @@ channel 1 database.' --set service.url="$CI_ENVIRONMENT_URL" \ --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ + --set ingress.canary.weight="${percentage}" \ --set postgresql.enabled="$old_postgres_enabled" \ --set postgresql.managed="$postgres_managed" \ --set postgresql.managedClassSelector="$postgres_managed_selector" \ @@ -333,6 +333,7 @@ channel 1 database.' --set service.url="$CI_ENVIRONMENT_URL" \ --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ + --set ingress.canary.weight="${percentage}" \ --set postgresql.enabled="$old_postgres_enabled_for_track" \ --set postgresql.managed="$postgres_managed" \ --set postgresql.managedClassSelector="$postgres_managed_selector" \ @@ -363,12 +364,13 @@ function scale() { name=$(deploy_name "$track") local replicas - replicas=$(get_replicas "$track" "$percentage") + replicas=$(get_replicas "$track") if [[ -n "$(helm ls --namespace "$KUBE_NAMESPACE" -q -f "^$name$")" ]]; then helm upgrade --reuse-values \ --wait \ --set replicaCount="$replicas" \ + --set ingress.canary.weight="${percentage}" \ --namespace="$KUBE_NAMESPACE" \ "$name" \ chart/ @@ -463,7 +465,6 @@ function deploy_name() { # shellcheck disable=SC2153 # incorrectly thinks replicas vs REPLICAS is a misspelling function get_replicas() { local track="${1:-stable}" - local percentage="${2:-100}" local env_track env_track=$(echo $track | tr '[:lower:]' '[:upper:]') @@ -471,32 +472,21 @@ function get_replicas() { local env_slug env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr '[:lower:]' '[:upper:]') - local new_replicas - if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then - # for stable track get number of replicas from `PRODUCTION_REPLICAS` - eval new_replicas=\$${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - new_replicas=$REPLICAS - fi - else - # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` - eval new_replicas=\$${env_track}_${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - eval new_replicas=\$${env_track}_REPLICAS - fi - fi + local environment_track_replicas + local environment_replicas + eval environment_track_replicas=\$${env_track}_${env_slug}_REPLICAS + eval environment_replicas=\$${env_slug}_REPLICAS - local replicas="${new_replicas:-1}" - replicas="$((replicas * percentage / 100))" + local new_replicas + new_replicas=${environment_track_replicas} + new_replicas=${new_replicas:-$environment_replicas} + new_replicas=${new_replicas:-$REPLICAS} - if [[ $new_replicas == 0 ]]; then + if [[ -n "$new_replicas" ]]; then # If zero replicas requested, then return 0 echo "$new_replicas" - elif [[ $replicas -gt 0 ]]; then - echo "$replicas" else - # Return one if calculated replicas is zero - # E.g. 25% of 2 replicas is 0 (integer division) + # Return one if replicas is not specified echo 1 fi }