Commit b8c725a4 authored by Shinya Maeda's avatar Shinya Maeda
Browse files

feat: Canary Ingress 

This commit introduces the Canary Ingress in the Auto Deploy
architecture in order for advanced traffic routing.

BREAKING CHANGE: Due to the resource rearchitecturing,
charts older than v2.0 are not compatible.
parent 22f10278

.gitlab/ci/test.gitlab-ci.yml

+40 −32
Original line number Diff line number Diff line
@@ -132,41 +132,46 @@ test-get-replicas: 
  variables:

    GIT_STRATEGY: none

    CI_ENVIRONMENT_SLUG: production

  script:

    - replicas=$(auto-deploy get_replicas "stable" "100")

    - |

      if [[ $replicas != 1 ]]; then

        echo "$replicas should equal 1"

        exit 1

      fi



test-get-replicas-multiple:

  <<: *test-job

  variables:

    GIT_STRATEGY: none

    CI_ENVIRONMENT_SLUG: production

    REPLICAS: "2"

  script:

    - replicas=$(auto-deploy get_replicas "stable" "100")

    - |

      if [[ $replicas != 2 ]]; then

        echo "$replicas should equal 2"

        exit 1

      fi



test-get-replicas-fraction:

    TRACK: stable

  script:

    # When `REPLICAS` variable is not specified

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 1 ]]; then echo "Unexpected replicas"; exit 1; fi

    # When `REPLICAS` variable is specified

    - export REPLICAS="2"

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 2 ]]; then echo "Unexpected replicas"; exit 1; fi

    # When `<env>_REPLICAS` variable is specified

    - export PRODUCTION_REPLICAS="3"

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 3 ]]; then echo "Unexpected replicas"; exit 1; fi

    # When `<track>_<env>_REPLICAS` variable is specified

    - export STABLE_PRODUCTION_REPLICAS="4"

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 4 ]]; then echo "Unexpected replicas"; exit 1; fi



test-get-replicas-canary:

  <<: *test-job

  variables:

    GIT_STRATEGY: none

    CI_ENVIRONMENT_SLUG: production

    REPLICAS: "2"

  script:

    - replicas=$(auto-deploy get_replicas "stable" "25")

    - |

      if [[ $replicas != 1 ]]; then

        echo "$replicas should 1, (25% of 2 is 0.5, so set a floor of 1)"

        exit 1

      fi

    TRACK: canary

  script:

    # When `REPLICAS` variable is not specified

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 1 ]]; then echo "Unexpected replicas"; exit 1; fi

    # When `REPLICAS` variable is specified

    - export REPLICAS="2"

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 2 ]]; then echo "Unexpected replicas"; exit 1; fi

    # When `<env>_REPLICAS` variable is specified

    - export PRODUCTION_REPLICAS="3"

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 3 ]]; then echo "Unexpected replicas"; exit 1; fi

    # When `<track>_<env>_REPLICAS` variable is specified

    - export CANARY_PRODUCTION_REPLICAS="4"

    - replicas=$(auto-deploy get_replicas ${TRACK})

    - if [[ $replicas != 4 ]]; then echo "Unexpected replicas"; exit 1; fi



test-get-replicas-zero:

  <<: *test-job
@@ -175,7 +180,7 @@ test-get-replicas-zero: 
    CI_ENVIRONMENT_SLUG: production

    REPLICAS: "0"

  script:

    - replicas=$(auto-deploy get_replicas "stable" "100")

    - replicas=$(auto-deploy get_replicas "stable")

    - |

      if [[ $replicas != 0 ]]; then

        echo "$replicas should equal 0, as requested"
@@ -375,6 +380,9 @@ test-deploy-canary: 
    - auto-deploy download_chart

    - auto-deploy deploy canary

    - helm get all production-canary

    # It should have Canary Ingress

    - kubectl describe ingress production-canary-auto-deploy -n $KUBE_NAMESPACE > ingress.spec

    - grep -q 'nginx.ingress.kubernetes.io/canary:.*true' ingress.spec || exit 1



test-deploy-modsecurity:

  extends: test-deploy

assets/auto-deploy-app/Chart.yaml

+1 −1
Original line number Diff line number Diff line 
apiVersion: v1

description: GitLab's Auto-deploy Helm Chart

name: auto-deploy-app

version: 2.0.0-beta.1

version: 2.0.0-beta.2

icon: https://gitlab.com/gitlab-com/gitlab-artwork/raw/master/logo/logo-square.png

assets/auto-deploy-app/templates/ingress.yaml

+8 −1
Original line number Diff line number Diff line 
{{- if and (.Values.service.enabled) (eq .Values.application.track "stable") (or (.Values.ingress.enabled) (not (hasKey .Values.ingress "enabled"))) -}}

{{- if and (.Values.service.enabled) (or (.Values.ingress.enabled) (not (hasKey .Values.ingress "enabled"))) -}}

apiVersion: extensions/v1beta1

kind: Ingress

metadata:
@@ -12,6 +12,13 @@ metadata: 
{{- if .Values.ingress.annotations }}

{{ toYaml .Values.ingress.annotations | indent 4 }}

{{- end }}

{{- if eq .Values.application.track "canary" }}

    nginx.ingress.kubernetes.io/canary: "true"

    nginx.ingress.kubernetes.io/canary-by-header: "canary"

{{- if .Values.ingress.canary.weight }}

    nginx.ingress.kubernetes.io/canary-weight: {{ .Values.ingress.canary.weight | quote }}

{{- end }}

{{- end }}

{{- with .Values.ingress.modSecurity }}

{{- if .enabled }}

    nginx.ingress.kubernetes.io/modsecurity-transaction-id: "$server_name-$request_id"

assets/auto-deploy-app/templates/service.yaml

+3 −1
Original line number Diff line number Diff line 
{{- if and (.Values.service.enabled) (eq .Values.application.track "stable") -}}

{{- if .Values.service.enabled -}}

apiVersion: v1

kind: Service

metadata:
@@ -13,6 +13,7 @@ metadata: 
{{- end }}

  labels:

    app: {{ template "appname" . }}

    track: "{{ .Values.application.track }}"

    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"

    release: {{ .Release.Name }}

    heritage: {{ .Release.Service }}
@@ -26,4 +27,5 @@ spec: 
  selector:

    app: {{ template "appname" . }}

    tier: "{{ .Values.application.tier }}"

    track: "{{ .Values.application.track }}"

{{- end -}}

assets/auto-deploy-app/test/template_test.go

+134 −26
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ import ( 
)



const (

	chartName     = "auto-deploy-app-2.0.0-beta.1"

	chartName     = "auto-deploy-app-2.0.0-beta.2"

	helmChartPath = ".."

)
@@ -719,6 +719,65 @@ SecRule REQUEST_HEADERS:Content-Type \"text/plain\" \"log,deny,id:\'20010\',stat 
	}

}



func TestIngressTemplate_DifferentTracks(t *testing.T) {

	templates := []string{"templates/ingress.yaml"}

	tcs := []struct {

		name        string

		releaseName string

		values      map[string]string



		expectedName                     string

		expectedLabels                   map[string]string

		expectedSelector                 map[string]string

		expectedAnnotations              map[string]string

		expectedInexistentAnnotationKeys []string

		expectedErrorRegexp              *regexp.Regexp

	}{

		{

			name:                             "defaults",

			releaseName:                      "production",

			expectedName:                     "production-auto-deploy",

			expectedAnnotations:              map[string]string{"kubernetes.io/ingress.class": "nginx"},

			expectedInexistentAnnotationKeys: []string{"nginx.ingress.kubernetes.io/canary"},

		},

		{

			name:                             "with canary track",

			releaseName:                      "production-canary",

			values:                           map[string]string{"application.track": "canary"},

			expectedName:                     "production-canary-auto-deploy",

			expectedAnnotations:              map[string]string{"nginx.ingress.kubernetes.io/canary": "true", "nginx.ingress.kubernetes.io/canary-by-header": "canary", "kubernetes.io/ingress.class": "nginx"},

			expectedInexistentAnnotationKeys: []string{"nginx.ingress.kubernetes.io/canary-weight"},

		},

		{

			name:                "with canary weight",

			releaseName:         "production-canary",

			values:              map[string]string{"application.track": "canary", "ingress.canary.weight": "25"},

			expectedName:        "production-canary-auto-deploy",

			expectedAnnotations: map[string]string{"nginx.ingress.kubernetes.io/canary-weight": "25"},

		},

	}



	for _, tc := range tcs {

		t.Run(tc.name, func(t *testing.T) {

			output, ret := renderTemplate(t, tc.values, tc.releaseName, templates, tc.expectedErrorRegexp)



			if ret == false {

				return

			}



			ingress := new(extensions.Ingress)

			helm.UnmarshalK8SYaml(t, output, ingress)

			require.Equal(t, tc.expectedName, ingress.ObjectMeta.Name)

			for key, value := range tc.expectedAnnotations {

				require.Equal(t, ingress.ObjectMeta.Annotations[key], value)

			}

			for _, key := range tc.expectedInexistentAnnotationKeys {

				require.Empty(t, ingress.ObjectMeta.Annotations[key])

			}

		})

	}

}



func TestIngressTemplate_Disable(t *testing.T) {

	templates := []string{"templates/ingress.yaml"}

	releaseName := "ingress-disable-test"
@@ -738,11 +797,6 @@ func TestIngressTemplate_Disable(t *testing.T) { 
			values:       map[string]string{"ingress.enabled": "null", "service.enabled": "true"},

			expectedName: releaseName + "-auto-deploy",

		},

		{

			name:                "with service enabled and track non-stable",

			values:              map[string]string{"service.enabled": "true", "application.track": "non-stable"},

			expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/ingress.yaml in chart"),

		},

		{

			name:                "with service disabled and track stable",

			values:              map[string]string{"service.enabled": "false", "application.track": "stable"},
@@ -758,11 +812,6 @@ func TestIngressTemplate_Disable(t *testing.T) { 
			values:              map[string]string{"ingress.enabled": "false"},

			expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/ingress.yaml in chart"),

		},

		{

			name:                "with ingress enabled and track non-stable",

			values:              map[string]string{"ingress.enabled": "true", "application.track": "non-stable"},

			expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/ingress.yaml in chart"),

		},

		{

			name:                "with ingress enabled and service disabled",

			values:              map[string]string{"ingress.enabled": "true", "service.enabled": "false"},
@@ -798,6 +847,56 @@ func TestIngressTemplate_Disable(t *testing.T) { 
	}

}



func TestServiceTemplate_DifferentTracks(t *testing.T) {

	templates := []string{"templates/service.yaml"}

	tcs := []struct {

		name        string

		releaseName string

		values      map[string]string



		expectedName        string

		expectedLabels      map[string]string

		expectedSelector    map[string]string

		expectedErrorRegexp *regexp.Regexp

	}{

		{

			name:             "defaults",

			releaseName:      "production",

			expectedName:     "production-auto-deploy",

			expectedLabels:   map[string]string{"app": "production", "release": "production", "track": "stable"},

			expectedSelector: map[string]string{"app": "production", "tier": "web", "track": "stable"},

		},

		{

			name:             "with canary track",

			releaseName:      "production-canary",

			values:           map[string]string{"application.track": "canary"},

			expectedName:     "production-canary-auto-deploy",

			expectedLabels:   map[string]string{"app": "production-canary", "release": "production-canary", "track": "canary"},

			expectedSelector: map[string]string{"app": "production-canary", "tier": "web", "track": "canary"},

		},

	}



	for _, tc := range tcs {

		t.Run(tc.name, func(t *testing.T) {

			output, ret := renderTemplate(t, tc.values, tc.releaseName, templates, tc.expectedErrorRegexp)



			if ret == false {

				return

			}



			service := new(coreV1.Service)

			helm.UnmarshalK8SYaml(t, output, service)

			require.Equal(t, tc.expectedName, service.ObjectMeta.Name)

			for key, value := range tc.expectedLabels {

				require.Equal(t, service.ObjectMeta.Labels[key], value)

			}

			for key, value := range tc.expectedSelector {

				require.Equal(t, service.Spec.Selector[key], value)

			}

		})

	}

}



func TestServiceTemplate_Disable(t *testing.T) {

	templates := []string{"templates/service.yaml"}

	releaseName := "service-disable-test"
@@ -812,11 +911,6 @@ func TestServiceTemplate_Disable(t *testing.T) { 
			name:         "defaults",

			expectedName: releaseName + "-auto-deploy",

		},

		{

			name:                "with service enabled and track non-stable",

			values:              map[string]string{"service.enabled": "true", "application.track": "non-stable"},

			expectedErrorRegexp: regexp.MustCompile("Error: could not find template templates/service.yaml in chart"),

		},

		{

			name:                "with service disabled and track stable",

			values:              map[string]string{"service.enabled": "false", "application.track": "stable"},
@@ -831,17 +925,9 @@ func TestServiceTemplate_Disable(t *testing.T) { 


	for _, tc := range tcs {

		t.Run(tc.name, func(t *testing.T) {

			opts := &helm.Options{

				SetValues: tc.values,

			}

			output, err := helm.RenderTemplateE(t, opts, helmChartPath, releaseName, templates)

			output, ret := renderTemplate(t, tc.values, releaseName, templates, tc.expectedErrorRegexp)



			if tc.expectedErrorRegexp != nil {

				require.Regexp(t, tc.expectedErrorRegexp, err.Error())

				return

			}

			if err != nil {

				t.Error(err)

			if ret == false {

				return

			}
@@ -852,6 +938,28 @@ func TestServiceTemplate_Disable(t *testing.T) { 
	}

}



func renderTemplate(t *testing.T, values map[string]string, releaseName string, templates []string, expectedErrorRegexp *regexp.Regexp) (string, bool) {

	opts := &helm.Options{

		SetValues: values,

	}



	output, err := helm.RenderTemplateE(t, opts, helmChartPath, releaseName, templates)

	if expectedErrorRegexp != nil {

		if err == nil {

			t.Error("Expected error but didn't happen")

		} else {

			require.Regexp(t, expectedErrorRegexp, err.Error())

		}

		return "", false

	}

	if err != nil {

		t.Error(err)

		return "", false

	}



	return output, true

}



type workerDeploymentTestCase struct {

	ExpectedName         string

	ExpectedCmd          []string