Commit 7e2a0a96 authored by Hordur Freyr Yngvason's avatar Hordur Freyr Yngvason Committed by Shinya Maeda
Browse files

feat: add defaults to improve k8s agent experience

parent b0024f73

.gitlab/ci/test.gitlab-ci.yml

+54 −66
Original line number Diff line number Diff line
@@ -9,9 +9,6 @@ 
    matrix:

      - K3S_VERSION:

        - v1.18.19-k3s1

        - v1.19.9-k3s1

        - v1.20.7-k3s1

        - v1.21.1-k3s1

        - v1.22.2-k3s2



  services:
@@ -53,14 +50,6 @@ test-kube-domain: 
  script:

    - auto-deploy check_kube_domain



test-kube-domain-legacy:

  <<: *test-job

  variables:

    GIT_STRATEGY: none

    AUTO_DEVOPS_DOMAIN: example.com

  script:

    - auto-deploy check_kube_domain && expected_error || failed_as_expected



test-kube-domain_error:

  <<: *test-job

  variables:
@@ -207,7 +196,7 @@ test-create-secret: 
  <<: *test-job

  variables:

    GIT_STRATEGY: none

    KUBE_NAMESPACE: default

    EXPECTED_NAMESPACE: default

    CI_REGISTRY: example.com

    CI_DEPLOY_USER: ci-deploy-user

    CI_DEPLOY_PASSWORD: ci-deploy-password
@@ -215,7 +204,7 @@ test-create-secret: 
    CI_PROJECT_VISIBILITY: private

  script:

    - auto-deploy create_secret

    - kubectl get secret "gitlab-registry-${CI_PROJECT_PATH_SLUG}" -n $KUBE_NAMESPACE

    - kubectl get secret "gitlab-registry-${CI_PROJECT_PATH_SLUG}" -n $EXPECTED_NAMESPACE



test-create-secret-public-project:

  <<: *test-job
@@ -225,7 +214,7 @@ test-create-secret-public-project: 
    KUBE_NAMESPACE: default

  script:

    - auto-deploy create_secret

    - kubectl get secret "gitlab-registry-${CI_PROJECT_PATH_SLUG}" -n $KUBE_NAMESPACE && expected_error || failed_as_expected

    - kubectl get secret "gitlab-registry-${CI_PROJECT_PATH_SLUG}" -n $EXPECTED_NAMESPACE && expected_error || failed_as_expected



test-persist-environment-url:

  <<: *test-job
@@ -236,65 +225,55 @@ test-persist-environment-url: 
    - auto-deploy persist_environment_url

    - grep review-app.example.com environment_url.txt



test-install-postgres:

  <<: *test-job

  variables:

    GIT_STRATEGY: none

    CI_ENVIRONMENT_SLUG: production

    KUBE_NAMESPACE: default

    AUTO_DEVOPS_POSTGRES_CHANNEL: 2

    POSTGRES_ENABLED: "true"

    POSTGRES_USER: user

    POSTGRES_PASSWORD: testing-password

    POSTGRES_DB: $CI_ENVIRONMENT_SLUG

  script:

    - mkdir -p .gitlab

    - "echo 'custom_key: custom_value' > .gitlab/auto-deploy-postgres-values.yaml"

    - auto-deploy download_chart

    - auto-deploy install_postgresql

    - helm get values production-postgresql --namespace "$KUBE_NAMESPACE" --output json | grep -q '"custom_key":"custom_value"' || exit 1

    - kubectl get statefulset production-postgresql -n $KUBE_NAMESPACE



test-deploy:

  <<: *test-job

  variables: &deploy-variables

  variables:

    CI_APPLICATION_REPOSITORY: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/auto-build-image-with-psql"

    CI_APPLICATION_TAG: "5d248f6fa69a"

    CI_ENVIRONMENT_SLUG: production

    CI_ENVIRONMENT_URL: example.com

    ADDITIONAL_HOSTS: '*.example.com, extra.host.com'

    CI_PROJECT_PATH_SLUG: "gitlab-org/cluster-integration/auto-build-image"

    CI_PROJECT_VISIBILITY: public

    KUBE_NAMESPACE: default

    KUBE_INGRESS_BASE_DOMAIN: example.com

    POSTGRES_USER: user

    POSTGRES_PASSWORD: testing-password

    POSTGRES_ENABLED: "true"

    POSTGRES_DB: $CI_ENVIRONMENT_SLUG

    POSTGRES_ENABLED: "false"

    HELM_HOST: "localhost:44134"

    EXPECTED_NAMESPACE: default

  script:

    - auto-deploy use_kube_context

    - auto-deploy download_chart

    - auto-deploy ensure_namespace

    - auto-deploy deploy

    - helm get all production

    - helm get values production --output json | grep "postgres://user:testing-password@production-postgresql:5432/production"

    - ./test/verify-deployment-database production postgresql

    - helm -n "$EXPECTED_NAMESPACE" get all production



test-deploy-postgres-disabled:

test-deploy-custom-context:

  extends: test-deploy

  variables:

    POSTGRES_ENABLED: "false"

    KUBE_CONTEXT: default



test-deploy-custom-namespace:

  extends: test-deploy

  variables:

    KUBE_NAMESPACE: custom-namespace

    EXPECTED_NAMESPACE: custom-namespace



test-deploy-postgres-enabled:

  extends: test-deploy

  variables:

    POSTGRES_ENABLED: "true"

    POSTGRES_USER: user

    POSTGRES_PASSWORD: testing-password

    POSTGRES_DB: $CI_ENVIRONMENT_SLUG

  script:

    - auto-deploy initialize_tiller

    - auto-deploy download_chart

    - auto-deploy deploy

    - helm get production

    - helm list > releases.txt

    - if grep -q "postgres" releases.txt; then echo "postgresql should not be installed"; exit 1; fi

    - helm -n "$EXPECTED_NAMESPACE" get production

    - helm -n "$EXPECTED_NAMESPACE" get values production --output json | grep "postgres://user:testing-password@production-postgresql:5432/production"

    - ./test/verify-deployment-database production postgresql



test-deploy-atomic:

  extends: test-deploy

  variables:

    POSTGRES_ENABLED: "false"

    KUBE_INGRESS_BASE_DOMAIN: ""

  script:

    - auto-deploy download_chart
@@ -329,8 +308,6 @@ test-deploy-debug: 


test-deploy-when-stable-chart-repository-is-unreachable:

  extends: test-deploy

  variables:

    <<: *deploy-variables

  script:

    - echo "127.0.0.1 kubernetes-charts.storage.googleapis.com" >> /etc/hosts

    - auto-deploy initialize_tiller
@@ -352,14 +329,14 @@ test-scale-does-not-create-old-postgres: 
      fi



test-show-warning-for-legacy-in-cluster-postgresql:

  extends: test-deploy

  extends: test-deploy-postgres-enabled

  script:

    # Create a release/deployment

    - auto-deploy download_chart

    - auto-deploy deploy

    # Forcibly update the release that a legacy in-cluster postgresql exists in it

    - helm upgrade --reuse-values --wait --set postgresql.enabled="true" --namespace="$KUBE_NAMESPACE" "${CI_ENVIRONMENT_SLUG}" chart/

    - helm get values --namespace "$KUBE_NAMESPACE" --output json "${CI_ENVIRONMENT_SLUG}"

    - helm upgrade --reuse-values --wait --set postgresql.enabled="true" --namespace="$EXPECTED_NAMESPACE" "${CI_ENVIRONMENT_SLUG}" chart/

    - helm get values --namespace "$EXPECTED_NAMESPACE" --output json "${CI_ENVIRONMENT_SLUG}"

    # It should see an error when the deployment is upgraded

    - auto-deploy deploy| tee deploy.log || true

    - grep -q "Detected an existing PostgreSQL database" deploy.log || exit 1
@@ -371,33 +348,44 @@ test-deploy-canary: 
    - auto-deploy deploy canary

    - helm get all production-canary

    # It should have Canary Ingress

    - kubectl describe ingress production-canary-auto-deploy -n $KUBE_NAMESPACE > ingress.spec

    - kubectl describe ingress production-canary-auto-deploy -n $EXPECTED_NAMESPACE > ingress.spec

    - grep -q 'nginx.ingress.kubernetes.io/canary:.*true' ingress.spec || exit 1



test-deploy-modsecurity:

  extends: test-deploy

  variables:

    <<: *deploy-variables

    AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE: "On"

  script:

    - auto-deploy download_chart

    - auto-deploy deploy

    - $([[ $(kubectl get ingress production-auto-deploy -n $KUBE_NAMESPACE --no-headers=true -o custom-columns=:"metadata.annotations.nginx\.ingress\.kubernetes\.io/modsecurity-snippet") != "<none>" ]])

    - $([[ $(kubectl get ingress production-auto-deploy -n $EXPECTED_NAMESPACE --no-headers=true -o custom-columns=:"metadata.annotations.nginx\.ingress\.kubernetes\.io/modsecurity-snippet") != "<none>" ]])



test-create-application-secret:

  <<: *test-job

  variables:

    KUBE_NAMESPACE: default

    EXPECTED_NAMESPACE: default

    CI_ENVIRONMENT_SLUG: production

    K8S_SECRET_CODE: 12345

    K8S_SECRET_CODE_MULTILINE: "12345

    NEW LINE"

  script:

    - auto-deploy create_application_secret "stable"

    - kubectl get secrets -n $KUBE_NAMESPACE

    - kubectl get secrets production-secret -n $KUBE_NAMESPACE

    - kubectl get secrets -n $EXPECTED_NAMESPACE

    - kubectl get secrets production-secret -n $EXPECTED_NAMESPACE

    - ./test/verify-application-secret



test-install-postgres:

  extends: test-deploy-postgres-enabled

  variables:

    GIT_STRATEGY: none

  script:

    - mkdir -p .gitlab

    - "echo 'custom_key: custom_value' > .gitlab/auto-deploy-postgres-values.yaml"

    - auto-deploy download_chart

    - auto-deploy install_postgresql

    - helm get values production-postgresql --namespace "$EXPECTED_NAMESPACE" --output json | grep -q '"custom_key":"custom_value"' || exit 1

    - kubectl get statefulset production-postgresql -n $EXPECTED_NAMESPACE



test-delete:

  extends: test-deploy

  script:
@@ -423,22 +411,22 @@ test-delete-failed: 
    - helm get all production && expected_error || failed_as_expected



test-delete-postgresql:

  extends: test-deploy

  extends: test-deploy-postgres-enabled

  script:

    - auto-deploy download_chart

    - auto-deploy deploy

    - helm get all production

    - helm get all production-postgresql

    - pvc_before_delete=$(kubectl -n $KUBE_NAMESPACE get pvc -l release=production-postgresql)

    - pvc_before_delete=$(kubectl -n $EXPECTED_NAMESPACE get pvc -l release=production-postgresql)

    - if [[ -z "$pvc_before_delete" ]]; then "expected to find a postgresql pvc"; exit 1; fi

    - auto-deploy delete

    - helm get all production && expected_error || failed_as_expected

    - helm get all production-postgresql && expected_error || failed_as_expected

    - pvc_after_delete=$(kubectl -n $KUBE_NAMESPACE get pvc -l release=production-postgresql)

    - pvc_after_delete=$(kubectl -n $EXPECTED_NAMESPACE get pvc -l release=production-postgresql)

    - if [[ -n "$pvc_after_delete" ]]; then echo "no postgresql pvc should be present"; exit 1; fi



test-delete-postgresql-failed:

  extends: test-deploy

  extends: test-deploy-postgres-enabled

  script:

    - auto-deploy download_chart

    - auto-deploy deploy
@@ -453,12 +441,12 @@ test-delete-postgresql-failed: 
    - auto-deploy delete

    - helm get all production && expected_error || failed_as_expected

    - helm get all production-postgresql && expected_error || failed_as_expected

    - pvc_after_delete=$(kubectl -n $KUBE_NAMESPACE get pvc -l release=production-postgresql)

    - pvc_after_delete=$(kubectl -n $EXPECTED_NAMESPACE get pvc -l release=production-postgresql)

    - if [[ -n "$pvc_after_delete" ]]; then echo "no postgresql pvc should be present"; exit 1; fi





test-delete-canary-postgresql:

  extends: test-deploy

  extends: test-deploy-postgres-enabled

  script:

    - auto-deploy download_chart

    - auto-deploy deploy canary

doc/api.md

+2 −1
Original line number Diff line number Diff line
@@ -15,8 +15,9 @@ auto-deploy-image provides the following APIs to orchestrate [GitLab Auto Deploy 
| `CI_PROJECT_PATH_SLUG`                 | string | yes       | See [GitLab CI Predefined Variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html). | v0.1.0 ~ |

| `CI_PROJECT_VISIBILITY`                | string | yes       | See [GitLab CI Predefined Variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html). | v0.1.0 ~ |

| `CI_REGISTRY_IMAGE`                    | string | yes       | See [GitLab CI Predefined Variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html). | v0.1.0 ~ |

| `KUBE_CONTEXT`                         | string | no        | Context to use from within `KUBECONFIG` | v2.16.0 ~ |

| `KUBE_INGRESS_BASE_DOMAIN`             | string | yes       | See [GitLab Cluster Integration Deployment Variables](https://docs.gitlab.com/ee/user/project/clusters/). | v0.1.0 ~ |

| `KUBE_NAMESPACE`                       | string | yes       | See [GitLab Cluster Integration Deployment Variables](https://docs.gitlab.com/ee/user/project/clusters/). | v0.1.0 ~ |

| `KUBE_NAMESPACE`                       | string | no        | The deployment namespace. If not specified, the context default will be used. If the context has no default, falls back to `default` | v0.1.0 ~ |

| `KUBECONFIG`                           | string | yes       | See [GitLab Cluster Integration Deployment Variables](https://docs.gitlab.com/ee/user/project/clusters/). | v0.1.0 ~ |

| `AUTO_DEVOPS_DEPLOY_DEBUG`             | boolean | no       | See [Customizing Auto DevOps](https://docs.gitlab.com/ee/topics/autodevops/customize.html). | [v0.16.0](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/compare/v0.15.0...v0.16.0) ~ |

| `HELM_RELEASE_NAME`                    | string | no        | See [Customizing Auto DevOps](https://docs.gitlab.com/ee/topics/autodevops/customize.html). | v0.1.0 ~ |

src/bin/auto-deploy

+19 −2
Original line number Diff line number Diff line
@@ -12,6 +12,24 @@ export ASSETS_DIR='/assets' 
export ASSETS_CHART_DIR="${ASSETS_DIR}/auto-deploy-app"

export ROLLOUT_RESOURCE_TYPE="${ROLLOUT_RESOURCE_TYPE:-deployment}"



if [[ -z "$KUBE_NAMESPACE" ]]; then

  context_namespace=$(kubectl config view --minify -o jsonpath='{..namespace}')

  if [[ -n "$context_namespace" ]]; then

    export KUBE_NAMESPACE="$context_namespace"

  else

    export KUBE_NAMESPACE='default'

  fi

fi



function use_kube_context() {

  if [[ -z "$KUBE_CONTEXT" ]]; then

    echo "KUBE_CONTEXT not defined. The default context (if present) will be used"

    return

  fi

  echo "Using context '$KUBE_CONTEXT'"

  kubectl config use-context "$KUBE_CONTEXT"

}



function check_kube_domain() {

  if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then

    echo "In order to deploy or use Review Apps,"
@@ -57,7 +75,6 @@ function download_chart() { 
  else

    helm dependency update chart/

  fi



}



function add_chart_repositories() {
@@ -534,7 +551,7 @@ function get_replicas() { 


option=$1

case $option in



  use_kube_context) use_kube_context ;;

  check_kube_domain) check_kube_domain ;;

  download_chart) download_chart ;;

  ensure_namespace) ensure_namespace ;;

test/verify-application-secret

+2 −2
Original line number Diff line number Diff line 
#!/bin/bash -e



result=$(kubectl -n "$KUBE_NAMESPACE" get secret production-secret -o json | jq .data.CODE | xargs echo | base64 -d)

result=$(kubectl -n "$EXPECTED_NAMESPACE" get secret production-secret -o json | jq .data.CODE | xargs echo | base64 -d)

if [[ "$result" != "$K8S_SECRET_CODE" ]]; then exit 1; fi



result=$(kubectl -n "$KUBE_NAMESPACE" get secret production-secret -o json | jq .data.CODE_MULTILINE | xargs echo | base64 -d)

result=$(kubectl -n "$EXPECTED_NAMESPACE" get secret production-secret -o json | jq .data.CODE_MULTILINE | xargs echo | base64 -d)

if [[ "$result" != "$K8S_SECRET_CODE_MULTILINE" ]]; then exit 1; fi

test/verify-deployment-database

+3 −3
Original line number Diff line number Diff line
@@ -3,11 +3,11 @@ 
appLabel=$1

dbLabel=$2



pod_name=$(kubectl get pod -l "app=$appLabel" -n "$KUBE_NAMESPACE" --sort-by '{.metadata.creationTimestamp}' -o custom-columns=:metadata.name --no-headers | tail -n 1)

pod_name=$(kubectl -n "$EXPECTED_NAMESPACE" get pod -l "app=$appLabel" --sort-by '{.metadata.creationTimestamp}' -o custom-columns=:metadata.name --no-headers | tail -n 1)



# Wait for DB pod to be ready

count=0

while [[ $(kubectl get pods -l "app=$dbLabel" -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do

while [[ $(kubectl -n "$EXPECTED_NAMESPACE" get pods -l "app=$dbLabel" -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do

  [[ $count -lt 30 ]] || exit 1



  echo "waiting for pod" && sleep 1
@@ -16,4 +16,4 @@ done 


# SC2016 does not apply to the quote for command pass to sh -c

# shellcheck disable=SC2016

kubectl exec -i -n "$KUBE_NAMESPACE" "$pod_name" -- sh -c 'psql $DATABASE_URL -c "SELECT 1;"'
 
kubectl exec -i -n "$EXPECTED_NAMESPACE" "$pod_name" -- sh -c 'psql $DATABASE_URL -c "SELECT 1;"'