add e2e binary to Jenkins tests (#808)

* add auth plugin support to e2e tests * add e2e testing to Jenkins
kubernetes-retired · May 10, 2017 · 84ff0de · blacksagi · May 30, 2017 · 84ff0de
1 parent 141403b
commit 84ff0de
Show file tree

Hide file tree

Showing 509 changed files with 99,254 additions and 40 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -120,14 +120,26 @@ node {
             --fix-auth \
             --create-artifacts
       """
+
+      // Run the e2e test framework
+      sh """${env.ROOT}/contrib/jenkins/run_e2e.sh \
+            --registry gcr.io/${test_project}/catalog/ \
+	    --version ${version} \
+	    --cleanup \
+	    --create-artifacts
+      """
+
+      echo 'Run succeeded.'
     } catch (Exception e) {
+      echo 'Run failed.'
       currentBuild.result = 'FAILURE'
     } finally {
-      archiveArtifacts artifacts: 'etcd-backed*.txt', fingerprint: true
-      archiveArtifacts artifacts: 'tpr-backed*.txt', fingerprint: true
+      archiveArtifacts artifacts: 'walkthrough*.txt', fingerprint: true
+      archiveArtifacts artifacts: 'e2e*.txt', fingerprint: true
       try {
         sh """${env.ROOT}/contrib/jenkins/cleanup_cluster.sh --kubeconfig ${KUBECONFIG}"""
       } catch (Exception e) {
+        echo 'Exception caught during cleanup.'
         currentBuild.result = 'FAILURE'
       }
     }

diff --git a/contrib/hack/test_walkthrough.sh b/contrib/hack/test_walkthrough.sh
@@ -57,9 +57,11 @@ echo '-------------------'
 function cleanup() {
   if [[ -n "${CREATE_ARTIFACTS:-}" ]]; then
     echo 'Creating artifacts...'
-    PREFIX='etcd-backed'
+    PREFIX='walkthrough_'
     if [[ -n "${WITH_TPR:-}" ]]; then
-      PREFIX='tpr-backed'
+      PREFIX+='tpr-backed'
+    else
+      PREFIX+='etcd-backed'
     fi
 
     KUBECONFIG="${K8S_KUBECONFIG}" "${ROOT}/contrib/hack/create_artifacts.sh" \

diff --git a/contrib/jenkins/install_catalog.sh b/contrib/jenkins/install_catalog.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o nounset
+set -o errexit
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+. "${ROOT}/contrib/hack/utilities.sh" || { echo 'Cannot load bash utilities.'; exit 1; }
+
+while [[ $# -gt 0 ]]; do
+  case "${1}" in
+    --registry)               REGISTRY="${2:-}"; shift ;;
+    --version)                VERSION="${2:-}"; shift ;;
+    --release-name)           CATALOG_RELEASE="${2:-}"; shift ;;
+    --service-catalog-config) SC_KUBECONFIG="${2:-}"; shift ;;
+    --with-tpr)               WITH_TPR=1 ;;
+    --fix-auth)               FIX_CONFIGMAP=1 ;;
+    *) error_exit "Unrecognized command line parameter: $1" ;;
+  esac
+  shift
+done
+
+CATALOG_RELEASE="${CATALOG_RELEASE:-"catalog"}"
+
+K8S_KUBECONFIG="${KUBECONFIG:-~/.kube/config}"
+SC_KUBECONFIG="${SC_KUBECONFIG:-"/tmp/sc-kubeconfig"}"
+
+VERSION="${VERSION:-"canary"}"
+REGISTRY="${REGISTRY:-}"
+CONTROLLER_MANAGER_IMAGE="${REGISTRY}controller-manager:${VERSION}"
+APISERVER_IMAGE="${REGISTRY}apiserver:${VERSION}"
+
+echo 'INSTALLING CATALOG'
+echo '-------------------'
+echo "Using kubeconfig: ${K8S_KUBECONFIG}"
+echo "Using service catalog kubeconfig: ${SC_KUBECONFIG}"
+echo "Using controller-manager image: ${CONTROLLER_MANAGER_IMAGE}"
+echo "Using apiserver image: ${APISERVER_IMAGE}"
+echo '-------------------'
+
+# Deploying to cluster
+
+retry -n 10 \
+    kubectl --namespace kube-system get configmap extension-apiserver-authentication \
+  || error_exit 'Timed out waiting for extension-apiserver-authentication configmap to come up.'
+
+# The API server automatically provisions the configmap, but we need it to contain the requestheader CA as well,
+# which is only provisioned if you pass the appropriate flags to master.  GKE doesn't do this, so we work around it.
+if [[ -n "${FIX_CONFIGMAP:-}" ]] && [[ -z "$(kubectl --namespace kube-system get configmap extension-apiserver-authentication -o jsonpath="{ $.data['requestheader-client-ca-file'] }")" ]]; then
+    full_configmap=$(kubectl --namespace kube-system get configmap extension-apiserver-authentication -o json)
+    echo "$full_configmap" | jq '.data["requestheader-client-ca-file"] = .data["client-ca-file"]' | kubectl --namespace kube-system update configmap extension-apiserver-authentication -f -
+    [[ -n "$(kubectl --namespace kube-system get configmap extension-apiserver-authentication -o jsonpath="{ $.data['requestheader-client-ca-file'] }")" ]] || { echo "Could not add requestheader auth CA to extension-apiserver-authentication configmap."; exit 1; }
+fi
+
+VALUES='debug=true'
+VALUES+=',insecure=true'
+VALUES+=",controllerManager.image=${CONTROLLER_MANAGER_IMAGE}"
+VALUES+=",apiserver.image=${APISERVER_IMAGE}"
+VALUES+=',apiserver.service.type=LoadBalancer'
+if [[ -n "${WITH_TPR:-}" ]]; then
+  VALUES+=',apiserver.storage.type=tpr'
+  VALUES+=',apiserver.storage.tpr.globalNamespace=test-ns'
+fi
+
+retry -n 10 \
+    helm install "${ROOT}/charts/catalog" \
+    --name "${CATALOG_RELEASE}" \
+    --namespace "catalog" \
+    --set "${VALUES}" \
+  || error_exit 'Error deploying service catalog to cluster.'
+
+# Waiting for everything to come up
+
+echo 'Waiting on pods to come up...'
+
+wait_for_expected_output -x -e 'Pending' -n 10 \
+    kubectl get pods --namespace catalog \
+  || error_exit 'Timed out waiting for service catalog pods to come up.'
+
+wait_for_expected_output -x -e 'ContainerCreating' -n 10 \
+    kubectl get pods --namespace catalog \
+  || error_exit 'Timed out waiting for service catalog pods to come up.'
+
+[[ "$(kubectl get pods --namespace catalog | grep catalog-catalog-apiserver | awk '{print $3}')" == 'Running' ]] \
+  || {
+    POD_NAME="$(kubectl get pods --namespace catalog | grep catalog-catalog-apiserver | awk '{print $1}')"
+    kubectl get pod "${POD_NAME}" --namespace catalog
+    kubectl describe pod "${POD_NAME}" --namespace catalog
+    error_exit 'API server pod did not come up successfully.'
+  }
+
+[[ "$(kubectl get pods --namespace catalog | grep catalog-catalog-controller | awk '{print $3}')" == 'Running' ]] \
+  || {
+    POD_NAME="$(kubectl get pods --namespace catalog | grep catalog-catalog-controller | awk '{print $1}')"
+    kubectl get pod "${POD_NAME}" --namespace catalog
+    kubectl describe pod "${POD_NAME}" --namespace catalog
+    error_exit 'Controller manager pod did not come up successfully.'
+  }
+
+echo 'Waiting on external IP for service catalog API Server...'
+
+wait_for_expected_output -x -e 'pending' -n 10 \
+    kubectl get services --namespace catalog \
+  || error_exit 'Timed out waiting for external IP for service catalog API Server.'
+
+# Create kubeconfig for service catalog API server
+
+echo 'Connecting to service catalog API Server...'
+
+${ROOT}/contrib/jenkins/setup-sc-kubectl.sh --service-catalog-config "${SC_KUBECONFIG}" \
+  || error_exit 'Error connecting to service catalog API server.'
+
+echo 'Service catalog successfully installed.'
diff --git a/contrib/jenkins/run_e2e.sh b/contrib/jenkins/run_e2e.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o nounset
+set -o errexit
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+. "${ROOT}/contrib/hack/utilities.sh" || { echo 'Cannot load bash utilities.'; exit 1; }
+
+while [[ $# -gt 0 ]]; do
+  case "${1}" in
+    --registry)         REGISTRY="${2:-}"; shift ;;
+    --version)          VERSION="${2:-}"; shift ;;
+    --with-tpr)         WITH_TPR=1 ;;
+    --cleanup)          CLEANUP=1 ;;
+    --create-artifacts) CREATE_ARTIFACTS=1 ;;
+    --fix-auth)         FIX_CONFIGMAP=1 ;;
+    *) error_exit "Unrecognized command line parameter: $1" ;;
+  esac
+  shift
+done
+
+CATALOG_RELEASE="catalog"
+
+K8S_KUBECONFIG="${KUBECONFIG:-"~/.kube/config"}"
+SC_KUBECONFIG="/tmp/sc-kubeconfig"
+
+function cleanup() {
+  export KUBECONFIG="${K8S_KUBECONFIG}"
+
+  if [[ -n "${CREATE_ARTIFACTS:-}" ]]; then
+    echo 'Creating artifacts...'
+    PREFIX="e2e.test_"
+    if [[ -n "${WITH_TPR:-}" ]]; then
+      PREFIX+='tpr-backed'
+    else
+      PREFIX+='etcd-backed'
+    fi
+
+    "${ROOT}/contrib/hack/create_artifacts.sh" \
+        --prefix "${PREFIX}" --location "${ROOT}" \
+        &> /dev/null \
+        || true
+  fi
+
+  echo 'Cleaning up resources...'
+  {
+    helm delete --purge "${CATALOG_RELEASE}" || true
+    rm -f "${SC_KUBECONFIG}"
+
+    # TODO: Hack in order to delete TPRs. Will need to be removed when TPRs can be deleted
+    # by the catalog API server.
+    if [[ -n "${WITH_TPR:-}" ]]; then
+      kubectl delete thirdpartyresources binding.servicecatalog.k8s.io
+      kubectl delete thirdpartyresources instance.servicecatalog.k8s.io
+      kubectl delete thirdpartyresources broker.servicecatalog.k8s.io
+      kubectl delete thirdpartyresources service-class.servicecatalog.k8s.io
+    fi
+  } &> /dev/null
+}
+
+if [[ -n "${CLEANUP:-}" ]]; then
+  trap cleanup EXIT
+fi
+
+echo "Running 'e2e.test'..."
+
+# Install catalog
+ARGUMENTS="--registry ${REGISTRY}"
+ARGUMENTS+=" --version ${VERSION}"
+ARGUMENTS+=" --fix-auth"
+ARGUMENTS+=" --service-catalog-config ${SC_KUBECONFIG}"
+ARGUMENTS+=" --release-name ${CATALOG_RELEASE}"
+if [[ -n "${WITH_TPR:-}" ]]; then
+  ARGUMENTS+=" --with-tpr"
+fi
+
+${ROOT}/contrib/jenkins/install_catalog.sh ${ARGUMENTS} \
+  || error_exit "Error installing catalog in cluster."
+
+make bin/e2e.test \
+  || error_exit "Error when making e2e test binary."
+
+KUBECONFIG="${KUBECONFIG}" SERVICECATALOGCONFIG="${SC_KUBECONFIG}" ${ROOT}/bin/e2e.test \
+  || error_exit "Error while running e2e tests."
+
+echo "'e2e.test' completed successfully."
diff --git a/contrib/jenkins/setup-sc-kubectl.sh b/contrib/jenkins/setup-sc-kubectl.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o nounset
+set -o errexit
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+. "${ROOT}/contrib/hack/utilities.sh" || { echo 'Cannot load bash utilities.'; exit 1; }
+
+while [[ $# -gt 0 ]]; do
+  case "${1}" in
+    --service-catalog-config) SC_KUBECONFIG="${2:-}"; shift ;;
+    *) error_exit "Unrecognized command line parameter: $1" ;;
+  esac
+  shift
+done
+
+API_SERVER_HOST="$(kubectl get services -n catalog | grep 'catalog-catalog-apiserver' | awk '{print $3}')"
+
+[[ "${API_SERVER_HOST}" =~ ^[0-9.]+$ ]] \
+  || error_exit 'Error when fetching service catalog API Server IP address.'
+
+echo "Using API Server IP: ${API_SERVER_HOST}"
+
+export KUBECONFIG="${SC_KUBECONFIG:-"${KUBECONFIG}"}"
+echo "Using config file at: ${SC_KUBECONFIG}"
+kubectl config set-credentials service-catalog-creds --username=admin --password=admin
+kubectl config set-cluster service-catalog-cluster --server="http://${API_SERVER_HOST}:80"
+kubectl config set-context service-catalog-ctx --cluster=service-catalog-cluster --user=service-catalog-creds
+kubectl config use-context service-catalog-ctx
+
+retry -n 10 \
+  kubectl get brokers,serviceclasses,instances,bindings \
+  || error_exit 'Issue listing resources from service catalog API server.'
+
+echo 'Set up service catalog kubeconfig.'