#!/usr/bin/env bash # We created this because we could not monitor quotas easily in GCP monitoring (see # https://gitlab.com/gitlab-org/quality/engineering-productivity-infrastructure/-/issues/37) # # If this functionality ever becomes available, please replace this script with GCP monitoring! function k8s_resource_count() { local resource_name="${1}" kubectl get -A "${resource_name}" 2> /dev/null | wc -l | xargs } # ~13 services per review-app - ~230 review apps SERVICES_COUNT_THRESHOLD=3000 REVIEW_APPS_COUNT_THRESHOLD=200 # One review app currently deploys 4 PVCs PVCS_COUNT_THRESHOLD=$((REVIEW_APPS_COUNT_THRESHOLD * 4)) exit_with_error=false # In the current GKE cluster configuration, we should never go higher than 4096 services per cluster. services_count=$(kubectl get services -A | wc -l | xargs) if [ "${services_count}" -gt "${SERVICES_COUNT_THRESHOLD}" ]; then >&2 echo "❌ [ERROR] Services are above ${SERVICES_COUNT_THRESHOLD} (currently at ${services_count})" exit_with_error=true fi review_apps_count=$(helm ls -A | wc -l | xargs) if [ "${review_apps_count}" -gt "${REVIEW_APPS_COUNT_THRESHOLD}" ]; then >&2 echo "❌ [ERROR] Review apps count are above ${REVIEW_APPS_COUNT_THRESHOLD} (currently at ${review_apps_count})" exit_with_error=true fi namespaces_count=$(kubectl get namespaces -A | wc -l | xargs) if [ "$(echo $(($namespaces_count - $review_apps_count)) | sed 's/-//')" -gt 30 ]; then >&2 echo "❌ [ERROR] Difference between namespaces and deployed review-apps is above 30 (${namespaces_count} namespaces and ${review_apps_count} review-apps)" exit_with_error=true fi pvcs_count=$(kubectl get pvc -A | wc -l | xargs) if [ "${pvcs_count}" -gt "${PVCS_COUNT_THRESHOLD}" ]; then >&2 echo "❌ [ERROR] PVCs are above ${PVCS_COUNT_THRESHOLD} (currently at ${pvcs_count})" exit_with_error=true fi if [ "${exit_with_error}" = true ] ; then exit 1 fi echo -e "\nShow k8s resources count: " cat > k8s-resources-count.out <