#!/bin/bash # WiFi-DensePose Deployment Validation Script # This script validates that all deployment components are functioning correctly set -euo pipefail # Configuration NAMESPACE="wifi-densepose" MONITORING_NAMESPACE="monitoring" TIMEOUT=300 # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Logging functions log_info() { echo -e "${BLUE}[INFO]${NC} $1" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1" } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1" } log_error() { echo -e "${RED}[ERROR]${NC} $1" } # Check if kubectl is available and configured check_kubectl() { log_info "Checking kubectl configuration..." if ! command -v kubectl &> /dev/null; then log_error "kubectl is not installed or not in PATH" return 1 fi if ! kubectl cluster-info &> /dev/null; then log_error "kubectl is not configured or cluster is not accessible" return 1 fi log_success "kubectl is configured and cluster is accessible" return 0 } # Validate namespace exists validate_namespace() { local ns=$1 log_info "Validating namespace: $ns" if kubectl get namespace "$ns" &> /dev/null; then log_success "Namespace $ns exists" return 0 else log_error "Namespace $ns does not exist" return 1 fi } # Validate deployments are ready validate_deployments() { log_info "Validating deployments in namespace: $NAMESPACE" local deployments deployments=$(kubectl get deployments -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}') if [ -z "$deployments" ]; then log_warning "No deployments found in namespace $NAMESPACE" return 1 fi local failed=0 for deployment in $deployments; do log_info "Checking deployment: $deployment" if kubectl wait --for=condition=available --timeout="${TIMEOUT}s" "deployment/$deployment" -n "$NAMESPACE" &> /dev/null; then local ready_replicas ready_replicas=$(kubectl get deployment "$deployment" -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}') local desired_replicas desired_replicas=$(kubectl get deployment "$deployment" -n "$NAMESPACE" -o jsonpath='{.spec.replicas}') if [ "$ready_replicas" = "$desired_replicas" ]; then log_success "Deployment $deployment is ready ($ready_replicas/$desired_replicas replicas)" else log_warning "Deployment $deployment has $ready_replicas/$desired_replicas replicas ready" failed=1 fi else log_error "Deployment $deployment is not ready within ${TIMEOUT}s" failed=1 fi done return $failed } # Validate services are accessible validate_services() { log_info "Validating services in namespace: $NAMESPACE" local services services=$(kubectl get services -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}') if [ -z "$services" ]; then log_warning "No services found in namespace $NAMESPACE" return 1 fi local failed=0 for service in $services; do log_info "Checking service: $service" local endpoints endpoints=$(kubectl get endpoints "$service" -n "$NAMESPACE" -o jsonpath='{.subsets[*].addresses[*].ip}') if [ -n "$endpoints" ]; then log_success "Service $service has endpoints: $endpoints" else log_error "Service $service has no endpoints" failed=1 fi done return $failed } # Validate ingress configuration validate_ingress() { log_info "Validating ingress configuration in namespace: $NAMESPACE" local ingresses ingresses=$(kubectl get ingress -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}') if [ -z "$ingresses" ]; then log_warning "No ingress resources found in namespace $NAMESPACE" return 0 fi local failed=0 for ingress in $ingresses; do log_info "Checking ingress: $ingress" local hosts hosts=$(kubectl get ingress "$ingress" -n "$NAMESPACE" -o jsonpath='{.spec.rules[*].host}') if [ -n "$hosts" ]; then log_success "Ingress $ingress configured for hosts: $hosts" # Check if ingress has an IP/hostname assigned local address address=$(kubectl get ingress "$ingress" -n "$NAMESPACE" -o jsonpath='{.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}') if [ -n "$address" ]; then log_success "Ingress $ingress has address: $address" else log_warning "Ingress $ingress does not have an assigned address yet" fi else log_error "Ingress $ingress has no configured hosts" failed=1 fi done return $failed } # Validate ConfigMaps and Secrets validate_config() { log_info "Validating ConfigMaps and Secrets in namespace: $NAMESPACE" # Check ConfigMaps local configmaps configmaps=$(kubectl get configmaps -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}') if [ -n "$configmaps" ]; then log_success "ConfigMaps found: $configmaps" else log_warning "No ConfigMaps found in namespace $NAMESPACE" fi # Check Secrets local secrets secrets=$(kubectl get secrets -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -v "default-token" | tr '\n' ' ') if [ -n "$secrets" ]; then log_success "Secrets found: $secrets" else log_warning "No custom secrets found in namespace $NAMESPACE" fi return 0 } # Validate HPA configuration validate_hpa() { log_info "Validating Horizontal Pod Autoscaler in namespace: $NAMESPACE" local hpas hpas=$(kubectl get hpa -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}') if [ -z "$hpas" ]; then log_warning "No HPA resources found in namespace $NAMESPACE" return 0 fi local failed=0 for hpa in $hpas; do log_info "Checking HPA: $hpa" local current_replicas current_replicas=$(kubectl get hpa "$hpa" -n "$NAMESPACE" -o jsonpath='{.status.currentReplicas}') local desired_replicas desired_replicas=$(kubectl get hpa "$hpa" -n "$NAMESPACE" -o jsonpath='{.status.desiredReplicas}') if [ -n "$current_replicas" ] && [ -n "$desired_replicas" ]; then log_success "HPA $hpa: current=$current_replicas, desired=$desired_replicas" else log_warning "HPA $hpa metrics not available yet" fi done return $failed } # Test application health endpoints test_health_endpoints() { log_info "Testing application health endpoints..." # Get application pods local pods pods=$(kubectl get pods -n "$NAMESPACE" -l app=wifi-densepose -o jsonpath='{.items[*].metadata.name}') if [ -z "$pods" ]; then log_error "No application pods found" return 1 fi local failed=0 for pod in $pods; do log_info "Testing health endpoint for pod: $pod" # Port forward and test health endpoint kubectl port-forward "pod/$pod" 8080:8080 -n "$NAMESPACE" & local pf_pid=$! sleep 2 if curl -f http://localhost:8080/health &> /dev/null; then log_success "Health endpoint for pod $pod is responding" else log_error "Health endpoint for pod $pod is not responding" failed=1 fi kill $pf_pid 2>/dev/null || true sleep 1 done return $failed } # Validate monitoring stack validate_monitoring() { log_info "Validating monitoring stack in namespace: $MONITORING_NAMESPACE" if ! validate_namespace "$MONITORING_NAMESPACE"; then log_warning "Monitoring namespace not found, skipping monitoring validation" return 0 fi # Check Prometheus if kubectl get deployment prometheus-server -n "$MONITORING_NAMESPACE" &> /dev/null; then if kubectl wait --for=condition=available --timeout=60s deployment/prometheus-server -n "$MONITORING_NAMESPACE" &> /dev/null; then log_success "Prometheus is running" else log_error "Prometheus is not ready" fi else log_warning "Prometheus deployment not found" fi # Check Grafana if kubectl get deployment grafana -n "$MONITORING_NAMESPACE" &> /dev/null; then if kubectl wait --for=condition=available --timeout=60s deployment/grafana -n "$MONITORING_NAMESPACE" &> /dev/null; then log_success "Grafana is running" else log_error "Grafana is not ready" fi else log_warning "Grafana deployment not found" fi return 0 } # Validate logging stack validate_logging() { log_info "Validating logging stack..." # Check Fluentd DaemonSet if kubectl get daemonset fluentd -n kube-system &> /dev/null; then local desired desired=$(kubectl get daemonset fluentd -n kube-system -o jsonpath='{.status.desiredNumberScheduled}') local ready ready=$(kubectl get daemonset fluentd -n kube-system -o jsonpath='{.status.numberReady}') if [ "$desired" = "$ready" ]; then log_success "Fluentd DaemonSet is ready ($ready/$desired nodes)" else log_warning "Fluentd DaemonSet has $ready/$desired pods ready" fi else log_warning "Fluentd DaemonSet not found" fi return 0 } # Check resource usage check_resource_usage() { log_info "Checking resource usage..." # Check node resource usage log_info "Node resource usage:" kubectl top nodes 2>/dev/null || log_warning "Metrics server not available for node metrics" # Check pod resource usage log_info "Pod resource usage in namespace $NAMESPACE:" kubectl top pods -n "$NAMESPACE" 2>/dev/null || log_warning "Metrics server not available for pod metrics" return 0 } # Generate validation report generate_report() { local total_checks=$1 local failed_checks=$2 local passed_checks=$((total_checks - failed_checks)) echo "" log_info "=== Deployment Validation Report ===" echo "Total checks: $total_checks" echo "Passed: $passed_checks" echo "Failed: $failed_checks" if [ $failed_checks -eq 0 ]; then log_success "All validation checks passed! 🎉" return 0 else log_error "Some validation checks failed. Please review the output above." return 1 fi } # Main validation function main() { log_info "Starting WiFi-DensePose deployment validation..." local total_checks=0 local failed_checks=0 # Run validation checks checks=( "check_kubectl" "validate_namespace $NAMESPACE" "validate_deployments" "validate_services" "validate_ingress" "validate_config" "validate_hpa" "test_health_endpoints" "validate_monitoring" "validate_logging" "check_resource_usage" ) for check in "${checks[@]}"; do total_checks=$((total_checks + 1)) echo "" if ! eval "$check"; then failed_checks=$((failed_checks + 1)) fi done # Generate final report generate_report $total_checks $failed_checks } # Run main function main "$@"