#!/bin/bash source test-devworkspace-controller-load/provision-che-workspace-namespace.sh source test-devworkspace-controller-load/che-cert-bundle-utils.sh source test-devworkspace-controller-load/install-che-if-needed.sh MODE="binary" # or 'operator' EXECUTOR_MODE="shared-iterations" # or 'ramping-vus' LOAD_TEST_NAMESPACE="loadtest-devworkspaces" DWO_NAMESPACE="openshift-operators" SA_NAME="k6-devworkspace-tester" CLUSTERROLE_NAME="k6-devworkspace-role" ROLEBINDING_NAME="k6-devworkspace-binding" CONFIGMAP_NAME="k6-test-script" K6_CR_NAME="k6-test-run" K6_SCRIPT="test-devworkspace-controller-load/devworkspace_load_test.js" K6_OPERATOR_VERSION="v0.0.22" # Local template (avoids gist.githubusercontent.com timeouts under VU load). # Override with --devworkspace-link or DEVWORKSPACE_LINK=https://... if needed. _RUNK6_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DEVWORKSPACE_LINK="${_RUNK6_REPO_ROOT}/common/templates/dw-minimal.json" MAX_VUS="${MAX_VUS:-100}" DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="3600" SEPARATE_NAMESPACES="false" DELETE_DEVWORKSPACE_AFTER_READY="true" MAX_DEVWORKSPACES="-1" CREATE_AUTOMOUNT_RESOURCES="false" RUN_WITH_ECLIPSE_CHE="false" LOGS_DIR="logs" TEST_DURATION_IN_MINUTES="25" MIN_KUBECTL_VERSION="1.24.0" MIN_CURL_VERSION="7.0.0" MIN_K6_VERSION="1.1.0" CHE_NAMESPACE="" # Auto-detected based on platform CHE_CLUSTER_NAME="" # Auto-discovered from cluster TEST_CERTIFICATES_COUNT="750" # ConfigMap limit is 1048576 bytes; leave room for ~156 system certs SKIP_CLEANUP="${SKIP_CLEANUP:-false}" # Skip cleanup after load test (default: false) TOKEN_TTL="16h" # ----------- Helper Functions ----------- get_initial_pod_restart_counts() { local namespace="$1" local label_selector="$2" # Get pod restart counts as JSON object kubectl get pods -n "$namespace" -l "$label_selector" -o json 2>/dev/null | \ jq -r '.items | map({(.metadata.name): (.status.containerStatuses[0].restartCount // 0)}) | add // {}' || echo '{}' } # ----------- Main Execution Flow ----------- main() { parse_arguments "$@" check_prerequisites if [[ "$RUN_WITH_ECLIPSE_CHE" == "false" ]]; then create_namespace else install_che_if_needed detect_che_namespace provision_che_workspace_namespace "$LOAD_TEST_NAMESPACE" "$CHE_NAMESPACE" run_che_ca_bundle_e2e "$CHE_NAMESPACE" "$LOAD_TEST_NAMESPACE" "test-devworkspace" "$TEST_CERTIFICATES_COUNT" fi create_rbac start_background_watchers if [[ "$MODE" == "operator" ]]; then install_k6_operator create_k6_configmap delete_existing_testruns create_k6_test_run wait_for_test_completion fetch_test_logs elif [[ "$MODE" == "binary" ]]; then generate_token_and_api_url run_k6_binary_test else echo "โŒ Invalid mode: $MODE" exit 1 fi stop_background_watchers # Only delete namespace if SKIP_CLEANUP is not enabled if [[ "$SKIP_CLEANUP" != "true" ]]; then delete_namespace else echo "โ„น๏ธ Skipping namespace deletion - SKIP_CLEANUP is enabled" fi } # ----------- Helper Functions ----------- detect_che_namespace() { if [[ -z "$CHE_NAMESPACE" ]]; then echo "๐Ÿ” Auto-detecting CheCluster namespace..." # Try to find CheCluster in any namespace local checluster_ns checluster_ns=$(kubectl get checluster --all-namespaces -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null || true) if [[ -n "$checluster_ns" ]]; then CHE_NAMESPACE="$checluster_ns" echo "โœ… Found CheCluster in namespace: $CHE_NAMESPACE" else echo "โŒ No CheCluster found in any namespace. Please install Eclipse Che/Dev Spaces first." exit 1 fi fi } print_help() { cat < Mode to run the script (default: binary) --executor K6 executor mode (default: shared-iterations) --max-vus Number of virtual users for k6 (default: 100) --max-devworkspaces Maximum number of DevWorkspaces to create (by default, it's not specified) --separate-namespaces Use separate namespaces for workspaces (default: false) --delete-devworkspace-after-ready Delete DevWorkspace once it becomes Ready (default: true) --devworkspace-ready-timeout-seconds Timeout in seconds for workspace to become ready (default: 1200) --devworkspace-link DevWorkspace link (default: empty, opinionated DevWorkspace is created) --create-automount-resources Whether to create automount resources (default: false) --dwo-namespace DevWorkspace Operator namespace (default: loadtest-devworkspaces) --logs-dir Directory name where DevWorkspace and event logs would be dumped --test-duration-minutes Duration in minutes for which to run load tests (default: 25 minutes) --run-with-eclipse-che Whether these tests are supposed to be run with Eclipse Che (If yes additional certificates are mounted) --che-cluster-name CheCluster name (auto-discovered if not specified) --che-namespace CheCluster namespace (auto-detected: openshift-operators on OpenShift, eclipse-che otherwise) -h, --help Show this help message EOF } parse_arguments() { while [[ $# -gt 0 ]]; do case "$1" in --mode) MODE="$2"; shift 2;; --executor) EXECUTOR_MODE="$2"; shift 2;; --max-vus) MAX_VUS="$2"; shift 2;; --separate-namespaces) SEPARATE_NAMESPACES="$2"; shift 2;; --max-devworkspaces) MAX_DEVWORKSPACES="$2"; shift 2;; --delete-devworkspace-after-ready) DELETE_DEVWORKSPACE_AFTER_READY="$2"; shift 2;; --devworkspace-ready-timeout-seconds) DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="$2"; shift 2;; --devworkspace-link) DEVWORKSPACE_LINK="$2"; shift 2;; --create-automount-resources) CREATE_AUTOMOUNT_RESOURCES="$2"; shift 2;; --dwo-namespace) LOAD_TEST_NAMESPACE="$2"; shift 2;; --logs-dir) LOGS_DIR="$2"; shift 2;; --test-duration-minutes) TEST_DURATION_IN_MINUTES="$2"; shift 2;; --run-with-eclipse-che) RUN_WITH_ECLIPSE_CHE="$2"; shift 2;; --che-cluster-name) CHE_CLUSTER_NAME="$2"; shift 2;; --che-namespace) CHE_NAMESPACE="$2"; shift 2;; -h|--help) print_help; exit 0;; *) echo "โŒ Unknown option: $1" print_help; exit 1;; esac done } create_namespace() { echo "๐Ÿ”ง Creating Namespace: $LOAD_TEST_NAMESPACE" cat <> "$logfile" # Every 100 lines, truncate to keep only last max_lines if (( RANDOM % 100 == 0 )); then tail -n "$max_lines" "$logfile" > "$logfile.tmp" && mv "$logfile.tmp" "$logfile" fi done } start_background_watchers() { echo "๐Ÿ“ Creating logs dir ..." mkdir -p ${LOGS_DIR} TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S) echo "๐Ÿ” Starting background watchers (keeping last 1000 lines)..." kubectl get events --field-selector involvedObject.kind=Pod --watch --all-namespaces 2>&1 | \ rotate_log "${LOGS_DIR}/${TIMESTAMP}_events.log" 1000 & PID_EVENTS_WATCH=$! kubectl get dw --watch --all-namespaces 2>&1 | \ rotate_log "${LOGS_DIR}/${TIMESTAMP}_dw_watch.log" 1000 & PID_DW_WATCH=$! log_failed_devworkspaces & PID_FAILED_DW_POLL=$! } log_failed_devworkspaces() { echo "๐Ÿ“„ Starting periodic failed DevWorkspaces report (every 10s)..." POLL_INTERVAL=10 # in seconds ITERATIONS=$((((TEST_DURATION_IN_MINUTES-1) * 60) / POLL_INTERVAL)) for ((i = 0; i < ITERATIONS; i++)); do OUTPUT=$(kubectl get devworkspaces --all-namespaces -o json | jq -r ' .items[] | select(.status.phase == "Failed") | [ .metadata.namespace, .metadata.name, .status.phase, (.status.message // "No message") ] | @csv') if [ -n "$OUTPUT" ]; then echo "$OUTPUT" > "${LOGS_DIR}/dw_failure_report.csv" fi sleep "$POLL_INTERVAL" done } stop_background_watchers() { echo "๐Ÿ›‘ Stopping background watchers..." local pids=() [[ -n "${PID_EVENTS_WATCH:-}" ]] && pids+=("$PID_EVENTS_WATCH") [[ -n "${PID_DW_WATCH:-}" ]] && pids+=("$PID_DW_WATCH") [[ -n "${PID_FAILED_DW_POLL:-}" ]] && pids+=("$PID_FAILED_DW_POLL") if [[ ${#pids[@]} -gt 0 ]]; then kill "${pids[@]}" 2>/dev/null || true fi } install_k6_operator() { echo "๐Ÿ“ฆ Installing k6 operator..." curl -L "https://raw.githubusercontent.com/grafana/k6-operator/refs/tags/${K6_OPERATOR_VERSION}/bundle.yaml" | kubectl apply -f - echo "โณ Waiting until k6 operator deployment is ready..." kubectl rollout status deployment/k6-operator-controller-manager -n k6-operator-system --timeout=300s } create_k6_configmap() { echo "๐Ÿงฉ Creating ConfigMap from script file: $K6_SCRIPT" kubectl create configmap "$CONFIGMAP_NAME" \ --from-file=script.js="$K6_SCRIPT" \ --namespace "$LOAD_TEST_NAMESPACE" \ --dry-run=client -o yaml | kubectl apply -f - } delete_existing_testruns() { echo "๐Ÿงน Deleting any existing K6 TestRun resources in namespace: $LOAD_TEST_NAMESPACE" kubectl delete testrun --all -n "$LOAD_TEST_NAMESPACE" || true } create_k6_test_run() { echo "๐Ÿš€ Creating K6 TestRun custom resource..." cat </dev/null) if [[ "$stage" == "finished" ]]; then echo "TestRun $K6_CR_NAME is finished." break fi if (( SECONDS >= end )); then echo "Timeout waiting for TestRun $K6_CR_NAME to finish." exit 1 fi sleep "$INTERVAL" done } fetch_test_logs() { K6_TEST_POD=$(kubectl get pod -l k6_cr=$K6_CR_NAME,runner=true -n "${LOAD_TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}') echo "๐Ÿ“œ Fetching logs from completed K6 test pod: $K6_TEST_POD" kubectl logs "$K6_TEST_POD" -n "$LOAD_TEST_NAMESPACE" } check_prerequisites() { echo "๐Ÿ” Checking prerequisites..." check_command "kubectl" "$MIN_KUBECTL_VERSION" check_command "curl" "$MIN_CURL_VERSION" if [[ "$MODE" == "binary" ]]; then check_command "k6" "$MIN_K6_VERSION" fi } check_command() { local cmd="$1" local min_version="$2" local version if ! command -v "$cmd" &>/dev/null; then echo "โŒ Required command '$cmd' not found in PATH." exit 1 fi case "$cmd" in kubectl) version=$(kubectl version --client -o json | jq -r '.clientVersion.gitVersion' | sed 's/^v//') ;; curl) version=$($cmd --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1) ;; k6) version=$($cmd version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') ;; *) version="0.0.0" ;; esac if ! version_gte "$version" "$min_version"; then echo "โŒ $cmd version $version is less than required $min_version" exit 1 else echo "โœ… $cmd version $version (>= $min_version)" fi } version_gte() { [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] } run_k6_binary_test() { echo "๐Ÿš€ Running k6 load test..." # Get initial etcd restart counts before starting test echo "โ„น๏ธ Detecting etcd namespace and getting initial restart counts..." local etcd_namespace="openshift-etcd" local etcd_label="app=etcd" # Detect cluster type - check for OpenShift etcd first if kubectl get namespace openshift-etcd >/dev/null 2>&1; then if kubectl get pods -n openshift-etcd -l app=etcd --no-headers 2>/dev/null | grep -q .; then etcd_namespace="openshift-etcd" etcd_label="app=etcd" echo "โœ… Detected OpenShift cluster - etcd in openshift-etcd namespace" fi fi # If not OpenShift, try kube-system (standard Kubernetes/k3s) if [[ "$etcd_namespace" == "openshift-etcd" ]] && ! kubectl get pods -n openshift-etcd -l app=etcd --no-headers 2>/dev/null | grep -q .; then if kubectl get namespace kube-system >/dev/null 2>&1; then if kubectl get pods -n kube-system -l component=etcd --no-headers 2>/dev/null | grep -q .; then etcd_namespace="kube-system" etcd_label="component=etcd" echo "โœ… Detected Kubernetes/k3s cluster - etcd in kube-system namespace" fi fi fi INITIAL_ETCD_RESTARTS=$(get_initial_pod_restart_counts "$etcd_namespace" "$etcd_label") echo "โ„น๏ธ Initial etcd restart counts: $INITIAL_ETCD_RESTARTS" # Warn if no etcd pods found if [[ "$INITIAL_ETCD_RESTARTS" == "{}" ]]; then echo "โš ๏ธ Warning: No etcd pods found in namespace $etcd_namespace with label $etcd_label" echo "โ„น๏ธ Etcd metrics may not be available for this cluster" fi # Get initial operator restart counts INITIAL_OPERATOR_RESTARTS=$(get_initial_pod_restart_counts "$DWO_NAMESPACE" "app.kubernetes.io/name=devworkspace-controller") echo "โ„น๏ธ Initial operator restart counts: $INITIAL_OPERATOR_RESTARTS" if IN_CLUSTER='false' \ KUBE_TOKEN="${KUBE_TOKEN}" \ KUBE_API="${KUBE_API}" \ DWO_NAMESPACE="${DWO_NAMESPACE}" \ CREATE_AUTOMOUNT_RESOURCES="${CREATE_AUTOMOUNT_RESOURCES}" \ SEPARATE_NAMESPACES="${SEPARATE_NAMESPACES}" \ LOAD_TEST_NAMESPACE="${LOAD_TEST_NAMESPACE}" \ DEVWORKSPACE_LINK="${DEVWORKSPACE_LINK}" \ MAX_VUS="${MAX_VUS}" \ EXECUTOR_MODE="${EXECUTOR_MODE}" \ TEST_DURATION_MINUTES="${TEST_DURATION_IN_MINUTES}" \ DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}" \ DELETE_DEVWORKSPACE_AFTER_READY="${DELETE_DEVWORKSPACE_AFTER_READY}" \ MAX_DEVWORKSPACES="${MAX_DEVWORKSPACES}" \ SKIP_CLEANUP="${SKIP_CLEANUP}" \ INITIAL_ETCD_RESTARTS="${INITIAL_ETCD_RESTARTS}" \ INITIAL_OPERATOR_RESTARTS="${INITIAL_OPERATOR_RESTARTS}" \ k6 run "${K6_SCRIPT}"; then echo "โœ… k6 load test completed successfully" else echo "โš ๏ธ k6 load test failed (exit code $?). Continuing with cleanup and backup validation." fi } trap stop_background_watchers EXIT main "$@"