Sitelet https://github.com/kubernetes/kops/pull/18018/files
Skip to content

tests/ai-conformance: Install prometheus for metrics #18018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
k8s-ci-robot merged 10 commits into from
Mar 7, 2026
Merged

tests/ai-conformance: Install prometheus for metrics #18018

Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
145d18e
tests/ai-conformance: Install prometheus for metrics
ameukam Feb 26, 2026
7fd321f
Ensure DGCM exporter schedule on GPU nodes
ameukam Mar 1, 2026
43a2a33
Add more diagnostics for DGCM exporter pods
ameukam Mar 1, 2026
e5a2beb
Get more informations about the GPU operator
ameukam Mar 1, 2026
8d6c2e9
Remove gpu-helm-charts Helm repository
ameukam Mar 6, 2026
b752dc8
Enable DCGM exporter in GPU Operator with ServiceMonitor
ameukam Mar 6, 2026
2e3c67c
fixup! Enable DCGM exporter in GPU Operator with ServiceMonitor
ameukam Mar 6, 2026
f97c8f9
Remove standalone DCGM exporter installation
ameukam Mar 6, 2026
459ec97
Update DCGM verification for gpu-operator namespace
ameukam Mar 6, 2026
742f3b3
Add comprehensive GPU Operator debugging output
ameukam Mar 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 108 additions & 18 deletions tests/e2e/scenarios/ai-conformance/run-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ if [[ "${AVAILABILITY}" == "false" ]]; then
fi
rm -f "${SCENARIO_ROOT}/tools/check-aws-availability/check-aws-availability"


kops-acquire-latest

# Cluster Configuration
Expand Down Expand Up @@ -85,6 +84,9 @@ EOF

${KOPS} update cluster --name "${CLUSTER_NAME}" --yes --admin

echo "Listing node with their labels..."
kubectl get nodes --show-labels

echo "----------------------------------------------------------------"
echo "Deploying AI Conformance Components"
echo "----------------------------------------------------------------"
Expand All @@ -97,18 +99,45 @@ kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/downloa
echo "Installing cert-manager..."
kubectl apply --server-side -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml

# Setup helm repo for NVIDIA GPU Operator and DRA Driver
# Setup helm repos for monitoring and NVIDIA components
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

# Prometheus Stack (kube-prometheus-stack)
# Provides Prometheus, Grafana, Alertmanager, and ServiceMonitor CRDs
# Must be installed before dcgm-exporter so ServiceMonitor CRDs are available
echo "Installing kube-prometheus-stack..."
helm upgrade -i kube-prometheus-stack \
oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack \
--namespace monitoring \
--create-namespace \
--set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
--set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \
--set grafana.enabled=false \
--wait

# NVIDIA GPU Operator
# Manages the full NVIDIA stack: kernel driver, container toolkit, device plugin.
# Manages the full NVIDIA stack: kernel driver, container toolkit, device plugin, DCGM exporter.
# The driver is installed into /run/nvidia/driver on each node.
# DCGM exporter is enabled with ServiceMonitor for Prometheus integration.
echo "Installing NVIDIA GPU Operator with DCGM exporter..."
helm upgrade -i nvidia-gpu-operator --wait \
-n gpu-operator --create-namespace \
nvidia/gpu-operator \
--version=v25.10.1 \
--wait
-n gpu-operator --create-namespace \
nvidia/gpu-operator \
--version=v25.10.1 \
--set dcgmExporter.enabled=true \
--set dcgmExporter.serviceMonitor.enabled=true \
--set dcgmExporter.serviceMonitor.additionalLabels.release=kube-prometheus-stack

echo "GPU Operator installation complete. Checking component status..."
kubectl get pods -n gpu-operator -o wide || echo "Warning: GPU Operator pods not ready yet"

echo "DCGM Exporter pod status in gpu-operator namespace..."
kubectl get pods -n gpu-operator -l app=nvidia-dcgm-exporter -o wide || echo "Warning: DCGM exporter pods not found"

echo "GPU Operator deployment details for debugging:"
kubectl get all -n gpu-operator || true

PATH="$(pwd):$PATH"
export PATH
Expand All @@ -117,7 +146,7 @@ export PATH
# Uses the driver installed by GPU Operator at /run/nvidia/driver (the default).
echo "Installing NVIDIA DRA Driver..."

cat > values.yaml <<EOF
cat >values.yaml <<EOF
# The driver daemonset needs a toleration for the nvidia.com/gpu taint
kubeletPlugin:
tolerations:
Expand All @@ -127,15 +156,14 @@ kubeletPlugin:
EOF

helm upgrade -i nvidia-dra-driver-gpu nvidia/nvidia-dra-driver-gpu \
--version="25.12.0" \
--create-namespace \
--namespace nvidia-dra-driver-gpu \
--set resources.gpus.enabled=true \
--set nvidiaDriverRoot=/run/nvidia/driver \
--set gpuResourcesEnabledOverride=true \
-f values.yaml \
--wait

--version="25.12.0" \
--create-namespace \
--namespace nvidia-dra-driver-gpu \
--set resources.gpus.enabled=true \
--set nvidiaDriverRoot=/run/nvidia/driver \
--set gpuResourcesEnabledOverride=true \
-f values.yaml \
--wait

# KubeRay
echo "Installing KubeRay Operator..."
Expand All @@ -145,7 +173,6 @@ kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/conf
echo "Installing Kueue..."
kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.8/manifests.yaml


echo "----------------------------------------------------------------"
echo "Verifying Cluster and Components"
echo "----------------------------------------------------------------"
Expand All @@ -166,6 +193,20 @@ kubectl rollout status deployment -n kuberay-system kuberay-operator --timeout=5
echo "Verifying Gateway API..."
kubectl get gatewayclass || echo "Warning: GatewayClass not found"

echo "Verifying Prometheus Stack..."
kubectl rollout status deployment -n monitoring kube-prometheus-stack-operator --timeout=5m || echo "Warning: Prometheus Operator not ready yet"
kubectl rollout status statefulset -n monitoring prometheus-kube-prometheus-stack-prometheus --timeout=5m || echo "Warning: Prometheus not ready yet"

echo "Verifying DCGM Exporter in gpu-operator namespace..."
kubectl rollout status daemonset -n gpu-operator nvidia-dcgm-exporter --timeout=5m || echo "Warning: DCGM Exporter not ready yet"

echo "DCGM Exporter pod details for debugging:"
kubectl get pods -n gpu-operator -l app=nvidia-dcgm-exporter -o wide || echo "Warning: No DCGM exporter pods found"
kubectl describe pods -n gpu-operator -l app=nvidia-dcgm-exporter || true

echo "Verifying ServiceMonitor for DCGM in gpu-operator namespace..."
kubectl get servicemonitor -n gpu-operator nvidia-dcgm-exporter -o yaml || echo "Warning: DCGM ServiceMonitor not found"

echo "Verifying Allocatable GPUs..."
# Wait a bit for nodes to report resources
sleep 30
Expand Down Expand Up @@ -216,6 +257,55 @@ echo "Waiting for Sample Workload to Complete..."
kubectl wait --for=condition=complete job/test-gpu-pod --timeout=5m || true
kubectl logs job/test-gpu-pod || echo "Failed to get logs"

echo "Verifying GPU Metrics in Prometheus..."
# Wait for DCGM exporter to start collecting metrics
# Query Prometheus for DCGM GPU metrics with retries
PROM_POD=$(kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}')
if [ -n "${PROM_POD}" ]; then
echo "Querying Prometheus for DCGM_FI_DEV_GPU_UTIL metric (retrying up to 5 times)..."
for i in {1..5}; do
echo "Attempt $i..."
METRICS=$(kubectl exec -n monitoring "${PROM_POD}" -c prometheus -- \
wget -qO- 'http://localhost:9090/api/v1/query?query=DCGM_FI_DEV_GPU_UTIL' 2>/dev/null)
if echo "${METRICS}" | grep -q "result"; then
echo "Successfully retrieved GPU metrics:"
echo "${METRICS}" | head -c 500
break
fi
echo "Metrics not yet available, waiting 20s..."
sleep 20
done
else
echo "Warning: Prometheus pod not found"
fi

echo "----------------------------------------------------------------"
echo "GPU Operator Component Status for Debugging"
echo "----------------------------------------------------------------"

echo "All GPU Operator pods:"
kubectl get pods -n gpu-operator -o wide || true

echo ""
echo "GPU Operator DaemonSets:"
kubectl get daemonsets -n gpu-operator -o wide || true

echo ""
echo "DCGM Exporter DaemonSet details:"
kubectl describe daemonset -n gpu-operator nvidia-dcgm-exporter || true

echo ""
echo "DCGM Exporter Service:"
kubectl get service -n gpu-operator nvidia-dcgm-exporter -o yaml || echo "No DCGM service found"

echo ""
echo "DCGM Exporter ServiceMonitor:"
kubectl get servicemonitor -n gpu-operator nvidia-dcgm-exporter -o yaml || echo "No ServiceMonitor found"

echo ""
echo "Recent GPU Operator events:"
kubectl get events -n gpu-operator --sort-by='.lastTimestamp' | tail -20 || true

echo "AI Conformance Environment Setup Complete."

# Now run the actual AI conformance tests
Expand Down
Loading