Sitelet https://github.com/kubernetes/kops/commit/e3892f4abd851118c71098fdefe673e30c7618e6
Skip to content

Commit e3892f4

Browse files
ameukamameukam
committed
tests/ai-conformance: Install prometheus for metrics
Signed-off-by: Arnaud Meukam <ameukam@gmail.com>
1 parent 79fc586 commit e3892f4

1 file changed

Lines changed: 84 additions & 16 deletions

File tree

‎tests/e2e/scenarios/ai-conformance/run-test.sh‎

Lines changed: 84 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ if [[ "${AVAILABILITY}" == "false" ]]; then
4646
fi
4747
rm -f "${SCENARIO_ROOT}/tools/check-aws-availability/check-aws-availability"
4848

49-
5049
kops-acquire-latest
5150

5251
# Cluster Configuration
@@ -97,18 +96,33 @@ kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/downloa
9796
echo "Installing cert-manager..."
9897
kubectl apply --server-side -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml
9998

100-
# Setup helm repo for NVIDIA GPU Operator and DRA Driver
99+
# Setup helm repos for monitoring and NVIDIA components
101100
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
101+
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
102+
helm repo add gpu-helm-charts https://nvidia.github.io/dcgm-exporter/helm-charts
102103
helm repo update
103104

105+
# Prometheus Stack (kube-prometheus-stack)
106+
# Provides Prometheus, Grafana, Alertmanager, and ServiceMonitor CRDs
107+
# Must be installed before dcgm-exporter so ServiceMonitor CRDs are available
108+
echo "Installing kube-prometheus-stack..."
109+
helm upgrade -i kube-prometheus-stack \
110+
oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack \
111+
--namespace monitoring \
112+
--create-namespace \
113+
--set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
114+
--set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \
115+
--wait
116+
104117
# NVIDIA GPU Operator
105118
# Manages the full NVIDIA stack: kernel driver, container toolkit, device plugin.
106119
# The driver is installed into /run/nvidia/driver on each node.
107120
helm upgrade -i nvidia-gpu-operator --wait \
108-
-n gpu-operator --create-namespace \
109-
nvidia/gpu-operator \
110-
--version=v25.10.1 \
111-
--wait
121+
-n gpu-operator --create-namespace \
122+
nvidia/gpu-operator \
123+
--version=v25.10.1 \
124+
--set dcgmExporter.enabled=false \
125+
--wait
112126

113127
PATH="$(pwd):$PATH"
114128
export PATH
@@ -117,7 +131,7 @@ export PATH
117131
# Uses the driver installed by GPU Operator at /run/nvidia/driver (the default).
118132
echo "Installing NVIDIA DRA Driver..."
119133

120-
cat > values.yaml <<EOF
134+
cat >values.yaml <<EOF
121135
# The driver daemonset needs a toleration for the nvidia.com/gpu taint
122136
kubeletPlugin:
123137
tolerations:
@@ -127,15 +141,38 @@ kubeletPlugin:
127141
EOF
128142

129143
helm upgrade -i nvidia-dra-driver-gpu nvidia/nvidia-dra-driver-gpu \
130-
--version="25.12.0" \
131-
--create-namespace \
132-
--namespace nvidia-dra-driver-gpu \
133-
--set resources.gpus.enabled=true \
134-
--set nvidiaDriverRoot=/run/nvidia/driver \
135-
--set gpuResourcesEnabledOverride=true \
136-
-f values.yaml \
137-
--wait
144+
--version="25.12.0" \
145+
--create-namespace \
146+
--namespace nvidia-dra-driver-gpu \
147+
--set resources.gpus.enabled=true \
148+
--set nvidiaDriverRoot=/run/nvidia/driver \
149+
--set gpuResourcesEnabledOverride=true \
150+
-f values.yaml \
151+
--wait
152+
153+
# NVIDIA DCGM Exporter
154+
# Exports GPU metrics to Prometheus for monitoring GPU utilization, memory, temperature, etc.
155+
echo "Installing NVIDIA DCGM Exporter..."
156+
157+
cat >dcgm-exporter-values.yaml <<EOF
158+
# Tolerations to run on GPU nodes with nvidia.com/gpu taint
159+
tolerations:
160+
- key: nvidia.com/gpu
161+
operator: Exists
162+
effect: NoSchedule
163+
164+
# ServiceMonitor for Prometheus integration
165+
serviceMonitor:
166+
enabled: true
167+
interval: 15s
168+
additionalLabels:
169+
release: kube-prometheus-stack
170+
EOF
138171

172+
helm upgrade -i dcgm-exporter gpu-helm-charts/dcgm-exporter \
173+
--namespace monitoring \
174+
-f dcgm-exporter-values.yaml \
175+
--wait
139176

140177
# KubeRay
141178
echo "Installing KubeRay Operator..."
@@ -145,7 +182,6 @@ kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/conf
145182
echo "Installing Kueue..."
146183
kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.8/manifests.yaml
147184

148-
149185
echo "----------------------------------------------------------------"
150186
echo "Verifying Cluster and Components"
151187
echo "----------------------------------------------------------------"
@@ -166,6 +202,16 @@ kubectl rollout status deployment -n kuberay-system kuberay-operator --timeout=5
166202
echo "Verifying Gateway API..."
167203
kubectl get gatewayclass || echo "Warning: GatewayClass not found"
168204

205+
echo "Verifying Prometheus Stack..."
206+
kubectl rollout status deployment -n monitoring kube-prometheus-stack-operator --timeout=5m || echo "Warning: Prometheus Operator not ready yet"
207+
kubectl rollout status statefulset -n monitoring prometheus-kube-prometheus-stack-prometheus --timeout=5m || echo "Warning: Prometheus not ready yet"
208+
209+
echo "Verifying DCGM Exporter..."
210+
kubectl rollout status daemonset -n monitoring dcgm-exporter --timeout=5m || echo "Warning: DCGM Exporter not ready yet"
211+
212+
echo "Verifying ServiceMonitor for DCGM..."
213+
kubectl get servicemonitor -n monitoring dcgm-exporter || echo "Warning: DCGM ServiceMonitor not found"
214+
169215
echo "Verifying Allocatable GPUs..."
170216
# Wait a bit for nodes to report resources
171217
sleep 30
@@ -216,6 +262,28 @@ echo "Waiting for Sample Workload to Complete..."
216262
kubectl wait --for=condition=complete job/test-gpu-pod --timeout=5m || true
217263
kubectl logs job/test-gpu-pod || echo "Failed to get logs"
218264

265+
echo "Verifying GPU Metrics in Prometheus..."
266+
# Wait for DCGM exporter to start collecting metrics
267+
# Query Prometheus for DCGM GPU metrics with retries
268+
PROM_POD=$(kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}')
269+
if [ -n "${PROM_POD}" ]; then
270+
echo "Querying Prometheus for DCGM_FI_DEV_GPU_UTIL metric (retrying up to 5 times)..."
271+
for i in {1..5}; do
272+
echo "Attempt $i..."
273+
METRICS=$(kubectl exec -n monitoring "${PROM_POD}" -c prometheus -- \
274+
wget -qO- 'http://localhost:9090/api/v1/query?query=DCGM_FI_DEV_GPU_UTIL' 2>/dev/null)
275+
if echo "${METRICS}" | grep -q "result"; then
276+
echo "Successfully retrieved GPU metrics:"
277+
echo "${METRICS}" | head -c 500
278+
break
279+
fi
280+
echo "Metrics not yet available, waiting 20s..."
281+
sleep 20
282+
done
283+
else
284+
echo "Warning: Prometheus pod not found"
285+
fi
286+
219287
echo "AI Conformance Environment Setup Complete."
220288

221289
# Now run the actual AI conformance tests

0 commit comments

Comments
 (0)