Sitelet https://github.com/kubernetes/kops/commit/6e4add41074c838db585eacc5573b11a24048538
Skip to content

Commit 6e4add4

Browse files
committed
[aiconformance]: expose vLLM metrics from prometheus
This feels a bit hacky, but this is apparently par for the course?
1 parent d9095a5 commit 6e4add4

1 file changed

Lines changed: 31 additions & 0 deletions

File tree

‎tests/e2e/scenarios/ai-conformance/run-test.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,32 @@ helm upgrade -i kube-prometheus-stack \
132132
--set grafana.enabled=false \
133133
--wait
134134

135+
# Prometheus Adapter
136+
# Bridges Prometheus metrics to the Kubernetes custom metrics API (custom.metrics.k8s.io),
137+
# required for HPA to scale on custom metrics like vllm:num_requests_waiting.
138+
echo "Installing prometheus-adapter..."
139+
cat >prometheus-adapter-values.yaml <<'ADAPTER_EOF'
140+
prometheus:
141+
url: http://kube-prometheus-stack-prometheus.monitoring.svc
142+
port: 9090
143+
rules:
144+
custom:
145+
- seriesQuery: 'vllm:num_requests_waiting{namespace!="",pod!=""}'
146+
resources:
147+
overrides:
148+
namespace: {resource: "namespace"}
149+
pod: {resource: "pod"}
150+
name:
151+
matches: "^(.*)"
152+
as: "${1}"
153+
metricsQuery: '<<.Series>>{<<.LabelMatchers>>}'
154+
ADAPTER_EOF
155+
helm upgrade -i prometheus-adapter prometheus-community/prometheus-adapter \
156+
--namespace monitoring \
157+
-f prometheus-adapter-values.yaml \
158+
--wait
159+
rm -f prometheus-adapter-values.yaml
160+
135161
# NVIDIA GPU Operator
136162
# Manages the full NVIDIA stack: kernel driver, container toolkit, device plugin, DCGM exporter.
137163
# The driver is installed into /run/nvidia/driver on each node.
@@ -237,6 +263,11 @@ echo "Verifying Prometheus Stack..."
237263
kubectl rollout status deployment -n monitoring kube-prometheus-stack-operator --timeout=5m || echo "Warning: Prometheus Operator not ready yet"
238264
kubectl rollout status statefulset -n monitoring prometheus-kube-prometheus-stack-prometheus --timeout=5m || echo "Warning: Prometheus not ready yet"
239265

266+
echo "Verifying Prometheus Adapter..."
267+
kubectl rollout status deployment -n monitoring prometheus-adapter --timeout=5m || echo "Warning: Prometheus Adapter not ready yet"
268+
echo "Checking custom metrics API registration..."
269+
kubectl get apiservice v1beta1.custom.metrics.k8s.io || echo "Warning: custom metrics API not registered"
270+
240271
echo "Verifying DCGM Exporter in gpu-operator namespace..."
241272
kubectl rollout status daemonset -n gpu-operator nvidia-dcgm-exporter --timeout=5m || echo "Warning: DCGM Exporter not ready yet"
242273

0 commit comments

Comments
 (0)