kubernetes · k8s-ci-robot · May 30, 2026 · May 30, 2026
diff --git a/channels/pkg/cmd/apply_channel.go b/channels/pkg/cmd/apply_channel.go
@@ -94,14 +94,25 @@ func runApplyChannelIteration(ctx context.Context, f *ChannelsFactory, out io.Wr
 // ChannelsFactory per iteration drops cached REST configs and the discovery
 // cache, picking up cert rotation and new CRDs without a restart.
 func runApplyChannelLoop(ctx context.Context, out io.Writer, options *ApplyChannelOptions, args []string) error {
+	// In daemon mode kops-channels runs as a system-node-critical static pod; serve a
+	// readiness probe reporting the last apply outcome, so a persistent failure surfaces
+	// as NotReady (failing `kops validate cluster`, which gates rolling updates) instead
+	// of only being logged. Starts NotReady until the first successful apply.
+	readiness, err := serveReadiness(ctx)
+	if err != nil {
+		return fmt.Errorf("serving readiness probe: %w", err)
+	}
+
 	// Retry quickly until the first success: the apiserver is usually
 	// unreachable while the control plane is still coming up.
 	const startupRetryInterval = 5 * time.Second
 
 	settled := false
 	for {
 		interval := options.Interval
-		if err := runApplyChannelIteration(ctx, NewChannelsFactory(), out, options, args); err != nil {
+		err := runApplyChannelIteration(ctx, NewChannelsFactory(), out, options, args)
+		readiness.recordApplyResult(err)
+		if err != nil {
 			if !settled {
 				interval = min(startupRetryInterval, options.Interval)
 			}

diff --git a/channels/pkg/cmd/readiness.go b/channels/pkg/cmd/readiness.go
@@ -0,0 +1,86 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/kops/pkg/wellknownports"
+)
+
+type applyChannelReadiness struct {
+	ready atomic.Bool
+	addr  string // resolved listen address; read only by tests (which bind :0)
+}
+
+func (r *applyChannelReadiness) recordApplyResult(err error) {
+	r.ready.Store(err == nil)
+}
+
+// serveReadiness serves /readyz on loopback for the kubelet readiness probe until ctx is cancelled:
+// 200 when ready is true, 503 otherwise. The pod runs with hostNetwork, so the kubelet reaches it
+// via 127.0.0.1 in the host network namespace.
+func serveReadiness(ctx context.Context) (*applyChannelReadiness, error) {
+	addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(wellknownports.KopsChannelsHealthCheck))
+	return serveReadinessOnAddr(ctx, addr)
+}
+
+func serveReadinessOnAddr(ctx context.Context, addr string) (*applyChannelReadiness, error) {
+	readiness := &applyChannelReadiness{}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
+		if readiness.ready.Load() {
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte("ok\n"))
+		} else {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			_, _ = w.Write([]byte("apply iterations are failing\n"))
+		}
+	})
+
+	listener, err := net.Listen("tcp", addr)
+	if err != nil {
+		return nil, fmt.Errorf("listening on %s: %w", addr, err)
+	}
+	readiness.addr = listener.Addr().String()
+
+	server := &http.Server{
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	go func() {
+		<-ctx.Done()
+		_ = server.Close()
+	}()
+	go func() {
+		if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			klog.Fatalf("kops-channels readiness server stopped: %v", err)
+		}
+	}()
+	return readiness, nil
+}
diff --git a/channels/pkg/cmd/readiness_test.go b/channels/pkg/cmd/readiness_test.go
@@ -0,0 +1,72 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestServeReadinessReportsApplyOutcome(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	readiness, err := serveReadinessOnAddr(ctx, "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("serveReadinessOnAddr returned error: %v", err)
+	}
+
+	assertReadinessStatus(t, readiness.addr, http.StatusServiceUnavailable)
+
+	readiness.recordApplyResult(nil)
+	assertReadinessStatus(t, readiness.addr, http.StatusOK)
+
+	readiness.recordApplyResult(errors.New("apply failed"))
+	assertReadinessStatus(t, readiness.addr, http.StatusServiceUnavailable)
+}
+
+func TestServeReadinessReturnsBindError(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to reserve test port: %v", err)
+	}
+	defer listener.Close()
+
+	_, err = serveReadinessOnAddr(context.Background(), listener.Addr().String())
+	if err == nil {
+		t.Fatalf("expected bind error")
+	}
+}
+
+func assertReadinessStatus(t *testing.T, addr string, expectedStatus int) {
+	t.Helper()
+
+	client := &http.Client{Timeout: time.Second}
+	resp, err := client.Get("http://" + addr + "/readyz")
+	if err != nil {
+		t.Fatalf("GET /readyz failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != expectedStatus {
+		t.Fatalf("expected status %d, got %d", expectedStatus, resp.StatusCode)
+	}
+}
diff --git a/pkg/model/components/channels/model.go b/pkg/model/components/channels/model.go
@@ -23,12 +23,14 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 
 	kopsroot "k8s.io/kops"
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/pkg/wellknownusers"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
@@ -156,6 +158,23 @@ func (b *ChannelsBuilder) buildPod(channels []string) (*v1.Pod, error) {
 				v1.ResourceMemory: resource.MustParse("50Mi"),
 			},
 		},
+		// kops-channels is system-node-critical, so a NotReady container fails `kops validate
+		// cluster`. The apply loop serves /readyz on loopback, publishing the most recent apply
+		// outcome. failureThreshold 2 (~20s at periodSeconds 10) trips within one apply
+		// interval (channelsInterval, 60s), so a single failed apply surfaces as NotReady; 2
+		// (not 1) rides out one flaky probe sample.
+		ReadinessProbe: &v1.Probe{
+			ProbeHandler: v1.ProbeHandler{
+				HTTPGet: &v1.HTTPGetAction{
+					Host: "127.0.0.1",
+					Path: "/readyz",
+					Port: intstr.FromInt(wellknownports.KopsChannelsHealthCheck),
+				},
+			},
+			InitialDelaySeconds: 30,
+			PeriodSeconds:       10,
+			FailureThreshold:    2,
+		},
 		// ko-distroless's default nonroot uid can't read /var/lib/kops/kubeconfig.
 		SecurityContext: &v1.SecurityContext{
 			RunAsUser:    fi.PtrTo(int64(wellknownusers.KopsChannelsID)),

diff --git a/pkg/model/components/channels/tests/container_registry/tasks.yaml b/pkg/model/components/channels/tests/container_registry/tasks.yaml
@@ -30,6 +30,14 @@ Contents: |
         value: us-test-1
       image: my-mirror.example.com/kops-channels:1.36.0-alpha.1
       name: kops-channels
+      readinessProbe:
+        failureThreshold: 2
+        httpGet:
+          host: 127.0.0.1
+          path: /readyz
+          port: 3986
+        initialDelaySeconds: 30
+        periodSeconds: 10
       resources:
         requests:
           cpu: 50m

diff --git a/pkg/model/components/channels/tests/minimal/tasks.yaml b/pkg/model/components/channels/tests/minimal/tasks.yaml
@@ -30,6 +30,14 @@ Contents: |
         value: us-test-1
       image: registry.k8s.io/kops/channels:1.36.0-alpha.1
       name: kops-channels
+      readinessProbe:
+        failureThreshold: 2
+        httpGet:
+          host: 127.0.0.1
+          path: /readyz
+          port: 3986
+        initialDelaySeconds: 30
+        periodSeconds: 10
       resources:
         requests:
           cpu: 50m

diff --git a/pkg/wellknownports/wellknownports.go b/pkg/wellknownports/wellknownports.go
@@ -26,6 +26,9 @@ const (
 	// EtcdMetricsPort is used to serve etcd metrics
 	EtcdMetricsPort = 2382
 
+	// KopsChannelsHealthCheck is the loopback port the kops-channels static pod serves /readyz on.
+	KopsChannelsHealthCheck = 3986
+
 	// NodeupChallenge is the port where nodeup listens for challenges.
 	NodeupChallenge = 3987
 

diff --git a/...ate_cluster/additionalobjects/data/aws_s3_object_manifests-channels-kops-channels_content b/...ate_cluster/additionalobjects/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m

diff --git a/...update_cluster/apiservernodes/data/aws_s3_object_manifests-channels-kops-channels_content b/...update_cluster/apiservernodes/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m

diff --git a/...ate_cluster/aws-lb-controller/data/aws_s3_object_manifests-channels-kops-channels_content b/...ate_cluster/aws-lb-controller/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m

diff --git a/...r/bastionadditional_user-data/data/aws_s3_object_manifests-channels-kops-channels_content b/...r/bastionadditional_user-data/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m

diff --git a/...aler-priority-expander-custom/data/aws_s3_object_manifests-channels-kops-channels_content b/...aler-priority-expander-custom/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m

diff --git a/...-autoscaler-priority-expander/data/aws_s3_object_manifests-channels-kops-channels_content b/...-autoscaler-priority-expander/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m

diff --git a/...ration/update_cluster/complex/data/aws_s3_object_manifests-channels-kops-channels_content b/...ration/update_cluster/complex/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m

diff --git a/...ation/update_cluster/compress/data/aws_s3_object_manifests-channels-kops-channels_content b/...ation/update_cluster/compress/data/aws_s3_object_manifests-channels-kops-channels_content
@@ -28,6 +28,14 @@ spec:
       value: us-test-1
     image: registry.k8s.io/kops/channels:1.34.0-beta.1
     name: kops-channels
+    readinessProbe:
+      failureThreshold: 2
+      httpGet:
+        host: 127.0.0.1
+        path: /readyz
+        port: 3986
+      initialDelaySeconds: 30
+      periodSeconds: 10
     resources:
       requests:
         cpu: 50m