kubernetes · kubernetes-prow · Jul 5, 2026 · Jul 4, 2026
diff --git a/docs/releases/1.36-NOTES.md b/docs/releases/1.36-NOTES.md
@@ -110,6 +110,8 @@ This feature is currently supported on Debian-family distributions only (Debian,
 
 ## Hetzner
 * Enable Cluster Autoscaler ([#18226](https://github.com/kubernetes/kops/pull/18226), [#18135](https://github.com/kubernetes/kops/pull/18135))
+* Node-role servers now carry the `hcloud/node-group` label, which the Cluster Autoscaler Hetzner provider uses for node group membership. Servers created by earlier kOps versions receive the label when they are replaced.
+* Lowering the `minSize` of an instance group no longer removes surplus servers; `minSize` now acts as a floor. Remove specific servers with `kops delete instance`, or rely on Cluster Autoscaler scale-down.
 * Upgrade `hcloud-cloud-controller-manager` to v1.31.0 ([#18281](https://github.com/kubernetes/kops/pull/18281), [#18317](https://github.com/kubernetes/kops/pull/18317))
 * Upgrade `hcloud-csi-driver` to v2.20.2 and reorder the CSI driver Deployment before the DaemonSet ([#18318](https://github.com/kubernetes/kops/pull/18318))
 * Split the hcloud Secret into its own addon and let the CSI driver consume the CCM-provided secret ([#18317](https://github.com/kubernetes/kops/pull/18317), [#18318](https://github.com/kubernetes/kops/pull/18318))

diff --git a/pkg/model/context.go b/pkg/model/context.go
@@ -202,6 +202,9 @@ func (b *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma
 		labels[hetzner.TagKubernetesInstanceRole] = string(ig.Spec.Role)
 		labels[hetzner.TagKubernetesClusterName] = b.ClusterName()
 		labels[hetzner.TagKubernetesInstanceGroup] = ig.Name
+		if ig.Spec.Role == kops.InstanceGroupRoleNode {
+			labels[hetzner.TagClusterAutoscalerNodeGroup] = ig.Name
+		}
 	case kops.CloudProviderGCE:
 		clusterLabel := gce.LabelForCluster(b.ClusterName())
 		roleLabel := gce.GceLabelNameRolePrefix + ig.Spec.Role.ToLowerString()

diff --git a/tests/integration/update_cluster/gossip-hetzner/kubernetes.tf b/tests/integration/update_cluster/gossip-hetzner/kubernetes.tf
@@ -284,6 +284,7 @@ resource "hcloud_server" "nodes-fsn1" {
   count = 1
   image = "ubuntu-26.04"
   labels = {
+    "hcloud/node-group"                                   = "nodes-fsn1"
     "kops.k8s.io/cluster"                                 = "gossip.k8s.local"
     "kops.k8s.io/instance-group"                          = "nodes-fsn1"
     "kops.k8s.io/instance-role"                           = "Node"

diff --git a/...e_cluster/minimal_hetzner/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...e_cluster/minimal_hetzner/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -29,7 +29,7 @@ spec:
       k8s-addon: limit-range.addons.k8s.io
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: e2156b91aef97324702360be3b28e7e1cd024c128d023c1f05464e3b4aa5b340
+    manifestHash: 43f620a4cd2eb6ebb7c87b3def7c6988cb6dc45d13e90a4ff3c3aa595a64cd4e
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -295,6 +295,7 @@ data:
             "node-role.kubernetes.io/node": ""
           },
           "serverLabels": {
+            "hcloud/node-group": "nodes-fsn1",
             "kops.k8s.io/cluster": "minimal.example.com",
             "kops.k8s.io/instance-group": "nodes-fsn1",
             "kops.k8s.io/instance-role": "Node",
@@ -344,7 +345,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/hcloud-cluster-config: 6b7e37db999d86fc14b812600dc73848da7c6aab60d3316be300af59701e643c
+        checksum/hcloud-cluster-config: fec176327c61c674272b4f0473ab1c595833b6377bc81c2f0db9be3c0ca33efc
         prometheus.io/port: "8085"
         prometheus.io/scrape: "true"
       labels:

diff --git a/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf b/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf
@@ -284,6 +284,7 @@ resource "hcloud_server" "nodes-fsn1" {
   count = 1
   image = "ubuntu-26.04"
   labels = {
+    "hcloud/node-group"                                   = "nodes-fsn1"
     "kops.k8s.io/cluster"                                 = "minimal.example.com"
     "kops.k8s.io/instance-group"                          = "nodes-fsn1"
     "kops.k8s.io/instance-role"                           = "Node"

diff --git a/upup/pkg/fi/cloudup/hetzner/cloud.go b/upup/pkg/fi/cloudup/hetzner/cloud.go
@@ -43,6 +43,8 @@ const (
 	TagKubernetesInstanceNeedsUpdate = "kops.k8s.io/needs-update"
 	TagKubernetesVolumeRole          = "kops.k8s.io/volume-role"
 	TagKubernetesNodeLabelPrefix     = "node-label.kops.k8s.io."
+	// TagClusterAutoscalerNodeGroup is the label Cluster Autoscaler uses to determine membership of a server.
+	TagClusterAutoscalerNodeGroup = "hcloud/node-group"
 )
 
 // HetznerCloud exposes all the interfaces required to operate on Hetzner Cloud resources

diff --git a/upup/pkg/fi/cloudup/hetznertasks/servergroup.go b/upup/pkg/fi/cloudup/hetznertasks/servergroup.go
@@ -86,49 +86,58 @@ func (v *ServerGroup) Find(c *fi.CloudupContext) (*ServerGroup, error) {
 	v.Labels[hetzner.TagKubernetesInstanceUserData] = userDataHash
 
 	actual := *v
-	actual.Count = len(servers)
 
-	// Find servers that need to be updated
-	for i, server := range servers {
+	needUpdate, actualCount := v.classifyServers(servers, userDataHash)
+	actual.NeedUpdate = needUpdate
+	actual.Count = actualCount
+
+	return &actual, nil
+}
+
+// classifyServers compares the real servers against the expected group template and returns the
+// servers that need to be replaced (needUpdate) and the count to report as actual.
+func (v *ServerGroup) classifyServers(servers []*hcloud.Server, userDataHash string) (needUpdate []string, actualCount int) {
+	for _, server := range servers {
 		// Ignore servers that are already labeled as needing update
 		if _, ok := server.Labels[hetzner.TagKubernetesInstanceNeedsUpdate]; ok {
 			continue
 		}
 
-		// Check if server index is higher than desired count
-		if i >= v.Count {
-			actual.NeedUpdate = append(actual.NeedUpdate, server.Name)
-			continue
-		}
-
 		// Check if server matches the expected group template
 		if server.Labels[hetzner.TagKubernetesInstanceUserData] != userDataHash {
-			actual.NeedUpdate = append(actual.NeedUpdate, server.Name)
+			needUpdate = append(needUpdate, server.Name)
 			continue
 		}
 		if server.Datacenter == nil || server.Datacenter.Location == nil || server.Datacenter.Location.Name != v.Location {
-			actual.NeedUpdate = append(actual.NeedUpdate, server.Name)
+			needUpdate = append(needUpdate, server.Name)
 			continue
 		}
 		if server.ServerType == nil || server.ServerType.Name != v.Size {
-			actual.NeedUpdate = append(actual.NeedUpdate, server.Name)
+			needUpdate = append(needUpdate, server.Name)
 			continue
 		}
 		if server.Image == nil || server.Image.Name != v.Image {
-			actual.NeedUpdate = append(actual.NeedUpdate, server.Name)
+			needUpdate = append(needUpdate, server.Name)
 			continue
 		}
 		if (server.PublicNet.IPv4.IP != nil) != v.EnableIPv4 {
-			actual.NeedUpdate = append(actual.NeedUpdate, server.Name)
+			needUpdate = append(needUpdate, server.Name)
 			continue
 		}
 		if (server.PublicNet.IPv6.IP != nil) != v.EnableIPv6 {
-			actual.NeedUpdate = append(actual.NeedUpdate, server.Name)
+			needUpdate = append(needUpdate, server.Name)
 			continue
 		}
 	}
 
-	return &actual, nil
+	// Count is a floor: surplus servers are left in place.
+	// Shrinking is done via `kops delete instance` or Cluster Autoscaler scale-down.
+	actualCount = len(servers)
+	if actualCount > v.Count {
+		actualCount = v.Count
+	}
+
+	return needUpdate, actualCount
 }
 
 func (v *ServerGroup) Run(c *fi.CloudupContext) error {

diff --git a/upup/pkg/fi/cloudup/hetznertasks/servergroup_test.go b/upup/pkg/fi/cloudup/hetznertasks/servergroup_test.go
@@ -0,0 +1,239 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hetznertasks
+
+import (
+	"net"
+	"reflect"
+	"testing"
+
+	"github.com/hetznercloud/hcloud-go/v2/hcloud"
+	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
+)
+
+const (
+	testUserDataHash = "sha256.newhash"
+	testOldHash      = "sha256.oldhash"
+	testLocation     = "fsn1"
+	testSize         = "cx21"
+	testImage        = "ubuntu-22.04"
+	testNodeGroup    = "nodes"
+)
+
+// baseGroup returns a ServerGroup with a template that a matching server satisfies.
+func baseGroup(count int) *ServerGroup {
+	return &ServerGroup{
+		Count:      count,
+		Location:   testLocation,
+		Size:       testSize,
+		Image:      testImage,
+		EnableIPv4: true,
+		EnableIPv6: false,
+		Labels: map[string]string{
+			hetzner.TagClusterAutoscalerNodeGroup: testNodeGroup,
+			hetzner.TagKubernetesInstanceUserData: testUserDataHash,
+		},
+	}
+}
+
+// matchingServer returns a server that fully satisfies baseGroup's template.
+func matchingServer(name string) *hcloud.Server {
+	return &hcloud.Server{
+		Name: name,
+		Labels: map[string]string{
+			hetzner.TagClusterAutoscalerNodeGroup: testNodeGroup,
+			hetzner.TagKubernetesInstanceUserData: testUserDataHash,
+		},
+		Datacenter: &hcloud.Datacenter{
+			Location: &hcloud.Location{Name: testLocation},
+		},
+		ServerType: &hcloud.ServerType{Name: testSize},
+		Image:      &hcloud.Image{Name: testImage},
+		PublicNet: hcloud.ServerPublicNet{
+			IPv4: hcloud.ServerPublicNetIPv4{IP: net.ParseIP("1.2.3.4")},
+		},
+	}
+}
+
+func TestClassifyServers(t *testing.T) {
+	tests := []struct {
+		name           string
+		group          *ServerGroup
+		servers        []*hcloud.Server
+		wantNeedUpdate []string
+		wantCount      int
+	}{
+		{
+			name:      "matching server not marked",
+			group:     baseGroup(1),
+			servers:   []*hcloud.Server{matchingServer("s1")},
+			wantCount: 1,
+		},
+		{
+			name:  "wrong userdata hash marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.Labels[hetzner.TagKubernetesInstanceUserData] = testOldHash
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "wrong location marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.Datacenter.Location.Name = "nbg1"
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "nil datacenter marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.Datacenter = nil
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "wrong server type marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.ServerType.Name = "cx31"
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "nil server type marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.ServerType = nil
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "wrong image marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.Image.Name = "debian-12"
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "nil image marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.Image = nil
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "IPv4 presence mismatch marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.PublicNet.IPv4.IP = nil
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "IPv6 presence mismatch marks needUpdate",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				s := matchingServer("s1")
+				s.PublicNet.IPv6.IP = net.ParseIP("2001:db8::1")
+				return []*hcloud.Server{s}
+			}(),
+			wantNeedUpdate: []string{"s1"},
+			wantCount:      1,
+		},
+		{
+			name:  "already needs-update server is not re-marked",
+			group: baseGroup(1),
+			servers: func() []*hcloud.Server {
+				// Mismatched (wrong image) but already labeled needs-update.
+				s := matchingServer("s1")
+				s.Image.Name = "debian-12"
+				s.Labels[hetzner.TagKubernetesInstanceNeedsUpdate] = ""
+				return []*hcloud.Server{s}
+			}(),
+			wantCount: 1,
+		},
+		{
+			name:  "no index-based marking with more matching servers than count",
+			group: baseGroup(1),
+			servers: []*hcloud.Server{
+				matchingServer("s1"),
+				matchingServer("s2"),
+				matchingServer("s3"),
+			},
+			wantCount: 1,
+		},
+		{
+			name:      "count clamp 1 server count 3",
+			group:     baseGroup(3),
+			servers:   []*hcloud.Server{matchingServer("s1")},
+			wantCount: 1,
+		},
+		{
+			name:  "count clamp 5 servers count 2",
+			group: baseGroup(2),
+			servers: []*hcloud.Server{
+				matchingServer("s1"),
+				matchingServer("s2"),
+				matchingServer("s3"),
+				matchingServer("s4"),
+				matchingServer("s5"),
+			},
+			wantCount: 2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			needUpdate, actualCount := tt.group.classifyServers(tt.servers, testUserDataHash)
+
+			if !reflect.DeepEqual(needUpdate, tt.wantNeedUpdate) {
+				t.Errorf("needUpdate = %v, want %v", needUpdate, tt.wantNeedUpdate)
+			}
+			if actualCount != tt.wantCount {
+				t.Errorf("actualCount = %d, want %d", actualCount, tt.wantCount)
+			}
+		})
+	}
+}