Sitelet https://github.com/kubernetes/kops/commit/31c024c6f304bcadbb66990cfaeb3e80b8c4430c
Skip to content

Commit 31c024c

Browse files
kubernetes-prow[bot]kubernetes-prow[bot]
authored
Merge pull request #18525 from hakman/etcd-custom
Support running custom etcd versions by setting version and image
2 parents 86784e0 + e746b3b commit 31c024c

15 files changed

Lines changed: 272 additions & 204 deletions

‎docs/advanced/experimental.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ The following experimental features are currently available:
1515
* `+SpotinstOceanTemplate` - Enables the use of Spot Ocean object as a template for Virtual Node Groups
1616
* `+SpotinstHybrid` - Toggles between hybrid and full instance group implementations
1717
* `-SpotinstController` - Toggles the installation of the Spot controller addon off
18-
* `+SkipEtcdVersionCheck` - Bypasses the check that etcd-manager is using a supported etcd version
1918
* `+EtcdEventsHTTP` - Enables HTTP (non-TLS) for the events etcd cluster, matching GCE scale test patterns
2019
* `+APIServerNodes` - Enables support for dedicated API server nodes
2120
* `+ExperimentalRoles` - Not fully implemented. Enable support for dedicated Etcd, Scheduler, CloudControllerManager and KubeControllerManager nodes.

‎docs/cluster_spec.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,40 @@ etcdClusters:
181181

182182
The etcd version used by kOps follows the recommended etcd version for the given kubernetes version. It is possible to override this by adding the `version` key to each of the etcd clusters.
183183

184+
### Running a custom etcd version
185+
{{ kops_feature_table(kops_added_default='1.37') }}
186+
187+
To run an etcd version that kOps does not bundle, set both `version` and `image` on each of the etcd clusters. When `image` is set, the etcd and etcdctl binaries for `version` are sourced from that image instead of the default `registry.k8s.io/etcd` image. The image must contain `/usr/local/bin/etcd` and `/usr/local/bin/etcdctl` matching `version`, like the official `gcr.io/etcd-development/etcd` images do, and must provide the architectures of the control-plane nodes.
188+
189+
```yaml
190+
etcdClusters:
191+
- etcdMembers:
192+
- instanceGroup: master-us-east-1a
193+
name: a
194+
image: gcr.io/etcd-development/etcd:v3.6.99
195+
name: main
196+
version: 3.6.99
197+
- etcdMembers:
198+
- instanceGroup: master-us-east-1a
199+
name: a
200+
image: gcr.io/etcd-development/etcd:v3.6.99
201+
name: events
202+
version: 3.6.99
203+
```
204+
205+
The same fields can be set when creating a cluster:
206+
207+
```sh
208+
kops create cluster \
209+
--set 'spec.etcdClusters[*].version=3.6.99' \
210+
--set 'spec.etcdClusters[*].image=gcr.io/etcd-development/etcd:v3.6.99' \
211+
...
212+
```
213+
214+
Both `version` and `image` must be set together. When a custom image is set, only the binaries for the selected version are made available to etcd-manager; restoring backups, which can require the binaries of a bundled version, is not supported.
215+
216+
Note that versions not bundled by kOps have not been tested with kOps and that all etcd clusters of a Kubernetes cluster must run the same etcd version. A custom image can only be set when creating a new cluster: once the etcd cluster exists, the image cannot be added, changed or removed, and the version cannot be changed while an image is set.
217+
184218
### The `leases` etcd shard for megaclusters
185219

186220
kOps supports configuring an optional `leases` etcd shard. This shard is specifically designed for megaclusters to isolate the massive volume of high-frequency write traffic generated by node heartbeats. By redirecting this traffic to a dedicated `leases` shard, it prevents the primary etcd database from being overwhelmed, ensuring overall control plane stability.

‎pkg/apis/kops/validation/cluster.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,13 @@ func validateEtcdClusterUpdate(fp *field.Path, obj kops.EtcdClusterSpec, status
7979

8080
// If the etcd cluster has been created (i.e. if we have status) then we can't support some changes
8181
if etcdClusterStatus != nil {
82+
if obj.Image != old.Image {
83+
allErrs = append(allErrs, field.Forbidden(fp.Child("image"), "image can only be set when creating a new etcd cluster"))
84+
}
85+
if obj.Image != "" && obj.Version != old.Version {
86+
allErrs = append(allErrs, field.Forbidden(fp.Child("version"), "version cannot be changed when image is set"))
87+
}
88+
8289
newMembers := make(map[string]kops.EtcdMemberSpec)
8390
for _, member := range obj.Members {
8491
newMembers[member.Name] = member

‎pkg/apis/kops/validation/cluster_test.go‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package validation
1919
import (
2020
"testing"
2121

22+
"k8s.io/apimachinery/pkg/util/validation/field"
2223
"k8s.io/kops/upup/pkg/fi"
2324

2425
"k8s.io/kops/pkg/apis/kops"
@@ -159,3 +160,124 @@ func TestValidEtcdChanges(t *testing.T) {
159160
}
160161
}
161162
}
163+
164+
func TestEtcdImageChanges(t *testing.T) {
165+
createdStatus := &kops.ClusterStatus{
166+
EtcdClusters: []kops.EtcdClusterStatus{
167+
{
168+
Name: "main",
169+
},
170+
},
171+
}
172+
173+
grid := []struct {
174+
Details string
175+
OldSpec kops.EtcdClusterSpec
176+
NewSpec kops.EtcdClusterSpec
177+
Status *kops.ClusterStatus
178+
ExpectedErrors []string
179+
}{
180+
{
181+
Details: "image cannot be added to an existing etcd cluster",
182+
OldSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99"},
183+
NewSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
184+
Status: createdStatus,
185+
ExpectedErrors: []string{
186+
"Forbidden::spec.etcdClusters[main].image",
187+
},
188+
},
189+
{
190+
Details: "image cannot be changed on an existing etcd cluster",
191+
OldSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
192+
NewSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "example.com/etcd:v3.6.99"},
193+
Status: createdStatus,
194+
ExpectedErrors: []string{
195+
"Forbidden::spec.etcdClusters[main].image",
196+
},
197+
},
198+
{
199+
Details: "image cannot be removed from an existing etcd cluster",
200+
OldSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
201+
NewSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99"},
202+
Status: createdStatus,
203+
ExpectedErrors: []string{
204+
"Forbidden::spec.etcdClusters[main].image",
205+
},
206+
},
207+
{
208+
Details: "version cannot be changed when image is set",
209+
OldSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
210+
NewSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.100", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
211+
Status: createdStatus,
212+
ExpectedErrors: []string{
213+
"Forbidden::spec.etcdClusters[main].version",
214+
},
215+
},
216+
{
217+
Details: "unchanged image and version are allowed",
218+
OldSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
219+
NewSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
220+
Status: createdStatus,
221+
},
222+
{
223+
Details: "version changes without image are allowed",
224+
OldSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.11"},
225+
NewSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.12"},
226+
Status: createdStatus,
227+
},
228+
{
229+
Details: "image can be set before the etcd cluster is created",
230+
OldSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99"},
231+
NewSpec: kops.EtcdClusterSpec{Name: "main", Version: "3.6.99", Image: "gcr.io/etcd-development/etcd:v3.6.99"},
232+
Status: &kops.ClusterStatus{},
233+
},
234+
}
235+
236+
for _, g := range grid {
237+
fp := field.NewPath("spec", "etcdClusters").Key(g.NewSpec.Name)
238+
errorList := validateEtcdClusterUpdate(fp, g.NewSpec, g.Status, g.OldSpec)
239+
testErrors(t, g.Details, errorList, g.ExpectedErrors)
240+
}
241+
}
242+
243+
func TestEtcdVersionRequiredWithImage(t *testing.T) {
244+
grid := []struct {
245+
Details string
246+
Spec kops.EtcdClusterSpec
247+
ExpectedErrors []string
248+
}{
249+
{
250+
Details: "image requires version",
251+
Spec: kops.EtcdClusterSpec{
252+
Name: "main",
253+
Members: []kops.EtcdMemberSpec{{Name: "a", InstanceGroup: fi.PtrTo("eu-central-1a")}},
254+
Image: "gcr.io/etcd-development/etcd:v3.6.99",
255+
},
256+
ExpectedErrors: []string{
257+
"Required value::spec.etcdClusters[0].version",
258+
},
259+
},
260+
{
261+
Details: "image with version is valid",
262+
Spec: kops.EtcdClusterSpec{
263+
Name: "main",
264+
Members: []kops.EtcdMemberSpec{{Name: "a", InstanceGroup: fi.PtrTo("eu-central-1a")}},
265+
Version: "3.6.99",
266+
Image: "gcr.io/etcd-development/etcd:v3.6.99",
267+
},
268+
},
269+
{
270+
Details: "neither image nor version is valid",
271+
Spec: kops.EtcdClusterSpec{
272+
Name: "main",
273+
Members: []kops.EtcdMemberSpec{{Name: "a", InstanceGroup: fi.PtrTo("eu-central-1a")}},
274+
},
275+
},
276+
}
277+
278+
for _, g := range grid {
279+
fp := field.NewPath("spec", "etcdClusters").Index(0)
280+
errorList := validateEtcdClusterSpec(g.Spec, nil, fp)
281+
testErrors(t, g.Details, errorList, g.ExpectedErrors)
282+
}
283+
}

‎pkg/apis/kops/validation/validation.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1535,6 +1535,9 @@ func validateEtcdClusterSpec(spec kops.EtcdClusterSpec, c *kops.Cluster, fieldPa
15351535
// Not technically a requirement, but doesn't really make sense to allow
15361536
allErrs = append(allErrs, field.Invalid(fieldPath.Child("etcdMembers"), len(spec.Members), "Should be an odd number of control-plane-zones for quorum. Use --zones and --control-plane-zones to declare node zones and control-plane zones separately"))
15371537
}
1538+
if spec.Image != "" && spec.Version == "" {
1539+
allErrs = append(allErrs, field.Required(fieldPath.Child("version"), "version must be set when image is set"))
1540+
}
15381541
allErrs = append(allErrs, validateEtcdVersion(spec, fieldPath, nil)...)
15391542
for i, m := range spec.Members {
15401543
allErrs = append(allErrs, validateEtcdMemberSpec(m, fieldPath.Child("etcdMembers").Index(i))...)

‎pkg/featureflag/featureflag.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,6 @@ var (
6969
SpotinstController = new("SpotinstController", Bool(true))
7070
// VPCSkipEnableDNSSupport if set will make that a VPC does not need DNSSupport enabled.
7171
VPCSkipEnableDNSSupport = new("VPCSkipEnableDNSSupport", Bool(false))
72-
// SkipEtcdVersionCheck will bypass the check that etcd-manager is using a supported etcd version
73-
SkipEtcdVersionCheck = new("SkipEtcdVersionCheck", Bool(false))
7472
// EtcdEventsHTTP enables HTTP (non-TLS) for the events etcd cluster.
7573
// This matches the pattern used by GCE scale tests and can help with
7674
// TLS handshake overhead for the ephemeral events data.

‎pkg/model/components/etcdmanager/model.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -274,8 +274,16 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
274274
}
275275
}
276276

277+
etcdVersions := etcdSupportedVersions()
278+
if etcdCluster.Image != "" {
279+
// With a custom image, only the selected version's binaries are made
280+
// available in the pod; restoring backups, which can require the
281+
// binaries of a bundled version, is not supported.
282+
etcdVersions = []etcdVersion{{Version: strings.TrimPrefix(etcdCluster.Version, "v"), Image: etcdCluster.Image}}
283+
}
284+
277285
if b.Cluster.HasImageVolumesSupport() {
278-
for _, etcdVersion := range etcdSupportedVersions() {
286+
for _, etcdVersion := range etcdVersions {
279287
if etcdVersion.SymlinkToVersion == "" {
280288
volume := v1.Volume{
281289
Name: "etcd-v" + strings.ReplaceAll(etcdVersion.Version, ".", "-"),
@@ -311,7 +319,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
311319
}
312320

313321
symlinkToVersions := sets.NewString()
314-
for _, etcdVersion := range etcdSupportedVersions() {
322+
for _, etcdVersion := range etcdVersions {
315323
if etcdVersion.SymlinkToVersion != "" {
316324
symlinkToVersions.Insert(etcdVersion.SymlinkToVersion)
317325
continue
@@ -336,7 +344,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
336344
for _, symlinkToVersion := range symlinkToVersions.List() {
337345
targetVersions := sets.NewString()
338346

339-
for _, etcdVersion := range etcdSupportedVersions() {
347+
for _, etcdVersion := range etcdVersions {
340348
if etcdVersion.SymlinkToVersion == symlinkToVersion {
341349
targetVersions.Insert(etcdVersion.Version)
342350
}
@@ -386,7 +394,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
386394
container.Image = b.AssetBuilder.RemapImage(container.Image)
387395

388396
if b.Cluster.HasImageVolumesSupport() {
389-
for _, etcdVersion := range etcdSupportedVersions() {
397+
for _, etcdVersion := range etcdVersions {
390398
volumeMount := v1.VolumeMount{
391399
MountPath: "/opt/etcd-v" + etcdVersion.Version,
392400
}

‎pkg/model/components/etcdmanager/options.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import (
2424
"github.com/blang/semver/v4"
2525
"k8s.io/klog/v2"
2626
"k8s.io/kops/pkg/apis/kops"
27-
"k8s.io/kops/pkg/featureflag"
2827
"k8s.io/kops/pkg/model/components"
2928
"k8s.io/kops/upup/pkg/fi/loader"
3029
)
@@ -65,17 +64,16 @@ func (b *EtcdManagerOptionsBuilder) BuildOptions(o *kops.Cluster) error {
6564
}
6665

6766
if !etcdVersionIsSupported(etcdCluster.Version) {
68-
if featureflag.SkipEtcdVersionCheck.Enabled() {
69-
klog.Warningf("etcd version %q is not known to be supported, but ignoring because of SkipEtcdVersionCheck feature flag", etcdCluster.Version)
67+
if etcdCluster.Image != "" {
68+
klog.Warningf("etcd version %q is not bundled by kOps and has not been tested; using binaries from custom image %q", etcdCluster.Version, etcdCluster.Image)
7069
} else {
7170
klog.Warningf("Unsupported etcd version %q detected; please update etcd version.", etcdCluster.Version)
72-
klog.Warningf("Use export KOPS_FEATURE_FLAGS=SkipEtcdVersionCheck to override this check.")
7371
var versions []string
7472
for _, v := range etcdSupportedVersions() {
7573
versions = append(versions, v.Version)
7674
}
7775
klog.Warningf("Supported etcd versions: %s", strings.Join(versions, ", "))
78-
return fmt.Errorf("etcd version %q is not supported with etcd-manager, please specify a supported version or remove the value to use the recommended version", etcdCluster.Version)
76+
return fmt.Errorf("etcd version %q is not supported with etcd-manager, please specify a supported version, remove the value to use the recommended version, or also set the image field to run a custom version", etcdCluster.Version)
7977
}
8078
}
8179
}

‎pkg/model/components/etcdmanager/options_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,84 @@ limitations under the License.
1717
package etcdmanager
1818

1919
import (
20+
"fmt"
2021
"testing"
22+
23+
"github.com/blang/semver/v4"
24+
"k8s.io/kops/pkg/apis/kops"
25+
"k8s.io/kops/pkg/assets"
26+
"k8s.io/kops/pkg/model/components"
27+
"k8s.io/kops/util/pkg/vfs"
2128
)
2229

30+
func TestBuildOptionsEtcdVersionCheck(t *testing.T) {
31+
latest36 := components.LatestEtcd36Version
32+
sv := semver.MustParse(latest36)
33+
next36 := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch+1)
34+
35+
tests := []struct {
36+
name string
37+
version string
38+
image string
39+
expectError bool
40+
}{
41+
{
42+
name: "bundled version",
43+
version: latest36,
44+
},
45+
{
46+
name: "version not bundled",
47+
version: next36,
48+
expectError: true,
49+
},
50+
{
51+
name: "version not bundled, with custom image",
52+
version: next36,
53+
image: "gcr.io/etcd-development/etcd:v" + next36,
54+
},
55+
{
56+
name: "bundled version, with custom image",
57+
version: latest36,
58+
image: "gcr.io/etcd-development/etcd:v" + latest36,
59+
},
60+
}
61+
62+
for _, test := range tests {
63+
t.Run(test.name, func(t *testing.T) {
64+
cluster := &kops.Cluster{
65+
Spec: kops.ClusterSpec{
66+
KubernetesVersion: "1.34.0",
67+
ConfigStore: kops.ConfigStoreSpec{
68+
Base: "memfs://clusters.example.com/minimal.example.com",
69+
},
70+
EtcdClusters: []kops.EtcdClusterSpec{
71+
{
72+
Name: "main",
73+
Version: test.version,
74+
Image: test.image,
75+
},
76+
},
77+
},
78+
}
79+
80+
assetBuilder := assets.NewAssetBuilder(vfs.Context, cluster.Spec.Assets, false)
81+
optionsContext, err := components.NewOptionsContext(cluster, assetBuilder, assetBuilder.KubeletSupportedVersion)
82+
if err != nil {
83+
t.Fatalf("unexpected error from NewOptionsContext: %v", err)
84+
}
85+
86+
builder := &EtcdManagerOptionsBuilder{OptionsContext: optionsContext}
87+
err = builder.BuildOptions(cluster)
88+
if test.expectError && err == nil {
89+
t.Errorf("expected error from BuildOptions, got none")
90+
}
91+
if !test.expectError && err != nil {
92+
t.Errorf("unexpected error from BuildOptions: %v", err)
93+
}
94+
})
95+
}
96+
}
97+
2398
func TestJoin(t *testing.T) {
2499
tests := []struct {
25100
base string

0 commit comments

Comments
 (0)