Sitelet https://github.com/kubernetes/kops/pull/18684/files
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions cmd/kops/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,14 @@ func TestMinimalIPv6(t *testing.T) {
runTestTerraformAWS(t)
}

// TestMinimalIPv6DNSNone runs the test on a minimum IPv6 configuration with --dns=none.
// Nodes in an IPv6-only cluster have no IPv4 connectivity, so the addresses baked into their
// bootstrap config must be IPv6 only.
func TestMinimalIPv6DNSNone(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-dns-none").
runTestTerraformAWS(t)
}

// TestMinimalIPv6Calico runs the test on a minimum IPv6 configuration with Calico
func TestMinimalIPv6Calico(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-calico").
Expand Down
32 changes: 22 additions & 10 deletions pkg/kopscontrollerclient/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,28 @@ type Client struct {
// BaseURL is the base URL for the server
BaseURL url.URL

// Backoff controls how long a single Query keeps retrying BaseURL before giving up.
// If unset, DefaultBackoff is used. Callers that have more than one server to try
// should set a shorter backoff, so that one unreachable server does not consume the
// whole budget.
Backoff wait.Backoff

httpClient *http.Client
}

// DefaultBackoff is the retry behaviour of a Query whose caller has not set Client.Backoff.
// The interval is capped so a control plane that takes a long time to become reachable does not
// push the next attempt tens of minutes out. Without a cap, doubling from 1s reaches a 17
// minute wait by attempt 11, so a node that has been failing for 17 minutes then sits idle
// for another 17 even once kops-controller is serving.
var DefaultBackoff = wait.Backoff{
Duration: 1 * time.Second,
Factor: 2,
Jitter: 0.1,
Cap: 30 * time.Second,
Steps: 100,
}

func New(authenticator bootstrap.Authenticator, cas []byte, baseURL url.URL) *Client {
return NewWithTLSServerName(authenticator, cas, baseURL, "")
}
Expand Down Expand Up @@ -101,16 +120,9 @@ func (b *Client) Query(ctx context.Context, req any, resp any) error {
bootstrapURL := b.BaseURL
bootstrapURL.Path = path.Join(bootstrapURL.Path, "/bootstrap")

// Cap the interval so a control plane that takes a long time to become reachable does not
// push the next attempt tens of minutes out. Without a cap, doubling from 1s reaches a 17
// minute wait by attempt 11, so a node that has been failing for 17 minutes then sits idle
// for another 17 even once kops-controller is serving.
backoff := wait.Backoff{
Duration: 1 * time.Second,
Factor: 2,
Jitter: 0.1,
Cap: 30 * time.Second,
Steps: 100,
backoff := b.Backoff
if backoff.Steps == 0 {
backoff = DefaultBackoff
}

var response *http.Response
Expand Down
102 changes: 60 additions & 42 deletions pkg/nodemodel/nodeupconfigbuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -291,48 +291,9 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, wellKnownAddre
}

// Set API server address to an IP from the cluster network CIDR
var controlPlaneIPs []string
switch cluster.GetCloudProvider() {
case kops.CloudProviderAWS, kops.CloudProviderHetzner, kops.CloudProviderOpenstack:
// Use a private IP address that belongs to the cluster network CIDR, or any IPv6 addresses (some additional addresses may be FQDNs or public IPs)
for _, additionalIP := range wellKnownAddresses[wellknownservices.KubeAPIServer] {
for _, networkCIDR := range append(cluster.Spec.Networking.AdditionalNetworkCIDRs, cluster.Spec.Networking.NetworkCIDR) {
cidr, err := netip.ParsePrefix(networkCIDR)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse network CIDR %q: %w", networkCIDR, err)
}
ip, err := netip.ParseAddr(additionalIP)
if err != nil {
continue
}
if cidr.Contains(ip) || ip.Is6() {
controlPlaneIPs = append(controlPlaneIPs, additionalIP)
}
}
}

case kops.CloudProviderGCE:
// Use the IP address of the internal load balancer (forwarding-rule)
// Note that on GCE subnets have IP ranges, networks do not
for _, apiserverIP := range wellKnownAddresses[wellknownservices.KubeAPIServer] {
for _, subnet := range cluster.Spec.Networking.Subnets {
cidr, err := netip.ParsePrefix(subnet.CIDR)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse subnet CIDR %q: %w", subnet.CIDR, err)
}
ip, err := netip.ParseAddr(apiserverIP)
if err != nil {
continue
}
if cidr.Contains(ip) {
controlPlaneIPs = append(controlPlaneIPs, apiserverIP)
}
}
}

case kops.CloudProviderDO, kops.CloudProviderScaleway, kops.CloudProviderAzure, kops.CloudProviderMetal:
// Use any IP address that is found (including public ones)
controlPlaneIPs = append(controlPlaneIPs, wellKnownAddresses[wellknownservices.KubeAPIServer]...)
controlPlaneIPs, err := selectControlPlaneIPs(cluster, wellKnownAddresses[wellknownservices.KubeAPIServer])
if err != nil {
return nil, nil, err
}

// Bake control-plane IPs into /etc/hosts (for api.internal and kops-controller.internal):
Expand Down Expand Up @@ -418,6 +379,63 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, wellKnownAddre
return config, bootConfig, nil
}

// selectControlPlaneIPs narrows the addresses that reach the API server down to the ones a node
// in this cluster can actually connect to. Some of the addresses may be FQDNs or public IPs.
func selectControlPlaneIPs(cluster *kops.Cluster, apiserverAddresses []string) ([]string, error) {
var controlPlaneIPs []string

switch cluster.GetCloudProvider() {
case kops.CloudProviderAWS, kops.CloudProviderHetzner, kops.CloudProviderOpenstack:
// Use a private IP address that belongs to the cluster network CIDR, or any IPv6 addresses (some additional addresses may be FQDNs or public IPs)
for _, additionalIP := range apiserverAddresses {
for _, networkCIDR := range append(cluster.Spec.Networking.AdditionalNetworkCIDRs, cluster.Spec.Networking.NetworkCIDR) {
cidr, err := netip.ParsePrefix(networkCIDR)
if err != nil {
return nil, fmt.Errorf("failed to parse network CIDR %q: %w", networkCIDR, err)
}
ip, err := netip.ParseAddr(additionalIP)
if err != nil {
continue
}
// Nodes in an IPv6-only cluster sit in subnets that have no IPv4 CIDR, so an IPv4
// address is unroutable from them even though it is inside the network CIDR. Handing
// one out only stalls bootstrap on an address that can never answer.
if cluster.Spec.IsIPv6Only() && !ip.Is6() {
continue
}
if cidr.Contains(ip) || ip.Is6() {
controlPlaneIPs = append(controlPlaneIPs, additionalIP)
}
}
}

case kops.CloudProviderGCE:
// Use the IP address of the internal load balancer (forwarding-rule)
// Note that on GCE subnets have IP ranges, networks do not
for _, apiserverIP := range apiserverAddresses {
for _, subnet := range cluster.Spec.Networking.Subnets {
cidr, err := netip.ParsePrefix(subnet.CIDR)
if err != nil {
return nil, fmt.Errorf("failed to parse subnet CIDR %q: %w", subnet.CIDR, err)
}
ip, err := netip.ParseAddr(apiserverIP)
if err != nil {
continue
}
if cidr.Contains(ip) {
controlPlaneIPs = append(controlPlaneIPs, apiserverIP)
}
}
}

case kops.CloudProviderDO, kops.CloudProviderScaleway, kops.CloudProviderAzure, kops.CloudProviderMetal:
// Use any IP address that is found (including public ones)
controlPlaneIPs = append(controlPlaneIPs, apiserverAddresses...)
}

return controlPlaneIPs, nil
}

func buildConfigServerOptions(clusterName string, caCertificates string, apiserverIPs []string) *nodeup.ConfigServerOptions {
kopsControllerName := "kops-controller.internal." + clusterName
hosts := []string{kopsControllerName}
Expand Down
62 changes: 62 additions & 0 deletions pkg/nodemodel/nodeupconfigbuilder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,70 @@ package nodemodel
import (
"reflect"
"testing"

"k8s.io/kops/pkg/apis/kops"
)

func TestSelectControlPlaneIPs(t *testing.T) {
// The addresses of an API load balancer, as FindAddresses returns them: the DNS name,
// then the ENI addresses sorted as strings, which puts IPv4 ahead of IPv6.
addresses := []string{
"api-abc123.elb.us-test-1.amazonaws.com",
"172.20.6.26",
"2001:db8:0:113::a",
}

grid := []struct {
name string
nonMasqueradeCIDR string
addresses []string
want []string
}{
{
name: "ipv4 cluster keeps the private IPv4 address",
nonMasqueradeCIDR: "100.64.0.0/10",
addresses: addresses,
want: []string{"172.20.6.26", "2001:db8:0:113::a"},
},
{
// Nodes in an IPv6-only cluster have no IPv4 address at all, so 172.20.6.26 is
// unroutable from them even though it is inside the network CIDR.
name: "ipv6-only cluster drops the IPv4 address",
nonMasqueradeCIDR: "::/0",
addresses: addresses,
want: []string{"2001:db8:0:113::a"},
},
{
name: "public IPv4 addresses are excluded",
nonMasqueradeCIDR: "100.64.0.0/10",
addresses: []string{"203.0.113.7"},
want: nil,
},
}

for _, g := range grid {
t.Run(g.name, func(t *testing.T) {
cluster := &kops.Cluster{
Spec: kops.ClusterSpec{
CloudProvider: kops.CloudProviderSpec{AWS: &kops.AWSSpec{}},
Networking: kops.NetworkingSpec{
NetworkCIDR: "172.20.0.0/16",
NonMasqueradeCIDR: g.nonMasqueradeCIDR,
},
},
}

got, err := selectControlPlaneIPs(cluster, g.addresses)
if err != nil {
t.Fatalf("selectControlPlaneIPs failed: %v", err)
}
if !reflect.DeepEqual(got, g.want) {
t.Errorf("selectControlPlaneIPs = %v, want %v", got, g.want)
}
})
}
}

func TestBuildConfigServerOptionsUsesTLSServerNameForIPServers(t *testing.T) {
options := buildConfigServerOptions("cluster.k8s.local", "ca-data", []string{"10.0.1.2"})

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "Service": "ec2.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "Service": "ec2.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
Loading
Loading