Sitelet https://github.com/kubernetes/kops/pull/18630/files
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions pkg/model/resources/nodeup.go
Original file line number Diff line number Diff line change
Expand Up @@ -448,11 +448,7 @@ func buildEnvironmentVariables(cluster *kops.Cluster, ig *kops.InstanceGroup) (m
if err != nil {
return nil, err
}
if region == "" {
klog.Warningf("unable to determine cluster region")
} else {
env["AWS_REGION"] = region
}
env["AWS_REGION"] = region
}

if cluster.GetCloudProvider() == kops.CloudProviderAzure {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ spec:
id: subnet-abcdef
name: utility-us-test-1a
type: Utility
zone: us-test-1a

---

Expand Down
15 changes: 12 additions & 3 deletions upup/pkg/fi/cloudup/awsup/aws_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,14 +92,19 @@ func ValidateRegion(ctx context.Context, region string) error {
func FindRegion(cluster *kops.Cluster) (string, error) {
region := ""

nodeZones := make(map[string]bool)
for _, subnet := range cluster.Spec.Networking.Subnets {
if subnet.Zone == "" {
// The zone of a subnet specified by ID is looked up from the cloud later.
if subnet.ID == "" {
return "", fmt.Errorf("subnet %q must specify a zone or the ID of an existing subnet", subnet.Name)
}
continue
}

if len(subnet.Zone) <= 2 {
return "", fmt.Errorf("invalid AWS zone: %q in subnet %q", subnet.Zone, subnet.Name)
}

nodeZones[subnet.Zone] = true

zoneRegion := subnet.Zone[:len(subnet.Zone)-1]
if region != "" && zoneRegion != region {
return "", fmt.Errorf("error Clusters cannot span multiple regions (found zone %q, but region is %q)", subnet.Zone, region)
Expand All @@ -108,6 +113,10 @@ func FindRegion(cluster *kops.Cluster) (string, error) {
region = zoneRegion
}

if region == "" {
return "", fmt.Errorf("could not determine cluster region: no subnet specifies a zone")
}

return region, nil
}

Expand Down
51 changes: 51 additions & 0 deletions upup/pkg/fi/cloudup/awsup/aws_utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,57 @@ func TestFindRegion(t *testing.T) {
t.Fatalf("unexpected region for zone: %q vs %q", expected, region)
}
}

testCases := []struct {
name string
subnets []kops.ClusterSubnetSpec
expected string
expectError bool
}{
{
name: "ignores subnets specified by ID without a zone",
subnets: []kops.ClusterSubnetSpec{
{Name: "subnet-a", ID: "subnet-12345678"},
{Name: "subnet-b", Zone: "us-east-2b"},
},
expected: "us-east-2",
},
{
name: "errors when no subnet specifies a zone",
subnets: []kops.ClusterSubnetSpec{
{Name: "subnet-a", ID: "subnet-12345678"},
},
expectError: true,
},
{
name: "errors when a subnet has neither zone nor ID",
subnets: []kops.ClusterSubnetSpec{
{Name: "subnet-a"},
{Name: "subnet-b", Zone: "us-east-2b"},
},
expectError: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
c := &kops.Cluster{}
c.Spec.Networking.Subnets = tc.subnets

region, err := FindRegion(c)
if tc.expectError {
if err == nil {
t.Fatalf("expected error finding region, got %q", region)
}
return
}
if err != nil {
t.Fatalf("unexpected error finding region: %v", err)
}
if region != tc.expected {
t.Fatalf("unexpected region: %q vs %q", region, tc.expected)
}
})
}
}

func TestEC2TagSpecification(t *testing.T) {
Expand Down
30 changes: 27 additions & 3 deletions upup/pkg/fi/cloudup/subnets.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,19 @@ func assignCIDRsToSubnets(c *kops.Cluster, cloud fi.Cloud) error {
// TODO: We probably could query for the existing subnets & allocate appropriately
// for now we'll require users to set CIDRs themselves

if allSubnetsHaveCIDRs(c) {
// On AWS, subnets specified by ID may omit the zone; we look it up from the cloud so that the
// rest of kOps can rely on the zone being set.
needZones := false
if c.GetCloudProvider() == kops.CloudProviderAWS {
for _, subnet := range c.Spec.Networking.Subnets {
if subnet.ID != "" && subnet.Zone == "" {
needZones = true
break
}
}
}

if allSubnetsHaveCIDRs(c) && !needZones {
klog.V(4).Infof("All subnets have CIDRs; skipping assignment logic")
return nil
}
Expand Down Expand Up @@ -75,21 +87,33 @@ func assignCIDRsToSubnets(c *kops.Cluster, cloud fi.Cloud) error {
}
if subnet.CIDR == "" {
subnet.CIDR = cloudSubnet.CIDR
if subnet.CIDR == "" {
// IPv6-only private subnets do not have an IPv4 CIDR
if subnet.CIDR == "" && (subnet.IPv6CIDR == "" || subnet.Type != kops.SubnetTypePrivate) {
return fmt.Errorf("Subnet %q did not have CIDR", subnet.ID)
}
} else if subnet.CIDR != cloudSubnet.CIDR {
return fmt.Errorf("Subnet %q has configured CIDR %q, but the actual CIDR found was %q", subnet.ID, subnet.CIDR, cloudSubnet.CIDR)
}

if subnet.Zone != cloudSubnet.Zone {
if needZones && subnet.Zone == "" {
subnet.Zone = cloudSubnet.Zone
} else if subnet.Zone != cloudSubnet.Zone {
return fmt.Errorf("Subnet %q has configured Zone %q, but the actual Zone found was %q", subnet.ID, subnet.Zone, cloudSubnet.Zone)
}

}
}
}

if needZones {
for i := range c.Spec.Networking.Subnets {
subnet := &c.Spec.Networking.Subnets[i]
if subnet.ID != "" && subnet.Zone == "" {
return fmt.Errorf("could not determine the zone of subnet %q; specify the zone in the cluster spec", subnet.Name)
}
}
}

if allSubnetsHaveCIDRs(c) {
klog.V(4).Infof("All subnets have CIDRs; skipping assignment logic")
return nil
Expand Down
74 changes: 74 additions & 0 deletions upup/pkg/fi/cloudup/subnets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ import (
"reflect"
"testing"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"k8s.io/kops/cloudmock/aws/mockec2"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
)

func Test_AssignSubnets(t *testing.T) {
Expand Down Expand Up @@ -147,3 +151,73 @@ func Test_AssignSubnets(t *testing.T) {
})
}
}

func Test_AssignSubnetZones(t *testing.T) {
buildCloud := func() *awsup.MockAWSCloud {
cloud := awsup.BuildMockAWSCloud("us-test-1", "abc")
mockEC2 := &mockec2.MockEC2{}
cloud.MockEC2 = mockEC2

mockEC2.CreateVpcWithId(&ec2.CreateVpcInput{
CidrBlock: aws.String("172.20.0.0/16"),
}, "vpc-12345678")
mockEC2.CreateSubnetWithId(&ec2.CreateSubnetInput{
VpcId: aws.String("vpc-12345678"),
AvailabilityZone: aws.String("us-test-1a"),
CidrBlock: aws.String("172.20.32.0/19"),
}, "subnet-a")
mockEC2.CreateSubnetWithId(&ec2.CreateSubnetInput{
VpcId: aws.String("vpc-12345678"),
AvailabilityZone: aws.String("us-test-1b"),
Ipv6CidrBlock: aws.String("2001:db8::/64"),
}, "subnet-b")
return cloud
}

buildCluster := func(subnets []kops.ClusterSubnetSpec) *kops.Cluster {
c := &kops.Cluster{}
c.Spec.CloudProvider.AWS = &kops.AWSSpec{}
c.Spec.Networking.NetworkID = "vpc-12345678"
c.Spec.Networking.NetworkCIDR = "172.20.0.0/16"
c.Spec.Networking.Subnets = subnets
return c
}

t.Run("zone is backfilled from the cloud", func(t *testing.T) {
c := buildCluster([]kops.ClusterSubnetSpec{
{Name: "a", ID: "subnet-a", CIDR: "172.20.32.0/19", Type: kops.SubnetTypePublic},
})

if err := assignCIDRsToSubnets(c, buildCloud()); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if zone := c.Spec.Networking.Subnets[0].Zone; zone != "us-test-1a" {
t.Fatalf("unexpected zone: %q", zone)
}
})

t.Run("IPv6-only private subnets do not require an IPv4 CIDR", func(t *testing.T) {
c := buildCluster([]kops.ClusterSubnetSpec{
{Name: "a", ID: "subnet-a", CIDR: "172.20.32.0/19", Type: kops.SubnetTypePublic},
{Name: "b", ID: "subnet-b", IPv6CIDR: "2001:db8::/64", Zone: "us-test-1b", Type: kops.SubnetTypePrivate},
})

if err := assignCIDRsToSubnets(c, buildCloud()); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if zone := c.Spec.Networking.Subnets[0].Zone; zone != "us-test-1a" {
t.Fatalf("unexpected zone: %q", zone)
}
})

t.Run("errors when the zone cannot be determined", func(t *testing.T) {
c := buildCluster([]kops.ClusterSubnetSpec{
{Name: "a", ID: "subnet-a", CIDR: "172.20.32.0/19", Type: kops.SubnetTypePublic},
})
c.Spec.Networking.NetworkID = ""

if err := assignCIDRsToSubnets(c, buildCloud()); err == nil {
t.Fatalf("expected error assigning zones")
}
})
}
5 changes: 4 additions & 1 deletion upup/pkg/fi/cloudup/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,10 @@ func BuildCloud(cluster *kops.Cluster) (fi.Cloud, error) {

var zoneNames []string
for _, subnet := range cluster.Spec.Networking.Subnets {
zoneNames = append(zoneNames, subnet.Zone)
// The zone of a subnet specified by ID is looked up from the cloud later.
if subnet.Zone != "" {
zoneNames = append(zoneNames, subnet.Zone)
}
}
err = awsup.ValidateZones(zoneNames, awsCloud)
if err != nil {
Expand Down
Loading