Sitelet https://github.com/kubernetes/kops/pull/18640/files
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion channels/pkg/cmd/apply_channel.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"net/url"
"os"
"os/signal"
"strings"
"syscall"
"time"

Expand All @@ -46,15 +47,24 @@ type ApplyChannelOptions struct {
Yes bool
Interval time.Duration
NodeName string

// Comma delimited label,value pairs to add to the node. Eg "kops.k8s.io/cloud-controller-manager,foo=bar"
NodeLabels map[string]string
}

func NewCmdApplyChannel(f *ChannelsFactory, out io.Writer) *cobra.Command {
var options ApplyChannelOptions
var rawLabels string

cmd := &cobra.Command{
Use: "channel CHANNEL...",
Short: "Applies updates from the given channel(s)",
RunE: func(cmd *cobra.Command, args []string) error {
var err error
options.NodeLabels, err = parseLabels(rawLabels)
if err != nil {
return err
}
if options.Interval > 0 {
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
Expand All @@ -67,10 +77,27 @@ func NewCmdApplyChannel(f *ChannelsFactory, out io.Writer) *cobra.Command {
cmd.Flags().BoolVar(&options.Yes, "yes", false, "Apply update")
cmd.Flags().DurationVar(&options.Interval, "interval", 0, "If non-zero, re-apply the channel on this interval until interrupted (e.g. 60s)")
cmd.Flags().StringVar(&options.NodeName, "node-name", "", "If set, patch the named node with the mandatory control-plane labels each iteration; typically supplied via the downward API.")
cmd.Flags().StringVar(&rawLabels, "node-labels", "", "If set, patch the named node with each of the label,value pairs each iteration; typically supplied via the downward API.")

return cmd
}

func parseLabels(rawLabels string) (map[string]string, error) {
labels := make(map[string]string)
pairs := strings.Split(rawLabels, ",")
for _, rawpair := range pairs {
pair := strings.Split(rawpair, "=")
if len(pair) > 2 {
return nil, fmt.Errorf("Error too many '=' (%d) in %s", len(pair), pair)
} else if len(pair) == 2 {
labels[pair[0]] = pair[1]
} else {
labels[rawpair] = ""
}
}
return labels, nil
}

// runApplyChannelIteration patches node labels (when --node-name is set) then
// applies the channel. Labels go first so addons targeting the control-plane
// label can schedule on the local node as soon as their manifests land.
Expand All @@ -80,7 +107,7 @@ func runApplyChannelIteration(ctx context.Context, f *ChannelsFactory, out io.Wr
labelerClient, err := f.KubernetesClient()
if err != nil {
merr = multierr.Append(merr, fmt.Errorf("building kubernetes client for node labeler: %w", err))
} else if err := nodelabeler.BootstrapControlPlaneNodeLabels(ctx, labelerClient, options.NodeName); err != nil {
} else if err := nodelabeler.BootstrapControlPlaneNodeLabels(ctx, labelerClient, options.NodeName, options.NodeLabels); err != nil {
merr = multierr.Append(merr, fmt.Errorf("bootstrapping node labels: %w", err))
}
}
Expand Down
4 changes: 2 additions & 2 deletions channels/pkg/nodelabeler/labeler.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type nodePatchMetadata struct {

// BootstrapControlPlaneNodeLabels applies labels to the current node so that it acts as a control-plane.
// Safe to call repeatedly: the patch is skipped when the labels already match.
func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Interface, nodeName string) error {
func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Interface, nodeName string, nodeLabels map[string]string) error {
if nodeName == "" {
return fmt.Errorf("node name is required")
}
Expand All @@ -49,7 +49,7 @@ func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Inte
return fmt.Errorf("querying node %q: %w", nodeName, err)
}

labels := nodelabels.BuildMandatoryControlPlaneLabels()
labels := nodelabels.BuildMandatoryControlPlaneLabels(nodeLabels)

shouldPatch := false
for k, v := range labels {
Expand Down
3 changes: 0 additions & 3 deletions cmd/kops/create_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,6 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
if featureflag.ExperimentalRoles.Enabled() {
cmd.Flags().Int32Var(&options.EtcdCount, "etcd-count", options.EtcdCount, "Number of etcd nodes. Defaults to 0.")
cmd.Flags().Int32Var(&options.SchedulerCount, "scheduler-count", options.SchedulerCount, "Number of scheduler nodes. Defaults to 0.")
cmd.Flags().Int32Var(&options.CloudControllerManagerCount, "ccm-count", options.CloudControllerManagerCount, "Number of cloud-controller-manager nodes. Defaults to 0.")
Comment thread
cheftako marked this conversation as resolved.
cmd.Flags().Int32Var(&options.KubeControllerManagerCount, "kcm-count", options.KubeControllerManagerCount, "Number of kube-controller-manager nodes. Defaults to 0.")
}

Expand Down Expand Up @@ -335,8 +334,6 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
cmd.RegisterFlagCompletionFunc("etcd-size", completeMachineType)
cmd.Flags().StringSliceVar(&options.SchedulerSizes, "scheduler-size", options.SchedulerSizes, "Machine type(s) for scheduler nodes")
cmd.RegisterFlagCompletionFunc("scheduler-size", completeMachineType)
cmd.Flags().StringSliceVar(&options.CloudControllerManagerSizes, "ccm-size", options.CloudControllerManagerSizes, "Machine type(s) for cloud-controller-manager nodes")
cmd.RegisterFlagCompletionFunc("ccm-size", completeMachineType)
cmd.Flags().StringSliceVar(&options.KubeControllerManagerSizes, "kcm-size", options.KubeControllerManagerSizes, "Machine type(s) for kube-controller-manager nodes")
cmd.RegisterFlagCompletionFunc("kcm-size", completeMachineType)
}
Expand Down
8 changes: 8 additions & 0 deletions cmd/kops/create_cluster_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,14 @@ func TestCreateClusterGCE(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/gce_byo_sa", "v1alpha2")
}

// TestCreateClusterExperimentalRoles tests kops create cluster with ExperimentalRoles and control-plane-count=0
func TestCreateClusterExperimentalRoles(t *testing.T) {
featureflag.ParseFlags("+APIServerNodes,+ExperimentalRoles")
defer featureflag.ParseFlags("-APIServerNodes,-ExperimentalRoles")

runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/experimental-roles", "v1alpha2")
}

// TestCreateClusterHASharedZone tests kops create cluster when the master count is bigger than the number of zones
func TestCreateClusterHASharedZone(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_shared_zone", "v1alpha2")
Expand Down
5 changes: 1 addition & 4 deletions cmd/kops/create_instancegroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,7 @@ func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
if r.HasScheduler() {
continue
}
if r.HasCloudControllerManager() {
continue
}
if r.HasKubControllerManager() {
if r.HasKubeControllerManager() {
continue
}
}
Expand Down
2 changes: 1 addition & 1 deletion docs/cli/kops_rolling-update_cluster.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/cli/kops_update_cluster.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions hack/verify-ig-role-comparisons.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ files=$(find . -name "*.go" -not -path "./vendor/*" -not -path "./.build/*" -not
# constant == variable
# variable != constant
# constant != variable
REGEX='==[[:space:]]*([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer|Etcd|Scheduler|CloudControllerManager|KubeControllerManager)\b|\b([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer)[[:space:]]*==|!=[[:space:]]*([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer)\b|\b([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer)[[:space:]]*!='
REGEX='==[[:space:]]*([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer|Etcd|Scheduler|KubeControllerManager)\b|\b([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer)[[:space:]]*==|!=[[:space:]]*([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer)\b|\b([a-zA-Z0-9_]+\.)?InstanceGroupRole(ControlPlane|Node|Bastion|APIServer)[[:space:]]*!='

for file in $files; do
if grep -E "${REGEX}" "${file}" > /dev/null; then
Expand All @@ -45,7 +45,7 @@ for file in $files; do
done

if [ "${errors}" -ne 0 ]; then
echo "Error: Found ${errors} files with direct InstanceGroupRole comparisons (== or !=). Use HasControlPlane(), HasNode(), HasBastion(), HasAPIServer(), HasEtcd(), HasScheduler(), HasCloudControllerManager() or HasKubeControllerManager() instead."
echo "Error: Found ${errors} files with direct InstanceGroupRole comparisons (== or !=). Use HasControlPlane(), HasNode(), HasBastion(), HasAPIServer(), HasEtcd(), HasScheduler() or HasKubeControllerManager() instead."
exit 1
fi

Expand Down
51 changes: 39 additions & 12 deletions pkg/apis/kops/instancegroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,8 @@ const (
InstanceGroupRoleEtcd InstanceGroupRole = "Etcd"
// InstanceGroupRoleScheduler is a Scheduler role.
InstanceGroupRoleScheduler InstanceGroupRole = "Scheduler"
// InstanceGroupRoleCloudControllerManager is a CloudControllerManager role.
InstanceGroupRoleCloudControllerManager InstanceGroupRole = "CloudControllerManager"
// InstanceGroupRoleKubControllerManager is a KubControllerManager role.
InstanceGroupRoleKubControllerManager InstanceGroupRole = "KubControllerManager"
// InstanceGroupRoleKubeControllerManager is a KubeControllerManager role.
InstanceGroupRoleKubeControllerManager InstanceGroupRole = "KubeControllerManager"
)

// AllInstanceGroupRoles is a slice of all valid InstanceGroupRole values
Expand All @@ -79,6 +77,9 @@ var AllInstanceGroupRoles = []InstanceGroupRole{
InstanceGroupRoleAPIServer,
InstanceGroupRoleNode,
InstanceGroupRoleBastion,
InstanceGroupRoleEtcd,
InstanceGroupRoleScheduler,
InstanceGroupRoleKubeControllerManager,
}

func (r InstanceGroupRole) HasControlPlane() bool {
Expand All @@ -105,16 +106,12 @@ func (r InstanceGroupRole) HasScheduler() bool {
return r == InstanceGroupRoleScheduler
}

func (r InstanceGroupRole) HasCloudControllerManager() bool {
return r == InstanceGroupRoleCloudControllerManager
}

func (r InstanceGroupRole) HasKubControllerManager() bool {
return r == InstanceGroupRoleKubControllerManager
func (r InstanceGroupRole) HasKubeControllerManager() bool {
return r == InstanceGroupRoleKubeControllerManager
}

func (r InstanceGroupRole) IsControlPlaneType() bool {
return r.HasControlPlane() || r.HasAPIServer()
return r.HasControlPlane() || r.HasAPIServer() || r.HasEtcd() || r.HasScheduler() || r.HasKubeControllerManager()
}

const (
Expand Down Expand Up @@ -408,10 +405,40 @@ func (g *InstanceGroup) IsAPIServerOnly() bool {
}

// hasAPIServer checks if instanceGroup runs an API Server
func (g *InstanceGroup) HasAPIServer() bool {
func (g *InstanceGroup) RunsAPIServer() bool {
return g.IsControlPlane() || g.IsAPIServerOnly()
}

// IsEtcdOnly checks if instanceGroup runs only Etcd
func (g *InstanceGroup) IsEtcdOnly() bool {
return g.Spec.Role.HasEtcd()
}

// HasEtcd checks if instanceGroup runs Etcd
func (g *InstanceGroup) RunsEtcd() bool {
return g.IsControlPlane() || g.IsEtcdOnly()
}

// IsSchedulerOnly checks if instanceGroup runs only Scheduler
func (g *InstanceGroup) IsSchedulerOnly() bool {
return g.Spec.Role.HasScheduler()
}

// HasScheduler checks if instanceGroup runs Scheduler
func (g *InstanceGroup) RunsScheduler() bool {
return g.IsControlPlane() || g.IsSchedulerOnly()
}

// IsKubeControllerManagerOnly checks if instanceGroup runs only KubeControllerManager
func (g *InstanceGroup) IsKubeControllerManagerOnly() bool {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I commented on the other PR (or it might be pending). It feels like calling IsKubeControllerManagerOnly would be an antipattern, we should be calling HasKubeControllerManager

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure I entirely agree. Maybe it should be private, assuming the packaging works. However I suspect we internally want to run checks on exactly which components are local. For instance we need to know IsKubeControllerManagerOnly && !IsAPIServerOnly to configure load balancing.

return g.Spec.Role.HasKubeControllerManager()
}

// RunsKubeControllerManager checks if instanceGroup runs KubeControllerManager
func (g *InstanceGroup) RunsKubeControllerManager() bool {
return g.IsControlPlane() || g.IsKubeControllerManagerOnly()
}

// HasGVisor checks if instanceGroup is a worker that has the gVisor (runsc) runtime enabled.
// gVisor is only valid on workers; ValidateInstanceGroup rejects it on other roles.
func (g *InstanceGroup) HasGVisor() bool {
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/kops/validation/instancegroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ func validateKarpenterInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster
if cluster.GetCloudProvider() != kops.CloudProviderAWS {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "manager"), "Karpenter InstanceGroups are only supported on AWS"))
}
if g.Spec.Role != kops.InstanceGroupRoleNode {
if !g.Spec.Role.HasNode() {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Karpenter InstanceGroups must have role Node"))
}
if g.Spec.MaxSize != nil && *g.Spec.MaxSize <= 0 {
Expand Down
8 changes: 4 additions & 4 deletions pkg/apis/nodeup/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
config.EnableLifecycleHook = true
}

if instanceGroup.HasAPIServer() {
if instanceGroup.RunsAPIServer() {
config.DisableSecurityGroupIngress = aws.DisableSecurityGroupIngress
config.ElbSecurityGroup = aws.ElbSecurityGroup
config.NLBSecurityGroupMode = aws.NLBSecurityGroupMode
Expand Down Expand Up @@ -365,7 +365,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
config.KubeletConfig = *instanceGroup.Spec.Kubelet
}

if instanceGroup.HasAPIServer() {
if instanceGroup.RunsAPIServer() {
config.APIServerConfig = &APIServerConfig{
ClusterDNSDomain: cluster.Spec.ClusterDNSDomain,
KubeAPIServer: cluster.Spec.KubeAPIServer,
Expand All @@ -391,14 +391,14 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
}
}

if instanceGroup.HasAPIServer() {
if instanceGroup.RunsAPIServer() {
config.ConfigStore = &kops.ConfigStoreSpec{
Keypairs: cluster.Spec.ConfigStore.Keypairs,
Secrets: cluster.Spec.ConfigStore.Secrets,
}
}

if instanceGroup.HasAPIServer() {
if instanceGroup.RunsAPIServer() {
config.Networking.EgressProxy = cluster.Spec.Networking.EgressProxy
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/model/awsmodel/autoscalinggroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ func (b *AutoscalingGroupModelBuilder) buildSecurityGroups(c *fi.CloudupModelBui

securityGroups := []*awstasks.SecurityGroup{sgLink}

if ig.HasAPIServer() &&
if ig.RunsAPIServer() &&
b.UseNetworkLoadBalancer() {
for _, id := range b.Cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups {
sgTask := &awstasks.SecurityGroup{
Expand Down Expand Up @@ -490,7 +490,7 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.CloudupMo
// hybrid (+SpotinstHybrid) instance groups.
if !featureflag.Spotinst.Enabled() ||
(featureflag.SpotinstHybrid.Enabled() && !HybridInstanceGroup(ig)) {
if b.UseLoadBalancerForAPI() && ig.HasAPIServer() {
if b.UseLoadBalancerForAPI() && ig.RunsAPIServer() {
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp"))
if b.Cluster.UsesLoadBalancerForKopsController() && ig.IsControlPlane() {
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("kops-controller"))
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/awsmodel/spotinst.go
Original file line number Diff line number Diff line change
Expand Up @@ -840,7 +840,7 @@ func (b *SpotInstanceGroupModelBuilder) buildLoadBalancers(c *fi.CloudupModelBui
var loadBalancers []*awstasks.ClassicLoadBalancer
var targetGroups []*awstasks.TargetGroup

if b.UseLoadBalancerForAPI() && ig.HasAPIServer() {
if b.UseLoadBalancerForAPI() && ig.RunsAPIServer() {
targetGroups = append(targetGroups, b.LinkToTargetGroup("tcp"))
if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
targetGroups = append(targetGroups, b.LinkToTargetGroup("tls"))
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/bootstrapscript.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ func KeypairNamesForInstanceGroup(cluster *kops.Cluster, ig *kops.InstanceGroup)
}
}

if ig.HasAPIServer() {
if ig.RunsAPIServer() {
keypairs = append(keypairs, "apiserver-aggregator-ca", "service-account", "etcd-clients-ca")
}

Expand Down
14 changes: 14 additions & 0 deletions pkg/model/components/channels/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"k8s.io/kops/pkg/k8scodecs"
"k8s.io/kops/pkg/kubemanifest"
"k8s.io/kops/pkg/model"
"k8s.io/kops/pkg/nodelabels"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/pkg/wellknownusers"
"k8s.io/kops/upup/pkg/fi"
Expand Down Expand Up @@ -121,11 +122,24 @@ func (b *ChannelsBuilder) buildPod(channels []string) (*v1.Pod, error) {
},
}

// Kops Clusters with ControlPlane nodes will run kops-channel/kops-controller there
// For those clusters we should label the cluster with the control plane label
// For Split Control Plane clusters we run kops-channel/kops-controller on the APIServer node.
// For those clusters we should label the cluster with the appropriate specific KOPS labels.
nodeLabel := nodelabels.RoleLabelKopsCCM + "," + nodelabels.RoleLabelKopsChannel + "," + nodelabels.RoleLabelKopsController
for _, ig := range b.AllInstanceGroups {
if ig.IsControlPlane() {
nodeLabel = nodelabels.RoleLabelControlPlane20
break
}
}

args := []string{
"apply", "channel",
"--v=4",
"--yes",
"--interval=" + channelsInterval.String(),
"--node-labels=" + nodeLabel,
"--node-name=$(NODE_NAME)",
}
args = append(args, channels...)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Contents: |
- --v=4
- --yes
- --interval=1m0s
- --node-labels=node-role.kubernetes.io/control-plane
- --node-name=$(NODE_NAME)
- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
env:
Expand Down
1 change: 1 addition & 0 deletions pkg/model/components/channels/tests/minimal/tasks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Contents: |
- --v=4
- --yes
- --interval=1m0s
- --node-labels=node-role.kubernetes.io/control-plane
- --node-name=$(NODE_NAME)
- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
env:
Expand Down
Loading
Loading