Sitelet https://github.com/kubernetes/kops/pull/18035/files
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/contributing/ports.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ See also pkg/wellknownports/wellknownports.go
| Port | Description |
| ---- | ---------------------------------------- |
| 22 | SSH |
| 443 | Kubernetes API |
| 179 | Calico |
| 443 | Kubernetes API |
| 2380 | etcd main peering |
| 2381 | etcd events peering |
| 2382 | etcd cilium peering |
| 3988 | kops controller serving port |
| 3989 | node local dns health check |
| 3990 | Kube API health check |
Expand All @@ -27,6 +28,7 @@ See also pkg/wellknownports/wellknownports.go
| 4000 | protokube gossip member list |
| 4001 | etcd main client |
| 4002 | etcd events client |
| 4003 | etcd cilium client |
| 4789 | VXLAN |
| 6942 | Cilium operator prometheus port |
| 9090 | Cilium prometheus port |
Expand Down
23 changes: 12 additions & 11 deletions pkg/model/awsmodel/firewall.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"

"k8s.io/klog/v2"
"k8s.io/kops/pkg/wellknownports"
)

type Protocol int
Expand Down Expand Up @@ -125,19 +126,19 @@ func (b *FirewallModelBuilder) applyNodeToMasterBlockSpecificPorts(c *fi.Cloudup
tcpBlocked := make(map[int]bool)

// Don't allow nodes to access etcd client port
tcpBlocked[4001] = true
tcpBlocked[4002] = true
tcpBlocked[wellknownports.EtcdMainClientPort] = true
tcpBlocked[wellknownports.EtcdEventsClientPort] = true

// Don't allow nodes to access etcd peer port
tcpBlocked[2380] = true
tcpBlocked[2381] = true
tcpBlocked[wellknownports.EtcdMainPeerPort] = true
tcpBlocked[wellknownports.EtcdEventsPeerPort] = true

udpRanges := []portRange{{From: 1, To: 65535}}
protocols := []Protocol{}

if b.Cluster.Spec.Networking.Cilium != nil && b.Cluster.Spec.Networking.Cilium.EtcdManaged {
// Block the etcd peer port
tcpBlocked[2382] = true
tcpBlocked[wellknownports.EtcdCiliumPeerPort] = true
}

if b.Cluster.Spec.Networking.Calico != nil {
Expand Down Expand Up @@ -311,13 +312,13 @@ func (b *AWSModelContext) GetSecurityGroups(role kops.InstanceGroupRole) ([]Secu
VPC: b.LinkToVPC(),
Description: fi.PtrTo("Security group for masters"),
RemoveExtraRules: []string{
"port=22", // SSH
"port=443", // k8s api
"port=2380", // etcd main peer
"port=2381", // etcd events peer
"port=22", // SSH
"port=443", // k8s api
"port=" + strconv.Itoa(wellknownports.EtcdMainPeerPort), // etcd main peer
"port=" + strconv.Itoa(wellknownports.EtcdEventsPeerPort), // etcd events peer
"port=3988", // kops-controller
"port=4001", // etcd main
"port=4002", // etcd events
"port=" + strconv.Itoa(wellknownports.EtcdMainClientPort), // etcd main
"port=" + strconv.Itoa(wellknownports.EtcdEventsClientPort), // etcd events
"port=4789", // VXLAN
"port=179", // Calico
"port=8443", // k8s api secondary listener
Expand Down
4 changes: 2 additions & 2 deletions pkg/model/azuremodel/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
SourceApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupNodes())},
SourcePortRange: fi.PtrTo("*"),
DestinationApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupControlPlane())},
DestinationPortRange: fi.PtrTo("2380-2381"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.EtcdMainPeerPort) + "-" + strconv.Itoa(wellknownports.EtcdEventsPeerPort)),
})
nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
Name: fi.PtrTo("DenyNodesToEtcd"),
Expand All @@ -196,7 +196,7 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
SourceApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupNodes())},
SourcePortRange: fi.PtrTo("*"),
DestinationApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupControlPlane())},
DestinationPortRange: fi.PtrTo("4000-4001"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.ProtokubeGossipMemberlist) + "-" + strconv.Itoa(wellknownports.EtcdMainClientPort)),
})
nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
Name: fi.PtrTo("AllowNodesToControlPlane"),
Expand Down
5 changes: 3 additions & 2 deletions pkg/model/components/apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/featureflag"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/loader"

Expand Down Expand Up @@ -146,14 +147,14 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(cluster *kops.Cluster) error
for _, etcdCluster := range clusterSpec.EtcdClusters {
switch etcdCluster.Name {
case "main":
c.EtcdServers = append(c.EtcdServers, "https://127.0.0.1:4001")
c.EtcdServers = append(c.EtcdServers, fmt.Sprintf("https://127.0.0.1:%d", wellknownports.EtcdMainClientPort))
case "events":
// Use HTTP for events etcd when EtcdEventsHTTP feature flag is enabled
scheme := "https"
if featureflag.EtcdEventsHTTP.Enabled() {
scheme = "http"
}
c.EtcdServersOverrides = append(c.EtcdServersOverrides, fmt.Sprintf("/events#%s://127.0.0.1:4002", scheme))
c.EtcdServersOverrides = append(c.EtcdServersOverrides, fmt.Sprintf("/events#%s://127.0.0.1:%d", scheme, wellknownports.EtcdEventsClientPort))
}
}

Expand Down
12 changes: 6 additions & 6 deletions pkg/model/components/etcdmanager/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -739,23 +739,23 @@ func PortsForCluster(etcdCluster kops.EtcdClusterSpec) (Ports, error) {
GRPCPort: wellknownports.EtcdMainGRPC,
// TODO: Use a socket file for the quarantine port
QuarantinedGRPCPort: wellknownports.EtcdMainQuarantinedClientPort,
ClientPort: 4001,
PeerPort: 2380,
ClientPort: wellknownports.EtcdMainClientPort,
PeerPort: wellknownports.EtcdMainPeerPort,
}, nil

case "events":
return Ports{
GRPCPort: wellknownports.EtcdEventsGRPC,
QuarantinedGRPCPort: wellknownports.EtcdEventsQuarantinedClientPort,
ClientPort: 4002,
PeerPort: 2381,
ClientPort: wellknownports.EtcdEventsClientPort,
PeerPort: wellknownports.EtcdEventsPeerPort,
}, nil
case "cilium":
return Ports{
GRPCPort: wellknownports.EtcdCiliumGRPC,
QuarantinedGRPCPort: wellknownports.EtcdCiliumQuarantinedClientPort,
ClientPort: 4003,
PeerPort: 2382,
ClientPort: wellknownports.EtcdCiliumClientPort,
PeerPort: wellknownports.EtcdCiliumPeerPort,
}, nil

default:
Expand Down
12 changes: 6 additions & 6 deletions pkg/model/openstackmodel/firewall.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,16 +158,16 @@ func (b *FirewallModelBuilder) addETCDRules(c *fi.CloudupModelBuilderContext, sg
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(IPV4),
PortRangeMin: i(4001),
PortRangeMax: i(4002),
PortRangeMin: i(wellknownports.EtcdMainClientPort),
PortRangeMax: i(wellknownports.EtcdEventsClientPort),
}
etcdPeerRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(IPV4),
PortRangeMin: i(2380),
PortRangeMax: i(2381),
PortRangeMin: i(wellknownports.EtcdMainPeerPort),
PortRangeMax: i(wellknownports.EtcdEventsPeerPort),
}
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdRule)
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdPeerRule)
Expand All @@ -178,8 +178,8 @@ func (b *FirewallModelBuilder) addETCDRules(c *fi.CloudupModelBuilderContext, sg
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(IPV4),
PortRangeMin: i(2382),
PortRangeMax: i(2382),
PortRangeMin: i(wellknownports.EtcdCiliumPeerPort),
PortRangeMax: i(wellknownports.EtcdCiliumPeerPort),
}
etcdCiliumGRPCRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Expand Down
13 changes: 11 additions & 2 deletions pkg/wellknownports/wellknownports.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,20 @@ const (
// ProtokubeGossipMemberlist is the port where protokube listens for the memberlist-backed gossip
ProtokubeGossipMemberlist = 4000

// 4001 is etcd main, 4002 is etcd events

// EtcdMainClientPort is the client port for the main etcd cluster
EtcdMainClientPort = 4001
// EtcdEventsClientPort is the client port for the events etcd cluster
EtcdEventsClientPort = 4002
// EtcdCiliumClientPort is the port were the Cilium etcd cluster listens
EtcdCiliumClientPort = 4003

// EtcdMainPeerPort is the peer port for the main etcd cluster
EtcdMainPeerPort = 2380
// EtcdEventsPeerPort is the peer port for the events etcd cluster
EtcdEventsPeerPort = 2381
// EtcdCiliumPeerPort is the peer port for the cilium etcd cluster
EtcdCiliumPeerPort = 2382

// CiliumOperatorPrometheusPort is the port the Cilium Operator exposes metrics
CiliumPrometheusOperatorPort = 6942

Expand Down
Loading