Sitelet https://github.com/kubernetes/kops/pull/18699/files
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 89 additions & 11 deletions tests/e2e/kubetest2-kops/deployer/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,14 @@ import (

"github.com/blang/semver/v4"
"k8s.io/klog/v2"
"k8s.io/kops/pkg/resources"
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
"k8s.io/kops/tests/e2e/pkg/target"
"k8s.io/kops/tests/e2e/pkg/util"
"sigs.k8s.io/kubetest2/pkg/boskos"
"sigs.k8s.io/kubetest2/pkg/exec"
"sigs.k8s.io/yaml"
)

func (d *deployer) init() error {
Expand Down Expand Up @@ -146,17 +149,7 @@ func (d *deployer) initialize() error {
}
d.terraform = t
}
if d.commonOptions.ShouldTest() {
for _, envvar := range d.env() {
// Set all of the env vars we use for kops in the current process
// so that the tester inherits them when shelling out to kops
if i := strings.Index(envvar, "="); i != -1 {
os.Setenv(envvar[0:i], envvar[i+1:])
} else {
os.Setenv(envvar, "")
}
}
}
d.exportEnvForTester()
return nil
}

Expand Down Expand Up @@ -227,6 +220,91 @@ func (d *deployer) resolveSSHKeys() error {
return nil
}

// resolveSSHUserFromCluster fills in SSHUser by asking kops which user it registered the SSH key
// for, which is only answerable once the cluster's instances exist.
//
// It is deliberately a last resort, after the --ssh-user flag, the users the deployer assigns
// itself (azure, digitalocean) and KUBE_SSH_USER. A job that sets KUBE_SSH_USER keeps exactly the
// user it has today, so this can be adopted one job at a time by dropping that variable.
//
// Failure is not fatal: the user simply stays empty, and callers omit --ssh-user so that kops
// applies its own default rather than an unusable empty value.
func (d *deployer) resolveSSHUserFromCluster() {
if d.SSHUser != "" {
return
}

args := []string{
d.KopsBinaryPath, "toolbox", "dump",
"--name", d.ClusterName,
"-o", "yaml",
}
klog.Info(strings.Join(args, " "))

// Without --dir this only lists cloud resources; it does not SSH anywhere, so it does not
// need the credentials we are trying to determine.
cmd := exec.Command(args[0], args[1:]...)
cmd.SetEnv(d.env()...)
cmd.SetStderr(os.Stderr)
output, err := exec.Output(cmd)
if err != nil {
klog.Warningf("failed to determine the SSH user from the cluster: %v", err)
return
}

var dump resources.Dump
if err := yaml.Unmarshal(output, &dump); err != nil {
klog.Warningf("failed to parse the cluster dump while determining the SSH user: %v", err)
return
}

sshUser := sshUserFromDump(&dump)
if sshUser == "" {
// Older kops releases do not report sshUser for every cloud; GCE gained it in 1.37.
klog.Warningf("cluster dump reported no SSH user; kops will fall back to its own default")
return
}

d.SSHUser = sshUser
klog.V(1).Infof("Determined SSH user from the cluster: [%s]", d.SSHUser)
}

// sshUserFromDump picks the SSH user to use for the cluster, preferring a control plane instance
// because that is the one host every dump path needs to reach.
func sshUserFromDump(dump *resources.Dump) string {
fallback := ""
for _, instance := range dump.Instances {
if instance.SSHUser == "" {
continue
}
for _, role := range instance.Roles {
if role == "control-plane" {
return instance.SSHUser
}
}
if fallback == "" {
fallback = instance.SSHUser
}
}
return fallback
}

// exportEnvForTester sets the env vars we pass to kops in the current process, so that the tester
// inherits them when it shells out. It is called once during initialize() and again once the
// cluster is up, because values such as the SSH user are not knowable before then.
func (d *deployer) exportEnvForTester() {
if !d.commonOptions.ShouldTest() {
return
}
for _, envvar := range d.env() {
if k, v, ok := strings.Cut(envvar, "="); ok {
os.Setenv(k, v)
} else {
os.Setenv(envvar, "")
}
}
}

// verifyKopsFlags ensures common fields are set for kops commands
func (d *deployer) verifyKopsFlags() error {
if d.ClusterName == "" {
Expand Down
77 changes: 77 additions & 0 deletions tests/e2e/kubetest2-kops/deployer/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"strings"
"testing"

"k8s.io/kops/pkg/resources"
"k8s.io/kops/tests/e2e/kubetest2-kops/builder"
)

Expand Down Expand Up @@ -298,3 +299,79 @@ func TestEnvExportsSSHKeyAndUser(t *testing.T) {
})
}
}

// The dump reports a user per instance. Control plane hosts are what every dump path has to
// reach, so they win over workers.
func TestSSHUserFromDump(t *testing.T) {
cases := []struct {
name string
instances []*resources.Instance
expected string
}{
{
name: "prefers a control plane instance",
instances: []*resources.Instance{
{Roles: []string{"node"}, SSHUser: "worker-user"},
{Roles: []string{"control-plane"}, SSHUser: "ubuntu"},
},
expected: "ubuntu",
},
{
name: "falls back to any instance when no control plane is reported",
instances: []*resources.Instance{
{Roles: []string{"node"}, SSHUser: "admin"},
},
expected: "admin",
},
{
name: "ignores instances with no user",
instances: []*resources.Instance{
{Roles: []string{"control-plane"}},
{Roles: []string{"node"}, SSHUser: "rocky"},
},
expected: "rocky",
},
{
// Older kops releases do not populate sshUser on every cloud.
name: "no user anywhere",
instances: []*resources.Instance{
{Roles: []string{"control-plane"}},
},
expected: "",
},
{
name: "no instances at all",
instances: nil,
expected: "",
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if actual := sshUserFromDump(&resources.Dump{Instances: tc.instances}); actual != tc.expected {
t.Errorf("sshUserFromDump() = %q, expected %q", actual, tc.expected)
}
})
}
}

// Discovery is a last resort. Anything that already set a user must survive untouched, which is
// what lets jobs opt in one at a time by dropping KUBE_SSH_USER.
func TestResolveSSHUserFromClusterKeepsExistingUser(t *testing.T) {
for _, user := range []string{"prow", "ec2-user", "kops", "root"} {
t.Run(user, func(t *testing.T) {
// KopsBinaryPath is deliberately bogus: if the deployer tried to shell out we would
// see it fail rather than silently keep the value.
d := &deployer{
CloudProvider: "gce",
ClusterName: "test.k8s.local",
SSHUser: user,
KopsBinaryPath: "/nonexistent/kops",
}
d.resolveSSHUserFromCluster()
if d.SSHUser != user {
t.Errorf("SSHUser = %q, expected it to stay %q", d.SSHUser, user)
}
})
}
}
12 changes: 10 additions & 2 deletions tests/e2e/kubetest2-kops/deployer/dumplogs.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,18 @@ func (d *deployer) DumpClusterLogs() error {
}
defer yamlFile.Close()

// Up() may not have run in this process, or may have failed before it could work this out.
d.resolveSSHUserFromCluster()

args := []string{
d.KopsBinaryPath, "toolbox", "dump",
"--name", d.ClusterName,
"--dir", d.ArtifactsDir,
"--private-key", d.SSHPrivateKeyPath,
"--ssh-user", d.SSHUser,
}
// Passing an empty --ssh-user would override the kops default with an unusable value.
if d.SSHUser != "" {
args = append(args, "--ssh-user", d.SSHUser)
}

if d.MaxNodesToDump != "" {
Expand Down Expand Up @@ -223,9 +229,11 @@ func (d *deployer) dumpClusterInfoSSH() error {
d.KopsBinaryPath, "toolbox", "dump",
"--name", d.ClusterName,
"--private-key", d.SSHPrivateKeyPath,
"--ssh-user", d.SSHUser,
"-o", "yaml",
}
if d.SSHUser != "" {
toolboxDumpArgs = append(toolboxDumpArgs, "--ssh-user", d.SSHUser)
}
klog.Info(strings.Join(toolboxDumpArgs, " "))

cmd := exec.Command(toolboxDumpArgs[0], toolboxDumpArgs[1:]...)
Expand Down
8 changes: 8 additions & 0 deletions tests/e2e/kubetest2-kops/deployer/up.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,14 @@ func (d *deployer) Up() error {

time.Sleep(10 * time.Second)

// The instances exist by this point, so kops can tell us which user it registered the SSH key
// for. Done before the validation below so that a cluster which fails to validate is still
// reachable for log collection. Re-export afterwards because the tester's environment was
// built during init(), before any of this was knowable.
d.resolveSSHUserFromCluster()
klog.V(1).Infof("Using SSH user: [%s]", d.SSHUser)
d.exportEnvForTester()

isUp, err := d.IsUp()
if err != nil {
return err
Expand Down
Loading