Sitelet https://github.com/kubernetes/kops/commit/36052d500f7c5f950f187974581c178a14c04c58
Skip to content

Commit 36052d5

Browse files
JefftreeJefftree
committed
dump: add --node-dump-timeout flag for per-node dump timeout
1 parent 1782152 commit 36052d5

5 files changed

Lines changed: 38 additions & 18 deletions

File tree

‎cmd/kops/toolbox_dump.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"os"
2525
"path/filepath"
2626
"strings"
27+
"time"
2728

2829
"github.com/spf13/cobra"
2930
"golang.org/x/crypto/ssh"
@@ -62,11 +63,12 @@ type ToolboxDumpOptions struct {
6263

6364
ClusterName string
6465

65-
Dir string
66-
PrivateKey string
67-
SSHUser string
68-
MaxNodes int
69-
K8sResources bool
66+
Dir string
67+
PrivateKey string
68+
SSHUser string
69+
MaxNodes int
70+
NodeDumpTimeout time.Duration
71+
K8sResources bool
7072

7173
// CloudResources controls whether we dump the cloud resources
7274
CloudResources bool
@@ -77,6 +79,7 @@ func (o *ToolboxDumpOptions) InitDefaults() {
7779
o.PrivateKey = "~/.ssh/id_rsa"
7880
o.SSHUser = "ubuntu"
7981
o.MaxNodes = 500
82+
o.NodeDumpTimeout = time.Minute
8083
o.K8sResources = k8sResources != ""
8184
o.CloudResources = true
8285
}
@@ -107,6 +110,7 @@ func NewCmdToolboxDump(f commandutils.Factory, out io.Writer) *cobra.Command {
107110
cmd.Flags().BoolVar(&options.K8sResources, "k8s-resources", options.K8sResources, "Include k8s resources in the dump")
108111
cmd.Flags().BoolVar(&options.CloudResources, "cloud-resources", options.CloudResources, "Include cloud resources in the dump")
109112
cmd.Flags().IntVar(&options.MaxNodes, "max-nodes", options.MaxNodes, "The maximum number of nodes from which to dump logs")
113+
cmd.Flags().DurationVar(&options.NodeDumpTimeout, "node-dump-timeout", options.NodeDumpTimeout, "Timeout for connecting to and dumping logs from a single node")
110114
cmd.Flags().StringVar(&options.PrivateKey, "private-key", options.PrivateKey, "File containing private key to use for SSH access to instances")
111115
cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "The remote user for SSH access to instances")
112116
cmd.RegisterFlagCompletionFunc("ssh-user", cobra.NoFileCompletions)
@@ -241,7 +245,7 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
241245
}
242246
}
243247

244-
dumper := dump.NewLogDumper(bastionAddress, sshConfig, keyRing, options.Dir)
248+
dumper := dump.NewLogDumper(bastionAddress, sshConfig, keyRing, options.Dir, options.NodeDumpTimeout)
245249

246250
if err := dumper.DumpAllNodes(ctx, nodes, options.MaxNodes, cloudResources); err != nil {
247251
klog.Warningf("error dumping nodes: %v", err)

‎docs/cli/kops_toolbox_dump.md‎

Lines changed: 9 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/dump/dumper.go‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,16 @@ type logDumper struct {
4040

4141
artifactsDir string
4242

43+
// nodeDumpTimeout bounds the time spent connecting to and dumping a single node.
44+
nodeDumpTimeout time.Duration
45+
4346
services []string
4447
files []string
4548
podSelectors []string
4649
}
4750

4851
// NewLogDumper is the constructor for a logDumper
49-
func NewLogDumper(bastionAddress string, sshConfig *ssh.ClientConfig, keyRing agent.Agent, artifactsDir string) *logDumper {
52+
func NewLogDumper(bastionAddress string, sshConfig *ssh.ClientConfig, keyRing agent.Agent, artifactsDir string, nodeDumpTimeout time.Duration) *logDumper {
5053
sshClientFactory := &sshClientFactoryImplementation{
5154
keyRing: keyRing,
5255
sshConfig: sshConfig,
@@ -56,9 +59,14 @@ func NewLogDumper(bastionAddress string, sshConfig *ssh.ClientConfig, keyRing ag
5659
sshClientFactory.bastion = bastionAddress
5760
}
5861

62+
if nodeDumpTimeout <= 0 {
63+
nodeDumpTimeout = defaultNodeDumpTimeout
64+
}
65+
5966
d := &logDumper{
6067
sshClientFactory: sshClientFactory,
6168
artifactsDir: artifactsDir,
69+
nodeDumpTimeout: nodeDumpTimeout,
6270
}
6371

6472
d.services = []string{
@@ -245,10 +253,13 @@ func (d *logDumper) dumpNotRegistered(ctx context.Context, node *resources.Insta
245253
return "", fmt.Errorf("no known addresses for node %s", node.Name)
246254
}
247255

248-
// nodeDumpTimeout bounds the time spent connecting to and dumping a single node.
256+
// defaultNodeDumpTimeout bounds the time spent connecting to and dumping a single node.
249257
// A healthy node dumps in well under a minute. Without this cap, an SSH operation
250258
// against an unreachable node blocks until the OS TCP timeout (~15 min) expires.
251-
const nodeDumpTimeout = time.Minute
259+
// Large clusters dump multi-GB logs per node and need a higher value, configurable
260+
// via the --node-dump-timeout flag, because the files are dumped sequentially and a
261+
// single oversized log can otherwise exhaust the budget before the rest are read.
262+
const defaultNodeDumpTimeout = time.Minute
252263

253264
// DumpNode connects to a node and dumps the logs.
254265
func (d *logDumper) dumpNode(ctx context.Context, name string, ip string, useBastion bool) error {
@@ -258,7 +269,7 @@ func (d *logDumper) dumpNode(ctx context.Context, name string, ip string, useBas
258269

259270
klog.Infof("Dumping node %s", name)
260271

261-
ctx, cancel := context.WithTimeout(ctx, nodeDumpTimeout)
272+
ctx, cancel := context.WithTimeout(ctx, d.nodeDumpTimeout)
262273
defer cancel()
263274

264275
n, err := d.connectToNode(ctx, name, ip, useBastion)

‎tests/e2e/kubetest2-kops/deployer/deployer.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ type deployer struct {
7070
ValidationCount int `flag:"validation-count" desc:"how many times should a validation pass"`
7171
ValidationInterval time.Duration `flag:"validation-interval" desc:"time in duration to wait between validation attempts"`
7272
MaxNodesToDump string `flag:"max-nodes-to-dump" desc:"max number of nodes to dump logs from, helpful to set when running scale tests"`
73+
NodeDumpTimeout time.Duration `flag:"node-dump-timeout" desc:"timeout for connecting to and dumping logs from a single node, helpful to raise when running scale tests"`
7374

7475
TemplatePath string `flag:"template-path" desc:"The path to the manifest template used for cluster creation"`
7576

‎tests/e2e/kubetest2-kops/deployer/dumplogs.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ func (d *deployer) DumpClusterLogs() error {
4747
if d.MaxNodesToDump != "" {
4848
args = append(args, "--max-nodes", d.MaxNodesToDump)
4949
}
50+
if d.NodeDumpTimeout > 0 {
51+
args = append(args, "--node-dump-timeout", d.NodeDumpTimeout.String())
52+
}
5053
klog.Info(strings.Join(args, " "))
5154
cmd := exec.Command(args[0], args[1:]...)
5255
cmd.SetEnv(append(d.env(), "KOPS_TOOLBOX_DUMP_K8S_RESOURCES=1")...)

0 commit comments

Comments
 (0)