Sitelet https://github.com/kubernetes/kops/pull/18666/files
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion docs/operations/asset-repository.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,20 @@ spec:
fileRepository: s3://example-bucket/files
```

{{ kops_feature_table(kops_added_default='1.37') }}

On Azure, the repository can also be an `azureblob://<account>/<container>/<prefix>` URL.
Nodes then read it with their system-assigned managed identity, which allows the storage
account to be private. The managed identities of the instance groups have to be granted
`Storage Blob Data Reader` on the assets container. Do not grant access to the state-store
storage account, as that would let nodes read the cluster PKI.

```yaml
spec:
assets:
fileRepository: azureblob://exampleaccount/assets/files
```

## Copying assets into repositories

{{ kops_feature_table(kops_added_default='1.22') }}
Expand All @@ -80,9 +94,11 @@ You can copy assets into their repositories either by running `kops get assets -
When running `kops get assets --copy`, kOps copies assets into their respective repositories if
they do not already exist there.

For file assets, kOps only supports copying to a repository that is either an S3 or GCS bucket.
For file assets, kOps only supports copying to a repository that is an S3 bucket, a GCS bucket,
or an Azure Blob Storage container.
An S3 bucket must be configured with a prefix of `s3://` or using the [regional naming conventions of S3](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region).
A GCS bucket must be configured with a prefix of `https://storage.googleapis.com/` or `gs://`.
An Azure Blob Storage container must be configured with a prefix of `azureblob://`.

## Listing assets

Expand Down
2 changes: 2 additions & 0 deletions docs/releases/1.37-NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ As part of this removal, the `protokube` component, whose only remaining respons

* Private cluster asset repositories now support AWS `s3://` URLs in addition to existing GCE `gs://` URLs, both for `KOPS_BASE_URL` (nodeup download) and `spec.assets.fileRepository`. Nodes authenticate with their instance credentials; see the [asset repository documentation](https://kops.sigs.k8s.io/operations/asset-repository/) for the required permissions. The AWS nodeup download requires node images with curl 8.0 or newer.

* Private cluster asset repositories now also support Azure `azureblob://<account>/<container>/<prefix>` URLs, both for `KOPS_BASE_URL` (nodeup download) and `spec.assets.fileRepository`. Nodes authenticate with their system-assigned managed identity, which has to be granted the `Storage Blob Data Reader` role on the assets container; see the [asset repository documentation](https://kops.sigs.k8s.io/operations/asset-repository/) for the details and warnings.

# Breaking changes

* Support for AWS Classic Load Balancer (CLB) for the API has been removed. Clusters with `spec.api.loadBalancer.class: Classic` (or with no explicit `class`, which previously defaulted to Classic) fail validation, and the long-deprecated `kops create cluster --api-loadbalancer-class` flag has been removed. Existing clusters using a CLB must migrate to a Network Load Balancer (NLB) using kOps 1.36 or earlier before upgrading to kOps 1.37, following the [CLB to NLB migration guide](https://github.com/kubernetes/kops/blob/master/permalinks/acm_nlb.md). Attaching instance groups to externally-managed Classic Load Balancers via `spec.externalLoadBalancers[].loadBalancerName` remains supported.
Expand Down
11 changes: 10 additions & 1 deletion pkg/apis/kops/validation/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -806,8 +806,17 @@ func validateFileRepository(s string, fieldPath *field.Path, cloudProvider kops.
if cloudProvider != kops.CloudProviderAWS {
allErrs = append(allErrs, field.Invalid(fieldPath, s, fmt.Sprintf("s3:// fileRepository is only supported on AWS, but the cloud provider is %q", cloudProvider)))
}
case "azureblob":
// Only Azure instances can authenticate to Azure Blob Storage with their managed identity.
if cloudProvider != kops.CloudProviderAzure {
allErrs = append(allErrs, field.Invalid(fieldPath, s, fmt.Sprintf("azureblob:// fileRepository is only supported on Azure, but the cloud provider is %q", cloudProvider)))
}
// Without a container, each remapped asset would treat its first path segment as the container.
if container, _, _ := strings.Cut(strings.TrimPrefix(u.Path, "/"), "/"); container == "" {
allErrs = append(allErrs, field.Invalid(fieldPath, s, "azureblob:// fileRepository must include a container: azureblob://<account>/<container>/<path>"))
}
default:
allErrs = append(allErrs, field.Invalid(fieldPath, s, "fileRepository must be an http://, https://, gs://, or s3:// URL"))
allErrs = append(allErrs, field.Invalid(fieldPath, s, "fileRepository must be an http://, https://, gs://, s3://, or azureblob:// URL"))
}
if u.Host == "" {
allErrs = append(allErrs, field.Invalid(fieldPath, s, "fileRepository must include a host"))
Expand Down
19 changes: 19 additions & 0 deletions pkg/apis/kops/validation/validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2305,6 +2305,25 @@ func TestValidateFileRepository(t *testing.T) {
Input: "s3://example-k8s-assets/kops",
ExpectedErrors: []string{"Invalid value::spec.assets.fileRepository"},
},
{
Input: "azureblob://exampleaccount/assets/kops",
CloudProvider: kops.CloudProviderAzure,
},
{
Input: "azureblob://exampleaccount/assets/kops",
CloudProvider: kops.CloudProviderGCE,
ExpectedErrors: []string{"Invalid value::spec.assets.fileRepository"},
},
{
Input: "azureblob://exampleaccount/assets/kops",
ExpectedErrors: []string{"Invalid value::spec.assets.fileRepository"},
},
{
// A container is required so that remapped assets share one container.
Input: "azureblob://exampleaccount",
CloudProvider: kops.CloudProviderAzure,
ExpectedErrors: []string{"Invalid value::spec.assets.fileRepository"},
},
{
// Nodes download from GCS with the credentials of their service account.
Input: "gs://example-k8s-assets/kops",
Expand Down
4 changes: 2 additions & 2 deletions pkg/assets/assetcopy/copyfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,11 +176,11 @@ func writeFile(ctx context.Context, cluster *kops.Cluster, p vfs.Path, data []by
return nil
}

// buildVFSPath returns local paths and memfs://, file://, gs://, and s3:// URLs unchanged.
// buildVFSPath returns local paths and memfs://, file://, gs://, s3://, and azureblob:// URLs unchanged.
// It converts recognized S3 or GCS HTTPS URLs to their native VFS form.
func buildVFSPath(target string) (string, error) {
if !strings.Contains(target, "://") || strings.HasPrefix(target, "memfs://") || strings.HasPrefix(target, "file://") ||
strings.HasPrefix(target, "gs://") || strings.HasPrefix(target, "s3://") {
strings.HasPrefix(target, "gs://") || strings.HasPrefix(target, "s3://") || strings.HasPrefix(target, "azureblob://") {
return target, nil
}

Expand Down
5 changes: 5 additions & 0 deletions pkg/assets/assetcopy/copyfile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,11 @@ func Test_BuildVFSPath(t *testing.T) {
"s3://k8s-for-greeks-kops/kubernetes-release/release/v1.7.2/bin/linux/amd64/kubectl",
true,
},
{
"azureblob://exampleaccount/assets/kubernetes-release/release/v1.7.2/bin/linux/amd64/kubectl",
"azureblob://exampleaccount/assets/kubernetes-release/release/v1.7.2/bin/linux/amd64/kubectl",
true,
},
{
"https://foo/k8s-for-greeks-kops/kubernetes-release/release/v1.7.2/bin/linux/amd64/kubectl",
"",
Expand Down
83 changes: 73 additions & 10 deletions pkg/model/resources/nodeup.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ imds-get() {
curl -s -f --noproxy '*' --connect-timeout 2 --max-time 5 \
-H "X-aws-ec2-metadata-token: $1" "http://169.254.169.254/latest/$2"
}
{{- end }}

{{- if or UseGCSDownload UseS3Download UseBlobDownload }}

# Extract a string field from a JSON object. args: json, field
json-field() {
Expand Down Expand Up @@ -107,13 +110,14 @@ download-or-bust() {
while true; do
for url in "${urls[@]}"; do
{{- if UseGCSDownload }}
# Use the IP of the metadata server, to not depend on DNS this early in boot
local metadata_server="http://169.254.169.254"
local token
local response token
echo "== Downloading ${url} =="
# Pipe the service account token to curl, to keep it out of the logs
if ! token=$(curl -s -f --noproxy '*' --connect-timeout 2 --max-time 5 -H 'Metadata-Flavor: Google' "${metadata_server}/computeMetadata/v1/instance/service-accounts/default/token" | grep -o '"access_token" *: *"[^"]*"' | cut -d '"' -f 4); then
# Use the IP of the metadata server, to not depend on DNS this early in boot
if ! response=$(curl -s -f --noproxy '*' --connect-timeout 2 --max-time 5 -H 'Metadata-Flavor: Google' "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"); then
echo "== Failed to get a service account token =="
elif ! token=$(json-field "${response}" access_token); then
echo "== Failed to parse the service account token =="
# Pass the token through stdin so it does not appear in files, logs, or process arguments.
elif ! echo "Authorization: Bearer ${token}" | curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 -H @- "https://storage.googleapis.com/${url#gs://}"; then
echo "== Failed to download ${url} =="
elif ! validate-hash "${file}" "${hash}"; then
Expand All @@ -123,6 +127,30 @@ download-or-bust() {
echo "== Downloaded ${url} with hash ${hash} =="
return 0
fi
{{- else if UseBlobDownload }}
local rest account response token
echo "== Downloading ${url} =="
rest="${url#azureblob://}"
account="${rest%%/*}"
rest="${rest#*/}"
# Use the IP of the metadata server, to not depend on DNS this early in boot.
# The token request is brokered to Entra ID, so allow more time than for other clouds.
if ! response=$(curl -s -f --noproxy '*' --connect-timeout 5 --max-time 30 -H 'Metadata: true' "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fstorage.azure.com%2F"); then
echo "== Failed to get a managed identity token =="
elif ! token=$(json-field "${response}" access_token); then
echo "== Failed to parse the managed identity token =="
# Pass the token through stdin so it does not appear in files, logs, or process arguments.
elif ! printf 'Authorization: Bearer %s\nx-ms-version: 2017-11-09\n' "${token}" |
curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 -H @- \
"https://${account}.blob.core.windows.net/${rest}"; then
echo "== Failed to download ${url} =="
elif ! validate-hash "${file}" "${hash}"; then
echo "== Failed to validate hash for ${url} =="
rm -f "${file}"
else
echo "== Downloaded ${url} with hash ${hash} =="
return 0
fi
{{- else if UseS3Download }}
local imds_token profile creds access_key secret_key session_token
echo "== Downloading ${url} =="
Expand Down Expand Up @@ -267,10 +295,16 @@ func (b *NodeUpScript) nodeUpSource(arch architectures.Architecture) (string, er

locations := slices.Clone(asset.Locations)
for i, location := range locations {
if !strings.HasPrefix(location, "s3://") {
var escape func(string) (string, error)
switch {
case strings.HasPrefix(location, "s3://"):
escape = escapeS3Location
case strings.HasPrefix(location, "azureblob://"):
escape = escapeBlobLocation
default:
continue
}
escaped, err := escapeS3Location(location)
escaped, err := escape(location)
if err != nil {
return "", fmt.Errorf("escaping nodeup source %q: %w", location, err)
}
Expand All @@ -291,6 +325,21 @@ func escapeS3Location(location string) (string, error) {
return "s3://" + u.Host + httpbinding.EscapePath(u.Path, false), nil
}

func escapeBlobLocation(location string) (string, error) {
u, err := url.Parse(location)
if err != nil {
return "", fmt.Errorf("parsing Azure Blob location: %w", err)
}
container, key, _ := strings.Cut(strings.TrimPrefix(u.Path, "/"), "/")
// Reject ports, IPv6 hosts, userinfo, queries, and fragments, which the account-based
// blob.core.windows.net URL cannot represent, so they fail here instead of in the boot retry loop.
if u.Scheme != "azureblob" || u.Host == "" || u.Hostname() != u.Host || u.User != nil || u.RawQuery != "" || u.Fragment != "" || container == "" || key == "" {
return "", fmt.Errorf("invalid Azure Blob location; expected azureblob://<account>/<container>/<key>")
}

return "azureblob://" + u.Host + httpbinding.EscapePath(u.Path, false), nil
}

func (b *NodeUpScript) Build() (fi.Resource, error) {
if b.ProxyEnv == nil {
b.ProxyEnv = funcEmptyString
Expand All @@ -303,6 +352,14 @@ func (b *NodeUpScript) Build() (fi.Resource, error) {
return nil, fmt.Errorf("ResolveS3Region must be called before building a nodeup script with an s3:// source")
}

if b.useBlobDownload() {
// The script hard-codes the public cloud blob.core.windows.net endpoint suffix.
// Azure environment names are case-insensitive; AzureCloud is the CLI name of the public cloud.
if azureEnv := os.Getenv("AZURE_ENVIRONMENT"); azureEnv != "" && !strings.EqualFold(azureEnv, "AzurePublicCloud") && !strings.EqualFold(azureEnv, "AzureCloud") {
return nil, fmt.Errorf("downloading nodeup from an azureblob:// URL is not supported in Azure environment %q", azureEnv)
}
}

functions := template.FuncMap{
"NodeUpSourceAmd64": func() (string, error) {
return b.nodeUpSource(architectures.ArchitectureAmd64)
Expand Down Expand Up @@ -349,9 +406,10 @@ func (b *NodeUpScript) Build() (fi.Resource, error) {
"ProxyEnv": b.ProxyEnv,
"EnvironmentVariables": b.EnvironmentVariables,

"UseGCSDownload": b.useGCSDownload,
"UseS3Download": b.useS3Download,
"S3Region": func() string { return b.S3Region },
"UseGCSDownload": b.useGCSDownload,
"UseS3Download": b.useS3Download,
"UseBlobDownload": b.useBlobDownload,
"S3Region": func() string { return b.S3Region },
}

return newTemplateResource("nodeup", nodeUpTemplate, functions, nil)
Expand Down Expand Up @@ -384,6 +442,11 @@ func (b *NodeUpScript) useS3Download() bool {
return b.CloudProvider == string(kops.CloudProviderAWS) && b.firstLocationWithScheme("s3://") != ""
}

// Azure Blob downloads require a managed identity on the instance.
func (b *NodeUpScript) useBlobDownload() bool {
return b.CloudProvider == string(kops.CloudProviderAzure) && b.firstLocationWithScheme("azureblob://") != ""
}

// ResolveS3Region resolves the bucket region because SigV4 requires it but s3:// URLs omit it.
func (b *NodeUpScript) ResolveS3Region(ctx context.Context, vfsContext *vfs.VFSContext) error {
if b.CloudProvider != string(kops.CloudProviderAWS) {
Expand Down
Loading
Loading