Sitelet https://github.com/kubernetes/kops/pull/18260/files
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ UPLOAD_CMD=$(KOPS_ROOT)/hack/upload ${UPLOAD_ARGS}
unexport AWS_ACCESS_KEY_ID AWS_REGION AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN CNI_VERSION_URL DNS_IGNORE_NS_CHECK DNSCONTROLLER_IMAGE DO_ACCESS_TOKEN GOOGLE_APPLICATION_CREDENTIALS
unexport KOPS_BASE_URL KOPS_CLUSTER_NAME KOPS_RUN_OBSOLETE_VERSION KOPS_STATE_STORE KOPS_STATE_S3_ACL KUBE_API_VERSIONS NODEUP_URL OPENSTACK_CREDENTIAL_FILE SKIP_PACKAGE_UPDATE
unexport SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCESS_KEY HCLOUD_TOKEN SCW_ACCESS_KEY SCW_SECRET_KEY SCW_DEFAULT_PROJECT_ID SCW_PROFILE
unexport AZURE_CLIENT_ID AZURE_CLIENT_SECRET AZURE_STORAGE_ACCOUNT AZURE_SUBSCRIPTION_ID AZURE_TENANT_ID
unexport AZURE_CLIENT_ID AZURE_CLIENT_SECRET AZURE_SUBSCRIPTION_ID AZURE_TENANT_ID


VERSION=$(shell tools/get_version.sh | grep VERSION | awk '{print $$2}')
Expand Down
1 change: 0 additions & 1 deletion cmd/kops/create_cluster_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ func TestCreateClusterGossipAWS(t *testing.T) {

// TestCreateClusterGossipAzure creates a minimal Azure gossip cluster
func TestCreateClusterGossipAzure(t *testing.T) {
t.Setenv("AZURE_STORAGE_ACCOUNT", "teststorage")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/gossip-azure", "v1alpha2")
}

Expand Down
1 change: 0 additions & 1 deletion cmd/kops/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1792,7 +1792,6 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
}

func (i *integrationTest) runTestTerraformAzure(t *testing.T) {
t.Setenv("AZURE_STORAGE_ACCOUNT", "teststorage")
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")

featureflag.ParseFlags("+Azure,+AzureTerraform")
Expand Down
11 changes: 9 additions & 2 deletions docs/getting_started/azure.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,12 @@ export KOPS_FEATURE_FLAGS="Azure"

```bash
export AZURE_SUBSCRIPTION_ID=<subscription-id>
export AZURE_STORAGE_ACCOUNT=<storage-account-name>
```

### kOps-specific

```bash
export KOPS_STATE_STORE=azureblob://<container-name>
export KOPS_STATE_STORE=azureblob://<storage-account-name>/<container-name>
```

## Creating a Single Master Cluster
Expand Down Expand Up @@ -90,3 +89,11 @@ kOps for Azure currently does not support the following features:
## Next steps

Now that you have a working kOps cluster, read through the recommendations for [production setups guide](production.md) to learn more about how to configure kOps for production workloads.

## Migrating from earlier alpha versions

Older alpha releases used `azureblob://<container>/...` URLs and read the storage account from `AZURE_STORAGE_ACCOUNT`. To upgrade an existing cluster:

1. `unset AZURE_STORAGE_ACCOUNT` and re-export `KOPS_STATE_STORE` in the new shape.
2. `kops edit cluster` to update `spec.configStore.base` to the updated URL.
3. `kops update cluster --yes` and `kops rolling-update cluster --yes`.
2 changes: 1 addition & 1 deletion hack/update-expected.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ unset AWS_ACCESS_KEY_ID AWS_REGION AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN CNI_V
unset KOPS_CLUSTER_NAME KOPS_RUN_OBSOLETE_VERSION KOPS_STATE_STORE KOPS_STATE_S3_ACL KUBE_API_VERSIONS NODEUP_URL OPENSTACK_CREDENTIAL_FILE PROTOKUBE_IMAGE SKIP_PACKAGE_UPDATE
unset SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCESS_KEY
unset SCW_ACCESS_KEY SCW_SECRET_KEY SCW_DEFAULT_PROJECT_ID SCW_PROFILE
unset AZURE_CLIENT_ID AZURE_CLIENT_SECRET AZURE_STORAGE_ACCOUNT AZURE_SUBSCRIPTION_ID AZURE_TENANT_ID
unset AZURE_CLIENT_ID AZURE_CLIENT_SECRET AZURE_SUBSCRIPTION_ID AZURE_TENANT_ID
unset DIGITALOCEAN_ACCESS_TOKEN

# Run the tests in "autofix mode"
Expand Down
4 changes: 0 additions & 4 deletions nodeup/pkg/bootstrap/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,6 @@ func (i *Installation) buildEnvFile() *nodetasks.InstallFile {
envVars["OSS_REGION"] = os.Getenv("OSS_REGION")
}

if os.Getenv("AZURE_STORAGE_ACCOUNT") != "" {
envVars["AZURE_STORAGE_ACCOUNT"] = os.Getenv("AZURE_STORAGE_ACCOUNT")
}

if os.Getenv("SCW_PROFILE") != "" || os.Getenv("SCW_SECRET_KEY") != "" {
profile, err := scaleway.CreateValidScalewayProfile()
if err != nil {
Expand Down
4 changes: 0 additions & 4 deletions nodeup/pkg/model/protokube.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,6 @@ func (t *ProtokubeBuilder) buildEnvFile() (*nodetasks.File, error) {
envVars["OSS_REGION"] = os.Getenv("OSS_REGION")
}

if os.Getenv("AZURE_STORAGE_ACCOUNT") != "" {
envVars["AZURE_STORAGE_ACCOUNT"] = os.Getenv("AZURE_STORAGE_ACCOUNT")
}

if t.CloudProvider() == kops.CloudProviderScaleway {
if os.Getenv("SCW_PROFILE") != "" || os.Getenv("SCW_SECRET_KEY") != "" {
profile, err := scaleway.CreateValidScalewayProfile()
Expand Down
79 changes: 79 additions & 0 deletions pkg/apis/kops/validation/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"k8s.io/kops/pkg/model/iam"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/utils"
"k8s.io/kops/util/pkg/vfs"
)

func newValidateCluster(cluster *kops.Cluster, strict bool) field.ErrorList {
Expand Down Expand Up @@ -199,6 +200,8 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
}
}

allErrs = append(allErrs, validateAzureBlobAccountUniformity(spec, fieldPath)...)

if spec.ContainerRuntime != "" {
allErrs = append(allErrs, validateContainerRuntime(c, spec.ContainerRuntime, fieldPath.Child("containerRuntime"))...)
}
Expand Down Expand Up @@ -1521,6 +1524,82 @@ func validateEtcdBackupStore(specs []kops.EtcdClusterSpec, fieldPath *field.Path
return allErrs
}

// azureBlobAccount returns the storage account encoded in an azureblob:// URL,
// or "" with no error if the URL is not azureblob://. Returns an error only if
// the URL has the azureblob:// prefix but fails to parse.
func azureBlobAccount(rawURL string) (string, error) {
if !strings.HasPrefix(rawURL, "azureblob://") {
return "", nil
}
p, err := vfs.Context.BuildVfsPath(rawURL)
if err != nil {
return "", err
}
azPath, ok := p.(*vfs.AzureBlobPath)
if !ok {
return "", fmt.Errorf("expected azureblob:// URL, got %q", rawURL)
}
return azPath.Account(), nil
}

// validateAzureBlobAccountUniformity enforces that every azureblob:// URL in
// the cluster spec uses the same storage account as configStore.base. Any
// azureblob:// URL elsewhere in the spec is rejected when configStore.base is
// not itself azureblob://.
func validateAzureBlobAccountUniformity(spec *kops.ClusterSpec, fieldPath *field.Path) field.ErrorList {
var allErrs field.ErrorList
csPath := fieldPath.Child("configStore")

canonical := ""
if strings.HasPrefix(spec.ConfigStore.Base, "azureblob://") {
account, err := azureBlobAccount(spec.ConfigStore.Base)
if err != nil {
allErrs = append(allErrs, field.Invalid(csPath.Child("base"), spec.ConfigStore.Base, err.Error()))
return allErrs
}
canonical = account
}

type entry struct {
path *field.Path
url string
}
others := []entry{
{csPath.Child("keypairs"), spec.ConfigStore.Keypairs},
{csPath.Child("secrets"), spec.ConfigStore.Secrets},
}
for i, ec := range spec.EtcdClusters {
if ec.Backups != nil {
others = append(others, entry{
fieldPath.Child("etcdClusters").Index(i).Child("backups", "backupStore"),
ec.Backups.BackupStore,
})
}
}

for _, e := range others {
if !strings.HasPrefix(e.url, "azureblob://") {
continue
}
account, err := azureBlobAccount(e.url)
if err != nil {
allErrs = append(allErrs, field.Invalid(e.path, e.url, err.Error()))
continue
}
if canonical == "" {
allErrs = append(allErrs, field.Invalid(e.path, e.url,
"azureblob:// URL requires configStore.base to also be azureblob://"))
continue
}
if account != canonical {
allErrs = append(allErrs, field.Invalid(e.path, e.url,
fmt.Sprintf("storage account %q does not match configStore.base account %q", account, canonical)))
}
}

return allErrs
}

// validateEtcdStorage is responsible for checking versions are identical.
func validateEtcdStorage(specs []kops.EtcdClusterSpec, fieldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
Expand Down
155 changes: 155 additions & 0 deletions pkg/apis/kops/validation/validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1931,3 +1931,158 @@ func TestValidateNetworkingLinode(t *testing.T) {
})
}
}

func TestValidateAzureBlobAccountUniformity(t *testing.T) {
tests := []struct {
name string
spec kops.ClusterSpec
expected []*field.Error
}{
{
name: "all matching azureblob URLs",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "azureblob://kopsstate/state/cluster.example.com",
Keypairs: "azureblob://kopsstate/state/cluster.example.com/pki",
Secrets: "azureblob://kopsstate/state/cluster.example.com/secrets",
},
EtcdClusters: []kops.EtcdClusterSpec{{
Backups: &kops.EtcdBackupSpec{
BackupStore: "azureblob://kopsstate/state/cluster.example.com/backups/etcd/main",
},
}},
},
},
{
name: "non-azure cluster is unaffected",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "s3://my-bucket/cluster.example.com",
Keypairs: "s3://my-bucket/cluster.example.com/pki",
},
EtcdClusters: []kops.EtcdClusterSpec{{
Backups: &kops.EtcdBackupSpec{
BackupStore: "s3://my-bucket/cluster.example.com/backups/etcd/main",
},
}},
},
},
{
name: "keypairs uses different storage account",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "azureblob://kopsstate/state/cluster.example.com",
Keypairs: "azureblob://otheracct/state/cluster.example.com/pki",
},
},
expected: []*field.Error{
{
Type: field.ErrorTypeInvalid,
Field: "spec.configStore.keypairs",
},
},
},
{
name: "secrets uses different storage account",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "azureblob://kopsstate/state/cluster.example.com",
Secrets: "azureblob://otheracct/state/cluster.example.com/secrets",
},
},
expected: []*field.Error{
{
Type: field.ErrorTypeInvalid,
Field: "spec.configStore.secrets",
},
},
},
{
name: "etcd backupStore uses different storage account",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "azureblob://kopsstate/state/cluster.example.com",
},
EtcdClusters: []kops.EtcdClusterSpec{{
Backups: &kops.EtcdBackupSpec{
BackupStore: "azureblob://otheracct/backups/etcd/main",
},
}},
},
expected: []*field.Error{
{
Type: field.ErrorTypeInvalid,
Field: "spec.etcdClusters[0].backups.backupStore",
},
},
},
{
name: "azureblob backupStore with non-azure configStore.base is rejected",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "s3://my-bucket/cluster.example.com",
},
EtcdClusters: []kops.EtcdClusterSpec{{
Backups: &kops.EtcdBackupSpec{
BackupStore: "azureblob://kopsstate/backups/etcd/main",
},
}},
},
expected: []*field.Error{
{
Type: field.ErrorTypeInvalid,
Field: "spec.etcdClusters[0].backups.backupStore",
},
},
},
{
name: "malformed azureblob configStore.base is rejected",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "azureblob://kopsstate",
},
},
expected: []*field.Error{
{
Type: field.ErrorTypeInvalid,
Field: "spec.configStore.base",
},
},
},
{
name: "malformed azureblob keypairs is rejected",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "azureblob://kopsstate/state/cluster.example.com",
Keypairs: "azureblob://kopsstate",
},
},
expected: []*field.Error{
{
Type: field.ErrorTypeInvalid,
Field: "spec.configStore.keypairs",
},
},
},
{
name: "non-azure backup store with azure config base is allowed",
spec: kops.ClusterSpec{
ConfigStore: kops.ConfigStoreSpec{
Base: "azureblob://kopsstate/state/cluster.example.com",
},
EtcdClusters: []kops.EtcdClusterSpec{{
Backups: &kops.EtcdBackupSpec{
BackupStore: "memfs://tests/cluster.example.com/backups/etcd/main",
},
}},
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
errList := validateAzureBlobAccountUniformity(&tt.spec, field.NewPath("spec"))
testFieldErrors(t, errList, tt.expected)
})
}
}
Loading
Loading