kubernetes
diff --git a/‎tests/e2e/scenarios/ai-conformance/validators/kube.go‎
Lines changed: 41 additions & 1 deletion b/‎tests/e2e/scenarios/ai-conformance/validators/kube.go‎
Lines changed: 41 additions & 1 deletion
@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/kops/tests/e2e/scenarios/ai-conformance/testartifacts"
+	"sigs.k8s.io/yaml"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -199,18 +200,57 @@ func (h *ValidatorHarness) TestNamespace() string {
 
 			startTime := time.Now()
 
+			deploymentGVR := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}
+
+			// HACK: Clean up deployments before deleting the namespace, to work around an issue where GPU pods can get stuck terminating due to finalizers related to GPU resource cleanup.
+			h.Logf("Deleting deployments in namespace %q", ns)
+			deleteOptions := metav1.DeleteOptions{}
+			gracePeriod := int64(1)
+			deleteOptions.GracePeriodSeconds = &gracePeriod
+			if err := h.DynamicClient().Resource(deploymentGVR).Namespace(ns).DeleteCollection(ctx, deleteOptions, metav1.ListOptions{}); err != nil {
+				h.Errorf("failed to delete deployments in namespace %q: %v", ns, err)
+			}
+
 			h.Logf("Deleting test namespace %q", ns)
 			if err := h.DynamicClient().Resource(namespaceGVR).Delete(ctx, ns, metav1.DeleteOptions{}); err != nil {
 				h.Errorf("failed to delete test namespace: %v", err)
 			}
 
+			// Wait for deployment deletion to complete.
+			if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 5*time.Minute, false, func(ctx context.Context) (done bool, err error) {
+				deploymentList, err := h.DynamicClient().Resource(deploymentGVR).Namespace(ns).List(ctx, metav1.ListOptions{})
+				if err != nil {
+					return false, fmt.Errorf("error checking for deployment deletion: %w", err)
+				}
+				if len(deploymentList.Items) == 0 {
+					return true, nil
+				}
+				var names []string
+				for _, deployment := range deploymentList.Items {
+					names = append(names, deployment.GetName())
+				}
+				h.Logf("Waiting for %d deployments to be deleted (%v)...", len(deploymentList.Items), names)
+
+				y, _ := yaml.Marshal(deploymentList)
+				h.Logf("Current deployments in namespace %q: %s", ns, string(y))
+
+				return false, nil
+			}); err != nil {
+				h.Errorf("error waiting for deployment deletion: %v", err)
+			}
+
 			// Wait for namespace deletion to complete so that we don't have leftover namespaces consuming resources.
 			if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 5*time.Minute, false, func(ctx context.Context) (done bool, err error) {
-				if _, err := h.DynamicClient().Resource(namespaceGVR).Get(ctx, ns, metav1.GetOptions{}); err != nil {
+				if ns, err := h.DynamicClient().Resource(namespaceGVR).Get(ctx, ns, metav1.GetOptions{}); err != nil {
 					if apierrors.IsNotFound(err) {
 						return true, nil
 					}
 					return false, fmt.Errorf("error checking for namespace deletion: %w", err)
+				} else {
+					h.Logf("Waiting for namespace %q to be deleted...", ns)
+
+					y, _ := yaml.Marshal(ns)
+					h.Logf("Current state of namespace %q: %s", ns, string(y))
 				}
 				return false, nil
 			}); err != nil {