Wait for server to acknowledge app deletion before trying ignore-cleanup

There are certain situations where the app delete succeeds and the call
to ignore-cleanup fails because it thinks that the app is not being
deleted. This change will retry on this error to avoid the necessity of
having to delete the app and then ignore-cleanup.

Signed-off-by: Donnie Adams <[email protected]>

Author: thedadams

pkg/cli/rm_helper.go

@@ -2,7 +2,9 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/http"
 	"strings"
 	"time"
 
@@ -11,6 +13,7 @@ import (
 	"github.com/pterm/pterm"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	k8swait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/retry"
 
 	"github.com/acorn-io/runtime/pkg/client"
@@ -147,7 +150,19 @@ func removeAcorn(ctx context.Context, c client.Client, arg string, ignoreCleanup
 	}
 
 	if ignoreCleanup {
-		if err := c.AppIgnoreDeleteCleanup(ctx, arg); err != nil {
+		// There are situations where an app being deleted the first time with the --ignore-cleanup flag will fail at this
+		// step because the server thinks that the app is not being deleted. Retrying here will work around this issue.
+		if err = retry.OnError(k8swait.Backoff{
+			Steps:    5,
+			Duration: 500 * time.Millisecond,
+			Factor:   2,
+			Jitter:   0.1,
+		}, func(err error) bool {
+			var statusErr *apierrors.StatusError
+			return errors.As(err, &statusErr) && statusErr.Status().Code == http.StatusBadRequest && strings.HasSuffix(statusErr.Status().Message, "it is not being deleted")
+		}, func() error {
+			return c.AppIgnoreDeleteCleanup(ctx, arg)
+		}); err != nil {
 			return fmt.Errorf("skipping cleanup for app %s: %w", arg, err)
 		}
 	}

pkg/server/registry/apigroups/acorn/apps/ignorecleanup.go

@@ -59,7 +59,7 @@ func (s *ignoreCleanupStrategy) Create(ctx context.Context, obj types.Object) (t
 		}
 
 		if app.DeletionTimestamp.IsZero() {
-			return fmt.Errorf("cannot force delete app %s because it is not being deleted", app.Name)
+			return apierrors.NewBadRequest(fmt.Sprintf("cannot force delete app %s because it is not being deleted", app.Name))
 		}
 
 		// If the app has the destroy job finalizer, remove it to force delete