[WIP]Allow force deleting an actor from any state.

cmd/ateapi/internal/controlapi/delete_actor.go

@@ -45,7 +45,7 @@ func (s *Service) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequ
 	actorRef := resources.ActorRefFromObjectRef(req.GetActor())
 	setSpanActorRefAttributes(ctx, actorRef)
 
-	deleted, err = s.actorWorkflow.DeleteActor(ctx, req.GetActor().GetAtespace(), req.GetActor().GetName())
+	deleted, err = s.actorWorkflow.DeleteActor(ctx, req.GetActor().GetAtespace(), req.GetActor().GetName(), req.GetForce())
 	if err != nil {
 		return nil, err
 	}

cmd/ateapi/internal/controlapi/delete_actor_test.go

@@ -26,6 +26,8 @@ import (
 	"github.com/agent-substrate/substrate/internal/resources"
 	"github.com/agent-substrate/substrate/internal/volume"
 	"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
@@ -207,3 +209,137 @@ func TestDeleteActor_MultipleVolumeDeletionFailures(t *testing.T) {
 		t.Errorf("expected error message to contain both volume failure details, got: %v", errMsg)
 	}
 }
+
+func TestDeleteActor_Force_Success(t *testing.T) {
+	ns := namespaceForTest("ns-delete-force-succ")
+	tc := setupTest(t, ns)
+	defer tc.cleanup()
+	createTemplate(t, tc, ns)
+	createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1")
+
+	runningActor := &ateapipb.Actor{
+		Metadata: &ateapipb.ResourceMetadata{
+			Atespace: testAtespace,
+			Name:     "running-actor",
+		},
+		Status:                 ateapipb.Actor_STATUS_RUNNING,
+		ActorTemplateNamespace: ns,
+		ActorTemplateName:      "tmpl1",
+		WorkerAssignment: &ateapipb.WorkerAssignment{
+			WorkerNamespace: ns,
+			WorkerPod:       "worker-1",
+			WorkerPool:      "pool1",
+		},
+	}
+	if _, err := tc.persistence.CreateActor(context.Background(), runningActor); err != nil {
+		t.Fatalf("CreateActor: %v", err)
+	}
+
+	worker, err := tc.persistence.GetWorker(context.Background(), ns, "pool1", "worker-1")
+	if err != nil {
+		t.Fatalf("GetWorker: %v", err)
+	}
+	worker.Assignment = &ateapipb.Assignment{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "running-actor"},
+	}
+	if err := tc.persistence.UpdateWorker(context.Background(), worker, worker.Version); err != nil {
+		t.Fatalf("UpdateWorker: %v", err)
+	}
+
+	deleted, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "running-actor"},
+		Force: true,
+	})
+	if err != nil {
+		t.Fatalf("DeleteActor with force failed: %v", err)
+	}
+	if deleted.GetMetadata().GetName() != "running-actor" {
+		t.Errorf("deleted actor name = %q, want %q", deleted.GetMetadata().GetName(), "running-actor")
+	}
+
+	// Verify actor is removed from store
+	if _, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: "running-actor"}); err == nil {
+		t.Errorf("expected actor to be deleted from store, but it still exists")
+	}
+
+	// Verify worker assignment is cleared
+	w, err := tc.persistence.GetWorker(context.Background(), ns, "pool1", "worker-1")
+	if err != nil {
+		t.Fatalf("GetWorker after delete failed: %v", err)
+	}
+	if w.Assignment != nil {
+		t.Errorf("expected worker assignment to be nil, got: %v", w.Assignment)
+	}
+
+	// Verify atelet Terminate was called
+	if !tc.fakeAtelet.TerminateCalled {
+		t.Errorf("expected atelet Terminate to be called for force delete")
+	}
+}
+
+func TestDeleteActor_Force_SuspendedAllowed(t *testing.T) {
+	ns := namespaceForTest("ns-delete-force-susp")
+	tc := setupTest(t, ns)
+	defer tc.cleanup()
+	createTemplate(t, tc, ns)
+
+	suspendedActor := &ateapipb.Actor{
+		Metadata: &ateapipb.ResourceMetadata{
+			Atespace: testAtespace,
+			Name:     "suspended-actor",
+		},
+		Status:                 ateapipb.Actor_STATUS_SUSPENDED,
+		ActorTemplateNamespace: ns,
+		ActorTemplateName:      "tmpl1",
+	}
+	if _, err := tc.persistence.CreateActor(context.Background(), suspendedActor); err != nil {
+		t.Fatalf("CreateActor: %v", err)
+	}
+
+	deleted, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "suspended-actor"},
+		Force: true,
+	})
+	if err != nil {
+		t.Fatalf("expected DeleteActor with force on suspended actor to succeed, got: %v", err)
+	}
+	if deleted.GetMetadata().GetName() != "suspended-actor" {
+		t.Errorf("deleted.Name = %q, want %q", deleted.GetMetadata().GetName(), "suspended-actor")
+	}
+}
+
+func TestDeleteActor_Force_WorkerNotFound(t *testing.T) {
+	ns := namespaceForTest("ns-delete-force-noworker")
+	tc := setupTest(t, ns)
+	defer tc.cleanup()
+	createTemplate(t, tc, ns)
+
+	runningActor := &ateapipb.Actor{
+		Metadata: &ateapipb.ResourceMetadata{
+			Atespace: testAtespace,
+			Name:     "running-actor",
+		},
+		Status:                 ateapipb.Actor_STATUS_RUNNING,
+		ActorTemplateNamespace: ns,
+		ActorTemplateName:      "tmpl1",
+		WorkerAssignment: &ateapipb.WorkerAssignment{
+			WorkerNamespace: ns,
+			WorkerPod:       "non-existent-worker",
+			WorkerPool:      "pool1",
+		},
+	}
+	if _, err := tc.persistence.CreateActor(context.Background(), runningActor); err != nil {
+		t.Fatalf("CreateActor: %v", err)
+	}
+
+	_, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "running-actor"},
+		Force: true,
+	})
+	if err == nil {
+		t.Fatalf("expected DeleteActor with force on non-existent worker to fail, but it succeeded")
+	}
+	if status.Code(err) != codes.NotFound {
+		t.Errorf("status.Code(err) = %v, want %v (err: %v)", status.Code(err), codes.NotFound, err)
+	}
+}

cmd/ateapi/internal/controlapi/functional_test.go

@@ -180,6 +180,10 @@ type FakeAteletServer struct {
 	RestoreRequest *ateletpb.RestoreRequest
 	FailRestore    error
 	RestoreDelay   time.Duration
+
+	TerminateCalled  bool
+	TerminateRequest *ateletpb.TerminateRequest
+	FailTerminate    error
 }
 
 func (f *FakeAteletServer) Reset() {
@@ -197,6 +201,23 @@ func (f *FakeAteletServer) Reset() {
 	f.RestoreRequest = nil
 	f.FailRestore = nil
 	f.RestoreDelay = 0
+
+	f.TerminateCalled = false
+	f.TerminateRequest = nil
+	f.FailTerminate = nil
+}
+
+func (f *FakeAteletServer) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) (*ateletpb.TerminateResponse, error) {
+	f.Lock.Lock()
+	defer f.Lock.Unlock()
+
+	f.TerminateCalled = true
+	f.TerminateRequest = proto.Clone(req).(*ateletpb.TerminateRequest)
+	if f.FailTerminate != nil {
+		return nil, f.FailTerminate
+	}
+
+	return &ateletpb.TerminateResponse{}, nil
 }
 
 func (f *FakeAteletServer) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.RunResponse, error) {
@@ -3291,20 +3312,10 @@ func TestDeleteActor_Crashed(t *testing.T) {
 		t.Fatalf("UpdateActor failed: %v", err)
 	}
 
-	deleted, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{
-		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"},
-	})
-	if err != nil {
-		t.Fatalf("DeleteActor of crashed actor failed: %v", err)
-	}
-	if got := deleted.GetStatus(); got != ateapipb.Actor_STATUS_DELETING {
-		t.Errorf("deleted actor status = %v, want %v", got, ateapipb.Actor_STATUS_DELETING)
-	}
-
-	_, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{
+	_, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{
 		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"},
 	})
-	assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found")
+	assertGrpcError(t, err, codes.FailedPrecondition, "Actor test-atespace/id1 is not in a deletable status (status: STATUS_CRASHED)")
 }
 
 func TestDeleteActor_NotFound(t *testing.T) {
@@ -3318,6 +3329,88 @@ func TestDeleteActor_NotFound(t *testing.T) {
 	assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/non-existent not found")
 }
 
+func TestDeleteActor_Force(t *testing.T) {
+	ns := namespaceForTest("ns-delete-force")
+	tc := setupTest(t, ns)
+	defer tc.cleanup()
+
+	createTemplate(t, tc, ns)
+	createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1")
+
+	// 1. Create and resume actor to running status
+	_, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
+		Metadata:               &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "force-actor"},
+		ActorTemplateNamespace: ns,
+		ActorTemplateName:      "tmpl1",
+	}})
+	if err != nil {
+		t.Fatalf("CreateActor failed: %v", err)
+	}
+
+	resumeResp, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "force-actor"},
+	})
+	if err != nil {
+		t.Fatalf("ResumeActor failed: %v", err)
+	}
+	if resumeResp.GetActor().GetStatus() != ateapipb.Actor_STATUS_RUNNING {
+		t.Fatalf("expected status STATUS_RUNNING, got %v", resumeResp.GetActor().GetStatus())
+	}
+
+	// 2. Force delete running actor
+	deleted, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "force-actor"},
+		Force: true,
+	})
+	if err != nil {
+		t.Fatalf("DeleteActor with force failed: %v", err)
+	}
+	if deleted.GetMetadata().GetName() != "force-actor" {
+		t.Errorf("deleted actor name = %q, want %q", deleted.GetMetadata().GetName(), "force-actor")
+	}
+
+	// Verify actor is removed from store
+	_, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "force-actor"},
+	})
+	assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/force-actor not found")
+
+	// Verify atelet Terminate was invoked
+	if !tc.fakeAtelet.TerminateCalled {
+		t.Errorf("expected atelet Terminate to have been called")
+	}
+
+	// Verify worker is unassigned
+	w, err := tc.persistence.GetWorker(context.Background(), ns, "pool1", "worker-1")
+	if err != nil {
+		t.Fatalf("GetWorker failed: %v", err)
+	}
+	if w.Assignment != nil {
+		t.Errorf("expected worker assignment to be nil, got: %v", w.Assignment)
+	}
+
+	// 3. Verify force delete on suspended actor succeeds
+	_, err = tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
+		Metadata:               &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "susp-actor"},
+		ActorTemplateNamespace: ns,
+		ActorTemplateName:      "tmpl1",
+	}})
+	if err != nil {
+		t.Fatalf("CreateActor failed: %v", err)
+	}
+
+	deletedSusp, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{
+		Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "susp-actor"},
+		Force: true,
+	})
+	if err != nil {
+		t.Fatalf("expected DeleteActor with force on suspended actor to succeed, got %v", err)
+	}
+	if deletedSusp.GetMetadata().GetName() != "susp-actor" {
+		t.Errorf("deleted actor name = %q, want %q", deletedSusp.GetMetadata().GetName(), "susp-actor")
+	}
+}
+
 func assertGrpcErrorRegex(t *testing.T, err error, wantCode codes.Code, wantMsg string) {
 	t.Helper()
 	fn := func(got string) (string, bool) {

cmd/ateapi/internal/controlapi/workflow.go

@@ -281,10 +281,11 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor
 }
 
 // DeleteActor executes the workflow to delete an actor. Idempotent.
-func (w *ActorWorkflow) DeleteActor(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) {
+func (w *ActorWorkflow) DeleteActor(ctx context.Context, atespace, name string, force bool) (*ateapipb.Actor, error) {
 	actorRef := resources.ActorRef{Atespace: atespace, Name: name}
 	input := &DeleteInput{
 		ActorRef: actorRef,
+		Force:    force,
 	}
 	state := &DeleteState{}
 
@@ -295,7 +296,11 @@ func (w *ActorWorkflow) DeleteActor(ctx context.Context, atespace, name string)
 	defer lock.Close()
 
 	steps := []WorkflowStep[*DeleteInput, *DeleteState]{
-		&LoadActorForDeleteStep{store: w.store},
+		&LoadActorForDeleteStep{store: w.store, actorTemplateLister: w.actorTemplateLister},
+		&MarkTerminatingStep{store: w.store},
+		&CallAteletTerminateStep{store: w.store, dialer: w.dialer},
+		&DetachVolumesForDeleteStep{store: w.store},
+		&ReleaseWorkerStep{store: w.store},
 		&MarkDeletingStep{store: w.store},
 		&DeleteVolumesStep{store: w.store},
 		&FinalizeDeletedStep{store: w.store},