Publish ActorTemplate golden snapshots as tags

benchmarking/locust/common/ateapi_pb2_grpc.py

@@ -473,7 +473,7 @@ def ListActorTemplates(self, request, context):
 
     def DeleteActorTemplate(self, request, context):
         """Delete an ActorTemplate together with its golden actor and golden
-        snapshot in the ActorTemplate's namespace.
+        tag in the reserved ate-golden atespace.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')

benchmarking/workloads/deploy.sh

@@ -149,7 +149,7 @@ wait_actortemplate_ready() {
 
   while ((SECONDS < deadline)); do
     if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then
-      snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenSnapshot.snapshotUri // empty' <<<"${json}")
+      snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}")
       if [[ -n "${snapshot}" ]]; then
         return 0
       fi

cmd/ate-setup/internal/steps/substrate.go

@@ -97,7 +97,7 @@ func WaitActorTemplateGolden(ctx context.Context, client *ateclient.Client, ref
 		lastErr = err
 		if err == nil {
 			goldenStatus := template.GetStatus().GetGoldenSnapshotStatus()
-			if goldenStatus.GetGoldenSnapshot().GetSnapshotUri() != "" {
+			if goldenStatus.GetGoldenTag().GetName() != "" {
 				return nil
 			}
 			if msg := goldenStatus.GetErrorMessage(); msg != "" {

cmd/ateapi/internal/controlapi/actor.go

@@ -86,14 +86,29 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor)
 		return nil, err
 	}
 
-	// If a source tag is requested, resolve it to the external
-	// snapshot the new Actor starts from.
+	// Resolve the explicit tag, or freeze the template's current golden default.
+	tagRef := inActor.GetSourceTag()
+	if tagRef == nil {
+		tagRef = template.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()
+	} else {
+		for _, volume := range template.GetVolumes() {
+			if volume.GetExternalVolumeTemplate() != nil {
+				// TODO: Permit cloning after CSI volume snapshots are supported.
+				return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes")
+			}
+		}
+	}
 	var sourceTag *ateapipb.Tag
-	if tagRef := inActor.GetSourceTag(); tagRef != nil {
+	if tagRef != nil {
 		sourceTag, err = s.resolveTagSource(ctx, inActor.GetMetadata().GetAtespace(), tagRef, template)
 		if err != nil {
 			return nil, err
 		}
+		if inActor.GetSourceTag() == nil {
+			if err := validateGoldenSnapshotScope(sourceTag.GetStatus().GetSnapshot()); err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	atespace := inActor.GetMetadata().GetAtespace()
@@ -176,12 +191,6 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string
 	if tag.GetStatus().GetActorTemplateUid() != template.GetMetadata().GetUid() {
 		return nil, status.Errorf(codes.FailedPrecondition, "source Tag must be taken from an actor with ActorTemplate uid %q", tag.GetStatus().GetActorTemplateUid())
 	}
-	for _, volume := range template.GetVolumes() {
-		if volume.GetExternalVolumeTemplate() != nil {
-			// TODO: Permit cloning after CSI volume snapshots are supported.
-			return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes")
-		}
-	}
 	return tag, nil
 }
 

cmd/ateapi/internal/controlapi/actor_template.go

@@ -155,6 +155,26 @@ func (s *RPCService) DeleteActorTemplate(ctx context.Context, req *ateapipb.Dele
 	}
 
 	templateRef := resources.ActorTemplateRefFromObjectRef(req.GetActorTemplate())
+	// Serialize cleanup against golden actor/tag creation by the reconciler.
+	ctx, lease, err := acquireLease(ctx, s.impl, "lease:actortemplate:"+templateRef.Atespace+":"+templateRef.Name, "ActorTemplate "+templateRef.String())
+	if err != nil {
+		return nil, err
+	}
+	defer lease.Close()
+	tmpl, err := s.impl.GetActorTemplate(ctx, templateRef)
+	if errors.Is(err, store.ErrNotFound) {
+		return nil, status.Errorf(codes.NotFound, "ActorTemplate %s not found", templateRef)
+	}
+	if err != nil {
+		return nil, err
+	}
+	goldenRef := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}
+	if _, err := s.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: goldenRef, AnyState: true}); err != nil && status.Code(err) != codes.NotFound {
+		return nil, fmt.Errorf("while deleting golden actor: %w", err)
+	}
+	if _, err := s.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: goldenRef}); err != nil && status.Code(err) != codes.NotFound {
+		return nil, fmt.Errorf("while deleting golden tag: %w", err)
+	}
 	deleted, err := s.impl.DeleteActorTemplate(ctx, templateRef)
 	if err != nil {
 		if errors.Is(err, store.ErrNotFound) {

cmd/ateapi/internal/controlapi/actor_template_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
+	"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest"
 	"github.com/agent-substrate/substrate/internal/resources"
 	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
 	listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
@@ -314,7 +315,7 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) {
 		// Server-owned status a client must not be able to set.
 		tmpl.Status = &ateapipb.ActorTemplateStatus{
 			GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
-				GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://my-bucket/snapshots/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/sneaky"},
+				GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
 			},
 		}
 	})
@@ -339,6 +340,125 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) {
 	}
 }
 
+func TestDeleteActorTemplate(t *testing.T) {
+	tests := []struct {
+		name         string
+		actorDeleted bool
+		tagDeleted   bool
+		pendingTag   bool
+		// failPrefix makes object storage fail cleanup for this resource kind.
+		failPrefix            string
+		wantActorAfterFailure bool
+	}{
+		{name: "golden actor and tag"},
+		{name: "golden actor already deleted", actorDeleted: true},
+		{name: "golden tag absent", tagDeleted: true},
+		{name: "no golden resources", actorDeleted: true, tagDeleted: true},
+		{name: "incomplete golden tag", pendingTag: true},
+		{name: "actor cleanup failure", failPrefix: "/actors/", wantActorAfterFailure: true},
+		{name: "tag cleanup failure", failPrefix: "/tags/"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := t.Context()
+			persistence := newTestPersistence(t)
+			tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl")
+			templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl)
+			goldenRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}
+			actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{
+				Metadata:      &ateapipb.ResourceMetadata{Atespace: goldenRef.Atespace, Name: goldenRef.Name},
+				ActorTemplate: templateRef.ToObjectRef(),
+				Status:        &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
+			})
+			workflow, objects := newFinalizeWorkflow(persistence)
+			actorURI := mustActorSnapshotURI(t, tmpl, actor, "snapshot")
+			objects.PutSnapshot(t, actorURI, "manifest.json")
+			actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) {
+				s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorURI.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}
+				s.CurrentActorTemplateUid = tmpl.GetMetadata().GetUid()
+			})
+			var tag *ateapipb.Tag
+			if tt.pendingTag {
+				tag = storetest.MustCreateTag(t, ctx, persistence, newPendingTestTag(t, goldenRef.Name, actor))
+			} else {
+				var err error
+				tag, err = workflow.TagActorSnapshot(ctx, tagToCreate(goldenRef, goldenRef.Name))
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			tagRef := resources.TagRefFromTag(tag)
+			tagURI := mustReservedTagSnapshotURI(t, tag)
+			objects.PutSnapshot(t, tagURI, "manifest.json")
+			svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, objectStore: objects}
+			// The handler must request AnyState to clean up an active golden actor.
+			mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) {
+				s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING
+			})
+			if tt.actorDeleted {
+				if _, err := workflow.DeleteActor(ctx, goldenRef, true); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if tt.tagDeleted {
+				if _, err := svc.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: tagRef.ToObjectRef()}); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if tt.failPrefix != "" {
+				objects.OnDelete = func(_, key string) error {
+					if strings.Contains(key, tt.failPrefix) {
+						return errObjectStore
+					}
+					return nil
+				}
+			}
+			req := &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()}
+			deleted, err := svc.DeleteActorTemplate(ctx, req)
+			if tt.failPrefix != "" {
+				if !errors.Is(err, errObjectStore) {
+					t.Fatalf("DeleteActorTemplate = %v, want object storage error", err)
+				}
+				if _, err := persistence.GetActorTemplate(ctx, templateRef); err != nil {
+					t.Fatalf("template lost after cleanup failure: %v", err)
+				}
+				if _, err := persistence.GetTag(ctx, tagRef); err != nil {
+					t.Fatalf("tag lost after cleanup failure: %v", err)
+				}
+				_, actorErr := persistence.GetActor(ctx, goldenRef)
+				if tt.wantActorAfterFailure && actorErr != nil || !tt.wantActorAfterFailure && !errors.Is(actorErr, store.ErrNotFound) {
+					t.Fatalf("GetActor after failure = %v, want present %v", actorErr, tt.wantActorAfterFailure)
+				}
+				objects.OnDelete = nil
+				deleted, err = svc.DeleteActorTemplate(ctx, req)
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(tmpl, deleted, protocmp.Transform()); diff != "" {
+				t.Fatalf("deleted template mismatch (-want +got):\n%s", diff)
+			}
+			if _, err := persistence.GetActorTemplate(ctx, templateRef); !errors.Is(err, store.ErrNotFound) {
+				t.Fatalf("GetActorTemplate after delete = %v, want NotFound", err)
+			}
+			if _, err := persistence.GetActor(ctx, goldenRef); !errors.Is(err, store.ErrNotFound) {
+				t.Fatalf("GetActor after delete = %v, want NotFound", err)
+			}
+			if _, err := persistence.GetTag(ctx, tagRef); !errors.Is(err, store.ErrNotFound) {
+				t.Fatalf("GetTag after delete = %v, want NotFound", err)
+			}
+			for _, uri := range []resources.SnapshotURI{actorURI, tagURI} {
+				if got := objects.Snapshot(t, uri); len(got) != 0 {
+					t.Errorf("snapshot %s still holds %v", uri, got)
+				}
+			}
+			if _, err := svc.DeleteActorTemplate(ctx, req); status.Code(err) != codes.NotFound {
+				t.Fatalf("delete missing template = %v, want NotFound", err)
+			}
+		})
+	}
+}
+
 func TestValidateGetActorTemplateRequest(t *testing.T) {
 	tests := []struct {
 		name string
@@ -1224,7 +1344,7 @@ func TestUpdateActorTemplateMetadata(t *testing.T) {
 	// A server-owned status write passes validation and bumps the version.
 	updated, err := persistence.UpdateActorTemplate(ctx, ref, store.PreconditionFrom(created), func(tmpl *ateapipb.ActorTemplate) error {
 		tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
-			GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"},
+			GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
 		}}
 		return nil
 	})

cmd/ateapi/internal/controlapi/actor_test.go

@@ -1436,3 +1436,90 @@ func TestValidateSuspendActorRequest(t *testing.T) {
 		})
 	}
 }
+
+func TestCreateActor_GoldenTagDefault(t *testing.T) {
+	for _, scenario := range []string{"default", "explicit tag", "own snapshot", "missing", "pending", "wrong template", "data scope"} {
+		t.Run(scenario, func(t *testing.T) {
+			ctx := t.Context()
+			persistence := newTestPersistence(t)
+			storetest.MustCreateAtespace(t, ctx, persistence, "team-a")
+			storetest.MustCreateAtespace(t, ctx, persistence, resources.GoldenActorAtespace)
+			tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl")
+			ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "golden"}
+			tag := &ateapipb.Tag{
+				Metadata:    &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name},
+				SourceActor: ref,
+				Scope:       ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
+				Status: &ateapipb.TagStatus{
+					ActorTemplateUid: tmpl.GetMetadata().GetUid(),
+					Snapshot:         &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
+				},
+			}
+			wantCode := codes.OK
+			switch scenario {
+			case "missing":
+				wantCode = codes.NotFound
+			case "pending":
+				tag.Status.Snapshot = nil
+				wantCode = codes.FailedPrecondition
+			case "wrong template":
+				tag.Status.ActorTemplateUid = "other"
+				wantCode = codes.FailedPrecondition
+			case "data scope":
+				tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA
+				wantCode = codes.FailedPrecondition
+			}
+			if scenario != "missing" {
+				if _, err := persistence.CreateTag(ctx, tag); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if _, err := persistence.UpdateActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl), store.PreconditionFrom(tmpl), func(db *ateapipb.ActorTemplate) error {
+				db.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: ref}}
+				return nil
+			}); err != nil {
+				t.Fatal(err)
+			}
+			actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()}
+			if scenario == "explicit tag" {
+				tag.Metadata.Name = "explicit"
+				tag.Status.Snapshot.SnapshotUri = "gs://bucket/atespaces/ate-golden/tags/explicit"
+				if _, err := persistence.CreateTag(ctx, tag); err != nil {
+					t.Fatal(err)
+				}
+				actor.SourceTag = &ateapipb.ObjectRef{Atespace: ref.Atespace, Name: "explicit"}
+			}
+			svc := &ServiceImpl{store: persistence}
+			created, err := svc.CreateActor(ctx, actor)
+			if status.Code(err) != wantCode {
+				t.Fatalf("CreateActor = %v, want %v", err, wantCode)
+			}
+			if err != nil {
+				return
+			}
+			if got := created.GetStatus(); got.GetExternalSnapshot().GetSnapshotUri() != tag.GetStatus().GetSnapshot().GetSnapshotUri() || got.GetCurrentActorTemplateUid() != tmpl.GetMetadata().GetUid() {
+				t.Fatalf("incorrect initial status: %v", got)
+			}
+			if scenario == "own snapshot" {
+				uri, err := resources.NewActorSnapshotURI(tmpl.GetSnapshotsConfig().GetStorageLocation(), "team-a", created.GetMetadata().GetUid(), "snapshot")
+				if err != nil {
+					t.Fatal(err)
+				}
+				if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapipb.Actor) error {
+					db.Status.ExternalSnapshot.SnapshotUri = uri.String()
+					return nil
+				}); err != nil {
+					t.Fatal(err)
+				}
+			}
+			workflow := &ActorWorkflow{store: persistence}
+			_, _, src, err := workflow.loadActorForResume(ctx, resources.ActorRefFromActor(created))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if src.SnapshotURI.IsZero() {
+				t.Fatalf("missing snapshot source for %s", scenario)
+			}
+		})
+	}
+}