From 4ae9e52087325b2afaec28d8780df46b5a2e8d91 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Sun, 6 Sep 2026 11:51:10 +0000 Subject: [PATCH 1/3] Publish template golden snapshots as tags Copy each warmed golden actor snapshot into a published tag before deleting the temporary actor. Resolve that tag as the default when creating actors, and use it for data-only golden restores. Clean up golden resources with their template and recover interrupted tag builds. Signed-off-by: Eitan Yarmush --- benchmarking/locust/common/ateapi_pb2.py | 344 +++++++++--------- benchmarking/locust/common/ateapi_pb2_grpc.py | 2 +- benchmarking/workloads/deploy.sh | 2 +- cmd/ate-setup/internal/steps/substrate.go | 2 +- cmd/ateapi/internal/controlapi/actor.go | 27 +- .../internal/controlapi/actor_template.go | 20 + .../controlapi/actor_template_test.go | 4 +- cmd/ateapi/internal/controlapi/actor_test.go | 87 +++++ .../functionaltest/actor_template_test.go | 99 ++++- .../controlapi/functionaltest/actor_test.go | 28 +- .../controlapi/functionaltest/common_test.go | 27 +- .../controlapi/template_reconciler.go | 78 ++-- .../controlapi/template_reconciler_test.go | 173 +++++++-- .../internal/controlapi/validation_test.go | 13 - .../internal/controlapi/workflow_resume.go | 37 +- .../controlapi/workflow_resume_test.go | 103 +++--- .../controlapi/zz_generated.validation.go | 27 +- cmd/kubectl-ate/README.md | 4 +- cmd/kubectl-ate/internal/printer/printer.go | 4 +- .../internal/printer/printer_test.go | 10 +- docs/api-guide.md | 14 +- hack/install-ate.sh | 2 +- hack/verify-atenet-drain.sh | 2 +- internal/e2e/template.go | 2 +- pkg/proto/ateapipb/ateapi.pb.go | 22 +- pkg/proto/ateapipb/ateapi.proto | 11 +- pkg/proto/ateapipb/ateapi_grpc.pb.go | 4 +- 27 files changed, 779 insertions(+), 369 deletions(-) diff --git a/benchmarking/locust/common/ateapi_pb2.py b/benchmarking/locust/common/ateapi_pb2.py index e0a6d66d74..466e2f6c1c 100644 --- a/benchmarking/locust/common/ateapi_pb2.py +++ b/benchmarking/locust/common/ateapi_pb2.py @@ -40,7 +40,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61teapi.proto\x12\x06\x61teapi\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"]\n\x10\x45xternalSnapshot\x12\x14\n\x0csnapshot_uri\x18\x01 \x01(\t\x12\x33\n\rcontent_scope\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"\x86\x01\n\x11LocalSnapshotInfo\x12\x15\n\rsnapshot_name\x18\x01 \x01(\t\x12%\n\x1dnode_vms_with_local_snapshots\x18\x02 \x03(\t\x12\x33\n\rcontent_scope\x18\x03 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"w\n\x08Selector\x12\x37\n\x0cmatch_labels\x18\x01 \x03(\x0b\x32!.ateapi.Selector.MatchLabelsEntry\x1a\x32\n\x10MatchLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb2\x01\n\x10ResourceMetadata\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0bupdate_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xdc\x02\n\x0e\x45xternalVolume\x12\x13\n\x0bvolume_name\x18\x01 \x01(\t\x12\x19\n\x11storage_volume_id\x18\x02 \x01(\t\x12\x13\n\x0bvolume_type\x18\x03 \x01(\t\x12-\n\x06status\x18\x04 \x01(\x0e\x32\x1d.ateapi.ExternalVolume.Status\x12\x41\n\x0evolume_context\x18\x05 \x03(\x0b\x32).ateapi.ExternalVolume.VolumeContextEntry\x1a\x34\n\x12VolumeContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x12\n\x0eSTATUS_CREATED\x10\x02\x12\x13\n\x0fSTATUS_DELETING\x10\x03\"\xd5\x01\n\x05\x41\x63tor\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0e\x61\x63tor_template\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x0fworker_selector\x18\x05 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\nsource_tag\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x06status\x18\x07 \x01(\x0b\x32\x13.ateapi.ActorStatus\"]\n\x0c\x45gressPolicy\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x05rules\x18\x02 \x03(\x0b\x32\x12.ateapi.EgressRule\"{\n\nEgressRule\x12\'\n\thostnames\x18\x01 \x01(\x0b\x32\x14.ateapi.HostnameRule\x12\x1f\n\x05\x63idrs\x18\x02 \x01(\x0b\x32\x10.ateapi.CIDRRule\x12#\n\x03\x61ll\x18\x03 \x01(\x0b\x32\x16.google.protobuf.Empty\"L\n\x0cHostnameRule\x12\x10\n\x08patterns\x18\x01 \x03(\t\x12*\n\x07\x65\x66\x66\x65\x63ts\x18\x02 \x01(\x0b\x32\x19.ateapi.EgressRuleEffects\"\x19\n\x08\x43IDRRule\x12\r\n\x05\x63idrs\x18\x01 \x03(\t\"U\n\x11\x45gressRuleEffects\x12@\n\x15inject_static_headers\x18\x01 \x03(\x0b\x32!.ateapi.CredentialHeaderInjection\"S\n\x19\x43redentialHeaderInjection\x12\x0e\n\x06header\x18\x01 \x01(\t\x12\x0e\n\x06prefix\x18\x02 \x01(\t\x12\x16\n\x0e\x63redential_uri\x18\x03 \x01(\t\"\xf1\x02\n\x0b\x41\x63torStatus\x12!\n\x05state\x18\x01 \x01(\x0e\x32\x12.ateapi.ActorState\x12\x33\n\x11worker_assignment\x18\x02 \x01(\x0b\x32\x18.ateapi.WorkerAssignment\x12!\n\x19in_progress_snapshot_name\x18\x03 \x01(\t\x12\x33\n\x11\x65xternal_snapshot\x18\x04 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x36\n\x13local_snapshot_info\x18\x05 \x01(\x0b\x32\x19.ateapi.LocalSnapshotInfo\x12-\n\ractor_volumes\x18\x07 \x03(\x0b\x32\x16.ateapi.ExternalVolume\x12\'\n\x1fin_progress_local_snapshot_name\x18\x08 \x01(\t\x12\"\n\x1a\x63urrent_actor_template_uid\x18\x0b \x01(\t\"\xa7\x01\n\x10WorkerAssignment\x12!\n\x06worker\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10worker_namespace\x18\x01 \x01(\t\x12\x13\n\x0bworker_pool\x18\x02 \x01(\t\x12\x12\n\nworker_pod\x18\x03 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x04 \x01(\t\x12\x15\n\rworker_pod_ip\x18\x05 \x01(\t\"\x87\x01\n\tTagStatus\x12*\n\x08snapshot\x18\x01 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x1a\n\x12\x61\x63tor_template_uid\x18\x02 \x01(\t\x12\x18\n\x10storage_location\x18\x03 \x01(\t\x12\x18\n\x10source_actor_uid\x18\x04 \x01(\t\"\x9e\x01\n\x03Tag\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x06status\x18\x02 \x01(\x0b\x32\x11.ateapi.TagStatus\x12\x1f\n\x05scope\x18\x03 \x01(\x0e\x32\x10.ateapi.TagScope\x12\'\n\x0csource_actor\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\"6\n\x08\x41tespace\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\"+\n\tObjectRef\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xe3\x02\n\rActorTemplate\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0fworker_selector\x18\x02 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\ncontainers\x18\x03 \x03(\x0b\x32\x11.ateapi.Container\x12\x1f\n\x07volumes\x18\x04 \x03(\x0b\x32\x0e.ateapi.Volume\x12\x31\n\x10snapshots_config\x18\x05 \x01(\x0b\x32\x17.ateapi.SnapshotsConfig\x12-\n\x0esandbox_config\x18\x06 \x01(\x0b\x32\x15.ateapi.SandboxConfig\x12$\n\tresources\x18\x07 \x01(\x0b\x32\x11.ateapi.Resources\x12+\n\x06status\x18\x08 \x01(\x0b\x32\x1b.ateapi.ActorTemplateStatus\"+\n\tResources\x12\x1e\n\x06limits\x18\x01 \x03(\x0b\x32\x0e.ateapi.Limits\"(\n\x06Limits\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08quantity\x18\x02 \x01(\t\"\x9d\x01\n\x14GoldenSnapshotStatus\x12\x31\n\x0fgolden_snapshot\x18\x01 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12;\n\x17take_golden_snapshot_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rerror_message\x18\x03 \x01(\t\"S\n\x13\x41\x63torTemplateStatus\x12<\n\x16golden_snapshot_status\x18\x01 \x01(\x0b\x32\x1c.ateapi.GoldenSnapshotStatus\"Q\n\rSandboxConfig\x12+\n\rsandbox_class\x18\x01 \x01(\x0e\x32\x14.ateapi.SandboxClass\x12\x13\n\x0b\x63onfig_name\x18\x02 \x01(\t\"\xb7\x01\n\x0fSnapshotsConfig\x12.\n\x08on_pause\x18\x01 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12/\n\ton_commit\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12)\n\ton_resume\x18\x03 \x01(\x0b\x32\x16.ateapi.OnResumeConfig\x12\x18\n\x10storage_location\x18\x04 \x01(\t\"9\n\x0eOnResumeConfig\x12\'\n\tfrom_data\x18\x01 \x01(\x0e\x32\x14.ateapi.ResumeSource\"\x92\x02\n\tContainer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x0c\n\x04\x61rgs\x18\x04 \x03(\t\x12\x1b\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x0e.ateapi.EnvVar\x12\'\n\x06readyz\x18\x06 \x01(\x0b\x32\x17.ateapi.ContainerReadyz\x12*\n\rvolume_mounts\x18\x07 \x03(\x0b\x32\x13.ateapi.VolumeMount\x12\x31\n\x10security_context\x18\x08 \x01(\x0b\x32\x17.ateapi.SecurityContext\x12$\n\tresources\x18\t \x01(\x0b\x32\x11.ateapi.Resources\"=\n\x0fSecurityContext\x12*\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32\x14.ateapi.Capabilities\")\n\x0c\x43\x61pabilities\x12\x0b\n\x03\x61\x64\x64\x18\x01 \x03(\t\x12\x0c\n\x04\x64rop\x18\x02 \x03(\t\"%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"S\n\x0f\x43ontainerReadyz\x12\'\n\x08http_get\x18\x01 \x01(\x0b\x32\x15.ateapi.HTTPGetAction\x12\x17\n\x0ftimeout_seconds\x18\x02 \x01(\x05\"+\n\rHTTPGetAction\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xec\x01\n\x06Volume\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x0b\x64urable_dir\x18\x02 \x01(\x0b\x32\x1e.ateapi.DurableDirVolumeSource\x12@\n\x18\x65xternal_volume_template\x18\x03 \x01(\x0b\x32\x1e.ateapi.ExternalVolumeTemplate\x12\x33\n\x0bsystem_info\x18\x05 \x01(\x0b\x32\x1e.ateapi.SystemInfoVolumeSource\x12(\n\x05image\x18\x06 \x01(\x0b\x32\x19.ateapi.ImageVolumeSource\"&\n\x11ImageVolumeSource\x12\x11\n\treference\x18\x01 \x01(\t\"\x18\n\x16\x44urableDirVolumeSource\"F\n\x16\x45xternalVolumeTemplate\x12\x10\n\x08\x63\x61pacity\x18\x01 \x01(\t\x12\x1a\n\x12storage_class_name\x18\x02 \x01(\t\"L\n\x16SystemInfoVolumeSource\x12\x32\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x1c.ateapi.SystemInfoDataSource\"\x84\x01\n\x14SystemInfoDataSource\x12\x37\n\x0e\x61\x63tor_metadata\x18\x01 \x01(\x0b\x32\x1f.ateapi.ActorMetadataDataSource\x12\x33\n\x0ctrust_bundle\x18\x02 \x01(\x0b\x32\x1d.ateapi.TrustBundleDataSource\"C\n\x17\x41\x63torMetadataDataSource\x12(\n\x05items\x18\x01 \x03(\x0b\x32\x19.ateapi.ActorMetadataItem\"L\n\x11\x41\x63torMetadataItem\x12)\n\x05\x66ield\x18\x01 \x01(\x0e\x32\x1a.ateapi.ActorMetadataField\x12\x0c\n\x04path\x18\x02 \x01(\t\"3\n\x15TrustBundleDataSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"/\n\x0bVolumeMount\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nmount_path\x18\x02 \x01(\t\";\n\x15\x43reateAtespaceRequest\x12\"\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x10.ateapi.Atespace\"9\n\x12GetAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"=\n\x14ListAtespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"U\n\x15ListAtespacesResponse\x12#\n\tatespaces\x18\x01 \x03(\x0b\x32\x10.ateapi.Atespace\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"<\n\x15\x44\x65leteAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1a\x43reateActorTemplateRequest\x12-\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x15.ateapi.ActorTemplate\"D\n\x17GetActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorTemplatesRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorTemplatesResponse\x12.\n\x0f\x61\x63tor_templates\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorTemplate\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"G\n\x1a\x44\x65leteActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"3\n\x0fGetActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12\x43reateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"2\n\x12UpdateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"7\n\x13SuspendActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"4\n\x14SuspendActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"5\n\x11PauseActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12PauseActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"6\n\x12ResumeActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"D\n\x13ResumeActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\"I\n\x12\x44\x65leteActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tany_state\x18\x02 \x01(\x08\"?\n\x1bGetActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"o\n\x1e\x43reateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"o\n\x1eUpdateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"B\n\x1e\x44\x65leteActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"/\n\rGetTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"\\\n\x13MintActorJWTRequest\x12 \n\x05\x61\x63tor\x18\x05 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x07 \x01(\t\x12\x10\n\x08\x61udience\x18\x01 \x03(\t\")\n\x14MintActorJWTResponse\x12\x11\n\tactor_jwt\x18\x01 \x01(\t\"\xa9\x01\n\x1bMintActorCertificateRequest\x12 \n\x05\x61\x63tor\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x07 \x01(\t\x12#\n\x1b\x63\x65rtificate_signing_request\x18\x02 \x01(\x0c\x12\x30\n\x07purpose\x18\x04 \x01(\x0e\x32\x1f.ateapi.ActorCertificatePurpose\":\n\x1cMintActorCertificateResponse\x12\x1a\n\x12\x61\x63tor_certificates\x18\x01 \x03(\x0c\"D\n\x17GetActorSnapshotRequest\x12)\n\x0e\x61\x63tor_snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1aGetActorSnapshotTagRequest\x12-\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"J\n\x0fListTagsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"F\n\x10ListTagsResponse\x12\x19\n\x04tags\x18\x01 \x03(\x0b\x32\x0b.ateapi.Tag\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\",\n\x10\x43reateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\",\n\x10UpdateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\"2\n\x10\x44\x65leteTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"-\n\rDeleteOptions\x12\x0f\n\x07version\x18\x01 \x01(\x03\x12\x0b\n\x03uid\x18\x02 \x01(\t\"m\n!ListWorkerActorAssignmentsRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"q\n\"ListWorkerActorAssignmentsResponse\x12\x32\n\x11\x61\x63tor_assignments\x18\x01 \x03(\x0b\x32\x17.ateapi.ActorAssignment\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\";\n\x12ListWorkersRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"O\n\x13ListWorkersResponse\x12\x1f\n\x07workers\x18\x01 \x03(\x0b\x32\x0e.ateapi.Worker\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"5\n\x10GetWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"5\n\x13\x43reateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"5\n\x13UpdateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"`\n\x13\x44\x65leteWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12&\n\x07options\x18\x02 \x01(\x0b\x32\x15.ateapi.DeleteOptions\"7\n\x12\x44rainWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"L\n\x11ListActorsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"L\n\x12ListActorsResponse\x12\x1d\n\x06\x61\x63tors\x18\x01 \x03(\x0b\x32\r.ateapi.Actor\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"\xc6\x02\n\x06Worker\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12\x18\n\x10worker_namespace\x18\x02 \x01(\t\x12\x13\n\x0bworker_pool\x18\x03 \x01(\t\x12\x12\n\nworker_pod\x18\x04 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x05 \x01(\t\x12\x11\n\tnode_name\x18\x06 \x01(\t\x12\n\n\x02ip\x18\x07 \x01(\t\x12\x15\n\rsandbox_class\x18\x08 \x01(\t\x12*\n\x06labels\x18\t \x03(\x0b\x32\x1a.ateapi.Worker.LabelsEntry\x12$\n\x06status\x18\x0b \x01(\x0b\x32\x14.ateapi.WorkerStatus\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x89\x01\n\x0cWorkerStatus\x12\"\n\x05state\x18\x01 \x01(\x0e\x32\x13.ateapi.WorkerState\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\x12*\n\tallocated\x18\x03 \x01(\x0b\x32\x17.ateapi.WorkerResources\"G\n\x0fWorkerResources\x12$\n\tresources\x18\x01 \x01(\x0b\x32\x11.ateapi.Resources\x12\x0e\n\x06\x61\x63tors\x18\x02 \x01(\x05\"\xc7\x01\n\x0f\x41\x63torAssignment\x12*\n\x08metadata\x18\x06 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12 \n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x03 \x01(\t\x12-\n\x12\x61\x63tor_template_ref\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12$\n\tresources\x18\x05 \x01(\x0b\x32\x11.ateapi.Resources\"h\n\x18SetWorkerCapacityRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\";\n\x19SetWorkerCapacityResponse\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker*\x80\x01\n\x14SnapshotContentScope\x12&\n\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*V\n\x08TagScope\x12\x19\n\x15TAG_SCOPE_UNSPECIFIED\x10\x00\x12\x16\n\x12TAG_SCOPE_ATESPACE\x10\x01\x12\x17\n\x13TAG_SCOPE_PUBLISHED\x10\x02*\xf7\x01\n\nActorState\x12\x1b\n\x17\x41\x43TOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x41\x43TOR_STATE_RESUMING\x10\x01\x12\x17\n\x13\x41\x43TOR_STATE_RUNNING\x10\x02\x12\x1a\n\x16\x41\x43TOR_STATE_SUSPENDING\x10\x03\x12\x19\n\x15\x41\x43TOR_STATE_SUSPENDED\x10\x04\x12\x17\n\x13\x41\x43TOR_STATE_PAUSING\x10\x05\x12\x16\n\x12\x41\x43TOR_STATE_PAUSED\x10\x06\x12\x17\n\x13\x41\x43TOR_STATE_CRASHED\x10\x07\x12\x18\n\x14\x41\x43TOR_STATE_DELETING\x10\x08*b\n\x0cSandboxClass\x12\x1d\n\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n\x15SANDBOX_CLASS_MICROVM\x10\x02*d\n\x0cResumeSource\x12\x1d\n\x19RESUME_SOURCE_UNSPECIFIED\x10\x00\x12\x1b\n\x17RESUME_SOURCE_COLD_BOOT\x10\x01\x12\x18\n\x14RESUME_SOURCE_GOLDEN\x10\x02*\x9a\x01\n\x12\x41\x63torMetadataField\x12$\n ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x41\x43TOR_METADATA_FIELD_NAME\x10\x01\x12!\n\x1d\x41\x43TOR_METADATA_FIELD_ATESPACE\x10\x02\x12\x1c\n\x18\x41\x43TOR_METADATA_FIELD_UID\x10\x03*k\n\x17\x41\x63torCertificatePurpose\x12)\n%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x01*_\n\x0bWorkerState\x12\x1c\n\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n\x15WORKER_STATE_DRAINING\x10\x02\x32\xa5\x13\n\x07\x43ontrol\x12\x34\n\x08GetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0b\x43reateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0bUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\x00\x12K\n\x0cSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12\x45\n\nPauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\x00\x12H\n\x0bResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12:\n\x0b\x44\x65leteActor\x12\x1a.ateapi.DeleteActorRequest\x1a\r.ateapi.Actor\"\x00\x12S\n\x14GetActorEgressPolicy\x12#.ateapi.GetActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x43reateActorEgressPolicy\x12&.ateapi.CreateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17UpdateActorEgressPolicy\x12&.ateapi.UpdateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x44\x65leteActorEgressPolicy\x12&.ateapi.DeleteActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12K\n\x0cMintActorJWT\x12\x1b.ateapi.MintActorJWTRequest\x1a\x1c.ateapi.MintActorJWTResponse\"\x00\x12\x63\n\x14MintActorCertificate\x12#.ateapi.MintActorCertificateRequest\x1a$.ateapi.MintActorCertificateResponse\"\x00\x12\x34\n\tCreateTag\x12\x18.ateapi.CreateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12.\n\x06GetTag\x12\x15.ateapi.GetTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12?\n\x08ListTags\x12\x17.ateapi.ListTagsRequest\x1a\x18.ateapi.ListTagsResponse\"\x00\x12\x34\n\tUpdateTag\x12\x18.ateapi.UpdateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12\x34\n\tDeleteTag\x12\x18.ateapi.DeleteTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12H\n\x0bListWorkers\x12\x1a.ateapi.ListWorkersRequest\x1a\x1b.ateapi.ListWorkersResponse\"\x00\x12\x37\n\tGetWorker\x12\x18.ateapi.GetWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x43reateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0cUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x44\x65leteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n\x0b\x44rainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12u\n\x1aListWorkerActorAssignments\x12).ateapi.ListWorkerActorAssignmentsRequest\x1a*.ateapi.ListWorkerActorAssignmentsResponse\"\x00\x12\x45\n\nListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12\x43\n\x0e\x43reateAtespace\x12\x1d.ateapi.CreateAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12=\n\x0bGetAtespace\x12\x1a.ateapi.GetAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12N\n\rListAtespaces\x12\x1c.ateapi.ListAtespacesRequest\x1a\x1d.ateapi.ListAtespacesResponse\"\x00\x12\x43\n\x0e\x44\x65leteAtespace\x12\x1d.ateapi.DeleteAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12R\n\x13\x43reateActorTemplate\x12\".ateapi.CreateActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12L\n\x10GetActorTemplate\x12\x1f.ateapi.GetActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12]\n\x12ListActorTemplates\x12!.ateapi.ListActorTemplatesRequest\x1a\".ateapi.ListActorTemplatesResponse\"\x00\x12R\n\x13\x44\x65leteActorTemplate\x12\".ateapi.DeleteActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x32i\n\rWorkerService\x12X\n\x11SetWorkerCapacity\x12 .ateapi.SetWorkerCapacityRequest\x1a!.ateapi.SetWorkerCapacityResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61teapi.proto\x12\x06\x61teapi\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"]\n\x10\x45xternalSnapshot\x12\x14\n\x0csnapshot_uri\x18\x01 \x01(\t\x12\x33\n\rcontent_scope\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"\x86\x01\n\x11LocalSnapshotInfo\x12\x15\n\rsnapshot_name\x18\x01 \x01(\t\x12%\n\x1dnode_vms_with_local_snapshots\x18\x02 \x03(\t\x12\x33\n\rcontent_scope\x18\x03 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"w\n\x08Selector\x12\x37\n\x0cmatch_labels\x18\x01 \x03(\x0b\x32!.ateapi.Selector.MatchLabelsEntry\x1a\x32\n\x10MatchLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb2\x01\n\x10ResourceMetadata\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0bupdate_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xdc\x02\n\x0e\x45xternalVolume\x12\x13\n\x0bvolume_name\x18\x01 \x01(\t\x12\x19\n\x11storage_volume_id\x18\x02 \x01(\t\x12\x13\n\x0bvolume_type\x18\x03 \x01(\t\x12-\n\x06status\x18\x04 \x01(\x0e\x32\x1d.ateapi.ExternalVolume.Status\x12\x41\n\x0evolume_context\x18\x05 \x03(\x0b\x32).ateapi.ExternalVolume.VolumeContextEntry\x1a\x34\n\x12VolumeContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x12\n\x0eSTATUS_CREATED\x10\x02\x12\x13\n\x0fSTATUS_DELETING\x10\x03\"\xd5\x01\n\x05\x41\x63tor\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0e\x61\x63tor_template\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x0fworker_selector\x18\x05 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\nsource_tag\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x06status\x18\x07 \x01(\x0b\x32\x13.ateapi.ActorStatus\"]\n\x0c\x45gressPolicy\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x05rules\x18\x02 \x03(\x0b\x32\x12.ateapi.EgressRule\"{\n\nEgressRule\x12\'\n\thostnames\x18\x01 \x01(\x0b\x32\x14.ateapi.HostnameRule\x12\x1f\n\x05\x63idrs\x18\x02 \x01(\x0b\x32\x10.ateapi.CIDRRule\x12#\n\x03\x61ll\x18\x03 \x01(\x0b\x32\x16.google.protobuf.Empty\"L\n\x0cHostnameRule\x12\x10\n\x08patterns\x18\x01 \x03(\t\x12*\n\x07\x65\x66\x66\x65\x63ts\x18\x02 \x01(\x0b\x32\x19.ateapi.EgressRuleEffects\"\x19\n\x08\x43IDRRule\x12\r\n\x05\x63idrs\x18\x01 \x03(\t\"U\n\x11\x45gressRuleEffects\x12@\n\x15inject_static_headers\x18\x01 \x03(\x0b\x32!.ateapi.CredentialHeaderInjection\"S\n\x19\x43redentialHeaderInjection\x12\x0e\n\x06header\x18\x01 \x01(\t\x12\x0e\n\x06prefix\x18\x02 \x01(\t\x12\x16\n\x0e\x63redential_uri\x18\x03 \x01(\t\"\xf1\x02\n\x0b\x41\x63torStatus\x12!\n\x05state\x18\x01 \x01(\x0e\x32\x12.ateapi.ActorState\x12\x33\n\x11worker_assignment\x18\x02 \x01(\x0b\x32\x18.ateapi.WorkerAssignment\x12!\n\x19in_progress_snapshot_name\x18\x03 \x01(\t\x12\x33\n\x11\x65xternal_snapshot\x18\x04 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x36\n\x13local_snapshot_info\x18\x05 \x01(\x0b\x32\x19.ateapi.LocalSnapshotInfo\x12-\n\ractor_volumes\x18\x07 \x03(\x0b\x32\x16.ateapi.ExternalVolume\x12\'\n\x1fin_progress_local_snapshot_name\x18\x08 \x01(\t\x12\"\n\x1a\x63urrent_actor_template_uid\x18\x0b \x01(\t\"\xa7\x01\n\x10WorkerAssignment\x12!\n\x06worker\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10worker_namespace\x18\x01 \x01(\t\x12\x13\n\x0bworker_pool\x18\x02 \x01(\t\x12\x12\n\nworker_pod\x18\x03 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x04 \x01(\t\x12\x15\n\rworker_pod_ip\x18\x05 \x01(\t\"\x87\x01\n\tTagStatus\x12*\n\x08snapshot\x18\x01 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x1a\n\x12\x61\x63tor_template_uid\x18\x02 \x01(\t\x12\x18\n\x10storage_location\x18\x03 \x01(\t\x12\x18\n\x10source_actor_uid\x18\x04 \x01(\t\"\x9e\x01\n\x03Tag\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x06status\x18\x02 \x01(\x0b\x32\x11.ateapi.TagStatus\x12\x1f\n\x05scope\x18\x03 \x01(\x0e\x32\x10.ateapi.TagScope\x12\'\n\x0csource_actor\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\"6\n\x08\x41tespace\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\"+\n\tObjectRef\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xe3\x02\n\rActorTemplate\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0fworker_selector\x18\x02 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\ncontainers\x18\x03 \x03(\x0b\x32\x11.ateapi.Container\x12\x1f\n\x07volumes\x18\x04 \x03(\x0b\x32\x0e.ateapi.Volume\x12\x31\n\x10snapshots_config\x18\x05 \x01(\x0b\x32\x17.ateapi.SnapshotsConfig\x12-\n\x0esandbox_config\x18\x06 \x01(\x0b\x32\x15.ateapi.SandboxConfig\x12$\n\tresources\x18\x07 \x01(\x0b\x32\x11.ateapi.Resources\x12+\n\x06status\x18\x08 \x01(\x0b\x32\x1b.ateapi.ActorTemplateStatus\"+\n\tResources\x12\x1e\n\x06limits\x18\x01 \x03(\x0b\x32\x0e.ateapi.Limits\"(\n\x06Limits\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08quantity\x18\x02 \x01(\t\"\x91\x01\n\x14GoldenSnapshotStatus\x12%\n\ngolden_tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12;\n\x17take_golden_snapshot_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rerror_message\x18\x03 \x01(\t\"S\n\x13\x41\x63torTemplateStatus\x12<\n\x16golden_snapshot_status\x18\x01 \x01(\x0b\x32\x1c.ateapi.GoldenSnapshotStatus\"Q\n\rSandboxConfig\x12+\n\rsandbox_class\x18\x01 \x01(\x0e\x32\x14.ateapi.SandboxClass\x12\x13\n\x0b\x63onfig_name\x18\x02 \x01(\t\"\xb7\x01\n\x0fSnapshotsConfig\x12.\n\x08on_pause\x18\x01 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12/\n\ton_commit\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12)\n\ton_resume\x18\x03 \x01(\x0b\x32\x16.ateapi.OnResumeConfig\x12\x18\n\x10storage_location\x18\x04 \x01(\t\"9\n\x0eOnResumeConfig\x12\'\n\tfrom_data\x18\x01 \x01(\x0e\x32\x14.ateapi.ResumeSource\"\x92\x02\n\tContainer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x0c\n\x04\x61rgs\x18\x04 \x03(\t\x12\x1b\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x0e.ateapi.EnvVar\x12\'\n\x06readyz\x18\x06 \x01(\x0b\x32\x17.ateapi.ContainerReadyz\x12*\n\rvolume_mounts\x18\x07 \x03(\x0b\x32\x13.ateapi.VolumeMount\x12\x31\n\x10security_context\x18\x08 \x01(\x0b\x32\x17.ateapi.SecurityContext\x12$\n\tresources\x18\t \x01(\x0b\x32\x11.ateapi.Resources\"=\n\x0fSecurityContext\x12*\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32\x14.ateapi.Capabilities\")\n\x0c\x43\x61pabilities\x12\x0b\n\x03\x61\x64\x64\x18\x01 \x03(\t\x12\x0c\n\x04\x64rop\x18\x02 \x03(\t\"%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"S\n\x0f\x43ontainerReadyz\x12\'\n\x08http_get\x18\x01 \x01(\x0b\x32\x15.ateapi.HTTPGetAction\x12\x17\n\x0ftimeout_seconds\x18\x02 \x01(\x05\"+\n\rHTTPGetAction\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xec\x01\n\x06Volume\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x0b\x64urable_dir\x18\x02 \x01(\x0b\x32\x1e.ateapi.DurableDirVolumeSource\x12@\n\x18\x65xternal_volume_template\x18\x03 \x01(\x0b\x32\x1e.ateapi.ExternalVolumeTemplate\x12\x33\n\x0bsystem_info\x18\x05 \x01(\x0b\x32\x1e.ateapi.SystemInfoVolumeSource\x12(\n\x05image\x18\x06 \x01(\x0b\x32\x19.ateapi.ImageVolumeSource\"&\n\x11ImageVolumeSource\x12\x11\n\treference\x18\x01 \x01(\t\"\x18\n\x16\x44urableDirVolumeSource\"F\n\x16\x45xternalVolumeTemplate\x12\x10\n\x08\x63\x61pacity\x18\x01 \x01(\t\x12\x1a\n\x12storage_class_name\x18\x02 \x01(\t\"L\n\x16SystemInfoVolumeSource\x12\x32\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x1c.ateapi.SystemInfoDataSource\"\x84\x01\n\x14SystemInfoDataSource\x12\x37\n\x0e\x61\x63tor_metadata\x18\x01 \x01(\x0b\x32\x1f.ateapi.ActorMetadataDataSource\x12\x33\n\x0ctrust_bundle\x18\x02 \x01(\x0b\x32\x1d.ateapi.TrustBundleDataSource\"C\n\x17\x41\x63torMetadataDataSource\x12(\n\x05items\x18\x01 \x03(\x0b\x32\x19.ateapi.ActorMetadataItem\"L\n\x11\x41\x63torMetadataItem\x12)\n\x05\x66ield\x18\x01 \x01(\x0e\x32\x1a.ateapi.ActorMetadataField\x12\x0c\n\x04path\x18\x02 \x01(\t\"3\n\x15TrustBundleDataSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"/\n\x0bVolumeMount\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nmount_path\x18\x02 \x01(\t\";\n\x15\x43reateAtespaceRequest\x12\"\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x10.ateapi.Atespace\"9\n\x12GetAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"=\n\x14ListAtespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"U\n\x15ListAtespacesResponse\x12#\n\tatespaces\x18\x01 \x03(\x0b\x32\x10.ateapi.Atespace\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"<\n\x15\x44\x65leteAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1a\x43reateActorTemplateRequest\x12-\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x15.ateapi.ActorTemplate\"D\n\x17GetActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorTemplatesRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorTemplatesResponse\x12.\n\x0f\x61\x63tor_templates\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorTemplate\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"G\n\x1a\x44\x65leteActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"3\n\x0fGetActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12\x43reateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"2\n\x12UpdateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"7\n\x13SuspendActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"4\n\x14SuspendActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"5\n\x11PauseActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12PauseActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"6\n\x12ResumeActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"D\n\x13ResumeActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\"I\n\x12\x44\x65leteActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tany_state\x18\x02 \x01(\x08\"?\n\x1bGetActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"o\n\x1e\x43reateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"o\n\x1eUpdateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"B\n\x1e\x44\x65leteActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"/\n\rGetTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"\\\n\x13MintActorJWTRequest\x12 \n\x05\x61\x63tor\x18\x05 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x07 \x01(\t\x12\x10\n\x08\x61udience\x18\x01 \x03(\t\")\n\x14MintActorJWTResponse\x12\x11\n\tactor_jwt\x18\x01 \x01(\t\"\xa9\x01\n\x1bMintActorCertificateRequest\x12 \n\x05\x61\x63tor\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x07 \x01(\t\x12#\n\x1b\x63\x65rtificate_signing_request\x18\x02 \x01(\x0c\x12\x30\n\x07purpose\x18\x04 \x01(\x0e\x32\x1f.ateapi.ActorCertificatePurpose\":\n\x1cMintActorCertificateResponse\x12\x1a\n\x12\x61\x63tor_certificates\x18\x01 \x03(\x0c\"D\n\x17GetActorSnapshotRequest\x12)\n\x0e\x61\x63tor_snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1aGetActorSnapshotTagRequest\x12-\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"J\n\x0fListTagsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"F\n\x10ListTagsResponse\x12\x19\n\x04tags\x18\x01 \x03(\x0b\x32\x0b.ateapi.Tag\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\",\n\x10\x43reateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\",\n\x10UpdateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\"2\n\x10\x44\x65leteTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"-\n\rDeleteOptions\x12\x0f\n\x07version\x18\x01 \x01(\x03\x12\x0b\n\x03uid\x18\x02 \x01(\t\"m\n!ListWorkerActorAssignmentsRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"q\n\"ListWorkerActorAssignmentsResponse\x12\x32\n\x11\x61\x63tor_assignments\x18\x01 \x03(\x0b\x32\x17.ateapi.ActorAssignment\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\";\n\x12ListWorkersRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"O\n\x13ListWorkersResponse\x12\x1f\n\x07workers\x18\x01 \x03(\x0b\x32\x0e.ateapi.Worker\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"5\n\x10GetWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"5\n\x13\x43reateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"5\n\x13UpdateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"`\n\x13\x44\x65leteWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12&\n\x07options\x18\x02 \x01(\x0b\x32\x15.ateapi.DeleteOptions\"7\n\x12\x44rainWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"L\n\x11ListActorsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"L\n\x12ListActorsResponse\x12\x1d\n\x06\x61\x63tors\x18\x01 \x03(\x0b\x32\r.ateapi.Actor\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"\xc6\x02\n\x06Worker\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12\x18\n\x10worker_namespace\x18\x02 \x01(\t\x12\x13\n\x0bworker_pool\x18\x03 \x01(\t\x12\x12\n\nworker_pod\x18\x04 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x05 \x01(\t\x12\x11\n\tnode_name\x18\x06 \x01(\t\x12\n\n\x02ip\x18\x07 \x01(\t\x12\x15\n\rsandbox_class\x18\x08 \x01(\t\x12*\n\x06labels\x18\t \x03(\x0b\x32\x1a.ateapi.Worker.LabelsEntry\x12$\n\x06status\x18\x0b \x01(\x0b\x32\x14.ateapi.WorkerStatus\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x89\x01\n\x0cWorkerStatus\x12\"\n\x05state\x18\x01 \x01(\x0e\x32\x13.ateapi.WorkerState\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\x12*\n\tallocated\x18\x03 \x01(\x0b\x32\x17.ateapi.WorkerResources\"G\n\x0fWorkerResources\x12$\n\tresources\x18\x01 \x01(\x0b\x32\x11.ateapi.Resources\x12\x0e\n\x06\x61\x63tors\x18\x02 \x01(\x05\"\xc7\x01\n\x0f\x41\x63torAssignment\x12*\n\x08metadata\x18\x06 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12 \n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x03 \x01(\t\x12-\n\x12\x61\x63tor_template_ref\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12$\n\tresources\x18\x05 \x01(\x0b\x32\x11.ateapi.Resources\"h\n\x18SetWorkerCapacityRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\";\n\x19SetWorkerCapacityResponse\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker*\x80\x01\n\x14SnapshotContentScope\x12&\n\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*V\n\x08TagScope\x12\x19\n\x15TAG_SCOPE_UNSPECIFIED\x10\x00\x12\x16\n\x12TAG_SCOPE_ATESPACE\x10\x01\x12\x17\n\x13TAG_SCOPE_PUBLISHED\x10\x02*\xf7\x01\n\nActorState\x12\x1b\n\x17\x41\x43TOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x41\x43TOR_STATE_RESUMING\x10\x01\x12\x17\n\x13\x41\x43TOR_STATE_RUNNING\x10\x02\x12\x1a\n\x16\x41\x43TOR_STATE_SUSPENDING\x10\x03\x12\x19\n\x15\x41\x43TOR_STATE_SUSPENDED\x10\x04\x12\x17\n\x13\x41\x43TOR_STATE_PAUSING\x10\x05\x12\x16\n\x12\x41\x43TOR_STATE_PAUSED\x10\x06\x12\x17\n\x13\x41\x43TOR_STATE_CRASHED\x10\x07\x12\x18\n\x14\x41\x43TOR_STATE_DELETING\x10\x08*b\n\x0cSandboxClass\x12\x1d\n\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n\x15SANDBOX_CLASS_MICROVM\x10\x02*d\n\x0cResumeSource\x12\x1d\n\x19RESUME_SOURCE_UNSPECIFIED\x10\x00\x12\x1b\n\x17RESUME_SOURCE_COLD_BOOT\x10\x01\x12\x18\n\x14RESUME_SOURCE_GOLDEN\x10\x02*\x9a\x01\n\x12\x41\x63torMetadataField\x12$\n ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x41\x43TOR_METADATA_FIELD_NAME\x10\x01\x12!\n\x1d\x41\x43TOR_METADATA_FIELD_ATESPACE\x10\x02\x12\x1c\n\x18\x41\x43TOR_METADATA_FIELD_UID\x10\x03*k\n\x17\x41\x63torCertificatePurpose\x12)\n%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x01*_\n\x0bWorkerState\x12\x1c\n\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n\x15WORKER_STATE_DRAINING\x10\x02\x32\xa5\x13\n\x07\x43ontrol\x12\x34\n\x08GetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0b\x43reateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0bUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\x00\x12K\n\x0cSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12\x45\n\nPauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\x00\x12H\n\x0bResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12:\n\x0b\x44\x65leteActor\x12\x1a.ateapi.DeleteActorRequest\x1a\r.ateapi.Actor\"\x00\x12S\n\x14GetActorEgressPolicy\x12#.ateapi.GetActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x43reateActorEgressPolicy\x12&.ateapi.CreateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17UpdateActorEgressPolicy\x12&.ateapi.UpdateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x44\x65leteActorEgressPolicy\x12&.ateapi.DeleteActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12K\n\x0cMintActorJWT\x12\x1b.ateapi.MintActorJWTRequest\x1a\x1c.ateapi.MintActorJWTResponse\"\x00\x12\x63\n\x14MintActorCertificate\x12#.ateapi.MintActorCertificateRequest\x1a$.ateapi.MintActorCertificateResponse\"\x00\x12\x34\n\tCreateTag\x12\x18.ateapi.CreateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12.\n\x06GetTag\x12\x15.ateapi.GetTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12?\n\x08ListTags\x12\x17.ateapi.ListTagsRequest\x1a\x18.ateapi.ListTagsResponse\"\x00\x12\x34\n\tUpdateTag\x12\x18.ateapi.UpdateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12\x34\n\tDeleteTag\x12\x18.ateapi.DeleteTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12H\n\x0bListWorkers\x12\x1a.ateapi.ListWorkersRequest\x1a\x1b.ateapi.ListWorkersResponse\"\x00\x12\x37\n\tGetWorker\x12\x18.ateapi.GetWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x43reateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0cUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x44\x65leteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n\x0b\x44rainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12u\n\x1aListWorkerActorAssignments\x12).ateapi.ListWorkerActorAssignmentsRequest\x1a*.ateapi.ListWorkerActorAssignmentsResponse\"\x00\x12\x45\n\nListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12\x43\n\x0e\x43reateAtespace\x12\x1d.ateapi.CreateAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12=\n\x0bGetAtespace\x12\x1a.ateapi.GetAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12N\n\rListAtespaces\x12\x1c.ateapi.ListAtespacesRequest\x1a\x1d.ateapi.ListAtespacesResponse\"\x00\x12\x43\n\x0e\x44\x65leteAtespace\x12\x1d.ateapi.DeleteAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12R\n\x13\x43reateActorTemplate\x12\".ateapi.CreateActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12L\n\x10GetActorTemplate\x12\x1f.ateapi.GetActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12]\n\x12ListActorTemplates\x12!.ateapi.ListActorTemplatesRequest\x1a\".ateapi.ListActorTemplatesResponse\"\x00\x12R\n\x13\x44\x65leteActorTemplate\x12\".ateapi.DeleteActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x32i\n\rWorkerService\x12X\n\x11SetWorkerCapacity\x12 .ateapi.SetWorkerCapacityRequest\x1a!.ateapi.SetWorkerCapacityResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -54,22 +54,22 @@ _globals['_EXTERNALVOLUME_VOLUMECONTEXTENTRY']._serialized_options = b'8\001' _globals['_WORKER_LABELSENTRY']._loaded_options = None _globals['_WORKER_LABELSENTRY']._serialized_options = b'8\001' - _globals['_SNAPSHOTCONTENTSCOPE']._serialized_start=9371 - _globals['_SNAPSHOTCONTENTSCOPE']._serialized_end=9499 - _globals['_TAGSCOPE']._serialized_start=9501 - _globals['_TAGSCOPE']._serialized_end=9587 - _globals['_ACTORSTATE']._serialized_start=9590 - _globals['_ACTORSTATE']._serialized_end=9837 - _globals['_SANDBOXCLASS']._serialized_start=9839 - _globals['_SANDBOXCLASS']._serialized_end=9937 - _globals['_RESUMESOURCE']._serialized_start=9939 - _globals['_RESUMESOURCE']._serialized_end=10039 - _globals['_ACTORMETADATAFIELD']._serialized_start=10042 - _globals['_ACTORMETADATAFIELD']._serialized_end=10196 - _globals['_ACTORCERTIFICATEPURPOSE']._serialized_start=10198 - _globals['_ACTORCERTIFICATEPURPOSE']._serialized_end=10305 - _globals['_WORKERSTATE']._serialized_start=10307 - _globals['_WORKERSTATE']._serialized_end=10402 + _globals['_SNAPSHOTCONTENTSCOPE']._serialized_start=9359 + _globals['_SNAPSHOTCONTENTSCOPE']._serialized_end=9487 + _globals['_TAGSCOPE']._serialized_start=9489 + _globals['_TAGSCOPE']._serialized_end=9575 + _globals['_ACTORSTATE']._serialized_start=9578 + _globals['_ACTORSTATE']._serialized_end=9825 + _globals['_SANDBOXCLASS']._serialized_start=9827 + _globals['_SANDBOXCLASS']._serialized_end=9925 + _globals['_RESUMESOURCE']._serialized_start=9927 + _globals['_RESUMESOURCE']._serialized_end=10027 + _globals['_ACTORMETADATAFIELD']._serialized_start=10030 + _globals['_ACTORMETADATAFIELD']._serialized_end=10184 + _globals['_ACTORCERTIFICATEPURPOSE']._serialized_start=10186 + _globals['_ACTORCERTIFICATEPURPOSE']._serialized_end=10293 + _globals['_WORKERSTATE']._serialized_start=10295 + _globals['_WORKERSTATE']._serialized_end=10390 _globals['_EXTERNALSNAPSHOT']._serialized_start=86 _globals['_EXTERNALSNAPSHOT']._serialized_end=179 _globals['_LOCALSNAPSHOTINFO']._serialized_start=182 @@ -119,159 +119,159 @@ _globals['_LIMITS']._serialized_start=3029 _globals['_LIMITS']._serialized_end=3069 _globals['_GOLDENSNAPSHOTSTATUS']._serialized_start=3072 - _globals['_GOLDENSNAPSHOTSTATUS']._serialized_end=3229 - _globals['_ACTORTEMPLATESTATUS']._serialized_start=3231 - _globals['_ACTORTEMPLATESTATUS']._serialized_end=3314 - _globals['_SANDBOXCONFIG']._serialized_start=3316 - _globals['_SANDBOXCONFIG']._serialized_end=3397 - _globals['_SNAPSHOTSCONFIG']._serialized_start=3400 - _globals['_SNAPSHOTSCONFIG']._serialized_end=3583 - _globals['_ONRESUMECONFIG']._serialized_start=3585 - _globals['_ONRESUMECONFIG']._serialized_end=3642 - _globals['_CONTAINER']._serialized_start=3645 - _globals['_CONTAINER']._serialized_end=3919 - _globals['_SECURITYCONTEXT']._serialized_start=3921 - _globals['_SECURITYCONTEXT']._serialized_end=3982 - _globals['_CAPABILITIES']._serialized_start=3984 - _globals['_CAPABILITIES']._serialized_end=4025 - _globals['_ENVVAR']._serialized_start=4027 - _globals['_ENVVAR']._serialized_end=4064 - _globals['_CONTAINERREADYZ']._serialized_start=4066 - _globals['_CONTAINERREADYZ']._serialized_end=4149 - _globals['_HTTPGETACTION']._serialized_start=4151 - _globals['_HTTPGETACTION']._serialized_end=4194 - _globals['_VOLUME']._serialized_start=4197 - _globals['_VOLUME']._serialized_end=4433 - _globals['_IMAGEVOLUMESOURCE']._serialized_start=4435 - _globals['_IMAGEVOLUMESOURCE']._serialized_end=4473 - _globals['_DURABLEDIRVOLUMESOURCE']._serialized_start=4475 - _globals['_DURABLEDIRVOLUMESOURCE']._serialized_end=4499 - _globals['_EXTERNALVOLUMETEMPLATE']._serialized_start=4501 - _globals['_EXTERNALVOLUMETEMPLATE']._serialized_end=4571 - _globals['_SYSTEMINFOVOLUMESOURCE']._serialized_start=4573 - _globals['_SYSTEMINFOVOLUMESOURCE']._serialized_end=4649 - _globals['_SYSTEMINFODATASOURCE']._serialized_start=4652 - _globals['_SYSTEMINFODATASOURCE']._serialized_end=4784 - _globals['_ACTORMETADATADATASOURCE']._serialized_start=4786 - _globals['_ACTORMETADATADATASOURCE']._serialized_end=4853 - _globals['_ACTORMETADATAITEM']._serialized_start=4855 - _globals['_ACTORMETADATAITEM']._serialized_end=4931 - _globals['_TRUSTBUNDLEDATASOURCE']._serialized_start=4933 - _globals['_TRUSTBUNDLEDATASOURCE']._serialized_end=4984 - _globals['_VOLUMEMOUNT']._serialized_start=4986 - _globals['_VOLUMEMOUNT']._serialized_end=5033 - _globals['_CREATEATESPACEREQUEST']._serialized_start=5035 - _globals['_CREATEATESPACEREQUEST']._serialized_end=5094 - _globals['_GETATESPACEREQUEST']._serialized_start=5096 - _globals['_GETATESPACEREQUEST']._serialized_end=5153 - _globals['_LISTATESPACESREQUEST']._serialized_start=5155 - _globals['_LISTATESPACESREQUEST']._serialized_end=5216 - _globals['_LISTATESPACESRESPONSE']._serialized_start=5218 - _globals['_LISTATESPACESRESPONSE']._serialized_end=5303 - _globals['_DELETEATESPACEREQUEST']._serialized_start=5305 - _globals['_DELETEATESPACEREQUEST']._serialized_end=5365 - _globals['_CREATEACTORTEMPLATEREQUEST']._serialized_start=5367 - _globals['_CREATEACTORTEMPLATEREQUEST']._serialized_end=5442 - _globals['_GETACTORTEMPLATEREQUEST']._serialized_start=5444 - _globals['_GETACTORTEMPLATEREQUEST']._serialized_end=5512 - _globals['_LISTACTORTEMPLATESREQUEST']._serialized_start=5514 - _globals['_LISTACTORTEMPLATESREQUEST']._serialized_end=5598 - _globals['_LISTACTORTEMPLATESRESPONSE']._serialized_start=5600 - _globals['_LISTACTORTEMPLATESRESPONSE']._serialized_end=5701 - _globals['_DELETEACTORTEMPLATEREQUEST']._serialized_start=5703 - _globals['_DELETEACTORTEMPLATEREQUEST']._serialized_end=5774 - _globals['_GETACTORREQUEST']._serialized_start=5776 - _globals['_GETACTORREQUEST']._serialized_end=5827 - _globals['_CREATEACTORREQUEST']._serialized_start=5829 - _globals['_CREATEACTORREQUEST']._serialized_end=5879 - _globals['_UPDATEACTORREQUEST']._serialized_start=5881 - _globals['_UPDATEACTORREQUEST']._serialized_end=5931 - _globals['_SUSPENDACTORREQUEST']._serialized_start=5933 - _globals['_SUSPENDACTORREQUEST']._serialized_end=5988 - _globals['_SUSPENDACTORRESPONSE']._serialized_start=5990 - _globals['_SUSPENDACTORRESPONSE']._serialized_end=6042 - _globals['_PAUSEACTORREQUEST']._serialized_start=6044 - _globals['_PAUSEACTORREQUEST']._serialized_end=6097 - _globals['_PAUSEACTORRESPONSE']._serialized_start=6099 - _globals['_PAUSEACTORRESPONSE']._serialized_end=6149 - _globals['_RESUMEACTORREQUEST']._serialized_start=6151 - _globals['_RESUMEACTORREQUEST']._serialized_end=6205 - _globals['_RESUMEACTORRESPONSE']._serialized_start=6207 - _globals['_RESUMEACTORRESPONSE']._serialized_end=6275 - _globals['_DELETEACTORREQUEST']._serialized_start=6277 - _globals['_DELETEACTORREQUEST']._serialized_end=6350 - _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_start=6352 - _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_end=6415 - _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_start=6417 - _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_end=6528 - _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_start=6530 - _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_end=6641 - _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_start=6643 - _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_end=6709 - _globals['_GETTAGREQUEST']._serialized_start=6711 - _globals['_GETTAGREQUEST']._serialized_end=6758 - _globals['_MINTACTORJWTREQUEST']._serialized_start=6760 - _globals['_MINTACTORJWTREQUEST']._serialized_end=6852 - _globals['_MINTACTORJWTRESPONSE']._serialized_start=6854 - _globals['_MINTACTORJWTRESPONSE']._serialized_end=6895 - _globals['_MINTACTORCERTIFICATEREQUEST']._serialized_start=6898 - _globals['_MINTACTORCERTIFICATEREQUEST']._serialized_end=7067 - _globals['_MINTACTORCERTIFICATERESPONSE']._serialized_start=7069 - _globals['_MINTACTORCERTIFICATERESPONSE']._serialized_end=7127 - _globals['_GETACTORSNAPSHOTREQUEST']._serialized_start=7129 - _globals['_GETACTORSNAPSHOTREQUEST']._serialized_end=7197 - _globals['_GETACTORSNAPSHOTTAGREQUEST']._serialized_start=7199 - _globals['_GETACTORSNAPSHOTTAGREQUEST']._serialized_end=7274 - _globals['_LISTTAGSREQUEST']._serialized_start=7276 - _globals['_LISTTAGSREQUEST']._serialized_end=7350 - _globals['_LISTTAGSRESPONSE']._serialized_start=7352 - _globals['_LISTTAGSRESPONSE']._serialized_end=7422 - _globals['_CREATETAGREQUEST']._serialized_start=7424 - _globals['_CREATETAGREQUEST']._serialized_end=7468 - _globals['_UPDATETAGREQUEST']._serialized_start=7470 - _globals['_UPDATETAGREQUEST']._serialized_end=7514 - _globals['_DELETETAGREQUEST']._serialized_start=7516 - _globals['_DELETETAGREQUEST']._serialized_end=7566 - _globals['_DELETEOPTIONS']._serialized_start=7568 - _globals['_DELETEOPTIONS']._serialized_end=7613 - _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_start=7615 - _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_end=7724 - _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_start=7726 - _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_end=7839 - _globals['_LISTWORKERSREQUEST']._serialized_start=7841 - _globals['_LISTWORKERSREQUEST']._serialized_end=7900 - _globals['_LISTWORKERSRESPONSE']._serialized_start=7902 - _globals['_LISTWORKERSRESPONSE']._serialized_end=7981 - _globals['_GETWORKERREQUEST']._serialized_start=7983 - _globals['_GETWORKERREQUEST']._serialized_end=8036 - _globals['_CREATEWORKERREQUEST']._serialized_start=8038 - _globals['_CREATEWORKERREQUEST']._serialized_end=8091 - _globals['_UPDATEWORKERREQUEST']._serialized_start=8093 - _globals['_UPDATEWORKERREQUEST']._serialized_end=8146 - _globals['_DELETEWORKERREQUEST']._serialized_start=8148 - _globals['_DELETEWORKERREQUEST']._serialized_end=8244 - _globals['_DRAINWORKERREQUEST']._serialized_start=8246 - _globals['_DRAINWORKERREQUEST']._serialized_end=8301 - _globals['_LISTACTORSREQUEST']._serialized_start=8303 - _globals['_LISTACTORSREQUEST']._serialized_end=8379 - _globals['_LISTACTORSRESPONSE']._serialized_start=8381 - _globals['_LISTACTORSRESPONSE']._serialized_end=8457 - _globals['_WORKER']._serialized_start=8460 - _globals['_WORKER']._serialized_end=8786 - _globals['_WORKER_LABELSENTRY']._serialized_start=8741 - _globals['_WORKER_LABELSENTRY']._serialized_end=8786 - _globals['_WORKERSTATUS']._serialized_start=8789 - _globals['_WORKERSTATUS']._serialized_end=8926 - _globals['_WORKERRESOURCES']._serialized_start=8928 - _globals['_WORKERRESOURCES']._serialized_end=8999 - _globals['_ACTORASSIGNMENT']._serialized_start=9002 - _globals['_ACTORASSIGNMENT']._serialized_end=9201 - _globals['_SETWORKERCAPACITYREQUEST']._serialized_start=9203 - _globals['_SETWORKERCAPACITYREQUEST']._serialized_end=9307 - _globals['_SETWORKERCAPACITYRESPONSE']._serialized_start=9309 - _globals['_SETWORKERCAPACITYRESPONSE']._serialized_end=9368 - _globals['_CONTROL']._serialized_start=10405 - _globals['_CONTROL']._serialized_end=12874 - _globals['_WORKERSERVICE']._serialized_start=12876 - _globals['_WORKERSERVICE']._serialized_end=12981 + _globals['_GOLDENSNAPSHOTSTATUS']._serialized_end=3217 + _globals['_ACTORTEMPLATESTATUS']._serialized_start=3219 + _globals['_ACTORTEMPLATESTATUS']._serialized_end=3302 + _globals['_SANDBOXCONFIG']._serialized_start=3304 + _globals['_SANDBOXCONFIG']._serialized_end=3385 + _globals['_SNAPSHOTSCONFIG']._serialized_start=3388 + _globals['_SNAPSHOTSCONFIG']._serialized_end=3571 + _globals['_ONRESUMECONFIG']._serialized_start=3573 + _globals['_ONRESUMECONFIG']._serialized_end=3630 + _globals['_CONTAINER']._serialized_start=3633 + _globals['_CONTAINER']._serialized_end=3907 + _globals['_SECURITYCONTEXT']._serialized_start=3909 + _globals['_SECURITYCONTEXT']._serialized_end=3970 + _globals['_CAPABILITIES']._serialized_start=3972 + _globals['_CAPABILITIES']._serialized_end=4013 + _globals['_ENVVAR']._serialized_start=4015 + _globals['_ENVVAR']._serialized_end=4052 + _globals['_CONTAINERREADYZ']._serialized_start=4054 + _globals['_CONTAINERREADYZ']._serialized_end=4137 + _globals['_HTTPGETACTION']._serialized_start=4139 + _globals['_HTTPGETACTION']._serialized_end=4182 + _globals['_VOLUME']._serialized_start=4185 + _globals['_VOLUME']._serialized_end=4421 + _globals['_IMAGEVOLUMESOURCE']._serialized_start=4423 + _globals['_IMAGEVOLUMESOURCE']._serialized_end=4461 + _globals['_DURABLEDIRVOLUMESOURCE']._serialized_start=4463 + _globals['_DURABLEDIRVOLUMESOURCE']._serialized_end=4487 + _globals['_EXTERNALVOLUMETEMPLATE']._serialized_start=4489 + _globals['_EXTERNALVOLUMETEMPLATE']._serialized_end=4559 + _globals['_SYSTEMINFOVOLUMESOURCE']._serialized_start=4561 + _globals['_SYSTEMINFOVOLUMESOURCE']._serialized_end=4637 + _globals['_SYSTEMINFODATASOURCE']._serialized_start=4640 + _globals['_SYSTEMINFODATASOURCE']._serialized_end=4772 + _globals['_ACTORMETADATADATASOURCE']._serialized_start=4774 + _globals['_ACTORMETADATADATASOURCE']._serialized_end=4841 + _globals['_ACTORMETADATAITEM']._serialized_start=4843 + _globals['_ACTORMETADATAITEM']._serialized_end=4919 + _globals['_TRUSTBUNDLEDATASOURCE']._serialized_start=4921 + _globals['_TRUSTBUNDLEDATASOURCE']._serialized_end=4972 + _globals['_VOLUMEMOUNT']._serialized_start=4974 + _globals['_VOLUMEMOUNT']._serialized_end=5021 + _globals['_CREATEATESPACEREQUEST']._serialized_start=5023 + _globals['_CREATEATESPACEREQUEST']._serialized_end=5082 + _globals['_GETATESPACEREQUEST']._serialized_start=5084 + _globals['_GETATESPACEREQUEST']._serialized_end=5141 + _globals['_LISTATESPACESREQUEST']._serialized_start=5143 + _globals['_LISTATESPACESREQUEST']._serialized_end=5204 + _globals['_LISTATESPACESRESPONSE']._serialized_start=5206 + _globals['_LISTATESPACESRESPONSE']._serialized_end=5291 + _globals['_DELETEATESPACEREQUEST']._serialized_start=5293 + _globals['_DELETEATESPACEREQUEST']._serialized_end=5353 + _globals['_CREATEACTORTEMPLATEREQUEST']._serialized_start=5355 + _globals['_CREATEACTORTEMPLATEREQUEST']._serialized_end=5430 + _globals['_GETACTORTEMPLATEREQUEST']._serialized_start=5432 + _globals['_GETACTORTEMPLATEREQUEST']._serialized_end=5500 + _globals['_LISTACTORTEMPLATESREQUEST']._serialized_start=5502 + _globals['_LISTACTORTEMPLATESREQUEST']._serialized_end=5586 + _globals['_LISTACTORTEMPLATESRESPONSE']._serialized_start=5588 + _globals['_LISTACTORTEMPLATESRESPONSE']._serialized_end=5689 + _globals['_DELETEACTORTEMPLATEREQUEST']._serialized_start=5691 + _globals['_DELETEACTORTEMPLATEREQUEST']._serialized_end=5762 + _globals['_GETACTORREQUEST']._serialized_start=5764 + _globals['_GETACTORREQUEST']._serialized_end=5815 + _globals['_CREATEACTORREQUEST']._serialized_start=5817 + _globals['_CREATEACTORREQUEST']._serialized_end=5867 + _globals['_UPDATEACTORREQUEST']._serialized_start=5869 + _globals['_UPDATEACTORREQUEST']._serialized_end=5919 + _globals['_SUSPENDACTORREQUEST']._serialized_start=5921 + _globals['_SUSPENDACTORREQUEST']._serialized_end=5976 + _globals['_SUSPENDACTORRESPONSE']._serialized_start=5978 + _globals['_SUSPENDACTORRESPONSE']._serialized_end=6030 + _globals['_PAUSEACTORREQUEST']._serialized_start=6032 + _globals['_PAUSEACTORREQUEST']._serialized_end=6085 + _globals['_PAUSEACTORRESPONSE']._serialized_start=6087 + _globals['_PAUSEACTORRESPONSE']._serialized_end=6137 + _globals['_RESUMEACTORREQUEST']._serialized_start=6139 + _globals['_RESUMEACTORREQUEST']._serialized_end=6193 + _globals['_RESUMEACTORRESPONSE']._serialized_start=6195 + _globals['_RESUMEACTORRESPONSE']._serialized_end=6263 + _globals['_DELETEACTORREQUEST']._serialized_start=6265 + _globals['_DELETEACTORREQUEST']._serialized_end=6338 + _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_start=6340 + _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_end=6403 + _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_start=6405 + _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_end=6516 + _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_start=6518 + _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_end=6629 + _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_start=6631 + _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_end=6697 + _globals['_GETTAGREQUEST']._serialized_start=6699 + _globals['_GETTAGREQUEST']._serialized_end=6746 + _globals['_MINTACTORJWTREQUEST']._serialized_start=6748 + _globals['_MINTACTORJWTREQUEST']._serialized_end=6840 + _globals['_MINTACTORJWTRESPONSE']._serialized_start=6842 + _globals['_MINTACTORJWTRESPONSE']._serialized_end=6883 + _globals['_MINTACTORCERTIFICATEREQUEST']._serialized_start=6886 + _globals['_MINTACTORCERTIFICATEREQUEST']._serialized_end=7055 + _globals['_MINTACTORCERTIFICATERESPONSE']._serialized_start=7057 + _globals['_MINTACTORCERTIFICATERESPONSE']._serialized_end=7115 + _globals['_GETACTORSNAPSHOTREQUEST']._serialized_start=7117 + _globals['_GETACTORSNAPSHOTREQUEST']._serialized_end=7185 + _globals['_GETACTORSNAPSHOTTAGREQUEST']._serialized_start=7187 + _globals['_GETACTORSNAPSHOTTAGREQUEST']._serialized_end=7262 + _globals['_LISTTAGSREQUEST']._serialized_start=7264 + _globals['_LISTTAGSREQUEST']._serialized_end=7338 + _globals['_LISTTAGSRESPONSE']._serialized_start=7340 + _globals['_LISTTAGSRESPONSE']._serialized_end=7410 + _globals['_CREATETAGREQUEST']._serialized_start=7412 + _globals['_CREATETAGREQUEST']._serialized_end=7456 + _globals['_UPDATETAGREQUEST']._serialized_start=7458 + _globals['_UPDATETAGREQUEST']._serialized_end=7502 + _globals['_DELETETAGREQUEST']._serialized_start=7504 + _globals['_DELETETAGREQUEST']._serialized_end=7554 + _globals['_DELETEOPTIONS']._serialized_start=7556 + _globals['_DELETEOPTIONS']._serialized_end=7601 + _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_start=7603 + _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_end=7712 + _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_start=7714 + _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_end=7827 + _globals['_LISTWORKERSREQUEST']._serialized_start=7829 + _globals['_LISTWORKERSREQUEST']._serialized_end=7888 + _globals['_LISTWORKERSRESPONSE']._serialized_start=7890 + _globals['_LISTWORKERSRESPONSE']._serialized_end=7969 + _globals['_GETWORKERREQUEST']._serialized_start=7971 + _globals['_GETWORKERREQUEST']._serialized_end=8024 + _globals['_CREATEWORKERREQUEST']._serialized_start=8026 + _globals['_CREATEWORKERREQUEST']._serialized_end=8079 + _globals['_UPDATEWORKERREQUEST']._serialized_start=8081 + _globals['_UPDATEWORKERREQUEST']._serialized_end=8134 + _globals['_DELETEWORKERREQUEST']._serialized_start=8136 + _globals['_DELETEWORKERREQUEST']._serialized_end=8232 + _globals['_DRAINWORKERREQUEST']._serialized_start=8234 + _globals['_DRAINWORKERREQUEST']._serialized_end=8289 + _globals['_LISTACTORSREQUEST']._serialized_start=8291 + _globals['_LISTACTORSREQUEST']._serialized_end=8367 + _globals['_LISTACTORSRESPONSE']._serialized_start=8369 + _globals['_LISTACTORSRESPONSE']._serialized_end=8445 + _globals['_WORKER']._serialized_start=8448 + _globals['_WORKER']._serialized_end=8774 + _globals['_WORKER_LABELSENTRY']._serialized_start=8729 + _globals['_WORKER_LABELSENTRY']._serialized_end=8774 + _globals['_WORKERSTATUS']._serialized_start=8777 + _globals['_WORKERSTATUS']._serialized_end=8914 + _globals['_WORKERRESOURCES']._serialized_start=8916 + _globals['_WORKERRESOURCES']._serialized_end=8987 + _globals['_ACTORASSIGNMENT']._serialized_start=8990 + _globals['_ACTORASSIGNMENT']._serialized_end=9189 + _globals['_SETWORKERCAPACITYREQUEST']._serialized_start=9191 + _globals['_SETWORKERCAPACITYREQUEST']._serialized_end=9295 + _globals['_SETWORKERCAPACITYRESPONSE']._serialized_start=9297 + _globals['_SETWORKERCAPACITYRESPONSE']._serialized_end=9356 + _globals['_CONTROL']._serialized_start=10393 + _globals['_CONTROL']._serialized_end=12862 + _globals['_WORKERSERVICE']._serialized_start=12864 + _globals['_WORKERSERVICE']._serialized_end=12969 # @@protoc_insertion_point(module_scope) diff --git a/benchmarking/locust/common/ateapi_pb2_grpc.py b/benchmarking/locust/common/ateapi_pb2_grpc.py index 63d5cbb699..97ef55f6fb 100644 --- a/benchmarking/locust/common/ateapi_pb2_grpc.py +++ b/benchmarking/locust/common/ateapi_pb2_grpc.py @@ -473,7 +473,7 @@ def ListActorTemplates(self, request, context): def DeleteActorTemplate(self, request, context): """Delete an ActorTemplate together with its golden actor and golden - snapshot in the ActorTemplate's namespace. + tag in the reserved ate-golden atespace. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 3a225600f6..0fa2c4c6c0 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -149,7 +149,7 @@ wait_actortemplate_ready() { while ((SECONDS < deadline)); do if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then - snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenSnapshot.snapshotUri // empty' <<<"${json}") + snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}") if [[ -n "${snapshot}" ]]; then return 0 fi diff --git a/cmd/ate-setup/internal/steps/substrate.go b/cmd/ate-setup/internal/steps/substrate.go index 0de3c180f3..db9429fddc 100644 --- a/cmd/ate-setup/internal/steps/substrate.go +++ b/cmd/ate-setup/internal/steps/substrate.go @@ -97,7 +97,7 @@ func WaitActorTemplateGolden(ctx context.Context, client *ateclient.Client, ref lastErr = err if err == nil { goldenStatus := template.GetStatus().GetGoldenSnapshotStatus() - if goldenStatus.GetGoldenSnapshot().GetSnapshotUri() != "" { + if goldenStatus.GetGoldenTag().GetName() != "" { return nil } if msg := goldenStatus.GetErrorMessage(); msg != "" { diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index 5bcff40702..f5bdc59442 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -86,14 +86,29 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor) return nil, err } - // If a source tag is requested, resolve it to the external - // snapshot the new Actor starts from. + // Resolve the explicit tag, or freeze the template's current golden default. + tagRef := inActor.GetSourceTag() + if tagRef == nil { + tagRef = template.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() + } else { + for _, volume := range template.GetVolumes() { + if volume.GetExternalVolumeTemplate() != nil { + // TODO: Permit cloning after CSI volume snapshots are supported. + return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes") + } + } + } var sourceTag *ateapipb.Tag - if tagRef := inActor.GetSourceTag(); tagRef != nil { + if tagRef != nil { sourceTag, err = s.resolveTagSource(ctx, inActor.GetMetadata().GetAtespace(), tagRef, template) if err != nil { return nil, err } + if inActor.GetSourceTag() == nil { + if err := validateGoldenSnapshotScope(sourceTag.GetStatus().GetSnapshot()); err != nil { + return nil, err + } + } } atespace := inActor.GetMetadata().GetAtespace() @@ -176,12 +191,6 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string if tag.GetStatus().GetActorTemplateUid() != template.GetMetadata().GetUid() { return nil, status.Errorf(codes.FailedPrecondition, "source Tag must be taken from an actor with ActorTemplate uid %q", tag.GetStatus().GetActorTemplateUid()) } - for _, volume := range template.GetVolumes() { - if volume.GetExternalVolumeTemplate() != nil { - // TODO: Permit cloning after CSI volume snapshots are supported. - return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes") - } - } return tag, nil } diff --git a/cmd/ateapi/internal/controlapi/actor_template.go b/cmd/ateapi/internal/controlapi/actor_template.go index 7ec2b53a29..8f2181cc2e 100644 --- a/cmd/ateapi/internal/controlapi/actor_template.go +++ b/cmd/ateapi/internal/controlapi/actor_template.go @@ -155,6 +155,26 @@ func (s *RPCService) DeleteActorTemplate(ctx context.Context, req *ateapipb.Dele } templateRef := resources.ActorTemplateRefFromObjectRef(req.GetActorTemplate()) + // Serialize cleanup against golden actor/tag creation by the reconciler. + ctx, lease, err := acquireLease(ctx, s.impl, "lease:actortemplate:"+templateRef.Atespace+":"+templateRef.Name, "ActorTemplate "+templateRef.String()) + if err != nil { + return nil, err + } + defer lease.Close() + tmpl, err := s.impl.GetActorTemplate(ctx, templateRef) + if errors.Is(err, store.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "ActorTemplate %s not found", templateRef) + } + if err != nil { + return nil, err + } + goldenRef := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()} + if _, err := s.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: goldenRef, AnyState: true}); err != nil && status.Code(err) != codes.NotFound { + return nil, fmt.Errorf("while deleting golden actor: %w", err) + } + if _, err := s.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: goldenRef}); err != nil && status.Code(err) != codes.NotFound { + return nil, fmt.Errorf("while deleting golden tag: %w", err) + } deleted, err := s.impl.DeleteActorTemplate(ctx, templateRef) if err != nil { if errors.Is(err, store.ErrNotFound) { diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 1cad1d9a5f..15b3b1ffd1 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -314,7 +314,7 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) { // Server-owned status a client must not be able to set. tmpl.Status = &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://my-bucket/snapshots/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/sneaky"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, }, } }) @@ -1224,7 +1224,7 @@ func TestUpdateActorTemplateMetadata(t *testing.T) { // A server-owned status write passes validation and bumps the version. updated, err := persistence.UpdateActorTemplate(ctx, ref, store.PreconditionFrom(created), func(tmpl *ateapipb.ActorTemplate) error { tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, }} return nil }) diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index de6a77705f..2d932b0e89 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -1436,3 +1436,90 @@ func TestValidateSuspendActorRequest(t *testing.T) { }) } } + +func TestCreateActor_GoldenTagDefault(t *testing.T) { + for _, scenario := range []string{"default", "explicit tag", "own snapshot", "missing", "pending", "wrong template", "data scope"} { + t.Run(scenario, func(t *testing.T) { + ctx := t.Context() + persistence := newTestPersistence(t) + storetest.MustCreateAtespace(t, ctx, persistence, "team-a") + storetest.MustCreateAtespace(t, ctx, persistence, resources.GoldenActorAtespace) + tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl") + ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "golden"} + tag := &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, + SourceActor: ref, + Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, + Status: &ateapipb.TagStatus{ + ActorTemplateUid: tmpl.GetMetadata().GetUid(), + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + }, + } + wantCode := codes.OK + switch scenario { + case "missing": + wantCode = codes.NotFound + case "pending": + tag.Status.Snapshot = nil + wantCode = codes.FailedPrecondition + case "wrong template": + tag.Status.ActorTemplateUid = "other" + wantCode = codes.FailedPrecondition + case "data scope": + tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA + wantCode = codes.FailedPrecondition + } + if scenario != "missing" { + if _, err := persistence.CreateTag(ctx, tag); err != nil { + t.Fatal(err) + } + } + if _, err := persistence.UpdateActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl), store.PreconditionFrom(tmpl), func(db *ateapipb.ActorTemplate) error { + db.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: ref}} + return nil + }); err != nil { + t.Fatal(err) + } + actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()} + if scenario == "explicit tag" { + tag.Metadata.Name = "explicit" + tag.Status.Snapshot.SnapshotUri = "gs://bucket/atespaces/ate-golden/tags/explicit" + if _, err := persistence.CreateTag(ctx, tag); err != nil { + t.Fatal(err) + } + actor.SourceTag = &ateapipb.ObjectRef{Atespace: ref.Atespace, Name: "explicit"} + } + svc := &ServiceImpl{store: persistence} + created, err := svc.CreateActor(ctx, actor) + if status.Code(err) != wantCode { + t.Fatalf("CreateActor = %v, want %v", err, wantCode) + } + if err != nil { + return + } + if got := created.GetStatus(); got.GetExternalSnapshot().GetSnapshotUri() != tag.GetStatus().GetSnapshot().GetSnapshotUri() || got.GetCurrentActorTemplateUid() != tmpl.GetMetadata().GetUid() { + t.Fatalf("incorrect initial status: %v", got) + } + if scenario == "own snapshot" { + uri, err := resources.NewActorSnapshotURI(tmpl.GetSnapshotsConfig().GetStorageLocation(), "team-a", created.GetMetadata().GetUid(), "snapshot") + if err != nil { + t.Fatal(err) + } + if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapipb.Actor) error { + db.Status.ExternalSnapshot.SnapshotUri = uri.String() + return nil + }); err != nil { + t.Fatal(err) + } + } + workflow := &ActorWorkflow{store: persistence} + _, _, src, err := workflow.loadActorForResume(ctx, resources.ActorRefFromActor(created)) + if err != nil { + t.Fatal(err) + } + if src.SnapshotURI.IsZero() { + t.Fatalf("missing snapshot source for %s", scenario) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go index 32bd246fc3..6d83b2289e 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go @@ -16,7 +16,12 @@ package functionaltest import ( "context" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" + "github.com/agent-substrate/substrate/internal/resources" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/util/wait" "testing" + "time" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" @@ -44,7 +49,7 @@ func TestActorTemplateCRUD(t *testing.T) { // Server-owned status on the request is ignored. Status: &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://my-bucket/snapshots/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/sneaky"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, }, }, }, @@ -134,6 +139,98 @@ func TestActorTemplateCRUD(t *testing.T) { assertGrpcErrorRegex(t, err, codes.InvalidArgument, `sandbox_config\.config_name`) } +func TestGoldenTagLifecycle(t *testing.T) { + ns := namespaceForTest("golden-tag") + tc := setupTest(t, ns) + defer tc.cleanup() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) + tmpl := createTemplateWithSelector(t, tc, "golden-template", &ateapipb.Selector{MatchLabels: map[string]string{poolLabelKey: ns}}) + workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl) + // Created before readiness: a later golden tag must not change its source. + early, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "early"}, ActorTemplate: templateRef.ToObjectRef(), + }}) + if err != nil { + t.Fatal(err) + } + controlapi.NewActorTemplateReconciler(tc.persistence, tc.service).Start(ctx) + var goldenRef *ateapipb.ObjectRef + err = wait.PollUntilContextTimeout(ctx, 50*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) { + current, err := tc.client.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()}) + goldenRef = current.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() + return goldenRef != nil, err + }) + if err != nil { + t.Fatalf("waiting for golden tag: %v", err) + } + golden, err := tc.client.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenRef}) + if err != nil { + t.Fatal(err) + } + uri := golden.GetStatus().GetSnapshot().GetSnapshotUri() + parsed, err := resources.ParseSnapshotURI(uri) + if err != nil { + t.Fatal(err) + } + if !parsed.OwnedBy(resources.TagSnapshotOwner(resources.GoldenActorAtespace, parsed.Name())) { + t.Fatal("golden snapshot is not tag-owned") + } + assertSnapshotPresent(t, tc, uri) + _, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: goldenRef}) + if status.Code(err) != codes.NotFound { + t.Fatalf("golden actor was not deleted: %v", err) + } + late, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "late"}, ActorTemplate: templateRef.ToObjectRef(), + }}) + if err != nil { + t.Fatal(err) + } + if late.GetStatus().GetExternalSnapshot().GetSnapshotUri() != uri || late.GetStatus().GetCurrentActorTemplateUid() != tmpl.GetMetadata().GetUid() { + t.Fatal("actor did not inherit golden tag snapshot and template UID") + } + waitForWorkerAvailable(t, tc, workerName) + tc.fakeAtelet.Lock.Lock() + tc.fakeAtelet.RunCalled = false + tc.fakeAtelet.Lock.Unlock() + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: resources.ActorRefFromActor(early).ToObjectRef()}); err != nil { + t.Fatal(err) + } + if !tc.fakeAtelet.RunCalled { + t.Fatal("actor created before golden readiness did not cold boot") + } + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: resources.ActorRefFromActor(early).ToObjectRef()}); err != nil { + t.Fatal(err) + } + waitForWorkerAvailable(t, tc, workerName) + if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: resources.ActorRefFromActor(late).ToObjectRef()}); err != nil { + t.Fatal(err) + } + if !tc.fakeAtelet.RestoreCalled { + t.Fatal("actor did not restore golden tag") + } + if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: resources.ActorRefFromActor(late).ToObjectRef()}); err != nil { + t.Fatal(err) + } + assertSnapshotPresent(t, tc, uri) + for _, actor := range []*ateapipb.Actor{early, late} { + if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: resources.ActorRefFromActor(actor).ToObjectRef(), AnyState: true}); err != nil { + t.Fatal(err) + } + } + if _, err := tc.client.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()}); err != nil { + t.Fatal(err) + } + assertSnapshotCollected(t, tc, uri) + _, err = tc.client.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenRef}) + if status.Code(err) != codes.NotFound { + t.Fatalf("golden tag was not deleted: %v", err) + } +} + func TestListActorTemplates_InvalidPageToken(t *testing.T) { ns := namespaceForTest("ns-template-invalid-token") tc := setupTest(t, ns) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index fdb9ee2a00..07ce4cd75a 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -51,7 +51,7 @@ func TestCreateActor_Success(t *testing.T) { tc := setupTest(t, ns) defer tc.cleanup() - createTemplate(t, tc, ns) + tmpl := createTemplate(t, tc, ns) createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{ @@ -67,9 +67,13 @@ func TestCreateActor_Success(t *testing.T) { } want := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(), + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + }, WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, } @@ -663,7 +667,7 @@ func TestUpdateActor_Success(t *testing.T) { tc := setupTest(t, ns) defer tc.cleanup() - createTemplate(t, tc, ns) + tmpl := createTemplate(t, tc, ns) toUpdate, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, @@ -687,7 +691,11 @@ func TestUpdateActor_Success(t *testing.T) { wantActor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(), + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, }, @@ -830,7 +838,11 @@ func TestUpdateActor(t *testing.T) { wantActor := &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, - Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, + CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(), + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + }, WorkerSelector: &ateapipb.Selector{ MatchLabels: map[string]string{"tier": "paid"}, }, @@ -1800,6 +1812,7 @@ func TestResumeActor(t *testing.T) { Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_RUNNING, CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(), + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, WorkerAssignment: &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: podUID}, WorkerNamespace: ns, @@ -2544,6 +2557,7 @@ func TestPauseActor(t *testing.T) { ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, }, CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(), + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, }, } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index f79912f301..4a62d08a51 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -341,15 +341,13 @@ func assertSnapshotCollected(t *testing.T, tc *testContext, snapshotURI string) } } -// goldenSnapshotURI is the golden snapshot the test templates record: the -// golden Actor owns it under its own prefix in the reserved atespace, the way -// the ActorTemplateReconciler's checkpoint would leave it. +// goldenSnapshotURI is the snapshot owned by the test template's golden tag. func goldenSnapshotURI(t *testing.T) string { t.Helper() - const goldenActorUID = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94" - uri, err := resources.NewActorSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenActorUID, "golden") + const goldenSnapshotName = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94" + uri, err := resources.NewTagSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenSnapshotName) if err != nil { - t.Fatalf("NewActorSnapshotURI: %v", err) + t.Fatalf("NewTagSnapshotURI: %v", err) } return uri.String() } @@ -455,6 +453,21 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st t.Fatalf("failed to create actor template: %v", err) } + createAtespace(t, tc, resources.GoldenActorAtespace) + tag, err := tc.persistence.CreateTag(context.Background(), &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, + SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()}, + Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, + Status: &ateapipb.TagStatus{ + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + ActorTemplateUid: created.GetMetadata().GetUid(), + SourceActorUid: "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94", + }, + }) + if err != nil { + t.Fatalf("create golden tag: %v", err) + } + // Record the golden snapshot on the template's status directly in the // store, as the ActorTemplateReconciler's checkpoint would: there is no // status RPC, and the reconciler does not run in this test environment. @@ -463,7 +476,7 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st func(dbTemplate *ateapipb.ActorTemplate) error { dbTemplate.Status = &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}, + GoldenTag: resources.TagRefFromTag(tag).ToObjectRef(), }, } return nil diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 132fa034f1..07f2828866 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -26,7 +26,6 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" @@ -64,6 +63,10 @@ type templateReconcilerStore interface { // goldenActorControl is the in-process slice of the Control service the // reconciler drives golden actors through. *RPCService satisfies it. type goldenActorControl interface { + GetTag(ctx context.Context, req *ateapipb.GetTagRequest) (*ateapipb.Tag, error) + CreateTag(ctx context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error) + DeleteTag(ctx context.Context, req *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error) + DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) @@ -200,11 +203,29 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource // The snapshot has already failed. return 0, nil } - if goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri() != "" { + if goldenSnapshotStatus.GetGoldenTag() != nil { // The golden snapshot exists already. return 0, nil } + // A completed tag survives a crash during actor deletion or checkpointing. + tag, err := r.control.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenActorRef}) + if err != nil && status.Code(err) != codes.NotFound { + return 0, fmt.Errorf("while getting golden tag: %w", err) + } + if err == nil { + if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) { + return 0, fmt.Errorf("golden tag belongs to another actor or template") + } + if tag.GetStatus().GetSnapshot().GetSnapshotUri() != "" { + return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef) + } + // CreateTag cannot resume an incomplete copy. Delete it before retrying. + if _, err := r.control.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: goldenActorRef}); err != nil && status.Code(err) != codes.NotFound { + return 0, fmt.Errorf("while deleting incomplete golden tag: %w", err) + } + } + actor, err := r.ensureActorExists(ctx, tmpl, goldenActorRef) if err != nil { if status.Code(err) == codes.InvalidArgument { @@ -238,19 +259,19 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource return rem, nil } // Warmup done: suspend the golden actor and record its snapshot. - snapshot, err := r.suspendActor(ctx, goldenActorRef) + err := r.suspendActor(ctx, goldenActorRef) if err != nil { return 0, err } - return 0, r.saveGoldenSnapshot(ctx, tmpl, snapshot) + return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef) case ateapipb.ActorState_ACTOR_STATE_SUSPENDING: // A previous pass died mid-suspend; retry suspend. - snapshot, err := r.suspendActor(ctx, goldenActorRef) + err := r.suspendActor(ctx, goldenActorRef) if err != nil { return 0, err } - return 0, r.saveGoldenSnapshot(ctx, tmpl, snapshot) + return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef) case ateapipb.ActorState_ACTOR_STATE_RESUMING, ateapipb.ActorState_ACTOR_STATE_SUSPENDED: @@ -260,7 +281,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource // Golden actors never start from a source snapshot, so an // existing snapshot means an earlier suspend completed // without being recorded. - return 0, r.saveGoldenSnapshot(ctx, tmpl, actor.GetStatus().GetExternalSnapshot()) + return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef) } if _, err := r.control.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: goldenActorRef}); err != nil { // A crash during resume is observed as CRASHED on the retry. @@ -283,29 +304,42 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource } } -// suspendActor suspends the golden actor and returns the external snapshot it -// wrote. Reentrant: SuspendActor completes an in-flight suspend and is a no-op -// on an already-suspended actor, returning the existing snapshot either way. -func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) (*ateapipb.ExternalSnapshot, error) { +// suspendActor waits for the golden actor to produce an external snapshot. +// SuspendActor completes an in-flight suspend and is a no-op if already suspended. +func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) error { resp, err := r.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenRef}) if err != nil { // A crash during suspend is observed as CRASHED on the retry. - return nil, fmt.Errorf("while suspending golden actor: %w", err) + return fmt.Errorf("while suspending golden actor: %w", err) } suspended := resp.GetActor().GetStatus().GetExternalSnapshot() if suspended.GetSnapshotUri() == "" { - return nil, fmt.Errorf("suspending golden actor produced no external snapshot") + return fmt.Errorf("suspending golden actor produced no external snapshot") } - return suspended, nil + return nil } -// saveGoldenSnapshot records the golden actor's external snapshot, the -// terminal success state that marks the template ready for use, ending the -// reconcile pass. The golden actor keeps owning those objects; the template -// only points at them. -func (r *ActorTemplateReconciler) saveGoldenSnapshot(ctx context.Context, observed *ateapipb.ActorTemplate, golden *ateapipb.ExternalSnapshot) error { - _, err := r.checkpoint(ctx, observed, func(snapshotStatus *ateapipb.GoldenSnapshotStatus) { - snapshotStatus.GoldenSnapshot = proto.CloneOf(golden) +// tagGoldenActor copies the snapshot into a tag before releasing the actor's copy. +func (r *ActorTemplateReconciler) tagGoldenActor(ctx context.Context, tmpl *ateapipb.ActorTemplate, ref *ateapipb.ObjectRef) error { + _, err := r.control.CreateTag(ctx, &ateapipb.CreateTagRequest{Tag: &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()}, + SourceActor: ref, + Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, + }}) + if err != nil { + return fmt.Errorf("while creating golden tag: %w", err) + } + return r.saveGoldenTag(ctx, tmpl, ref) +} + +// saveGoldenTag finishes cleanup before recording terminal success, so retries +// can rediscover the tag even if deletion or the status write fails. +func (r *ActorTemplateReconciler) saveGoldenTag(ctx context.Context, tmpl *ateapipb.ActorTemplate, ref *ateapipb.ObjectRef) error { + if _, err := r.control.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}); err != nil && status.Code(err) != codes.NotFound { + return fmt.Errorf("while deleting golden actor: %w", err) + } + _, err := r.checkpoint(ctx, tmpl, func(snapshotStatus *ateapipb.GoldenSnapshotStatus) { + snapshotStatus.GoldenTag = ref }) return err } @@ -345,7 +379,7 @@ func (r *ActorTemplateReconciler) fail(ctx context.Context, observed *ateapipb.A // goldenSnapshotDone reports whether the golden snapshot build reached a // terminal state: the snapshot was recorded, or the build failed. func goldenSnapshotDone(snapshotStatus *ateapipb.GoldenSnapshotStatus) bool { - return snapshotStatus.GetGoldenSnapshot().GetSnapshotUri() != "" || snapshotStatus.GetErrorMessage() != "" + return snapshotStatus.GetGoldenTag() != nil || snapshotStatus.GetErrorMessage() != "" } // goldenSnapshotWarmupFor returns 0 when every container has a readyz probe diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index aac5806dcd..db7649c29a 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -143,7 +143,13 @@ func (s *fakeTemplateStore) storedStatus(t *testing.T, ref resources.ActorTempla // from that observation. Tests seed mid-lifecycle states via exists / // goldenState / goldenSnapshot. type fakeGoldenControl struct { - mu sync.Mutex + mu sync.Mutex + tag *ateapipb.Tag + tagErr error + deleteErr error + deleteTagErr error + tagReqs []*ateapipb.CreateTagRequest + deleteReqs []*ateapipb.DeleteActorRequest createErr error resumeErr error @@ -235,6 +241,52 @@ func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.Suspen }, nil } +func (c *fakeGoldenControl) GetTag(_ context.Context, _ *ateapipb.GetTagRequest) (*ateapipb.Tag, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.tag == nil { + return nil, status.Error(codes.NotFound, "no tag") + } + return proto.CloneOf(c.tag), nil +} + +func (c *fakeGoldenControl) CreateTag(_ context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.tagReqs = append(c.tagReqs, req) + if c.tagErr != nil { + return nil, c.tagErr + } + c.tag = proto.CloneOf(req.GetTag()) + c.tag.Status = &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}} + return proto.CloneOf(c.tag), nil +} + +func (c *fakeGoldenControl) DeleteTag(_ context.Context, _ *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.deleteTagErr != nil { + return nil, c.deleteTagErr + } + tag := c.tag + c.tag = nil + return tag, nil +} + +func (c *fakeGoldenControl) DeleteActor(_ context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.deleteReqs = append(c.deleteReqs, req) + if c.deleteErr != nil { + return nil, c.deleteErr + } + if !c.exists { + return nil, status.Error(codes.NotFound, "no actor") + } + c.exists = false + return &ateapipb.Actor{}, nil +} + func (c *fakeGoldenControl) callCounts() (creates, resumes, suspends int) { c.mu.Lock() defer c.mu.Unlock() @@ -286,9 +338,9 @@ func withSnapshotDeadline(at time.Time) func(*ateapipb.ActorTemplate) { } } -func withGoldenSnapshot(snapshotURI string) func(*ateapipb.ActorTemplate) { +func withGoldenTag() func(*ateapipb.ActorTemplate) { return func(tmpl *ateapipb.ActorTemplate) { - seededGoldenStatus(tmpl).GoldenSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: snapshotURI} + seededGoldenStatus(tmpl).GoldenTag = &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID} } } @@ -340,9 +392,8 @@ func TestReconcileOne(t *testing.T) { // stored error message must be empty. Checked when template is seeded. wantFailedReason string wantMessage string - // wantSnapshot must equal the stored golden snapshot URI; empty means - // the snapshot must not be recorded. - wantSnapshot string + // wantTag indicates that the golden tag should be recorded. + wantTag bool // wantDeadline asserts whether take_golden_snapshot_at is set. wantDeadline bool wantCreates int @@ -353,7 +404,7 @@ func TestReconcileOne(t *testing.T) { name: "happy path creates, resumes, and snapshots the golden actor", template: testTemplate(), control: &fakeGoldenControl{snapshot: goldenSnapshot}, - wantSnapshot: goldenSnapshot, + wantTag: true, wantDeadline: true, wantCreates: 1, wantResumes: 1, @@ -382,7 +433,7 @@ func TestReconcileOne(t *testing.T) { template: testTemplate( withSnapshotDeadline(time.Now().Add(-time.Minute))), control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_RUNNING, snapshot: goldenSnapshot}, - wantSnapshot: goldenSnapshot, + wantTag: true, wantSuspends: 1, }, { @@ -405,14 +456,14 @@ func TestReconcileOne(t *testing.T) { name: "suspending golden actor is completed and recorded", template: testTemplate(), control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, snapshot: goldenSnapshot}, - wantSnapshot: goldenSnapshot, + wantTag: true, wantSuspends: 1, }, { - name: "suspended golden actor with a snapshot is recorded without more control calls", - template: testTemplate(), - control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: goldenSnapshot}, - wantSnapshot: goldenSnapshot, + name: "suspended golden actor with a snapshot is recorded without more control calls", + template: testTemplate(), + control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: goldenSnapshot}, + wantTag: true, }, { name: "create AlreadyExists requeues for the retry to observe", @@ -488,10 +539,10 @@ func TestReconcileOne(t *testing.T) { control: &fakeGoldenControl{}, }, { - name: "terminal golden snapshot is a noop", - template: testTemplate(withGoldenSnapshot(goldenSnapshot)), - control: &fakeGoldenControl{}, - wantSnapshot: goldenSnapshot, + name: "terminal golden snapshot is a noop", + template: testTemplate(withGoldenTag()), + control: &fakeGoldenControl{}, + wantTag: true, }, { name: "terminal error message is a noop", @@ -538,8 +589,8 @@ func TestReconcileOne(t *testing.T) { t.Errorf("stored error message = %q, want it to contain %q", errorMessage, tt.wantMessage) } } - if got := snapshotStatus.GetGoldenSnapshot().GetSnapshotUri(); got != tt.wantSnapshot { - t.Errorf("stored golden snapshot uri = %q, want %q", got, tt.wantSnapshot) + if got := snapshotStatus.GetGoldenTag(); (got != nil) != tt.wantTag { + t.Errorf("stored golden tag = %v, want tag %v", got, tt.wantTag) } if tt.wantDeadline && snapshotStatus.GetTakeGoldenSnapshotAt() == nil { t.Error("stored take_golden_snapshot_at is nil, want set") @@ -591,13 +642,12 @@ func TestReconcileOne_GoldenActorRequests(t *testing.T) { func TestCheckpoint_TerminalStateErrors(t *testing.T) { ctx := context.Background() - goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1" for _, seed := range []struct { name string opt func(*ateapipb.ActorTemplate) }{ - {"golden snapshot taken", withGoldenSnapshot(goldenSnapshot)}, + {"golden snapshot taken", withGoldenTag()}, {"failed", withFailed(reasonGoldenActorCrashed)}, } { t.Run(seed.name, func(t *testing.T) { @@ -658,7 +708,6 @@ func drainQueue(r *ActorTemplateReconciler) []resources.ActorTemplateRef { } func TestResync_QueuesOnlyActionableTemplates(t *testing.T) { - goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1" tests := []struct { name string @@ -667,7 +716,7 @@ func TestResync_QueuesOnlyActionableTemplates(t *testing.T) { }{ {"empty status", nil, true}, {"mid warmup", []func(*ateapipb.ActorTemplate){withSnapshotDeadline(time.Now().Add(time.Hour))}, true}, - {"golden snapshot taken", []func(*ateapipb.ActorTemplate){withGoldenSnapshot(goldenSnapshot)}, false}, + {"golden snapshot taken", []func(*ateapipb.ActorTemplate){withGoldenTag()}, false}, {"failed", []func(*ateapipb.ActorTemplate){withFailed(reasonGoldenActorCrashed)}, false}, } @@ -714,3 +763,81 @@ func TestResync_FollowsPagination(t *testing.T) { t.Errorf("queued %d templates, want 3 (all pages walked)", got) } } + +func TestReconcileOne_GoldenTagRecovery(t *testing.T) { + ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID} + completed := &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, + SourceActor: ref, + Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, + Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/tag-snapshot"}}, + } + for _, scenario := range []string{"completed tag", "actor already deleted", "incomplete tag", "copy failure", "actor deletion failure", "tag deletion failure", "foreign tag"} { + t.Run(scenario, func(t *testing.T) { + control := &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: "gs://bucket/actor-snapshot"} + switch scenario { + case "completed tag", "actor already deleted", "foreign tag": + control.tag = proto.CloneOf(completed) + control.exists = scenario != "actor already deleted" + if scenario == "foreign tag" { + control.tag.Status.ActorTemplateUid = "another-template" + } + case "incomplete tag", "tag deletion failure": + control.tag = proto.CloneOf(completed) + control.tag.Status.Snapshot = nil + if scenario == "tag deletion failure" { + control.deleteTagErr = errors.New("storage unavailable") + } + case "copy failure": + control.tagErr = errors.New("copy interrupted") + case "actor deletion failure": + control.deleteErr = errors.New("storage unavailable") + } + st := newFakeTemplateStore(testTemplate()) + r := newTestTemplateReconciler(st, control) + defer r.queue.ShutDown() + _, err := r.reconcileOne(t.Context(), testTemplateRef) + wantErr := strings.Contains(scenario, "failure") || scenario == "foreign tag" + if (err != nil) != wantErr { + t.Fatalf("reconcile = %v, want error %v", err, wantErr) + } + if wantErr { + if st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag() != nil { + t.Fatal("marked ready before cleanup completed") + } + if !control.exists { + t.Fatal("deleted actor after tag failure") + } + if scenario == "foreign tag" { + return + } + control.tagErr, control.deleteErr, control.deleteTagErr = nil, nil, nil + if _, err := r.reconcileOne(t.Context(), testTemplateRef); err != nil { + t.Fatal(err) + } + } + if control.exists { + t.Fatal("golden actor still exists") + } + if !proto.Equal(st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag(), ref) { + t.Fatal("golden tag not recorded") + } + if control.tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" { + t.Fatal("golden tag has no snapshot") + } + if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 { + t.Fatal("repeated golden actor warmup") + } + if scenario == "completed tag" || scenario == "actor already deleted" { + if len(control.tagReqs) != 0 { + t.Fatal("recreated completed tag") + } + } + for _, req := range control.tagReqs { + if !proto.Equal(req.Tag.SourceActor, ref) || req.Tag.Scope != ateapipb.TagScope_TAG_SCOPE_PUBLISHED || req.Tag.Metadata.Name != ref.Name { + t.Fatalf("incorrect golden tag request: %v", req) + } + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/validation_test.go b/cmd/ateapi/internal/controlapi/validation_test.go index 5e8953c18a..9ee3948a12 100644 --- a/cmd/ateapi/internal/controlapi/validation_test.go +++ b/cmd/ateapi/internal/controlapi/validation_test.go @@ -1314,19 +1314,6 @@ func TestValidateNestedExternalSnapshot(t *testing.T) { return Validate_Tag(ctx, op, nil, obj, nil) }, }, - { - name: "actor_template.status.golden_snapshot_status.golden_snapshot", - path: field.NewPath("golden_snapshot_status", "golden_snapshot"), - validate: func(ctx context.Context) field.ErrorList { - op := operation.Operation{Type: operation.Create} - obj := &ateapipb.ActorTemplateStatus{ - GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: badExternalSnapshot(), - }, - } - return Validate_ActorTemplateStatus(ctx, op, nil, obj, nil) - }, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index cb1229f2d8..b263c62179 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -38,9 +38,9 @@ import ( // and passed by value to the restore step — never mutated after resolution. type resumeSnapshotSource struct { // SnapshotURI is the storage location of the durable snapshot to restore - // from: the actor's own latest snapshot when one exists, the template's - // golden snapshot otherwise. Zero means cold boot from the spec (unless - // the actor holds a local snapshot, which takes precedence at restore). + // from: the actor's latest snapshot, including a tag borrowed at creation. + // Zero means cold boot from the spec (unless the actor holds a local + // snapshot, which takes precedence at restore). SnapshotURI resources.SnapshotURI Scope ateapipb.SnapshotContentScope // GoldenSnapshotURI is the storage location of the ActorTemplate's golden @@ -179,7 +179,6 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc if err != nil { return nil, nil, src, err } - goldenSnapshotStatus := actorTemplate.GetStatus().GetGoldenSnapshotStatus() if uri := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); uri != "" { if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil { return nil, nil, src, status.Errorf(codes.DataLoss, "Actor %s external snapshot: %v", actorRef, err) @@ -192,14 +191,6 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc // as well; it is already disallowed at admission time. builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid() src.TemplateReplaced = builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid() - } else if goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri(); goldenURI != "" { - if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil { - return nil, nil, src, err - } - if src.SnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil { - return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err) - } - src.Scope = goldenSnapshotStatus.GetGoldenSnapshot().GetContentScope() } // The template's onResume configuration selects the boot source for the @@ -218,13 +209,25 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc dataOnly = src.Scope == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA } if dataOnly { - goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri() - if goldenURI == "" { - return nil, nil, src, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden snapshot, which is not available") + ref := actorTemplate.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag() + if ref == nil { + return nil, nil, src, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden tag, which is not available") + } + tag, err := w.store.GetTag(ctx, resources.TagRefFromObjectRef(ref)) + if errors.Is(err, store.ErrNotFound) { + return nil, nil, src, status.Error(codes.FailedPrecondition, "ActorTemplate golden tag is not available") + } + if err != nil { + return nil, nil, src, fmt.Errorf("while getting golden tag: %w", err) + } + golden := tag.GetStatus().GetSnapshot() + if golden.GetSnapshotUri() == "" || tag.GetStatus().GetActorTemplateUid() != actorTemplate.GetMetadata().GetUid() { + return nil, nil, src, status.Error(codes.FailedPrecondition, "ActorTemplate golden tag is incomplete or belongs to another template") } - if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil { + if err := validateGoldenSnapshotScope(golden); err != nil { return nil, nil, src, err } + goldenURI := golden.GetSnapshotUri() if src.GoldenSnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil { return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err) } @@ -753,7 +756,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou _, err = client.Restore(ctx, req) return tele, maybeCrashActor(ctx, w.store, actorRef, err, "while restoring durable snapshot", ateattr.OperationResume) } else { - slog.InfoContext(ctx, "Actor has no snapshot; ActorTemplate has no golden snapshot; Booting from ActorTemplate spec") + slog.InfoContext(ctx, "Actor has no snapshot; Booting from ActorTemplate spec") tele.SnapshotKind = ateattr.SnapshotKindBoot // Booting from scratch: resolve the sandbox binaries from the diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 488e830a1f..17fc25159e 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "net" - "strings" "sync" "testing" "time" @@ -1028,12 +1027,27 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { } if tt.goldenURI != "" { tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"}, }} } - if _, err := persistence.CreateActorTemplate(ctx, tmpl); err != nil { + stored, err := persistence.CreateActorTemplate(ctx, tmpl) + if err != nil { t.Fatalf("create template: %v", err) } + if tt.goldenURI != "" { + _, err := persistence.CreateTag(ctx, &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "golden"}, + SourceActor: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"}, + Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, + Status: &ateapipb.TagStatus{ + ActorTemplateUid: stored.GetMetadata().GetUid(), + Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope}, + }, + }) + if err != nil { + t.Fatalf("create golden tag: %v", err) + } + } w := &ActorWorkflow{store: persistence} _, _, src, err := w.loadActorForResume(ctx, actorRef) @@ -1053,41 +1067,25 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { } } -// TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden covers the -// golden-fallback branch (actor with no snapshot of its own): a golden -// snapshot recorded with a non-Full scope holds no guest state, so the resume -// must fail with a clear error instead of forwarding its scope to atelet -// with no golden location (which atelet rejects with a confusing -// "missing bucket" validation error). -func TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden(t *testing.T) { +// A golden tag becoming ready after creation does not change an actor's source. +func TestLoadActorForResume_DoesNotDefaultGolden(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED) - storetest.MustCreateAtespace(t, ctx, persistence, "ns") if _, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, - Status: &ateapipb.ActorTemplateStatus{ - GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1"), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - }, - }, - }, + Status: &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ + GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"}, + }}, }); err != nil { - t.Fatalf("create template: %v", err) + t.Fatal(err) } - w := &ActorWorkflow{store: persistence} - _, _, _, err := w.loadActorForResume(ctx, actorRef) - if got := status.Code(err); got != codes.FailedPrecondition { - t.Fatalf("status.Code(err) = %v, want FailedPrecondition (err: %v)", got, err) - } - if !strings.Contains(err.Error(), "regenerate the golden snapshot") { - t.Errorf("error %q does not tell the operator to regenerate the golden snapshot", err) + _, _, src, err := w.loadActorForResume(ctx, actorRef) + if err != nil || !src.SnapshotURI.IsZero() { + t.Fatalf("source = %+v, err = %v; want cold boot", src, err) } } @@ -1326,7 +1324,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { type templateSeed struct { // onPause is the template's pause scope. onPause ateapipb.SnapshotContentScope - // golden seeds Status.GoldenSnapshotStatus.GoldenSnapshot. + // golden seeds the template's golden tag snapshot. golden *ateapipb.ExternalSnapshot // fromData is the template's onResume boot-source policy. fromData ateapipb.ResumeSource @@ -1366,8 +1364,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { want: restoreWant{run: true}, }, { - name: "03 golden fallback restores the golden snapshot in Full", - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, + name: "03 inherited golden snapshot restores in Full", + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"}, + tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, want: restoreWant{ checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, snapshotURI: goldenURI, @@ -1375,9 +1374,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, }, { - // With no actor snapshot the restore is not data-only, so the - // golden rides in ExternalConfig and GoldenSnapshotUri stays empty. - name: "04 golden fallback under Golden fromData is a plain Full restore", + // An inherited Full golden snapshot needs no data-only overlay. + name: "04 inherited golden under Golden fromData is a plain Full restore", + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"}, tmpl: templateSeed{ golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, fromData: fromGolden, @@ -1389,27 +1388,21 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, }, { - name: "05 golden fallback rejects a non-Full golden", + name: "05 late non-Full golden does not change a cold boot", tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: dataScope}}, - want: restoreWant{code: codes.FailedPrecondition}, + want: restoreWant{run: true}, }, { - name: "06 golden fallback rejects a malformed golden URI", - tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}}, - want: restoreWant{code: codes.DataLoss}, + name: "06 inherited golden snapshot rejects a malformed URI", + actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}, tmplUID: "current"}, + tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}}, + want: restoreWant{code: codes.DataLoss}, }, { - // TemplateReplaced is derived from the actor's own durable - // snapshot; without one, a repoint cannot downgrade the golden - // fallback. - name: "07 template repoint does not affect the golden fallback", - actor: actorSeed{tmplUID: "mismatch"}, + name: "07 template repoint with a late golden still cold-boots", + actor: actorSeed{tmplUID: "old-template-uid"}, tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}}, - want: restoreWant{ - checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, - snapshotURI: goldenURI, - scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, - }, + want: restoreWant{run: true}, }, { name: "08 Full durable snapshot restores itself in Full", @@ -1732,13 +1725,23 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { } if tt.tmpl.golden != nil { tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: tt.tmpl.golden, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"}, }} } createdTmpl, err := persistence.CreateActorTemplate(ctx, tmpl) if err != nil { t.Fatalf("create template: %v", err) } + if tt.tmpl.golden != nil { + if _, err := persistence.CreateTag(ctx, &ateapipb.Tag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "golden"}, + SourceActor: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"}, + Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, + Status: &ateapipb.TagStatus{ActorTemplateUid: createdTmpl.GetMetadata().GetUid(), Snapshot: tt.tmpl.golden}, + }); err != nil { + t.Fatalf("create golden tag: %v", err) + } + } if createdTmpl.GetMetadata().GetUid() == "" { t.Fatal("created template has no UID; the matching tmplUID case would be vacuous") } diff --git a/cmd/ateapi/internal/controlapi/zz_generated.validation.go b/cmd/ateapi/internal/controlapi/zz_generated.validation.go index 8f9ad88bac..2488d421be 100644 --- a/cmd/ateapi/internal/controlapi/zz_generated.validation.go +++ b/cmd/ateapi/internal/controlapi/zz_generated.validation.go @@ -3609,10 +3609,10 @@ func Validate_GoldenSnapshotStatus( ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *ateapipb.GoldenSnapshotStatus) (errs field.ErrorList) { - { // field ateapipb.GoldenSnapshotStatus.GoldenSnapshot + { // field ateapipb.GoldenSnapshotStatus.GoldenTag fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.ExternalSnapshot, + obj, oldObj *ateapipb.ObjectRef, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -3628,15 +3628,30 @@ func Validate_GoldenSnapshotStatus( if earlyReturn { return // do not proceed } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() // call the type's validation function - errs = append(errs, Validate_ExternalSnapshot(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.GoldenSnapshotStatus) *ateapipb.ExternalSnapshot { - return oldObj.GoldenSnapshot + func(oldObj *ateapipb.GoldenSnapshotStatus) *ateapipb.ObjectRef { + return oldObj.GoldenTag }) - errs = append(errs, fn(fldPath.Child("golden_snapshot"), obj.GoldenSnapshot, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("golden_tag"), obj.GoldenTag, oldVal, oldObj != nil)...) } // field ateapipb.GoldenSnapshotStatus.TakeGoldenSnapshotAt has no validation diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 09ee1c50f9..94a9c00437 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -107,7 +107,7 @@ kubectl ate get workers -l | Column | Meaning | |---|---| | `ATESPACE` | The atespace the actor belongs to. Part of the actor's identity; folded into the storage key as `actor::`. | -| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises during `ResumeGoldenActor`. | +| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises while building its golden tag. | | `TEMPLATE` | The `ActorTemplate` the actor was created from, displayed as `/`. | | `STATE` | One of `ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`. | | `WORKER POD` | The worker pod (namespace/name) currently hosting the actor. Empty while suspended. | @@ -184,7 +184,7 @@ for a complete manifest example. | `ATESPACE` | The atespace the template belongs to. | | `NAME` | The template's name. | | `SANDBOX CLASS` | The sandbox runtime family (`SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`). | -| `GOLDEN SNAPSHOT` | The golden snapshot's name once it exists (actors can be created); empty while the golden build is still running. | +| `GOLDEN TAG` | The golden tag's name once it exists; empty while the golden build is still running. | | `ERROR` | `ERROR` when the golden build failed; `-o yaml` shows the full message. | | `AGE` | Time elapsed since the template was created. | diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index 5c93880650..d757e86f2b 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -291,7 +291,7 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f return printProto(out, &ateapipb.ListActorTemplatesResponse{ActorTemplates: templates}, format) case "table": w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "ATESPACE\tNAME\tSANDBOX CLASS\tGOLDEN SNAPSHOT\tERROR\tAGE") + fmt.Fprintln(w, "ATESPACE\tNAME\tSANDBOX CLASS\tGOLDEN TAG\tERROR\tAGE") for _, t := range templates { gss := t.GetStatus().GetGoldenSnapshotStatus() // Error messages are too long for a table cell. @@ -302,7 +302,7 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", t.GetMetadata().GetAtespace(), t.GetMetadata().GetName(), t.GetSandboxConfig().GetSandboxClass(), - gss.GetGoldenSnapshot().GetSnapshotUri(), errFlag, + gss.GetGoldenTag().GetName(), errFlag, formatAge(t.GetMetadata().GetCreateTime())) } return w.Flush() diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index 1a85fd3c33..51fa9db4b7 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -440,7 +440,7 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { }, Status: &ateapipb.ActorTemplateStatus{ GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/snap-1"}, + GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"}, }, }, }, @@ -481,10 +481,10 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) { // Sorted by atespace, then name. The ERROR column only flags that an // error message exists; the full text is available via json/yaml. - expected := `ATESPACE NAME SANDBOX CLASS GOLDEN SNAPSHOT ERROR AGE -ate-demo-counter-substrate counter SANDBOX_CLASS_GVISOR gs://private/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/snap-1 5m -ate-demo-counter-substrate counter-2 SANDBOX_CLASS_GVISOR 3d -ate-demo-counter-substrate-microvm counter-microvm SANDBOX_CLASS_MICROVM ERROR 5h + expected := `ATESPACE NAME SANDBOX CLASS GOLDEN TAG ERROR AGE +ate-demo-counter-substrate counter SANDBOX_CLASS_GVISOR golden-tag 5m +ate-demo-counter-substrate counter-2 SANDBOX_CLASS_GVISOR 3d +ate-demo-counter-substrate-microvm counter-microvm SANDBOX_CLASS_MICROVM ERROR 5h ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) diff --git a/docs/api-guide.md b/docs/api-guide.md index b8aaef6f27..b6d185006b 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -360,13 +360,13 @@ snapshotsConfig: /atespaces//tags/ ``` -The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden actor lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/actors//snapshots/`. +The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden tag lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/tags/`. An actor takes a series of snapshots over its life, so it gets a prefix of its own and each snapshot sits below it. A tag holds exactly one, so the tag's prefix *is* its snapshot's. Both owners are keyed on their UID, so recreating an actor or tag under the same name never inherits its predecessor's objects. A pending tag records its base location in `status.storageLocation`; together with its atespace and UID, this identifies any partial copy to collect if creation fails. An owner is collected by deleting everything under its prefix, and it can delete nothing else. That is what makes a borrowed snapshot safe: an actor created from a tag points at a URI under `tags/`, which its own prefix does not cover. See [Snapshot lifetime](#snapshot-lifetime). -An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot`, a `Tag` in `status.snapshot`, and an `ActorTemplate` its golden one in `status.goldenSnapshotStatus.goldenSnapshot` — each an `ExternalSnapshot` carrying `snapshotUri` and the `contentScope` it captured. The URI is recorded when the snapshot is written. All three are server-owned: do not send them on input, and parse a URI only against the scheme above. +An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot` and a `Tag` in `status.snapshot`, each an `ExternalSnapshot` carrying `snapshotUri` and `contentScope`. The URI is recorded when the snapshot is written. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above. An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like: @@ -434,10 +434,12 @@ See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble ### The Golden Snapshot When an `ActorTemplate` is created: -1. Substrate starts a temporary **Golden Pod**. -2. It executes your workload containers as defined in the template. -3. Once the process is initialized, gVisor takes a **Golden Snapshot** (Version 0). -4. The template enters the `Ready` phase. +1. Substrate creates and resumes a temporary golden actor in `ate-golden`. +2. It waits for readiness (or the warm-up interval), then suspends the actor. +3. It creates a published tag named after the template UID, copying the snapshot into tag-owned storage. +4. It deletes the golden actor and records the tag reference in the template status. + +`CreateActor` uses an explicit `sourceTag` when supplied; otherwise it resolves the template's golden tag and records that snapshot on the new actor. If the golden tag is not ready yet, the actor starts without a snapshot and cold-boots even if the tag becomes ready before its first resume. The default does not populate the caller-owned `sourceTag` field. Deleting the template collects its golden tag and any unfinished golden actor. ### Resumption Lifecycle Once a template is `Ready`, creating an actor logically (via `kubectl ate create actor`) allows it to be resumed instantly on any free worker in the referenced `WorkerPool`. Substrate bypasses the standard container boot and restores the process directly from its last saved state. diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 3c30367a86..5a2f411428 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -1204,7 +1204,7 @@ wait_actortemplate_ready() { while ((SECONDS < deadline)); do if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then - snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenSnapshot.snapshotUri // empty' <<<"${json}") + snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}") if [[ -n "${snapshot}" ]]; then return 0 fi diff --git a/hack/verify-atenet-drain.sh b/hack/verify-atenet-drain.sh index 3678a2eae8..ff023f9fc8 100755 --- a/hack/verify-atenet-drain.sh +++ b/hack/verify-atenet-drain.sh @@ -94,7 +94,7 @@ log_step "preflight" # Ready means the template's golden snapshot exists; protojson omits empty # fields, so the key is only present once it is set. run_kubectl_ate get actor-template "${DEMO_POOL}" -a "${ATESPACE}" -o json 2>/dev/null \ - | grep -q '"goldenSnapshot"' \ + | grep -q '"goldenTag"' \ || fail "actor template ${ATESPACE}/${DEMO_POOL} has no golden snapshot; install the counter demo first" # Column 4 is STATUS; the header row's "ASSIGNED ACTOR" column name must not # trip the check. diff --git a/internal/e2e/template.go b/internal/e2e/template.go index 064dc3f226..f16198992e 100644 --- a/internal/e2e/template.go +++ b/internal/e2e/template.go @@ -162,7 +162,7 @@ func WaitForSubstrateTemplateReady(ctx context.Context, t *testing.T, clients *C }) if err == nil { lastStatus = at.GetStatus().GetGoldenSnapshotStatus() - if lastStatus.GetGoldenSnapshot().GetSnapshotUri() != "" { + if lastStatus.GetGoldenTag().GetName() != "" { return } if msg := lastStatus.GetErrorMessage(); msg != "" { diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 755d4ca550..1d958381fa 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -2271,12 +2271,11 @@ func (x *Limits) GetQuantity() string { type GoldenSnapshotStatus struct { state protoimpl.MessageState `protogen:"open.v1"` - // golden_snapshot is the external snapshot built for this version by - // ate-api, taken from the golden Actor in the reserved ate-golden system - // atespace. Set once state is READY. The golden Actor owns it. - // + // golden_tag owns the immutable snapshot built by the template controller. + // Set after tagging the snapshot and deleting the temporary golden Actor. // +k8s:optional - GoldenSnapshot *ExternalSnapshot `protobuf:"bytes,1,opt,name=golden_snapshot,json=goldenSnapshot,proto3" json:"golden_snapshot,omitempty"` + // +k8s:subfield(atespace)=+k8s:required + GoldenTag *ObjectRef `protobuf:"bytes,1,opt,name=golden_tag,json=goldenTag,proto3" json:"golden_tag,omitempty"` // take_golden_snapshot_at is when the golden-actor warmup ends and the // golden snapshot may be taken. TakeGoldenSnapshotAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=take_golden_snapshot_at,json=takeGoldenSnapshotAt,proto3" json:"take_golden_snapshot_at,omitempty"` @@ -2315,9 +2314,9 @@ func (*GoldenSnapshotStatus) Descriptor() ([]byte, []int) { return file_ateapi_proto_rawDescGZIP(), []int{21} } -func (x *GoldenSnapshotStatus) GetGoldenSnapshot() *ExternalSnapshot { +func (x *GoldenSnapshotStatus) GetGoldenTag() *ObjectRef { if x != nil { - return x.GoldenSnapshot + return x.GoldenTag } return nil } @@ -6957,9 +6956,10 @@ const file_ateapi_proto_rawDesc = "" + "\x06limits\x18\x01 \x03(\v2\x0e.ateapi.LimitsR\x06limits\"8\n" + "\x06Limits\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" + - "\bquantity\x18\x02 \x01(\tR\bquantity\"\xd1\x01\n" + - "\x14GoldenSnapshotStatus\x12A\n" + - "\x0fgolden_snapshot\x18\x01 \x01(\v2\x18.ateapi.ExternalSnapshotR\x0egoldenSnapshot\x12Q\n" + + "\bquantity\x18\x02 \x01(\tR\bquantity\"\xc0\x01\n" + + "\x14GoldenSnapshotStatus\x120\n" + + "\n" + + "golden_tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\tgoldenTag\x12Q\n" + "\x17take_golden_snapshot_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x14takeGoldenSnapshotAt\x12#\n" + "\rerror_message\x18\x03 \x01(\tR\ferrorMessage\"i\n" + "\x13ActorTemplateStatus\x12R\n" + @@ -7439,7 +7439,7 @@ var file_ateapi_proto_depIdxs = []int32{ 28, // 37: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources 31, // 38: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus 29, // 39: ateapi.Resources.limits:type_name -> ateapi.Limits - 9, // 40: ateapi.GoldenSnapshotStatus.golden_snapshot:type_name -> ateapi.ExternalSnapshot + 26, // 40: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef 108, // 41: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp 30, // 42: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus 3, // 43: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 6b8db07811..2a119f6a16 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -138,7 +138,7 @@ service Control { rpc ListActorTemplates(ListActorTemplatesRequest) returns (ListActorTemplatesResponse) {} // Delete an ActorTemplate together with its golden actor and golden - // snapshot in the ActorTemplate's namespace. + // tag in the reserved ate-golden atespace. rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) {} } @@ -807,12 +807,11 @@ message Limits { } message GoldenSnapshotStatus { - // golden_snapshot is the external snapshot built for this version by - // ate-api, taken from the golden Actor in the reserved ate-golden system - // atespace. Set once state is READY. The golden Actor owns it. - // + // golden_tag owns the immutable snapshot built by the template controller. + // Set after tagging the snapshot and deleting the temporary golden Actor. // +k8s:optional - ExternalSnapshot golden_snapshot = 1; + // +k8s:subfield(atespace)=+k8s:required + ObjectRef golden_tag = 1; // take_golden_snapshot_at is when the golden-actor warmup ends and the // golden snapshot may be taken. diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 69a154a2c1..f88dbd6700 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -157,7 +157,7 @@ type ControlClient interface { GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) ListActorTemplates(ctx context.Context, in *ListActorTemplatesRequest, opts ...grpc.CallOption) (*ListActorTemplatesResponse, error) // Delete an ActorTemplate together with its golden actor and golden - // snapshot in the ActorTemplate's namespace. + // tag in the reserved ate-golden atespace. DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) } @@ -597,7 +597,7 @@ type ControlServer interface { GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) ListActorTemplates(context.Context, *ListActorTemplatesRequest) (*ListActorTemplatesResponse, error) // Delete an ActorTemplate together with its golden actor and golden - // snapshot in the ActorTemplate's namespace. + // tag in the reserved ate-golden atespace. DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) mustEmbedUnimplementedControlServer() } From c81763135f18d1f3c1ea94e92b18ffc5b741a491 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Mon, 14 Sep 2026 13:09:21 +0000 Subject: [PATCH 2/3] Stop golden snapshot retries when the tag belongs to another resource Record ownership conflicts as terminal template failures so reconciliation does not retry indefinitely. Keep conflicting tags untouched and cover actor and template mismatches, repeated reconciliation, and periodic resync. Signed-off-by: Eitan Yarmush --- .../controlapi/template_reconciler.go | 3 +- .../controlapi/template_reconciler_test.go | 52 +++++++++++++++---- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 07f2828866..b17f8d3578 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -46,6 +46,7 @@ const ( ) const ( + reasonGoldenTagConflict = "GoldenTagConflict" reasonGoldenActorInvalid = "GoldenActorInvalid" reasonGoldenActorCrashed = "GoldenActorCrashed" reasonUnexpectedState = "GoldenActorUnexpectedState" @@ -215,7 +216,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource } if err == nil { if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) { - return 0, fmt.Errorf("golden tag belongs to another actor or template") + return 0, r.fail(ctx, tmpl, reasonGoldenTagConflict, "golden tag belongs to another actor or template") } if tag.GetStatus().GetSnapshot().GetSnapshotUri() != "" { return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef) diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index db7649c29a..d9fba7b2df 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -772,16 +772,13 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/tag-snapshot"}}, } - for _, scenario := range []string{"completed tag", "actor already deleted", "incomplete tag", "copy failure", "actor deletion failure", "tag deletion failure", "foreign tag"} { + for _, scenario := range []string{"completed tag", "actor already deleted", "incomplete tag", "copy failure", "actor deletion failure", "tag deletion failure"} { t.Run(scenario, func(t *testing.T) { control := &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: "gs://bucket/actor-snapshot"} switch scenario { - case "completed tag", "actor already deleted", "foreign tag": + case "completed tag", "actor already deleted": control.tag = proto.CloneOf(completed) control.exists = scenario != "actor already deleted" - if scenario == "foreign tag" { - control.tag.Status.ActorTemplateUid = "another-template" - } case "incomplete tag", "tag deletion failure": control.tag = proto.CloneOf(completed) control.tag.Status.Snapshot = nil @@ -797,7 +794,7 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { r := newTestTemplateReconciler(st, control) defer r.queue.ShutDown() _, err := r.reconcileOne(t.Context(), testTemplateRef) - wantErr := strings.Contains(scenario, "failure") || scenario == "foreign tag" + wantErr := strings.Contains(scenario, "failure") if (err != nil) != wantErr { t.Fatalf("reconcile = %v, want error %v", err, wantErr) } @@ -808,9 +805,6 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { if !control.exists { t.Fatal("deleted actor after tag failure") } - if scenario == "foreign tag" { - return - } control.tagErr, control.deleteErr, control.deleteTagErr = nil, nil, nil if _, err := r.reconcileOne(t.Context(), testTemplateRef); err != nil { t.Fatal(err) @@ -841,3 +835,43 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { }) } } + +func TestReconcileOne_GoldenTagConflict(t *testing.T) { + for _, scenario := range []string{"template", "actor name", "actor atespace"} { + t.Run(scenario, func(t *testing.T) { + tag := &ateapipb.Tag{ + SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}, + Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID}, + } + switch scenario { + case "template": + tag.Status.ActorTemplateUid = "another-template" + case "actor name": + tag.SourceActor.Name = "another-actor" + case "actor atespace": + tag.SourceActor.Atespace = "another-atespace" + } + control := &fakeGoldenControl{tag: proto.CloneOf(tag), exists: true} + st := newFakeTemplateStore(testTemplate()) + r := newTestTemplateReconciler(st, control) + defer r.queue.ShutDown() + for range 2 { + after, err := r.reconcileOne(t.Context(), testTemplateRef) + if err != nil || after != 0 { + t.Fatalf("reconcile = (%v, %v), want terminal failure without retry", after, err) + } + snapshotStatus := st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus() + if snapshotStatus.GetErrorMessage() != reasonGoldenTagConflict+": golden tag belongs to another actor or template" || snapshotStatus.GetGoldenTag() != nil { + t.Fatalf("unexpected golden snapshot status: %v", snapshotStatus) + } + if !proto.Equal(control.tag, tag) || !control.exists || len(control.tagReqs) != 0 || len(control.deleteReqs) != 0 { + t.Fatal("modified golden resources after ownership conflict") + } + } + r.resync(t.Context()) + if r.queue.Len() != 0 { + t.Fatal("resync queued a terminally failed template") + } + }) + } +} From 92b1fc1f0110df71e64246c5ca909a75d11399a2 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 15 Sep 2026 18:35:13 +0000 Subject: [PATCH 3/3] Cover template deletion and clarify golden tag recovery tests Exercise golden resource cleanup, partial failures, and retries through DeleteActorTemplate. Declare recovery and ownership-conflict inputs in tables and assert the number of tag copy attempts. Signed-off-by: Eitan Yarmush --- .../controlapi/actor_template_test.go | 120 ++++++++++++++++++ .../controlapi/template_reconciler_test.go | 78 +++++++----- 2 files changed, 164 insertions(+), 34 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/actor_template_test.go b/cmd/ateapi/internal/controlapi/actor_template_test.go index 15b3b1ffd1..cc02e6920f 100644 --- a/cmd/ateapi/internal/controlapi/actor_template_test.go +++ b/cmd/ateapi/internal/controlapi/actor_template_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/internal/resources" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" @@ -339,6 +340,125 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) { } } +func TestDeleteActorTemplate(t *testing.T) { + tests := []struct { + name string + actorDeleted bool + tagDeleted bool + pendingTag bool + // failPrefix makes object storage fail cleanup for this resource kind. + failPrefix string + wantActorAfterFailure bool + }{ + {name: "golden actor and tag"}, + {name: "golden actor already deleted", actorDeleted: true}, + {name: "golden tag absent", tagDeleted: true}, + {name: "no golden resources", actorDeleted: true, tagDeleted: true}, + {name: "incomplete golden tag", pendingTag: true}, + {name: "actor cleanup failure", failPrefix: "/actors/", wantActorAfterFailure: true}, + {name: "tag cleanup failure", failPrefix: "/tags/"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + persistence := newTestPersistence(t) + tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl") + templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl) + goldenRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()} + actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: goldenRef.Atespace, Name: goldenRef.Name}, + ActorTemplate: templateRef.ToObjectRef(), + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + }) + workflow, objects := newFinalizeWorkflow(persistence) + actorURI := mustActorSnapshotURI(t, tmpl, actor, "snapshot") + objects.PutSnapshot(t, actorURI, "manifest.json") + actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { + s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorURI.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL} + s.CurrentActorTemplateUid = tmpl.GetMetadata().GetUid() + }) + var tag *ateapipb.Tag + if tt.pendingTag { + tag = storetest.MustCreateTag(t, ctx, persistence, newPendingTestTag(t, goldenRef.Name, actor)) + } else { + var err error + tag, err = workflow.TagActorSnapshot(ctx, tagToCreate(goldenRef, goldenRef.Name)) + if err != nil { + t.Fatal(err) + } + } + tagRef := resources.TagRefFromTag(tag) + tagURI := mustReservedTagSnapshotURI(t, tag) + objects.PutSnapshot(t, tagURI, "manifest.json") + svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, objectStore: objects} + // The handler must request AnyState to clean up an active golden actor. + mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) { + s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + }) + if tt.actorDeleted { + if _, err := workflow.DeleteActor(ctx, goldenRef, true); err != nil { + t.Fatal(err) + } + } + if tt.tagDeleted { + if _, err := svc.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: tagRef.ToObjectRef()}); err != nil { + t.Fatal(err) + } + } + if tt.failPrefix != "" { + objects.OnDelete = func(_, key string) error { + if strings.Contains(key, tt.failPrefix) { + return errObjectStore + } + return nil + } + } + req := &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()} + deleted, err := svc.DeleteActorTemplate(ctx, req) + if tt.failPrefix != "" { + if !errors.Is(err, errObjectStore) { + t.Fatalf("DeleteActorTemplate = %v, want object storage error", err) + } + if _, err := persistence.GetActorTemplate(ctx, templateRef); err != nil { + t.Fatalf("template lost after cleanup failure: %v", err) + } + if _, err := persistence.GetTag(ctx, tagRef); err != nil { + t.Fatalf("tag lost after cleanup failure: %v", err) + } + _, actorErr := persistence.GetActor(ctx, goldenRef) + if tt.wantActorAfterFailure && actorErr != nil || !tt.wantActorAfterFailure && !errors.Is(actorErr, store.ErrNotFound) { + t.Fatalf("GetActor after failure = %v, want present %v", actorErr, tt.wantActorAfterFailure) + } + objects.OnDelete = nil + deleted, err = svc.DeleteActorTemplate(ctx, req) + } + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tmpl, deleted, protocmp.Transform()); diff != "" { + t.Fatalf("deleted template mismatch (-want +got):\n%s", diff) + } + if _, err := persistence.GetActorTemplate(ctx, templateRef); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("GetActorTemplate after delete = %v, want NotFound", err) + } + if _, err := persistence.GetActor(ctx, goldenRef); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("GetActor after delete = %v, want NotFound", err) + } + if _, err := persistence.GetTag(ctx, tagRef); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("GetTag after delete = %v, want NotFound", err) + } + for _, uri := range []resources.SnapshotURI{actorURI, tagURI} { + if got := objects.Snapshot(t, uri); len(got) != 0 { + t.Errorf("snapshot %s still holds %v", uri, got) + } + } + if _, err := svc.DeleteActorTemplate(ctx, req); status.Code(err) != codes.NotFound { + t.Fatalf("delete missing template = %v, want NotFound", err) + } + }) + } +} + func TestValidateGetActorTemplateRequest(t *testing.T) { tests := []struct { name string diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index d9fba7b2df..159383b972 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -772,29 +772,40 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED, Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/tag-snapshot"}}, } - for _, scenario := range []string{"completed tag", "actor already deleted", "incomplete tag", "copy failure", "actor deletion failure", "tag deletion failure"} { - t.Run(scenario, func(t *testing.T) { - control := &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: "gs://bucket/actor-snapshot"} - switch scenario { - case "completed tag", "actor already deleted": - control.tag = proto.CloneOf(completed) - control.exists = scenario != "actor already deleted" - case "incomplete tag", "tag deletion failure": - control.tag = proto.CloneOf(completed) - control.tag.Status.Snapshot = nil - if scenario == "tag deletion failure" { - control.deleteTagErr = errors.New("storage unavailable") - } - case "copy failure": - control.tagErr = errors.New("copy interrupted") - case "actor deletion failure": - control.deleteErr = errors.New("storage unavailable") + incomplete := proto.CloneOf(completed) + incomplete.Status.Snapshot = nil + tests := []struct { + name string + // tag is the golden tag an earlier pass left behind, if any. + tag *ateapipb.Tag + // actorDeleted seeds a pass that died after deleting the golden actor. + actorDeleted bool + // These errors fail one step of the first pass; the retry succeeds. + createTagErr error + deleteActorErr error + deleteTagErr error + // wantCreateTags counts copy attempts across both passes. + wantCreateTags int + }{ + {name: "completed tag", tag: completed, wantCreateTags: 0}, + {name: "actor already deleted", tag: completed, actorDeleted: true, wantCreateTags: 0}, + {name: "incomplete tag", tag: incomplete, wantCreateTags: 1}, + {name: "copy failure", createTagErr: errors.New("copy interrupted"), wantCreateTags: 2}, + {name: "actor deletion failure", deleteActorErr: errors.New("storage unavailable"), wantCreateTags: 1}, + {name: "tag deletion failure", tag: incomplete, deleteTagErr: errors.New("storage unavailable"), wantCreateTags: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + control := &fakeGoldenControl{ + tag: proto.CloneOf(tt.tag), exists: !tt.actorDeleted, + goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: "gs://bucket/actor-snapshot", + tagErr: tt.createTagErr, deleteErr: tt.deleteActorErr, deleteTagErr: tt.deleteTagErr, } st := newFakeTemplateStore(testTemplate()) r := newTestTemplateReconciler(st, control) defer r.queue.ShutDown() _, err := r.reconcileOne(t.Context(), testTemplateRef) - wantErr := strings.Contains(scenario, "failure") + wantErr := tt.createTagErr != nil || tt.deleteActorErr != nil || tt.deleteTagErr != nil if (err != nil) != wantErr { t.Fatalf("reconcile = %v, want error %v", err, wantErr) } @@ -822,10 +833,8 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 { t.Fatal("repeated golden actor warmup") } - if scenario == "completed tag" || scenario == "actor already deleted" { - if len(control.tagReqs) != 0 { - t.Fatal("recreated completed tag") - } + if got := len(control.tagReqs); got != tt.wantCreateTags { + t.Fatalf("CreateTag calls = %d, want %d", got, tt.wantCreateTags) } for _, req := range control.tagReqs { if !proto.Equal(req.Tag.SourceActor, ref) || req.Tag.Scope != ateapipb.TagScope_TAG_SCOPE_PUBLISHED || req.Tag.Metadata.Name != ref.Name { @@ -837,19 +846,20 @@ func TestReconcileOne_GoldenTagRecovery(t *testing.T) { } func TestReconcileOne_GoldenTagConflict(t *testing.T) { - for _, scenario := range []string{"template", "actor name", "actor atespace"} { - t.Run(scenario, func(t *testing.T) { + tests := []struct { + name string + templateUID string + sourceActor *ateapipb.ObjectRef + }{ + {name: "template", templateUID: "another-template", sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}}, + {name: "actor name", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "another-actor"}}, + {name: "actor atespace", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: "another-atespace", Name: testTemplateUID}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { tag := &ateapipb.Tag{ - SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}, - Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID}, - } - switch scenario { - case "template": - tag.Status.ActorTemplateUid = "another-template" - case "actor name": - tag.SourceActor.Name = "another-actor" - case "actor atespace": - tag.SourceActor.Atespace = "another-atespace" + SourceActor: tt.sourceActor, + Status: &ateapipb.TagStatus{ActorTemplateUid: tt.templateUID}, } control := &fakeGoldenControl{tag: proto.CloneOf(tag), exists: true} st := newFakeTemplateStore(testTemplate())