Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions cmd/ateom-gvisor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"github.com/agent-substrate/substrate/internal/childreap"
"github.com/agent-substrate/substrate/internal/contextlogging"
"github.com/agent-substrate/substrate/internal/imagecache"
"github.com/agent-substrate/substrate/internal/ocispec"
"github.com/agent-substrate/substrate/internal/otlprelay"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
"github.com/agent-substrate/substrate/internal/readyz"
Expand Down Expand Up @@ -637,7 +638,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload
// Contract with atelet:
//
// * Correct runsc version is downloaded and placed on disk.
// * All OCI bundles are set up, including for "pause" container.
// * All OCI bundles are set up, including for the pause container.

egress, err := s.prepareActorEgress(ctx, req.GetActorUid(), req.GetEgressGateway())
if err != nil {
Expand Down Expand Up @@ -687,14 +688,14 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload
// upper — because mounting is ateom's job (atelet runs with no
// capabilities); runsc's gofer resolves the mount in this pod's mount
// namespace.
if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), "pause")); err != nil {
if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ocispec.PauseContainer)); err != nil {
return nil, fmt.Errorf("while composing pause rootfs: %w", err)
}
containersToDelete = append(containersToDelete, "pause")
if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil {
containersToDelete = append(containersToDelete, ocispec.PauseContainer)
if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil {
return nil, fmt.Errorf("while creating pause container: %w", err)
}
if err := rcmd.cmdStart(ctx, os.Stdout, "pause"); err != nil {
if err := rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer); err != nil {
return nil, fmt.Errorf("while starting pause container: %w", err)
}

Expand Down Expand Up @@ -773,7 +774,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
if !hasDurableVolumes(req.GetSpec().GetContainers()) {
return nil, fmt.Errorf("no durable-dir volumes found for DATA snapshot")
}
if err := rcmd.cmdPause(ctx, "pause"); err != nil {
if err := rcmd.cmdPause(ctx, ocispec.PauseContainer); err != nil {
return nil, fmt.Errorf("while pausing pause container: %w", err)
}
tarErr := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointPath)
Expand All @@ -782,7 +783,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
// fail the resume instantly and leave the sandbox paused forever.
resumeCtx, cancelResume := context.WithTimeout(context.WithoutCancel(ctx), resumeTimeout)
defer cancelResume()
if err := rcmd.cmdResume(resumeCtx, "pause"); err != nil {
if err := rcmd.cmdResume(resumeCtx, ocispec.PauseContainer); err != nil {
return nil, fmt.Errorf("while resuming pause container: %w", err)
}
if tarErr != nil {
Expand All @@ -791,7 +792,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL:
// Checkpoint pause container (root of the sandbox)
// TODO: Consider pause -> tar -> resume -> checkpoint order for better failure handling.
if err := rcmd.cmdCheckpoint(ctx, "pause", checkpointPath); err != nil {
if err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath); err != nil {
return nil, fmt.Errorf("while checkpointing pause: %w", err)
}
if hasDurableVolumes(req.GetSpec().GetContainers()) {
Expand Down Expand Up @@ -860,15 +861,15 @@ func (r *runsc) stopContainers(ctx context.Context, containers []*ateompb.Contai
_ = r.cmdKill(ctx, ctr.GetName(), "SIGKILL")
_ = r.cmdWait(ctx, ctr.GetName())
}
_ = r.cmdKill(ctx, "pause", "SIGKILL")
_ = r.cmdWait(ctx, "pause")
_ = r.cmdKill(ctx, ocispec.PauseContainer, "SIGKILL")
_ = r.cmdWait(ctx, ocispec.PauseContainer)
}

func (r *runsc) cleanupContainers(ctx context.Context, containers []*ateompb.Container) error {
// Check state of all containers to mimic containerd.
//
// Without this, `runsc delete` occasionally throws an error.
if err := r.cmdState(ctx, "pause"); err != nil {
if err := r.cmdState(ctx, ocispec.PauseContainer); err != nil {
return fmt.Errorf("while checking state of pause container: %w", err)
}
for _, ctr := range containers {
Expand All @@ -883,7 +884,7 @@ func (r *runsc) cleanupContainers(ctx context.Context, containers []*ateompb.Con
}
}

if err := r.cmdDelete(ctx, "pause"); err != nil {
if err := r.cmdDelete(ctx, ocispec.PauseContainer); err != nil {
return fmt.Errorf("while deleting pause container: %w", err)
}

Expand Down Expand Up @@ -915,7 +916,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
// Contract with atelet:
//
// * Correct runsc version is downloaded and placed on disk.
// * All OCI bundles are set up, including for "pause" container.
// * All OCI bundles are set up, including for the pause container.
// * Checkpoint downloaded and placed on disk

egress, err := s.prepareActorEgress(ctx, req.GetActorUid(), req.GetEgressGateway())
Expand Down Expand Up @@ -967,27 +968,27 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
// Compose the pause rootfs before create (see RunWorkload). runsc restore
// only needs the rootfs to hold the correct content; whether it came from
// an untar or an overlay of cached layers is transparent to it.
if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), "pause")); err != nil {
if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ocispec.PauseContainer)); err != nil {
return nil, fmt.Errorf("while composing pause rootfs: %w", err)
}

switch req.GetScope() {
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA:
// Create and start pause container (cold boot with durable-dir volumes restored)
containersToDelete = append(containersToDelete, "pause")
if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil {
containersToDelete = append(containersToDelete, ocispec.PauseContainer)
if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil {
return nil, fmt.Errorf("while creating pause container: %w", err)
}
if err := rcmd.cmdStart(ctx, os.Stdout, "pause"); err != nil {
if err := rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer); err != nil {
return nil, fmt.Errorf("while starting pause container: %w", err)
}
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN:
// Create and restore pause container
containersToDelete = append(containersToDelete, "pause")
if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil {
containersToDelete = append(containersToDelete, ocispec.PauseContainer)
if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil {
return nil, fmt.Errorf("while creating pause container: %w", err)
}
if err := rcmd.cmdRestore(ctx, os.Stdout, "pause", checkpointDir); err != nil {
if err := rcmd.cmdRestore(ctx, os.Stdout, ocispec.PauseContainer, checkpointDir); err != nil {
return nil, fmt.Errorf("while restoring pause container: %w", err)
}
default:
Expand Down
9 changes: 5 additions & 4 deletions cmd/ateom-gvisor/runsc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"testing"

"github.com/agent-substrate/substrate/internal/ateompath"
"github.com/agent-substrate/substrate/internal/ocispec"
)

func TestKillArgs(t *testing.T) {
Expand Down Expand Up @@ -70,13 +71,13 @@ func TestPauseArgs(t *testing.T) {
actorUID: "test-actor-123",
}

got := r.pauseArgs("pause")
got := r.pauseArgs(ocispec.PauseContainer)
want := []string{
"-log-format", "json",
"--alsologtostderr",
"-root", ateompath.RunSCStateDir("test-actor-123"),
"pause",
"pause",
ocispec.PauseContainer,
}

if !reflect.DeepEqual(got, want) {
Expand All @@ -90,13 +91,13 @@ func TestResumeArgs(t *testing.T) {
actorUID: "test-actor-123",
}

got := r.resumeArgs("pause")
got := r.resumeArgs(ocispec.PauseContainer)
want := []string{
"-log-format", "json",
"--alsologtostderr",
"-root", ateompath.RunSCStateDir("test-actor-123"),
"resume",
"pause",
ocispec.PauseContainer,
}

if !reflect.DeepEqual(got, want) {
Expand Down
7 changes: 4 additions & 3 deletions cmd/ateom-gvisor/stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"google.golang.org/grpc/status"

"github.com/agent-substrate/substrate/cmd/ateom-gvisor/internal/cgroupstats"
"github.com/agent-substrate/substrate/internal/ocispec"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
"github.com/agent-substrate/substrate/internal/resources"
)
Expand All @@ -44,7 +45,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup"
// the sentry. runsc starts that process from the root container's create and
// from inside that container's cgroup — container.createRoot wraps the sandbox
// and gofer spawn in cgroup.RunInCgroup — so the sentry lands in the leaf of
// "pause", the first container RunWorkload and RestoreWorkload create.
// the pause container, the first one RunWorkload and RestoreWorkload create.
//
// The leaf is a direct child of the delegated scope rather than of ateom's own
// cgroup, because runsc resolves cgroupsPath against the parent of the cgroup
Expand All @@ -70,7 +71,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup"
// What the leaf holds besides the actor's own work: the sentry's own overhead
// (its Go heap, page tables, netstack) and the gofers. Process listings taken
// on a live node in #161 put runsc-sandbox and both gofers — the pause
// container's and the actor container's — in the "pause" cgroup. Those runs
// container's and the actor container's — in the pause cgroup. Those runs
// predate #496, so they establish the leaf name and the fact that everything
// lands in one leaf, not the absolute path, which #496's delegation moved under
// the pod scope.
Expand All @@ -86,7 +87,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup"
// The name has to agree with the cgroupsPath convention in
// ocispec.ShapeGVisor, which is "/" + containerName relative to the same
// scope.
const sandboxCgroupContainer = "pause"
const sandboxCgroupContainer = ocispec.PauseContainer

// GetWorkloadStats implements ateompb.Ateom/GetWorkloadStats.
//
Expand Down
6 changes: 4 additions & 2 deletions internal/ocispec/gvisor.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@ import (
"github.com/opencontainers/runtime-spec/specs-go"
)

// PauseContainer is the name of the sandbox root container.
const PauseContainer = "pause"
// PauseContainer is the name of the sandbox root container. The underscore
// keeps it outside the k8s-short-name an ActorTemplate container
// name is drawn from, so no actor container can collide with it.
const PauseContainer = "_pause"

// resolvConf is the host resolver config bound into the sandbox.
const resolvConf = "/etc/resolv.conf"
Expand Down
Loading