diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f895107 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +.claude +dist +*.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f7fb3ec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1 +# +# Host agent image: stages the static linuxaid-cli onto the node and runs the +# OpenVox agent in the host's namespaces via nsenter (see deploy/entrypoint.sh). +# Runs as the per-node Job that the fan-out launcher spawns. The node's cert is +# provided pre-signed (obmondo-clientcert), so there is nothing to enroll. + +FROM golang:1.24-alpine AS build +WORKDIR /src +RUN apk add --no-cache git +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . +ARG VERSION=spike +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 go build -trimpath -ldflags="-X main.Version=${VERSION} -s -w" -o /out/linuxaid-cli ./cmd/linuxaid-cli + +FROM alpine:3.20 +# git + openssh-client: apply mode clones the puppet code in-container into the +# /opt/obmondo hostPath (the host itself is not required to have git). +RUN apk add --no-cache bash util-linux ca-certificates git openssh-client +COPY --from=build /out/linuxaid-cli /usr/local/bin/linuxaid-cli +COPY deploy/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/Dockerfile.launcher b/Dockerfile.launcher new file mode 100644 index 0000000..91c295c --- /dev/null +++ b/Dockerfile.launcher @@ -0,0 +1,21 @@ +# syntax=docker/dockerfile:1 +# +# Fan-out launcher image: the in-cluster controller (linuxaid-agent) that lists +# nodes and creates one per-node agent Job each run. It is only a Kubernetes API +# client — no host access — so it runs non-root with no extra tooling. + +FROM golang:1.24-alpine AS build +WORKDIR /src +RUN apk add --no-cache git +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . +ARG VERSION=spike +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 go build -trimpath -ldflags="-X main.Version=${VERSION} -s -w" -o /out/linuxaid-agent ./cmd/linuxaid-agent + +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates +COPY --from=build /out/linuxaid-agent /usr/local/bin/linuxaid-agent +USER 65534:65534 +ENTRYPOINT ["/usr/local/bin/linuxaid-agent"] diff --git a/cmd/linuxaid-agent/main.go b/cmd/linuxaid-agent/main.go new file mode 100644 index 0000000..dff47c6 --- /dev/null +++ b/cmd/linuxaid-agent/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "log/slog" + "os" + + "github.com/spf13/cobra" +) + +var Version string + +var rootCmd = &cobra.Command{ + Use: "linuxaid-agent", + Short: "In-cluster controller for running LinuxAid OpenVox agents on Kubernetes nodes", + Version: Version, + CompletionOptions: cobra.CompletionOptions{ + HiddenDefaultCmd: true, + }, +} + +func main() { + if err := rootCmd.Execute(); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } +} diff --git a/cmd/linuxaid-agent/operator.go b/cmd/linuxaid-agent/operator.go new file mode 100644 index 0000000..b860c34 --- /dev/null +++ b/cmd/linuxaid-agent/operator.go @@ -0,0 +1,531 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/spf13/cobra" +) + +const saDir = "/var/run/secrets/kubernetes.io/serviceaccount" + +const ( + defaultReconcileInterval = 4 * time.Hour + defaultTTLSeconds = 600 + defaultActiveDeadlineSeconds = 1200 +) + +var ( + opImage string + opImagePullPolicy string + opCertname string + opEnforce bool + opOpenvoxEnv string + opPuppetServer string + opSecretName string + opHostPath string + opNodeSelector string + opNamePrefix string + opNamespace string + opTTLSeconds int + opActiveDeadline int + opBackoffLimit int + opInterval time.Duration + opNode string + opControlRepoURL string + opControlRepoRef string + opHieraConfigMap string + opGitSecretName string +) + +var operatorCmd = &cobra.Command{ + Use: "operator", + Short: "Reconcile one agent Job per node as a daemon, or once for a single --node", + RunE: func(_ *cobra.Command, _ []string) error { + return runOperator() + }, +} + +func init() { + f := operatorCmd.Flags() + f.StringVar(&opImage, "image", "", "agent container image (required)") + f.StringVar(&opImagePullPolicy, "image-pull-policy", "IfNotPresent", "imagePullPolicy for the agent Jobs") + f.StringVar(&opCertname, "certname", "", "obmondo-clientcert CN, shared by all nodes (required)") + f.BoolVar(&opEnforce, "enforce", false, "apply changes (puppet --no-noop) instead of report-only") + f.StringVar(&opOpenvoxEnv, "openvox-environment", "master", "LinuxAid/OpenVox environment") + f.StringVar(&opPuppetServer, "puppet-server", "", "optional puppetserver override") + f.StringVar(&opSecretName, "cert-secret", "obmondo-clientcert", "secret holding the pre-signed client cert") + f.StringVar(&opHostPath, "host-path", "/opt/obmondo", "host path for staging the CLI binary") + f.StringVar(&opNodeSelector, "node-selector", "", "label selector to filter nodes (empty = all)") + f.StringVar(&opNamePrefix, "name-prefix", "linuxaid-agents", "prefix for the per-node Job names") + f.StringVar(&opNamespace, "namespace", "", "namespace to create Jobs in (default: the pod's namespace)") + f.IntVar(&opTTLSeconds, "ttl-seconds", defaultTTLSeconds, "ttlSecondsAfterFinished on the agent Jobs") + f.IntVar(&opActiveDeadline, "active-deadline-seconds", defaultActiveDeadlineSeconds, "activeDeadlineSeconds on the agent Jobs") + f.IntVar(&opBackoffLimit, "backoff-limit", 1, "backoffLimit on the agent Jobs") + f.DurationVar(&opInterval, "interval", defaultReconcileInterval, "reconcile interval in daemon mode") + f.StringVar(&opNode, "node", "", "create a Job for a single node and exit (skips if one already exists)") + f.StringVar(&opControlRepoURL, "control-repo-url", "", "git URL of the puppet control-repo; when set the agent Jobs run masterless puppet apply") + f.StringVar(&opControlRepoRef, "control-repo-ref", "", "tag (or ref) of the control-repo to checkout, from the chart's linuxaid.tag (default: latest tag)") + f.StringVar(&opHieraConfigMap, "hiera-configmap", "", "ConfigMap with the rendered Helm-values hiera data, mounted into the agent Jobs at /hiera-data") + f.StringVar(&opGitSecretName, "git-secret", "", "secret with git credentials (ssh-privatekey or token key) mounted into the agent Jobs at /git-credentials") + rootCmd.AddCommand(operatorCmd) +} + +// runOperator dispatches to a single-node run (--node) or the daemon loop. +func runOperator() error { + if opImage == "" || opCertname == "" { + return fmt.Errorf("--image and --certname are required") + } + + cfg, err := inClusterConfig() + if err != nil { + return err + } + + if opNode != "" { + return runOnce(cfg, opNode) + } + return runDaemon(cfg) +} + +// runOnce creates the Job for a single named node and returns. A pre-existing Job +// (still running or within its TTL) is left untouched, so this is safe to trigger +// repeatedly for the same node. +func runOnce(cfg *clusterConfig, node string) error { + ok, err := cfg.nodeExists(node) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("node %q not found in cluster", node) + } + + created, err := createJobForNode(cfg, node) + if err != nil { + return err + } + if !created { + slog.Info("job already present, skipped", slog.String("node", node)) + return nil + } + slog.Info("created job", slog.String("node", node)) + return nil +} + +// runDaemon reconciles immediately, then every opInterval, until SIGINT/SIGTERM. +// A failed reconcile is logged and retried on the next tick rather than crashing +// the daemon. +func runDaemon(cfg *clusterConfig) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + ticker := time.NewTicker(opInterval) + defer ticker.Stop() + + slog.Info("operator started", + slog.Duration("interval", opInterval), + slog.String("namespace", cfg.namespace), + slog.Bool("enforce", opEnforce)) + + for { + if err := reconcile(cfg); err != nil { + slog.Error("reconcile failed", slog.Any("error", err)) + } + select { + case <-ctx.Done(): + slog.Info("shutting down") + return nil + case <-ticker.C: + } + } +} + +// reconcile lists the matching nodes and ensures one agent Job exists per node. +func reconcile(cfg *clusterConfig) error { + nodes, err := cfg.listNodes(opNodeSelector) + if err != nil { + return err + } + if len(nodes) == 0 { + slog.Warn("no nodes matched", slog.String("selector", opNodeSelector)) + return nil + } + + slog.Info("reconciling", slog.Int("nodes", len(nodes)), slog.Bool("enforce", opEnforce)) + + var failed int + for _, node := range nodes { + created, err := createJobForNode(cfg, node) + switch { + case err != nil: + slog.Error("create job", slog.String("node", node), slog.Any("error", err)) + failed++ + case created: + slog.Info("created job", slog.String("node", node)) + default: + slog.Info("skipped, already present", slog.String("node", node)) + } + } + + if failed > 0 { + return fmt.Errorf("%d of %d nodes failed", failed, len(nodes)) + } + return nil +} + +// createJobForNode builds and POSTs the agent Job for one node. created=false with a +// nil error means a Job already exists (HTTP 409) — the source of at-most-one Job per node. +func createJobForNode(cfg *clusterConfig, node string) (bool, error) { + slug := slugify(node) + name := jobName(opNamePrefix, slug) + + manifest, err := buildJobManifest(node, name, slug) + if err != nil { + return false, fmt.Errorf("build job for %s: %w", node, err) + } + return cfg.createJob(manifest) +} + +type clusterConfig struct { + host string + namespace string + client *http.Client +} + +func inClusterConfig() (*clusterConfig, error) { + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if host == "" || port == "" { + return nil, fmt.Errorf("not running in-cluster: KUBERNETES_SERVICE_HOST/PORT are unset") + } + + // Sanity-check the token exists now; it is re-read per request (see do) because the + // kubelet rotates bound service-account tokens on disk during a long-running daemon. + if _, err := os.ReadFile(saDir + "/token"); err != nil { + return nil, fmt.Errorf("read service account token: %w", err) + } + + caPEM, err := os.ReadFile(saDir + "/ca.crt") + if err != nil { + return nil, fmt.Errorf("read cluster CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("parse cluster CA") + } + + namespace := opNamespace + if namespace == "" { + b, err := os.ReadFile(saDir + "/namespace") + if err != nil { + return nil, fmt.Errorf("read namespace: %w", err) + } + namespace = strings.TrimSpace(string(b)) + } + + return &clusterConfig{ + host: fmt.Sprintf("https://%s:%s", host, port), + namespace: namespace, + client: &http.Client{ + Timeout: 30 * time.Second, //nolint:mnd + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}, + }, + }, nil +} + +func (c *clusterConfig) do(method, path string, body []byte) (*http.Response, error) { + token, err := os.ReadFile(saDir + "/token") + if err != nil { + return nil, fmt.Errorf("read service account token: %w", err) + } + + var r io.Reader = http.NoBody + if body != nil { + r = bytes.NewReader(body) + } + req, err := http.NewRequest(method, c.host+path, r) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(token))) + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return c.client.Do(req) +} + +// nodeList is the trimmed shape decoded from the Kubernetes node-list response. +type nodeList struct { + Items []nodeListItem `json:"items"` +} + +type nodeListItem struct { + Metadata objMeta `json:"metadata"` +} + +func (c *clusterConfig) listNodes(selector string) ([]string, error) { + path := "/api/v1/nodes" + if selector != "" { + path += "?labelSelector=" + url.QueryEscape(selector) + } + + resp, err := c.do(http.MethodGet, path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("list nodes: %s: %s", resp.Status, strings.TrimSpace(string(b))) + } + + var out nodeList + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + + names := make([]string, 0, len(out.Items)) + for _, item := range out.Items { + names = append(names, item.Metadata.Name) + } + return names, nil +} + +// nodeExists reports whether a node object with the given name is present, so a --node +// run fails fast on a typo instead of creating a Job that never schedules. +func (c *clusterConfig) nodeExists(name string) (bool, error) { + resp, err := c.do(http.MethodGet, "/api/v1/nodes/"+url.PathEscape(name), nil) + if err != nil { + return false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + default: + b, _ := io.ReadAll(resp.Body) + return false, fmt.Errorf("get node %q: %s: %s", name, resp.Status, strings.TrimSpace(string(b))) + } +} + +// createJob POSTs the Job; a 409 means it already exists (still running) and is +// treated as a skip, which is what gives us no-concurrency per node. +func (c *clusterConfig) createJob(manifest []byte) (created bool, err error) { + resp, err := c.do(http.MethodPost, fmt.Sprintf("/apis/batch/v1/namespaces/%s/jobs", c.namespace), manifest) + if err != nil { + return false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusCreated, http.StatusOK: + return true, nil + case http.StatusConflict: + return false, nil + default: + b, _ := io.ReadAll(resp.Body) + return false, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b))) + } +} + +func buildJobManifest(node, name, slug string) ([]byte, error) { + env := []envVar{ + {Name: "CERTNAME", Value: opCertname}, + {Name: "ENFORCE", Value: fmt.Sprintf("%t", opEnforce)}, + {Name: "OPENVOX_ENVIRONMENT", Value: opOpenvoxEnv}, + {Name: "CLIENT_CERT_DIR", Value: "/obmondo-clientcert"}, + } + if opPuppetServer != "" { + env = append(env, envVar{Name: "PUPPET_SERVER", Value: opPuppetServer}) + } + if opControlRepoURL != "" { + env = append(env, envVar{Name: "CONTROL_REPO_URL", Value: opControlRepoURL}) + } + if opControlRepoRef != "" { + env = append(env, envVar{Name: "CONTROL_REPO_REF", Value: opControlRepoRef}) + } + + mounts := []volumeMount{ + {Name: "obmondo-clientcert", MountPath: "/obmondo-clientcert", ReadOnly: true}, + {Name: "host-obmondo", MountPath: opHostPath}, + } + volumes := []volume{ + {Name: "obmondo-clientcert", Secret: &secretVol{SecretName: opSecretName}}, + {Name: "host-obmondo", HostPath: &hostPathVol{Path: opHostPath, Type: "DirectoryOrCreate"}}, + } + if opGitSecretName != "" { + mounts = append(mounts, volumeMount{Name: "git-credentials", MountPath: "/git-credentials", ReadOnly: true}) + volumes = append(volumes, volume{Name: "git-credentials", Secret: &secretVol{SecretName: opGitSecretName}}) + } + if opHieraConfigMap != "" { + mounts = append(mounts, volumeMount{Name: "hiera-data", MountPath: "/hiera-data", ReadOnly: true}) + volumes = append(volumes, volume{Name: "hiera-data", ConfigMap: &configMapVol{Name: opHieraConfigMap}}) + } + + labels := map[string]string{ + "app.kubernetes.io/name": "linuxaid-agents", + "app.kubernetes.io/managed-by": "linuxaid-agent", + "linuxaid-agents.obmondo.com/node": slug, + } + + j := k8sJob{ + APIVersion: "batch/v1", + Kind: "Job", + Metadata: objMeta{Name: name, Labels: labels}, + Spec: jobSpec{ + BackoffLimit: opBackoffLimit, + ActiveDeadlineSeconds: opActiveDeadline, + TTLSecondsAfterFinished: opTTLSeconds, + Template: podTemplate{ + Metadata: objMeta{Labels: labels}, + Spec: podSpec{ + RestartPolicy: "Never", + HostPID: true, + AutomountServiceAccountToken: false, + NodeName: node, + Tolerations: []toleration{{Operator: "Exists"}}, + Containers: []container{{ + Name: "agent", + Image: opImage, + ImagePullPolicy: opImagePullPolicy, + SecurityContext: secCtx{Privileged: true, RunAsUser: 0}, + Env: env, + VolumeMounts: mounts, + }}, + Volumes: volumes, + }, + }, + }, + } + return json.Marshal(j) +} + +// slugify turns a node name into a DNS-1123 label fragment. +func slugify(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } else { + b.WriteRune('-') + } + } + slug := b.String() + for strings.Contains(slug, "--") { + slug = strings.ReplaceAll(slug, "--", "-") + } + slug = strings.Trim(slug, "-") + if len(slug) > 40 { //nolint:mnd + slug = strings.Trim(slug[:40], "-") + } + return slug +} + +// jobName keeps the name a valid DNS-1123 label with room for the pod suffix. +func jobName(prefix, slug string) string { + name := prefix + "-" + slug + if len(name) > 57 { //nolint:mnd + name = strings.TrimRight(name[:57], "-") + } + return name +} + +type k8sJob struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata objMeta `json:"metadata"` + Spec jobSpec `json:"spec"` +} + +type objMeta struct { + Name string `json:"name,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +type jobSpec struct { + BackoffLimit int `json:"backoffLimit"` + ActiveDeadlineSeconds int `json:"activeDeadlineSeconds"` + TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` + Template podTemplate `json:"template"` +} + +type podTemplate struct { + Metadata objMeta `json:"metadata,omitempty"` + Spec podSpec `json:"spec"` +} + +type podSpec struct { + RestartPolicy string `json:"restartPolicy"` + HostPID bool `json:"hostPID"` + AutomountServiceAccountToken bool `json:"automountServiceAccountToken"` + NodeName string `json:"nodeName"` + Tolerations []toleration `json:"tolerations,omitempty"` + Containers []container `json:"containers"` + Volumes []volume `json:"volumes"` +} + +type toleration struct { + Operator string `json:"operator"` +} + +type container struct { + Name string `json:"name"` + Image string `json:"image"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + SecurityContext secCtx `json:"securityContext"` + Env []envVar `json:"env,omitempty"` + VolumeMounts []volumeMount `json:"volumeMounts"` +} + +type secCtx struct { + Privileged bool `json:"privileged"` + RunAsUser int `json:"runAsUser"` +} + +type envVar struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type volumeMount struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + ReadOnly bool `json:"readOnly,omitempty"` +} + +type volume struct { + Name string `json:"name"` + Secret *secretVol `json:"secret,omitempty"` + HostPath *hostPathVol `json:"hostPath,omitempty"` + ConfigMap *configMapVol `json:"configMap,omitempty"` +} + +type secretVol struct { + SecretName string `json:"secretName"` +} + +type configMapVol struct { + Name string `json:"name"` +} + +type hostPathVol struct { + Path string `json:"path"` + Type string `json:"type,omitempty"` +} diff --git a/cmd/linuxaid-cli/run-openvox.go b/cmd/linuxaid-cli/run-openvox.go index 295a916..67b90fd 100644 --- a/cmd/linuxaid-cli/run-openvox.go +++ b/cmd/linuxaid-cli/run-openvox.go @@ -1,8 +1,13 @@ package main import ( + "fmt" "log/slog" + "os" + "path/filepath" + "strings" + "gitea.obmondo.com/EnableIT/linuxaid-cli/constant" "gitea.obmondo.com/EnableIT/linuxaid-cli/helper" "gitea.obmondo.com/EnableIT/linuxaid-cli/pkg/checkconnectivity" api "gitea.obmondo.com/EnableIT/linuxaid-cli/pkg/obmondo" @@ -10,61 +15,128 @@ import ( "github.com/spf13/cobra" ) +var ( + // runOpenvoxEnforce applies changes (puppet --no-noop) instead of the default report-only (--noop). + runOpenvoxEnforce bool + // runOpenvoxApply compiles the catalog locally (puppet apply) from a cloned + // control-repo instead of asking a puppetserver (puppet agent). + runOpenvoxApply bool + runOpenvoxEnvPath string + runOpenvoxEnvName string + runOpenvoxHieraConf string +) + var runOpenvoxCmd = &cobra.Command{ Use: "run-openvox", Short: "Execute run-openvox command", Long: "A longer description of run-openvox command", Example: `$ linuxaid-cli run-openvox --certname web01.example`, Run: func(*cobra.Command, []string) { - RunOpenvox() + RunOpenvox(runOpenvoxEnforce) }, } -// Run the puppet agent in noop mode for now -func runOpenvoxAgent() error { - // Puppet run execution returns total 5 status codes - // - // 0: The run succeeded with no changes or failures; the system was already in the desired state. - // 1: The run failed, or wasn't attempted due to another run already in progress. - // 2: The run succeeded, and some resources were changed. - // 4: The run succeeded, and some resources failed. - // 6: The run succeeded, and included both changes and failures. - // [Source: https://www.puppet.com/docs/puppet/7/man/agent.html#usage-notes] - // - // We throw error at status code 1, and return. - // Status codes other than 2 are considered as warning. - // Status code 0 doesn't count as error, so no need to handle it. - +// runPuppet executes a puppet command and maps its exit status. +// +// Puppet run execution returns total 5 status codes +// +// 0: The run succeeded with no changes or failures; the system was already in the desired state. +// 1: The run failed, or wasn't attempted due to another run already in progress. +// 2: The run succeeded, and some resources were changed. +// 4: The run succeeded, and some resources failed. +// 6: The run succeeded, and included both changes and failures. +// [Source: https://www.puppet.com/docs/puppet/7/man/agent.html#usage-notes] +// +// We throw error at status code 1, and return. +// Status codes other than 2 are considered as warning. +// Status code 0 doesn't count as error, so no need to handle it. +func runPuppet(puppetCmd string) error { statusCodeFailed := 1 statusCodeSucceededWithChanges := 2 - slog.Info("executing the puppet agent command") - cmdPipe := script.Exec("/opt/puppetlabs/bin/puppet agent -t --noop") + slog.Info("executing the puppet command", slog.String("command", puppetCmd)) + cmdPipe := script.Exec(puppetCmd) _, err := cmdPipe.Stdout() if err != nil { // When encountering status code 1, consider it as an error, and return. if cmdPipe.ExitStatus() == statusCodeFailed { - slog.Error("puppet agent command execution failed", slog.String("status", err.Error())) + slog.Error("puppet command execution failed", slog.String("status", err.Error())) return err } // When encountering status codes other than 2, just log it as a warning. if cmdPipe.ExitStatus() != statusCodeSucceededWithChanges { - slog.Warn("puppet agent run succeeded, but with failures", slog.String("status", err.Error())) + slog.Warn("puppet run succeeded, but with failures", slog.String("status", err.Error())) } } - slog.Info("completed the puppet agent command execution") + slog.Info("completed the puppet command execution") return nil } +// runOpenvoxAgent runs the puppet agent against the puppetserver: report-only (--noop) +// by default, or applying changes (--no-noop --detailed-exitcodes) when enforce is set. +func runOpenvoxAgent(enforce bool) error { + puppetCmd := "/opt/puppetlabs/bin/puppet agent -t --noop" + if enforce { + puppetCmd = "/opt/puppetlabs/bin/puppet agent -t --no-noop --detailed-exitcodes" + } + + slog.Info("executing the puppet agent command", slog.Bool("enforce", enforce)) + return runPuppet(puppetCmd) +} + +// applyOpenvox runs masterless puppet apply against the environment cloned under +// environmentPath. The environment's own environment.conf and hiera.yaml drive the +// modulepath and hierarchy, exactly as on the puppetserver; hieraConfig injects the +// Helm-values data as the (higher-precedence) global hiera layer. +func applyOpenvox(enforce bool, environmentPath, environment, hieraConfig string) error { + sitePP := filepath.Join(environmentPath, environment, "manifests", "site.pp") + if _, err := os.Stat(sitePP); err != nil { + return fmt.Errorf("site.pp not found at %s, is the control-repo cloned: %w", sitePP, err) + } + + args := []string{ + "/opt/puppetlabs/bin/puppet", "apply", + "--detailed-exitcodes", + "--environmentpath", environmentPath, + "--environment", environment, + } + if hieraConfig != "" { + args = append(args, "--hiera_config", hieraConfig) + } + if !enforce { + args = append(args, "--noop") + } + args = append(args, sitePP) + + slog.Info("executing the puppet apply command", slog.Bool("enforce", enforce), slog.String("environment", environment)) + return runPuppet(strings.Join(args, " ")) +} + // Entry point -func RunOpenvox() { +func RunOpenvox(enforce bool) { helper.LoadPuppetEnv() obmondoAPI := api.NewObmondoClient(api.GetObmondoURL(), false) certname := helper.GetCertname() + + // Masterless mode: the catalog is compiled locally from the cloned repo, so + // there is no puppetserver to resolve or reach. + if runOpenvoxApply { + // nolint:errcheck + obmondoAPI.ServerPing() + + if err := applyOpenvox(enforce, runOpenvoxEnvPath, runOpenvoxEnvName, runOpenvoxHieraConf); err != nil { + slog.Error("unable to run puppet apply", slog.String("error", err.Error())) + } + + // nolint:errcheck + obmondoAPI.UpdatePuppetLastRunReport() + return + } + prometheusHost, puppetServerHost := resolveCustomerURLs(obmondoAPI, certname) slog.Info("resolved customer URLs", slog.String("prometheus", prometheusHost), @@ -82,7 +154,7 @@ func RunOpenvox() { obmondoAPI.ServerPing() // Need to have case here later in future, when we migrate the endpoints in go-api - if err := runOpenvoxAgent(); err != nil { + if err := runOpenvoxAgent(enforce); err != nil { slog.Error("unable to run the puppet agent", slog.String("error", err.Error())) } @@ -91,5 +163,16 @@ func RunOpenvox() { } func init() { + f := runOpenvoxCmd.Flags() + f.BoolVar(&runOpenvoxEnforce, constant.CobraFlagEnforce, false, + "Apply changes by running puppet with --no-noop (default is report-only --noop)") + f.BoolVar(&runOpenvoxApply, constant.CobraFlagApply, false, + "Run masterless puppet apply against a locally cloned control-repo instead of puppet agent") + f.StringVar(&runOpenvoxEnvPath, constant.CobraFlagEnvironmentPath, constant.DefaultEnvironmentPath, + "Path holding the cloned puppet environments (apply mode)") + f.StringVar(&runOpenvoxEnvName, constant.CobraFlagOpenvoxEnv, constant.DefaultOpenvoxEnv, + "LinuxAid/OpenVox environment to apply (apply mode)") + f.StringVar(&runOpenvoxHieraConf, constant.CobraFlagHieraConfig, constant.DefaultHieraConfig, + "Global-layer hiera.yaml for puppet apply, empty to use the environment hierarchy only (apply mode)") rootCmd.AddCommand(runOpenvoxCmd) } diff --git a/constant/constants.go b/constant/constants.go index a7daa57..f7d79aa 100644 --- a/constant/constants.go +++ b/constant/constants.go @@ -21,6 +21,13 @@ const ( DefaultPuppetServerCustomerID = "enableit" DefaultPuppetServerDomainSuffix = ".puppet.obmondo.com" DefaultOpenvoxEnv = "master" + // DefaultEnvironmentPath is where the per-node Job clones the puppet code for + // masterless apply; it must live under the /opt/obmondo hostPath mount so the + // host-side puppet sees it at the same absolute path. + DefaultEnvironmentPath = "/opt/obmondo/openvox/code/environments" + // DefaultHieraConfig is the global-layer hiera.yaml the Job generates; its + // hierarchy points at the Helm-values data staged under /opt/obmondo. + DefaultHieraConfig = "/opt/obmondo/openvox/hiera.yaml" // Progress Bar BarProgressSize = 100 @@ -40,6 +47,10 @@ const ( CobraFlagOpenvoxEnv = "openvox-environment" CobraFlagNoReboot = "no-reboot" CobraFlagSkipOpenvox = "skip-openvox" + CobraFlagEnforce = "enforce" + CobraFlagApply = "apply" + CobraFlagEnvironmentPath = "environmentpath" + CobraFlagHieraConfig = "hiera-config" CobraFlagSecurityExporterURL = "security-exporter-url" ObmondoEnv = "OBMONDO_ENV" diff --git a/deploy/chart/linuxaid-agent/Chart.yaml b/deploy/chart/linuxaid-agent/Chart.yaml new file mode 100644 index 0000000..19eb021 --- /dev/null +++ b/deploy/chart/linuxaid-agent/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: linuxaid-agent +description: >- + LinuxAid OpenVox agent for Kubernetes nodes: an operator Deployment that + reconciles one per-node Job every interval; each Job clones the puppet + control-repo at the pinned tag and runs masterless puppet apply on its node, + with the chart values as the global hiera layer. +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/deploy/chart/linuxaid-agent/templates/configmap.yaml b/deploy/chart/linuxaid-agent/templates/configmap.yaml new file mode 100644 index 0000000..09f79e9 --- /dev/null +++ b/deploy/chart/linuxaid-agent/templates/configmap.yaml @@ -0,0 +1,14 @@ +# The rendered Helm values ARE the node's hiera data: the operator mounts this +# ConfigMap into every per-node Job at /hiera-data, the Job stages it onto the +# host, and puppet apply reads it as the (highest-precedence) global hiera layer. +# linuxaid.tag is injected as common::system::openvox::environment so the puppet +# code knows which control-repo tag it was applied from. +apiVersion: v1 +kind: ConfigMap +metadata: + name: linuxaid-agent-hiera + labels: + app.kubernetes.io/name: linuxaid-agent +data: + values.yaml: | + {{- toYaml (merge (dict "common::system::openvox::environment" (required "linuxaid.tag is required" .Values.linuxaid.tag)) .Values.hiera) | nindent 4 }} diff --git a/deploy/chart/linuxaid-agent/templates/deployment.yaml b/deploy/chart/linuxaid-agent/templates/deployment.yaml new file mode 100644 index 0000000..c0a3d78 --- /dev/null +++ b/deploy/chart/linuxaid-agent/templates/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: linuxaid-agent + labels: + app.kubernetes.io/name: linuxaid-agent +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: linuxaid-agent + template: + metadata: + labels: + app.kubernetes.io/name: linuxaid-agent + spec: + serviceAccountName: linuxaid-agent + containers: + - name: operator + image: {{ .Values.linuxaid.images.operator }} + args: + - operator + - --image={{ .Values.linuxaid.images.agent }} + - --certname={{ required "linuxaid.certname is required" .Values.linuxaid.certname }} + - --control-repo-url={{ .Values.linuxaid.controlRepoURL }} + - --control-repo-ref={{ required "linuxaid.tag is required" .Values.linuxaid.tag }} + - --hiera-configmap=linuxaid-agent-hiera + - --cert-secret={{ .Values.linuxaid.certSecret }} + {{- if .Values.linuxaid.gitSecret }} + - --git-secret={{ .Values.linuxaid.gitSecret }} + {{- end }} + - --openvox-environment={{ .Values.linuxaid.openvoxEnvironment }} + - --interval={{ .Values.linuxaid.interval }} + {{- if .Values.linuxaid.enforce }} + - --enforce + {{- end }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] diff --git a/deploy/chart/linuxaid-agent/templates/rbac.yaml b/deploy/chart/linuxaid-agent/templates/rbac.yaml new file mode 100644 index 0000000..187e404 --- /dev/null +++ b/deploy/chart/linuxaid-agent/templates/rbac.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: linuxaid-agent + labels: + app.kubernetes.io/name: linuxaid-agent +--- +# Nodes are cluster-scoped: list drives the fan-out, get backs --node runs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: linuxaid-agent-nodes + labels: + app.kubernetes.io/name: linuxaid-agent +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: linuxaid-agent-nodes + labels: + app.kubernetes.io/name: linuxaid-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: linuxaid-agent-nodes +subjects: + - kind: ServiceAccount + name: linuxaid-agent + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linuxaid-agent-jobs + labels: + app.kubernetes.io/name: linuxaid-agent +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linuxaid-agent-jobs + labels: + app.kubernetes.io/name: linuxaid-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: linuxaid-agent-jobs +subjects: + - kind: ServiceAccount + name: linuxaid-agent + namespace: {{ .Release.Namespace }} diff --git a/deploy/chart/linuxaid-agent/values.yaml b/deploy/chart/linuxaid-agent/values.yaml new file mode 100644 index 0000000..2b4b09c --- /dev/null +++ b/deploy/chart/linuxaid-agent/values.yaml @@ -0,0 +1,27 @@ +linuxaid: + # Control-repo tag to apply on every node. Pins the git checkout AND is + # injected into the hiera data as common::system::openvox::environment. + tag: "" + controlRepoURL: https://github.com/Obmondo/linuxaid.git + # obmondo-clientcert CN, . + certname: "" + # false = report-only (puppet --noop); true = apply changes + enforce: false + # reconcile interval of the operator (one Job per node per cycle) + interval: 4h + openvoxEnvironment: master + images: + operator: gitea.obmondo.com/enableit/linuxaid-agent:latest + agent: gitea.obmondo.com/enableit/linuxaid-cli:latest + # secret with the pre-signed puppet client cert (tls.crt / tls.key / ca.crt) + certSecret: obmondo-clientcert + # secret with git credentials for the control-repo: key `ssh-privatekey` + # (SSH remotes) or `token` (https remotes). Empty = public repo, no credentials. + gitSecret: "" + +# Free-form hiera data for every node in the cluster, rendered verbatim into +# the ConfigMap the Jobs mount — anything hiera-legal works, e.g.: +# classes: +# - role::kubeaid +# common::monitor::prometheus::server: prometheus.demo.example.com +hiera: {} diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh new file mode 100644 index 0000000..24552f9 --- /dev/null +++ b/deploy/entrypoint.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# +# Launcher for the LinuxAid OpenVox agent on a Kubernetes node. +# +# The pod is only a delivery + scheduling shell: it stages the static linuxaid-cli +# binary onto the host and runs the agent inside the host's namespaces via nsenter, +# because package/user management must happen on the host, not the container. +# Designed to run once (as a Job) and exit. +# +# The obmondo-clientcert is mounted and installed into the host puppet SSL tree +# under CERTNAME (already signed — nothing to enroll). If the openvox agent is +# missing on the host, it is installed first from the voxpupuli openvox8 apt repo. +# +# Two run modes: +# - agent (default): puppet agent -t against the customer's puppetserver. +# - apply (CONTROL_REPO_URL set): masterless puppet apply. The control-repo is +# cloned at CONTROL_REPO_REF (the chart's linuxaid.tag; latest tag when unset) +# into CODE_DIR under the /opt/obmondo hostPath — so the host sees it at the +# same absolute path — and the Helm-values hiera data mounted at HIERA_SRC_DIR +# is staged next to it as the global hiera layer. +set -euo pipefail + +CERTNAME="${CERTNAME:?CERTNAME (obmondo-clientcert CN, e.g. .) is required}" +ENFORCE="${ENFORCE:-false}" +OPENVOX_ENVIRONMENT="${OPENVOX_ENVIRONMENT:-master}" +PUPPET_SERVER="${PUPPET_SERVER:-}" # agent mode only; run-openvox also resolves it from the Obmondo API +CLIENT_CERT_DIR="${CLIENT_CERT_DIR:-/obmondo-clientcert}" # mounted obmondo-clientcert (tls.crt / tls.key / ca.crt) +OPENVOX_RELEASE_DEB="${OPENVOX_RELEASE_DEB:-https://apt.voxpupuli.org/openvox8-release-ubuntu26.04.deb}" + +CONTROL_REPO_URL="${CONTROL_REPO_URL:-}" # apply-mode switch: git URL of the puppet control-repo (environment code) +CONTROL_REPO_REF="${CONTROL_REPO_REF:-}" # tag to checkout, from the chart's linuxaid.tag (default: latest tag) +HIERA_SRC_DIR="${HIERA_SRC_DIR:-/hiera-data}" # mounted ConfigMap with the rendered Helm-values hiera data +GIT_CRED_DIR="${GIT_CRED_DIR:-/git-credentials}" # optional secret mount: ssh-privatekey or token + +OPENVOX_DIR=/opt/obmondo/openvox # must live under the /opt/obmondo hostPath mount +CODE_DIR="${CODE_DIR:-${OPENVOX_DIR}/code}" +DATA_DIR="${OPENVOX_DIR}/data" +HIERA_CONF="${OPENVOX_DIR}/hiera.yaml" + +HOST_BIN_DIR=/opt/obmondo/bin +HOST_CLI="${HOST_BIN_DIR}/linuxaid-cli" +PUPPET_BIN=/opt/puppetlabs/bin/puppet +PUPPET_SSL=/etc/puppetlabs/puppet/ssl +PUPPET_CONF=/etc/puppetlabs/puppet/puppet.conf + +# Run a command in the host's namespaces (needs pod hostPID + container privileged). +ns() { nsenter --target 1 --mount --uts --ipc --net --pid -- "$@"; } +log() { printf '[entrypoint] %s\n' "$*"; } + +# git with credentials from GIT_CRED_DIR when present: an ssh-privatekey (copied to +# 0600 first — secret mounts are world-readable and ssh refuses such keys) or a +# Gitea token for https remotes. +git_cmd() { + if [ -f "${GIT_CRED_DIR}/ssh-privatekey" ]; then + if [ ! -f /tmp/git-ssh-key ]; then + install -m 0600 "${GIT_CRED_DIR}/ssh-privatekey" /tmp/git-ssh-key + fi + GIT_SSH_COMMAND="ssh -i /tmp/git-ssh-key -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/tmp/git-known-hosts" git "$@" + elif [ -f "${GIT_CRED_DIR}/token" ]; then + git -c http.extraHeader="Authorization: token $(cat "${GIT_CRED_DIR}/token")" "$@" + else + git "$@" + fi +} + +# Clone a repo at the given ref; without one, at its latest tag (version-sorted), +# falling back to the default-branch tip for untagged repos. +clone_repo() { + local url=$1 dest=$2 ref=$3 + rm -rf "${dest}" + mkdir -p "$(dirname "${dest}")" + + if [ -z "${ref}" ]; then + ref="$(git_cmd ls-remote --refs --sort='-v:refname' --tags "${url}" | awk -F'refs/tags/' 'NR==1{print $2}')" + fi + if [ -n "${ref}" ]; then + log "cloning ${url} at ${ref}" + git_cmd clone --quiet --depth 1 --branch "${ref}" "${url}" "${dest}" + else + log "no tags on ${url} — cloning default-branch tip" + git_cmd clone --quiet --depth 1 "${url}" "${dest}" + fi +} + +log "certname=${CERTNAME} enforce=${ENFORCE} env=${OPENVOX_ENVIRONMENT} mode=$([ -n "${CONTROL_REPO_URL}" ] && echo apply || echo agent)" + +# 1) Stage the static CLI onto the host via the /opt/obmondo hostPath mount. +mkdir -p "${HOST_BIN_DIR}" +install -m 0755 /usr/local/bin/linuxaid-cli "${HOST_CLI}" + +# 2) Install the mounted obmondo-clientcert into the host puppet SSL tree under CERTNAME. +# (`< file` is resolved in the container mount ns, piped into a host-side writer.) +ns install -d -m 0755 "${PUPPET_SSL}/certs" +ns install -d -m 0700 "${PUPPET_SSL}/private_keys" +ns tee "${PUPPET_SSL}/certs/${CERTNAME}.pem" >/dev/null < "${CLIENT_CERT_DIR}/tls.crt" +ns tee "${PUPPET_SSL}/private_keys/${CERTNAME}.pem" >/dev/null < "${CLIENT_CERT_DIR}/tls.key" +ns chmod 0644 "${PUPPET_SSL}/certs/${CERTNAME}.pem" +ns chmod 0600 "${PUPPET_SSL}/private_keys/${CERTNAME}.pem" +if [ -f "${CLIENT_CERT_DIR}/ca.crt" ]; then + ns tee "${PUPPET_SSL}/certs/ca.pem" >/dev/null < "${CLIENT_CERT_DIR}/ca.crt" + ns chmod 0644 "${PUPPET_SSL}/certs/ca.pem" +fi + +# 3) Install the openvox agent on the host if it is missing, from the voxpupuli +# openvox8 apt repo. Runs on the host (Ubuntu) via nsenter. +if ! ns test -x "${PUPPET_BIN}"; then + log "openvox agent not found on host — installing from ${OPENVOX_RELEASE_DEB}" + ns bash -c ' + set -eu + export DEBIAN_FRONTEND=noninteractive + url="$1" + deb="$(mktemp --suffix=.deb)" + if command -v curl >/dev/null 2>&1; then + curl -fsSL -o "$deb" "$url" + else + wget -qO "$deb" "$url" + fi + apt-get install -y "$deb" + apt-get update + apt-get install -y openvox-agent + rm -f "$deb" + ' _ "${OPENVOX_RELEASE_DEB}" +fi + +if [ -n "${CONTROL_REPO_URL}" ]; then + # 4a) Masterless apply: materialize the puppet code + hiera data on the host. + # The clone runs in the container (which has git+ssh) into the hostPath, + # so the host-side puppet apply reads it at the same absolute path. + if [ ! -f "${HIERA_SRC_DIR}/values.yaml" ]; then + log "no hiera data at ${HIERA_SRC_DIR}/values.yaml — is the hiera ConfigMap mounted?" + exit 1 + fi + + env_dir="${CODE_DIR}/environments/${OPENVOX_ENVIRONMENT}" + clone_repo "${CONTROL_REPO_URL}" "${env_dir}" "${CONTROL_REPO_REF}" + + # Stage the Helm-values hiera data and point a global-layer hiera.yaml at it: + # global beats the environment layer, so chart values win over repo defaults. + mkdir -p "${DATA_DIR}" + install -m 0644 "${HIERA_SRC_DIR}/values.yaml" "${DATA_DIR}/values.yaml" + cat > "${HIERA_CONF}" <<-EOF + version: 5 + defaults: + datadir: ${DATA_DIR} + data_hash: yaml_data + hierarchy: + - name: "Helm chart values" + path: values.yaml + EOF + + # Minimal puppet.conf on the host — no server: the catalog is compiled locally. + ns install -d -m 0755 "$(dirname "${PUPPET_CONF}")" + { + echo "[main]" + echo "certname = ${CERTNAME}" + echo "environment = ${OPENVOX_ENVIRONMENT}" + echo "report = true" + } | ns tee "${PUPPET_CONF}" >/dev/null + + # 5a) Run masterless apply on the host. Report-only (--noop) unless ENFORCE=true. + args=(run-openvox --apply + --environmentpath "${CODE_DIR}/environments" + --openvox-environment "${OPENVOX_ENVIRONMENT}" + --hiera-config "${HIERA_CONF}") + if [ "${ENFORCE}" = "true" ]; then args+=(--enforce); fi + log "running: linuxaid-cli ${args[*]}" + ns env CERTNAME="${CERTNAME}" OBMONDO_SKIP_CONNECTIVITY_METRICS=1 PATH="/opt/puppetlabs/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + "${HOST_CLI}" "${args[@]}" +else + # 4b) Agent mode: minimal puppet.conf on the host, pointing at the puppetserver. + ns install -d -m 0755 "$(dirname "${PUPPET_CONF}")" + { + echo "[main]" + if [ -n "${PUPPET_SERVER}" ]; then echo "server = ${PUPPET_SERVER}"; fi + echo "certname = ${CERTNAME}" + echo "masterport = 443" + echo "environment = ${OPENVOX_ENVIRONMENT}" + echo + echo "[agent]" + echo "report = true" + } | ns tee "${PUPPET_CONF}" >/dev/null + + # 5b) Run the agent on the host. Report-only (--noop) unless ENFORCE=true. + args=(run-openvox) + if [ "${ENFORCE}" = "true" ]; then args+=(--enforce); fi + log "running: linuxaid-cli ${args[*]}" + ns env CERTNAME="${CERTNAME}" OBMONDO_SKIP_CONNECTIVITY_METRICS=1 PATH="/opt/puppetlabs/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + "${HOST_CLI}" "${args[@]}" +fi diff --git a/pkg/checkconnectivity/checkconnectivity.go b/pkg/checkconnectivity/checkconnectivity.go index 00dcf49..8bfe8d4 100644 --- a/pkg/checkconnectivity/checkconnectivity.go +++ b/pkg/checkconnectivity/checkconnectivity.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "os" "time" "github.com/prometheus/client_golang/prometheus" @@ -16,6 +17,10 @@ const ( metricsFile = "/var/lib/node_exporter/obmondo_domains_reachable.prom" apiHost = "api.obmondo.com" + + // skipMetricsEnv, when set, skips writing the node_exporter textfile metric + // (used to disable it in environments like Kubernetes that lack it). + skipMetricsEnv = "OBMONDO_SKIP_CONNECTIVITY_METRICS" ) var runPuppetMetric *prometheus.GaugeVec @@ -64,9 +69,13 @@ func CheckTCPConnection(prometheusHost, puppetServerHost string) bool { runPuppetMetric.WithLabelValues(host, port).Set(0) } - if err := prometheus.WriteToTextfile(metricsFile, registry); err != nil { - slog.Info("Error writing metrics to file:", slog.String("error", err.Error())) - return false + // The .prom metric feeds the fleet's node_exporter textfile collector; skip it + // where that isn't present (e.g. Kubernetes) so a failed write can't abort the run. + if os.Getenv(skipMetricsEnv) == "" { + if err := prometheus.WriteToTextfile(metricsFile, registry); err != nil { + slog.Info("Error writing metrics to file:", slog.String("error", err.Error())) + return false + } } return allAPIReachable