From 1c13a0d4713ebffeb3fe93d0f7095cd5b4c7c8e7 Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 1 Jul 2026 12:40:22 +0530 Subject: [PATCH 1/8] feat(run-openvox): add --enforce flag to apply puppet changes run-openvox has always run report-only (`puppet agent -t --noop`), even though puppet.conf unconditionally sets `noop = true`. There was no way to actually apply changes through the CLI itself. Add an opt-in --enforce flag: when set, run-openvox runs `puppet agent -t --no-noop --detailed-exitcodes` instead, which overrides the config-level noop setting and applies changes. Default behavior (flag unset) is unchanged. The connectivity gate, ServerPing, and last-run-report wrapper around the puppet invocation are untouched. This flag is consumed by the launcher image added in this branch (Dockerfile / deploy/entrypoint.sh), which gates it behind an ENFORCE env var and defaults to report-only. --- cmd/linuxaid-cli/run-openvox.go | 26 +++++++++++++++++++------- constant/constants.go | 1 + 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/linuxaid-cli/run-openvox.go b/cmd/linuxaid-cli/run-openvox.go index 295a916..38ae338 100644 --- a/cmd/linuxaid-cli/run-openvox.go +++ b/cmd/linuxaid-cli/run-openvox.go @@ -3,6 +3,7 @@ package main import ( "log/slog" + "gitea.obmondo.com/EnableIT/linuxaid-cli/constant" "gitea.obmondo.com/EnableIT/linuxaid-cli/helper" "gitea.obmondo.com/EnableIT/linuxaid-cli/pkg/checkconnectivity" api "gitea.obmondo.com/EnableIT/linuxaid-cli/pkg/obmondo" @@ -10,18 +11,22 @@ import ( "github.com/spf13/cobra" ) +// runOpenvoxEnforce applies changes (puppet --no-noop) instead of the default report-only (--noop). +var runOpenvoxEnforce bool + var runOpenvoxCmd = &cobra.Command{ Use: "run-openvox", Short: "Execute run-openvox command", Long: "A longer description of run-openvox command", Example: `$ linuxaid-cli run-openvox --certname web01.example`, Run: func(*cobra.Command, []string) { - RunOpenvox() + RunOpenvox(runOpenvoxEnforce) }, } -// Run the puppet agent in noop mode for now -func runOpenvoxAgent() error { +// runOpenvoxAgent runs the puppet agent: report-only (--noop) by default, or applying changes +// (--no-noop --detailed-exitcodes) when enforce is set. +func runOpenvoxAgent(enforce bool) error { // Puppet run execution returns total 5 status codes // // 0: The run succeeded with no changes or failures; the system was already in the desired state. @@ -38,8 +43,13 @@ func runOpenvoxAgent() error { statusCodeFailed := 1 statusCodeSucceededWithChanges := 2 - slog.Info("executing the puppet agent command") - cmdPipe := script.Exec("/opt/puppetlabs/bin/puppet agent -t --noop") + puppetCmd := "/opt/puppetlabs/bin/puppet agent -t --noop" + if enforce { + puppetCmd = "/opt/puppetlabs/bin/puppet agent -t --no-noop --detailed-exitcodes" + } + + slog.Info("executing the puppet agent command", slog.Bool("enforce", enforce)) + cmdPipe := script.Exec(puppetCmd) _, err := cmdPipe.Stdout() if err != nil { // When encountering status code 1, consider it as an error, and return. @@ -59,7 +69,7 @@ func runOpenvoxAgent() error { } // Entry point -func RunOpenvox() { +func RunOpenvox(enforce bool) { helper.LoadPuppetEnv() obmondoAPI := api.NewObmondoClient(api.GetObmondoURL(), false) @@ -82,7 +92,7 @@ func RunOpenvox() { obmondoAPI.ServerPing() // Need to have case here later in future, when we migrate the endpoints in go-api - if err := runOpenvoxAgent(); err != nil { + if err := runOpenvoxAgent(enforce); err != nil { slog.Error("unable to run the puppet agent", slog.String("error", err.Error())) } @@ -91,5 +101,7 @@ func RunOpenvox() { } func init() { + runOpenvoxCmd.Flags().BoolVar(&runOpenvoxEnforce, constant.CobraFlagEnforce, false, + "Apply changes by running puppet with --no-noop (default is report-only --noop)") rootCmd.AddCommand(runOpenvoxCmd) } diff --git a/constant/constants.go b/constant/constants.go index a7daa57..99ada32 100644 --- a/constant/constants.go +++ b/constant/constants.go @@ -40,6 +40,7 @@ const ( CobraFlagOpenvoxEnv = "openvox-environment" CobraFlagNoReboot = "no-reboot" CobraFlagSkipOpenvox = "skip-openvox" + CobraFlagEnforce = "enforce" CobraFlagSecurityExporterURL = "security-exporter-url" ObmondoEnv = "OBMONDO_ENV" From 632c280acb7a959b9732e503fb3e8b71a47040c4 Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 1 Jul 2026 12:41:05 +0530 Subject: [PATCH 2/8] feat(docker): add launcher image to run linuxaid-cli as a per-node Job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a multi-stage Dockerfile that builds the static linuxaid-cli binary and packages it into a minimal alpine runtime (bash + util-linux for nsenter). The image is a delivery/scheduling shell only: deploy/entrypoint.sh stages the binary onto the host and runs the agent inside the host's namespaces via nsenter, because package/user/sudo management must happen on the host, not inside the container. The entrypoint installs a pre-signed obmondo-clientcert (mounted into the pod) into the host's puppet SSL tree under CERTNAME, writes a minimal puppet.conf, and runs run-openvox, gating the --enforce flag added in the previous commit behind an ENFORCE env var (default false, report-only). The image ships no linuxaid-install — the cert is provided pre-signed, so there is nothing to enroll — and fails fast if the openvox agent isn't already installed on the host. Intended to run as a per-node Kubernetes Job with hostPID and privileged/CAP_SYS_ADMIN for nsenter, and /opt/obmondo hostPath mounted for binary staging. --- .dockerignore | 4 +++ Dockerfile | 27 ++++++++++++++++ deploy/entrypoint.sh | 76 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 deploy/entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f895107 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +.claude +dist +*.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..10d6661 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +# syntax=docker/dockerfile:1 +# +# linuxaid-agents: a thin launcher image for running the LinuxAid OpenVox agent +# on a Kubernetes node. The container is only a delivery + scheduling shell — it +# stages the static linuxaid-cli binary onto the host and runs the agent inside +# the host's namespaces via nsenter (see deploy/entrypoint.sh). Intended to run +# as a per-node Job. The node's cert is provided pre-signed (obmondo-clientcert), +# so no enrollment/linuxaid-install is needed. + +# ---- build the static binary ---- +FROM golang:1.24-alpine AS build +WORKDIR /src +RUN apk add --no-cache git +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . +ARG VERSION=spike +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 go build -trimpath -ldflags="-X main.Version=${VERSION} -s -w" -o /out/linuxaid-cli ./cmd/linuxaid-cli + +# ---- runtime: launcher that nsenters into the host ---- +FROM alpine:3.20 +RUN apk add --no-cache bash util-linux ca-certificates +COPY --from=build /out/linuxaid-cli /usr/local/bin/linuxaid-cli +COPY deploy/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh new file mode 100644 index 0000000..fbe4d9d --- /dev/null +++ b/deploy/entrypoint.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Launcher for the LinuxAid OpenVox agent on a Kubernetes node. +# +# The pod is only a delivery + scheduling shell: it stages the static +# linuxaid-cli binary onto the host and runs the agent inside the host's +# namespaces via nsenter, because package/user/sudo management must happen on +# the host, not in the container. Designed to run once (as a Job) and exit. +# +# Cert model: the pod mounts the cluster's obmondo-clientcert; this script +# installs it into the host puppet SSL tree under CERTNAME so that both +# `puppet agent` and linuxaid-cli's Obmondo API client pick it up. +set -euo pipefail + +CERTNAME="${CERTNAME:?CERTNAME (obmondo-clientcert CN, e.g. .) is required}" +ENFORCE="${ENFORCE:-false}" +OPENVOX_ENVIRONMENT="${OPENVOX_ENVIRONMENT:-master}" +PUPPET_SERVER="${PUPPET_SERVER:-}" # optional; run-openvox also resolves it from the Obmondo API +CLIENT_CERT_DIR="${CLIENT_CERT_DIR:-/obmondo-clientcert}" # mounted obmondo-clientcert (tls.crt / tls.key / ca.crt) + +HOST_BIN_DIR=/opt/obmondo/bin +PUPPET_SSL=/etc/puppetlabs/puppet/ssl +PUPPET_CONF=/etc/puppetlabs/puppet/puppet.conf + +# Run a command in the host's namespaces (needs pod hostPID + container privileged). +ns() { nsenter --target 1 --mount --uts --ipc --net --pid -- "$@"; } +log() { printf '[entrypoint] %s\n' "$*"; } + +log "certname=${CERTNAME} enforce=${ENFORCE} env=${OPENVOX_ENVIRONMENT} server=${PUPPET_SERVER:-}" + +# 1) Stage the static CLI onto the host via the /opt/obmondo hostPath mount +# (visible in the host mount namespace after nsenter). +mkdir -p "${HOST_BIN_DIR}" +install -m 0755 /usr/local/bin/linuxaid-cli "${HOST_BIN_DIR}/linuxaid-cli" + +# The openvox/puppet agent must already be installed on the host — this image +# ships neither the agent nor linuxaid-install (the cert is provided pre-signed, +# so there is nothing to enroll). Fail fast with a clear message if it is absent. +if ! ns test -x /opt/puppetlabs/bin/puppet; then + log "ERROR: /opt/puppetlabs/bin/puppet not found on the host — pre-install the openvox agent on this node." + exit 1 +fi + +# 2) Install the mounted obmondo-clientcert into the host puppet SSL tree under +# CERTNAME. The `< file` redirection is resolved in the container mount ns, so +# the cert content is piped into a host-side writer. +ns install -d -m 0755 "${PUPPET_SSL}/certs" +ns install -d -m 0700 "${PUPPET_SSL}/private_keys" +ns tee "${PUPPET_SSL}/certs/${CERTNAME}.pem" >/dev/null < "${CLIENT_CERT_DIR}/tls.crt" +ns tee "${PUPPET_SSL}/private_keys/${CERTNAME}.pem" >/dev/null < "${CLIENT_CERT_DIR}/tls.key" +ns chmod 0644 "${PUPPET_SSL}/certs/${CERTNAME}.pem" +ns chmod 0600 "${PUPPET_SSL}/private_keys/${CERTNAME}.pem" +if [ -f "${CLIENT_CERT_DIR}/ca.crt" ]; then + ns tee "${PUPPET_SSL}/certs/ca.pem" >/dev/null < "${CLIENT_CERT_DIR}/ca.crt" + ns chmod 0644 "${PUPPET_SSL}/certs/ca.pem" +fi + +# 3) Minimal puppet.conf on the host. +ns install -d -m 0755 "$(dirname "${PUPPET_CONF}")" +{ + echo "[main]" + if [ -n "${PUPPET_SERVER}" ]; then echo "server = ${PUPPET_SERVER}"; fi + echo "certname = ${CERTNAME}" + echo "masterport = 443" + echo "environment = ${OPENVOX_ENVIRONMENT}" + echo + echo "[agent]" + echo "report = true" +} | ns tee "${PUPPET_CONF}" >/dev/null + +# 4) Run the agent on the host. Report-only (--noop) unless ENFORCE=true. +args=(run-openvox) +if [ "${ENFORCE}" = "true" ]; then args+=(--enforce); fi +log "running: linuxaid-cli ${args[*]}" +ns env CERTNAME="${CERTNAME}" PATH="/opt/puppetlabs/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + "${HOST_BIN_DIR}/linuxaid-cli" "${args[@]}" From 0f05c3faafc59c072ef09edd7407e81a0bce020d Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 1 Jul 2026 15:26:28 +0530 Subject: [PATCH 3/8] feat(linuxaid-agent): add fan-out launcher and split images Add cmd/linuxaid-agent, an in-cluster controller that lists nodes over the k8s REST API (SA token + CA, no client-go) and creates one per-node agent Job each run; a deterministic name means a still-running Job is skipped (no-concurrency). Split the image build: Dockerfile builds the host agent (linuxaid-cli + nsenter entrypoint), Dockerfile.launcher builds the minimal non-root linuxaid-agent controller. --- Dockerfile | 12 +- Dockerfile.launcher | 21 ++ cmd/linuxaid-agent/fanout.go | 396 +++++++++++++++++++++++++++++++++++ cmd/linuxaid-agent/main.go | 26 +++ 4 files changed, 447 insertions(+), 8 deletions(-) create mode 100644 Dockerfile.launcher create mode 100644 cmd/linuxaid-agent/fanout.go create mode 100644 cmd/linuxaid-agent/main.go diff --git a/Dockerfile b/Dockerfile index 10d6661..52aeafc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,10 @@ # syntax=docker/dockerfile:1 # -# linuxaid-agents: a thin launcher image for running the LinuxAid OpenVox agent -# on a Kubernetes node. The container is only a delivery + scheduling shell — it -# stages the static linuxaid-cli binary onto the host and runs the agent inside -# the host's namespaces via nsenter (see deploy/entrypoint.sh). Intended to run -# as a per-node Job. The node's cert is provided pre-signed (obmondo-clientcert), -# so no enrollment/linuxaid-install is needed. +# Host agent image: stages the static linuxaid-cli onto the node and runs the +# OpenVox agent in the host's namespaces via nsenter (see deploy/entrypoint.sh). +# Runs as the per-node Job that the fan-out launcher spawns. The node's cert is +# provided pre-signed (obmondo-clientcert), so there is nothing to enroll. -# ---- build the static binary ---- FROM golang:1.24-alpine AS build WORKDIR /src RUN apk add --no-cache git @@ -18,7 +15,6 @@ ARG VERSION=spike RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 go build -trimpath -ldflags="-X main.Version=${VERSION} -s -w" -o /out/linuxaid-cli ./cmd/linuxaid-cli -# ---- runtime: launcher that nsenters into the host ---- FROM alpine:3.20 RUN apk add --no-cache bash util-linux ca-certificates COPY --from=build /out/linuxaid-cli /usr/local/bin/linuxaid-cli diff --git a/Dockerfile.launcher b/Dockerfile.launcher new file mode 100644 index 0000000..91c295c --- /dev/null +++ b/Dockerfile.launcher @@ -0,0 +1,21 @@ +# syntax=docker/dockerfile:1 +# +# Fan-out launcher image: the in-cluster controller (linuxaid-agent) that lists +# nodes and creates one per-node agent Job each run. It is only a Kubernetes API +# client — no host access — so it runs non-root with no extra tooling. + +FROM golang:1.24-alpine AS build +WORKDIR /src +RUN apk add --no-cache git +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . +ARG VERSION=spike +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 go build -trimpath -ldflags="-X main.Version=${VERSION} -s -w" -o /out/linuxaid-agent ./cmd/linuxaid-agent + +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates +COPY --from=build /out/linuxaid-agent /usr/local/bin/linuxaid-agent +USER 65534:65534 +ENTRYPOINT ["/usr/local/bin/linuxaid-agent"] diff --git a/cmd/linuxaid-agent/fanout.go b/cmd/linuxaid-agent/fanout.go new file mode 100644 index 0000000..42c808b --- /dev/null +++ b/cmd/linuxaid-agent/fanout.go @@ -0,0 +1,396 @@ +package main + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/spf13/cobra" +) + +const saDir = "/var/run/secrets/kubernetes.io/serviceaccount" + +var ( + faImage string + faCertname string + faEnforce bool + faOpenvoxEnv string + faPuppetServer string + faSecretName string + faHostPath string + faNodeSelector string + faNamePrefix string + faNamespace string + faTTLSeconds int + faActiveDeadline int + faBackoffLimit int +) + +var fanoutCmd = &cobra.Command{ + Use: "fanout", + Short: "List cluster nodes and create one agent Job per node", + RunE: func(_ *cobra.Command, _ []string) error { + return runFanout() + }, +} + +func init() { + f := fanoutCmd.Flags() + f.StringVar(&faImage, "image", "", "agent container image (required)") + f.StringVar(&faCertname, "certname", "", "obmondo-clientcert CN, shared by all nodes (required)") + f.BoolVar(&faEnforce, "enforce", false, "apply changes (puppet --no-noop) instead of report-only") + f.StringVar(&faOpenvoxEnv, "openvox-environment", "master", "LinuxAid/OpenVox environment") + f.StringVar(&faPuppetServer, "puppet-server", "", "optional puppetserver override") + f.StringVar(&faSecretName, "cert-secret", "obmondo-clientcert", "secret holding the pre-signed client cert") + f.StringVar(&faHostPath, "host-path", "/opt/obmondo", "host path for staging the CLI binary") + f.StringVar(&faNodeSelector, "node-selector", "", "label selector to filter nodes (empty = all)") + f.StringVar(&faNamePrefix, "name-prefix", "linuxaid-agents", "prefix for the per-node Job names") + f.StringVar(&faNamespace, "namespace", "", "namespace to create Jobs in (default: the pod's namespace)") + f.IntVar(&faTTLSeconds, "ttl-seconds", 600, "ttlSecondsAfterFinished on the agent Jobs") + f.IntVar(&faActiveDeadline, "active-deadline-seconds", 1200, "activeDeadlineSeconds on the agent Jobs") + f.IntVar(&faBackoffLimit, "backoff-limit", 1, "backoffLimit on the agent Jobs") + rootCmd.AddCommand(fanoutCmd) +} + +func runFanout() error { + if faImage == "" || faCertname == "" { + return fmt.Errorf("--image and --certname are required") + } + + cfg, err := inClusterConfig() + if err != nil { + return err + } + + nodes, err := cfg.listNodes(faNodeSelector) + if err != nil { + return err + } + if len(nodes) == 0 { + slog.Warn("no nodes matched", slog.String("selector", faNodeSelector)) + return nil + } + + slog.Info("fanning out agent jobs", + slog.Int("nodes", len(nodes)), + slog.String("namespace", cfg.namespace), + slog.Bool("enforce", faEnforce)) + + var failed int + for _, node := range nodes { + slug := slugify(node) + name := jobName(faNamePrefix, slug) + + manifest, err := buildJobManifest(node, name, slug) + if err != nil { + slog.Error("build job", slog.String("node", node), slog.Any("error", err)) + failed++ + continue + } + + created, err := cfg.createJob(manifest) + switch { + case err != nil: + slog.Error("create job", slog.String("node", node), slog.Any("error", err)) + failed++ + case created: + slog.Info("created job", slog.String("job", name), slog.String("node", node)) + default: + slog.Info("skipped, already present (no-concurrency)", slog.String("job", name), slog.String("node", node)) + } + } + + if failed > 0 { + return fmt.Errorf("%d of %d nodes failed", failed, len(nodes)) + } + return nil +} + +type clusterConfig struct { + host string + token string + namespace string + client *http.Client +} + +func inClusterConfig() (*clusterConfig, error) { + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if host == "" || port == "" { + return nil, fmt.Errorf("not running in-cluster: KUBERNETES_SERVICE_HOST/PORT are unset") + } + + token, err := os.ReadFile(saDir + "/token") + if err != nil { + return nil, fmt.Errorf("read service account token: %w", err) + } + + caPEM, err := os.ReadFile(saDir + "/ca.crt") + if err != nil { + return nil, fmt.Errorf("read cluster CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("parse cluster CA") + } + + namespace := faNamespace + if namespace == "" { + b, err := os.ReadFile(saDir + "/namespace") + if err != nil { + return nil, fmt.Errorf("read namespace: %w", err) + } + namespace = strings.TrimSpace(string(b)) + } + + return &clusterConfig{ + host: fmt.Sprintf("https://%s:%s", host, port), + token: strings.TrimSpace(string(token)), + namespace: namespace, + client: &http.Client{ + Timeout: 30 * time.Second, //nolint:mnd + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}, + }, + }, nil +} + +func (c *clusterConfig) do(method, path string, body []byte) (*http.Response, error) { + var r io.Reader = http.NoBody + if body != nil { + r = bytes.NewReader(body) + } + req, err := http.NewRequest(method, c.host+path, r) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return c.client.Do(req) +} + +func (c *clusterConfig) listNodes(selector string) ([]string, error) { + path := "/api/v1/nodes" + if selector != "" { + path += "?labelSelector=" + url.QueryEscape(selector) + } + + resp, err := c.do(http.MethodGet, path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("list nodes: %s: %s", resp.Status, strings.TrimSpace(string(b))) + } + + var out struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + } `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + + names := make([]string, 0, len(out.Items)) + for _, item := range out.Items { + names = append(names, item.Metadata.Name) + } + return names, nil +} + +// createJob POSTs the Job; a 409 means it already exists (still running) and is +// treated as a skip, which is what gives us no-concurrency per node. +func (c *clusterConfig) createJob(manifest []byte) (created bool, err error) { + resp, err := c.do(http.MethodPost, fmt.Sprintf("/apis/batch/v1/namespaces/%s/jobs", c.namespace), manifest) + if err != nil { + return false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusCreated, http.StatusOK: + return true, nil + case http.StatusConflict: + return false, nil + default: + b, _ := io.ReadAll(resp.Body) + return false, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b))) + } +} + +func buildJobManifest(node, name, slug string) ([]byte, error) { + env := []envVar{ + {Name: "CERTNAME", Value: faCertname}, + {Name: "ENFORCE", Value: fmt.Sprintf("%t", faEnforce)}, + {Name: "OPENVOX_ENVIRONMENT", Value: faOpenvoxEnv}, + {Name: "CLIENT_CERT_DIR", Value: "/obmondo-clientcert"}, + } + if faPuppetServer != "" { + env = append(env, envVar{Name: "PUPPET_SERVER", Value: faPuppetServer}) + } + + labels := map[string]string{ + "app.kubernetes.io/name": "linuxaid-agents", + "app.kubernetes.io/managed-by": "linuxaid-agent", + "linuxaid-agents.obmondo.com/node": slug, + } + + j := k8sJob{ + APIVersion: "batch/v1", + Kind: "Job", + Metadata: objMeta{Name: name, Labels: labels}, + Spec: jobSpec{ + BackoffLimit: faBackoffLimit, + ActiveDeadlineSeconds: faActiveDeadline, + TTLSecondsAfterFinished: faTTLSeconds, + Template: podTemplate{ + Metadata: objMeta{Labels: labels}, + Spec: podSpec{ + RestartPolicy: "Never", + HostPID: true, + AutomountServiceAccountToken: false, + NodeName: node, + Tolerations: []toleration{{Operator: "Exists"}}, + Containers: []container{{ + Name: "agent", + Image: faImage, + SecurityContext: secCtx{Privileged: true, RunAsUser: 0}, + Env: env, + VolumeMounts: []volumeMount{ + {Name: "obmondo-clientcert", MountPath: "/obmondo-clientcert", ReadOnly: true}, + {Name: "host-obmondo", MountPath: faHostPath}, + }, + }}, + Volumes: []volume{ + {Name: "obmondo-clientcert", Secret: &secretVol{SecretName: faSecretName}}, + {Name: "host-obmondo", HostPath: &hostPathVol{Path: faHostPath, Type: "DirectoryOrCreate"}}, + }, + }, + }, + }, + } + return json.Marshal(j) +} + +// slugify turns a node name into a DNS-1123 label fragment. +func slugify(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } else { + b.WriteRune('-') + } + } + slug := b.String() + for strings.Contains(slug, "--") { + slug = strings.ReplaceAll(slug, "--", "-") + } + slug = strings.Trim(slug, "-") + if len(slug) > 40 { //nolint:mnd + slug = strings.Trim(slug[:40], "-") + } + return slug +} + +// jobName keeps the name a valid DNS-1123 label with room for the pod suffix. +func jobName(prefix, slug string) string { + name := prefix + "-" + slug + if len(name) > 57 { //nolint:mnd + name = strings.TrimRight(name[:57], "-") + } + return name +} + +type k8sJob struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata objMeta `json:"metadata"` + Spec jobSpec `json:"spec"` +} + +type objMeta struct { + Name string `json:"name,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +type jobSpec struct { + BackoffLimit int `json:"backoffLimit"` + ActiveDeadlineSeconds int `json:"activeDeadlineSeconds"` + TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` + Template podTemplate `json:"template"` +} + +type podTemplate struct { + Metadata objMeta `json:"metadata,omitempty"` + Spec podSpec `json:"spec"` +} + +type podSpec struct { + RestartPolicy string `json:"restartPolicy"` + HostPID bool `json:"hostPID"` + AutomountServiceAccountToken bool `json:"automountServiceAccountToken"` + NodeName string `json:"nodeName"` + Tolerations []toleration `json:"tolerations,omitempty"` + Containers []container `json:"containers"` + Volumes []volume `json:"volumes"` +} + +type toleration struct { + Operator string `json:"operator"` +} + +type container struct { + Name string `json:"name"` + Image string `json:"image"` + SecurityContext secCtx `json:"securityContext"` + Env []envVar `json:"env,omitempty"` + VolumeMounts []volumeMount `json:"volumeMounts"` +} + +type secCtx struct { + Privileged bool `json:"privileged"` + RunAsUser int `json:"runAsUser"` +} + +type envVar struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type volumeMount struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + ReadOnly bool `json:"readOnly,omitempty"` +} + +type volume struct { + Name string `json:"name"` + Secret *secretVol `json:"secret,omitempty"` + HostPath *hostPathVol `json:"hostPath,omitempty"` +} + +type secretVol struct { + SecretName string `json:"secretName"` +} + +type hostPathVol struct { + Path string `json:"path"` + Type string `json:"type,omitempty"` +} diff --git a/cmd/linuxaid-agent/main.go b/cmd/linuxaid-agent/main.go new file mode 100644 index 0000000..dff47c6 --- /dev/null +++ b/cmd/linuxaid-agent/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "log/slog" + "os" + + "github.com/spf13/cobra" +) + +var Version string + +var rootCmd = &cobra.Command{ + Use: "linuxaid-agent", + Short: "In-cluster controller for running LinuxAid OpenVox agents on Kubernetes nodes", + Version: Version, + CompletionOptions: cobra.CompletionOptions{ + HiddenDefaultCmd: true, + }, +} + +func main() { + if err := rootCmd.Execute(); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } +} From 23d9a92acdf6b1d3d331bcb9040e364929a93a9b Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 8 Jul 2026 16:20:18 +0530 Subject: [PATCH 4/8] feat(linuxaid-agent): rename fanout to operator, run as a reconcile daemon The operator reconciles one agent Job per node immediately and then every --interval (default 4h), re-reading the rotated SA token per request. A single node can be triggered on demand with --node , which skips cleanly when that node's Job already exists. --- cmd/linuxaid-agent/fanout.go | 396 ------------------------ cmd/linuxaid-agent/operator.go | 531 +++++++++++++++++++++++++++++++++ 2 files changed, 531 insertions(+), 396 deletions(-) delete mode 100644 cmd/linuxaid-agent/fanout.go create mode 100644 cmd/linuxaid-agent/operator.go diff --git a/cmd/linuxaid-agent/fanout.go b/cmd/linuxaid-agent/fanout.go deleted file mode 100644 index 42c808b..0000000 --- a/cmd/linuxaid-agent/fanout.go +++ /dev/null @@ -1,396 +0,0 @@ -package main - -import ( - "bytes" - "crypto/tls" - "crypto/x509" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "net/url" - "os" - "strings" - "time" - - "github.com/spf13/cobra" -) - -const saDir = "/var/run/secrets/kubernetes.io/serviceaccount" - -var ( - faImage string - faCertname string - faEnforce bool - faOpenvoxEnv string - faPuppetServer string - faSecretName string - faHostPath string - faNodeSelector string - faNamePrefix string - faNamespace string - faTTLSeconds int - faActiveDeadline int - faBackoffLimit int -) - -var fanoutCmd = &cobra.Command{ - Use: "fanout", - Short: "List cluster nodes and create one agent Job per node", - RunE: func(_ *cobra.Command, _ []string) error { - return runFanout() - }, -} - -func init() { - f := fanoutCmd.Flags() - f.StringVar(&faImage, "image", "", "agent container image (required)") - f.StringVar(&faCertname, "certname", "", "obmondo-clientcert CN, shared by all nodes (required)") - f.BoolVar(&faEnforce, "enforce", false, "apply changes (puppet --no-noop) instead of report-only") - f.StringVar(&faOpenvoxEnv, "openvox-environment", "master", "LinuxAid/OpenVox environment") - f.StringVar(&faPuppetServer, "puppet-server", "", "optional puppetserver override") - f.StringVar(&faSecretName, "cert-secret", "obmondo-clientcert", "secret holding the pre-signed client cert") - f.StringVar(&faHostPath, "host-path", "/opt/obmondo", "host path for staging the CLI binary") - f.StringVar(&faNodeSelector, "node-selector", "", "label selector to filter nodes (empty = all)") - f.StringVar(&faNamePrefix, "name-prefix", "linuxaid-agents", "prefix for the per-node Job names") - f.StringVar(&faNamespace, "namespace", "", "namespace to create Jobs in (default: the pod's namespace)") - f.IntVar(&faTTLSeconds, "ttl-seconds", 600, "ttlSecondsAfterFinished on the agent Jobs") - f.IntVar(&faActiveDeadline, "active-deadline-seconds", 1200, "activeDeadlineSeconds on the agent Jobs") - f.IntVar(&faBackoffLimit, "backoff-limit", 1, "backoffLimit on the agent Jobs") - rootCmd.AddCommand(fanoutCmd) -} - -func runFanout() error { - if faImage == "" || faCertname == "" { - return fmt.Errorf("--image and --certname are required") - } - - cfg, err := inClusterConfig() - if err != nil { - return err - } - - nodes, err := cfg.listNodes(faNodeSelector) - if err != nil { - return err - } - if len(nodes) == 0 { - slog.Warn("no nodes matched", slog.String("selector", faNodeSelector)) - return nil - } - - slog.Info("fanning out agent jobs", - slog.Int("nodes", len(nodes)), - slog.String("namespace", cfg.namespace), - slog.Bool("enforce", faEnforce)) - - var failed int - for _, node := range nodes { - slug := slugify(node) - name := jobName(faNamePrefix, slug) - - manifest, err := buildJobManifest(node, name, slug) - if err != nil { - slog.Error("build job", slog.String("node", node), slog.Any("error", err)) - failed++ - continue - } - - created, err := cfg.createJob(manifest) - switch { - case err != nil: - slog.Error("create job", slog.String("node", node), slog.Any("error", err)) - failed++ - case created: - slog.Info("created job", slog.String("job", name), slog.String("node", node)) - default: - slog.Info("skipped, already present (no-concurrency)", slog.String("job", name), slog.String("node", node)) - } - } - - if failed > 0 { - return fmt.Errorf("%d of %d nodes failed", failed, len(nodes)) - } - return nil -} - -type clusterConfig struct { - host string - token string - namespace string - client *http.Client -} - -func inClusterConfig() (*clusterConfig, error) { - host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") - if host == "" || port == "" { - return nil, fmt.Errorf("not running in-cluster: KUBERNETES_SERVICE_HOST/PORT are unset") - } - - token, err := os.ReadFile(saDir + "/token") - if err != nil { - return nil, fmt.Errorf("read service account token: %w", err) - } - - caPEM, err := os.ReadFile(saDir + "/ca.crt") - if err != nil { - return nil, fmt.Errorf("read cluster CA: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("parse cluster CA") - } - - namespace := faNamespace - if namespace == "" { - b, err := os.ReadFile(saDir + "/namespace") - if err != nil { - return nil, fmt.Errorf("read namespace: %w", err) - } - namespace = strings.TrimSpace(string(b)) - } - - return &clusterConfig{ - host: fmt.Sprintf("https://%s:%s", host, port), - token: strings.TrimSpace(string(token)), - namespace: namespace, - client: &http.Client{ - Timeout: 30 * time.Second, //nolint:mnd - Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}, - }, - }, nil -} - -func (c *clusterConfig) do(method, path string, body []byte) (*http.Response, error) { - var r io.Reader = http.NoBody - if body != nil { - r = bytes.NewReader(body) - } - req, err := http.NewRequest(method, c.host+path, r) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+c.token) - req.Header.Set("Accept", "application/json") - if body != nil { - req.Header.Set("Content-Type", "application/json") - } - return c.client.Do(req) -} - -func (c *clusterConfig) listNodes(selector string) ([]string, error) { - path := "/api/v1/nodes" - if selector != "" { - path += "?labelSelector=" + url.QueryEscape(selector) - } - - resp, err := c.do(http.MethodGet, path, nil) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("list nodes: %s: %s", resp.Status, strings.TrimSpace(string(b))) - } - - var out struct { - Items []struct { - Metadata struct { - Name string `json:"name"` - } `json:"metadata"` - } `json:"items"` - } - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return nil, err - } - - names := make([]string, 0, len(out.Items)) - for _, item := range out.Items { - names = append(names, item.Metadata.Name) - } - return names, nil -} - -// createJob POSTs the Job; a 409 means it already exists (still running) and is -// treated as a skip, which is what gives us no-concurrency per node. -func (c *clusterConfig) createJob(manifest []byte) (created bool, err error) { - resp, err := c.do(http.MethodPost, fmt.Sprintf("/apis/batch/v1/namespaces/%s/jobs", c.namespace), manifest) - if err != nil { - return false, err - } - defer resp.Body.Close() - - switch resp.StatusCode { - case http.StatusCreated, http.StatusOK: - return true, nil - case http.StatusConflict: - return false, nil - default: - b, _ := io.ReadAll(resp.Body) - return false, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b))) - } -} - -func buildJobManifest(node, name, slug string) ([]byte, error) { - env := []envVar{ - {Name: "CERTNAME", Value: faCertname}, - {Name: "ENFORCE", Value: fmt.Sprintf("%t", faEnforce)}, - {Name: "OPENVOX_ENVIRONMENT", Value: faOpenvoxEnv}, - {Name: "CLIENT_CERT_DIR", Value: "/obmondo-clientcert"}, - } - if faPuppetServer != "" { - env = append(env, envVar{Name: "PUPPET_SERVER", Value: faPuppetServer}) - } - - labels := map[string]string{ - "app.kubernetes.io/name": "linuxaid-agents", - "app.kubernetes.io/managed-by": "linuxaid-agent", - "linuxaid-agents.obmondo.com/node": slug, - } - - j := k8sJob{ - APIVersion: "batch/v1", - Kind: "Job", - Metadata: objMeta{Name: name, Labels: labels}, - Spec: jobSpec{ - BackoffLimit: faBackoffLimit, - ActiveDeadlineSeconds: faActiveDeadline, - TTLSecondsAfterFinished: faTTLSeconds, - Template: podTemplate{ - Metadata: objMeta{Labels: labels}, - Spec: podSpec{ - RestartPolicy: "Never", - HostPID: true, - AutomountServiceAccountToken: false, - NodeName: node, - Tolerations: []toleration{{Operator: "Exists"}}, - Containers: []container{{ - Name: "agent", - Image: faImage, - SecurityContext: secCtx{Privileged: true, RunAsUser: 0}, - Env: env, - VolumeMounts: []volumeMount{ - {Name: "obmondo-clientcert", MountPath: "/obmondo-clientcert", ReadOnly: true}, - {Name: "host-obmondo", MountPath: faHostPath}, - }, - }}, - Volumes: []volume{ - {Name: "obmondo-clientcert", Secret: &secretVol{SecretName: faSecretName}}, - {Name: "host-obmondo", HostPath: &hostPathVol{Path: faHostPath, Type: "DirectoryOrCreate"}}, - }, - }, - }, - }, - } - return json.Marshal(j) -} - -// slugify turns a node name into a DNS-1123 label fragment. -func slugify(s string) string { - var b strings.Builder - for _, r := range strings.ToLower(s) { - if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { - b.WriteRune(r) - } else { - b.WriteRune('-') - } - } - slug := b.String() - for strings.Contains(slug, "--") { - slug = strings.ReplaceAll(slug, "--", "-") - } - slug = strings.Trim(slug, "-") - if len(slug) > 40 { //nolint:mnd - slug = strings.Trim(slug[:40], "-") - } - return slug -} - -// jobName keeps the name a valid DNS-1123 label with room for the pod suffix. -func jobName(prefix, slug string) string { - name := prefix + "-" + slug - if len(name) > 57 { //nolint:mnd - name = strings.TrimRight(name[:57], "-") - } - return name -} - -type k8sJob struct { - APIVersion string `json:"apiVersion"` - Kind string `json:"kind"` - Metadata objMeta `json:"metadata"` - Spec jobSpec `json:"spec"` -} - -type objMeta struct { - Name string `json:"name,omitempty"` - Labels map[string]string `json:"labels,omitempty"` -} - -type jobSpec struct { - BackoffLimit int `json:"backoffLimit"` - ActiveDeadlineSeconds int `json:"activeDeadlineSeconds"` - TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` - Template podTemplate `json:"template"` -} - -type podTemplate struct { - Metadata objMeta `json:"metadata,omitempty"` - Spec podSpec `json:"spec"` -} - -type podSpec struct { - RestartPolicy string `json:"restartPolicy"` - HostPID bool `json:"hostPID"` - AutomountServiceAccountToken bool `json:"automountServiceAccountToken"` - NodeName string `json:"nodeName"` - Tolerations []toleration `json:"tolerations,omitempty"` - Containers []container `json:"containers"` - Volumes []volume `json:"volumes"` -} - -type toleration struct { - Operator string `json:"operator"` -} - -type container struct { - Name string `json:"name"` - Image string `json:"image"` - SecurityContext secCtx `json:"securityContext"` - Env []envVar `json:"env,omitempty"` - VolumeMounts []volumeMount `json:"volumeMounts"` -} - -type secCtx struct { - Privileged bool `json:"privileged"` - RunAsUser int `json:"runAsUser"` -} - -type envVar struct { - Name string `json:"name"` - Value string `json:"value"` -} - -type volumeMount struct { - Name string `json:"name"` - MountPath string `json:"mountPath"` - ReadOnly bool `json:"readOnly,omitempty"` -} - -type volume struct { - Name string `json:"name"` - Secret *secretVol `json:"secret,omitempty"` - HostPath *hostPathVol `json:"hostPath,omitempty"` -} - -type secretVol struct { - SecretName string `json:"secretName"` -} - -type hostPathVol struct { - Path string `json:"path"` - Type string `json:"type,omitempty"` -} diff --git a/cmd/linuxaid-agent/operator.go b/cmd/linuxaid-agent/operator.go new file mode 100644 index 0000000..b860c34 --- /dev/null +++ b/cmd/linuxaid-agent/operator.go @@ -0,0 +1,531 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/spf13/cobra" +) + +const saDir = "/var/run/secrets/kubernetes.io/serviceaccount" + +const ( + defaultReconcileInterval = 4 * time.Hour + defaultTTLSeconds = 600 + defaultActiveDeadlineSeconds = 1200 +) + +var ( + opImage string + opImagePullPolicy string + opCertname string + opEnforce bool + opOpenvoxEnv string + opPuppetServer string + opSecretName string + opHostPath string + opNodeSelector string + opNamePrefix string + opNamespace string + opTTLSeconds int + opActiveDeadline int + opBackoffLimit int + opInterval time.Duration + opNode string + opControlRepoURL string + opControlRepoRef string + opHieraConfigMap string + opGitSecretName string +) + +var operatorCmd = &cobra.Command{ + Use: "operator", + Short: "Reconcile one agent Job per node as a daemon, or once for a single --node", + RunE: func(_ *cobra.Command, _ []string) error { + return runOperator() + }, +} + +func init() { + f := operatorCmd.Flags() + f.StringVar(&opImage, "image", "", "agent container image (required)") + f.StringVar(&opImagePullPolicy, "image-pull-policy", "IfNotPresent", "imagePullPolicy for the agent Jobs") + f.StringVar(&opCertname, "certname", "", "obmondo-clientcert CN, shared by all nodes (required)") + f.BoolVar(&opEnforce, "enforce", false, "apply changes (puppet --no-noop) instead of report-only") + f.StringVar(&opOpenvoxEnv, "openvox-environment", "master", "LinuxAid/OpenVox environment") + f.StringVar(&opPuppetServer, "puppet-server", "", "optional puppetserver override") + f.StringVar(&opSecretName, "cert-secret", "obmondo-clientcert", "secret holding the pre-signed client cert") + f.StringVar(&opHostPath, "host-path", "/opt/obmondo", "host path for staging the CLI binary") + f.StringVar(&opNodeSelector, "node-selector", "", "label selector to filter nodes (empty = all)") + f.StringVar(&opNamePrefix, "name-prefix", "linuxaid-agents", "prefix for the per-node Job names") + f.StringVar(&opNamespace, "namespace", "", "namespace to create Jobs in (default: the pod's namespace)") + f.IntVar(&opTTLSeconds, "ttl-seconds", defaultTTLSeconds, "ttlSecondsAfterFinished on the agent Jobs") + f.IntVar(&opActiveDeadline, "active-deadline-seconds", defaultActiveDeadlineSeconds, "activeDeadlineSeconds on the agent Jobs") + f.IntVar(&opBackoffLimit, "backoff-limit", 1, "backoffLimit on the agent Jobs") + f.DurationVar(&opInterval, "interval", defaultReconcileInterval, "reconcile interval in daemon mode") + f.StringVar(&opNode, "node", "", "create a Job for a single node and exit (skips if one already exists)") + f.StringVar(&opControlRepoURL, "control-repo-url", "", "git URL of the puppet control-repo; when set the agent Jobs run masterless puppet apply") + f.StringVar(&opControlRepoRef, "control-repo-ref", "", "tag (or ref) of the control-repo to checkout, from the chart's linuxaid.tag (default: latest tag)") + f.StringVar(&opHieraConfigMap, "hiera-configmap", "", "ConfigMap with the rendered Helm-values hiera data, mounted into the agent Jobs at /hiera-data") + f.StringVar(&opGitSecretName, "git-secret", "", "secret with git credentials (ssh-privatekey or token key) mounted into the agent Jobs at /git-credentials") + rootCmd.AddCommand(operatorCmd) +} + +// runOperator dispatches to a single-node run (--node) or the daemon loop. +func runOperator() error { + if opImage == "" || opCertname == "" { + return fmt.Errorf("--image and --certname are required") + } + + cfg, err := inClusterConfig() + if err != nil { + return err + } + + if opNode != "" { + return runOnce(cfg, opNode) + } + return runDaemon(cfg) +} + +// runOnce creates the Job for a single named node and returns. A pre-existing Job +// (still running or within its TTL) is left untouched, so this is safe to trigger +// repeatedly for the same node. +func runOnce(cfg *clusterConfig, node string) error { + ok, err := cfg.nodeExists(node) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("node %q not found in cluster", node) + } + + created, err := createJobForNode(cfg, node) + if err != nil { + return err + } + if !created { + slog.Info("job already present, skipped", slog.String("node", node)) + return nil + } + slog.Info("created job", slog.String("node", node)) + return nil +} + +// runDaemon reconciles immediately, then every opInterval, until SIGINT/SIGTERM. +// A failed reconcile is logged and retried on the next tick rather than crashing +// the daemon. +func runDaemon(cfg *clusterConfig) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + ticker := time.NewTicker(opInterval) + defer ticker.Stop() + + slog.Info("operator started", + slog.Duration("interval", opInterval), + slog.String("namespace", cfg.namespace), + slog.Bool("enforce", opEnforce)) + + for { + if err := reconcile(cfg); err != nil { + slog.Error("reconcile failed", slog.Any("error", err)) + } + select { + case <-ctx.Done(): + slog.Info("shutting down") + return nil + case <-ticker.C: + } + } +} + +// reconcile lists the matching nodes and ensures one agent Job exists per node. +func reconcile(cfg *clusterConfig) error { + nodes, err := cfg.listNodes(opNodeSelector) + if err != nil { + return err + } + if len(nodes) == 0 { + slog.Warn("no nodes matched", slog.String("selector", opNodeSelector)) + return nil + } + + slog.Info("reconciling", slog.Int("nodes", len(nodes)), slog.Bool("enforce", opEnforce)) + + var failed int + for _, node := range nodes { + created, err := createJobForNode(cfg, node) + switch { + case err != nil: + slog.Error("create job", slog.String("node", node), slog.Any("error", err)) + failed++ + case created: + slog.Info("created job", slog.String("node", node)) + default: + slog.Info("skipped, already present", slog.String("node", node)) + } + } + + if failed > 0 { + return fmt.Errorf("%d of %d nodes failed", failed, len(nodes)) + } + return nil +} + +// createJobForNode builds and POSTs the agent Job for one node. created=false with a +// nil error means a Job already exists (HTTP 409) — the source of at-most-one Job per node. +func createJobForNode(cfg *clusterConfig, node string) (bool, error) { + slug := slugify(node) + name := jobName(opNamePrefix, slug) + + manifest, err := buildJobManifest(node, name, slug) + if err != nil { + return false, fmt.Errorf("build job for %s: %w", node, err) + } + return cfg.createJob(manifest) +} + +type clusterConfig struct { + host string + namespace string + client *http.Client +} + +func inClusterConfig() (*clusterConfig, error) { + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if host == "" || port == "" { + return nil, fmt.Errorf("not running in-cluster: KUBERNETES_SERVICE_HOST/PORT are unset") + } + + // Sanity-check the token exists now; it is re-read per request (see do) because the + // kubelet rotates bound service-account tokens on disk during a long-running daemon. + if _, err := os.ReadFile(saDir + "/token"); err != nil { + return nil, fmt.Errorf("read service account token: %w", err) + } + + caPEM, err := os.ReadFile(saDir + "/ca.crt") + if err != nil { + return nil, fmt.Errorf("read cluster CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("parse cluster CA") + } + + namespace := opNamespace + if namespace == "" { + b, err := os.ReadFile(saDir + "/namespace") + if err != nil { + return nil, fmt.Errorf("read namespace: %w", err) + } + namespace = strings.TrimSpace(string(b)) + } + + return &clusterConfig{ + host: fmt.Sprintf("https://%s:%s", host, port), + namespace: namespace, + client: &http.Client{ + Timeout: 30 * time.Second, //nolint:mnd + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}, + }, + }, nil +} + +func (c *clusterConfig) do(method, path string, body []byte) (*http.Response, error) { + token, err := os.ReadFile(saDir + "/token") + if err != nil { + return nil, fmt.Errorf("read service account token: %w", err) + } + + var r io.Reader = http.NoBody + if body != nil { + r = bytes.NewReader(body) + } + req, err := http.NewRequest(method, c.host+path, r) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(token))) + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return c.client.Do(req) +} + +// nodeList is the trimmed shape decoded from the Kubernetes node-list response. +type nodeList struct { + Items []nodeListItem `json:"items"` +} + +type nodeListItem struct { + Metadata objMeta `json:"metadata"` +} + +func (c *clusterConfig) listNodes(selector string) ([]string, error) { + path := "/api/v1/nodes" + if selector != "" { + path += "?labelSelector=" + url.QueryEscape(selector) + } + + resp, err := c.do(http.MethodGet, path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("list nodes: %s: %s", resp.Status, strings.TrimSpace(string(b))) + } + + var out nodeList + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + + names := make([]string, 0, len(out.Items)) + for _, item := range out.Items { + names = append(names, item.Metadata.Name) + } + return names, nil +} + +// nodeExists reports whether a node object with the given name is present, so a --node +// run fails fast on a typo instead of creating a Job that never schedules. +func (c *clusterConfig) nodeExists(name string) (bool, error) { + resp, err := c.do(http.MethodGet, "/api/v1/nodes/"+url.PathEscape(name), nil) + if err != nil { + return false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + default: + b, _ := io.ReadAll(resp.Body) + return false, fmt.Errorf("get node %q: %s: %s", name, resp.Status, strings.TrimSpace(string(b))) + } +} + +// createJob POSTs the Job; a 409 means it already exists (still running) and is +// treated as a skip, which is what gives us no-concurrency per node. +func (c *clusterConfig) createJob(manifest []byte) (created bool, err error) { + resp, err := c.do(http.MethodPost, fmt.Sprintf("/apis/batch/v1/namespaces/%s/jobs", c.namespace), manifest) + if err != nil { + return false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusCreated, http.StatusOK: + return true, nil + case http.StatusConflict: + return false, nil + default: + b, _ := io.ReadAll(resp.Body) + return false, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b))) + } +} + +func buildJobManifest(node, name, slug string) ([]byte, error) { + env := []envVar{ + {Name: "CERTNAME", Value: opCertname}, + {Name: "ENFORCE", Value: fmt.Sprintf("%t", opEnforce)}, + {Name: "OPENVOX_ENVIRONMENT", Value: opOpenvoxEnv}, + {Name: "CLIENT_CERT_DIR", Value: "/obmondo-clientcert"}, + } + if opPuppetServer != "" { + env = append(env, envVar{Name: "PUPPET_SERVER", Value: opPuppetServer}) + } + if opControlRepoURL != "" { + env = append(env, envVar{Name: "CONTROL_REPO_URL", Value: opControlRepoURL}) + } + if opControlRepoRef != "" { + env = append(env, envVar{Name: "CONTROL_REPO_REF", Value: opControlRepoRef}) + } + + mounts := []volumeMount{ + {Name: "obmondo-clientcert", MountPath: "/obmondo-clientcert", ReadOnly: true}, + {Name: "host-obmondo", MountPath: opHostPath}, + } + volumes := []volume{ + {Name: "obmondo-clientcert", Secret: &secretVol{SecretName: opSecretName}}, + {Name: "host-obmondo", HostPath: &hostPathVol{Path: opHostPath, Type: "DirectoryOrCreate"}}, + } + if opGitSecretName != "" { + mounts = append(mounts, volumeMount{Name: "git-credentials", MountPath: "/git-credentials", ReadOnly: true}) + volumes = append(volumes, volume{Name: "git-credentials", Secret: &secretVol{SecretName: opGitSecretName}}) + } + if opHieraConfigMap != "" { + mounts = append(mounts, volumeMount{Name: "hiera-data", MountPath: "/hiera-data", ReadOnly: true}) + volumes = append(volumes, volume{Name: "hiera-data", ConfigMap: &configMapVol{Name: opHieraConfigMap}}) + } + + labels := map[string]string{ + "app.kubernetes.io/name": "linuxaid-agents", + "app.kubernetes.io/managed-by": "linuxaid-agent", + "linuxaid-agents.obmondo.com/node": slug, + } + + j := k8sJob{ + APIVersion: "batch/v1", + Kind: "Job", + Metadata: objMeta{Name: name, Labels: labels}, + Spec: jobSpec{ + BackoffLimit: opBackoffLimit, + ActiveDeadlineSeconds: opActiveDeadline, + TTLSecondsAfterFinished: opTTLSeconds, + Template: podTemplate{ + Metadata: objMeta{Labels: labels}, + Spec: podSpec{ + RestartPolicy: "Never", + HostPID: true, + AutomountServiceAccountToken: false, + NodeName: node, + Tolerations: []toleration{{Operator: "Exists"}}, + Containers: []container{{ + Name: "agent", + Image: opImage, + ImagePullPolicy: opImagePullPolicy, + SecurityContext: secCtx{Privileged: true, RunAsUser: 0}, + Env: env, + VolumeMounts: mounts, + }}, + Volumes: volumes, + }, + }, + }, + } + return json.Marshal(j) +} + +// slugify turns a node name into a DNS-1123 label fragment. +func slugify(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } else { + b.WriteRune('-') + } + } + slug := b.String() + for strings.Contains(slug, "--") { + slug = strings.ReplaceAll(slug, "--", "-") + } + slug = strings.Trim(slug, "-") + if len(slug) > 40 { //nolint:mnd + slug = strings.Trim(slug[:40], "-") + } + return slug +} + +// jobName keeps the name a valid DNS-1123 label with room for the pod suffix. +func jobName(prefix, slug string) string { + name := prefix + "-" + slug + if len(name) > 57 { //nolint:mnd + name = strings.TrimRight(name[:57], "-") + } + return name +} + +type k8sJob struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata objMeta `json:"metadata"` + Spec jobSpec `json:"spec"` +} + +type objMeta struct { + Name string `json:"name,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +type jobSpec struct { + BackoffLimit int `json:"backoffLimit"` + ActiveDeadlineSeconds int `json:"activeDeadlineSeconds"` + TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` + Template podTemplate `json:"template"` +} + +type podTemplate struct { + Metadata objMeta `json:"metadata,omitempty"` + Spec podSpec `json:"spec"` +} + +type podSpec struct { + RestartPolicy string `json:"restartPolicy"` + HostPID bool `json:"hostPID"` + AutomountServiceAccountToken bool `json:"automountServiceAccountToken"` + NodeName string `json:"nodeName"` + Tolerations []toleration `json:"tolerations,omitempty"` + Containers []container `json:"containers"` + Volumes []volume `json:"volumes"` +} + +type toleration struct { + Operator string `json:"operator"` +} + +type container struct { + Name string `json:"name"` + Image string `json:"image"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + SecurityContext secCtx `json:"securityContext"` + Env []envVar `json:"env,omitempty"` + VolumeMounts []volumeMount `json:"volumeMounts"` +} + +type secCtx struct { + Privileged bool `json:"privileged"` + RunAsUser int `json:"runAsUser"` +} + +type envVar struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type volumeMount struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + ReadOnly bool `json:"readOnly,omitempty"` +} + +type volume struct { + Name string `json:"name"` + Secret *secretVol `json:"secret,omitempty"` + HostPath *hostPathVol `json:"hostPath,omitempty"` + ConfigMap *configMapVol `json:"configMap,omitempty"` +} + +type secretVol struct { + SecretName string `json:"secretName"` +} + +type configMapVol struct { + Name string `json:"name"` +} + +type hostPathVol struct { + Path string `json:"path"` + Type string `json:"type,omitempty"` +} From b2f58bf96adc2c8e62e5907c93284533dfa94547 Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 8 Jul 2026 16:20:19 +0530 Subject: [PATCH 5/8] feat(run-openvox): add masterless puppet apply mode --apply compiles the catalog locally from the cloned control-repo (--environmentpath/--openvox-environment) with the Helm-values data as the global hiera layer (--hiera-config), skipping puppetserver resolution and connectivity gating. Exit-code policy is shared with agent mode. --- cmd/linuxaid-cli/run-openvox.go | 127 +++++++++++++++++++++++++------- constant/constants.go | 10 +++ 2 files changed, 109 insertions(+), 28 deletions(-) diff --git a/cmd/linuxaid-cli/run-openvox.go b/cmd/linuxaid-cli/run-openvox.go index 38ae338..67b90fd 100644 --- a/cmd/linuxaid-cli/run-openvox.go +++ b/cmd/linuxaid-cli/run-openvox.go @@ -1,7 +1,11 @@ package main import ( + "fmt" "log/slog" + "os" + "path/filepath" + "strings" "gitea.obmondo.com/EnableIT/linuxaid-cli/constant" "gitea.obmondo.com/EnableIT/linuxaid-cli/helper" @@ -11,8 +15,16 @@ import ( "github.com/spf13/cobra" ) -// runOpenvoxEnforce applies changes (puppet --no-noop) instead of the default report-only (--noop). -var runOpenvoxEnforce bool +var ( + // runOpenvoxEnforce applies changes (puppet --no-noop) instead of the default report-only (--noop). + runOpenvoxEnforce bool + // runOpenvoxApply compiles the catalog locally (puppet apply) from a cloned + // control-repo instead of asking a puppetserver (puppet agent). + runOpenvoxApply bool + runOpenvoxEnvPath string + runOpenvoxEnvName string + runOpenvoxHieraConf string +) var runOpenvoxCmd = &cobra.Command{ Use: "run-openvox", @@ -24,50 +36,84 @@ var runOpenvoxCmd = &cobra.Command{ }, } -// runOpenvoxAgent runs the puppet agent: report-only (--noop) by default, or applying changes -// (--no-noop --detailed-exitcodes) when enforce is set. -func runOpenvoxAgent(enforce bool) error { - // Puppet run execution returns total 5 status codes - // - // 0: The run succeeded with no changes or failures; the system was already in the desired state. - // 1: The run failed, or wasn't attempted due to another run already in progress. - // 2: The run succeeded, and some resources were changed. - // 4: The run succeeded, and some resources failed. - // 6: The run succeeded, and included both changes and failures. - // [Source: https://www.puppet.com/docs/puppet/7/man/agent.html#usage-notes] - // - // We throw error at status code 1, and return. - // Status codes other than 2 are considered as warning. - // Status code 0 doesn't count as error, so no need to handle it. - +// runPuppet executes a puppet command and maps its exit status. +// +// Puppet run execution returns total 5 status codes +// +// 0: The run succeeded with no changes or failures; the system was already in the desired state. +// 1: The run failed, or wasn't attempted due to another run already in progress. +// 2: The run succeeded, and some resources were changed. +// 4: The run succeeded, and some resources failed. +// 6: The run succeeded, and included both changes and failures. +// [Source: https://www.puppet.com/docs/puppet/7/man/agent.html#usage-notes] +// +// We throw error at status code 1, and return. +// Status codes other than 2 are considered as warning. +// Status code 0 doesn't count as error, so no need to handle it. +func runPuppet(puppetCmd string) error { statusCodeFailed := 1 statusCodeSucceededWithChanges := 2 - puppetCmd := "/opt/puppetlabs/bin/puppet agent -t --noop" - if enforce { - puppetCmd = "/opt/puppetlabs/bin/puppet agent -t --no-noop --detailed-exitcodes" - } - - slog.Info("executing the puppet agent command", slog.Bool("enforce", enforce)) + slog.Info("executing the puppet command", slog.String("command", puppetCmd)) cmdPipe := script.Exec(puppetCmd) _, err := cmdPipe.Stdout() if err != nil { // When encountering status code 1, consider it as an error, and return. if cmdPipe.ExitStatus() == statusCodeFailed { - slog.Error("puppet agent command execution failed", slog.String("status", err.Error())) + slog.Error("puppet command execution failed", slog.String("status", err.Error())) return err } // When encountering status codes other than 2, just log it as a warning. if cmdPipe.ExitStatus() != statusCodeSucceededWithChanges { - slog.Warn("puppet agent run succeeded, but with failures", slog.String("status", err.Error())) + slog.Warn("puppet run succeeded, but with failures", slog.String("status", err.Error())) } } - slog.Info("completed the puppet agent command execution") + slog.Info("completed the puppet command execution") return nil } +// runOpenvoxAgent runs the puppet agent against the puppetserver: report-only (--noop) +// by default, or applying changes (--no-noop --detailed-exitcodes) when enforce is set. +func runOpenvoxAgent(enforce bool) error { + puppetCmd := "/opt/puppetlabs/bin/puppet agent -t --noop" + if enforce { + puppetCmd = "/opt/puppetlabs/bin/puppet agent -t --no-noop --detailed-exitcodes" + } + + slog.Info("executing the puppet agent command", slog.Bool("enforce", enforce)) + return runPuppet(puppetCmd) +} + +// applyOpenvox runs masterless puppet apply against the environment cloned under +// environmentPath. The environment's own environment.conf and hiera.yaml drive the +// modulepath and hierarchy, exactly as on the puppetserver; hieraConfig injects the +// Helm-values data as the (higher-precedence) global hiera layer. +func applyOpenvox(enforce bool, environmentPath, environment, hieraConfig string) error { + sitePP := filepath.Join(environmentPath, environment, "manifests", "site.pp") + if _, err := os.Stat(sitePP); err != nil { + return fmt.Errorf("site.pp not found at %s, is the control-repo cloned: %w", sitePP, err) + } + + args := []string{ + "/opt/puppetlabs/bin/puppet", "apply", + "--detailed-exitcodes", + "--environmentpath", environmentPath, + "--environment", environment, + } + if hieraConfig != "" { + args = append(args, "--hiera_config", hieraConfig) + } + if !enforce { + args = append(args, "--noop") + } + args = append(args, sitePP) + + slog.Info("executing the puppet apply command", slog.Bool("enforce", enforce), slog.String("environment", environment)) + return runPuppet(strings.Join(args, " ")) +} + // Entry point func RunOpenvox(enforce bool) { helper.LoadPuppetEnv() @@ -75,6 +121,22 @@ func RunOpenvox(enforce bool) { obmondoAPI := api.NewObmondoClient(api.GetObmondoURL(), false) certname := helper.GetCertname() + + // Masterless mode: the catalog is compiled locally from the cloned repo, so + // there is no puppetserver to resolve or reach. + if runOpenvoxApply { + // nolint:errcheck + obmondoAPI.ServerPing() + + if err := applyOpenvox(enforce, runOpenvoxEnvPath, runOpenvoxEnvName, runOpenvoxHieraConf); err != nil { + slog.Error("unable to run puppet apply", slog.String("error", err.Error())) + } + + // nolint:errcheck + obmondoAPI.UpdatePuppetLastRunReport() + return + } + prometheusHost, puppetServerHost := resolveCustomerURLs(obmondoAPI, certname) slog.Info("resolved customer URLs", slog.String("prometheus", prometheusHost), @@ -101,7 +163,16 @@ func RunOpenvox(enforce bool) { } func init() { - runOpenvoxCmd.Flags().BoolVar(&runOpenvoxEnforce, constant.CobraFlagEnforce, false, + f := runOpenvoxCmd.Flags() + f.BoolVar(&runOpenvoxEnforce, constant.CobraFlagEnforce, false, "Apply changes by running puppet with --no-noop (default is report-only --noop)") + f.BoolVar(&runOpenvoxApply, constant.CobraFlagApply, false, + "Run masterless puppet apply against a locally cloned control-repo instead of puppet agent") + f.StringVar(&runOpenvoxEnvPath, constant.CobraFlagEnvironmentPath, constant.DefaultEnvironmentPath, + "Path holding the cloned puppet environments (apply mode)") + f.StringVar(&runOpenvoxEnvName, constant.CobraFlagOpenvoxEnv, constant.DefaultOpenvoxEnv, + "LinuxAid/OpenVox environment to apply (apply mode)") + f.StringVar(&runOpenvoxHieraConf, constant.CobraFlagHieraConfig, constant.DefaultHieraConfig, + "Global-layer hiera.yaml for puppet apply, empty to use the environment hierarchy only (apply mode)") rootCmd.AddCommand(runOpenvoxCmd) } diff --git a/constant/constants.go b/constant/constants.go index 99ada32..f7d79aa 100644 --- a/constant/constants.go +++ b/constant/constants.go @@ -21,6 +21,13 @@ const ( DefaultPuppetServerCustomerID = "enableit" DefaultPuppetServerDomainSuffix = ".puppet.obmondo.com" DefaultOpenvoxEnv = "master" + // DefaultEnvironmentPath is where the per-node Job clones the puppet code for + // masterless apply; it must live under the /opt/obmondo hostPath mount so the + // host-side puppet sees it at the same absolute path. + DefaultEnvironmentPath = "/opt/obmondo/openvox/code/environments" + // DefaultHieraConfig is the global-layer hiera.yaml the Job generates; its + // hierarchy points at the Helm-values data staged under /opt/obmondo. + DefaultHieraConfig = "/opt/obmondo/openvox/hiera.yaml" // Progress Bar BarProgressSize = 100 @@ -41,6 +48,9 @@ const ( CobraFlagNoReboot = "no-reboot" CobraFlagSkipOpenvox = "skip-openvox" CobraFlagEnforce = "enforce" + CobraFlagApply = "apply" + CobraFlagEnvironmentPath = "environmentpath" + CobraFlagHieraConfig = "hiera-config" CobraFlagSecurityExporterURL = "security-exporter-url" ObmondoEnv = "OBMONDO_ENV" From 514a48f37821f29c72976452688ee216aa9e38a5 Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 8 Jul 2026 16:20:28 +0530 Subject: [PATCH 6/8] feat(deploy): clone the control-repo at the pinned tag and stage values-as-hiera Apply mode (CONTROL_REPO_URL set) shallow-clones the repo at CONTROL_REPO_REF into the /opt/obmondo hostPath, stages the mounted ConfigMap values and a generated global-layer hiera.yaml next to it, and runs linuxaid-cli run-openvox --apply on the host. Agent mode is unchanged; the runtime image gains git and openssh-client for the in-container clone. --- Dockerfile | 4 +- deploy/entrypoint.sh | 197 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 158 insertions(+), 43 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52aeafc..f7fb3ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,9 @@ RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache CGO_ENABLED=0 go build -trimpath -ldflags="-X main.Version=${VERSION} -s -w" -o /out/linuxaid-cli ./cmd/linuxaid-cli FROM alpine:3.20 -RUN apk add --no-cache bash util-linux ca-certificates +# git + openssh-client: apply mode clones the puppet code in-container into the +# /opt/obmondo hostPath (the host itself is not required to have git). +RUN apk add --no-cache bash util-linux ca-certificates git openssh-client COPY --from=build /out/linuxaid-cli /usr/local/bin/linuxaid-cli COPY deploy/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index fbe4d9d..24552f9 100644 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -2,23 +2,44 @@ # # Launcher for the LinuxAid OpenVox agent on a Kubernetes node. # -# The pod is only a delivery + scheduling shell: it stages the static -# linuxaid-cli binary onto the host and runs the agent inside the host's -# namespaces via nsenter, because package/user/sudo management must happen on -# the host, not in the container. Designed to run once (as a Job) and exit. +# The pod is only a delivery + scheduling shell: it stages the static linuxaid-cli +# binary onto the host and runs the agent inside the host's namespaces via nsenter, +# because package/user management must happen on the host, not the container. +# Designed to run once (as a Job) and exit. # -# Cert model: the pod mounts the cluster's obmondo-clientcert; this script -# installs it into the host puppet SSL tree under CERTNAME so that both -# `puppet agent` and linuxaid-cli's Obmondo API client pick it up. +# The obmondo-clientcert is mounted and installed into the host puppet SSL tree +# under CERTNAME (already signed — nothing to enroll). If the openvox agent is +# missing on the host, it is installed first from the voxpupuli openvox8 apt repo. +# +# Two run modes: +# - agent (default): puppet agent -t against the customer's puppetserver. +# - apply (CONTROL_REPO_URL set): masterless puppet apply. The control-repo is +# cloned at CONTROL_REPO_REF (the chart's linuxaid.tag; latest tag when unset) +# into CODE_DIR under the /opt/obmondo hostPath — so the host sees it at the +# same absolute path — and the Helm-values hiera data mounted at HIERA_SRC_DIR +# is staged next to it as the global hiera layer. set -euo pipefail CERTNAME="${CERTNAME:?CERTNAME (obmondo-clientcert CN, e.g. .) is required}" ENFORCE="${ENFORCE:-false}" OPENVOX_ENVIRONMENT="${OPENVOX_ENVIRONMENT:-master}" -PUPPET_SERVER="${PUPPET_SERVER:-}" # optional; run-openvox also resolves it from the Obmondo API +PUPPET_SERVER="${PUPPET_SERVER:-}" # agent mode only; run-openvox also resolves it from the Obmondo API CLIENT_CERT_DIR="${CLIENT_CERT_DIR:-/obmondo-clientcert}" # mounted obmondo-clientcert (tls.crt / tls.key / ca.crt) +OPENVOX_RELEASE_DEB="${OPENVOX_RELEASE_DEB:-https://apt.voxpupuli.org/openvox8-release-ubuntu26.04.deb}" + +CONTROL_REPO_URL="${CONTROL_REPO_URL:-}" # apply-mode switch: git URL of the puppet control-repo (environment code) +CONTROL_REPO_REF="${CONTROL_REPO_REF:-}" # tag to checkout, from the chart's linuxaid.tag (default: latest tag) +HIERA_SRC_DIR="${HIERA_SRC_DIR:-/hiera-data}" # mounted ConfigMap with the rendered Helm-values hiera data +GIT_CRED_DIR="${GIT_CRED_DIR:-/git-credentials}" # optional secret mount: ssh-privatekey or token + +OPENVOX_DIR=/opt/obmondo/openvox # must live under the /opt/obmondo hostPath mount +CODE_DIR="${CODE_DIR:-${OPENVOX_DIR}/code}" +DATA_DIR="${OPENVOX_DIR}/data" +HIERA_CONF="${OPENVOX_DIR}/hiera.yaml" HOST_BIN_DIR=/opt/obmondo/bin +HOST_CLI="${HOST_BIN_DIR}/linuxaid-cli" +PUPPET_BIN=/opt/puppetlabs/bin/puppet PUPPET_SSL=/etc/puppetlabs/puppet/ssl PUPPET_CONF=/etc/puppetlabs/puppet/puppet.conf @@ -26,24 +47,49 @@ PUPPET_CONF=/etc/puppetlabs/puppet/puppet.conf ns() { nsenter --target 1 --mount --uts --ipc --net --pid -- "$@"; } log() { printf '[entrypoint] %s\n' "$*"; } -log "certname=${CERTNAME} enforce=${ENFORCE} env=${OPENVOX_ENVIRONMENT} server=${PUPPET_SERVER:-}" +# git with credentials from GIT_CRED_DIR when present: an ssh-privatekey (copied to +# 0600 first — secret mounts are world-readable and ssh refuses such keys) or a +# Gitea token for https remotes. +git_cmd() { + if [ -f "${GIT_CRED_DIR}/ssh-privatekey" ]; then + if [ ! -f /tmp/git-ssh-key ]; then + install -m 0600 "${GIT_CRED_DIR}/ssh-privatekey" /tmp/git-ssh-key + fi + GIT_SSH_COMMAND="ssh -i /tmp/git-ssh-key -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/tmp/git-known-hosts" git "$@" + elif [ -f "${GIT_CRED_DIR}/token" ]; then + git -c http.extraHeader="Authorization: token $(cat "${GIT_CRED_DIR}/token")" "$@" + else + git "$@" + fi +} + +# Clone a repo at the given ref; without one, at its latest tag (version-sorted), +# falling back to the default-branch tip for untagged repos. +clone_repo() { + local url=$1 dest=$2 ref=$3 + rm -rf "${dest}" + mkdir -p "$(dirname "${dest}")" -# 1) Stage the static CLI onto the host via the /opt/obmondo hostPath mount -# (visible in the host mount namespace after nsenter). + if [ -z "${ref}" ]; then + ref="$(git_cmd ls-remote --refs --sort='-v:refname' --tags "${url}" | awk -F'refs/tags/' 'NR==1{print $2}')" + fi + if [ -n "${ref}" ]; then + log "cloning ${url} at ${ref}" + git_cmd clone --quiet --depth 1 --branch "${ref}" "${url}" "${dest}" + else + log "no tags on ${url} — cloning default-branch tip" + git_cmd clone --quiet --depth 1 "${url}" "${dest}" + fi +} + +log "certname=${CERTNAME} enforce=${ENFORCE} env=${OPENVOX_ENVIRONMENT} mode=$([ -n "${CONTROL_REPO_URL}" ] && echo apply || echo agent)" + +# 1) Stage the static CLI onto the host via the /opt/obmondo hostPath mount. mkdir -p "${HOST_BIN_DIR}" -install -m 0755 /usr/local/bin/linuxaid-cli "${HOST_BIN_DIR}/linuxaid-cli" - -# The openvox/puppet agent must already be installed on the host — this image -# ships neither the agent nor linuxaid-install (the cert is provided pre-signed, -# so there is nothing to enroll). Fail fast with a clear message if it is absent. -if ! ns test -x /opt/puppetlabs/bin/puppet; then - log "ERROR: /opt/puppetlabs/bin/puppet not found on the host — pre-install the openvox agent on this node." - exit 1 -fi +install -m 0755 /usr/local/bin/linuxaid-cli "${HOST_CLI}" -# 2) Install the mounted obmondo-clientcert into the host puppet SSL tree under -# CERTNAME. The `< file` redirection is resolved in the container mount ns, so -# the cert content is piped into a host-side writer. +# 2) Install the mounted obmondo-clientcert into the host puppet SSL tree under CERTNAME. +# (`< file` is resolved in the container mount ns, piped into a host-side writer.) ns install -d -m 0755 "${PUPPET_SSL}/certs" ns install -d -m 0700 "${PUPPET_SSL}/private_keys" ns tee "${PUPPET_SSL}/certs/${CERTNAME}.pem" >/dev/null < "${CLIENT_CERT_DIR}/tls.crt" @@ -55,22 +101,89 @@ if [ -f "${CLIENT_CERT_DIR}/ca.crt" ]; then ns chmod 0644 "${PUPPET_SSL}/certs/ca.pem" fi -# 3) Minimal puppet.conf on the host. -ns install -d -m 0755 "$(dirname "${PUPPET_CONF}")" -{ - echo "[main]" - if [ -n "${PUPPET_SERVER}" ]; then echo "server = ${PUPPET_SERVER}"; fi - echo "certname = ${CERTNAME}" - echo "masterport = 443" - echo "environment = ${OPENVOX_ENVIRONMENT}" - echo - echo "[agent]" - echo "report = true" -} | ns tee "${PUPPET_CONF}" >/dev/null - -# 4) Run the agent on the host. Report-only (--noop) unless ENFORCE=true. -args=(run-openvox) -if [ "${ENFORCE}" = "true" ]; then args+=(--enforce); fi -log "running: linuxaid-cli ${args[*]}" -ns env CERTNAME="${CERTNAME}" PATH="/opt/puppetlabs/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ - "${HOST_BIN_DIR}/linuxaid-cli" "${args[@]}" +# 3) Install the openvox agent on the host if it is missing, from the voxpupuli +# openvox8 apt repo. Runs on the host (Ubuntu) via nsenter. +if ! ns test -x "${PUPPET_BIN}"; then + log "openvox agent not found on host — installing from ${OPENVOX_RELEASE_DEB}" + ns bash -c ' + set -eu + export DEBIAN_FRONTEND=noninteractive + url="$1" + deb="$(mktemp --suffix=.deb)" + if command -v curl >/dev/null 2>&1; then + curl -fsSL -o "$deb" "$url" + else + wget -qO "$deb" "$url" + fi + apt-get install -y "$deb" + apt-get update + apt-get install -y openvox-agent + rm -f "$deb" + ' _ "${OPENVOX_RELEASE_DEB}" +fi + +if [ -n "${CONTROL_REPO_URL}" ]; then + # 4a) Masterless apply: materialize the puppet code + hiera data on the host. + # The clone runs in the container (which has git+ssh) into the hostPath, + # so the host-side puppet apply reads it at the same absolute path. + if [ ! -f "${HIERA_SRC_DIR}/values.yaml" ]; then + log "no hiera data at ${HIERA_SRC_DIR}/values.yaml — is the hiera ConfigMap mounted?" + exit 1 + fi + + env_dir="${CODE_DIR}/environments/${OPENVOX_ENVIRONMENT}" + clone_repo "${CONTROL_REPO_URL}" "${env_dir}" "${CONTROL_REPO_REF}" + + # Stage the Helm-values hiera data and point a global-layer hiera.yaml at it: + # global beats the environment layer, so chart values win over repo defaults. + mkdir -p "${DATA_DIR}" + install -m 0644 "${HIERA_SRC_DIR}/values.yaml" "${DATA_DIR}/values.yaml" + cat > "${HIERA_CONF}" <<-EOF + version: 5 + defaults: + datadir: ${DATA_DIR} + data_hash: yaml_data + hierarchy: + - name: "Helm chart values" + path: values.yaml + EOF + + # Minimal puppet.conf on the host — no server: the catalog is compiled locally. + ns install -d -m 0755 "$(dirname "${PUPPET_CONF}")" + { + echo "[main]" + echo "certname = ${CERTNAME}" + echo "environment = ${OPENVOX_ENVIRONMENT}" + echo "report = true" + } | ns tee "${PUPPET_CONF}" >/dev/null + + # 5a) Run masterless apply on the host. Report-only (--noop) unless ENFORCE=true. + args=(run-openvox --apply + --environmentpath "${CODE_DIR}/environments" + --openvox-environment "${OPENVOX_ENVIRONMENT}" + --hiera-config "${HIERA_CONF}") + if [ "${ENFORCE}" = "true" ]; then args+=(--enforce); fi + log "running: linuxaid-cli ${args[*]}" + ns env CERTNAME="${CERTNAME}" OBMONDO_SKIP_CONNECTIVITY_METRICS=1 PATH="/opt/puppetlabs/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + "${HOST_CLI}" "${args[@]}" +else + # 4b) Agent mode: minimal puppet.conf on the host, pointing at the puppetserver. + ns install -d -m 0755 "$(dirname "${PUPPET_CONF}")" + { + echo "[main]" + if [ -n "${PUPPET_SERVER}" ]; then echo "server = ${PUPPET_SERVER}"; fi + echo "certname = ${CERTNAME}" + echo "masterport = 443" + echo "environment = ${OPENVOX_ENVIRONMENT}" + echo + echo "[agent]" + echo "report = true" + } | ns tee "${PUPPET_CONF}" >/dev/null + + # 5b) Run the agent on the host. Report-only (--noop) unless ENFORCE=true. + args=(run-openvox) + if [ "${ENFORCE}" = "true" ]; then args+=(--enforce); fi + log "running: linuxaid-cli ${args[*]}" + ns env CERTNAME="${CERTNAME}" OBMONDO_SKIP_CONNECTIVITY_METRICS=1 PATH="/opt/puppetlabs/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + "${HOST_CLI}" "${args[@]}" +fi From 7a91f6d2f414922e9eeb52e3f3c083a0551325a0 Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 8 Jul 2026 16:20:37 +0530 Subject: [PATCH 7/8] feat(chart): add linuxaid-agent helm chart linuxaid.tag pins the control-repo checkout and is injected into the rendered hiera ConfigMap as common::system::openvox::environment; the free-form hiera block is the cluster-wide node data. Templates cover the operator Deployment and its ServiceAccount/RBAC (nodes get+list, jobs create). --- deploy/chart/linuxaid-agent/Chart.yaml | 10 ++++ .../linuxaid-agent/templates/configmap.yaml | 14 +++++ .../linuxaid-agent/templates/deployment.yaml | 47 +++++++++++++++ .../chart/linuxaid-agent/templates/rbac.yaml | 59 +++++++++++++++++++ deploy/chart/linuxaid-agent/values.yaml | 27 +++++++++ 5 files changed, 157 insertions(+) create mode 100644 deploy/chart/linuxaid-agent/Chart.yaml create mode 100644 deploy/chart/linuxaid-agent/templates/configmap.yaml create mode 100644 deploy/chart/linuxaid-agent/templates/deployment.yaml create mode 100644 deploy/chart/linuxaid-agent/templates/rbac.yaml create mode 100644 deploy/chart/linuxaid-agent/values.yaml diff --git a/deploy/chart/linuxaid-agent/Chart.yaml b/deploy/chart/linuxaid-agent/Chart.yaml new file mode 100644 index 0000000..19eb021 --- /dev/null +++ b/deploy/chart/linuxaid-agent/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: linuxaid-agent +description: >- + LinuxAid OpenVox agent for Kubernetes nodes: an operator Deployment that + reconciles one per-node Job every interval; each Job clones the puppet + control-repo at the pinned tag and runs masterless puppet apply on its node, + with the chart values as the global hiera layer. +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/deploy/chart/linuxaid-agent/templates/configmap.yaml b/deploy/chart/linuxaid-agent/templates/configmap.yaml new file mode 100644 index 0000000..09f79e9 --- /dev/null +++ b/deploy/chart/linuxaid-agent/templates/configmap.yaml @@ -0,0 +1,14 @@ +# The rendered Helm values ARE the node's hiera data: the operator mounts this +# ConfigMap into every per-node Job at /hiera-data, the Job stages it onto the +# host, and puppet apply reads it as the (highest-precedence) global hiera layer. +# linuxaid.tag is injected as common::system::openvox::environment so the puppet +# code knows which control-repo tag it was applied from. +apiVersion: v1 +kind: ConfigMap +metadata: + name: linuxaid-agent-hiera + labels: + app.kubernetes.io/name: linuxaid-agent +data: + values.yaml: | + {{- toYaml (merge (dict "common::system::openvox::environment" (required "linuxaid.tag is required" .Values.linuxaid.tag)) .Values.hiera) | nindent 4 }} diff --git a/deploy/chart/linuxaid-agent/templates/deployment.yaml b/deploy/chart/linuxaid-agent/templates/deployment.yaml new file mode 100644 index 0000000..c0a3d78 --- /dev/null +++ b/deploy/chart/linuxaid-agent/templates/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: linuxaid-agent + labels: + app.kubernetes.io/name: linuxaid-agent +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: linuxaid-agent + template: + metadata: + labels: + app.kubernetes.io/name: linuxaid-agent + spec: + serviceAccountName: linuxaid-agent + containers: + - name: operator + image: {{ .Values.linuxaid.images.operator }} + args: + - operator + - --image={{ .Values.linuxaid.images.agent }} + - --certname={{ required "linuxaid.certname is required" .Values.linuxaid.certname }} + - --control-repo-url={{ .Values.linuxaid.controlRepoURL }} + - --control-repo-ref={{ required "linuxaid.tag is required" .Values.linuxaid.tag }} + - --hiera-configmap=linuxaid-agent-hiera + - --cert-secret={{ .Values.linuxaid.certSecret }} + {{- if .Values.linuxaid.gitSecret }} + - --git-secret={{ .Values.linuxaid.gitSecret }} + {{- end }} + - --openvox-environment={{ .Values.linuxaid.openvoxEnvironment }} + - --interval={{ .Values.linuxaid.interval }} + {{- if .Values.linuxaid.enforce }} + - --enforce + {{- end }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] diff --git a/deploy/chart/linuxaid-agent/templates/rbac.yaml b/deploy/chart/linuxaid-agent/templates/rbac.yaml new file mode 100644 index 0000000..187e404 --- /dev/null +++ b/deploy/chart/linuxaid-agent/templates/rbac.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: linuxaid-agent + labels: + app.kubernetes.io/name: linuxaid-agent +--- +# Nodes are cluster-scoped: list drives the fan-out, get backs --node runs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: linuxaid-agent-nodes + labels: + app.kubernetes.io/name: linuxaid-agent +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: linuxaid-agent-nodes + labels: + app.kubernetes.io/name: linuxaid-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: linuxaid-agent-nodes +subjects: + - kind: ServiceAccount + name: linuxaid-agent + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linuxaid-agent-jobs + labels: + app.kubernetes.io/name: linuxaid-agent +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linuxaid-agent-jobs + labels: + app.kubernetes.io/name: linuxaid-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: linuxaid-agent-jobs +subjects: + - kind: ServiceAccount + name: linuxaid-agent + namespace: {{ .Release.Namespace }} diff --git a/deploy/chart/linuxaid-agent/values.yaml b/deploy/chart/linuxaid-agent/values.yaml new file mode 100644 index 0000000..2b4b09c --- /dev/null +++ b/deploy/chart/linuxaid-agent/values.yaml @@ -0,0 +1,27 @@ +linuxaid: + # Control-repo tag to apply on every node. Pins the git checkout AND is + # injected into the hiera data as common::system::openvox::environment. + tag: "" + controlRepoURL: https://github.com/Obmondo/linuxaid.git + # obmondo-clientcert CN, . + certname: "" + # false = report-only (puppet --noop); true = apply changes + enforce: false + # reconcile interval of the operator (one Job per node per cycle) + interval: 4h + openvoxEnvironment: master + images: + operator: gitea.obmondo.com/enableit/linuxaid-agent:latest + agent: gitea.obmondo.com/enableit/linuxaid-cli:latest + # secret with the pre-signed puppet client cert (tls.crt / tls.key / ca.crt) + certSecret: obmondo-clientcert + # secret with git credentials for the control-repo: key `ssh-privatekey` + # (SSH remotes) or `token` (https remotes). Empty = public repo, no credentials. + gitSecret: "" + +# Free-form hiera data for every node in the cluster, rendered verbatim into +# the ConfigMap the Jobs mount — anything hiera-legal works, e.g.: +# classes: +# - role::kubeaid +# common::monitor::prometheus::server: prometheus.demo.example.com +hiera: {} From 77c5d4f23bad5036ca4553586a65c31494ad73b4 Mon Sep 17 00:00:00 2001 From: Ashish Jaiswal Date: Wed, 8 Jul 2026 16:20:43 +0530 Subject: [PATCH 8/8] fix(checkconnectivity): allow skipping the node_exporter textfile metric OBMONDO_SKIP_CONNECTIVITY_METRICS disables writing the .prom textfile on hosts without the node_exporter textfile collector (e.g. Kubernetes nodes), so a failed write cannot abort the run. --- pkg/checkconnectivity/checkconnectivity.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/checkconnectivity/checkconnectivity.go b/pkg/checkconnectivity/checkconnectivity.go index 00dcf49..8bfe8d4 100644 --- a/pkg/checkconnectivity/checkconnectivity.go +++ b/pkg/checkconnectivity/checkconnectivity.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "os" "time" "github.com/prometheus/client_golang/prometheus" @@ -16,6 +17,10 @@ const ( metricsFile = "/var/lib/node_exporter/obmondo_domains_reachable.prom" apiHost = "api.obmondo.com" + + // skipMetricsEnv, when set, skips writing the node_exporter textfile metric + // (used to disable it in environments like Kubernetes that lack it). + skipMetricsEnv = "OBMONDO_SKIP_CONNECTIVITY_METRICS" ) var runPuppetMetric *prometheus.GaugeVec @@ -64,9 +69,13 @@ func CheckTCPConnection(prometheusHost, puppetServerHost string) bool { runPuppetMetric.WithLabelValues(host, port).Set(0) } - if err := prometheus.WriteToTextfile(metricsFile, registry); err != nil { - slog.Info("Error writing metrics to file:", slog.String("error", err.Error())) - return false + // The .prom metric feeds the fleet's node_exporter textfile collector; skip it + // where that isn't present (e.g. Kubernetes) so a failed write can't abort the run. + if os.Getenv(skipMetricsEnv) == "" { + if err := prometheus.WriteToTextfile(metricsFile, registry); err != nil { + slog.Info("Error writing metrics to file:", slog.String("error", err.Error())) + return false + } } return allAPIReachable