Maintain dumps-nfs mountpoints

.github/workflows/maintain-dumps-nfs.yaml

@@ -0,0 +1,18 @@
+---
+
+name: build and push maintain-dumps-nfs
+
+'on':
+  pull_request_target:
+    paths:
+      - images/maintain-dumps-nfs/**
+
+jobs:
+  build-and-push:
+    name: build and push maintain-dumps-nfs
+    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v4
+    with:
+      imagename: maintain-dumps-nfs
+    secrets:
+      quay_user: ${{ secrets.QUAY_USER }}
+      quay_password: ${{ secrets.QUAY_PASSWORD }}

.github/workflows/update-container-tags.yaml

@@ -31,7 +31,7 @@ jobs:
 
       - name: update values.yaml
         run: |
-          for i in renderer nbserve paws-hub jobber singleuser minesweeper ; do
+          for i in renderer nbserve paws-hub jobber singleuser minesweeper maintain-dumps-nfs ; do
             if [[ $(git diff remotes/toolforgepaws/main -- images/${i}/) ]]; then
             sed -i "s/tag: .* # ${i} tag managed by github actions$/tag: pr-${{ github.event.number }} # ${i} tag managed by github actions/" paws/values.yaml
             fi

images/maintain-dumps-nfs/Dockerfile

@@ -0,0 +1,8 @@
+FROM alpine:3.20.0
+
+RUN apk add --no-cache procps python3 py3-pip py3-psutil nfs-utils util-linux kmod
+RUN python3 -mpip install --break-system-packages --no-cache --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3 -mpip install --break-system-packages --no-cache -r /tmp/requirements.txt
+
+ENV PYTHONUNBUFFERED=1

images/maintain-dumps-nfs/requirements.in

@@ -0,0 +1 @@
+kubernetes

images/maintain-dumps-nfs/requirements.txt

@@ -0,0 +1,54 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --output-file=./requirements.txt ./requirements.in
+#
+cachetools==5.3.1
+    # via google-auth
+certifi==2024.7.4
+    # via
+    #   kubernetes
+    #   requests
+charset-normalizer==3.2.0
+    # via requests
+google-auth==2.22.0
+    # via kubernetes
+idna==3.7
+    # via requests
+kubernetes==27.2.0
+    # via -r requirements.in
+oauthlib==3.2.2
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+pyasn1==0.5.0
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
+python-dateutil==2.8.2
+    # via kubernetes
+pyyaml==6.0.1
+    # via kubernetes
+requests==2.32.4
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+requests-oauthlib==1.3.1
+    # via kubernetes
+rsa==4.9
+    # via google-auth
+six==1.16.0
+    # via
+    #   google-auth
+    #   kubernetes
+    #   python-dateutil
+urllib3==1.26.19
+    # via
+    #   google-auth
+    #   kubernetes
+    #   requests
+websocket-client==1.6.1
+    # via kubernetes

paws/codfw.yaml

@@ -27,6 +27,17 @@ jupyterhub:
       HUB_DOMAIN: "hub-paws-dev.codfw1dev.wmcloud.org"  # Check jupyterhub.ingress.hosts
 minesweeper:
   enabled: true
+maintainDumpsNfs:
+  enabled: false
+  server: pawsdev-nfs.pawsdev.codfw1dev.wikimedia.cloud
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: DoesNotExist
+
 localdev:
   enabled: false
 pawspublic:

paws/files/maintain-dumps-nfs/maintain-dumps-nfs.py

@@ -0,0 +1,223 @@
+"""Daemon that mounts the dumps NFS export on each node and keeps it healthy.
+
+Runs as a DaemonSet with Bidirectional mount propagation so the mount is
+visible on the host.  Also manages clouddumps100[12]-compat symlinks within
+/mnt/nfs and a /public/dumps/public -> /mnt/nfs/dumps symlink for singleuser
+pods.
+
+Similar to dumps-nfs-client-sitter in the puppet repo, but differs in that it
+also manages symlinks as opposed to puppet in production.
+
+/mnt/nfs and /public/dumps are then bind-mounted from the host into singleuser
+pods by the jupyterhub spawner.
+"""
+
+import errno
+import json
+import logging
+import os
+import subprocess
+import time
+
+import kubernetes
+import kubernetes.client
+import kubernetes.config
+import kubernetes.watch
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger("maintain-dumps-nfs")
+
+CONFIG_PATH = "/etc/maintain-dumps-nfs"
+
+MOUNT_OPTIONS = "ro,bg,soft,tcp,noatime,lookupcache=all,nofsc,timeo=20,retrans=1"
+
+DUMPS_SERVER = "dumps-nfs.wikimedia.org"
+DUMPS_MOUNT = "/host/mnt/nfs/dumps"
+
+
+def load_config():
+    """Load daemonset config from the configmap JSON file."""
+    config_file = os.path.join(CONFIG_PATH, "maintain-dumps-nfs.json")
+    if os.path.exists(config_file):
+        with open(config_file) as f:
+            return json.load(f)
+    return {}
+
+
+def read_mountinfo():
+    """Read the container's own mount table from /proc/self/mountinfo."""
+    try:
+        with open("/proc/self/mountinfo") as f:
+            return f.readlines()
+    except FileNotFoundError:
+        logger.error("Cannot read /proc/self/mountinfo")
+        return []
+
+
+def is_nfs_mounted(mount_point):
+    """Check whether an NFS filesystem is mounted at mount_point."""
+    for line in read_mountinfo():
+        parts = line.split()
+        if len(parts) < 10:
+            continue
+        try:
+            sep = parts.index("-")
+        except ValueError:
+            continue
+        fstype = parts[sep + 1]
+        mp = parts[4]
+        if mp == mount_point and fstype in ("nfs", "nfs4"):
+            return True
+    return False
+
+
+def check_mount_healthy(mount_name, mount_info):
+    """Check that the NFS mount exists, is accessible, and is not stale."""
+    host_path = mount_info["host_path"]
+    if not is_nfs_mounted(host_path):
+        logger.warning("Mount %s (%s) not present", mount_name, host_path)
+        return False
+    try:
+        os.listdir(host_path)
+    except OSError as e:
+        if "Stale file handle" in str(e) or "ESTALE" in str(e):
+            logger.warning("Mount %s (%s) has ESTALE", mount_name, host_path)
+        else:
+            logger.warning("Mount %s (%s) unhealthy: %s", mount_name, host_path, e)
+        return False
+    logger.debug("Mount %s (%s) is healthy", mount_name, host_path)
+    return True
+
+
+def umount_host(mount_point):
+    """Force-unmount a filesystem."""
+    logger.info("Unmounting %s", mount_point)
+    try:
+        subprocess.run(["umount", "-f", mount_point], capture_output=True, text=True, check=True)
+    except subprocess.CalledProcessError as e:
+        logger.warning("umount failed: %s", e.stderr.strip())
+
+
+def mount_host(server, mount_point):
+    """Mount an NFS export. The mount propagates to the host via Bidirectional propagation."""
+    logger.info("Mounting %s on %s", server, mount_point)
+    os.makedirs(mount_point, exist_ok=True)
+    try:
+        subprocess.run(
+            ["mount", "-t", "nfs", "-o", MOUNT_OPTIONS, f"{server}:", mount_point],
+            capture_output=True, text=True, check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        logger.warning("mount failed: %s", e.stderr.strip())
+
+
+def ensure_compat_symlinks(host_mnt_nfs):
+    """Create or repair compat symlinks inside the NFS mount root based on config."""
+    config = load_config()
+    compat_symlinks = config.get("compatSymlinks", [])
+    for name in compat_symlinks:
+        link_path = os.path.join(host_mnt_nfs, name)
+        target = "dumps"
+        current_target = None
+        try:
+            current_target = os.readlink(link_path)
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                logger.info("Compat symlink %s missing — creating", link_path)
+            elif os.path.isdir(link_path):
+                try:
+                    entries = os.listdir(link_path)
+                except OSError:
+                    entries = []
+                if entries:
+                    logger.warning(
+                        "Compat symlink %s is a non-empty directory — skipping",
+                        link_path,
+                    )
+                    continue
+                logger.info(
+                    "Compat symlink %s is an empty directory — removing",
+                    link_path,
+                )
+                try:
+                    os.rmdir(link_path)
+                except OSError:
+                    logger.warning("Failed to remove empty directory %s", link_path)
+            else:
+                logger.warning("Cannot read symlink %s: %s", link_path, e)
+                continue
+        if current_target != target:
+            if current_target is not None:
+                logger.info(
+                    "Compat symlink %s points to %s, updating to %s",
+                    link_path, current_target, target,
+                )
+                try:
+                    os.unlink(link_path)
+                except OSError:
+                    logger.warning("Failed to remove symlink %s", link_path)
+            try:
+                os.symlink(target, link_path)
+            except OSError as e:
+                logger.warning("Failed to create symlink %s -> %s: %s", link_path, target, e)
+        else:
+            logger.debug("Compat symlink %s -> %s ok", link_path, target)
+
+
+def ensure_public_dumps_symlink():
+    """Create /host/public/dumps/public -> /mnt/nfs/dumps so singleuser pods can reach dumps via /public/dumps."""
+    dumps_dir = "/host/public/dumps"
+    public_path = os.path.join(dumps_dir, "public")
+    target = "/mnt/nfs/dumps"
+    try:
+        current = os.readlink(public_path)
+        if current == target:
+            logger.info("Symlink %s -> %s already exists", public_path, target)
+            return
+        logger.info("Symlink %s points to %s, updating", public_path, current)
+        os.unlink(public_path)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            logger.info("Symlink %s missing — creating", public_path)
+        elif os.path.isdir(public_path):
+            logger.error(
+                "%s exists and is a directory — cannot create compat symlink",
+                public_path,
+            )
+            return
+        else:
+            logger.info("Removing existing %s to create symlink", public_path)
+            try:
+                os.unlink(public_path)
+            except OSError:
+                logger.warning("Failed to remove %s", public_path)
+                return
+    os.makedirs(dumps_dir, exist_ok=True)
+    os.symlink(target, public_path)
+
+
+def main():
+    """Mount NFS dumps, create compat symlinks, then loop for health checks and remounts."""
+    config = load_config()
+    logger.info("Config: %s", json.dumps(config, default=str))
+
+    server = config.get("server", DUMPS_SERVER)
+    mount_info = {"host_path": DUMPS_MOUNT}
+
+    ensure_public_dumps_symlink()
+    ensure_compat_symlinks("/host/mnt/nfs")
+
+    while True:
+        if not is_nfs_mounted(DUMPS_MOUNT):
+            logger.info("Mount dumps-src (%s) not present — mounting", DUMPS_MOUNT)
+            mount_host(server, DUMPS_MOUNT)
+        elif not check_mount_healthy("dumps-src", mount_info):
+            logger.info("Mount dumps-src (%s) unhealthy — remounting", DUMPS_MOUNT)
+            umount_host(DUMPS_MOUNT)
+            time.sleep(2)
+            mount_host(server, DUMPS_MOUNT)
+        time.sleep(60)
+
+
+if __name__ == "__main__":
+    main()