From 03014d13bb8b45b899499ba917a0a983e8605be6 Mon Sep 17 00:00:00 2001 From: Filippo Giunchedi Date: Tue, 28 Jul 2026 19:04:08 +0200 Subject: [PATCH] add maintain-dumps-nfs --- .github/workflows/maintain-dumps-nfs.yaml | 18 ++ .github/workflows/update-container-tags.yaml | 2 +- images/maintain-dumps-nfs/Dockerfile | 8 + images/maintain-dumps-nfs/requirements.in | 1 + images/maintain-dumps-nfs/requirements.txt | 54 +++++ paws/codfw.yaml | 11 + .../maintain-dumps-nfs/maintain-dumps-nfs.py | 223 ++++++++++++++++++ .../maintain-dumps-nfs_test.py | 182 ++++++++++++++ paws/production.yaml | 10 + .../maintain-dumps-nfs/configmap.yaml | 27 +++ .../maintain-dumps-nfs/daemonset.yaml | 89 +++++++ paws/templates/maintain-dumps-nfs/rbac.yaml | 39 +++ paws/values.yaml | 14 ++ 13 files changed, 677 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/maintain-dumps-nfs.yaml create mode 100644 images/maintain-dumps-nfs/Dockerfile create mode 100644 images/maintain-dumps-nfs/requirements.in create mode 100644 images/maintain-dumps-nfs/requirements.txt create mode 100644 paws/files/maintain-dumps-nfs/maintain-dumps-nfs.py create mode 100644 paws/files/maintain-dumps-nfs/maintain-dumps-nfs_test.py create mode 100644 paws/templates/maintain-dumps-nfs/configmap.yaml create mode 100644 paws/templates/maintain-dumps-nfs/daemonset.yaml create mode 100644 paws/templates/maintain-dumps-nfs/rbac.yaml diff --git a/.github/workflows/maintain-dumps-nfs.yaml b/.github/workflows/maintain-dumps-nfs.yaml new file mode 100644 index 00000000..cff034ba --- /dev/null +++ b/.github/workflows/maintain-dumps-nfs.yaml @@ -0,0 +1,18 @@ +--- + +name: build and push maintain-dumps-nfs + +'on': + pull_request_target: + paths: + - images/maintain-dumps-nfs/** + +jobs: + build-and-push: + name: build and push maintain-dumps-nfs + uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v4 + with: + imagename: maintain-dumps-nfs + secrets: + quay_user: ${{ secrets.QUAY_USER }} + quay_password: ${{ secrets.QUAY_PASSWORD }} diff --git a/.github/workflows/update-container-tags.yaml b/.github/workflows/update-container-tags.yaml index d9a07a2a..cdd56db0 100644 --- a/.github/workflows/update-container-tags.yaml +++ b/.github/workflows/update-container-tags.yaml @@ -31,7 +31,7 @@ jobs: - name: update values.yaml run: | - for i in renderer nbserve paws-hub jobber singleuser minesweeper ; do + for i in renderer nbserve paws-hub jobber singleuser minesweeper maintain-dumps-nfs ; do if [[ $(git diff remotes/toolforgepaws/main -- images/${i}/) ]]; then sed -i "s/tag: .* # ${i} tag managed by github actions$/tag: pr-${{ github.event.number }} # ${i} tag managed by github actions/" paws/values.yaml fi diff --git a/images/maintain-dumps-nfs/Dockerfile b/images/maintain-dumps-nfs/Dockerfile new file mode 100644 index 00000000..3eb4cbf4 --- /dev/null +++ b/images/maintain-dumps-nfs/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.20.0 + +RUN apk add --no-cache procps python3 py3-pip py3-psutil nfs-utils util-linux kmod +RUN python3 -mpip install --break-system-packages --no-cache --upgrade pip +COPY requirements.txt /tmp/requirements.txt +RUN python3 -mpip install --break-system-packages --no-cache -r /tmp/requirements.txt + +ENV PYTHONUNBUFFERED=1 diff --git a/images/maintain-dumps-nfs/requirements.in b/images/maintain-dumps-nfs/requirements.in new file mode 100644 index 00000000..807e21be --- /dev/null +++ b/images/maintain-dumps-nfs/requirements.in @@ -0,0 +1 @@ +kubernetes diff --git a/images/maintain-dumps-nfs/requirements.txt b/images/maintain-dumps-nfs/requirements.txt new file mode 100644 index 00000000..05f350d2 --- /dev/null +++ b/images/maintain-dumps-nfs/requirements.txt @@ -0,0 +1,54 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=./requirements.txt ./requirements.in +# +cachetools==5.3.1 + # via google-auth +certifi==2024.7.4 + # via + # kubernetes + # requests +charset-normalizer==3.2.0 + # via requests +google-auth==2.22.0 + # via kubernetes +idna==3.7 + # via requests +kubernetes==27.2.0 + # via -r requirements.in +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +python-dateutil==2.8.2 + # via kubernetes +pyyaml==6.0.1 + # via kubernetes +requests==2.32.4 + # via + # kubernetes + # requests-oauthlib +requests-oauthlib==1.3.1 + # via kubernetes +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # google-auth + # kubernetes + # python-dateutil +urllib3==1.26.19 + # via + # google-auth + # kubernetes + # requests +websocket-client==1.6.1 + # via kubernetes diff --git a/paws/codfw.yaml b/paws/codfw.yaml index 35dc1aec..b50a1861 100644 --- a/paws/codfw.yaml +++ b/paws/codfw.yaml @@ -27,6 +27,17 @@ jupyterhub: HUB_DOMAIN: "hub-paws-dev.codfw1dev.wmcloud.org" # Check jupyterhub.ingress.hosts minesweeper: enabled: true +maintainDumpsNfs: + enabled: false + server: pawsdev-nfs.pawsdev.codfw1dev.wikimedia.cloud + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + localdev: enabled: false pawspublic: diff --git a/paws/files/maintain-dumps-nfs/maintain-dumps-nfs.py b/paws/files/maintain-dumps-nfs/maintain-dumps-nfs.py new file mode 100644 index 00000000..065718a1 --- /dev/null +++ b/paws/files/maintain-dumps-nfs/maintain-dumps-nfs.py @@ -0,0 +1,223 @@ +"""Daemon that mounts the dumps NFS export on each node and keeps it healthy. + +Runs as a DaemonSet with Bidirectional mount propagation so the mount is +visible on the host. Also manages clouddumps100[12]-compat symlinks within +/mnt/nfs and a /public/dumps/public -> /mnt/nfs/dumps symlink for singleuser +pods. + +Similar to dumps-nfs-client-sitter in the puppet repo, but differs in that it +also manages symlinks as opposed to puppet in production. + +/mnt/nfs and /public/dumps are then bind-mounted from the host into singleuser +pods by the jupyterhub spawner. +""" + +import errno +import json +import logging +import os +import subprocess +import time + +import kubernetes +import kubernetes.client +import kubernetes.config +import kubernetes.watch + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger("maintain-dumps-nfs") + +CONFIG_PATH = "/etc/maintain-dumps-nfs" + +MOUNT_OPTIONS = "ro,bg,soft,tcp,noatime,lookupcache=all,nofsc,timeo=20,retrans=1" + +DUMPS_SERVER = "dumps-nfs.wikimedia.org" +DUMPS_MOUNT = "/host/mnt/nfs/dumps" + + +def load_config(): + """Load daemonset config from the configmap JSON file.""" + config_file = os.path.join(CONFIG_PATH, "maintain-dumps-nfs.json") + if os.path.exists(config_file): + with open(config_file) as f: + return json.load(f) + return {} + + +def read_mountinfo(): + """Read the container's own mount table from /proc/self/mountinfo.""" + try: + with open("/proc/self/mountinfo") as f: + return f.readlines() + except FileNotFoundError: + logger.error("Cannot read /proc/self/mountinfo") + return [] + + +def is_nfs_mounted(mount_point): + """Check whether an NFS filesystem is mounted at mount_point.""" + for line in read_mountinfo(): + parts = line.split() + if len(parts) < 10: + continue + try: + sep = parts.index("-") + except ValueError: + continue + fstype = parts[sep + 1] + mp = parts[4] + if mp == mount_point and fstype in ("nfs", "nfs4"): + return True + return False + + +def check_mount_healthy(mount_name, mount_info): + """Check that the NFS mount exists, is accessible, and is not stale.""" + host_path = mount_info["host_path"] + if not is_nfs_mounted(host_path): + logger.warning("Mount %s (%s) not present", mount_name, host_path) + return False + try: + os.listdir(host_path) + except OSError as e: + if "Stale file handle" in str(e) or "ESTALE" in str(e): + logger.warning("Mount %s (%s) has ESTALE", mount_name, host_path) + else: + logger.warning("Mount %s (%s) unhealthy: %s", mount_name, host_path, e) + return False + logger.debug("Mount %s (%s) is healthy", mount_name, host_path) + return True + + +def umount_host(mount_point): + """Force-unmount a filesystem.""" + logger.info("Unmounting %s", mount_point) + try: + subprocess.run(["umount", "-f", mount_point], capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + logger.warning("umount failed: %s", e.stderr.strip()) + + +def mount_host(server, mount_point): + """Mount an NFS export. The mount propagates to the host via Bidirectional propagation.""" + logger.info("Mounting %s on %s", server, mount_point) + os.makedirs(mount_point, exist_ok=True) + try: + subprocess.run( + ["mount", "-t", "nfs", "-o", MOUNT_OPTIONS, f"{server}:", mount_point], + capture_output=True, text=True, check=True, + ) + except subprocess.CalledProcessError as e: + logger.warning("mount failed: %s", e.stderr.strip()) + + +def ensure_compat_symlinks(host_mnt_nfs): + """Create or repair compat symlinks inside the NFS mount root based on config.""" + config = load_config() + compat_symlinks = config.get("compatSymlinks", []) + for name in compat_symlinks: + link_path = os.path.join(host_mnt_nfs, name) + target = "dumps" + current_target = None + try: + current_target = os.readlink(link_path) + except OSError as e: + if e.errno == errno.ENOENT: + logger.info("Compat symlink %s missing — creating", link_path) + elif os.path.isdir(link_path): + try: + entries = os.listdir(link_path) + except OSError: + entries = [] + if entries: + logger.warning( + "Compat symlink %s is a non-empty directory — skipping", + link_path, + ) + continue + logger.info( + "Compat symlink %s is an empty directory — removing", + link_path, + ) + try: + os.rmdir(link_path) + except OSError: + logger.warning("Failed to remove empty directory %s", link_path) + else: + logger.warning("Cannot read symlink %s: %s", link_path, e) + continue + if current_target != target: + if current_target is not None: + logger.info( + "Compat symlink %s points to %s, updating to %s", + link_path, current_target, target, + ) + try: + os.unlink(link_path) + except OSError: + logger.warning("Failed to remove symlink %s", link_path) + try: + os.symlink(target, link_path) + except OSError as e: + logger.warning("Failed to create symlink %s -> %s: %s", link_path, target, e) + else: + logger.debug("Compat symlink %s -> %s ok", link_path, target) + + +def ensure_public_dumps_symlink(): + """Create /host/public/dumps/public -> /mnt/nfs/dumps so singleuser pods can reach dumps via /public/dumps.""" + dumps_dir = "/host/public/dumps" + public_path = os.path.join(dumps_dir, "public") + target = "/mnt/nfs/dumps" + try: + current = os.readlink(public_path) + if current == target: + logger.info("Symlink %s -> %s already exists", public_path, target) + return + logger.info("Symlink %s points to %s, updating", public_path, current) + os.unlink(public_path) + except OSError as e: + if e.errno == errno.ENOENT: + logger.info("Symlink %s missing — creating", public_path) + elif os.path.isdir(public_path): + logger.error( + "%s exists and is a directory — cannot create compat symlink", + public_path, + ) + return + else: + logger.info("Removing existing %s to create symlink", public_path) + try: + os.unlink(public_path) + except OSError: + logger.warning("Failed to remove %s", public_path) + return + os.makedirs(dumps_dir, exist_ok=True) + os.symlink(target, public_path) + + +def main(): + """Mount NFS dumps, create compat symlinks, then loop for health checks and remounts.""" + config = load_config() + logger.info("Config: %s", json.dumps(config, default=str)) + + server = config.get("server", DUMPS_SERVER) + mount_info = {"host_path": DUMPS_MOUNT} + + ensure_public_dumps_symlink() + ensure_compat_symlinks("/host/mnt/nfs") + + while True: + if not is_nfs_mounted(DUMPS_MOUNT): + logger.info("Mount dumps-src (%s) not present — mounting", DUMPS_MOUNT) + mount_host(server, DUMPS_MOUNT) + elif not check_mount_healthy("dumps-src", mount_info): + logger.info("Mount dumps-src (%s) unhealthy — remounting", DUMPS_MOUNT) + umount_host(DUMPS_MOUNT) + time.sleep(2) + mount_host(server, DUMPS_MOUNT) + time.sleep(60) + + +if __name__ == "__main__": + main() diff --git a/paws/files/maintain-dumps-nfs/maintain-dumps-nfs_test.py b/paws/files/maintain-dumps-nfs/maintain-dumps-nfs_test.py new file mode 100644 index 00000000..21e35df0 --- /dev/null +++ b/paws/files/maintain-dumps-nfs/maintain-dumps-nfs_test.py @@ -0,0 +1,182 @@ +import json +import os +import errno +import sys +import importlib.util +from unittest.mock import MagicMock, patch + +sys.modules["kubernetes"] = MagicMock() +sys.modules["kubernetes.client"] = MagicMock() +sys.modules["kubernetes.config"] = MagicMock() +sys.modules["kubernetes.watch"] = MagicMock() + +spec = importlib.util.spec_from_file_location( + "maintain_dumps_nfs", + os.path.join(os.path.dirname(__file__), "maintain-dumps-nfs.py"), +) +mdn = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mdn) + +SAMPLE_MOUNTINFO_NFS = ( + "1 2 3:4 / / rw,relatime shared:1 - ext4 /dev/root rw\n" + "5 6 7:8 /mnt/nfs/dumps /mnt/nfs/dumps rw,noatime shared:2 - nfs4 server:/ rw,soft,noatime\n" + "9 10 11:12 /public/dumps /public/dumps rw,relatime shared:3 - ext4 /dev/sda1 rw\n" +) + + +class TestIsNfsMounted: + def test_returns_true_when_nfs_mounted(self): + with patch.object(mdn, "read_mountinfo", return_value=SAMPLE_MOUNTINFO_NFS.splitlines(keepends=True)): + assert mdn.is_nfs_mounted("/mnt/nfs/dumps") is True + + +class TestCheckMountHealthy: + def test_returns_false_on_stale_handle(self): + def raiser(_): + raise OSError("ESTALE: Stale file handle") + + with ( + patch.object(mdn, "is_nfs_mounted", return_value=True), + patch.object(mdn.os, "listdir", side_effect=raiser), + ): + assert mdn.check_mount_healthy("test", {"host_path": "/mnt/nfs/dumps"}) is False + + def test_returns_false_on_ls_failure(self): + def raiser(_): + raise OSError("some error") + + with ( + patch.object(mdn, "is_nfs_mounted", return_value=True), + patch.object(mdn.os, "listdir", side_effect=raiser), + ): + assert mdn.check_mount_healthy("test", {"host_path": "/mnt/nfs/dumps"}) is False + + +class TestEnsureCompatSymlinks: + def test_creates_missing_symlink(self): + os_err = OSError() + os_err.errno = errno.ENOENT + with ( + patch.object(mdn, "load_config", return_value={"compatSymlinks": ["old-name"]}), + patch.object(mdn.os, "readlink", side_effect=os_err), + patch.object(mdn.os.path, "isdir"), + patch.object(mdn.os, "symlink") as mock_symlink, + ): + mdn.ensure_compat_symlinks("/host/mnt/nfs") + mock_symlink.assert_called_once_with("dumps", "/host/mnt/nfs/old-name") + + def test_updates_wrong_symlink(self): + with ( + patch.object(mdn, "load_config", return_value={"compatSymlinks": ["old-name"]}), + patch.object(mdn.os, "readlink", return_value="wrong-target"), + patch.object(mdn.os, "unlink") as mock_unlink, + patch.object(mdn.os, "symlink") as mock_symlink, + ): + mdn.ensure_compat_symlinks("/host/mnt/nfs") + mock_unlink.assert_called_once_with("/host/mnt/nfs/old-name") + mock_symlink.assert_called_once_with("dumps", "/host/mnt/nfs/old-name") + + def test_removes_empty_dir(self): + os_err = OSError() + os_err.errno = errno.EINVAL + with ( + patch.object(mdn, "load_config", return_value={"compatSymlinks": ["old-name"]}), + patch.object(mdn.os, "readlink", side_effect=os_err), + patch.object(mdn.os.path, "isdir", return_value=True), + patch.object(mdn.os, "listdir", return_value=[]), + patch.object(mdn.os, "rmdir") as mock_rmdir, + patch.object(mdn.os, "symlink") as mock_symlink, + ): + mdn.ensure_compat_symlinks("/host/mnt/nfs") + mock_rmdir.assert_called_once_with("/host/mnt/nfs/old-name") + mock_symlink.assert_called_once_with("dumps", "/host/mnt/nfs/old-name") + + def test_skips_non_empty_dir(self): + os_err = OSError() + os_err.errno = errno.EINVAL + with ( + patch.object(mdn, "load_config", return_value={"compatSymlinks": ["old-name"]}), + patch.object(mdn.os, "readlink", side_effect=os_err), + patch.object(mdn.os.path, "isdir", return_value=True), + patch.object(mdn.os, "listdir", return_value=["file.txt"]), + patch.object(mdn.os, "rmdir") as mock_rmdir, + patch.object(mdn.os, "symlink") as mock_symlink, + ): + mdn.ensure_compat_symlinks("/host/mnt/nfs") + mock_rmdir.assert_not_called() + mock_symlink.assert_not_called() + + +class TestEnsurePublicDumpsSymlink: + def test_skips_when_symlink_exists(self): + with ( + patch.object(mdn.os, "readlink", return_value="/mnt/nfs/dumps"), + patch.object(mdn.os, "symlink") as mock_symlink, + patch.object(mdn.os, "makedirs") as mock_mkdir, + ): + mdn.ensure_public_dumps_symlink() + mock_symlink.assert_not_called() + mock_mkdir.assert_not_called() + + def test_creates_when_missing(self): + os_err = OSError() + os_err.errno = errno.ENOENT + with ( + patch.object(mdn.os, "readlink", side_effect=os_err), + patch.object(mdn.os.path, "isdir"), + patch.object(mdn.os, "makedirs") as mock_mkdir, + patch.object(mdn.os, "symlink") as mock_symlink, + ): + mdn.ensure_public_dumps_symlink() + mock_mkdir.assert_called_once_with("/host/public/dumps", exist_ok=True) + mock_symlink.assert_called_once_with("/mnt/nfs/dumps", "/host/public/dumps/public") + + def test_errors_when_path_is_directory(self): + os_err = OSError() + os_err.errno = errno.EINVAL + with ( + patch.object(mdn.os, "readlink", side_effect=os_err), + patch.object(mdn.os.path, "isdir", return_value=True), + patch.object(mdn.os, "symlink") as mock_symlink, + patch.object(mdn.os, "makedirs") as mock_mkdir, + ): + mdn.ensure_public_dumps_symlink() + mock_symlink.assert_not_called() + mock_mkdir.assert_not_called() + + +class TestMountHost: + def test_creates_dir_and_mounts(self): + with ( + patch.object(mdn.os, "makedirs") as mock_mkdir, + patch.object(mdn.subprocess, "run") as mock_run, + ): + mdn.mount_host("nfs.example.com", "/mnt/nfs/test") + mock_mkdir.assert_called_once_with("/mnt/nfs/test", exist_ok=True) + mock_run.assert_called_once_with( + ["mount", "-t", "nfs", "-o", mdn.MOUNT_OPTIONS, "nfs.example.com:", "/mnt/nfs/test"], + capture_output=True, text=True, check=True, + ) + + +class TestReadMountinfo: + def test_returns_lines_when_file_exists(self): + with patch("builtins.open") as mock_open: + mock_open.return_value.__enter__.return_value.readlines.return_value = ["line1\n", "line2\n"] + result = mdn.read_mountinfo() + assert result == ["line1\n", "line2\n"] + + def test_returns_empty_when_file_missing(self): + with patch("builtins.open", side_effect=FileNotFoundError): + result = mdn.read_mountinfo() + assert result == [] + + +class TestLoadConfig: + def test_loads_existing_config(self, tmp_path): + config_dir = tmp_path / "etc" / "maintain-dumps-nfs" + config_dir.mkdir(parents=True) + config_file = config_dir / "maintain-dumps-nfs.json" + config_file.write_text(json.dumps({"enabled": True, "server": "test.example.com"})) + with patch.object(mdn, "CONFIG_PATH", str(config_dir)): + assert mdn.load_config() == {"enabled": True, "server": "test.example.com"} diff --git a/paws/production.yaml b/paws/production.yaml index f680f43b..a1a6c280 100644 --- a/paws/production.yaml +++ b/paws/production.yaml @@ -27,6 +27,16 @@ jupyterhub: HUB_DOMAIN: "hub-paws.wmcloud.org" # Check jupyterhub.ingress.hosts minesweeper: enabled: true +maintainDumpsNfs: + enabled: false + server: dumps-nfs.wikimedia.org + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist localdev: enabled: false pawspublic: diff --git a/paws/templates/maintain-dumps-nfs/configmap.yaml b/paws/templates/maintain-dumps-nfs/configmap.yaml new file mode 100644 index 00000000..22bc49a4 --- /dev/null +++ b/paws/templates/maintain-dumps-nfs/configmap.yaml @@ -0,0 +1,27 @@ +--- + +{{- /* configmap for maintain-dumps-nfs source files */}} +kind: ConfigMap +apiVersion: v1 +metadata: + name: maintain-dumps-nfs-src + labels: + app: maintain-dumps-nfs + component: maintain-dumps-nfs-src + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +data: +{{- (.Files.Glob "files/maintain-dumps-nfs/*").AsConfig | nindent 2 }} +{{- /* configmap for maintain-dumps-nfs configuration from values */}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: maintain-dumps-nfs-config + labels: + app: maintain-dumps-nfs + component: maintain-dumps-nfs-config + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: + "maintain-dumps-nfs.json": {{ toJson .Values.maintainDumpsNfs | quote }} diff --git a/paws/templates/maintain-dumps-nfs/daemonset.yaml b/paws/templates/maintain-dumps-nfs/daemonset.yaml new file mode 100644 index 00000000..da47e0a1 --- /dev/null +++ b/paws/templates/maintain-dumps-nfs/daemonset.yaml @@ -0,0 +1,89 @@ +--- + +{{- if .Values.maintainDumpsNfs.enabled -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: maintain-dumps-nfs +spec: + selector: + matchLabels: + name: maintain-dumps-nfs + release: {{ .Release.Name }} + template: + metadata: + labels: + name: maintain-dumps-nfs + app: binder + component: maintain-dumps-nfs + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/maintain-dumps-nfs/configmap.yaml") . | sha256sum }} + + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- with .Values.maintainDumpsNfs.affinity }} + affinity: +{{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + - effect: NoSchedule + key: hub.jupyter.org/dedicated + operator: Equal + value: user + - effect: NoSchedule + key: hub.jupyter.org_dedicated + operator: Equal + value: user + serviceAccountName: maintain-dumps-nfs + + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- . | toYaml | nindent 8 }} + {{- end }} + + containers: + - name: maintain-dumps-nfs + image: {{ tpl .Values.maintainDumpsNfs.image.template . | quote}} + securityContext: + privileged: true + runAsUser: 0 + runAsGroup: 0 + {{- with .Values.maintainDumpsNfs.resources }} + resources: {{ toJson . }} + {{- end }} + volumeMounts: + - name: host-nfs + mountPath: /host/mnt/nfs + mountPropagation: Bidirectional + - name: host-public-dumps + mountPath: /host/public/dumps + mountPropagation: Bidirectional + - name: config + mountPath: /etc/maintain-dumps-nfs + readOnly: true + - name: src + mountPath: /srv/maintain-dumps-nfs + readOnly: true + command: + - python3 + - /srv/maintain-dumps-nfs/maintain-dumps-nfs.py + terminationGracePeriodSeconds: 30 + volumes: + - name: host-nfs + hostPath: + path: /mnt/nfs + type: DirectoryOrCreate + - name: host-public-dumps + hostPath: + path: /public/dumps + type: DirectoryOrCreate + - name: src + configMap: + name: maintain-dumps-nfs-src + - name: config + configMap: + name: maintain-dumps-nfs-config +{{- end }} diff --git a/paws/templates/maintain-dumps-nfs/rbac.yaml b/paws/templates/maintain-dumps-nfs/rbac.yaml new file mode 100644 index 00000000..83935504 --- /dev/null +++ b/paws/templates/maintain-dumps-nfs/rbac.yaml @@ -0,0 +1,39 @@ +--- + +{{- if .Values.maintainDumpsNfs.enabled -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: maintain-dumps-nfs +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: maintain-dumps-nfs +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: maintain-dumps-nfs +roleRef: + kind: Role + name: maintain-dumps-nfs + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: maintain-dumps-nfs +{{- end }} diff --git a/paws/values.yaml b/paws/values.yaml index 7fc0076c..c911bf25 100644 --- a/paws/values.yaml +++ b/paws/values.yaml @@ -301,6 +301,20 @@ minesweeper: name: quay.io/wikimedia-paws-prod/minesweeper tag: pr-487 # minesweeper tag managed by github actions template: "{{ .Values.minesweeper.image.name }}:{{ .Values.minesweeper.image.tag }}" +maintainDumpsNfs: + enabled: false + image: + name: quay.io/wikimedia-paws-prod/maintain-dumps-nfs + tag: pr-1 # maintain-dumps-nfs tag managed by github actions + template: "{{ .Values.maintainDumpsNfs.image.name }}:{{ .Values.maintainDumpsNfs.image.tag }}" + server: dumps-nfs.wikimedia.org + compatSymlinks: + - dumps-clouddumps1001.wikimedia.org + - dumps-clouddumps1002.wikimedia.org + resources: + requests: + cpu: "100m" + memory: "128Mi" # If not deployed for prod use, we use the some hacks for testing localdev: enabled: true