diff --git a/deploy/ansible/codecarbon_cli_as_a_service/host_vars/yourservername.yourdomain.com.yml b/deploy/ansible/codecarbon_cli_as_a_service/host_vars/yourservername.yourdomain.com.yml new file mode 100644 index 000000000..2250cc746 --- /dev/null +++ b/deploy/ansible/codecarbon_cli_as_a_service/host_vars/yourservername.yourdomain.com.yml @@ -0,0 +1,4 @@ +--- +# One experiment per machine, so each one gets its own series on the dashboard. +# The file name must match the host exactly as written in the `hosts` inventory. +experiment_id: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx diff --git a/deploy/ansible/codecarbon_cli_as_a_service/tasks/install_codecarbon.yml b/deploy/ansible/codecarbon_cli_as_a_service/tasks/install_codecarbon.yml index 1b8cfa4b2..f05784c0e 100644 --- a/deploy/ansible/codecarbon_cli_as_a_service/tasks/install_codecarbon.yml +++ b/deploy/ansible/codecarbon_cli_as_a_service/tasks/install_codecarbon.yml @@ -7,6 +7,14 @@ become_method: sudo tasks: + - name: Check this host has its own dashboard experiment + assert: + that: experiment_id is defined + fail_msg: >- + No experiment_id for {{ inventory_hostname }}. Create the experiment on + the dashboard, then declare it in host_vars/{{ inventory_hostname }}.yml + rather than in vars/main.yml. + quiet: yes - name: Create CodeCarbon group group: name: "{{ codecarbon_group }}" @@ -31,17 +39,34 @@ apt: name: python3-venv state: present + # A venv hardcodes the Python minor version in lib/pythonX.Y/site-packages, + # but bin/python3 is only a symlink to /usr/bin/python3. After a distribution + # upgrade the interpreter no longer finds its own site-packages and even pip + # disappears. `creates:` would then skip the rebuild forever, so probe the + # venv instead of merely testing its presence. + - name: Check whether the existing virtual environment is still usable + command: "{{ codecarbon_venv }}/bin/python -c 'import pip'" + register: codecarbon_venv_check + changed_when: false + failed_when: false + - name: Remove the stale virtual environment + file: + path: "{{ codecarbon_venv }}" + state: absent + when: codecarbon_venv_check.rc != 0 + # The venv is built and owned by root: the service account only needs to read + # it, and it has no home directory for Ansible to write its temporary files. - name: Create Python virtual environment command: cmd: "python3 -m venv {{ codecarbon_venv }}" - creates: "{{ codecarbon_venv }}" - become_user: "{{ codecarbon_user }}" + creates: "{{ codecarbon_venv }}/bin/python" - name: Install CodeCarbon package pip: name: codecarbon + state: latest virtualenv: "{{ codecarbon_venv }}" - become_user: "{{ codecarbon_user }}" - name: Create CodeCarbon configuration file + register: codecarbon_config template: src: ../templates/codecarbon.config.j2 dest: "{{ codecarbon_home }}/.codecarbon.config" diff --git a/deploy/ansible/codecarbon_cli_as_a_service/tasks/rapl.yml b/deploy/ansible/codecarbon_cli_as_a_service/tasks/rapl.yml index 190738141..b708b8735 100644 --- a/deploy/ansible/codecarbon_cli_as_a_service/tasks/rapl.yml +++ b/deploy/ansible/codecarbon_cli_as_a_service/tasks/rapl.yml @@ -7,40 +7,55 @@ become_method: sudo tasks: - - name: Set RAPL directory permissions - shell: | - chmod -R g+r {{ rapl_base_path }}/* - chown -R root:{{ codecarbon_group }} {{ rapl_base_path }}/* - ignore_errors: yes # In case RAPL is not available - # This does not work because there is loop in folder symlink - # - name: Set RAPL directory permissions - # file: - # path: "{{ rapl_base_path }}" - # state: directory - # owner: root - # group: "{{ codecarbon_group }}" - # mode: "g+r" - # recurse: yes - # ignore_errors: yes # In case RAPL is not available - - name: Install sysfsutils - apt: - name: sysfsutils - state: present - - name: Find all RAPL energy_uj files + # A udev rule is used instead of a one-off chmod so the permissions survive + # reboots, and instead of sysfsutils so it does not depend on a Debian-only + # package. The rule fires once per powercap device, so every RAPL domain is + # covered without enumerating them. + # + # Only energy_uj is exposed, only to the CodeCarbon group, and read-only: + # energy counters are a power side channel (CVE-2020-8694, PLATYPUS). + - name: Install the udev rule granting RAPL access to the CodeCarbon group + copy: + dest: /etc/udev/rules.d/99-codecarbon-rapl.rules + owner: root + group: root + mode: "0644" + content: | + # Managed by Ansible - CodeCarbon + SUBSYSTEM=="powercap", ACTION=="add", TEST=="energy_uj", \ + RUN+="/bin/chgrp {{ codecarbon_group }} /sys%p/energy_uj", \ + RUN+="/bin/chmod 0440 /sys%p/energy_uj" + register: rapl_udev_rule + + - name: Apply the rule to the RAPL devices already present + command: "{{ item }}" + loop: + - udevadm control --reload-rules + - udevadm trigger --subsystem-match=powercap --action=add + when: rapl_udev_rule.changed + + - name: List the RAPL energy counters find: - paths: "{{ rapl_base_path }}" - patterns: - - "energy_uj" - - "name" - recurse: yes - register: rapl_files - - name: Configure sysfs for RAPL permissions - blockinfile: - path: /etc/sysfs.conf - create: yes - block: | - {% for file in rapl_files.files %} - mode {{ file.path | replace('/sys/','class/') }} = 0440 - owner {{ file.path | replace('/sys/','class/') }} = root:{{ codecarbon_group }} - {% endfor %} + paths: "{{ rapl_base_path }}" + patterns: energy_uj + recurse: yes + follow: no + register: rapl_counters + ignore_errors: yes # In case RAPL is not available + + # The check is done on the gid, not on gr_name: when another group shares the + # same gid, the name resolves to whichever entry comes first in /etc/group, + # which would report a failure even though access is actually granted. + - name: Look up the CodeCarbon group id + getent: + database: group + key: "{{ codecarbon_group }}" + - name: Report RAPL counters that are still unreadable by CodeCarbon + debug: + msg: >- + No RAPL counter readable by the {{ codecarbon_group }} group was found. + CodeCarbon will fall back to power estimation instead of measurement. + when: rapl_counters.files | default([]) + | selectattr('gid', 'equalto', ansible_facts.getent_group[codecarbon_group][1] | int) + | list | length == 0 diff --git a/deploy/ansible/codecarbon_cli_as_a_service/tasks/systemd_service.yml b/deploy/ansible/codecarbon_cli_as_a_service/tasks/systemd_service.yml index ada5cc34f..abb571b02 100644 --- a/deploy/ansible/codecarbon_cli_as_a_service/tasks/systemd_service.yml +++ b/deploy/ansible/codecarbon_cli_as_a_service/tasks/systemd_service.yml @@ -18,4 +18,13 @@ name: codecarbon enabled: yes state: started - daemon_reload: yes \ No newline at end of file + daemon_reload: yes + + # CodeCarbon only reads .codecarbon.config at startup, so a configuration + # change is a no-op until the service is restarted. The variable comes from + # the previous play, registered variables are kept per host across plays. + - name: Restart CodeCarbon to pick up the new configuration + systemd: + name: codecarbon + state: restarted + when: codecarbon_config.changed | default(false) \ No newline at end of file diff --git a/deploy/ansible/codecarbon_cli_as_a_service/vars/main.yml b/deploy/ansible/codecarbon_cli_as_a_service/vars/main.yml index 1685407d8..7280d9699 100644 --- a/deploy/ansible/codecarbon_cli_as_a_service/vars/main.yml +++ b/deploy/ansible/codecarbon_cli_as_a_service/vars/main.yml @@ -12,5 +12,8 @@ codecarbon_venv: "{{ codecarbon_home }}/venv" api_endpoint: https://api.codecarbon.io organization_id: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx project_id: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -experiment_id: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx api_key: xxx_xxxxxxxx_xxxxxxxxxxxx + +# experiment_id is per host: see host_vars/.yml. +# Do not define it here, vars_files takes precedence over host_vars and would +# silently send every machine to the same dashboard experiment. diff --git a/docs/how-to/ansible.md b/docs/how-to/ansible.md index c65a0ba4b..cdb5f4d84 100644 --- a/docs/how-to/ansible.md +++ b/docs/how-to/ansible.md @@ -29,6 +29,8 @@ The Ansible playbook automates the following tasks: ``` text codecarbon/deploy/ansible/codecarbon_cli_as_a_service/ ├── hosts +├── host_vars +│ └── yourservername.yourdomain.com.yml ├── tasks │ ├── install_codecarbon.yml │ ├── main.yml @@ -53,16 +55,52 @@ yourservername.yourdomain.com hostname=yourservername ansible_user=root ansibl ### Step 2: Update Ansible Variables -Update your CodeCarbon API credentials in `vars/main.yml`: +Update your CodeCarbon API credentials in `vars/main.yml`. These are shared by +every machine the playbook installs: ``` yaml organization_id: your_org_id project_id: your_project_id -experiment_id: your_experiment_id api_key: your_api_key ``` -### Step 3: Run the Playbook +### Step 3: Give Each Machine Its Own Experiment + +An experiment identifies a single monitored machine on the dashboard. If several +servers report under the same `experiment_id`, their measurements are mixed into +one series and you can no longer tell them apart, so create one experiment per +machine on the dashboard. + +Declare it in `host_vars/`, next to the `hosts` inventory. The file name must +match the host exactly as written in the inventory: + +``` yaml +# host_vars/yourservername.yourdomain.com.yml +experiment_id: your_experiment_id_for_this_server +``` + +Add one such file per machine. Ansible loads it automatically, no change to the +playbook is needed: + +``` text +host_vars +├── firstserver.yourdomain.com.yml +└── secondserver.yourdomain.com.yml +``` + +Do not put `experiment_id` back into `vars/main.yml`. That file is loaded with +`vars_files`, which takes precedence over `host_vars`, so a value left there +overrides every per-host file and silently sends all your machines to the same +experiment. The playbook stops with an explicit error if a host has no +`experiment_id` at all. + +You can check what each host resolves to before deploying: + +``` bash +ansible all -i hosts -m debug -a 'msg="{{ inventory_hostname }} -> {{ experiment_id }}"' +``` + +### Step 4: Run the Playbook Execute the Ansible playbook to deploy CodeCarbon: @@ -70,6 +108,11 @@ Execute the Ansible playbook to deploy CodeCarbon: ansible-playbook -i hosts tasks/main.yml ``` +The playbook is idempotent, so you can run it again to add a machine or to +update the configuration of the existing ones. Changing the configuration +restarts the service, because CodeCarbon only reads `.codecarbon.config` at +startup. + ## Next Steps - [Install CodeCarbon as a Linux Service](linux-service.md) for manual setup details diff --git a/docs/how-to/enable-rapl.md b/docs/how-to/enable-rapl.md index 8f4a76fe8..ce297380b 100644 --- a/docs/how-to/enable-rapl.md +++ b/docs/how-to/enable-rapl.md @@ -32,78 +32,138 @@ If the command returns directories (e.g., `intel-rapl:0`, `intel-rapl:1`), your ### Step 1: Understand the Security Issue -Due to [CVE-2020-8694](https://www.cve.org/CVERecord?id=CVE-2020-8694), Linux distributions restrict RAPL file permissions to root-only for security. This prevents unprivileged users from reading fine-grained power data. +Since [CVE-2020-8694](https://www.cve.org/CVERecord?id=CVE-2020-8694), the Linux kernel +restricts RAPL counters to root. The reason is the +[PLATYPUS attack](https://platypusattack.com/): power consumption is a side channel, and +`energy_uj` exposes it to software without needing an oscilloscope. + +**What an attacker actually needs.** The threat is a process running *on the same machine* +as the victim, under an account allowed to read the counters. Remote attackers, and local +accounts outside the group you grant, gain nothing. What such a process can do, in +increasing order of difficulty: + +| Capability | Requirements | +|---|---| +| Infer coarse activity: when a workload starts, roughly how busy it is, a covert channel between processes | Sampling the counter. Immediate. | +| Break KASLR (defeats a kernel exploit mitigation, does not by itself leak data) | ~20 seconds of sampling. | +| Recover a cryptographic key (AES-NI, RSA) | The victim must repeat the *same* operation with the *same* key tens of thousands of times while the attacker averages traces — 26 to 277 hours in the published results. The SGX variants also required privileged single-stepping (SGX-Step), not just counter access. | + +So the intuition that key recovery needs a victim doing the same encryption with the same +key, over and over, for a very long time is correct: it is not a realistic threat against a +general-purpose workload. The cheap and realistic gains are **activity inference** and +**KASLR defeat as one link in an exploit chain**. + +**How this guide limits the exposure:** + +- **Access goes to a dedicated group**, not to all users, so an untrusted local account + gains nothing. +- **Only `energy_uj` is exposed.** Power limits and other powercap attributes stay + root-only, and nothing becomes writable — no risk of an attacker throttling or + overheating the machine through this interface. + +Judge it accordingly: on a single-tenant server or a personal workstation, granting a +service account read access to an energy counter is a minor change. On a machine where +untrusted code runs under other local accounts — shared build servers, multi-tenant hosts, +CI runners executing third-party jobs — keep the group tight, and prefer running CodeCarbon +under its own service user rather than making the counters world-readable. ### Step 2: Temporary Access (Testing) -To quickly test RAPL without permanent changes: +To quickly check that RAPL works at all, without permanent changes: ```bash -sudo chmod -R a+r /sys/class/powercap/* +sudo chmod a+r /sys/class/powercap/*/energy_uj ``` -This grants read access to all users. However, **permissions are lost at next reboot**, so this is only for testing. +**Permissions are lost at next reboot**, so this is only a throwaway test — and it grants +access to every local user. Move on to step 3 for a real setup. ### Step 3: Permanent Access (Recommended) -For permanent access that survives reboots, use `sysfsutils`: +For permanent access that survives reboots, add a `udev` rule. The rule fires every time a +powercap device appears, so it covers all RAPL domains (`package`, `core`, `dram`, `psys`, +MMIO) without listing them one by one, and it works on any distribution using `systemd`. -**Step 3a: Install sysfsutils** +**Step 3a: Create a Dedicated Group** ```bash -sudo apt install sysfsutils +sudo groupadd codecarbon +sudo usermod -a -G codecarbon $USER ``` -**Step 3b: Configure RAPL Permissions** +Use the account that will run CodeCarbon — if it runs as a service, add that service user +instead of `$USER`. -Edit the sysfsutils configuration: +**Step 3b: Create the Rule** ```bash -sudo nano /etc/sysfs.conf +sudo tee /etc/udev/rules.d/99-codecarbon-rapl.rules <<'EOF' +SUBSYSTEM=="powercap", ACTION=="add", TEST=="energy_uj", \ + RUN+="/bin/chgrp codecarbon /sys%p/energy_uj", \ + RUN+="/bin/chmod 0440 /sys%p/energy_uj" +EOF ``` -Add this line at the end: +`%p` expands to the device path, so each RAPL domain is handled by its own event. +`TEST=="energy_uj"` skips devices that do not expose the counter. Only that one file +changes: mode `0440` means root and the `codecarbon` group can read it, nobody else, and +it stays read-only. -```text -mode class/powercap/intel-rapl:0/energy_uj = 0444 +**Step 3c: Apply It Without Rebooting** + +```bash +sudo udevadm control --reload-rules +sudo udevadm trigger --subsystem-match=powercap --action=add ``` -Save and exit (`Ctrl+X`, then `Y`, then `Enter`). +**Step 3d: Check the Result** -**Step 3c: Reboot to Apply Changes** +```bash +ls -l /sys/class/powercap/*/energy_uj +``` + +Each file should show `-r--r----- root codecarbon`. Log out and back in for your group +membership to take effect: ```bash -sudo reboot +logout +# Then log back in ``` -### Step 4: (Optional) More Restrictive Permissions +### Step 4: (Optional) Single-User Workstation -For better security, you can create a dedicated group instead of allowing all users: +On a personal machine where every account is yours, you may prefer to skip the group and +make the counters world-readable. Per step 1, this mainly means any local process can +observe your machine's activity and defeat KASLR: ```bash -# Create a codecarbon group -sudo groupadd codecarbon +sudo tee /etc/udev/rules.d/99-codecarbon-rapl.rules <<'EOF' +SUBSYSTEM=="powercap", ACTION=="add", TEST=="energy_uj", RUN+="/bin/chmod 0444 /sys%p/energy_uj" +EOF -# Add your user to the group -sudo usermod -a -G codecarbon $USER - -# Update sysfs.conf with group permissions -sudo nano /etc/sysfs.conf +sudo udevadm control --reload-rules +sudo udevadm trigger --subsystem-match=powercap --action=add ``` -Update the line to: +### Alternative: sysfsutils (Debian and Ubuntu Only) + +If you prefer a declarative configuration file, `sysfsutils` applies sysfs permissions at +boot through its own `systemd` unit. It is only packaged for Debian-based distributions, +and each RAPL domain must be listed explicitly: -```text +```bash +sudo apt install sysfsutils +sudo tee -a /etc/sysfs.conf <<'EOF' mode class/powercap/intel-rapl:0/energy_uj = 0440 owner class/powercap/intel-rapl:0/energy_uj = root:codecarbon +mode class/powercap/intel-rapl:0:0/energy_uj = 0440 +owner class/powercap/intel-rapl:0:0/energy_uj = root:codecarbon +EOF +sudo systemctl restart sysfsutils ``` -Log out and back in for group membership to take effect: - -```bash -logout -# Then log back in -``` +Add one `mode` and one `owner` line per domain reported by `ls /sys/class/powercap/`. +Missing a domain is easy here, which is why the `udev` rule is preferred. ### Step 5: Verify RAPL Access diff --git a/docs/how-to/linux-service.md b/docs/how-to/linux-service.md index 7d6e358ee..d5bfac1a9 100644 --- a/docs/how-to/linux-service.md +++ b/docs/how-to/linux-service.md @@ -81,15 +81,30 @@ EOF Give the CodeCarbon user permissions to read RAPL (Running Average Power Limit) energy information for accurate CPU power tracking: +A `udev` rule restores those permissions on every boot, for all RAPL domains. Only the +`energy_uj` counters are touched, and only the `codecarbon` group gains read access: + ``` bash -sudo chown -R root:codecarbon /sys/class/powercap/intel-rapl/* -sudo chmod g+r -R /sys/class/powercap/intel-rapl/* +sudo tee /etc/udev/rules.d/99-codecarbon-rapl.rules <<'EOF' +SUBSYSTEM=="powercap", ACTION=="add", TEST=="energy_uj", \ + RUN+="/bin/chgrp codecarbon /sys%p/energy_uj", \ + RUN+="/bin/chmod 0440 /sys%p/energy_uj" +EOF -sudo apt install sysfsutils -echo "mode class/powercap/intel-rapl:0/energy_uj = 0440" >> /etc/sysfs.conf -echo "owner class/powercap/intel-rapl:0/energy_uj = root:codecarbon" >> /etc/sysfs.conf +sudo udevadm control --reload-rules +sudo udevadm trigger --subsystem-match=powercap --action=add ``` +Check that the counters are now readable by the `codecarbon` group: + +``` bash +ls -l /sys/class/powercap/*/energy_uj +``` + +Energy counters are a side channel (see [Enable RAPL](enable-rapl.md) for the security +rationale and for the `sysfsutils` alternative). Keeping them restricted to the service +group means no other local account gains anything from this change. + ### Step 6: Create the CodeCarbon Configuration File Configure CodeCarbon with your dashboard credentials: