From 0eea792a35c8ce50ba6d9a7b985462ada8582ae0 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:44:49 +0200 Subject: [PATCH 01/29] feat: add telemetry module with Tier 1 payload builder --- codecarbon/telemetry.py | 21 +++++++++++++++++++++ tests/test_telemetry.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 codecarbon/telemetry.py create mode 100644 tests/test_telemetry.py diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py new file mode 100644 index 000000000..a25948aad --- /dev/null +++ b/codecarbon/telemetry.py @@ -0,0 +1,21 @@ +TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +TELEMETRY_API_URL = "https://api.codecarbon.io" +TELEMETRY_EXPERIMENT_ID = None # Set before enabling Tier 2 (see Task 4) + +_TIER1_SENT = False # module-level dedup: send once per Python session + +_TIER1_FIELDS = [ + "python_version", + "os", + "cpu_count", + "cpu_model", + "gpu_count", + "gpu_model", + "ram_total_size", + "codecarbon_version", + "tracking_mode", +] + + +def collect_tier1_payload(conf: dict) -> dict: + return {k: conf.get(k) for k in _TIER1_FIELDS} diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 000000000..6982ec8a7 --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,28 @@ +import sys +import platform +from codecarbon.telemetry import collect_tier1_payload + + +def test_collect_tier1_payload_has_required_fields(): + conf = { + "python_version": platform.python_version(), + "os": platform.platform(), + "cpu_count": 8, + "cpu_model": "Intel Core i7", + "gpu_count": 1, + "gpu_model": "NVIDIA RTX 3080", + "ram_total_size": 32.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "machine", + } + payload = collect_tier1_payload(conf) + assert "python_version" in payload + assert "os" in payload + assert "cpu_count" in payload + assert "cpu_model" in payload + assert "gpu_count" in payload + assert "gpu_model" in payload + assert "ram_total_size" in payload + assert "codecarbon_version" in payload + assert "tracking_mode" in payload + assert payload["python_version"] == conf["python_version"] From 0347f011a39031f7e730d2f08c7936f49f1dd7b3 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:47:22 +0200 Subject: [PATCH 02/29] fix: improve code quality for Task 1 (security, docs, types) - Move hardcoded TELEMETRY_API_KEY to environment variable with fallback - Add comment clarifying it's a public key with limited permissions - Add module docstring describing Tier 1 and Tier 2 telemetry - Improve type hints on collect_tier1_payload (dict[str, Any]) - Remove unused 'import sys' from test_telemetry.py - Convert test from function-based to class-based unittest.TestCase style - All tests passing --- codecarbon/telemetry.py | 19 +++++++++++++-- tests/test_telemetry.py | 53 ++++++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index a25948aad..60be883aa 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,4 +1,19 @@ -TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +""" +Telemetry module for CodeCarbon. + +This module handles two tiers of telemetry: +- Tier 1: Basic system information (always enabled) - python_version, os, cpu, gpu, ram, codecarbon_version, tracking_mode +- Tier 2: Detailed emissions data (opt-in) - requires CODECARBON_TELEMETRY_EXPERIMENT_ID environment variable +""" + +import os +from typing import Any + +# Public telemetry API key - limited permissions for metrics only +TELEMETRY_API_KEY = os.environ.get( + "CODECARBON_TELEMETRY_API_KEY", + "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +) TELEMETRY_API_URL = "https://api.codecarbon.io" TELEMETRY_EXPERIMENT_ID = None # Set before enabling Tier 2 (see Task 4) @@ -17,5 +32,5 @@ ] -def collect_tier1_payload(conf: dict) -> dict: +def collect_tier1_payload(conf: dict[str, Any]) -> dict[str, Any]: return {k: conf.get(k) for k in _TIER1_FIELDS} diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 6982ec8a7..aa6a28399 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,28 +1,33 @@ -import sys import platform +import unittest from codecarbon.telemetry import collect_tier1_payload -def test_collect_tier1_payload_has_required_fields(): - conf = { - "python_version": platform.python_version(), - "os": platform.platform(), - "cpu_count": 8, - "cpu_model": "Intel Core i7", - "gpu_count": 1, - "gpu_model": "NVIDIA RTX 3080", - "ram_total_size": 32.0, - "codecarbon_version": "2.0.0", - "tracking_mode": "machine", - } - payload = collect_tier1_payload(conf) - assert "python_version" in payload - assert "os" in payload - assert "cpu_count" in payload - assert "cpu_model" in payload - assert "gpu_count" in payload - assert "gpu_model" in payload - assert "ram_total_size" in payload - assert "codecarbon_version" in payload - assert "tracking_mode" in payload - assert payload["python_version"] == conf["python_version"] +class TestTelemetry(unittest.TestCase): + def test_collect_tier1_payload_has_required_fields(self): + conf = { + "python_version": platform.python_version(), + "os": platform.platform(), + "cpu_count": 8, + "cpu_model": "Intel Core i7", + "gpu_count": 1, + "gpu_model": "NVIDIA RTX 3080", + "ram_total_size": 32.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "machine", + } + payload = collect_tier1_payload(conf) + self.assertIn("python_version", payload) + self.assertIn("os", payload) + self.assertIn("cpu_count", payload) + self.assertIn("cpu_model", payload) + self.assertIn("gpu_count", payload) + self.assertIn("gpu_model", payload) + self.assertIn("ram_total_size", payload) + self.assertIn("codecarbon_version", payload) + self.assertIn("tracking_mode", payload) + self.assertEqual(payload["python_version"], conf["python_version"]) + + +if __name__ == "__main__": + unittest.main() From 0ae7783967da3c6839d49c6a62669b65cd15c1aa Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:49:45 +0200 Subject: [PATCH 03/29] feat: add Tier 1 telemetry send with session dedup and silent fail - Implement send_tier1_telemetry() function that POSTs environment metadata to the telemetry API once per session - Module-level _TIER1_SENT flag ensures deduplication (returns False on duplicate calls within same session) - Silent failure handling: catches all exceptions, logs error, returns False - Tests verify: deduplication prevents multiple POSTs, exceptions are caught and logged without being raised Co-Authored-By: Claude Haiku 4.5 --- codecarbon/telemetry.py | 37 ++++++++++++++++++++++++++++ tests/test_telemetry.py | 53 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 60be883aa..d3ca9a0e9 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -8,6 +8,9 @@ import os from typing import Any +import requests + +from codecarbon.external.logger import logger # Public telemetry API key - limited permissions for metrics only TELEMETRY_API_KEY = os.environ.get( @@ -34,3 +37,37 @@ def collect_tier1_payload(conf: dict[str, Any]) -> dict[str, Any]: return {k: conf.get(k) for k in _TIER1_FIELDS} + + +def send_tier1_telemetry(conf: dict[str, Any]) -> bool: + """Send Tier 1 telemetry metadata once per session. + + Posts environment metadata to the telemetry API endpoint. Uses module-level + deduplication flag to ensure data is sent only once per Python session. + Exceptions are caught and logged but not raised (silent fail). + + Args: + conf: Configuration dict with environment metadata (keys: python_version, + os, cpu_count, cpu_model, gpu_count, gpu_model, ram_total_size, + codecarbon_version, tracking_mode) + + Returns: + True if telemetry was sent successfully on this call, False if already + sent in this session or if an error occurred. + """ + global _TIER1_SENT + if _TIER1_SENT: + return False + try: + payload = collect_tier1_payload(conf) + requests.post( + f"{TELEMETRY_API_URL}/telemetry", + json=payload, + headers={"x-api-token": TELEMETRY_API_KEY}, + timeout=2, + ) + _TIER1_SENT = True + return True + except Exception as e: + logger.error(f"Telemetry Tier 1 failed (non-critical): {e}") + return False diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index aa6a28399..c53be0f2c 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,6 +1,8 @@ import platform import unittest -from codecarbon.telemetry import collect_tier1_payload +from unittest.mock import patch, MagicMock +import codecarbon.telemetry as telemetry_module +from codecarbon.telemetry import collect_tier1_payload, send_tier1_telemetry class TestTelemetry(unittest.TestCase): @@ -29,5 +31,54 @@ def test_collect_tier1_payload_has_required_fields(self): self.assertEqual(payload["python_version"], conf["python_version"]) +class TestTier1Send(unittest.TestCase): + def test_send_tier1_telemetry_sends_once_per_session(self): + """Verify deduplication: second call doesn't POST again.""" + telemetry_module._TIER1_SENT = False # reset between test runs + conf = { + "python_version": "3.11.0", + "os": "Linux", + "cpu_count": 4, + "cpu_model": "Intel i5", + "gpu_count": 0, + "gpu_model": None, + "ram_total_size": 16.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "process", + } + with patch("codecarbon.telemetry.requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=201) + result = send_tier1_telemetry(conf) + self.assertTrue(result) + assert mock_post.call_count == 1 + # second call is deduplicated + result = send_tier1_telemetry(conf) + self.assertFalse(result) + assert mock_post.call_count == 1 + + def test_send_tier1_telemetry_fails_silently(self): + """Verify exceptions are caught and logged, not raised.""" + telemetry_module._TIER1_SENT = False + conf = { + "python_version": "3.11.0", + "os": "Linux", + "cpu_count": 4, + "cpu_model": None, + "gpu_count": 0, + "gpu_model": None, + "ram_total_size": 8.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "process", + } + with patch("codecarbon.telemetry.requests.post", side_effect=Exception("network error")): + with patch("codecarbon.telemetry.logger.error") as mock_logger: + # must not raise + result = send_tier1_telemetry(conf) + self.assertFalse(result) + # verify error was logged + mock_logger.assert_called_once() + assert "network error" in str(mock_logger.call_args) + + if __name__ == "__main__": unittest.main() From ed2d3d2430eb7e371be28d9a5cae221ab9aed47d Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:59:22 +0200 Subject: [PATCH 04/29] feat: wire Tier 1 telemetry into BaseEmissionsTracker (opt-out via send_telemetry=False) --- codecarbon/emissions_tracker.py | 12 +++- tests/test_emissions_tracker.py | 17 +++-- tests/test_offline_emissions_tracker.py | 4 +- tests/test_telemetry.py | 88 +++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 9 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 862eba2b4..406c46bec 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -199,6 +199,7 @@ def __init__( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + send_telemetry: Optional[bool] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name @@ -352,6 +353,8 @@ def __init__( self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) + # Tier 1 telemetry (opt-out, default enabled) + self._set_from_conf(send_telemetry, "send_telemetry", default=True, return_type=bool) assert self._tracking_mode in ["machine", "process"] set_logger_level(self._log_level) @@ -448,6 +451,12 @@ def __init__( self._emissions: Emissions = Emissions( self._data_source, self._electricitymaps_api_token ) + + # Send Tier 1 telemetry if enabled + if self._send_telemetry: + from codecarbon.telemetry import send_tier1_telemetry + send_tier1_telemetry(self._conf) + self._init_output_methods(api_key=self._api_key) def _init_output_methods(self, *, api_key: str = None): @@ -1136,6 +1145,7 @@ def __init__( cloud_provider: Optional[str] = _sentinel, cloud_region: Optional[str] = _sentinel, country_2letter_iso_code: Optional[str] = _sentinel, + send_telemetry: Optional[bool] = _sentinel, **kwargs, ): """ @@ -1208,7 +1218,7 @@ def __init__( assert isinstance(self._country_2letter_iso_code, str) self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper() - super().__init__(*args, **kwargs) + super().__init__(*args, send_telemetry=send_telemetry, **kwargs) def _get_geo_metadata(self) -> GeoMetadata: return GeoMetadata( diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index ab4a0a275..b845962dd 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -99,7 +99,7 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( json=GEO_METADATA_CANADA, status=200, ) - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) # WHEN tracker.start() heavy_computation(run_time_secs=5) @@ -126,7 +126,7 @@ def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) mock_gpu = mock.MagicMock() from codecarbon.external.hardware import GPU @@ -164,7 +164,7 @@ def raise_timeout_exception(*args, **kwargs): mocked_requests_get.side_effect = raise_timeout_exception - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) # WHEN tracker.start() @@ -183,7 +183,7 @@ def test_graceful_start_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) def raise_exception(*args, **kwargs): raise Exception() @@ -202,7 +202,7 @@ def test_graceful_stop_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) def raise_exception(*args, **kwargs): raise Exception() @@ -346,6 +346,7 @@ def test_offline_tracker_country_name( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", + send_telemetry=False, ) tracker.start() heavy_computation(run_time_secs=2) @@ -369,6 +370,7 @@ def test_offline_tracker_invalid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", + send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_invalid_headers.csv" @@ -402,6 +404,7 @@ def test_offline_tracker_valid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", + send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_valid_headers.csv" @@ -445,7 +448,7 @@ def test_carbon_tracker_online_context_manager_TWO_GPU_PRIVATE_INFRA_CANADA( ) # WHEN - with EmissionsTracker(measure_power_secs=1, save_to_file=False) as tracker: + with EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) as tracker: heavy_computation(run_time_secs=5) # THEN @@ -608,6 +611,7 @@ def test_scheduler_warning_suppressed_when_stopped( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing + send_telemetry=False, ) as tracker: # Stop the scheduler to simulate task mode or manual stopping tracker._scheduler.stop() @@ -651,6 +655,7 @@ def test_scheduler_warning_shown_when_running( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing + send_telemetry=False, ) as tracker: # Ensure scheduler is running (default state) self.assertFalse(tracker._scheduler._stopped) diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 07adf403c..7e2ff40ac 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -41,7 +41,7 @@ def tearDown(self) -> None: self.temp_dir.cleanup() def test_offline_tracker(self): - tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path) + tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path, send_telemetry=False) tracker.start() heavy_computation(run_time_secs=2) tracker.stop() @@ -60,7 +60,7 @@ def test_offline_tracker(self): ) def test_offline_tracker_task(self): - tracker = OfflineEmissionsTracker() + tracker = OfflineEmissionsTracker(send_telemetry=False) tracker.start_task() heavy_computation(run_time_secs=2) task_emission_data = tracker.stop_task() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index c53be0f2c..ea6929dc7 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,8 +1,27 @@ +import os import platform +import shutil +import sys +import tempfile import unittest +from pathlib import Path from unittest.mock import patch, MagicMock import codecarbon.telemetry as telemetry_module from codecarbon.telemetry import collect_tier1_payload, send_tier1_telemetry +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from tests.testutils import get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch( + "codecarbon.core.cpu.IntelPowerGadget._setup_cli" + ) + + +empty_conf = "[codecarbon]" class TestTelemetry(unittest.TestCase): @@ -80,5 +99,74 @@ def test_send_tier1_telemetry_fails_silently(self): assert "network error" in str(mock_logger.call_args) +@mock_platform_cli_setup +class TestTrackerTelemetry(unittest.TestCase): + """Test that trackers wire Tier 1 telemetry into initialization.""" + + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.temp_path = Path(self.temp_dir.name) + self.patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + ) + self.mock_open = self.patcher.start() + + def tearDown(self) -> None: + self.patcher.stop() + self.temp_dir.cleanup() + + def test_emissions_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + """Tier 1 fires on EmissionsTracker initialization when send_telemetry=True (default).""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=201) + # Block geo lookup to isolate test + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + send_telemetry=True, save_to_api=False, save_to_file=False + ) + # Verify telemetry POST was called + assert mock_post.called, "Telemetry POST should have been called" + + def test_emissions_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + """Tier 1 does NOT fire when send_telemetry=False.""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + # Block geo lookup + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + send_telemetry=False, save_to_api=False, save_to_file=False + ) + # Verify telemetry POST was NOT called + assert not mock_post.called, "Telemetry POST should not have been called" + + def test_offline_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + """Tier 1 fires on OfflineEmissionsTracker initialization when send_telemetry=True (default).""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=201) + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + send_telemetry=True, + save_to_api=False, + save_to_file=False, + ) + # Verify telemetry POST was called + assert mock_post.called, "Telemetry POST should have been called" + + def test_offline_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + """Tier 1 does NOT fire when send_telemetry=False on OfflineEmissionsTracker.""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + send_telemetry=False, + save_to_api=False, + save_to_file=False, + ) + # Verify telemetry POST was NOT called + assert not mock_post.called, "Telemetry POST should not have been called" + + if __name__ == "__main__": unittest.main() From 5c568e7e9e17bdfc268796d62f0fea07373c4aa4 Mon Sep 17 00:00:00 2001 From: inimaz <93inigo93@gmail.com> Date: Sun, 3 May 2026 14:35:14 +0200 Subject: [PATCH 05/29] feat: first version of telemetry --- .../carbonserver/api/domain/telemetry.py | 10 + .../infra/database/telemetry_sql_models.py | 104 ++++++++++ .../repositories/repository_telemetry.py | 28 +++ .../carbonserver/api/routers/telemetry.py | 31 +++ .../carbonserver/api/schemas_telemetry.py | 178 ++++++++++++++++++ .../api/services/telemetry_service.py | 16 ++ carbonserver/carbonserver/container.py | 12 ++ carbonserver/main.py | 6 +- .../tests/api/routers/test_telemetry.py | 80 ++++++++ .../api/service/test_telemetry_service.py | 29 +++ codecarbon/core/telemetry_client.py | 61 ++++++ codecarbon/core/telemetry_schemas.py | 156 +++++++++++++++ tests/test_telemetry_client.py | 104 ++++++++++ 13 files changed, 814 insertions(+), 1 deletion(-) create mode 100644 carbonserver/carbonserver/api/domain/telemetry.py create mode 100644 carbonserver/carbonserver/api/infra/database/telemetry_sql_models.py create mode 100644 carbonserver/carbonserver/api/infra/repositories/repository_telemetry.py create mode 100644 carbonserver/carbonserver/api/routers/telemetry.py create mode 100644 carbonserver/carbonserver/api/schemas_telemetry.py create mode 100644 carbonserver/carbonserver/api/services/telemetry_service.py create mode 100644 carbonserver/tests/api/routers/test_telemetry.py create mode 100644 carbonserver/tests/api/service/test_telemetry_service.py create mode 100644 codecarbon/core/telemetry_client.py create mode 100644 codecarbon/core/telemetry_schemas.py create mode 100644 tests/test_telemetry_client.py diff --git a/carbonserver/carbonserver/api/domain/telemetry.py b/carbonserver/carbonserver/api/domain/telemetry.py new file mode 100644 index 000000000..c69201e93 --- /dev/null +++ b/carbonserver/carbonserver/api/domain/telemetry.py @@ -0,0 +1,10 @@ +import abc +from uuid import UUID + +from carbonserver.api import schemas_telemetry + + +class Telemetry(abc.ABC): + @abc.abstractmethod + def add_telemetry(self, telemetry: schemas_telemetry.TelemetryCreate) -> UUID: + raise NotImplementedError diff --git a/carbonserver/carbonserver/api/infra/database/telemetry_sql_models.py b/carbonserver/carbonserver/api/infra/database/telemetry_sql_models.py new file mode 100644 index 000000000..243ae79ee --- /dev/null +++ b/carbonserver/carbonserver/api/infra/database/telemetry_sql_models.py @@ -0,0 +1,104 @@ +"""SQLAlchemy models for telemetry data in the CarbonServer API.""" + +import uuid + +from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String +from sqlalchemy.dialects.postgresql import UUID + +from carbonserver.database.database import Base + + +class Telemetry(Base): + __tablename__ = "telemetry" + + id = Column(UUID(as_uuid=True), primary_key=True, index=True, default=uuid.uuid4) + timestamp = Column(DateTime, nullable=False) + telemetry_level = Column(String, nullable=False) + + os = Column(String, nullable=True) + country_name = Column(String, nullable=True) + country_iso_code = Column(String, nullable=True) + region = Column(String, nullable=True) + cloud_provider = Column(String, nullable=True) + cloud_region = Column(String, nullable=True) + longitude = Column(Float, nullable=True) + latitude = Column(Float, nullable=True) + + cpu_count = Column(Integer, nullable=True) + cpu_physical_count = Column(Integer, nullable=True) + cpu_model = Column(String, nullable=True) + cpu_architecture = Column(String, nullable=True) + gpu_count = Column(Integer, nullable=True) + gpu_model = Column(String, nullable=True) + gpu_driver_version = Column(String, nullable=True) + gpu_memory_total_gb = Column(Float, nullable=True) + ram_total_size_gb = Column(Float, nullable=True) + cuda_version = Column(String, nullable=True) + cudnn_version = Column(String, nullable=True) + + python_version = Column(String, nullable=True) + python_implementation = Column(String, nullable=True) + python_executable_hash = Column(String, nullable=True) + python_env_type = Column(String, nullable=True) + codecarbon_version = Column(String, nullable=True) + codecarbon_install_method = Column(String, nullable=True) + + total_emissions_kg = Column(Float, nullable=True) + emissions_rate_kg_per_sec = Column(Float, nullable=True) + energy_consumed_kwh = Column(Float, nullable=True) + cpu_energy_kwh = Column(Float, nullable=True) + gpu_energy_kwh = Column(Float, nullable=True) + ram_energy_kwh = Column(Float, nullable=True) + duration_seconds = Column(Float, nullable=True) + cpu_utilization_avg = Column(Float, nullable=True) + gpu_utilization_avg = Column(Float, nullable=True) + ram_utilization_avg = Column(Float, nullable=True) + + tracking_mode = Column(String, nullable=True) + api_mode = Column(String, nullable=True) + output_methods = Column(JSON, nullable=True) + hardware_tracked = Column(JSON, nullable=True) + task_tracking_used = Column(Boolean, nullable=True) + decorator_vs_context = Column(String, nullable=True) + measure_power_interval_secs = Column(Float, nullable=True) + + hardware_detection_success = Column(Boolean, nullable=True) + rapl_available = Column(Boolean, nullable=True) + gpu_detection_method = Column(String, nullable=True) + first_measurement_time_ms = Column(Float, nullable=True) + tracking_overhead_percent = Column(Float, nullable=True) + errors_encountered = Column(JSON, nullable=True) + warning_count = Column(Integer, nullable=True) + + ide_used = Column(String, nullable=True) + notebook_environment = Column(String, nullable=True) + ci_environment = Column(String, nullable=True) + python_package_manager = Column(String, nullable=True) + framework_detected = Column(String, nullable=True) + + has_torch = Column(Boolean, nullable=True) + torch_version = Column(String, nullable=True) + has_transformers = Column(Boolean, nullable=True) + transformers_version = Column(String, nullable=True) + has_diffusers = Column(Boolean, nullable=True) + diffusers_version = Column(String, nullable=True) + has_tensorflow = Column(Boolean, nullable=True) + tensorflow_version = Column(String, nullable=True) + has_keras = Column(Boolean, nullable=True) + keras_version = Column(String, nullable=True) + has_pytorch_lightning = Column(Boolean, nullable=True) + pytorch_lightning_version = Column(String, nullable=True) + has_fastai = Column(Boolean, nullable=True) + fastai_version = Column(String, nullable=True) + ml_framework_primary = Column(String, nullable=True) + + container_runtime = Column(String, nullable=True) + in_container = Column(Boolean, nullable=True) + host_machine_hash = Column(String, nullable=True) + + def __repr__(self): + return ( + f'' + ) diff --git a/carbonserver/carbonserver/api/infra/repositories/repository_telemetry.py b/carbonserver/carbonserver/api/infra/repositories/repository_telemetry.py new file mode 100644 index 000000000..b530cbdc8 --- /dev/null +++ b/carbonserver/carbonserver/api/infra/repositories/repository_telemetry.py @@ -0,0 +1,28 @@ +"""Repository implementation for telemetry data using SQLAlchemy.""" + +import uuid +from contextlib import AbstractContextManager +from uuid import UUID + +from dependency_injector.providers import Callable + +from carbonserver.api.domain.telemetry import Telemetry +from carbonserver.api.infra.database.telemetry_sql_models import ( + Telemetry as SqlModelTelemetry, +) +from carbonserver.api.schemas_telemetry import TelemetryCreate + + +class SqlAlchemyRepository(Telemetry): + def __init__(self, session_factory) -> Callable[..., AbstractContextManager]: + self.session_factory = session_factory + + def add_telemetry(self, telemetry: TelemetryCreate) -> UUID: + with self.session_factory() as session: + db_telemetry = SqlModelTelemetry( + id=uuid.uuid4(), + **telemetry.model_dump(), + ) + session.add(db_telemetry) + session.commit() + return db_telemetry.id diff --git a/carbonserver/carbonserver/api/routers/telemetry.py b/carbonserver/carbonserver/api/routers/telemetry.py new file mode 100644 index 000000000..a2e4da796 --- /dev/null +++ b/carbonserver/carbonserver/api/routers/telemetry.py @@ -0,0 +1,31 @@ +"""API router for handling telemetry data in the CarbonServer API.""" + +from uuid import UUID + +from dependency_injector.wiring import Provide, inject +from fastapi import APIRouter, Depends +from starlette import status + +from carbonserver.api.schemas_telemetry import TelemetryCreate +from carbonserver.api.services.telemetry_service import TelemetryService +from carbonserver.container import ServerContainer + +TELEMETRY_ROUTER_TAGS = ["Telemetry"] + +router = APIRouter() + + +@router.post( + "/telemetry", + tags=TELEMETRY_ROUTER_TAGS, + status_code=status.HTTP_201_CREATED, + response_model=UUID, +) +@inject +def add_telemetry( + telemetry: TelemetryCreate, + telemetry_service: TelemetryService = Depends( + Provide[ServerContainer.telemetry_service] + ), +) -> UUID: + return telemetry_service.add_telemetry(telemetry) diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py new file mode 100644 index 000000000..5517c2ff4 --- /dev/null +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -0,0 +1,178 @@ +"""Schemas for telemetry data submitted to the CarbonServer API.""" + +from datetime import datetime +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class TelemetryLevel(str, Enum): + disabled = "disabled" + minimal = "minimal" + extensive = "extensive" + + +class TelemetryBase(BaseModel): + model_config = ConfigDict( + extra="forbid", + use_enum_values=True, + json_schema_extra={ + "example": { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + "os": "Linux-5.10.0-x86_64", + "country_name": "France", + "country_iso_code": "FRA", + "cpu_count": 12, + "cpu_model": "Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz", + "python_version": "3.11.5", + "codecarbon_version": "3.0.0", + } + }, + ) + + timestamp: datetime + telemetry_level: TelemetryLevel + + os: Optional[str] = None + country_name: Optional[str] = None + country_iso_code: Optional[str] = Field(default=None, min_length=2, max_length=3) + region: Optional[str] = None + cloud_provider: Optional[str] = None + cloud_region: Optional[str] = None + longitude: Optional[float] = Field(default=None, ge=-180, le=180) + latitude: Optional[float] = Field(default=None, ge=-90, le=90) + + cpu_count: Optional[int] = Field(default=None, ge=0) + cpu_physical_count: Optional[int] = Field(default=None, ge=0) + cpu_model: Optional[str] = None + cpu_architecture: Optional[str] = None + gpu_count: Optional[int] = Field(default=None, ge=0) + gpu_model: Optional[str] = None + gpu_driver_version: Optional[str] = None + gpu_memory_total_gb: Optional[float] = Field(default=None, ge=0) + ram_total_size_gb: Optional[float] = Field(default=None, ge=0) + cuda_version: Optional[str] = None + cudnn_version: Optional[str] = None + + python_version: Optional[str] = None + python_implementation: Optional[str] = None + python_executable_hash: Optional[str] = Field( + default=None, min_length=64, max_length=64 + ) + python_env_type: Optional[str] = None + codecarbon_version: Optional[str] = None + codecarbon_install_method: Optional[str] = None + + total_emissions_kg: Optional[float] = Field(default=None, ge=0) + emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) + energy_consumed_kwh: Optional[float] = Field(default=None, ge=0) + cpu_energy_kwh: Optional[float] = Field(default=None, ge=0) + gpu_energy_kwh: Optional[float] = Field(default=None, ge=0) + ram_energy_kwh: Optional[float] = Field(default=None, ge=0) + duration_seconds: Optional[float] = Field(default=None, ge=0) + cpu_utilization_avg: Optional[float] = Field(default=None, ge=0, le=100) + gpu_utilization_avg: Optional[float] = Field(default=None, ge=0, le=100) + ram_utilization_avg: Optional[float] = Field(default=None, ge=0, le=100) + + tracking_mode: Optional[str] = None + api_mode: Optional[str] = None + output_methods: Optional[List[str]] = None + hardware_tracked: Optional[List[str]] = None + task_tracking_used: Optional[bool] = None + decorator_vs_context: Optional[str] = None + measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + + hardware_detection_success: Optional[bool] = None + rapl_available: Optional[bool] = None + gpu_detection_method: Optional[str] = None + first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) + tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) + errors_encountered: Optional[List[str]] = None + warning_count: Optional[int] = Field(default=None, ge=0) + + ide_used: Optional[str] = None + notebook_environment: Optional[str] = None + ci_environment: Optional[str] = None + python_package_manager: Optional[str] = None + framework_detected: Optional[str] = None + + has_torch: Optional[bool] = None + torch_version: Optional[str] = None + has_transformers: Optional[bool] = None + transformers_version: Optional[str] = None + has_diffusers: Optional[bool] = None + diffusers_version: Optional[str] = None + has_tensorflow: Optional[bool] = None + tensorflow_version: Optional[str] = None + has_keras: Optional[bool] = None + keras_version: Optional[str] = None + has_pytorch_lightning: Optional[bool] = None + pytorch_lightning_version: Optional[str] = None + has_fastai: Optional[bool] = None + fastai_version: Optional[str] = None + ml_framework_primary: Optional[str] = None + + container_runtime: Optional[str] = None + in_container: Optional[bool] = None + host_machine_hash: Optional[str] = None + + @model_validator(mode="after") + def validate_telemetry_level(self): + if self.telemetry_level == TelemetryLevel.disabled: + raise ValueError("Disabled telemetry must not be submitted") + + if self.telemetry_level == TelemetryLevel.minimal: + extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS + submitted_extensive_fields = [ + field + for field in extensive_fields + if getattr(self, field) not in (None, [], {}) + ] + if submitted_extensive_fields: + fields = ", ".join(sorted(submitted_extensive_fields)) + raise ValueError( + f"Minimal telemetry cannot include extensive fields: {fields}" + ) + + return self + + +MINIMAL_TELEMETRY_FIELDS = { + "timestamp", + "telemetry_level", + "os", + "country_name", + "country_iso_code", + "region", + "cloud_provider", + "cloud_region", + "longitude", + "latitude", + "cpu_count", + "cpu_physical_count", + "cpu_model", + "cpu_architecture", + "gpu_count", + "gpu_model", + "gpu_driver_version", + "gpu_memory_total_gb", + "ram_total_size_gb", + "cuda_version", + "cudnn_version", + "python_version", + "python_implementation", + "python_executable_hash", + "python_env_type", + "codecarbon_version", + "codecarbon_install_method", +} + + +class TelemetryCreate(TelemetryBase): + pass + + +class Telemetry(TelemetryBase): + id: str diff --git a/carbonserver/carbonserver/api/services/telemetry_service.py b/carbonserver/carbonserver/api/services/telemetry_service.py new file mode 100644 index 000000000..8e0161e83 --- /dev/null +++ b/carbonserver/carbonserver/api/services/telemetry_service.py @@ -0,0 +1,16 @@ +"""Service layer for handling telemetry data in the CarbonServer API.""" + +from uuid import UUID + +from carbonserver.api.infra.repositories.repository_telemetry import ( + SqlAlchemyRepository as TelemetrySqlRepository, +) +from carbonserver.api.schemas_telemetry import TelemetryCreate + + +class TelemetryService: + def __init__(self, telemetry_repository: TelemetrySqlRepository): + self._repository = telemetry_repository + + def add_telemetry(self, telemetry: TelemetryCreate) -> UUID: + return self._repository.add_telemetry(telemetry) diff --git a/carbonserver/carbonserver/container.py b/carbonserver/carbonserver/container.py index b730fb706..c09711495 100644 --- a/carbonserver/carbonserver/container.py +++ b/carbonserver/carbonserver/container.py @@ -8,6 +8,7 @@ repository_projects, repository_projects_tokens, repository_runs, + repository_telemetry, repository_users, ) from carbonserver.api.services.auth_context import AuthContext @@ -21,6 +22,7 @@ from carbonserver.api.services.project_token_service import ProjectTokenService from carbonserver.api.services.run_service import RunService from carbonserver.api.services.signup_service import SignUpService +from carbonserver.api.services.telemetry_service import TelemetryService from carbonserver.api.services.user_service import UserService from carbonserver.api.usecases.experiment.project_sum_by_experiment import ( ProjectSumsByExperimentUsecase, @@ -63,6 +65,11 @@ class ServerContainer(containers.DeclarativeContainer): session_factory=db.provided.session, ) + telemetry_repository = providers.Factory( + repository_telemetry.SqlAlchemyRepository, + session_factory=db.provided.session, + ) + experiment_repository = providers.Factory( repository_experiments.SqlAlchemyRepository, session_factory=db.provided.session, @@ -99,6 +106,11 @@ class ServerContainer(containers.DeclarativeContainer): emission_repository=emission_repository, ) + telemetry_service = providers.Factory( + TelemetryService, + telemetry_repository=telemetry_repository, + ) + experiment_service = providers.Factory( ExperimentService, auth_context=auth_context, diff --git a/carbonserver/main.py b/carbonserver/main.py index 8eb366059..b02119ecd 100644 --- a/carbonserver/main.py +++ b/carbonserver/main.py @@ -9,7 +9,7 @@ from starlette.responses import JSONResponse from carbonserver.api.errors import DBException, UserException, get_http_exception -from carbonserver.api.infra.database import sql_models +from carbonserver.api.infra.database import sql_models, telemetry_sql_models from carbonserver.api.routers import ( authenticate, emissions, @@ -18,6 +18,7 @@ project_api_tokens, projects, runs, + telemetry, users, ) from carbonserver.api.services import auth_service @@ -70,6 +71,7 @@ def init_container(): project_api_tokens, organizations, users, + telemetry, authenticate, auth_service, ] @@ -81,6 +83,7 @@ def init_db(container): db = container.db() db.create_database() sql_models.Base.metadata.create_all(bind=engine) + telemetry_sql_models.Base.metadata.create_all(bind=engine) def init_server(container): @@ -102,6 +105,7 @@ def init_server(container): server.include_router(experiments.router) server.include_router(runs.router) server.include_router(emissions.router) + server.include_router(telemetry.router) add_pagination(server) # Add CORS from env variable diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py new file mode 100644 index 000000000..a405b59e8 --- /dev/null +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -0,0 +1,80 @@ +from unittest import mock +from uuid import UUID + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from starlette import status + +from carbonserver.api.infra.repositories.repository_telemetry import ( + SqlAlchemyRepository as TelemetryRepository, +) +from carbonserver.api.routers import telemetry +from carbonserver.container import ServerContainer + +TELEMETRY_ID = "f52fe339-164d-4c2b-a8c0-f562dfce066d" + +MINIMAL_TELEMETRY_TO_CREATE = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + "os": "Linux-5.10.0-x86_64", + "country_name": "France", + "country_iso_code": "FRA", + "cpu_count": 12, + "python_version": "3.11.5", + "codecarbon_version": "3.2.6", +} + + +@pytest.fixture +def custom_test_server(): + container = ServerContainer() + container.wire(modules=[telemetry]) + app = FastAPI() + app.container = container + app.include_router(telemetry.router) + yield app + + +@pytest.fixture +def client(custom_test_server): + yield TestClient(custom_test_server) + + +def test_add_telemetry(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) + + with custom_test_server.container.telemetry_repository.override(repository_mock): + response = client.post("/telemetry", json=MINIMAL_TELEMETRY_TO_CREATE) + + assert response.status_code == status.HTTP_201_CREATED + assert response.json() == TELEMETRY_ID + + +def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + telemetry_with_extensive_field = { + **MINIMAL_TELEMETRY_TO_CREATE, + "total_emissions_kg": 0.42, + } + + with custom_test_server.container.telemetry_repository.override(repository_mock): + response = client.post("/telemetry", json=telemetry_with_extensive_field) + + assert response.status_code == 422 + repository_mock.add_telemetry.assert_not_called() + + +def test_disabled_telemetry_is_rejected(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + disabled_telemetry = { + **MINIMAL_TELEMETRY_TO_CREATE, + "telemetry_level": "disabled", + } + + with custom_test_server.container.telemetry_repository.override(repository_mock): + response = client.post("/telemetry", json=disabled_telemetry) + + assert response.status_code == 422 + repository_mock.add_telemetry.assert_not_called() diff --git a/carbonserver/tests/api/service/test_telemetry_service.py b/carbonserver/tests/api/service/test_telemetry_service.py new file mode 100644 index 000000000..08ef947e0 --- /dev/null +++ b/carbonserver/tests/api/service/test_telemetry_service.py @@ -0,0 +1,29 @@ +from unittest import mock + +from carbonserver.api.infra.repositories.repository_telemetry import ( + SqlAlchemyRepository, +) +from carbonserver.api.schemas_telemetry import TelemetryCreate +from carbonserver.api.services.telemetry_service import TelemetryService + +TELEMETRY_ID = "f52fe339-164d-4c2b-a8c0-f562dfce066d" + + +def test_telemetry_service_creates_telemetry(): + repository_mock: SqlAlchemyRepository = mock.Mock(spec=SqlAlchemyRepository) + telemetry_service = TelemetryService(repository_mock) + repository_mock.add_telemetry.return_value = TELEMETRY_ID + + telemetry_to_create = TelemetryCreate( + timestamp="2026-05-03T12:00:00+00:00", + telemetry_level="minimal", + os="Linux-5.10.0-x86_64", + cpu_count=12, + python_version="3.11.5", + codecarbon_version="3.2.6", + ) + + actual_saved_telemetry_id = telemetry_service.add_telemetry(telemetry_to_create) + + assert actual_saved_telemetry_id == TELEMETRY_ID + repository_mock.add_telemetry.assert_called_once_with(telemetry_to_create) diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py new file mode 100644 index 000000000..8dfdad05a --- /dev/null +++ b/codecarbon/core/telemetry_client.py @@ -0,0 +1,61 @@ +import json +from typing import Optional, Union + +import requests + +from codecarbon.core.telemetry_schemas import TelemetryCreate +from codecarbon.external.logger import logger + + +class TelemetryClient: + """ + Client dedicated to sending CodeCarbon telemetry payloads. + """ + + def __init__( + self, + endpoint_url="https://api.codecarbon.io", + telemetry: Optional[Union[TelemetryCreate, dict]] = None, + ): + self.endpoint_url = endpoint_url.rstrip("/") + self.telemetry_url = self.endpoint_url + "/telemetry" + self.headers = {"Content-Type": "application/json"} + self.telemetry = self._validate_telemetry(telemetry) if telemetry else None + + def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None): + telemetry_payload = ( + self._validate_telemetry(telemetry) if telemetry else self.telemetry + ) + if telemetry_payload is None: + logger.error("TelemetryClient.add_telemetry() needs a telemetry payload") + return None + payload = telemetry_payload.model_dump(mode="json", exclude_none=True) + + try: + response = requests.post( + url=self.telemetry_url, + json=payload, + timeout=2, + headers=self.headers, + ) + if response.status_code != 201: + self._log_error(payload, response) + return None + return response.json() + except Exception as e: + logger.error(e, exc_info=True) + return None + + @staticmethod + def _validate_telemetry(telemetry: Union[TelemetryCreate, dict]) -> TelemetryCreate: + if isinstance(telemetry, TelemetryCreate): + return telemetry + return TelemetryCreate(**telemetry) + + def _log_error(self, payload, response): + logger.error( + f"TelemetryClient Error when calling the API on {self.telemetry_url} with : {json.dumps(payload)}" + ) + logger.error( + f"TelemetryClient API return http code {response.status_code} and answer : {response.text}" + ) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry_schemas.py new file mode 100644 index 000000000..ea6249b65 --- /dev/null +++ b/codecarbon/core/telemetry_schemas.py @@ -0,0 +1,156 @@ +from datetime import datetime +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class TelemetryLevel(str, Enum): + disabled = "disabled" + minimal = "minimal" + extensive = "extensive" + + +class TelemetryBase(BaseModel): + model_config = ConfigDict(extra="forbid", use_enum_values=True) + + timestamp: datetime + telemetry_level: TelemetryLevel + + os: Optional[str] = None + country_name: Optional[str] = None + country_iso_code: Optional[str] = Field(default=None, min_length=2, max_length=3) + region: Optional[str] = None + cloud_provider: Optional[str] = None + cloud_region: Optional[str] = None + longitude: Optional[float] = Field(default=None, ge=-180, le=180) + latitude: Optional[float] = Field(default=None, ge=-90, le=90) + + cpu_count: Optional[int] = Field(default=None, ge=0) + cpu_physical_count: Optional[int] = Field(default=None, ge=0) + cpu_model: Optional[str] = None + cpu_architecture: Optional[str] = None + gpu_count: Optional[int] = Field(default=None, ge=0) + gpu_model: Optional[str] = None + gpu_driver_version: Optional[str] = None + gpu_memory_total_gb: Optional[float] = Field(default=None, ge=0) + ram_total_size_gb: Optional[float] = Field(default=None, ge=0) + cuda_version: Optional[str] = None + cudnn_version: Optional[str] = None + + python_version: Optional[str] = None + python_implementation: Optional[str] = None + python_executable_hash: Optional[str] = Field( + default=None, min_length=64, max_length=64 + ) + python_env_type: Optional[str] = None + codecarbon_version: Optional[str] = None + codecarbon_install_method: Optional[str] = None + + total_emissions_kg: Optional[float] = Field(default=None, ge=0) + emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) + energy_consumed_kwh: Optional[float] = Field(default=None, ge=0) + cpu_energy_kwh: Optional[float] = Field(default=None, ge=0) + gpu_energy_kwh: Optional[float] = Field(default=None, ge=0) + ram_energy_kwh: Optional[float] = Field(default=None, ge=0) + duration_seconds: Optional[float] = Field(default=None, ge=0) + cpu_utilization_avg: Optional[float] = Field(default=None, ge=0, le=100) + gpu_utilization_avg: Optional[float] = Field(default=None, ge=0, le=100) + ram_utilization_avg: Optional[float] = Field(default=None, ge=0, le=100) + + tracking_mode: Optional[str] = None + api_mode: Optional[str] = None + output_methods: Optional[List[str]] = None + hardware_tracked: Optional[List[str]] = None + task_tracking_used: Optional[bool] = None + decorator_vs_context: Optional[str] = None + measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + + hardware_detection_success: Optional[bool] = None + rapl_available: Optional[bool] = None + gpu_detection_method: Optional[str] = None + first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) + tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) + errors_encountered: Optional[List[str]] = None + warning_count: Optional[int] = Field(default=None, ge=0) + + ide_used: Optional[str] = None + notebook_environment: Optional[str] = None + ci_environment: Optional[str] = None + python_package_manager: Optional[str] = None + framework_detected: Optional[str] = None + + has_torch: Optional[bool] = None + torch_version: Optional[str] = None + has_transformers: Optional[bool] = None + transformers_version: Optional[str] = None + has_diffusers: Optional[bool] = None + diffusers_version: Optional[str] = None + has_tensorflow: Optional[bool] = None + tensorflow_version: Optional[str] = None + has_keras: Optional[bool] = None + keras_version: Optional[str] = None + has_pytorch_lightning: Optional[bool] = None + pytorch_lightning_version: Optional[str] = None + has_fastai: Optional[bool] = None + fastai_version: Optional[str] = None + ml_framework_primary: Optional[str] = None + + container_runtime: Optional[str] = None + in_container: Optional[bool] = None + host_machine_hash: Optional[str] = None + + @model_validator(mode="after") + def validate_telemetry_level(self): + if self.telemetry_level == TelemetryLevel.disabled: + raise ValueError("Disabled telemetry must not be submitted") + + if self.telemetry_level == TelemetryLevel.minimal: + extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS + submitted_extensive_fields = [ + field + for field in extensive_fields + if getattr(self, field) not in (None, [], {}) + ] + if submitted_extensive_fields: + fields = ", ".join(sorted(submitted_extensive_fields)) + raise ValueError( + f"Minimal telemetry cannot include extensive fields: {fields}" + ) + + return self + + +MINIMAL_TELEMETRY_FIELDS = { + "timestamp", + "telemetry_level", + "os", + "country_name", + "country_iso_code", + "region", + "cloud_provider", + "cloud_region", + "longitude", + "latitude", + "cpu_count", + "cpu_physical_count", + "cpu_model", + "cpu_architecture", + "gpu_count", + "gpu_model", + "gpu_driver_version", + "gpu_memory_total_gb", + "ram_total_size_gb", + "cuda_version", + "cudnn_version", + "python_version", + "python_implementation", + "python_executable_hash", + "python_env_type", + "codecarbon_version", + "codecarbon_install_method", +} + + +class TelemetryCreate(TelemetryBase): + pass diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py new file mode 100644 index 000000000..14ade8b7f --- /dev/null +++ b/tests/test_telemetry_client.py @@ -0,0 +1,104 @@ +import unittest + +import requests_mock +from pydantic import ValidationError + +from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_schemas import TelemetryCreate + + +class TestTelemetryClient(unittest.TestCase): + def test_init_sets_up_client_without_calling_api(self): + with requests_mock.Mocker() as m: + client = TelemetryClient( + endpoint_url="http://test.com/", + telemetry={ + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + }, + ) + + self.assertEqual(client.endpoint_url, "http://test.com") + self.assertEqual(client.telemetry_url, "http://test.com/telemetry") + self.assertIsInstance(client.telemetry, TelemetryCreate) + self.assertEqual(m.call_count, 0) + + def test_add_telemetry_posts_configured_payload(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + "os": "Linux-5.10.0-x86_64", + } + + with requests_mock.Mocker() as m: + m.post( + "http://test.com/telemetry", + json="f52fe339-164d-4c2b-a8c0-f562dfce066d", + status_code=201, + ) + client = TelemetryClient( + endpoint_url="http://test.com", telemetry=telemetry + ) + + actual_telemetry_id = client.add_telemetry() + + self.assertEqual( + actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" + ) + self.assertEqual(m.call_count, 1) + self.assertEqual( + m.last_request.json(), + { + **telemetry, + "timestamp": "2026-05-03T12:00:00Z", + }, + ) + + def test_add_telemetry_posts_call_payload(self): + telemetry = TelemetryCreate( + timestamp="2026-05-03T12:00:00+00:00", + telemetry_level="minimal", + os="Linux-5.10.0-x86_64", + ) + + with requests_mock.Mocker() as m: + m.post( + "http://test.com/telemetry", + json="f52fe339-164d-4c2b-a8c0-f562dfce066d", + status_code=201, + ) + client = TelemetryClient(endpoint_url="http://test.com") + + actual_telemetry_id = client.add_telemetry(telemetry) + + self.assertEqual( + actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" + ) + self.assertEqual(m.call_count, 1) + self.assertEqual( + m.last_request.json(), + { + "timestamp": "2026-05-03T12:00:00Z", + "telemetry_level": "minimal", + "os": "Linux-5.10.0-x86_64", + }, + ) + + def test_init_rejects_invalid_telemetry_without_calling_api(self): + with requests_mock.Mocker() as m: + with self.assertRaises(ValidationError): + TelemetryClient( + endpoint_url="http://test.com", + telemetry={ + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + "total_emissions_kg": 0.42, + }, + ) + + self.assertEqual(m.call_count, 0) + + def test_add_telemetry_returns_none_without_payload(self): + client = TelemetryClient(endpoint_url="http://test.com") + + self.assertIsNone(client.add_telemetry()) From 39eef0ba478f5c70508b727e469fc91ea9048c6d Mon Sep 17 00:00:00 2001 From: inimaz <93inigo93@gmail.com> Date: Sun, 3 May 2026 14:36:08 +0200 Subject: [PATCH 06/29] tests: add a test so that client and server do not differ --- .../tests/api/test_telemetry_schema_drift.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 carbonserver/tests/api/test_telemetry_schema_drift.py diff --git a/carbonserver/tests/api/test_telemetry_schema_drift.py b/carbonserver/tests/api/test_telemetry_schema_drift.py new file mode 100644 index 000000000..abc20d764 --- /dev/null +++ b/carbonserver/tests/api/test_telemetry_schema_drift.py @@ -0,0 +1,45 @@ +"""Test to ensure that the telemetry schema used by the CarbonServer API does not drift from the core telemetry schema defined in CodeCarbon.""" + +import importlib.util +from copy import deepcopy +from pathlib import Path + +from carbonserver.api.schemas_telemetry import TelemetryCreate as ServerTelemetryCreate + +REPO_ROOT = Path(__file__).resolve().parents[3] +CORE_TELEMETRY_SCHEMA_PATH = REPO_ROOT / "codecarbon" / "core" / "telemetry_schemas.py" + + +def _load_core_telemetry_create(): + spec = importlib.util.spec_from_file_location( + "core_telemetry_schemas", + CORE_TELEMETRY_SCHEMA_PATH, + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.TelemetryCreate + + +def _contract_schema(schema): + contract = deepcopy(schema) + contract.pop("description", None) + contract.pop("example", None) + contract.pop("examples", None) + contract.pop("title", None) + contract.pop("$defs", None) + + for property_schema in contract.get("properties", {}).values(): + property_schema.pop("description", None) + property_schema.pop("example", None) + property_schema.pop("examples", None) + property_schema.pop("title", None) + + return contract + + +def test_core_and_server_telemetry_schemas_do_not_drift(): + CoreTelemetryCreate = _load_core_telemetry_create() + + assert _contract_schema( + CoreTelemetryCreate.model_json_schema() + ) == _contract_schema(ServerTelemetryCreate.model_json_schema()) From 9e97561f31dbfdf7544163691595ff42d269a043 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 20:01:44 +0200 Subject: [PATCH 07/29] refactor: use TelemetryClient for minimal tracker telemetry Align Tier 1 send path with PR #1171 schemas and HTTP 201 contract while keeping session dedup and emissions tracker opt-out wiring. Co-authored-by: Cursor --- codecarbon/telemetry.py | 104 +++++++++++++++++-------------- tests/test_telemetry.py | 131 ++++++++++++++++++++-------------------- 2 files changed, 123 insertions(+), 112 deletions(-) diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index d3ca9a0e9..eb4efb9b7 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,73 +1,87 @@ -""" -Telemetry module for CodeCarbon. - -This module handles two tiers of telemetry: -- Tier 1: Basic system information (always enabled) - python_version, os, cpu, gpu, ram, codecarbon_version, tracking_mode -- Tier 2: Detailed emissions data (opt-in) - requires CODECARBON_TELEMETRY_EXPERIMENT_ID environment variable -""" +"""Tracker-facing telemetry helpers built on the shared TelemetryClient.""" import os +from datetime import datetime, timezone from typing import Any -import requests +from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.external.logger import logger -# Public telemetry API key - limited permissions for metrics only TELEMETRY_API_KEY = os.environ.get( "CODECARBON_TELEMETRY_API_KEY", - "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" + "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU", +) +TELEMETRY_API_URL = os.environ.get( + "CODECARBON_TELEMETRY_API_URL", "https://api.codecarbon.io" ) -TELEMETRY_API_URL = "https://api.codecarbon.io" -TELEMETRY_EXPERIMENT_ID = None # Set before enabling Tier 2 (see Task 4) +TELEMETRY_EXPERIMENT_ID = None -_TIER1_SENT = False # module-level dedup: send once per Python session +_TIER1_SENT = False -_TIER1_FIELDS = [ - "python_version", - "os", - "cpu_count", - "cpu_model", - "gpu_count", - "gpu_model", - "ram_total_size", - "codecarbon_version", - "tracking_mode", -] +def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: + """Build a minimal telemetry payload dict from tracker configuration. + Args: + conf: Tracker configuration dictionary. + + Returns: + Dictionary suitable for ``TelemetryCreate`` validation. + """ + payload: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc), + "telemetry_level": TelemetryLevel.minimal.value, + "os": conf.get("os"), + "country_iso_code": conf.get("country_iso_code"), + "region": conf.get("region"), + "cloud_provider": conf.get("provider"), + "cloud_region": conf.get("region"), + "longitude": conf.get("longitude"), + "latitude": conf.get("latitude"), + "cpu_count": conf.get("cpu_count"), + "cpu_physical_count": conf.get("cpu_physical_count"), + "cpu_model": conf.get("cpu_model"), + "gpu_count": conf.get("gpu_count"), + "gpu_model": conf.get("gpu_model"), + "ram_total_size_gb": conf.get("ram_total_size"), + "python_version": conf.get("python_version"), + "codecarbon_version": conf.get("codecarbon_version"), + } + return {key: value for key, value in payload.items() if value is not None} -def collect_tier1_payload(conf: dict[str, Any]) -> dict[str, Any]: - return {k: conf.get(k) for k in _TIER1_FIELDS} +collect_tier1_payload = build_minimal_telemetry_dict -def send_tier1_telemetry(conf: dict[str, Any]) -> bool: - """Send Tier 1 telemetry metadata once per session. - Posts environment metadata to the telemetry API endpoint. Uses module-level - deduplication flag to ensure data is sent only once per Python session. - Exceptions are caught and logged but not raised (silent fail). +def send_tier1_telemetry( + conf: dict[str, Any], endpoint_url: str | None = None +) -> bool: + """Send minimal telemetry once per Python session. Args: - conf: Configuration dict with environment metadata (keys: python_version, - os, cpu_count, cpu_model, gpu_count, gpu_model, ram_total_size, - codecarbon_version, tracking_mode) + conf: Tracker configuration dictionary. + endpoint_url: Optional API base URL override. Returns: True if telemetry was sent successfully on this call, False if already - sent in this session or if an error occurred. + sent in this session or if sending failed. """ global _TIER1_SENT if _TIER1_SENT: return False try: - payload = collect_tier1_payload(conf) - requests.post( - f"{TELEMETRY_API_URL}/telemetry", - json=payload, - headers={"x-api-token": TELEMETRY_API_KEY}, - timeout=2, + payload = build_minimal_telemetry_dict(conf) + client = TelemetryClient( + endpoint_url=endpoint_url or TELEMETRY_API_URL, + telemetry=payload, ) - _TIER1_SENT = True - return True - except Exception as e: - logger.error(f"Telemetry Tier 1 failed (non-critical): {e}") + if TELEMETRY_API_KEY: + client.headers["x-api-token"] = TELEMETRY_API_KEY + result = client.add_telemetry() + if result is not None: + _TIER1_SENT = True + return True + return False + except Exception as error: + logger.error(f"Telemetry Tier 1 failed (non-critical): {error}") return False diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index ea6929dc7..f70979d66 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,14 +1,18 @@ -import os import platform -import shutil import sys import tempfile import unittest from pathlib import Path -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch + import codecarbon.telemetry as telemetry_module -from codecarbon.telemetry import collect_tier1_payload, send_tier1_telemetry +from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from codecarbon.telemetry import ( + build_minimal_telemetry_dict, + collect_tier1_payload, + send_tier1_telemetry, +) from tests.testutils import get_custom_mock_open if sys.platform == "darwin": @@ -20,89 +24,84 @@ "codecarbon.core.cpu.IntelPowerGadget._setup_cli" ) - empty_conf = "[codecarbon]" class TestTelemetry(unittest.TestCase): - def test_collect_tier1_payload_has_required_fields(self): + def test_build_minimal_telemetry_dict_maps_tracker_conf(self): conf = { "python_version": platform.python_version(), "os": platform.platform(), "cpu_count": 8, + "cpu_physical_count": 4, "cpu_model": "Intel Core i7", "gpu_count": 1, "gpu_model": "NVIDIA RTX 3080", "ram_total_size": 32.0, "codecarbon_version": "2.0.0", "tracking_mode": "machine", + "country_iso_code": "FRA", + "provider": "aws", + "region": "eu-west-1", } - payload = collect_tier1_payload(conf) - self.assertIn("python_version", payload) - self.assertIn("os", payload) - self.assertIn("cpu_count", payload) - self.assertIn("cpu_model", payload) - self.assertIn("gpu_count", payload) - self.assertIn("gpu_model", payload) - self.assertIn("ram_total_size", payload) - self.assertIn("codecarbon_version", payload) - self.assertIn("tracking_mode", payload) + payload = build_minimal_telemetry_dict(conf) + self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) + self.assertIn("timestamp", payload) self.assertEqual(payload["python_version"], conf["python_version"]) + self.assertEqual(payload["ram_total_size_gb"], conf["ram_total_size"]) + self.assertEqual(payload["cloud_provider"], conf["provider"]) + self.assertEqual(payload["cloud_region"], conf["region"]) + + def test_collect_tier1_payload_delegates_to_builder(self): + self.assertIs(collect_tier1_payload, build_minimal_telemetry_dict) class TestTier1Send(unittest.TestCase): def test_send_tier1_telemetry_sends_once_per_session(self): - """Verify deduplication: second call doesn't POST again.""" - telemetry_module._TIER1_SENT = False # reset between test runs + telemetry_module._TIER1_SENT = False conf = { "python_version": "3.11.0", "os": "Linux", "cpu_count": 4, "cpu_model": "Intel i5", "gpu_count": 0, - "gpu_model": None, - "ram_total_size": 16.0, "codecarbon_version": "2.0.0", - "tracking_mode": "process", } - with patch("codecarbon.telemetry.requests.post") as mock_post: - mock_post.return_value = MagicMock(status_code=201) - result = send_tier1_telemetry(conf) + with patch( + "codecarbon.core.telemetry_client.requests.post" + ) as mock_post: + mock_post.return_value = MagicMock( + status_code=201, json=lambda: "telemetry-id" + ) + result = send_tier1_telemetry(conf, endpoint_url="http://test.com") self.assertTrue(result) - assert mock_post.call_count == 1 - # second call is deduplicated - result = send_tier1_telemetry(conf) + self.assertEqual(mock_post.call_count, 1) + body = mock_post.call_args.kwargs["json"] + self.assertEqual(body["telemetry_level"], "minimal") + result = send_tier1_telemetry(conf, endpoint_url="http://test.com") self.assertFalse(result) - assert mock_post.call_count == 1 + self.assertEqual(mock_post.call_count, 1) def test_send_tier1_telemetry_fails_silently(self): - """Verify exceptions are caught and logged, not raised.""" telemetry_module._TIER1_SENT = False conf = { "python_version": "3.11.0", "os": "Linux", "cpu_count": 4, - "cpu_model": None, - "gpu_count": 0, - "gpu_model": None, - "ram_total_size": 8.0, "codecarbon_version": "2.0.0", - "tracking_mode": "process", } - with patch("codecarbon.telemetry.requests.post", side_effect=Exception("network error")): - with patch("codecarbon.telemetry.logger.error") as mock_logger: - # must not raise - result = send_tier1_telemetry(conf) + with patch( + "codecarbon.core.telemetry_client.requests.post", + side_effect=Exception("network error"), + ): + with patch("codecarbon.core.telemetry_client.logger.error") as mock_logger: + result = send_tier1_telemetry(conf, endpoint_url="http://test.com") self.assertFalse(result) - # verify error was logged - mock_logger.assert_called_once() - assert "network error" in str(mock_logger.call_args) + mock_logger.assert_called() @mock_platform_cli_setup class TestTrackerTelemetry(unittest.TestCase): - """Test that trackers wire Tier 1 telemetry into initialization.""" - def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() self.temp_path = Path(self.temp_dir.name) @@ -116,56 +115,54 @@ def tearDown(self) -> None: self.temp_dir.cleanup() def test_emissions_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): - """Tier 1 fires on EmissionsTracker initialization when send_telemetry=True (default).""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - mock_post.return_value = MagicMock(status_code=201) - # Block geo lookup to isolate test + with patch( + "codecarbon.core.telemetry_client.requests.post" + ) as mock_post: + mock_post.return_value = MagicMock( + status_code=201, json=lambda: "telemetry-id" + ) with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( + EmissionsTracker( send_telemetry=True, save_to_api=False, save_to_file=False ) - # Verify telemetry POST was called - assert mock_post.called, "Telemetry POST should have been called" + self.assertTrue(mock_post.called) def test_emissions_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): - """Tier 1 does NOT fire when send_telemetry=False.""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - # Block geo lookup + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( + EmissionsTracker( send_telemetry=False, save_to_api=False, save_to_file=False ) - # Verify telemetry POST was NOT called - assert not mock_post.called, "Telemetry POST should not have been called" + self.assertFalse(mock_post.called) def test_offline_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): - """Tier 1 fires on OfflineEmissionsTracker initialization when send_telemetry=True (default).""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - mock_post.return_value = MagicMock(status_code=201) - tracker = OfflineEmissionsTracker( + with patch( + "codecarbon.core.telemetry_client.requests.post" + ) as mock_post: + mock_post.return_value = MagicMock( + status_code=201, json=lambda: "telemetry-id" + ) + OfflineEmissionsTracker( country_iso_code="CAN", send_telemetry=True, save_to_api=False, save_to_file=False, ) - # Verify telemetry POST was called - assert mock_post.called, "Telemetry POST should have been called" + self.assertTrue(mock_post.called) def test_offline_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): - """Tier 1 does NOT fire when send_telemetry=False on OfflineEmissionsTracker.""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - tracker = OfflineEmissionsTracker( + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + OfflineEmissionsTracker( country_iso_code="CAN", send_telemetry=False, save_to_api=False, save_to_file=False, ) - # Verify telemetry POST was NOT called - assert not mock_post.called, "Telemetry POST should not have been called" + self.assertFalse(mock_post.called) if __name__ == "__main__": From 453860f796caa7405e2f88544ff009696dc16d8d Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:36:17 +0200 Subject: [PATCH 08/29] feat: enhance telemetry module with minimal payload builder and improved session handling - Introduced `build_minimal_telemetry_dict` to create a minimal telemetry payload from tracker configuration. - Updated `send_tier1_telemetry` to utilize the new payload builder and handle optional API endpoint URLs. - Refactored tests to validate the new telemetry structure and ensure deduplication logic remains intact. - Improved error handling and logging for telemetry sending failures. --- codecarbon/cli/telemetry_cli.py | 249 ++++++++++++++ codecarbon/core/telemetry_settings.py | 153 +++++++++ docs/how-to/telemetry.md | 129 +++++++ .../2026-05-19-telemetry-configuration.md | 270 +++++++++++++++ tests/cli/test_telemetry_cli.py | 110 ++++++ tests/test_config_file_settings.py | 51 +++ tests/test_telemetry_config.py | 314 ++++++++++++++++++ tests/test_telemetry_settings.py | 132 ++++++++ 8 files changed, 1408 insertions(+) create mode 100644 codecarbon/cli/telemetry_cli.py create mode 100644 codecarbon/core/telemetry_settings.py create mode 100644 docs/how-to/telemetry.md create mode 100644 docs/plans/2026-05-19-telemetry-configuration.md create mode 100644 tests/cli/test_telemetry_cli.py create mode 100644 tests/test_config_file_settings.py create mode 100644 tests/test_telemetry_config.py create mode 100644 tests/test_telemetry_settings.py diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py new file mode 100644 index 000000000..218f2e941 --- /dev/null +++ b/codecarbon/cli/telemetry_cli.py @@ -0,0 +1,249 @@ +"""CLI commands to configure CodeCarbon product telemetry tiers.""" + +from pathlib import Path +from typing import Optional + +import questionary +import typer +from rich import print +from typing_extensions import Annotated + +from codecarbon.cli.cli_utils import ( + create_new_config_file, + get_config, + overwrite_local_config, +) +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import ( + DEFAULT_TELEMETRY_LEVEL, + is_telemetry_level_explicit, + parse_telemetry_level, + resolve_telemetry_level, +) + +telemetry_app = typer.Typer( + help="Configure product telemetry (disabled, minimal, or extensive).", + no_args_is_help=False, +) + +TIER_DESCRIPTIONS = { + "disabled": "No telemetry.", + "minimal": "Send hardware/environment metadata once per session (Tier 1).", + "extensive": "Tier 1 plus public emissions on stop (Tier 2 leaderboard).", +} + + +def normalize_telemetry_level(level: str) -> str: + """Validate and normalize a telemetry tier string for CLI use. + + Args: + level: User-provided tier name. + + Returns: + Canonical tier value. + + Raises: + typer.BadParameter: If the level is not a valid ``TelemetryLevel``. + """ + try: + return parse_telemetry_level(level).value + except ValueError as error: + raise typer.BadParameter(str(error)) from error + + +def resolve_config_path(config: Optional[Path], *, create: bool = False) -> Path: + """Resolve which config file to read or write. + + Args: + config: Explicit path from ``--config``, if any. + create: When True and no file exists, create ``./.codecarbon.config``. + + Returns: + Resolved config file path. + """ + if config is not None: + path = config.expanduser().resolve() + if create and not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + return path + local_path = Path.cwd().resolve() / ".codecarbon.config" + if local_path.exists(): + return local_path + global_path = (Path.home() / ".codecarbon.config").expanduser().resolve() + if global_path.exists(): + return global_path + if create: + local_path.write_text("[codecarbon]\n", encoding="utf-8") + return local_path + return local_path + + +def pick_config_path_interactive() -> Path: + """Prompt for which config file to update. + + Returns: + Path chosen by the user. + """ + home = Path.home() + global_path = (home / ".codecarbon.config").expanduser().resolve() + local_path = Path.cwd().resolve() / ".codecarbon.config" + options = [] + if global_path.exists(): + options.append(str(global_path)) + if local_path.exists() and local_path not in options: + options.append(str(local_path)) + options.append("Create new config file") + if not options: + options = ["Create new config file"] + choice = questionary.select( + "Which configuration file should store telemetry_level?", + choices=options, + ).ask() + if choice == "Create new config file": + return create_new_config_file() + return Path(choice).expanduser().resolve() + + +def write_telemetry_level(path: Path, level: str) -> None: + """Persist ``telemetry_level`` to a config file. + + Args: + path: Target ``.codecarbon.config`` path. + level: Validated tier value. + """ + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + overwrite_local_config("telemetry_level", level, path=path) + + +def print_telemetry_status(config_path: Optional[Path] = None) -> None: + """Print resolved telemetry settings. + + Without ``config_path``, uses the same merged file settings and env overlay + as ``EmissionsTracker``. With ``config_path``, inspects that file only. + + Args: + config_path: Optional single config file to inspect. + """ + if config_path is not None: + path = config_path.expanduser().resolve() + if not path.exists(): + print(f"[yellow]Config file not found:[/yellow] {path}") + print(f"Default tier: {DEFAULT_TELEMETRY_LEVEL.value} (not explicit)") + return + file_settings = get_config(path) + external_conf: dict[str, str] = {} + source_label = str(path) + else: + file_settings = get_config_file_settings() + external_conf = get_hierarchical_config() + source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" + + level = resolve_telemetry_level(file_settings) + explicit = is_telemetry_level_explicit(file_settings, external_conf=external_conf) + stored = file_settings.get("telemetry_level") + print(f"Config source: {source_label}") + print(f"telemetry_level in file(s): {stored!r}") + print(f"Resolved tier: {level.value}") + print(f"Explicitly configured: {explicit}") + if not explicit: + print( + "[yellow]Tier 1 minimal telemetry will be sent once per session " + "until you set telemetry_level.[/yellow]" + ) + + +def run_telemetry_interactive(config: Optional[Path] = None) -> None: + """Run the interactive telemetry configuration wizard. + + Args: + config: Optional fixed config path; otherwise prompt for file choice. + """ + print("CodeCarbon product telemetry") + print( + "Separate from your dashboard experiment (codecarbon config). " + "Controls optional usage analytics and public leaderboard data.\n" + ) + path = resolve_config_path(config) if config else None + if path is None or (config is None and not path.exists()): + path = pick_config_path_interactive() + else: + path = resolve_config_path(config, create=True) + + choices = [ + questionary.Choice("disabled — " + TIER_DESCRIPTIONS["disabled"], "disabled"), + questionary.Choice("minimal — " + TIER_DESCRIPTIONS["minimal"], "minimal"), + questionary.Choice( + "extensive — " + TIER_DESCRIPTIONS["extensive"], "extensive" + ), + ] + try: + current = get_config(path).get("telemetry_level") + except FileNotFoundError: + current = None + valid_levels = {member.value for member in TelemetryLevel} + default = current if current in valid_levels else "minimal" + level = questionary.select( + "Select telemetry_level:", + choices=choices, + default=default, + ).ask() + if level is None: + raise typer.Exit(0) + level = normalize_telemetry_level(level) + write_telemetry_level(path, level) + print(f"[green]Saved[/green] telemetry_level = {level} in {path}") + + +@telemetry_app.callback(invoke_without_command=True) +def telemetry_entry( + ctx: typer.Context, + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (default: local then global).", + ), + ] = None, +) -> None: + """Configure telemetry interactively when no subcommand is given.""" + if ctx.invoked_subcommand is None: + run_telemetry_interactive(config=config) + + +@telemetry_app.command("show") +def show( + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Inspect one file only; default matches EmissionsTracker merge.", + ), + ] = None, +) -> None: + """Show the resolved telemetry tier.""" + print_telemetry_status(config_path=config) + + +@telemetry_app.command("set") +def set_level( + level: Annotated[ + str, + typer.Argument(help="Telemetry tier: disabled, minimal, or extensive."), + ], + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (creates ./.codecarbon.config if missing).", + ), + ] = None, +) -> None: + """Write telemetry_level to a config file.""" + path = resolve_config_path(config, create=True) + normalized = normalize_telemetry_level(level) + write_telemetry_level(path, normalized) + print(f"[green]Saved[/green] telemetry_level = {normalized} in {path}") diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py new file mode 100644 index 000000000..ee78970c4 --- /dev/null +++ b/codecarbon/core/telemetry_settings.py @@ -0,0 +1,153 @@ +"""Resolve telemetry tier and API settings from config and environment.""" + +import os +from typing import Any + +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.external.logger import logger + +DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" +DEFAULT_TELEMETRY_API_KEY = "cpt_JZhj-vJdEVG28qErZL5mh1ftiqbnDIBYjWSxwvX3rfI" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "aa69b440-014a-4562-ac06-ba7eecb023f9" +DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal + +TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" + + +def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: + """Parse a telemetry tier from a string or enum value. + + Args: + raw: Tier name or ``TelemetryLevel`` member. + + Returns: + Parsed ``TelemetryLevel``. + + Raises: + ValueError: If ``raw`` is not a valid tier name. + """ + if isinstance(raw, TelemetryLevel): + return raw + try: + return TelemetryLevel(str(raw).lower()) + except ValueError as error: + raise ValueError( + f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." + ) from error + + +def is_telemetry_level_explicit( + config_file_conf: dict[str, Any], + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> bool: + """Return whether the user explicitly chose a telemetry tier. + + Explicit sources (in order): tracker ``telemetry_level`` argument, config file + ``telemetry_level``, environment ``CODECARBON_TELEMETRY_LEVEL`` / + ``telemetry_level``. Legacy ``CODECARBON_TELEMETRY`` / ``telemetry`` counts as + explicit for this check only. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). + override: Value passed to ``EmissionsTracker(telemetry_level=...)``. + external_conf: Merged config from file and environment. + + Returns: + True if any explicit source is set. + """ + if override is not None: + return True + if config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + return True + if external_conf is None: + return False + if external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + return True + return external_conf.get("telemetry") is not None + + +def resolve_telemetry_level( + config_file_conf: dict[str, Any], + *, + override: str | TelemetryLevel | None = None, +) -> TelemetryLevel: + """Resolve the active telemetry tier. + + Precedence: tracker ``telemetry_level`` argument, then ``telemetry_level`` in + ``.codecarbon.config``. Environment variables do not change the tier unless + passed as ``override`` from a future CLI integration. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). + override: Optional tier from ``EmissionsTracker(telemetry_level=...)``. + + Returns: + Resolved ``TelemetryLevel``. + """ + raw = ( + override + if override is not None + else config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) + ) + if raw is None: + return DEFAULT_TELEMETRY_LEVEL + try: + return parse_telemetry_level(raw) + except ValueError: + logger.error( + "Invalid telemetry_level %r; falling back to %r", + raw, + DEFAULT_TELEMETRY_LEVEL.value, + ) + return DEFAULT_TELEMETRY_LEVEL + + +def get_telemetry_api_url( + external_conf: dict[str, Any], + default: str = DEFAULT_TELEMETRY_API_URL, +) -> str: + """Return telemetry API base URL from config or environment. + + Args: + external_conf: Merged config from file and environment. + default: URL used when unset. + + Returns: + API base URL without trailing slash. + """ + url = external_conf.get("telemetry_api_url") or os.environ.get( + "CODECARBON_TELEMETRY_API_URL" + ) + return (url or default).rstrip("/") + + +def get_telemetry_api_key(external_conf: dict[str, Any]) -> str: + """Return telemetry API token from config, environment, or public default. + + Args: + external_conf: Merged config from file and environment. + + Returns: + API token string. + """ + key = external_conf.get("telemetry_api_key") or os.environ.get( + "CODECARBON_TELEMETRY_API_KEY" + ) + return key or DEFAULT_TELEMETRY_API_KEY + + +def get_telemetry_experiment_id(external_conf: dict[str, Any]) -> str: + """Return telemetry experiment id from config, environment, or public default. + + Args: + external_conf: Merged config from file and environment. + + Returns: + Experiment UUID string for Tier 2 / leaderboard linkage. + """ + experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( + "CODECARBON_TELEMETRY_EXPERIMENT_ID" + ) + return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md new file mode 100644 index 000000000..7a19bafa9 --- /dev/null +++ b/docs/how-to/telemetry.md @@ -0,0 +1,129 @@ +# Product telemetry + +CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people use, and (if you opt in) anonymous run emissions on a public leaderboard. + +This is **separate from** your own dashboard setup (`codecarbon config`, `codecarbon login`, `save_to_api`). Those commands configure **your** project and experiments. Product telemetry uses the shared settings below. + +## Telemetry tiers + +| Tier | `telemetry_level` | When | What is sent | +|------|-------------------|------|----------------| +| **0** | `disabled` | — | Nothing | +| **1** | `minimal` (default) | Once per Python process, when the tracker starts | Minimal hardware / environment metadata (see below) | +| **2** | `extensive` | Tier 1 on start **and** Tier 2 on `stop()` | Tier 1 plus one public emissions row for the run | + +If you never set `telemetry_level`, CodeCarbon uses **`minimal`** and logs a **one-time warning** per Python session telling you that Tier 1 will be sent. + +## Tier 1: what we collect today + +Tier 1 sends a single `POST` to `{telemetry_api_url}/telemetry` the first time an `EmissionsTracker` or `OfflineEmissionsTracker` is created in a process (not on every run). + +Only the fields below are included. Any value that is unknown is **omitted** from the payload (not sent as `null`). + +| Field | Description | +|-------|-------------| +| `timestamp` | UTC time when the tracker was initialized | +| `telemetry_level` | Always `minimal` for this tier | +| `os` | Operating system string (e.g. platform description) | +| `country_iso_code` | ISO country code when known (e.g. offline mode or geo lookup) | +| `region` | Region or cloud region when known | +| `cloud_provider` | Cloud provider name when known | +| `cloud_region` | Same as `region` when cloud metadata is available | +| `longitude` | Approximate longitude when known (degrees) | +| `latitude` | Approximate latitude when known (degrees) | +| `cpu_count` | Logical CPU count | +| `cpu_physical_count` | Physical CPU count | +| `cpu_model` | CPU model name | +| `gpu_count` | Number of GPUs detected | +| `gpu_model` | GPU model name(s) | +| `ram_total_size_gb` | Total RAM in GB | +| `python_version` | Python version string | +| `codecarbon_version` | Installed CodeCarbon version | + +### Tier 1: what we do **not** collect yet + +The API schema supports more “minimal” fields (framework versions, install method, executable hash, etc.). **The client does not send them today.** In particular, Tier 1 does **not** include: + +- Run duration, energy, or CO₂ emissions +- CPU/GPU utilization or power samples +- Project, experiment, or user identifiers from your dashboard config +- Python executable hash, virtualenv type, or ML framework versions +- IDE, CI, or notebook environment metadata + +If we add fields later, this page will be updated; the payload will stay limited to the minimal tier rules on the server. + +## Tier 2: extensive (public leaderboard) + +When `telemetry_level = extensive`, CodeCarbon still sends Tier 1 once per process, and on **`stop()`** posts **one** emissions summary to the shared telemetry experiment via the same API used for leaderboard data (`add_emission`). That is independent of `save_to_api` (your private dashboard uploads). + +Use this only if you are comfortable contributing anonymous run-level emissions to the public experiment. + +## Configure telemetry + +### Config file + +Add to `~/.codecarbon.config` and/or `./.codecarbon.config`: + +```ini +[codecarbon] +telemetry_level = minimal +``` + +Allowed values: `disabled`, `minimal`, `extensive`. + +Optional API overrides (defaults point at the public telemetry project): + +```ini +telemetry_api_url = https://api.codecarbon.io +telemetry_api_key = cpt_... +telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 +``` + +Environment variables for URL, key, and experiment id: `CODECARBON_TELEMETRY_API_URL`, `CODECARBON_TELEMETRY_API_KEY`, `CODECARBON_TELEMETRY_EXPERIMENT_ID`. + +**Tier resolution:** `telemetry_level` in the config file, or the tracker / CLI override below. Environment variables such as `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` can mark your choice as “explicit” (so the default warning is skipped) but **do not** change the tier unless the same value is also in the config file or passed to the tracker. + +### CLI + +```bash +# Interactive wizard +codecarbon telemetry + +# Set tier in a config file +codecarbon telemetry set disabled +codecarbon telemetry show + +# One-run override (does not write the config file) +codecarbon monitor --telemetry-level minimal -- python train.py +``` + +See the [CLI reference](../reference/cli.md#codecarbon-telemetry) for details. + +### Python + +```python +from codecarbon import EmissionsTracker + +tracker = EmissionsTracker(telemetry_level="disabled") +``` + +`OfflineEmissionsTracker`, `@track_emissions`, and `codecarbon monitor` accept the same `telemetry_level` argument. + +### Disable telemetry in tests + +```ini +[codecarbon] +telemetry_level = disabled +``` + +## Privacy notes + +- Tier 1 is **best-effort**: failures are logged and do not block tracking. +- Coordinates are only sent when the tracker already resolved them; they are not precise location tracking by themselves. +- Choose `disabled` if you do not want any product telemetry. +- Choose `extensive` only if you accept publishing one emissions row per process to the public telemetry experiment on stop. + +## Related + +- [Configure CodeCarbon](configuration.md) — general config file and environment variables +- [Use the Cloud API & Dashboard](cloud-api.md) — your own projects and experiments diff --git a/docs/plans/2026-05-19-telemetry-configuration.md b/docs/plans/2026-05-19-telemetry-configuration.md new file mode 100644 index 000000000..82bc5e6c1 --- /dev/null +++ b/docs/plans/2026-05-19-telemetry-configuration.md @@ -0,0 +1,270 @@ +# Telemetry Tier Configuration Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use executing-plans to implement this plan task-by-task. + +**Goal:** Configure telemetry tiers (`disabled`, `minimal`, `extensive`) with clear defaults, explicit opt-in/out, and optional public emissions on stop. Aligned with issue #1106 and `TelemetryLevel` from PR #1171. + +**Architecture (shipped):** Tier resolution: **tracker `telemetry_level` kwarg** (if set) → else **`telemetry_level` in `.codecarbon.config`**. Env `CODECARBON_TELEMETRY` / `telemetry` does **not** change the tier (only counts as “explicit” for the configure warning). API URL/key/experiment: config → env → `DEFAULT_*` in `telemetry_settings.py`. **Tier 1:** `TelemetryClient` → `POST /telemetry` once per session on init (404 logged as warning until prod deploy). **Tier 2:** `ApiClient.add_emission` to public experiment on `stop()` when `extensive`. One-time **warning** if tier was not set explicitly. No `send_telemetry` bool. + +**Tech Stack:** Python 3.11+, Pydantic v2, `configparser`, Typer CLI, `pytest`, `requests` / `requests_mock`. + +**Base branch:** `feat/add-telemetry` (stacked on `feat/telemetry-backend`, PR #1200) + +**Related:** +- Issue: [#1106](https://github.com/mlco2/codecarbon/issues/1106) +- Backend: [PR #1171](https://github.com/mlco2/codecarbon/pull/1171) +- Client integration: [PR #1200](https://github.com/mlco2/codecarbon/pull/1200) +- CLI import fix (separate, to `master`): [PR #1201](https://github.com/mlco2/codecarbon/pull/1201) — `amdsmi` / `libamd_smi.so` on macOS + +--- + +## Configuration contract (current) + +### What users can configure where + +| Setting | Config file | Env | Python kwarg | CLI | Default | +|---------|-------------|-----|--------------|-----|---------| +| **`telemetry_level`** (tier) | ✅ | ❌ (tier) / ✅ (explicit warning only)¹ | ✅ `telemetry_level=` | ✅ see below | `minimal` | +| `telemetry_api_url` | ✅ | `CODECARBON_TELEMETRY_API_URL` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_URL` | +| `telemetry_api_key` | ✅ | `CODECARBON_TELEMETRY_API_KEY` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_KEY` | +| `telemetry_experiment_id` | ✅ | `CODECARBON_TELEMETRY_EXPERIMENT_ID` | ❌ | ❌ | `DEFAULT_TELEMETRY_EXPERIMENT_ID` | + +¹ `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` suppress the “not configured” warning but do **not** change the resolved tier unless also in config or passed as kwarg. + +**CLI for `telemetry_level`:** + +| Command | Persists to config? | Notes | +|---------|---------------------|--------| +| `codecarbon telemetry` | ✅ | Interactive wizard | +| `codecarbon telemetry set ` | ✅ | `disabled` \| `minimal` \| `extensive` | +| `codecarbon telemetry show` | — | Read resolved tier | +| `codecarbon monitor --telemetry-level ` | ❌ | One-run override only (tracker kwarg) | + +Separate from `codecarbon config` (dashboard org/project/experiment). + +### Tier values + +| Value | Tier | On init | On stop | Visibility (product) | +|-------|------|---------|---------|----------------------| +| `disabled` | 0 | — | — | None | +| `minimal` | 1 | Tier 1 `POST /telemetry` (once/session) | — | Private metadata | +| `extensive` | 2 | Tier 1 `POST /telemetry` (once/session) | Tier 2 `add_emission` | Public leaderboard | + +### Default when not explicit + +If the user never sets `telemetry_level` (no config key, no kwarg, no env key counted as explicit): + +1. **Warning** (once per Python session): default is `minimal`; **Tier 1 minimal telemetry will be sent** once per session. +2. **Behaviour:** same as `telemetry_level = minimal` (Tier 1 HTTP on init). + +### Explicit configuration (no warning) + +Any of: + +- `telemetry_level = …` in `.codecarbon.config` (local or global) +- `EmissionsTracker(telemetry_level="…")` / `OfflineEmissionsTracker` / `@track_emissions` +- `codecarbon telemetry set …` (writes config) +- `codecarbon monitor --telemetry-level …` (run override; counts as explicit for that run) +- `CODECARBON_TELEMETRY_LEVEL` or `CODECARBON_TELEMETRY` in the environment + +### Tier resolution precedence + +1. Tracker kwarg `telemetry_level` (CLI `--telemetry-level` or Python arg) +2. `telemetry_level` in config file (local overrides global) +3. Built-in default: `minimal` + +### `.codecarbon.config` example + +```ini +[codecarbon] +telemetry_level = minimal +telemetry_api_url = https://api.codecarbon.io +telemetry_api_key = cpt_... +telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 +``` + +### Tests disabling telemetry + +```ini +[codecarbon] +telemetry_level = disabled +``` + +Pattern: `get_custom_mock_open("[codecarbon]\ntelemetry_level = disabled\n", ...)`. + +--- + +## Out of scope + +- `codecarbon config` wizard step for telemetry (use `codecarbon telemetry` instead) +- `CODECARBON_TELEMETRY` env var **changing** the tier (legacy key only affects “explicit” warning) +- First-run consent prompt, Alembic migration +- Leaderboard UI / server-side public visibility enforcement + +--- + +## Task 1: Telemetry settings module ✅ + +**Files:** `codecarbon/core/telemetry_settings.py`, `codecarbon/core/config.py`, `tests/test_telemetry_settings.py`, `tests/test_config_file_settings.py` + +**Done:** +- `DEFAULT_TELEMETRY_*` constants and `get_telemetry_api_*` helpers +- `get_config_file_settings()` — file only +- `resolve_telemetry_level(config_file_conf, override=…)` — kwarg overrides config +- `is_telemetry_level_explicit(config_file_conf, override=…, external_conf=…)` + +--- + +## Task 2: `codecarbon/telemetry.py` ✅ + +**Done:** +- `build_minimal_telemetry_dict` / `send_tier1_telemetry` — `TelemetryClient` → `POST /telemetry` + `x-api-token`; session dedup `_TIER1_SENT`; HTTP 404 → warning +- `send_tier2_public_emission` — `ApiClient` + `add_emission` to public experiment; `_TIER2_SENT` +- `warn_if_telemetry_not_configured` — one-time warning; message states Tier 1 will be sent +- Tests in `tests/test_telemetry.py` + +--- + +## Task 3: Tracker wiring ✅ + +**Files:** `codecarbon/emissions_tracker.py` + +```mermaid +flowchart LR + subgraph init [Tracker __init__] + W[warn if not explicit] + T1[Tier 1 POST /telemetry] + end + subgraph stop [Tracker stop] + T2[Tier 2 add_emission] + end + W --> T1 + T1 --> TelAPI[TelemetryClient] + T2 --> CCApi[ApiClient public experiment] +``` + +```python +self._config_file_conf = get_config_file_settings() +telemetry_override = None if telemetry_level is _sentinel else telemetry_level +self._telemetry_level = resolve_telemetry_level(self._config_file_conf, override=telemetry_override) +self._apply_init_telemetry(telemetry_override) # warn + Tier 1 when minimal/extensive +# stop(): +self._maybe_send_tier2_telemetry(emissions_data_delta) # extensive only +``` + +| Config | Init | Stop | +|--------|------|------| +| `disabled` | — | — | +| `minimal` | Tier 1 HTTP | — | +| `extensive` | Tier 1 HTTP | Tier 2 `add_emission` | + +Independent of `save_to_api`. Best-effort; never blocks tracker. + +--- + +## Task 4: Extensive `POST /telemetry` payload ⏸ + +Deferred — Tier 2 is public `add_emission` only. + +--- + +## Task 5: Config integration tests ✅ + +**File:** `tests/test_telemetry_config.py` + +- Tier behaviour per config (`disabled` / `minimal` / `extensive`) +- Env does not change tier; env API URL for Tier 2 +- Warning when config empty; no warning when config or kwarg explicit +- Tier 1 asserts `TelemetryClient` / `POST /telemetry` (mocked in unit tests) + +--- + +## Task 6: Telemetry CLI ✅ + +**Files:** `codecarbon/cli/telemetry_cli.py`, `codecarbon/cli/main.py`, `tests/cli/test_telemetry_cli.py` + +**Done:** +- `codecarbon.add_typer(telemetry_app, name="telemetry")` +- `codecarbon telemetry` — interactive (`questionary`): pick config path + tier +- `codecarbon telemetry set [--config PATH]` — write `telemetry_level` +- `codecarbon telemetry show [--config PATH]` — resolved tier + explicit flag +- `codecarbon monitor --telemetry-level ` — one-run override via tracker kwarg +- `parse_telemetry_level()` (core) + `normalize_telemetry_level()` (CLI Typer wrapper) +- `telemetry show` uses merged file settings by default (matches tracker) + +**Not done:** optional hook at end of `codecarbon config` wizard (deferred). + +--- + +## Task 7: Documentation ✅ + +**Files:** `docs/how-to/telemetry.md`, `docs/reference/cli.md`, `mkdocs.yml`, link from `docs/how-to/configuration.md` + +**Done:** +- User-facing telemetry how-to with tiers, config, CLI, Python opt-out +- Tier 1 documents **only fields sent today** by `build_minimal_telemetry_dict` plus explicit “not collected yet” list +- CLI reference: `codecarbon telemetry`, `--telemetry-level` + +--- + +## Task 8: PR hygiene ✅ + +```bash +uv run pytest tests/test_telemetry_settings.py tests/test_config_file_settings.py \ + tests/test_telemetry.py tests/test_telemetry_config.py tests/test_telemetry_client.py \ + tests/cli/test_telemetry_cli.py -v +uv run pytest tests/test_emissions_tracker.py tests/test_offline_emissions_tracker.py -q +uv run pytest carbonserver/tests/api/test_telemetry_schema_drift.py -v +uv run task test-package +``` + +**Results (2026-05-19):** 62 telemetry tests, 22 tracker tests, schema drift, **527** package unit tests passed. + +- [x] Public `DEFAULT_TELEMETRY_API_KEY` / `DEFAULT_TELEMETRY_EXPERIMENT_ID` +- [x] `telemetry_level` kwarg on tracker + `@track_emissions` +- [x] Configure warning + Tier 1 HTTP when not explicit +- [x] Telemetry CLI + `monitor --telemetry-level` +- [x] No `send_telemetry`; tests use config mocks +- [x] Full package test suite green (`uv run task test-package`) +- [x] Schema drift test passes +- [x] `test_config.py` isolates `get_config_file_settings` (no leak from real `~/.codecarbon.config`) +- [x] `test_gpu.py` amdsmi warning message aligned with `gpu_amd.py` +- [x] `ApiClient._create_run` handles `longitude`/`latitude` `None` + +--- + +## Related fix (separate PR, not telemetry feature) + +**[PR #1201](https://github.com/mlco2/codecarbon/pull/1201)** → `master`: `codecarbon/core/gpu_amd.py` catches `KeyError` / `OSError` when `amdsmi` is installed but `libamd_smi.so` is missing (macOS CLI crash). Branch: `fix/amdsmi-import-keyerror-macos`. + +--- + +## Risk notes + +| Risk | Mitigation | +|------|------------| +| Users surprised by default Tier 1 logging | One-time warning names Tier 1 explicitly | +| Env `CODECARBON_TELEMETRY` confusion | Document: warning-only, not tier resolution | +| Tier 1 log vs future HTTP | Doc + comment in `send_tier1_telemetry`; flip when API ready | +| CLI import before amdsmi fix merged | PR #1201 to master; rebase `feat/add-telemetry` after merge | + +--- + +## Progress summary + +| Task | Status | +|------|--------| +| 1. Telemetry settings | ✅ | +| 2. `telemetry.py` | ✅ | +| 3. Tracker wiring | ✅ | +| 4. Extensive `/telemetry` POST | ⏸ | +| 5. Integration tests | ✅ | +| 6. Telemetry CLI | ✅ | +| 7. Documentation | ✅ | +| 8. Full CI / PR hygiene | ✅ | + +--- + +## Execution handoff + +**Next step:** Merge PR #1200; rebase after [PR #1201](https://github.com/mlco2/codecarbon/pull/1201); deploy [#1171](https://github.com/mlco2/codecarbon/pull/1171) so prod `/telemetry` stops returning 404. diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py new file mode 100644 index 000000000..871bd13c3 --- /dev/null +++ b/tests/cli/test_telemetry_cli.py @@ -0,0 +1,110 @@ +"""Tests for codecarbon telemetry CLI commands.""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from codecarbon.cli import main as cli_main +from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app + + +def test_normalize_telemetry_level_accepts_valid_values(): + assert normalize_telemetry_level("MINIMAL") == "minimal" + assert normalize_telemetry_level("disabled") == "disabled" + + +def test_normalize_telemetry_level_rejects_invalid(): + with pytest.raises(Exception): + normalize_telemetry_level("bogus") + + +def test_telemetry_set_writes_config(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + result = runner.invoke( + telemetry_app, + ["set", "disabled", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "telemetry_level = disabled" in result.output + content = config_path.read_text() + assert "telemetry_level = disabled" in content + + +def test_telemetry_show_reports_stored_level(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + config_path.write_text("[codecarbon]\ntelemetry_level = extensive\n") + result = runner.invoke( + telemetry_app, + ["show", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "extensive" in result.output + assert "Explicitly configured: True" in result.output + + +def test_telemetry_show_merged_matches_tracker_precedence(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + result = runner.invoke(telemetry_app, ["show"]) + assert result.exit_code == 0 + assert "Resolved tier: disabled" in result.output + assert "merged" in result.output + + +def test_monitor_passes_telemetry_level_override(monkeypatch): + from codecarbon.cli import monitor as monitor_module + + captured = {} + + class FakeTracker: + def __init__(self, **kwargs): + captured.update(kwargs) + self._conf = {"output_file": "emissions.csv"} + + def start(self): + return None + + def stop(self): + return 0.0 + + monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + [ + "monitor", + "--no-api", + "--telemetry-level", + "disabled", + "--", + "echo", + "ok", + ], + ) + assert result.exit_code == 0 + assert captured.get("telemetry_level") == "disabled" + + +def test_monitor_rejects_invalid_telemetry_level(): + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + ["monitor", "--telemetry-level", "invalid"], + ) + assert result.exit_code != 0 diff --git a/tests/test_config_file_settings.py b/tests/test_config_file_settings.py new file mode 100644 index 000000000..6a155ef7e --- /dev/null +++ b/tests/test_config_file_settings.py @@ -0,0 +1,51 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config + + +class TestGetConfigFileSettings(unittest.TestCase): + def test_returns_empty_when_no_config_files(self): + with patch("codecarbon.core.config._config_file_paths") as mock_paths: + mock_paths.return_value = ("/nonexistent/global", "/nonexistent/local") + settings = get_config_file_settings() + self.assertEqual(settings, {}) + + def test_local_overrides_global_telemetry_level(self): + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + settings = get_config_file_settings() + self.assertEqual(settings["telemetry_level"], "disabled") + + def test_hierarchical_config_includes_env_but_file_settings_do_not(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + file_settings = get_config_file_settings() + hierarchical = get_hierarchical_config() + self.assertEqual(file_settings.get("telemetry_level"), "minimal") + self.assertNotIn("telemetry", file_settings) + self.assertEqual(hierarchical.get("telemetry"), "disabled") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py new file mode 100644 index 000000000..41cbc26d0 --- /dev/null +++ b/tests/test_telemetry_config.py @@ -0,0 +1,314 @@ +"""Integration tests for telemetry tier resolution and config contract (Task 5).""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +import codecarbon.telemetry as telemetry_module +from codecarbon.core.config import get_config_file_settings +from codecarbon.core.telemetry_settings import resolve_telemetry_level +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import get_telemetry_api_url +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from codecarbon.telemetry import ( + send_tier1_telemetry, + send_tier2_public_emission, + warn_if_telemetry_not_configured, +) +from tests.testutils import get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") + + +def _conf(level: str) -> str: + return f"[codecarbon]\ntelemetry_level = {level}\n" + + +class TestTelemetryConfigContract(unittest.TestCase): + def setUp(self) -> None: + telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + + def test_warns_once_when_telemetry_not_explicit(self): + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) + warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) + self.assertEqual(mock_warning.call_count, 1) + self.assertIn("Tier 1", mock_warning.call_args[0][0]) + + def test_no_warn_when_config_explicit(self): + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + warn_if_telemetry_not_configured( + {"telemetry_level": "disabled"}, TelemetryLevel.disabled + ) + mock_warning.assert_not_called() + + def test_minimal_tier1_posts_once_per_session(self): + telemetry_module._TIER1_SENT = False + conf = {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"} + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_client_cls.return_value = mock_client + self.assertTrue(send_tier1_telemetry(conf)) + self.assertFalse(send_tier1_telemetry(conf)) + mock_client_cls.assert_called_once() + payload = mock_client_cls.call_args.kwargs["telemetry"] + self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) + + def test_tier1_posts_to_telemetry_endpoint(self): + telemetry_module._TIER1_SENT = False + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + send_tier1_telemetry( + {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"}, + external_conf={"telemetry_api_url": "http://tier1.example"}, + ) + mock_post.assert_called_once() + self.assertEqual( + mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" + ) + self.assertEqual( + mock_post.call_args.kwargs["json"]["telemetry_level"], + TelemetryLevel.minimal.value, + ) + + def test_env_codecarbon_telemetry_does_not_change_tier(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + file_settings = get_config_file_settings() + level = resolve_telemetry_level(file_settings) + self.assertEqual(level, TelemetryLevel.minimal) + + def test_telemetry_api_url_env_used_for_tier2_client(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env-telemetry.example"}, + clear=False, + ): + url = get_telemetry_api_url({}) + self.assertEqual(url, "http://env-telemetry.example") + + def test_telemetry_api_url_from_config_overrides_default(self): + url = get_telemetry_api_url( + {"telemetry_api_url": "http://config-telemetry.example"} + ) + self.assertEqual(url, "http://config-telemetry.example") + + +@mock_platform_cli_setup +class TestTrackerTelemetryFromConfig(unittest.TestCase): + def setUp(self) -> None: + telemetry_module._TIER1_SENT = False + telemetry_module._TIER2_SENT = False + telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + self._config_patcher = None + + def tearDown(self) -> None: + if self._config_patcher: + self._config_patcher.stop() + + def _mock_config(self, conf: str) -> None: + self._config_patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self._config_patcher.start() + + def test_disabled_no_tier1_post_or_tier2_emission(self, mock_cli_setup): + self._mock_config(_conf("disabled")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_not_called() + mock_api_cls.assert_not_called() + + def test_minimal_posts_tier1_on_init_not_tier2_on_stop(self, mock_cli_setup): + self._mock_config(_conf("minimal")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + mock_api_cls.assert_not_called() + + def test_extensive_posts_tier1_and_tier2_on_stop(self, mock_cli_setup): + self._mock_config(_conf("extensive")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + mock_api.add_emission.assert_called_once() + + def test_offline_minimal_with_save_to_api_false_still_posts_tier1( + self, mock_cli_setup + ): + self._mock_config(_conf("minimal")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + mock_telemetry_cls.assert_called_once() + + def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + env_without_telemetry = { + key: value + for key, value in os.environ.items() + if key.lower() + not in ( + "codecarbon_telemetry", + "codecarbon_telemetry_level", + ) + } + with patch.dict(os.environ, env_without_telemetry, clear=True): + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + with patch( + "codecarbon.telemetry.TelemetryClient" + ) as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + configure_warnings = [ + c + for c in mock_warning.call_args_list + if c[0] and "telemetry" in str(c[0][0]).lower() + ] + self.assertEqual(len(configure_warnings), 1) + + def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + with patch("codecarbon.telemetry.TelemetryClient"): + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker( + telemetry_level="disabled", + save_to_api=False, + save_to_file=False, + ) + configure_warnings = [ + c for c in mock_warning.call_args_list if c[0] and "Tier 1" in str(c[0][0]) + ] + self.assertEqual(len(configure_warnings), 0) + + def test_env_telemetry_disabled_still_minimal_tier1_when_config_minimal( + self, mock_cli_setup + ): + self._mock_config(_conf("minimal")) + with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_telemetry_cls.assert_called_once() + + +class TestTier2ApiSettings(unittest.TestCase): + def test_tier2_uses_resolved_api_url(self): + telemetry_module._TIER2_SENT = False + from codecarbon.output_methods.emissions_data import EmissionsData + + emissions = EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=1.0, + emissions=0.001, + emissions_rate=0.001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="", + country_iso_code="", + region="", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + external_conf = {"telemetry_api_url": "http://custom.example"} + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + send_tier2_public_emission({}, emissions, external_conf=external_conf) + mock_api_cls.assert_called_once() + self.assertEqual( + mock_api_cls.call_args.kwargs["endpoint_url"], "http://custom.example" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py new file mode 100644 index 000000000..68254999c --- /dev/null +++ b/tests/test_telemetry_settings.py @@ -0,0 +1,132 @@ +import os +import unittest +from unittest.mock import patch + +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + get_telemetry_api_key, + get_telemetry_api_url, + get_telemetry_experiment_id, + is_telemetry_level_explicit, + parse_telemetry_level, + resolve_telemetry_level, +) + + +class TestParseTelemetryLevel(unittest.TestCase): + def test_parse_accepts_enum(self): + self.assertEqual( + parse_telemetry_level(TelemetryLevel.minimal), TelemetryLevel.minimal + ) + + def test_parse_normalizes_case(self): + self.assertEqual(parse_telemetry_level("EXTENSIVE"), TelemetryLevel.extensive) + + def test_parse_rejects_invalid(self): + with self.assertRaises(ValueError): + parse_telemetry_level("bogus") + + +class TestResolveTelemetryLevel(unittest.TestCase): + def test_default_is_minimal_when_unset(self): + level = resolve_telemetry_level({}) + self.assertEqual(level, TelemetryLevel.minimal) + + def test_telemetry_level_from_config_file(self): + level = resolve_telemetry_level({"telemetry_level": "disabled"}) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_telemetry_level_extensive(self): + level = resolve_telemetry_level({"telemetry_level": "extensive"}) + self.assertEqual(level, TelemetryLevel.extensive) + + def test_env_telemetry_key_ignored(self): + with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): + level = resolve_telemetry_level({"telemetry": "extensive"}) + self.assertEqual(level, TelemetryLevel.minimal) + + def test_invalid_level_falls_back_to_minimal(self): + with patch("codecarbon.core.telemetry_settings.logger.error") as mock_error: + level = resolve_telemetry_level({"telemetry_level": "bogus"}) + self.assertEqual(level, TelemetryLevel.minimal) + mock_error.assert_called_once() + + def test_override_kwarg_takes_precedence_over_config_file(self): + level = resolve_telemetry_level( + {"telemetry_level": "minimal"}, override="disabled" + ) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_is_explicit_with_config_file(self): + self.assertTrue(is_telemetry_level_explicit({"telemetry_level": "minimal"})) + + def test_is_explicit_with_override(self): + self.assertTrue(is_telemetry_level_explicit({}, override="disabled")) + + def test_is_explicit_with_env_telemetry_level(self): + self.assertTrue( + is_telemetry_level_explicit( + {}, external_conf={"telemetry_level": "minimal"} + ) + ) + + def test_is_not_explicit_when_unset(self): + self.assertFalse(is_telemetry_level_explicit({})) + + +class TestTelemetryApiSettings(unittest.TestCase): + def test_get_telemetry_api_url_from_conf(self): + url = get_telemetry_api_url({"telemetry_api_url": "http://test.example"}) + self.assertEqual(url, "http://test.example") + + def test_get_telemetry_api_url_default(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_URL" + } + with patch.dict(os.environ, env, clear=True): + url = get_telemetry_api_url({}) + self.assertEqual(url, DEFAULT_TELEMETRY_API_URL) + + def test_get_telemetry_api_url_env_fallback(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env.example"}, + clear=False, + ): + url = get_telemetry_api_url({}) + self.assertEqual(url, "http://env.example") + + def test_get_telemetry_api_key_from_conf(self): + key = get_telemetry_api_key({"telemetry_api_key": "cpt_test"}) + self.assertEqual(key, "cpt_test") + + def test_get_telemetry_api_key_uses_public_default_when_unset(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_KEY" + } + with patch.dict(os.environ, env, clear=True): + key = get_telemetry_api_key({}) + self.assertEqual(key, DEFAULT_TELEMETRY_API_KEY) + + def test_get_telemetry_experiment_id_from_conf(self): + experiment_id = get_telemetry_experiment_id( + {"telemetry_experiment_id": "00000000-0000-0000-0000-000000000001"} + ) + self.assertEqual(experiment_id, "00000000-0000-0000-0000-000000000001") + + def test_get_telemetry_experiment_id_uses_public_default_when_unset(self): + env = { + k: v + for k, v in os.environ.items() + if k != "CODECARBON_TELEMETRY_EXPERIMENT_ID" + } + with patch.dict(os.environ, env, clear=True): + experiment_id = get_telemetry_experiment_id({}) + self.assertEqual(experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + + +if __name__ == "__main__": + unittest.main() From 954910ef28ce2477367ecc153a6c61f50574b361 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:40:50 +0200 Subject: [PATCH 09/29] feat: enhance telemetry functionality with new configuration and session handling - Added support for telemetry level configuration via `.codecarbon.config`, allowing users to specify tiers (disabled, minimal, extensive). - Introduced `get_config_file_settings` to read configuration settings from global and local files. - Updated `BaseEmissionsTracker` to utilize the new telemetry level parameter, improving flexibility in telemetry data collection. - Refactored telemetry sending methods to handle new configurations and ensure proper logging and error handling. - Enhanced tests to cover new telemetry configurations and ensure correct behavior across different scenarios. --- .gitignore | 3 + codecarbon/cli/main.py | 13 ++ codecarbon/core/api_client.py | 12 +- codecarbon/core/config.py | 40 +++-- codecarbon/core/gpu_amd.py | 10 +- codecarbon/core/telemetry_client.py | 19 ++- codecarbon/emissions_tracker.py | 60 +++++-- codecarbon/telemetry.py | 129 +++++++++++--- docs/how-to/configuration.md | 6 + docs/how-to/telemetry.md | 149 ++++++++-------- docs/reference/cli.md | 29 ++++ mkdocs.yml | 1 + tests/test_api_call.py | 16 ++ tests/test_config.py | 33 +++- tests/test_emissions_tracker.py | 22 +-- tests/test_gpu.py | 2 +- tests/test_offline_emissions_tracker.py | 9 +- tests/test_telemetry.py | 215 ++++++++++++++++-------- tests/test_telemetry_client.py | 39 +++++ 19 files changed, 587 insertions(+), 220 deletions(-) diff --git a/.gitignore b/.gitignore index 86958c842..b88e0109f 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,6 @@ tests/test_data/rapl/* credentials* .codecarbon.config* scripts/agent-vm.personal.config.sh + +# Added by ggshield +.cache_ggshield diff --git a/codecarbon/cli/main.py b/codecarbon/cli/main.py index d5175ea9c..5174d6475 100644 --- a/codecarbon/cli/main.py +++ b/codecarbon/cli/main.py @@ -22,6 +22,7 @@ overwrite_local_config, ) from codecarbon.cli.monitor import run_and_monitor +from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app from codecarbon.core.api_client import ApiClient, get_datetime_with_timezone from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate, ProjectCreate from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker @@ -32,6 +33,7 @@ DEFAULT_ORGANIzATION_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1" codecarbon = typer.Typer(no_args_is_help=True) +codecarbon.add_typer(telemetry_app, name="telemetry") def main(): @@ -342,6 +344,15 @@ def monitor( str, typer.Option(help="Region/province for offline mode"), ] = None, + telemetry_level: Annotated[ + Optional[str], + typer.Option( + help=( + "Override telemetry tier for this run only " + "(disabled, minimal, or extensive)." + ), + ), + ] = None, ): """Monitor your machine's carbon emissions.""" @@ -350,6 +361,8 @@ def monitor( "measure_power_secs": measure_power_secs, "api_call_interval": api_call_interval, } + if telemetry_level is not None: + tracker_args["telemetry_level"] = normalize_telemetry_level(telemetry_level) # Set up the tracker arguments based on mode (offline vs online) and validate required args for each mode if offline: if not country_iso_code: diff --git a/codecarbon/core/api_client.py b/codecarbon/core/api_client.py index 34067c71c..de1988d34 100644 --- a/codecarbon/core/api_client.py +++ b/codecarbon/core/api_client.py @@ -30,6 +30,13 @@ def get_datetime_with_timezone(): return timestamp +def _round_coordinate(value, decimals: int = 1) -> float: + """Round a geographic coordinate for API payloads, treating None as zero.""" + if value is None: + return round(0.0, decimals) + return round(float(value), decimals) + + class ApiClient: # (AsyncClient) """ This class call the Code Carbon API @@ -263,9 +270,8 @@ def _create_run(self, experiment_id: str): cpu_model=self.conf.get("cpu_model"), gpu_count=self.conf.get("gpu_count"), gpu_model=self.conf.get("gpu_model"), - # Reduce precision for Privacy - longitude=round(self.conf.get("longitude", 0), 1), - latitude=round(self.conf.get("latitude", 0), 1), + longitude=_round_coordinate(self.conf.get("longitude")), + latitude=_round_coordinate(self.conf.get("latitude")), region=self.conf.get("region"), provider=self.conf.get("provider"), ram_total_size=self.conf.get("ram_total_size"), diff --git a/codecarbon/core/config.py b/codecarbon/core/config.py index 7cacea41d..65330f7a0 100644 --- a/codecarbon/core/config.py +++ b/codecarbon/core/config.py @@ -110,6 +110,31 @@ def normalize_gpu_ids( return None +def _config_file_paths() -> tuple[str, str]: + """Return resolved paths for global and local CodeCarbon config files.""" + cwd = Path.cwd() + home = Path.home() + global_path = str((home / ".codecarbon.config").expanduser().resolve()) + local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + return global_path, local_path + + +def get_config_file_settings() -> dict[str, str]: + """Return the ``[codecarbon]`` section from config files without environment overlay. + + Reads ``~/.codecarbon.config`` then ``./.codecarbon.config`` (local overrides global). + + Returns: + Configuration dict from files only. Empty when no file or section exists. + """ + config = configparser.ConfigParser() + global_path, local_path = _config_file_paths() + config.read([global_path, local_path]) + if "codecarbon" not in config: + return {} + return dict(config["codecarbon"]) + + def get_hierarchical_config(): """ Get the user-defined codecarbon configuration ConfigParser dictionnary @@ -137,13 +162,7 @@ def get_hierarchical_config(): dict: The final configuration dict parsed from global, local and environment configurations. **All values are strings**. """ - - config = configparser.ConfigParser() - - cwd = Path.cwd() - home = Path.home() - global_path = str((home / ".codecarbon.config").expanduser().resolve()) - local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + global_path, local_path = _config_file_paths() if Path(global_path).exists(): logger.info( f"Codecarbon is taking the configuration from global file: {global_path}" @@ -155,7 +174,6 @@ def get_hierarchical_config(): f"Codecarbon is taking the configuration from the local file {local_path}" ) - config.read([global_path, local_path]) - config.read_dict(parse_env_config()) - - return dict(config["codecarbon"]) + conf = get_config_file_settings() + conf.update(parse_env_config().get("codecarbon", {})) + return conf diff --git a/codecarbon/core/gpu_amd.py b/codecarbon/core/gpu_amd.py index 6cff5a287..9b1b3ddc1 100644 --- a/codecarbon/core/gpu_amd.py +++ b/codecarbon/core/gpu_amd.py @@ -28,14 +28,12 @@ def is_rocm_system(): "Please install amdsmi to get GPU metrics." ) AMDSMI_AVAILABLE = False -except AttributeError as e: +except (AttributeError, OSError, KeyError) as error: amdsmi = None - # In some environments, amdsmi may be present but not properly configured, leading to AttributeError when importing logger.warning( - "AMD GPU detected but amdsmi is not properly configured. " - "Please ensure amdsmi is correctly installed to get GPU metrics." - "Tips : check consistency between Python amdsmi package and ROCm versions, and ensure AMD drivers are up to date." - f" Error: {e}" + "amdsmi is installed but could not load libamd_smi (ROCm drivers missing " + "or incompatible). AMD GPU metrics will be unavailable. Error: %s", + error, ) AMDSMI_AVAILABLE = False diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py index 8dfdad05a..6cf821f28 100644 --- a/codecarbon/core/telemetry_client.py +++ b/codecarbon/core/telemetry_client.py @@ -14,14 +14,23 @@ class TelemetryClient: def __init__( self, - endpoint_url="https://api.codecarbon.io", + endpoint_url: str = "https://api.codecarbon.io", telemetry: Optional[Union[TelemetryCreate, dict]] = None, + api_key: Optional[str] = None, ): self.endpoint_url = endpoint_url.rstrip("/") self.telemetry_url = self.endpoint_url + "/telemetry" - self.headers = {"Content-Type": "application/json"} + self.api_key = api_key + self.headers = self._build_headers(api_key) self.telemetry = self._validate_telemetry(telemetry) if telemetry else None + @staticmethod + def _build_headers(api_key: Optional[str]) -> dict[str, str]: + headers = {"Content-Type": "application/json"} + if api_key: + headers["x-api-token"] = api_key + return headers + def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None): telemetry_payload = ( self._validate_telemetry(telemetry) if telemetry else self.telemetry @@ -38,6 +47,12 @@ def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None timeout=2, headers=self.headers, ) + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + self.telemetry_url, + ) + return None if response.status_code != 201: self._log_error(payload, response) return None diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 406c46bec..46d79759f 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -17,7 +17,18 @@ import psutil from codecarbon._version import __version__ -from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids +from codecarbon.core.config import ( + get_config_file_settings, + get_hierarchical_config, + normalize_gpu_ids, +) +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import resolve_telemetry_level +from codecarbon.telemetry import ( + send_tier1_telemetry, + send_tier2_public_emission, + warn_if_telemetry_not_configured, +) from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker from codecarbon.core.units import Energy, Power, Time, Water @@ -199,7 +210,7 @@ def __init__( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, - send_telemetry: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name @@ -273,10 +284,16 @@ def __init__( (CPU + chipset + PCIe). When False, uses package domains which are more reliable. Note: psys can report higher values than CPU TDP and may be unreliable on older systems. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). + Overrides ``telemetry_level`` in ``.codecarbon.config`` when set. """ - # logger.info("base tracker init") self._external_conf = get_hierarchical_config() + self._config_file_conf = get_config_file_settings() + telemetry_override = None if telemetry_level is _sentinel else telemetry_level + self._telemetry_level = resolve_telemetry_level( + self._config_file_conf, override=telemetry_override + ) self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", True, bool) if self._allow_multiple_runs: logger.warning( @@ -353,9 +370,6 @@ def __init__( self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) - # Tier 1 telemetry (opt-out, default enabled) - self._set_from_conf(send_telemetry, "send_telemetry", default=True, return_type=bool) - assert self._tracking_mode in ["machine", "process"] set_logger_level(self._log_level) set_logger_format(self._logger_preamble) @@ -452,13 +466,29 @@ def __init__( self._data_source, self._electricitymaps_api_token ) - # Send Tier 1 telemetry if enabled - if self._send_telemetry: - from codecarbon.telemetry import send_tier1_telemetry - send_tier1_telemetry(self._conf) - + self._apply_init_telemetry(telemetry_override) self._init_output_methods(api_key=self._api_key) + @suppress(Exception) + def _apply_init_telemetry(self, telemetry_override: str | None) -> None: + warn_if_telemetry_not_configured( + self._config_file_conf, + self._telemetry_level, + override=telemetry_override, + external_conf=self._external_conf, + ) + if self._telemetry_level in (TelemetryLevel.minimal, TelemetryLevel.extensive): + send_tier1_telemetry(self._conf, external_conf=self._external_conf) + + @suppress(Exception) + def _maybe_send_tier2_telemetry(self, emissions_data_delta: EmissionsData) -> None: + if self._telemetry_level == TelemetryLevel.extensive: + send_tier2_public_emission( + self._conf, + emissions_data_delta, + external_conf=self._external_conf, + ) + def _init_output_methods(self, *, api_key: str = None): """ Prepare the different output methods @@ -760,6 +790,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) + self._maybe_send_tier2_telemetry(emissions_data_delta) self._persist_data( total_emissions=emissions_data, @@ -1145,7 +1176,6 @@ def __init__( cloud_provider: Optional[str] = _sentinel, cloud_region: Optional[str] = _sentinel, country_2letter_iso_code: Optional[str] = _sentinel, - send_telemetry: Optional[bool] = _sentinel, **kwargs, ): """ @@ -1218,7 +1248,7 @@ def __init__( assert isinstance(self._country_2letter_iso_code, str) self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper() - super().__init__(*args, send_telemetry=send_telemetry, **kwargs) + super().__init__(*args, **kwargs) def _get_geo_metadata(self) -> GeoMetadata: return GeoMetadata( @@ -1323,6 +1353,7 @@ def track_emissions( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ Decorator that supports both `EmissionsTracker` and `OfflineEmissionsTracker` @@ -1406,6 +1437,7 @@ def track_emissions( When True, measures CPU package + DRAM. :param rapl_prefer_psys: Prefer psys over package domains for RAPL on Linux (default: False). When True, uses total platform power. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). :return: The decorated function """ @@ -1460,6 +1492,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) else: tracker = EmissionsTracker( @@ -1494,6 +1527,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) tracker.start() try: diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index eb4efb9b7..7bcbd6373 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,23 +1,64 @@ -"""Tracker-facing telemetry helpers built on the shared TelemetryClient.""" +"""Tracker-facing telemetry helpers (Tier 1 HTTP, Tier 2 public emissions).""" -import os +import dataclasses from datetime import datetime, timezone from typing import Any +from codecarbon.core.api_client import ApiClient from codecarbon.core.telemetry_client import TelemetryClient from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import ( + get_telemetry_api_key, + get_telemetry_api_url, + get_telemetry_experiment_id, + is_telemetry_level_explicit, +) from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData -TELEMETRY_API_KEY = os.environ.get( - "CODECARBON_TELEMETRY_API_KEY", - "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU", -) -TELEMETRY_API_URL = os.environ.get( - "CODECARBON_TELEMETRY_API_URL", "https://api.codecarbon.io" +_TIER1_SENT = False +_TIER2_SENT = False +_TELEMETRY_CONFIGURE_WARNED = False + +TELEMETRY_NOT_CONFIGURED_MESSAGE = ( + "CodeCarbon telemetry_level was not set explicitly; using default %r. " + "Tier 1 minimal telemetry (hardware and environment metadata) will be " + "sent once per Python session. Set telemetry_level in .codecarbon.config " + "(disabled, minimal, or extensive), pass telemetry_level=... to " + "EmissionsTracker / OfflineEmissionsTracker, or set CODECARBON_TELEMETRY_LEVEL " + "to opt out (disabled), keep minimal telemetry, or enable extensive (public " + "emissions on stop)." ) -TELEMETRY_EXPERIMENT_ID = None -_TIER1_SENT = False + +def warn_if_telemetry_not_configured( + config_file_conf: dict[str, Any], + active_level: TelemetryLevel, + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> None: + """Log a one-time warning when telemetry tier was not set explicitly. + + Args: + config_file_conf: File-only settings from ``get_config_file_settings()``. + active_level: Resolved tier in use for this tracker. + override: Optional ``telemetry_level`` tracker argument. + external_conf: Merged file/env settings for explicit-env detection. + """ + global _TELEMETRY_CONFIGURE_WARNED + if _TELEMETRY_CONFIGURE_WARNED: + return + if is_telemetry_level_explicit( + config_file_conf, override=override, external_conf=external_conf + ): + return + logger.warning( + TELEMETRY_NOT_CONFIGURED_MESSAGE, + active_level.value, + ) + _TELEMETRY_CONFIGURE_WARNED = True + def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: """Build a minimal telemetry payload dict from tracker configuration. @@ -50,38 +91,80 @@ def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in payload.items() if value is not None} -collect_tier1_payload = build_minimal_telemetry_dict - - def send_tier1_telemetry( - conf: dict[str, Any], endpoint_url: str | None = None + conf: dict[str, Any], + external_conf: dict[str, Any] | None = None, ) -> bool: - """Send minimal telemetry once per Python session. + """POST minimal telemetry once per Python session via ``TelemetryClient``. + + Sends ``POST {telemetry_api_url}/telemetry`` with a ``TelemetryCreate`` payload + built from tracker configuration. Best-effort: failures are logged and not raised. Args: conf: Tracker configuration dictionary. - endpoint_url: Optional API base URL override. + external_conf: Merged file/env config for telemetry API URL and key resolution. Returns: - True if telemetry was sent successfully on this call, False if already - sent in this session or if sending failed. + True if telemetry was posted successfully on this call, False if already sent + in this session or if the request failed. """ global _TIER1_SENT if _TIER1_SENT: return False + settings_conf = external_conf or {} try: payload = build_minimal_telemetry_dict(conf) + endpoint_url = get_telemetry_api_url(settings_conf) client = TelemetryClient( - endpoint_url=endpoint_url or TELEMETRY_API_URL, + endpoint_url=endpoint_url, telemetry=payload, + api_key=get_telemetry_api_key(settings_conf), ) - if TELEMETRY_API_KEY: - client.headers["x-api-token"] = TELEMETRY_API_KEY - result = client.add_telemetry() - if result is not None: + response = client.add_telemetry() + if response is not None: _TIER1_SENT = True return True return False except Exception as error: logger.error(f"Telemetry Tier 1 failed (non-critical): {error}") return False + + +def send_tier2_public_emission( + conf: dict[str, Any], + emissions_data: EmissionsData, + external_conf: dict[str, Any] | None = None, +) -> bool: + """Send run emissions to the public telemetry experiment via ApiClient. + + Mirrors ``CodeCarbonAPIOutput`` / ``add_emission`` for the shared leaderboard + project. Best-effort: errors are logged and not raised. + + Args: + conf: Tracker configuration dictionary. + emissions_data: Delta or total emissions row for the run. + external_conf: Merged file/env config for API URL, key, and experiment id. + + Returns: + True if emission was posted successfully on this call, False otherwise. + """ + global _TIER2_SENT + if _TIER2_SENT: + return False + settings_conf = external_conf or {} + try: + api = ApiClient( + endpoint_url=get_telemetry_api_url(settings_conf), + experiment_id=get_telemetry_experiment_id(settings_conf), + api_key=get_telemetry_api_key(settings_conf), + conf=conf, + create_run_automatically=True, + ) + posted = api.add_emission(dataclasses.asdict(emissions_data)) + if posted: + _TIER2_SENT = True + return True + return False + except Exception as error: + logger.error(f"Telemetry Tier 2 failed (non-critical): {error}") + return False diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index e8b42b929..029135d65 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -107,3 +107,9 @@ os.environ["HTTPS_PROXY"] = "http://0.0.0.0:0000" For more information, please read the [requests library proxy documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) + +## Product telemetry + +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. + +See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 7a19bafa9..c5a0dda36 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -1,104 +1,107 @@ # Product telemetry -CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people use, and (if you opt in) anonymous run emissions on a public leaderboard. +CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people run on, not what your code does. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. -This is **separate from** your own dashboard setup (`codecarbon config`, `codecarbon login`, `save_to_api`). Those commands configure **your** project and experiments. Product telemetry uses the shared settings below. +## Telemetry vs your dashboard data -## Telemetry tiers +| | Product telemetry | Your emissions (`save_to_api`) | +|--|-------------------|--------------------------------| +| Purpose | Improve CodeCarbon (aggregate usage) | Your projects and experiments | +| Config | `telemetry_level`, `codecarbon telemetry` | `codecarbon config`, `experiment_id` | +| Default experiment | Public telemetry project (built-in defaults) | Your account / experiment | -| Tier | `telemetry_level` | When | What is sent | -|------|-------------------|------|----------------| -| **0** | `disabled` | — | Nothing | -| **1** | `minimal` (default) | Once per Python process, when the tracker starts | Minimal hardware / environment metadata (see below) | -| **2** | `extensive` | Tier 1 on start **and** Tier 2 on `stop()` | Tier 1 plus one public emissions row for the run | +You can use one without the other. -If you never set `telemetry_level`, CodeCarbon uses **`minimal`** and logs a **one-time warning** per Python session telling you that Tier 1 will be sent. +## Tiers -## Tier 1: what we collect today +| `telemetry_level` | What happens | +|-------------------|--------------| +| `disabled` | No product telemetry | +| `minimal` (default) | **Tier 1** once per Python process: minimal hardware/environment metadata | +| `extensive` | Tier 1 + **Tier 2** on tracker `stop()`: one public emissions row (leaderboard-style) | -Tier 1 sends a single `POST` to `{telemetry_api_url}/telemetry` the first time an `EmissionsTracker` or `OfflineEmissionsTracker` is created in a process (not on every run). +Tier is resolved in this order: -Only the fields below are included. Any value that is unknown is **omitted** from the payload (not sent as `null`). +1. `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` +2. `telemetry_level` in `.codecarbon.config` (local overrides global) +3. Default: `minimal` + +Environment variables `CODECARBON_TELEMETRY` / `CODECARBON_TELEMETRY_LEVEL` count as “explicit” configuration (they suppress the one-time setup warning) but **do not** change the tier unless you also set `telemetry_level` in a config file or pass the tracker argument. + +## Tier 1: what we collect (and what we do not) + +Tier 1 is intentionally small. The client only builds a **minimal** payload; the API schema rejects “extensive” fields when `telemetry_level` is `minimal`. + +### Sent at most once per process (if known) + +Only non-empty values are included: | Field | Description | |-------|-------------| -| `timestamp` | UTC time when the tracker was initialized | +| `timestamp` | UTC time of the send | | `telemetry_level` | Always `minimal` for this tier | -| `os` | Operating system string (e.g. platform description) | -| `country_iso_code` | ISO country code when known (e.g. offline mode or geo lookup) | -| `region` | Region or cloud region when known | -| `cloud_provider` | Cloud provider name when known | -| `cloud_region` | Same as `region` when cloud metadata is available | -| `longitude` | Approximate longitude when known (degrees) | -| `latitude` | Approximate latitude when known (degrees) | -| `cpu_count` | Logical CPU count | -| `cpu_physical_count` | Physical CPU count | -| `cpu_model` | CPU model name | -| `gpu_count` | Number of GPUs detected | -| `gpu_model` | GPU model name(s) | -| `ram_total_size_gb` | Total RAM in GB | -| `python_version` | Python version string | +| `os` | Platform string | +| `country_iso_code` | Country (e.g. from offline mode or geo) | +| `region` | Region / province | +| `cloud_provider` | Cloud provider name, if detected | +| `cloud_region` | Cloud region, if detected | +| `longitude`, `latitude` | Approximate location, if geo resolution ran | +| `cpu_count`, `cpu_physical_count`, `cpu_model` | CPU metadata | +| `gpu_count`, `gpu_model` | GPU metadata | +| `ram_total_size_gb` | Total RAM | +| `python_version` | Python version | | `codecarbon_version` | Installed CodeCarbon version | -### Tier 1: what we do **not** collect yet - -The API schema supports more “minimal” fields (framework versions, install method, executable hash, etc.). **The client does not send them today.** In particular, Tier 1 does **not** include: +### Not sent in Tier 1 -- Run duration, energy, or CO₂ emissions -- CPU/GPU utilization or power samples -- Project, experiment, or user identifiers from your dashboard config -- Python executable hash, virtualenv type, or ML framework versions -- IDE, CI, or notebook environment metadata +Tier 1 does **not** include: -If we add fields later, this page will be updated; the payload will stay limited to the minimal tier rules on the server. +- Emissions, energy, power, duration, or utilization +- Project name, experiment id, run id, or API keys +- Source code, file paths, hostnames, or user ids +- ML stack versions (PyTorch, TensorFlow, etc.) +- Output methods, tracking mode, or internal diagnostics +- Anything else defined as “extensive” in the telemetry schema -## Tier 2: extensive (public leaderboard) +Tier 2 (`extensive`) adds a single **public** emissions summary via the same mechanism as `add_emission` to the shared telemetry experiment—not a full extensive telemetry document. -When `telemetry_level = extensive`, CodeCarbon still sends Tier 1 once per process, and on **`stop()`** posts **one** emissions summary to the shared telemetry experiment via the same API used for leaderboard data (`add_emission`). That is independent of `save_to_api` (your private dashboard uploads). +### Transport -Use this only if you are comfortable contributing anonymous run-level emissions to the public experiment. +- HTTP `POST` to `{telemetry_api_url}/telemetry` +- Best-effort: failures are logged and do not stop your tracker +- If the endpoint is not deployed yet, you may see a warning (HTTP 404) ## Configure telemetry ### Config file -Add to `~/.codecarbon.config` and/or `./.codecarbon.config`: - ```ini [codecarbon] telemetry_level = minimal +# Optional overrides (defaults point at the public telemetry API): +# telemetry_api_url = https://api.codecarbon.io +# telemetry_api_key = ... +# telemetry_experiment_id = ... ``` -Allowed values: `disabled`, `minimal`, `extensive`. - -Optional API overrides (defaults point at the public telemetry project): - -```ini -telemetry_api_url = https://api.codecarbon.io -telemetry_api_key = cpt_... -telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 -``` - -Environment variables for URL, key, and experiment id: `CODECARBON_TELEMETRY_API_URL`, `CODECARBON_TELEMETRY_API_KEY`, `CODECARBON_TELEMETRY_EXPERIMENT_ID`. - -**Tier resolution:** `telemetry_level` in the config file, or the tracker / CLI override below. Environment variables such as `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` can mark your choice as “explicit” (so the default warning is skipped) but **do not** change the tier unless the same value is also in the config file or passed to the tracker. - ### CLI ```bash -# Interactive wizard +# Interactive wizard (pick config file + tier) codecarbon telemetry -# Set tier in a config file +# Set tier in config codecarbon telemetry set disabled +codecarbon telemetry set minimal +codecarbon telemetry set extensive + +# Show resolved tier (merged global + local config) codecarbon telemetry show -# One-run override (does not write the config file) -codecarbon monitor --telemetry-level minimal -- python train.py +# One-run override (does not write config) +codecarbon monitor --telemetry-level disabled -- python train.py ``` -See the [CLI reference](../reference/cli.md#codecarbon-telemetry) for details. - ### Python ```python @@ -107,23 +110,29 @@ from codecarbon import EmissionsTracker tracker = EmissionsTracker(telemetry_level="disabled") ``` -`OfflineEmissionsTracker`, `@track_emissions`, and `codecarbon monitor` accept the same `telemetry_level` argument. +## Opt out -### Disable telemetry in tests +Set any of: ```ini [codecarbon] telemetry_level = disabled ``` -## Privacy notes +```bash +codecarbon telemetry set disabled +``` + +```python +EmissionsTracker(telemetry_level="disabled") +``` + +## First run without explicit configuration -- Tier 1 is **best-effort**: failures are logged and do not block tracking. -- Coordinates are only sent when the tracker already resolved them; they are not precise location tracking by themselves. -- Choose `disabled` if you do not want any product telemetry. -- Choose `extensive` only if you accept publishing one emissions row per process to the public telemetry experiment on stop. +If you never set `telemetry_level`, CodeCarbon uses `minimal` and logs a **one-time warning** per Python session that Tier 1 will be sent. Set `telemetry_level` explicitly (config, CLI, or tracker argument) to silence it. ## Related -- [Configure CodeCarbon](configuration.md) — general config file and environment variables -- [Use the Cloud API & Dashboard](cloud-api.md) — your own projects and experiments +- [Configure CodeCarbon](configuration.md) — general `.codecarbon.config` hierarchy +- [CLI reference](../reference/cli.md#codecarbon-telemetry) — command flags +- [Cloud API & dashboard](cloud-api.md) — your own emissions data diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1e93a588f..6c550f625 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -45,6 +45,7 @@ Displays real-time emissions data for all processes on your machine. Press `Ctrl | `--offline` | flag | false | Run without internet access | | `--country-iso-code` | string | - | ISO 3166-1 alpha-3 country code (required in offline mode) | | `--log-level` | choice | INFO | Log level: DEBUG, INFO, WARNING, ERROR | +| `--telemetry-level` | string | - | One-run tier: `disabled`, `minimal`, or `extensive` | **Examples:** ```bash @@ -88,6 +89,34 @@ codecarbon monitor -- node app.js --port 8080 Same options as `codecarbon monitor` apply (see above). +### `codecarbon telemetry` + +Configure **product telemetry** (library usage metadata), separate from `codecarbon config` (dashboard org/project/experiment). + +**Usage:** + +```bash +codecarbon telemetry # interactive wizard +codecarbon telemetry set # disabled | minimal | extensive +codecarbon telemetry show # resolved tier and whether it was set explicitly +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--config PATH` | Use a specific `.codecarbon.config` (default: local then global) | + +See [Product telemetry](../how-to/telemetry.md) for what Tier 1 collects and how to opt out. + +### `codecarbon monitor --telemetry-level` + +Override the telemetry tier for a single monitor run (does not update config). + +```bash +codecarbon monitor --telemetry-level disabled -- python train.py +``` + ### `codecarbon detect` Detect and print hardware information. diff --git a/mkdocs.yml b/mkdocs.yml index b82641fb7..1de7399a8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -149,6 +149,7 @@ nav: - LLMs and Agents: how-to/agents.md - How-to Guides: - Configure CodeCarbon: how-to/configuration.md + - Product telemetry: how-to/telemetry.md - Compare Model Efficiency: tutorials/comparing-model-efficiency.md - Dashboard & Visualization: - Use the Cloud API & Dashboard: how-to/cloud-api.md diff --git a/tests/test_api_call.py b/tests/test_api_call.py index a4bb4cd7f..e0b61d01c 100644 --- a/tests/test_api_call.py +++ b/tests/test_api_call.py @@ -244,6 +244,22 @@ def test_add_emission_returns_false_on_unsuccessful_post(self): ) ) + def test_create_run_handles_none_coordinates(self): + conf_with_none_coords = {**conf, "longitude": None, "latitude": None} + with requests_mock.Mocker() as m: + m.post("http://test.com/runs", json={"id": "run-id"}, status_code=201) + api = ApiClient( + endpoint_url="http://test.com", + experiment_id="experiment_id", + api_key="Toto", + conf=conf_with_none_coords, + create_run_automatically=False, + ) + run_id = api._create_run("experiment_id") + self.assertEqual(run_id, "run-id") + self.assertEqual(m.last_request.json()["longitude"], 0.0) + self.assertEqual(m.last_request.json()["latitude"], 0.0) + def test_create_run_returns_none_on_unsuccessful_status(self): with requests_mock.Mocker() as m: m.post("http://test.com/runs", text="bad", status_code=400) diff --git a/tests/test_config.py b/tests/test_config.py index 181913c6c..1f431e236 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import configparser import os import unittest from textwrap import dedent @@ -16,6 +17,19 @@ from tests.testutils import get_custom_mock_open +def _file_settings_from_ini(global_conf: str, local_conf: str) -> dict[str, str]: + """Merge mocked global and local ``[codecarbon]`` sections like config files do.""" + merged: dict[str, str] = {} + for text in (global_conf, local_conf): + if not text.strip(): + continue + parser = configparser.ConfigParser() + parser.read_string(text) + if "codecarbon" in parser: + merged.update(dict(parser["codecarbon"])) + return merged + + class TestConfig(unittest.TestCase): def setUp(self): self._original_environ = os.environ.copy() @@ -23,6 +37,12 @@ def setUp(self): "CODECARBON_API_KEY", "CODECARBON_EXPERIMENT_ID", "CODECARBON_API_ENDPOINT", + "CODECARBON_TELEMETRY", + "CODECARBON_TELEMETRY_LEVEL", + "CODECARBON_TELEMETRY_PROJECT_TOKEN", + "CODECARBON_TELEMETRY_API_URL", + "CODECARBON_TELEMETRY_API_KEY", + "CODECARBON_TELEMETRY_EXPERIMENT_ID", "codecarbon_api_key", "codecarbon_experiment_id", "codecarbon_api_endpoint", @@ -115,7 +135,8 @@ def test_read_confs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -156,7 +177,8 @@ def test_read_confs_and_parse_envs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -171,12 +193,7 @@ def test_read_confs_and_parse_envs(self): self.assertDictEqual(conf, target) def test_empty_conf(self): - global_conf = "" - local_conf = "" - - with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) - ): + with patch("codecarbon.core.config.get_config_file_settings", return_value={}): conf = dict(get_hierarchical_config()) # allow_multiple_runs is set in pytest.ini and not mocked, so it's visible here. target = {"allow_multiple_runs": "True"} diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index b845962dd..2fe5a1199 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -35,7 +35,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" if sys.platform == "darwin": @@ -73,7 +73,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() @@ -99,7 +100,7 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( json=GEO_METADATA_CANADA, status=200, ) - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) # WHEN tracker.start() heavy_computation(run_time_secs=5) @@ -126,7 +127,7 @@ def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) mock_gpu = mock.MagicMock() from codecarbon.external.hardware import GPU @@ -164,7 +165,7 @@ def raise_timeout_exception(*args, **kwargs): mocked_requests_get.side_effect = raise_timeout_exception - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) # WHEN tracker.start() @@ -183,7 +184,7 @@ def test_graceful_start_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) def raise_exception(*args, **kwargs): raise Exception() @@ -202,7 +203,7 @@ def test_graceful_stop_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) def raise_exception(*args, **kwargs): raise Exception() @@ -346,7 +347,6 @@ def test_offline_tracker_country_name( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", - send_telemetry=False, ) tracker.start() heavy_computation(run_time_secs=2) @@ -370,7 +370,6 @@ def test_offline_tracker_invalid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", - send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_invalid_headers.csv" @@ -404,7 +403,6 @@ def test_offline_tracker_valid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", - send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_valid_headers.csv" @@ -448,7 +446,7 @@ def test_carbon_tracker_online_context_manager_TWO_GPU_PRIVATE_INFRA_CANADA( ) # WHEN - with EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) as tracker: + with EmissionsTracker(measure_power_secs=1, save_to_file=False) as tracker: heavy_computation(run_time_secs=5) # THEN @@ -611,7 +609,6 @@ def test_scheduler_warning_suppressed_when_stopped( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing - send_telemetry=False, ) as tracker: # Stop the scheduler to simulate task mode or manual stopping tracker._scheduler.stop() @@ -655,7 +652,6 @@ def test_scheduler_warning_shown_when_running( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing - send_telemetry=False, ) as tracker: # Ensure scheduler is running (default state) self.assertFalse(tracker._scheduler._stopped) diff --git a/tests/test_gpu.py b/tests/test_gpu.py index bfbc8e603..913c05017 100644 --- a/tests/test_gpu.py +++ b/tests/test_gpu.py @@ -153,7 +153,7 @@ def check_output(cmd, *args, **kwargs): sys.modules["pynvml"] = old_pynvml assert any( - "amdsmi is not properly configured" in str(c.args[0]) + "could not load libamd_smi" in str(c.args[0]) for c in warning_mock.call_args_list ) diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 7e2ff40ac..4b9b4b4ec 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -18,7 +18,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" class TestOfflineEmissionsTracker(unittest.TestCase): @@ -32,7 +32,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() @@ -41,7 +42,7 @@ def tearDown(self) -> None: self.temp_dir.cleanup() def test_offline_tracker(self): - tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path, send_telemetry=False) + tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path) tracker.start() heavy_computation(run_time_secs=2) tracker.stop() @@ -60,7 +61,7 @@ def test_offline_tracker(self): ) def test_offline_tracker_task(self): - tracker = OfflineEmissionsTracker(send_telemetry=False) + tracker = OfflineEmissionsTracker() tracker.start_task() heavy_computation(run_time_secs=2) task_emission_data = tracker.stop_task() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index f70979d66..7a235f76c 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -10,8 +10,8 @@ from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.telemetry import ( build_minimal_telemetry_dict, - collect_tier1_payload, send_tier1_telemetry, + send_tier2_public_emission, ) from tests.testutils import get_custom_mock_open @@ -20,11 +20,11 @@ "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" ) else: - mock_platform_cli_setup = patch( - "codecarbon.core.cpu.IntelPowerGadget._setup_cli" - ) + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" +minimal_conf = "[codecarbon]\ntelemetry_level = minimal\n" +extensive_conf = "[codecarbon]\ntelemetry_level = extensive\n" class TestTelemetry(unittest.TestCase): @@ -52,12 +52,9 @@ def test_build_minimal_telemetry_dict_maps_tracker_conf(self): self.assertEqual(payload["cloud_provider"], conf["provider"]) self.assertEqual(payload["cloud_region"], conf["region"]) - def test_collect_tier1_payload_delegates_to_builder(self): - self.assertIs(collect_tier1_payload, build_minimal_telemetry_dict) - class TestTier1Send(unittest.TestCase): - def test_send_tier1_telemetry_sends_once_per_session(self): + def test_send_tier1_telemetry_posts_once_per_session(self): telemetry_module._TIER1_SENT = False conf = { "python_version": "3.11.0", @@ -67,102 +64,188 @@ def test_send_tier1_telemetry_sends_once_per_session(self): "gpu_count": 0, "codecarbon_version": "2.0.0", } - with patch( - "codecarbon.core.telemetry_client.requests.post" - ) as mock_post: - mock_post.return_value = MagicMock( - status_code=201, json=lambda: "telemetry-id" - ) - result = send_tier1_telemetry(conf, endpoint_url="http://test.com") + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_client_cls.return_value = mock_client + result = send_tier1_telemetry(conf) self.assertTrue(result) - self.assertEqual(mock_post.call_count, 1) - body = mock_post.call_args.kwargs["json"] - self.assertEqual(body["telemetry_level"], "minimal") - result = send_tier1_telemetry(conf, endpoint_url="http://test.com") + result = send_tier1_telemetry(conf) self.assertFalse(result) - self.assertEqual(mock_post.call_count, 1) + mock_client_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + call_kwargs = mock_client_cls.call_args.kwargs + self.assertEqual( + call_kwargs["telemetry"]["telemetry_level"], TelemetryLevel.minimal.value + ) - def test_send_tier1_telemetry_fails_silently(self): + def test_send_tier1_telemetry_uses_resolved_api_url_and_key(self): telemetry_module._TIER1_SENT = False - conf = { - "python_version": "3.11.0", - "os": "Linux", - "cpu_count": 4, - "codecarbon_version": "2.0.0", + external_conf = { + "telemetry_api_url": "http://custom-tier1.example", + "telemetry_api_key": "cpt_custom", } + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_client_cls.return_value = mock_client + send_tier1_telemetry( + {"codecarbon_version": "2.0"}, external_conf=external_conf + ) + self.assertEqual( + mock_client_cls.call_args.kwargs["endpoint_url"], + "http://custom-tier1.example", + ) + self.assertEqual(mock_client_cls.call_args.kwargs["api_key"], "cpt_custom") + + def test_send_tier1_telemetry_fails_silently(self): + telemetry_module._TIER1_SENT = False with patch( - "codecarbon.core.telemetry_client.requests.post", - side_effect=Exception("network error"), + "codecarbon.telemetry.build_minimal_telemetry_dict", + side_effect=RuntimeError("build failed"), ): - with patch("codecarbon.core.telemetry_client.logger.error") as mock_logger: - result = send_tier1_telemetry(conf, endpoint_url="http://test.com") + with patch("codecarbon.telemetry.logger.error") as mock_logger: + result = send_tier1_telemetry({}) self.assertFalse(result) mock_logger.assert_called() +class TestTier2Send(unittest.TestCase): + def test_send_tier2_public_emission_sends_once_per_session(self): + telemetry_module._TIER2_SENT = False + from codecarbon.output_methods.emissions_data import EmissionsData + + emissions = EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=1.0, + emissions=0.001, + emissions_rate=0.001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="", + country_iso_code="", + region="", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + result = send_tier2_public_emission({}, emissions) + self.assertTrue(result) + mock_api.add_emission.assert_called_once() + result = send_tier2_public_emission({}, emissions) + self.assertFalse(result) + mock_api.add_emission.assert_called_once() + + @mock_platform_cli_setup class TestTrackerTelemetry(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() self.temp_path = Path(self.temp_dir.name) - self.patcher = patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) - ) - self.mock_open = self.patcher.start() + self.patcher = None def tearDown(self) -> None: - self.patcher.stop() + if self.patcher: + self.patcher.stop() self.temp_dir.cleanup() - def test_emissions_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + def _start_config_mock(self, conf: str) -> None: + self.patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self.patcher.start() + + def test_emissions_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch( - "codecarbon.core.telemetry_client.requests.post" - ) as mock_post: - mock_post.return_value = MagicMock( - status_code=201, json=lambda: "telemetry-id" - ) + self._start_config_mock(minimal_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker( - send_telemetry=True, save_to_api=False, save_to_file=False - ) - self.assertTrue(mock_post.called) + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() - def test_emissions_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + def test_emissions_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + self._start_config_mock(disabled_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker( - send_telemetry=False, save_to_api=False, save_to_file=False - ) - self.assertFalse(mock_post.called) + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_telemetry_cls.assert_not_called() - def test_offline_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + def test_offline_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch( - "codecarbon.core.telemetry_client.requests.post" - ) as mock_post: - mock_post.return_value = MagicMock( - status_code=201, json=lambda: "telemetry-id" - ) + self._start_config_mock(minimal_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client OfflineEmissionsTracker( country_iso_code="CAN", - send_telemetry=True, save_to_api=False, save_to_file=False, ) - self.assertTrue(mock_post.called) + mock_telemetry_cls.assert_called_once() - def test_offline_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + def test_offline_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + self._start_config_mock(disabled_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: OfflineEmissionsTracker( country_iso_code="CAN", - send_telemetry=False, save_to_api=False, save_to_file=False, ) - self.assertFalse(mock_post.called) + mock_telemetry_cls.assert_not_called() + + def test_extensive_sends_tier1_on_init_and_tier2_on_stop(self, mock_cli_setup): + telemetry_module._TIER1_SENT = False + telemetry_module._TIER2_SENT = False + self._start_config_mock(extensive_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_telemetry = MagicMock() + mock_telemetry.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_telemetry + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() + mock_telemetry.add_telemetry.assert_called_once() + mock_api.add_emission.assert_called_once() if __name__ == "__main__": diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 14ade8b7f..5ce145370 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch import requests_mock from pydantic import ValidationError @@ -102,3 +103,41 @@ def test_add_telemetry_returns_none_without_payload(self): client = TelemetryClient(endpoint_url="http://test.com") self.assertIsNone(client.add_telemetry()) + + def test_add_telemetry_logs_warning_on_404(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with requests_mock.Mocker() as m: + m.post( + "http://test.com/telemetry", + text='{"detail":"Not Found"}', + status_code=404, + ) + with patch("codecarbon.core.telemetry_client.logger") as mock_logger: + client = TelemetryClient( + endpoint_url="http://test.com", telemetry=telemetry + ) + result = client.add_telemetry() + self.assertIsNone(result) + mock_logger.warning.assert_called_once() + + def test_add_telemetry_sends_api_key_header_when_configured(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with requests_mock.Mocker() as m: + m.post( + "http://test.com/telemetry", + json="telemetry-id", + status_code=201, + ) + client = TelemetryClient( + endpoint_url="http://test.com", + telemetry=telemetry, + api_key="cpt_test_key", + ) + client.add_telemetry() + self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") From 2a865f6abfebc054ee7cc891e345b74a72e6acb6 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:44:00 +0200 Subject: [PATCH 10/29] refactor: enhance telemetry level resolution and configuration handling - Updated `resolve_telemetry_level` to incorporate external configuration, allowing for more flexible telemetry tier resolution. - Improved handling of telemetry level overrides from environment variables and configuration files. - Enhanced documentation to clarify telemetry configuration options and resolution order. - Added tests to validate new behavior for telemetry level resolution and ensure correct precedence of settings. --- codecarbon/cli/telemetry_cli.py | 5 +++- codecarbon/core/telemetry_settings.py | 37 ++++++++++++++++++--------- codecarbon/emissions_tracker.py | 6 +++-- codecarbon/telemetry.py | 9 +++---- docs/how-to/configuration.md | 2 +- docs/how-to/telemetry.md | 8 +++--- docs/reference/api.md | 14 ++++++++++ tests/test_telemetry_config.py | 31 +++++++++++++++++++--- tests/test_telemetry_settings.py | 23 +++++++++++++++++ 9 files changed, 107 insertions(+), 28 deletions(-) diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py index 218f2e941..e32cd730b 100644 --- a/codecarbon/cli/telemetry_cli.py +++ b/codecarbon/cli/telemetry_cli.py @@ -142,7 +142,10 @@ def print_telemetry_status(config_path: Optional[Path] = None) -> None: external_conf = get_hierarchical_config() source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" - level = resolve_telemetry_level(file_settings) + level = resolve_telemetry_level( + file_settings, + external_conf=external_conf or None, + ) explicit = is_telemetry_level_explicit(file_settings, external_conf=external_conf) stored = file_settings.get("telemetry_level") print(f"Config source: {source_label}") diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index ee78970c4..9e6d5ad91 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -69,29 +69,42 @@ def is_telemetry_level_explicit( def resolve_telemetry_level( - config_file_conf: dict[str, Any], + config_file_conf: dict[str, Any] | None = None, *, override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, ) -> TelemetryLevel: """Resolve the active telemetry tier. - Precedence: tracker ``telemetry_level`` argument, then ``telemetry_level`` in - ``.codecarbon.config``. Environment variables do not change the tier unless - passed as ``override`` from a future CLI integration. + Precedence: + + 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or + ``codecarbon monitor --telemetry-level`` + 2. ``external_conf`` — merged ``.codecarbon.config`` and ``CODECARBON_*`` env + (environment overrides file for the same key) + 3. ``config_file_conf`` — file-only settings when ``external_conf`` is omitted + 4. Default: ``minimal`` + + Legacy ``CODECARBON_TELEMETRY`` / config key ``telemetry`` only affect whether + the tier counts as explicitly configured, not tier resolution. Args: - config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). - override: Optional tier from ``EmissionsTracker(telemetry_level=...)``. + config_file_conf: Settings from ``get_config_file_settings()`` (optional). + override: Optional tier from tracker or CLI. + external_conf: Merged settings from ``get_hierarchical_config()`` (optional). Returns: Resolved ``TelemetryLevel``. """ - raw = ( - override - if override is not None - else config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) - ) - if raw is None: + if override is not None: + raw = override + elif external_conf is not None and external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + raw = external_conf[TELEMETRY_LEVEL_CONFIG_KEY] + elif config_file_conf is not None and config_file_conf.get( + TELEMETRY_LEVEL_CONFIG_KEY + ) is not None: + raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + else: return DEFAULT_TELEMETRY_LEVEL try: return parse_telemetry_level(raw) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 46d79759f..d88892d66 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -285,14 +285,16 @@ def __init__( are more reliable. Note: psys can report higher values than CPU TDP and may be unreliable on older systems. :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). - Overrides ``telemetry_level`` in ``.codecarbon.config`` when set. + Overrides config file and ``CODECARBON_TELEMETRY_LEVEL`` when set. """ self._external_conf = get_hierarchical_config() self._config_file_conf = get_config_file_settings() telemetry_override = None if telemetry_level is _sentinel else telemetry_level self._telemetry_level = resolve_telemetry_level( - self._config_file_conf, override=telemetry_override + self._config_file_conf, + override=telemetry_override, + external_conf=self._external_conf, ) self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", True, bool) if self._allow_multiple_runs: diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 7bcbd6373..ad4946e0c 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -23,11 +23,10 @@ TELEMETRY_NOT_CONFIGURED_MESSAGE = ( "CodeCarbon telemetry_level was not set explicitly; using default %r. " "Tier 1 minimal telemetry (hardware and environment metadata) will be " - "sent once per Python session. Set telemetry_level in .codecarbon.config " - "(disabled, minimal, or extensive), pass telemetry_level=... to " - "EmissionsTracker / OfflineEmissionsTracker, or set CODECARBON_TELEMETRY_LEVEL " - "to opt out (disabled), keep minimal telemetry, or enable extensive (public " - "emissions on stop)." + "sent once per Python session. Set telemetry_level in .codecarbon.config, " + "set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... to " + "EmissionsTracker / OfflineEmissionsTracker, or run " + "codecarbon telemetry set ." ) diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index 029135d65..34e32b2bb 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -110,6 +110,6 @@ documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) ## Product telemetry -Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index c5a0dda36..a99d42930 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -22,11 +22,11 @@ You can use one without the other. Tier is resolved in this order: -1. `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` -2. `telemetry_level` in `.codecarbon.config` (local overrides global) -3. Default: `minimal` +1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` (highest priority) +2. **Config + environment** — `telemetry_level` in `.codecarbon.config` (local overrides global), then `CODECARBON_TELEMETRY_LEVEL` overrides the file value when both are set (same rules as other `CODECARBON_*` settings) +3. **Default:** `minimal` -Environment variables `CODECARBON_TELEMETRY` / `CODECARBON_TELEMETRY_LEVEL` count as “explicit” configuration (they suppress the one-time setup warning) but **do not** change the tier unless you also set `telemetry_level` in a config file or pass the tracker argument. +Legacy `CODECARBON_TELEMETRY` / config key `telemetry` suppress the one-time setup warning when set, but do **not** set the tier (use `telemetry_level` or `CODECARBON_TELEMETRY_LEVEL`). ## Tier 1: what we collect (and what we do not) diff --git a/docs/reference/api.md b/docs/reference/api.md index 15117b123..37d2abea0 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -12,6 +12,20 @@ Parameters can be set via `EmissionsTracker()`, `OfflineEmissionsTracker()`, the If you use `CUDA_VISIBLE_DEVICES` or `ROCR_VISIBLE_DEVICES` to set GPUs, CodeCarbon will automatically populate `gpu_ids`. Manual `gpu_ids` overrides this. +## Product telemetry + +Optional library telemetry is controlled by **`telemetry_level`** on the tracker (same parameter on `OfflineEmissionsTracker` and `@track_emissions`): + +| Value | Behavior | +|-------|----------| +| `disabled` | No product telemetry | +| `minimal` (default) | Tier 1 hardware/environment metadata once per process | +| `extensive` | Tier 1 + public emissions summary on `stop()` | + +**Resolution order:** tracker argument → `.codecarbon.config` → `CODECARBON_TELEMETRY_LEVEL` → default `minimal`. The tracker argument overrides config and environment. + +This is separate from `save_to_api` (your dashboard experiment). See [Product telemetry](../how-to/telemetry.md). + ## EmissionsTracker / BaseEmissionsTracker `EmissionsTracker` and `OfflineEmissionsTracker` inherit from `BaseEmissionsTracker`. diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 41cbc26d0..57845f32f 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -81,7 +81,7 @@ def test_tier1_posts_to_telemetry_endpoint(self): TelemetryLevel.minimal.value, ) - def test_env_codecarbon_telemetry_does_not_change_tier(self): + def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): with tempfile.TemporaryDirectory() as tmp: local_path = Path(tmp) / ".codecarbon.config" local_path.write_text(_conf("minimal")) @@ -94,10 +94,35 @@ def test_env_codecarbon_telemetry_does_not_change_tier(self): {"CODECARBON_TELEMETRY": "disabled"}, clear=False, ): - file_settings = get_config_file_settings() - level = resolve_telemetry_level(file_settings) + from codecarbon.core.config import get_hierarchical_config + + level = resolve_telemetry_level( + get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) self.assertEqual(level, TelemetryLevel.minimal) + def test_env_codecarbon_telemetry_level_overrides_file(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, + clear=False, + ): + from codecarbon.core.config import get_hierarchical_config + + level = resolve_telemetry_level( + get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) + self.assertEqual(level, TelemetryLevel.disabled) + def test_telemetry_api_url_env_used_for_tier2_client(self): with patch.dict( os.environ, diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py index 68254999c..c09582c62 100644 --- a/tests/test_telemetry_settings.py +++ b/tests/test_telemetry_settings.py @@ -60,6 +60,29 @@ def test_override_kwarg_takes_precedence_over_config_file(self): ) self.assertEqual(level, TelemetryLevel.disabled) + def test_override_kwarg_takes_precedence_over_external_conf(self): + level = resolve_telemetry_level( + external_conf={"telemetry_level": "extensive"}, + override="disabled", + ) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_external_conf_env_overrides_file_when_merged(self): + level = resolve_telemetry_level( + {"telemetry_level": "minimal"}, + external_conf={"telemetry_level": "disabled"}, + ) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_env_telemetry_level_via_external_conf(self): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, clear=False + ): + from codecarbon.core.config import get_hierarchical_config + + level = resolve_telemetry_level(external_conf=get_hierarchical_config()) + self.assertEqual(level, TelemetryLevel.disabled) + def test_is_explicit_with_config_file(self): self.assertTrue(is_telemetry_level_explicit({"telemetry_level": "minimal"})) From 1ffc45dd2b8554943e1ad973657756e9bb520689 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Tue, 19 May 2026 21:46:57 +0200 Subject: [PATCH 11/29] Delete docs/plans/2026-05-19-telemetry-configuration.md --- .../2026-05-19-telemetry-configuration.md | 270 ------------------ 1 file changed, 270 deletions(-) delete mode 100644 docs/plans/2026-05-19-telemetry-configuration.md diff --git a/docs/plans/2026-05-19-telemetry-configuration.md b/docs/plans/2026-05-19-telemetry-configuration.md deleted file mode 100644 index 82bc5e6c1..000000000 --- a/docs/plans/2026-05-19-telemetry-configuration.md +++ /dev/null @@ -1,270 +0,0 @@ -# Telemetry Tier Configuration Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use executing-plans to implement this plan task-by-task. - -**Goal:** Configure telemetry tiers (`disabled`, `minimal`, `extensive`) with clear defaults, explicit opt-in/out, and optional public emissions on stop. Aligned with issue #1106 and `TelemetryLevel` from PR #1171. - -**Architecture (shipped):** Tier resolution: **tracker `telemetry_level` kwarg** (if set) → else **`telemetry_level` in `.codecarbon.config`**. Env `CODECARBON_TELEMETRY` / `telemetry` does **not** change the tier (only counts as “explicit” for the configure warning). API URL/key/experiment: config → env → `DEFAULT_*` in `telemetry_settings.py`. **Tier 1:** `TelemetryClient` → `POST /telemetry` once per session on init (404 logged as warning until prod deploy). **Tier 2:** `ApiClient.add_emission` to public experiment on `stop()` when `extensive`. One-time **warning** if tier was not set explicitly. No `send_telemetry` bool. - -**Tech Stack:** Python 3.11+, Pydantic v2, `configparser`, Typer CLI, `pytest`, `requests` / `requests_mock`. - -**Base branch:** `feat/add-telemetry` (stacked on `feat/telemetry-backend`, PR #1200) - -**Related:** -- Issue: [#1106](https://github.com/mlco2/codecarbon/issues/1106) -- Backend: [PR #1171](https://github.com/mlco2/codecarbon/pull/1171) -- Client integration: [PR #1200](https://github.com/mlco2/codecarbon/pull/1200) -- CLI import fix (separate, to `master`): [PR #1201](https://github.com/mlco2/codecarbon/pull/1201) — `amdsmi` / `libamd_smi.so` on macOS - ---- - -## Configuration contract (current) - -### What users can configure where - -| Setting | Config file | Env | Python kwarg | CLI | Default | -|---------|-------------|-----|--------------|-----|---------| -| **`telemetry_level`** (tier) | ✅ | ❌ (tier) / ✅ (explicit warning only)¹ | ✅ `telemetry_level=` | ✅ see below | `minimal` | -| `telemetry_api_url` | ✅ | `CODECARBON_TELEMETRY_API_URL` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_URL` | -| `telemetry_api_key` | ✅ | `CODECARBON_TELEMETRY_API_KEY` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_KEY` | -| `telemetry_experiment_id` | ✅ | `CODECARBON_TELEMETRY_EXPERIMENT_ID` | ❌ | ❌ | `DEFAULT_TELEMETRY_EXPERIMENT_ID` | - -¹ `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` suppress the “not configured” warning but do **not** change the resolved tier unless also in config or passed as kwarg. - -**CLI for `telemetry_level`:** - -| Command | Persists to config? | Notes | -|---------|---------------------|--------| -| `codecarbon telemetry` | ✅ | Interactive wizard | -| `codecarbon telemetry set ` | ✅ | `disabled` \| `minimal` \| `extensive` | -| `codecarbon telemetry show` | — | Read resolved tier | -| `codecarbon monitor --telemetry-level ` | ❌ | One-run override only (tracker kwarg) | - -Separate from `codecarbon config` (dashboard org/project/experiment). - -### Tier values - -| Value | Tier | On init | On stop | Visibility (product) | -|-------|------|---------|---------|----------------------| -| `disabled` | 0 | — | — | None | -| `minimal` | 1 | Tier 1 `POST /telemetry` (once/session) | — | Private metadata | -| `extensive` | 2 | Tier 1 `POST /telemetry` (once/session) | Tier 2 `add_emission` | Public leaderboard | - -### Default when not explicit - -If the user never sets `telemetry_level` (no config key, no kwarg, no env key counted as explicit): - -1. **Warning** (once per Python session): default is `minimal`; **Tier 1 minimal telemetry will be sent** once per session. -2. **Behaviour:** same as `telemetry_level = minimal` (Tier 1 HTTP on init). - -### Explicit configuration (no warning) - -Any of: - -- `telemetry_level = …` in `.codecarbon.config` (local or global) -- `EmissionsTracker(telemetry_level="…")` / `OfflineEmissionsTracker` / `@track_emissions` -- `codecarbon telemetry set …` (writes config) -- `codecarbon monitor --telemetry-level …` (run override; counts as explicit for that run) -- `CODECARBON_TELEMETRY_LEVEL` or `CODECARBON_TELEMETRY` in the environment - -### Tier resolution precedence - -1. Tracker kwarg `telemetry_level` (CLI `--telemetry-level` or Python arg) -2. `telemetry_level` in config file (local overrides global) -3. Built-in default: `minimal` - -### `.codecarbon.config` example - -```ini -[codecarbon] -telemetry_level = minimal -telemetry_api_url = https://api.codecarbon.io -telemetry_api_key = cpt_... -telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 -``` - -### Tests disabling telemetry - -```ini -[codecarbon] -telemetry_level = disabled -``` - -Pattern: `get_custom_mock_open("[codecarbon]\ntelemetry_level = disabled\n", ...)`. - ---- - -## Out of scope - -- `codecarbon config` wizard step for telemetry (use `codecarbon telemetry` instead) -- `CODECARBON_TELEMETRY` env var **changing** the tier (legacy key only affects “explicit” warning) -- First-run consent prompt, Alembic migration -- Leaderboard UI / server-side public visibility enforcement - ---- - -## Task 1: Telemetry settings module ✅ - -**Files:** `codecarbon/core/telemetry_settings.py`, `codecarbon/core/config.py`, `tests/test_telemetry_settings.py`, `tests/test_config_file_settings.py` - -**Done:** -- `DEFAULT_TELEMETRY_*` constants and `get_telemetry_api_*` helpers -- `get_config_file_settings()` — file only -- `resolve_telemetry_level(config_file_conf, override=…)` — kwarg overrides config -- `is_telemetry_level_explicit(config_file_conf, override=…, external_conf=…)` - ---- - -## Task 2: `codecarbon/telemetry.py` ✅ - -**Done:** -- `build_minimal_telemetry_dict` / `send_tier1_telemetry` — `TelemetryClient` → `POST /telemetry` + `x-api-token`; session dedup `_TIER1_SENT`; HTTP 404 → warning -- `send_tier2_public_emission` — `ApiClient` + `add_emission` to public experiment; `_TIER2_SENT` -- `warn_if_telemetry_not_configured` — one-time warning; message states Tier 1 will be sent -- Tests in `tests/test_telemetry.py` - ---- - -## Task 3: Tracker wiring ✅ - -**Files:** `codecarbon/emissions_tracker.py` - -```mermaid -flowchart LR - subgraph init [Tracker __init__] - W[warn if not explicit] - T1[Tier 1 POST /telemetry] - end - subgraph stop [Tracker stop] - T2[Tier 2 add_emission] - end - W --> T1 - T1 --> TelAPI[TelemetryClient] - T2 --> CCApi[ApiClient public experiment] -``` - -```python -self._config_file_conf = get_config_file_settings() -telemetry_override = None if telemetry_level is _sentinel else telemetry_level -self._telemetry_level = resolve_telemetry_level(self._config_file_conf, override=telemetry_override) -self._apply_init_telemetry(telemetry_override) # warn + Tier 1 when minimal/extensive -# stop(): -self._maybe_send_tier2_telemetry(emissions_data_delta) # extensive only -``` - -| Config | Init | Stop | -|--------|------|------| -| `disabled` | — | — | -| `minimal` | Tier 1 HTTP | — | -| `extensive` | Tier 1 HTTP | Tier 2 `add_emission` | - -Independent of `save_to_api`. Best-effort; never blocks tracker. - ---- - -## Task 4: Extensive `POST /telemetry` payload ⏸ - -Deferred — Tier 2 is public `add_emission` only. - ---- - -## Task 5: Config integration tests ✅ - -**File:** `tests/test_telemetry_config.py` - -- Tier behaviour per config (`disabled` / `minimal` / `extensive`) -- Env does not change tier; env API URL for Tier 2 -- Warning when config empty; no warning when config or kwarg explicit -- Tier 1 asserts `TelemetryClient` / `POST /telemetry` (mocked in unit tests) - ---- - -## Task 6: Telemetry CLI ✅ - -**Files:** `codecarbon/cli/telemetry_cli.py`, `codecarbon/cli/main.py`, `tests/cli/test_telemetry_cli.py` - -**Done:** -- `codecarbon.add_typer(telemetry_app, name="telemetry")` -- `codecarbon telemetry` — interactive (`questionary`): pick config path + tier -- `codecarbon telemetry set [--config PATH]` — write `telemetry_level` -- `codecarbon telemetry show [--config PATH]` — resolved tier + explicit flag -- `codecarbon monitor --telemetry-level ` — one-run override via tracker kwarg -- `parse_telemetry_level()` (core) + `normalize_telemetry_level()` (CLI Typer wrapper) -- `telemetry show` uses merged file settings by default (matches tracker) - -**Not done:** optional hook at end of `codecarbon config` wizard (deferred). - ---- - -## Task 7: Documentation ✅ - -**Files:** `docs/how-to/telemetry.md`, `docs/reference/cli.md`, `mkdocs.yml`, link from `docs/how-to/configuration.md` - -**Done:** -- User-facing telemetry how-to with tiers, config, CLI, Python opt-out -- Tier 1 documents **only fields sent today** by `build_minimal_telemetry_dict` plus explicit “not collected yet” list -- CLI reference: `codecarbon telemetry`, `--telemetry-level` - ---- - -## Task 8: PR hygiene ✅ - -```bash -uv run pytest tests/test_telemetry_settings.py tests/test_config_file_settings.py \ - tests/test_telemetry.py tests/test_telemetry_config.py tests/test_telemetry_client.py \ - tests/cli/test_telemetry_cli.py -v -uv run pytest tests/test_emissions_tracker.py tests/test_offline_emissions_tracker.py -q -uv run pytest carbonserver/tests/api/test_telemetry_schema_drift.py -v -uv run task test-package -``` - -**Results (2026-05-19):** 62 telemetry tests, 22 tracker tests, schema drift, **527** package unit tests passed. - -- [x] Public `DEFAULT_TELEMETRY_API_KEY` / `DEFAULT_TELEMETRY_EXPERIMENT_ID` -- [x] `telemetry_level` kwarg on tracker + `@track_emissions` -- [x] Configure warning + Tier 1 HTTP when not explicit -- [x] Telemetry CLI + `monitor --telemetry-level` -- [x] No `send_telemetry`; tests use config mocks -- [x] Full package test suite green (`uv run task test-package`) -- [x] Schema drift test passes -- [x] `test_config.py` isolates `get_config_file_settings` (no leak from real `~/.codecarbon.config`) -- [x] `test_gpu.py` amdsmi warning message aligned with `gpu_amd.py` -- [x] `ApiClient._create_run` handles `longitude`/`latitude` `None` - ---- - -## Related fix (separate PR, not telemetry feature) - -**[PR #1201](https://github.com/mlco2/codecarbon/pull/1201)** → `master`: `codecarbon/core/gpu_amd.py` catches `KeyError` / `OSError` when `amdsmi` is installed but `libamd_smi.so` is missing (macOS CLI crash). Branch: `fix/amdsmi-import-keyerror-macos`. - ---- - -## Risk notes - -| Risk | Mitigation | -|------|------------| -| Users surprised by default Tier 1 logging | One-time warning names Tier 1 explicitly | -| Env `CODECARBON_TELEMETRY` confusion | Document: warning-only, not tier resolution | -| Tier 1 log vs future HTTP | Doc + comment in `send_tier1_telemetry`; flip when API ready | -| CLI import before amdsmi fix merged | PR #1201 to master; rebase `feat/add-telemetry` after merge | - ---- - -## Progress summary - -| Task | Status | -|------|--------| -| 1. Telemetry settings | ✅ | -| 2. `telemetry.py` | ✅ | -| 3. Tracker wiring | ✅ | -| 4. Extensive `/telemetry` POST | ⏸ | -| 5. Integration tests | ✅ | -| 6. Telemetry CLI | ✅ | -| 7. Documentation | ✅ | -| 8. Full CI / PR hygiene | ✅ | - ---- - -## Execution handoff - -**Next step:** Merge PR #1200; rebase after [PR #1201](https://github.com/mlco2/codecarbon/pull/1201); deploy [#1171](https://github.com/mlco2/codecarbon/pull/1171) so prod `/telemetry` stops returning 404. From 0a319bcb480c4146a3dc16ec8453e2e7ad7c9f93 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:49:41 +0200 Subject: [PATCH 12/29] fix: improve error handling and logging for AMD GPU metrics - Updated warning message for amdsmi configuration issues to provide clearer guidance on potential causes and solutions. - Adjusted test to reflect the new warning message format, ensuring accurate validation of GPU import warnings. --- codecarbon/core/gpu_amd.py | 10 ++++++---- tests/test_gpu.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/codecarbon/core/gpu_amd.py b/codecarbon/core/gpu_amd.py index 9b1b3ddc1..bd8eeb226 100644 --- a/codecarbon/core/gpu_amd.py +++ b/codecarbon/core/gpu_amd.py @@ -28,12 +28,14 @@ def is_rocm_system(): "Please install amdsmi to get GPU metrics." ) AMDSMI_AVAILABLE = False -except (AttributeError, OSError, KeyError) as error: +except (AttributeError, OSError, KeyError) as e: amdsmi = None + # In some environments, amdsmi may be present but not properly configured, leading to AttributeError when importing logger.warning( - "amdsmi is installed but could not load libamd_smi (ROCm drivers missing " - "or incompatible). AMD GPU metrics will be unavailable. Error: %s", - error, + "AMD GPU detected but amdsmi is not properly configured. " + "Please ensure amdsmi is correctly installed to get GPU metrics." + "Tips : check consistency between Python amdsmi package and ROCm versions, and ensure AMD drivers are up to date." + f" Error: {e}" ) AMDSMI_AVAILABLE = False diff --git a/tests/test_gpu.py b/tests/test_gpu.py index 913c05017..bfbc8e603 100644 --- a/tests/test_gpu.py +++ b/tests/test_gpu.py @@ -153,7 +153,7 @@ def check_output(cmd, *args, **kwargs): sys.modules["pynvml"] = old_pynvml assert any( - "could not load libamd_smi" in str(c.args[0]) + "amdsmi is not properly configured" in str(c.args[0]) for c in warning_mock.call_args_list ) From dcec0c4e8b4544c3d9f73ff529f73d7b66296e21 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 23:09:46 +0200 Subject: [PATCH 13/29] feat: enhance telemetry schema and collection methods - Added new fields to the TelemetryBase schema, including `on_cloud`, `python_package_manager`, `integration_surface`, `offline_mode`, and `save_to_api_enabled`. - Updated MINIMAL_TELEMETRY_FIELDS to include new fields for improved telemetry data collection. - Refactored telemetry sending methods to utilize the new schema and ensure accurate data transmission. - Enhanced tests to validate the new telemetry fields and their integration into the existing telemetry framework. --- .../carbonserver/api/schemas_telemetry.py | 40 ++- .../tests/api/routers/test_telemetry.py | 2 +- codecarbon/cli/telemetry_cli.py | 6 +- codecarbon/core/telemetry_collect.py | 313 ++++++++++++++++++ codecarbon/core/telemetry_schemas.py | 41 ++- codecarbon/core/telemetry_settings.py | 33 +- codecarbon/emissions_tracker.py | 25 +- codecarbon/telemetry.py | 141 ++++---- docs/how-to/configuration.md | 2 +- docs/how-to/telemetry.md | 110 +++--- docs/reference/api.md | 4 +- tests/test_telemetry.py | 276 +++++++-------- tests/test_telemetry_client.py | 2 +- tests/test_telemetry_collect.py | 113 +++++++ tests/test_telemetry_config.py | 257 +++++++------- tests/testutils.py | 21 ++ 16 files changed, 887 insertions(+), 499 deletions(-) create mode 100644 codecarbon/core/telemetry_collect.py create mode 100644 tests/test_telemetry_collect.py diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index 5517c2ff4..dec7e0dfa 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -41,6 +41,7 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None + on_cloud: Optional[bool] = None longitude: Optional[float] = Field(default=None, ge=-180, le=180) latitude: Optional[float] = Field(default=None, ge=-90, le=90) @@ -64,6 +65,7 @@ class TelemetryBase(BaseModel): python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None + python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -83,6 +85,9 @@ class TelemetryBase(BaseModel): task_tracking_used: Optional[bool] = None decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + integration_surface: Optional[str] = None + offline_mode: Optional[bool] = None + save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None @@ -148,25 +153,50 @@ def validate_telemetry_level(self): "region", "cloud_provider", "cloud_region", - "longitude", - "latitude", + "on_cloud", "cpu_count", "cpu_physical_count", "cpu_model", "cpu_architecture", "gpu_count", "gpu_model", - "gpu_driver_version", "gpu_memory_total_gb", "ram_total_size_gb", "cuda_version", - "cudnn_version", "python_version", "python_implementation", - "python_executable_hash", "python_env_type", + "python_package_manager", "codecarbon_version", "codecarbon_install_method", + "tracking_mode", + "integration_surface", + "offline_mode", + "output_methods", + "save_to_api_enabled", + "task_tracking_used", + "measure_power_interval_secs", + "in_container", + "ci_environment", + "notebook_environment", + "has_torch", + "has_transformers", + "has_tensorflow", + "has_keras", + "has_diffusers", + "has_pytorch_lightning", + "has_fastai", + "ml_framework_primary", + "total_emissions_kg", + "emissions_rate_kg_per_sec", + "energy_consumed_kwh", + "cpu_energy_kwh", + "gpu_energy_kwh", + "ram_energy_kwh", + "duration_seconds", + "cpu_utilization_avg", + "gpu_utilization_avg", + "ram_utilization_avg", } diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index a405b59e8..733269bb0 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -56,7 +56,7 @@ def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) telemetry_with_extensive_field = { **MINIMAL_TELEMETRY_TO_CREATE, - "total_emissions_kg": 0.42, + "torch_version": "2.2.0", } with custom_test_server.container.telemetry_repository.override(repository_mock): diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py index e32cd730b..0cf7a8d77 100644 --- a/codecarbon/cli/telemetry_cli.py +++ b/codecarbon/cli/telemetry_cli.py @@ -28,9 +28,9 @@ ) TIER_DESCRIPTIONS = { - "disabled": "No telemetry.", - "minimal": "Send hardware/environment metadata once per session (Tier 1).", - "extensive": "Tier 1 plus public emissions on stop (Tier 2 leaderboard).", + "disabled": "No product telemetry.", + "minimal": "Tier 1: private telemetry per run at stop (environment, usage, frameworks, emissions).", + "extensive": "Tier 2: Tier 1 plus run emissions summary to the shared telemetry experiment (ApiClient).", } diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py new file mode 100644 index 000000000..bdd3ed676 --- /dev/null +++ b/codecarbon/core/telemetry_collect.py @@ -0,0 +1,313 @@ +"""Collect and project private product telemetry (Tier 1 / Tier 2) from tracker state.""" + +from __future__ import annotations + +import importlib.util +import os +import platform +import sys +from datetime import datetime, timezone +from typing import Any, Optional + +from codecarbon.core.cloud import get_env_cloud_details +from codecarbon.core.gpu import is_nvidia_system +from codecarbon.core.telemetry_schemas import ( + MINIMAL_TELEMETRY_FIELDS, + TelemetryLevel, +) +from codecarbon.output_methods.emissions_data import EmissionsData + +FRAMEWORK_PACKAGES = ( + ("torch", "has_torch", "torch_version"), + ("transformers", "has_transformers", "transformers_version"), + ("tensorflow", "has_tensorflow", "tensorflow_version"), + ("keras", "has_keras", "keras_version"), + ("diffusers", "has_diffusers", "diffusers_version"), + ("pytorch_lightning", "has_pytorch_lightning", "pytorch_lightning_version"), + ("fastai", "has_fastai", "fastai_version"), +) + +def _non_empty(value: Any) -> bool: + if value is None: + return False + if value == [] or value == {}: + return False + if isinstance(value, str) and value == "": + return False + return True + + +def _strip_none(data: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in data.items() if _non_empty(value)} + + +def _package_installed(name: str) -> bool: + return importlib.util.find_spec(name) is not None + + +def _package_version(name: str) -> Optional[str]: + if not _package_installed(name): + return None + try: + from importlib.metadata import version + + return version(name) + except Exception: + return None + + +def _detect_codecarbon_install_method() -> Optional[str]: + try: + from importlib.metadata import distribution + + dist = distribution("codecarbon") + if getattr(dist, "editable", False): + return "editable" + installer = (dist.metadata.get("Installer") or "").lower() + if "uv" in installer: + return "uv" + if "pip" in installer: + return "pip" + except Exception: + pass + return None + + +def _detect_python_env_type() -> Optional[str]: + if os.environ.get("CONDA_DEFAULT_ENV"): + return "conda" + if os.environ.get("VIRTUAL_ENV"): + return "venv" + if sys.prefix != getattr(sys, "base_prefix", sys.prefix): + return "venv" + return "system" + + +def _detect_python_package_manager() -> Optional[str]: + if os.environ.get("UV"): + return "uv" + if os.environ.get("POETRY_ACTIVE"): + return "poetry" + if os.environ.get("PIP_RUN"): + return "pip" + return None + + +def _detect_ci_environment() -> Optional[str]: + if os.environ.get("GITHUB_ACTIONS"): + return "github_actions" + if os.environ.get("GITLAB_CI"): + return "gitlab_ci" + if os.environ.get("CIRCLECI"): + return "circleci" + if os.environ.get("JENKINS_URL"): + return "jenkins" + if os.environ.get("CI"): + return "ci" + return None + + +def _detect_notebook_environment() -> Optional[str]: + if os.environ.get("COLAB_GPU") is not None or "google.colab" in sys.modules: + return "colab" + try: + from IPython import get_ipython + + shell = get_ipython().__class__.__name__ + if "ZMQInteractiveShell" in shell: + return "jupyter" + except Exception: + pass + return None + + +def _detect_in_container() -> bool: + if os.path.exists("/.dockerenv"): + return True + if os.environ.get("KUBERNETES_SERVICE_HOST"): + return True + return False + + +def _detect_integration_surface(tracker: Any) -> str: + from codecarbon.emissions_tracker import OfflineEmissionsTracker + + if isinstance(tracker, OfflineEmissionsTracker): + return "offline_tracker" + argv = " ".join(sys.argv) + if "codecarbon" in argv and "monitor" in argv: + return "cli_monitor" + return "library" + + +def _collect_output_methods(tracker: Any) -> list[str]: + methods: list[str] = [] + if getattr(tracker, "_save_to_file", False): + methods.append("file") + if getattr(tracker, "_save_to_api", False): + methods.append("api") + if getattr(tracker, "_save_to_logger", False): + methods.append("logger") + if getattr(tracker, "_emissions_endpoint", None): + methods.append("http") + if getattr(tracker, "_save_to_prometheus", False): + methods.append("prometheus") + if getattr(tracker, "_save_to_logfire", False): + methods.append("logfire") + return methods + + +def _raw_cloud_provider_and_region() -> tuple[Optional[str], Optional[str]]: + details = get_env_cloud_details() + if not details or not details.get("metadata"): + return None, None + provider = (details.get("provider") or "").lower() or None + metadata = details.get("metadata") or {} + region: Optional[str] = None + if provider == "aws": + region = metadata.get("region") + elif provider == "azure": + region = (metadata.get("compute") or {}).get("location") + elif provider == "gcp": + zone = metadata.get("zone") or "" + parts = zone.split("/") + region = parts[-1].rsplit("-", 1)[0] if parts else None + return provider, region + + +def _collect_framework_fields(include_versions: bool) -> dict[str, Any]: + fields: dict[str, Any] = {} + primary: Optional[str] = None + for package, has_field, version_field in FRAMEWORK_PACKAGES: + installed = _package_installed(package) + fields[has_field] = installed + if include_versions and installed: + fields[version_field] = _package_version(package) + if installed and primary is None: + primary = package + if primary: + fields["ml_framework_primary"] = primary + return fields + + +def _gpu_static_fields() -> dict[str, Any]: + fields: dict[str, Any] = {} + if not is_nvidia_system(): + return fields + try: + import pynvml + + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + mem = pynvml.nvmlDeviceGetMemoryInfo(handle) + fields["gpu_memory_total_gb"] = mem.total / (1024**3) + fields["gpu_driver_version"] = pynvml.nvmlSystemGetDriverVersion() + fields["cuda_version"] = pynvml.nvmlSystemGetCudaDriverVersion_v2() + if isinstance(fields["cuda_version"], int): + v = fields["cuda_version"] + fields["cuda_version"] = f"{v // 1000}.{(v % 1000) // 10}" + except Exception: + pass + return fields + + +def collect_telemetry_context( + tracker: Any, + emissions: EmissionsData, +) -> dict[str, Any]: + """Build a flat telemetry context from tracker state and emissions at stop. + + Args: + tracker: Active ``BaseEmissionsTracker`` instance. + emissions: Total emissions row from ``_prepare_emissions_data()``. + + Returns: + Flat dictionary for ``project_tier1``. + """ + conf = getattr(tracker, "_conf", {}) + raw_provider, raw_region = _raw_cloud_provider_and_region() + on_cloud = emissions.on_cloud == "Y" + cloud_provider = emissions.cloud_provider or raw_provider + cloud_region = emissions.cloud_region or raw_region + region = emissions.region or conf.get("region") + if on_cloud and cloud_region: + region = region or cloud_region + + integration_surface = _detect_integration_surface(tracker) + gpu_fields = _gpu_static_fields() + + context: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc), + "os": conf.get("os") or platform.platform(), + "python_version": conf.get("python_version") or platform.python_version(), + "python_implementation": platform.python_implementation(), + "python_env_type": _detect_python_env_type(), + "python_package_manager": _detect_python_package_manager(), + "codecarbon_version": conf.get("codecarbon_version"), + "codecarbon_install_method": _detect_codecarbon_install_method(), + "country_name": emissions.country_name, + "country_iso_code": emissions.country_iso_code, + "region": region, + "cloud_provider": cloud_provider, + "cloud_region": cloud_region, + "on_cloud": on_cloud, + "cpu_count": conf.get("cpu_count"), + "cpu_physical_count": conf.get("cpu_physical_count"), + "cpu_model": conf.get("cpu_model"), + "cpu_architecture": platform.machine(), + "gpu_count": conf.get("gpu_count"), + "gpu_model": conf.get("gpu_model"), + "ram_total_size_gb": conf.get("ram_total_size"), + "tracking_mode": conf.get("tracking_mode"), + "integration_surface": integration_surface, + "offline_mode": integration_surface == "offline_tracker", + "output_methods": _collect_output_methods(tracker), + "save_to_api_enabled": bool(getattr(tracker, "_save_to_api", False)), + "task_tracking_used": bool(getattr(tracker, "_tasks", {})), + "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), + "in_container": _detect_in_container(), + "ci_environment": _detect_ci_environment(), + "notebook_environment": _detect_notebook_environment(), + "duration_seconds": float(emissions.duration) if emissions.duration else None, + "total_emissions_kg": emissions.emissions, + "emissions_rate_kg_per_sec": emissions.emissions_rate, + "energy_consumed_kwh": emissions.energy_consumed, + "cpu_energy_kwh": emissions.cpu_energy, + "gpu_energy_kwh": emissions.gpu_energy, + "ram_energy_kwh": emissions.ram_energy, + "cpu_utilization_avg": emissions.cpu_utilization_percent, + "gpu_utilization_avg": emissions.gpu_utilization_percent, + "ram_utilization_avg": emissions.ram_utilization_percent, + **_collect_framework_fields(include_versions=False), + } + + for key in ("gpu_memory_total_gb", "cuda_version"): + if key in gpu_fields: + context[key] = gpu_fields[key] + + return _strip_none(context) + + +def project_tier1(context: dict[str, Any]) -> dict[str, Any]: + """Project context to Tier 1 (``telemetry_level=minimal``) fields.""" + payload = { + key: context[key] + for key in MINIMAL_TELEMETRY_FIELDS + if key in context + } + payload["telemetry_level"] = TelemetryLevel.minimal.value + return _strip_none(payload) + + +def build_tier1_payload(tracker: Any, emissions: EmissionsData) -> dict[str, Any]: + """Build a Tier 1 payload dict for ``TelemetryCreate``. + + Args: + tracker: Active emissions tracker. + emissions: Run emissions data. + + Returns: + Payload dict for ``POST /telemetry``. + """ + context = collect_telemetry_context(tracker, emissions) + return project_tier1(context) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry_schemas.py index ea6249b65..9dc3e2997 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry_schemas.py @@ -23,6 +23,7 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None + on_cloud: Optional[bool] = None longitude: Optional[float] = Field(default=None, ge=-180, le=180) latitude: Optional[float] = Field(default=None, ge=-90, le=90) @@ -46,6 +47,7 @@ class TelemetryBase(BaseModel): python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None + python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -65,6 +67,9 @@ class TelemetryBase(BaseModel): task_tracking_used: Optional[bool] = None decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + integration_surface: Optional[str] = None + offline_mode: Optional[bool] = None + save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None @@ -77,7 +82,6 @@ class TelemetryBase(BaseModel): ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None - python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -130,25 +134,50 @@ def validate_telemetry_level(self): "region", "cloud_provider", "cloud_region", - "longitude", - "latitude", + "on_cloud", "cpu_count", "cpu_physical_count", "cpu_model", "cpu_architecture", "gpu_count", "gpu_model", - "gpu_driver_version", "gpu_memory_total_gb", "ram_total_size_gb", "cuda_version", - "cudnn_version", "python_version", "python_implementation", - "python_executable_hash", "python_env_type", + "python_package_manager", "codecarbon_version", "codecarbon_install_method", + "tracking_mode", + "integration_surface", + "offline_mode", + "output_methods", + "save_to_api_enabled", + "task_tracking_used", + "measure_power_interval_secs", + "in_container", + "ci_environment", + "notebook_environment", + "has_torch", + "has_transformers", + "has_tensorflow", + "has_keras", + "has_diffusers", + "has_pytorch_lightning", + "has_fastai", + "ml_framework_primary", + "total_emissions_kg", + "emissions_rate_kg_per_sec", + "energy_consumed_kwh", + "cpu_energy_kwh", + "gpu_energy_kwh", + "ram_energy_kwh", + "duration_seconds", + "cpu_utilization_avg", + "gpu_utilization_avg", + "ram_utilization_avg", } diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index 9e6d5ad91..1d2f65c87 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -44,10 +44,8 @@ def is_telemetry_level_explicit( ) -> bool: """Return whether the user explicitly chose a telemetry tier. - Explicit sources (in order): tracker ``telemetry_level`` argument, config file - ``telemetry_level``, environment ``CODECARBON_TELEMETRY_LEVEL`` / - ``telemetry_level``. Legacy ``CODECARBON_TELEMETRY`` / ``telemetry`` counts as - explicit for this check only. + Explicit sources: tracker ``telemetry_level`` argument, config file + ``telemetry_level``, or environment ``CODECARBON_TELEMETRY_LEVEL``. Args: config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). @@ -63,9 +61,7 @@ def is_telemetry_level_explicit( return True if external_conf is None: return False - if external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: - return True - return external_conf.get("telemetry") is not None + return external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None def resolve_telemetry_level( @@ -78,15 +74,10 @@ def resolve_telemetry_level( Precedence: - 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or - ``codecarbon monitor --telemetry-level`` + 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI 2. ``external_conf`` — merged ``.codecarbon.config`` and ``CODECARBON_*`` env - (environment overrides file for the same key) 3. ``config_file_conf`` — file-only settings when ``external_conf`` is omitted - 4. Default: ``minimal`` - - Legacy ``CODECARBON_TELEMETRY`` / config key ``telemetry`` only affect whether - the tier counts as explicitly configured, not tier resolution. + 4. Default: ``minimal`` (Tier 1) Args: config_file_conf: Settings from ``get_config_file_settings()`` (optional). @@ -130,8 +121,10 @@ def get_telemetry_api_url( Returns: API base URL without trailing slash. """ - url = external_conf.get("telemetry_api_url") or os.environ.get( - "CODECARBON_TELEMETRY_API_URL" + url = ( + external_conf.get("telemetry_api_url") + or external_conf.get("api_endpoint") + or os.environ.get("CODECARBON_TELEMETRY_API_URL") ) return (url or default).rstrip("/") @@ -145,8 +138,10 @@ def get_telemetry_api_key(external_conf: dict[str, Any]) -> str: Returns: API token string. """ - key = external_conf.get("telemetry_api_key") or os.environ.get( - "CODECARBON_TELEMETRY_API_KEY" + key = ( + external_conf.get("telemetry_api_key") + or external_conf.get("api_key") + or os.environ.get("CODECARBON_TELEMETRY_API_KEY") ) return key or DEFAULT_TELEMETRY_API_KEY @@ -158,7 +153,7 @@ def get_telemetry_experiment_id(external_conf: dict[str, Any]) -> str: external_conf: Merged config from file and environment. Returns: - Experiment UUID string for Tier 2 / leaderboard linkage. + Experiment UUID string (legacy leaderboard / API helpers). """ experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( "CODECARBON_TELEMETRY_EXPERIMENT_ID" diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index d88892d66..fa4eeb166 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -25,8 +25,7 @@ from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import resolve_telemetry_level from codecarbon.telemetry import ( - send_tier1_telemetry, - send_tier2_public_emission, + send_product_telemetry_at_stop, warn_if_telemetry_not_configured, ) from codecarbon.core.emissions import Emissions @@ -415,6 +414,7 @@ def __init__( self._hardware = [] resource_tracker = ResourceTracker(self) resource_tracker.set_CPU_GPU_ram_tracking() + self._resource_tracker = resource_tracker self._conf["hardware"] = list(map(lambda x: x.description(), self._hardware)) @@ -458,7 +458,7 @@ def __init__( if cloud.is_on_private_infra: self._conf["longitude"] = self._geo.longitude self._conf["latitude"] = self._geo.latitude - self._conf["region"] = cloud.region + self._conf["region"] = self._geo.region self._conf["provider"] = cloud.provider else: self._conf["region"] = cloud.region @@ -479,17 +479,14 @@ def _apply_init_telemetry(self, telemetry_override: str | None) -> None: override=telemetry_override, external_conf=self._external_conf, ) - if self._telemetry_level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_tier1_telemetry(self._conf, external_conf=self._external_conf) - @suppress(Exception) - def _maybe_send_tier2_telemetry(self, emissions_data_delta: EmissionsData) -> None: - if self._telemetry_level == TelemetryLevel.extensive: - send_tier2_public_emission( - self._conf, - emissions_data_delta, - external_conf=self._external_conf, - ) + def _send_product_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: + send_product_telemetry_at_stop( + self, + emissions_data, + self._telemetry_level, + external_conf=self._external_conf, + ) def _init_output_methods(self, *, api_key: str = None): """ @@ -792,7 +789,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) - self._maybe_send_tier2_telemetry(emissions_data_delta) + self._send_product_telemetry_at_stop(emissions_data) self._persist_data( total_emissions=emissions_data, diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index ad4946e0c..397be674b 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,11 +1,11 @@ -"""Tracker-facing telemetry helpers (Tier 1 HTTP, Tier 2 public emissions).""" +"""Product telemetry sent at tracker stop (Tier 1 private, Tier 2 public emissions).""" import dataclasses -from datetime import datetime, timezone from typing import Any from codecarbon.core.api_client import ApiClient from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_collect import build_tier1_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import ( get_telemetry_api_key, @@ -16,16 +16,13 @@ from codecarbon.external.logger import logger from codecarbon.output_methods.emissions_data import EmissionsData -_TIER1_SENT = False -_TIER2_SENT = False _TELEMETRY_CONFIGURE_WARNED = False TELEMETRY_NOT_CONFIGURED_MESSAGE = ( "CodeCarbon telemetry_level was not set explicitly; using default %r. " - "Tier 1 minimal telemetry (hardware and environment metadata) will be " - "sent once per Python session. Set telemetry_level in .codecarbon.config, " - "set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... to " - "EmissionsTracker / OfflineEmissionsTracker, or run " + "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " + "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " + "to EmissionsTracker / OfflineEmissionsTracker, or run " "codecarbon telemetry set ." ) @@ -59,98 +56,66 @@ def warn_if_telemetry_not_configured( _TELEMETRY_CONFIGURE_WARNED = True -def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: - """Build a minimal telemetry payload dict from tracker configuration. +def _run_too_short_for_telemetry(emissions: EmissionsData) -> bool: + return emissions.duration is not None and emissions.duration < 1 - Args: - conf: Tracker configuration dictionary. - Returns: - Dictionary suitable for ``TelemetryCreate`` validation. - """ - payload: dict[str, Any] = { - "timestamp": datetime.now(timezone.utc), - "telemetry_level": TelemetryLevel.minimal.value, - "os": conf.get("os"), - "country_iso_code": conf.get("country_iso_code"), - "region": conf.get("region"), - "cloud_provider": conf.get("provider"), - "cloud_region": conf.get("region"), - "longitude": conf.get("longitude"), - "latitude": conf.get("latitude"), - "cpu_count": conf.get("cpu_count"), - "cpu_physical_count": conf.get("cpu_physical_count"), - "cpu_model": conf.get("cpu_model"), - "gpu_count": conf.get("gpu_count"), - "gpu_model": conf.get("gpu_model"), - "ram_total_size_gb": conf.get("ram_total_size"), - "python_version": conf.get("python_version"), - "codecarbon_version": conf.get("codecarbon_version"), - } - return {key: value for key, value in payload.items() if value is not None} - - -def send_tier1_telemetry( - conf: dict[str, Any], +def send_tier1_at_stop( + tracker: Any, + emissions: EmissionsData, external_conf: dict[str, Any] | None = None, ) -> bool: - """POST minimal telemetry once per Python session via ``TelemetryClient``. - - Sends ``POST {telemetry_api_url}/telemetry`` with a ``TelemetryCreate`` payload - built from tracker configuration. Best-effort: failures are logged and not raised. + """Send Tier 1 telemetry: private hardware/usage/run summary via ``POST /telemetry``. Args: - conf: Tracker configuration dictionary. - external_conf: Merged file/env config for telemetry API URL and key resolution. + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + external_conf: Merged config for telemetry API URL and key resolution. Returns: - True if telemetry was posted successfully on this call, False if already sent - in this session or if the request failed. + True if Tier 1 was accepted, False otherwise. """ - global _TIER1_SENT - if _TIER1_SENT: + if _run_too_short_for_telemetry(emissions): + logger.debug( + "Tier 1 telemetry not sent because run duration is shorter than 1 second." + ) return False settings_conf = external_conf or {} try: - payload = build_minimal_telemetry_dict(conf) - endpoint_url = get_telemetry_api_url(settings_conf) + payload = build_tier1_payload(tracker, emissions) client = TelemetryClient( - endpoint_url=endpoint_url, + endpoint_url=get_telemetry_api_url(settings_conf), telemetry=payload, api_key=get_telemetry_api_key(settings_conf), ) - response = client.add_telemetry() - if response is not None: - _TIER1_SENT = True - return True - return False + return client.add_telemetry() is not None except Exception as error: - logger.error(f"Telemetry Tier 1 failed (non-critical): {error}") + logger.error(f"Tier 1 telemetry failed (non-critical): {error}") return False -def send_tier2_public_emission( - conf: dict[str, Any], - emissions_data: EmissionsData, +def send_tier2_at_stop( + tracker: Any, + emissions: EmissionsData, external_conf: dict[str, Any] | None = None, ) -> bool: - """Send run emissions to the public telemetry experiment via ApiClient. - - Mirrors ``CodeCarbonAPIOutput`` / ``add_emission`` for the shared leaderboard - project. Best-effort: errors are logged and not raised. + """Send Tier 2 telemetry: run emissions to the shared experiment via ``ApiClient``. Args: - conf: Tracker configuration dictionary. - emissions_data: Delta or total emissions row for the run. - external_conf: Merged file/env config for API URL, key, and experiment id. + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + external_conf: Merged config for API URL, key, and experiment resolution. Returns: - True if emission was posted successfully on this call, False otherwise. + True if Tier 2 was posted successfully, False otherwise. """ - global _TIER2_SENT - if _TIER2_SENT: + if _run_too_short_for_telemetry(emissions): + logger.debug( + "Tier 2 telemetry not sent because run duration is shorter than 1 second." + ) return False settings_conf = external_conf or {} + conf = getattr(tracker, "_conf", {}) try: api = ApiClient( endpoint_url=get_telemetry_api_url(settings_conf), @@ -159,11 +124,33 @@ def send_tier2_public_emission( conf=conf, create_run_automatically=True, ) - posted = api.add_emission(dataclasses.asdict(emissions_data)) - if posted: - _TIER2_SENT = True - return True - return False + return bool(api.add_emission(dataclasses.asdict(emissions))) except Exception as error: - logger.error(f"Telemetry Tier 2 failed (non-critical): {error}") + logger.error(f"Tier 2 telemetry failed (non-critical): {error}") return False + + +def send_product_telemetry_at_stop( + tracker: Any, + emissions: EmissionsData, + level: TelemetryLevel, + external_conf: dict[str, Any] | None = None, +) -> None: + """Send product telemetry for the resolved tier at tracker ``stop()``. + + Tier 1 (``minimal``): private ``POST /telemetry`` only. + Tier 2 (``extensive``): Tier 1 plus ``ApiClient`` run summary. + + Args: + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + level: Resolved ``TelemetryLevel``. + external_conf: Merged config for API settings. + """ + if level == TelemetryLevel.disabled: + return + settings = external_conf or {} + if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): + send_tier1_at_stop(tracker, emissions, settings) + if level == TelemetryLevel.extensive: + send_tier2_at_stop(tracker, emissions, settings) diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index 34e32b2bb..5cabf13ee 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -110,6 +110,6 @@ documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) ## Product telemetry -Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends private product telemetry once per run at `stop()`—see [Product telemetry](telemetry.md). See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index a99d42930..181605047 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -1,6 +1,6 @@ # Product telemetry -CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people run on, not what your code does. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. +CodeCarbon can send **optional private product telemetry** to help improve the library: hardware, environment, how the package is used, and per-run carbon/energy summaries. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. ## Telemetry vs your dashboard data @@ -8,68 +8,53 @@ CodeCarbon can send **optional product telemetry** to help improve the library: |--|-------------------|--------------------------------| | Purpose | Improve CodeCarbon (aggregate usage) | Your projects and experiments | | Config | `telemetry_level`, `codecarbon telemetry` | `codecarbon config`, `experiment_id` | -| Default experiment | Public telemetry project (built-in defaults) | Your account / experiment | +| Default API target | Built-in telemetry project (private) | Your account / experiment | You can use one without the other. ## Tiers -| `telemetry_level` | What happens | -|-------------------|--------------| -| `disabled` | No product telemetry | -| `minimal` (default) | **Tier 1** once per Python process: minimal hardware/environment metadata | -| `extensive` | Tier 1 + **Tier 2** on tracker `stop()`: one public emissions row (leaderboard-style) | +| `telemetry_level` | Name | When | Transport | +|-------------------|------|------|-----------| +| `disabled` | — | — | Nothing | +| `minimal` | **Tier 1** | Each `stop()` | `POST /telemetry` (private) | +| `extensive` | **Tier 2** | Each `stop()` | Tier 1 (`POST /telemetry`) **and** Tier 2 (`ApiClient` → `/emissions`) | Tier is resolved in this order: -1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` (highest priority) -2. **Config + environment** — `telemetry_level` in `.codecarbon.config` (local overrides global), then `CODECARBON_TELEMETRY_LEVEL` overrides the file value when both are set (same rules as other `CODECARBON_*` settings) -3. **Default:** `minimal` +1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` +2. **Config + environment** — `telemetry_level` in `.codecarbon.config`, then `CODECARBON_TELEMETRY_LEVEL` when both are set +3. **Default:** `minimal` (Tier 1) -Legacy `CODECARBON_TELEMETRY` / config key `telemetry` suppress the one-time setup warning when set, but do **not** set the tier (use `telemetry_level` or `CODECARBON_TELEMETRY_LEVEL`). +## Lifecycle -## Tier 1: what we collect (and what we do not) - -Tier 1 is intentionally small. The client only builds a **minimal** payload; the API schema rejects “extensive” fields when `telemetry_level` is `minimal`. +```text +EmissionsTracker.__init__ → collect hardware/geo (no POST) +EmissionsTracker.stop() → minimal: Tier 1 only | extensive: Tier 1 + Tier 2 +``` -### Sent at most once per process (if known) +If the run lasts less than one second, telemetry is not sent. -Only non-empty values are included: +## Tier 1 (`minimal`) — per run -| Field | Description | -|-------|-------------| -| `timestamp` | UTC time of the send | -| `telemetry_level` | Always `minimal` for this tier | -| `os` | Platform string | -| `country_iso_code` | Country (e.g. from offline mode or geo) | -| `region` | Region / province | -| `cloud_provider` | Cloud provider name, if detected | -| `cloud_region` | Cloud region, if detected | -| `longitude`, `latitude` | Approximate location, if geo resolution ran | -| `cpu_count`, `cpu_physical_count`, `cpu_model` | CPU metadata | -| `gpu_count`, `gpu_model` | GPU metadata | -| `ram_total_size_gb` | Total RAM | -| `python_version` | Python version | -| `codecarbon_version` | Installed CodeCarbon version | +One private row per tracker run with: -### Not sent in Tier 1 +- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region +- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container hints +- **ML stack (presence):** `has_torch`, `has_transformers`, `has_tensorflow`, and related flags +- **Run outcome:** duration, emissions, energy (total and per component), utilization averages -Tier 1 does **not** include: +Tier 1 does **not** include project names, experiment ids, API keys, file paths, or survey demographics (role, industry, etc.). -- Emissions, energy, power, duration, or utilization -- Project name, experiment id, run id, or API keys -- Source code, file paths, hostnames, or user ids -- ML stack versions (PyTorch, TensorFlow, etc.) -- Output methods, tracking mode, or internal diagnostics -- Anything else defined as “extensive” in the telemetry schema +## Tier 2 (`extensive`) — per run -Tier 2 (`extensive`) adds a single **public** emissions summary via the same mechanism as `add_emission` to the shared telemetry experiment—not a full extensive telemetry document. +**Always sends Tier 1 first**, then adds a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. -### Transport +## Never collected -- HTTP `POST` to `{telemetry_api_url}/telemetry` -- Best-effort: failures are logged and do not stop your tracker -- If the endpoint is not deployed yet, you may see a warning (HTTP 404) +- Project name, experiment id, run id, API keys +- Source code, file paths, hostnames +- Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) ## Configure telemetry @@ -78,27 +63,13 @@ Tier 2 (`extensive`) adds a single **public** emissions summary via the same mec ```ini [codecarbon] telemetry_level = minimal -# Optional overrides (defaults point at the public telemetry API): -# telemetry_api_url = https://api.codecarbon.io -# telemetry_api_key = ... -# telemetry_experiment_id = ... ``` ### CLI ```bash -# Interactive wizard (pick config file + tier) -codecarbon telemetry - -# Set tier in config -codecarbon telemetry set disabled codecarbon telemetry set minimal -codecarbon telemetry set extensive - -# Show resolved tier (merged global + local config) codecarbon telemetry show - -# One-run override (does not write config) codecarbon monitor --telemetry-level disabled -- python train.py ``` @@ -107,32 +78,25 @@ codecarbon monitor --telemetry-level disabled -- python train.py ```python from codecarbon import EmissionsTracker -tracker = EmissionsTracker(telemetry_level="disabled") +tracker = EmissionsTracker(telemetry_level="minimal") +tracker.start() +# ... +tracker.stop() ``` ## Opt out -Set any of: - ```ini [codecarbon] telemetry_level = disabled ``` -```bash -codecarbon telemetry set disabled -``` - -```python -EmissionsTracker(telemetry_level="disabled") -``` - ## First run without explicit configuration -If you never set `telemetry_level`, CodeCarbon uses `minimal` and logs a **one-time warning** per Python session that Tier 1 will be sent. Set `telemetry_level` explicitly (config, CLI, or tracker argument) to silence it. +If you never set `telemetry_level`, CodeCarbon uses `minimal` (Tier 1) and logs a **one-time warning** per Python session. Set `telemetry_level` explicitly to silence it. ## Related -- [Configure CodeCarbon](configuration.md) — general `.codecarbon.config` hierarchy -- [CLI reference](../reference/cli.md#codecarbon-telemetry) — command flags -- [Cloud API & dashboard](cloud-api.md) — your own emissions data +- [Configure CodeCarbon](configuration.md) +- [CLI reference](../reference/cli.md#codecarbon-telemetry) +- [Cloud API & dashboard](cloud-api.md) diff --git a/docs/reference/api.md b/docs/reference/api.md index 37d2abea0..8b8bea306 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -19,8 +19,8 @@ Optional library telemetry is controlled by **`telemetry_level`** on the tracker | Value | Behavior | |-------|----------| | `disabled` | No product telemetry | -| `minimal` (default) | Tier 1 hardware/environment metadata once per process | -| `extensive` | Tier 1 + public emissions summary on `stop()` | +| `minimal` (default) | Tier 1 private telemetry at each `stop()` | +| `extensive` | Tier 1 + Tier 2 (private telemetry and public run summary) at each `stop()` | **Resolution order:** tracker argument → `.codecarbon.config` → `CODECARBON_TELEMETRY_LEVEL` → default `minimal`. The tracker argument overrides config and environment. diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 7a235f76c..741182d8f 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,19 +1,14 @@ -import platform import sys import tempfile import unittest from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch -import codecarbon.telemetry as telemetry_module from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker -from codecarbon.telemetry import ( - build_minimal_telemetry_dict, - send_tier1_telemetry, - send_tier2_public_emission, -) -from tests.testutils import get_custom_mock_open +from codecarbon.output_methods.emissions_data import EmissionsData +from codecarbon.telemetry import send_tier1_at_stop, send_tier2_at_stop +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": mock_platform_cli_setup = patch( @@ -27,102 +22,16 @@ extensive_conf = "[codecarbon]\ntelemetry_level = extensive\n" -class TestTelemetry(unittest.TestCase): - def test_build_minimal_telemetry_dict_maps_tracker_conf(self): - conf = { - "python_version": platform.python_version(), - "os": platform.platform(), - "cpu_count": 8, - "cpu_physical_count": 4, - "cpu_model": "Intel Core i7", - "gpu_count": 1, - "gpu_model": "NVIDIA RTX 3080", - "ram_total_size": 32.0, - "codecarbon_version": "2.0.0", - "tracking_mode": "machine", - "country_iso_code": "FRA", - "provider": "aws", - "region": "eu-west-1", - } - payload = build_minimal_telemetry_dict(conf) - self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) - self.assertIn("timestamp", payload) - self.assertEqual(payload["python_version"], conf["python_version"]) - self.assertEqual(payload["ram_total_size_gb"], conf["ram_total_size"]) - self.assertEqual(payload["cloud_provider"], conf["provider"]) - self.assertEqual(payload["cloud_region"], conf["region"]) - - -class TestTier1Send(unittest.TestCase): - def test_send_tier1_telemetry_posts_once_per_session(self): - telemetry_module._TIER1_SENT = False - conf = { - "python_version": "3.11.0", - "os": "Linux", - "cpu_count": 4, - "cpu_model": "Intel i5", - "gpu_count": 0, - "codecarbon_version": "2.0.0", - } - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_client_cls.return_value = mock_client - result = send_tier1_telemetry(conf) - self.assertTrue(result) - result = send_tier1_telemetry(conf) - self.assertFalse(result) - mock_client_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() - call_kwargs = mock_client_cls.call_args.kwargs - self.assertEqual( - call_kwargs["telemetry"]["telemetry_level"], TelemetryLevel.minimal.value - ) - - def test_send_tier1_telemetry_uses_resolved_api_url_and_key(self): - telemetry_module._TIER1_SENT = False - external_conf = { - "telemetry_api_url": "http://custom-tier1.example", - "telemetry_api_key": "cpt_custom", - } - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_client_cls.return_value = mock_client - send_tier1_telemetry( - {"codecarbon_version": "2.0"}, external_conf=external_conf - ) - self.assertEqual( - mock_client_cls.call_args.kwargs["endpoint_url"], - "http://custom-tier1.example", - ) - self.assertEqual(mock_client_cls.call_args.kwargs["api_key"], "cpt_custom") - - def test_send_tier1_telemetry_fails_silently(self): - telemetry_module._TIER1_SENT = False - with patch( - "codecarbon.telemetry.build_minimal_telemetry_dict", - side_effect=RuntimeError("build failed"), - ): - with patch("codecarbon.telemetry.logger.error") as mock_logger: - result = send_tier1_telemetry({}) - self.assertFalse(result) - mock_logger.assert_called() - - -class TestTier2Send(unittest.TestCase): - def test_send_tier2_public_emission_sends_once_per_session(self): - telemetry_module._TIER2_SENT = False - from codecarbon.output_methods.emissions_data import EmissionsData - - emissions = EmissionsData( +class TestTelemetryTiersAtStop(unittest.TestCase): + def _emissions(self) -> EmissionsData: + return EmissionsData( timestamp="2020-01-01T00:00:00", project_name="test", run_id="run-1", experiment_id="exp-1", - duration=1.0, + duration=10.0, emissions=0.001, - emissions_rate=0.001, + emissions_rate=0.0001, cpu_power=0.0, gpu_power=0.0, ram_power=0.0, @@ -131,16 +40,16 @@ def test_send_tier2_public_emission_sends_once_per_session(self): ram_energy=0.0, energy_consumed=0.01, water_consumed=0.0, - country_name="", - country_iso_code="", - region="", + country_name="France", + country_iso_code="FRA", + region="idf", cloud_provider="", cloud_region="", os="Linux", python_version="3.11", codecarbon_version="2.0", cpu_count=1.0, - cpu_model="", + cpu_model="cpu", gpu_count=0.0, gpu_model="", longitude=0.0, @@ -148,23 +57,54 @@ def test_send_tier2_public_emission_sends_once_per_session(self): ram_total_size=8.0, tracking_mode="machine", ) + + def test_tier1_posts_private_telemetry(self): + tracker = MagicMock() + emissions = self._emissions() + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = {"id": "ok"} + mock_client_cls.return_value = mock_client + result = send_tier1_at_stop(tracker, emissions, external_conf={}) + self.assertTrue(result) + posted = mock_client_cls.call_args.kwargs["telemetry"] + self.assertEqual(posted["telemetry_level"], "minimal") + self.assertEqual(posted["total_emissions_kg"], 0.001) + + def test_tier1_skips_short_duration(self): + tracker = MagicMock() + emissions = self._emissions() + emissions.duration = 0.5 + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + result = send_tier1_at_stop(tracker, emissions, external_conf={}) + self.assertFalse(result) + mock_client_cls.assert_not_called() + + def test_tier2_uses_api_client(self): + tracker = MagicMock() + tracker._conf = {"os": "Linux"} + emissions = self._emissions() with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api - result = send_tier2_public_emission({}, emissions) - self.assertTrue(result) - mock_api.add_emission.assert_called_once() - result = send_tier2_public_emission({}, emissions) - self.assertFalse(result) - mock_api.add_emission.assert_called_once() + result = send_tier2_at_stop(tracker, emissions, external_conf={}) + self.assertTrue(result) + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + mock_api_cls.assert_called_with( + endpoint_url=ANY, + experiment_id=ANY, + api_key=ANY, + conf=tracker._conf, + create_run_automatically=True, + ) @mock_platform_cli_setup class TestTrackerTelemetry(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() - self.temp_path = Path(self.temp_dir.name) self.patcher = None def tearDown(self) -> None: @@ -178,63 +118,22 @@ def _start_config_mock(self, conf: str) -> None: ) self.patcher.start() - def test_emissions_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False + def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker(save_to_api=False, save_to_file=False) - mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() - - def test_emissions_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False - self._start_config_mock(disabled_conf) with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) mock_telemetry_cls.assert_not_called() - def test_offline_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False + def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( + self, mock_cli_setup + ): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - OfflineEmissionsTracker( - country_iso_code="CAN", - save_to_api=False, - save_to_file=False, - ) - mock_telemetry_cls.assert_called_once() - - def test_offline_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False - self._start_config_mock(disabled_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - OfflineEmissionsTracker( - country_iso_code="CAN", - save_to_api=False, - save_to_file=False, - ) - mock_telemetry_cls.assert_not_called() - - def test_extensive_sends_tier1_on_init_and_tier2_on_stop(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False - telemetry_module._TIER2_SENT = False - self._start_config_mock(extensive_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_telemetry = MagicMock() - mock_telemetry.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_telemetry - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = {"id": "ok"} + mock_telemetry_cls.return_value = mock_client with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -244,9 +143,64 @@ def test_extensive_sends_tier1_on_init_and_tier2_on_stop(self, mock_cli_setup): tracker.start() tracker.stop() mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + payload = mock_telemetry_cls.call_args.kwargs["telemetry"] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertIn("total_emissions_kg", payload) + + def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): + self._start_config_mock(disabled_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_not_called() + + def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): + self._start_config_mock(extensive_conf) + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_telemetry = MagicMock() + mock_telemetry.add_telemetry.return_value = {"id": "ok"} + mock_telemetry_cls.return_value = mock_telemetry + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() mock_telemetry.add_telemetry.assert_called_once() + mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() + def test_offline_tracker_sends_on_stop(self, mock_cli_setup): + self._start_config_mock(minimal_conf) + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = {"id": "ok"} + mock_telemetry_cls.return_value = mock_client + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_client.add_telemetry.assert_called_once() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 5ce145370..faefc555a 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -93,7 +93,7 @@ def test_init_rejects_invalid_telemetry_without_calling_api(self): telemetry={ "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "total_emissions_kg": 0.42, + "torch_version": "2.2.0", }, ) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py new file mode 100644 index 000000000..774808d94 --- /dev/null +++ b/tests/test_telemetry_collect.py @@ -0,0 +1,113 @@ +import unittest +from unittest.mock import MagicMock, patch + +from codecarbon.core.telemetry_collect import ( + build_tier1_payload, + collect_telemetry_context, + project_tier1, +) +from codecarbon.output_methods.emissions_data import EmissionsData + + +def _sample_emissions(**overrides): + base = dict( + timestamp="2026-01-01T00:00:00", + project_name="p", + run_id="r", + experiment_id="e", + duration=10.0, + emissions=0.5, + emissions_rate=0.05, + cpu_power=1.0, + gpu_power=2.0, + ram_power=0.5, + cpu_energy=0.01, + gpu_energy=0.02, + ram_energy=0.001, + energy_consumed=0.031, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="3.0", + cpu_count=4, + cpu_model="cpu", + gpu_count=1, + gpu_model="gpu", + longitude=0.0, + latitude=0.0, + ram_total_size=16.0, + tracking_mode="machine", + ) + base.update(overrides) + return EmissionsData(**base) + + +class TestTelemetryCollect(unittest.TestCase): + def test_project_tier1_includes_run_and_framework_flags(self): + tracker = MagicMock() + tracker._conf = { + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + } + tracker._geo = None + tracker._save_to_file = True + tracker._save_to_api = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + tracker._hardware = [] + tracker._resource_tracker = None + + emissions = _sample_emissions() + with patch( + "codecarbon.core.telemetry_collect._package_installed", + side_effect=lambda name: name == "torch", + ): + context = collect_telemetry_context(tracker, emissions) + payload = project_tier1(context) + + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertEqual(payload["total_emissions_kg"], 0.5) + self.assertEqual(payload["duration_seconds"], 10.0) + self.assertTrue(payload["has_torch"]) + self.assertNotIn("torch_version", payload) + self.assertIn("file", payload["output_methods"]) + + def test_build_tier1_payload_excludes_framework_versions(self): + tracker = MagicMock() + tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} + tracker._geo = None + tracker._save_to_file = False + tracker._save_to_api = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + tracker._hardware = [] + tracker._resource_tracker = None + + emissions = _sample_emissions() + with patch( + "codecarbon.core.telemetry_collect._package_installed", + return_value=True, + ), patch( + "codecarbon.core.telemetry_collect._package_version", + return_value="2.0.0", + ): + payload = build_tier1_payload(tracker, emissions) + + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertTrue(payload["has_torch"]) + self.assertNotIn("torch_version", payload) diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 57845f32f..e90b5987f 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -4,6 +4,7 @@ import sys import tempfile import unittest +from datetime import datetime, timezone from pathlib import Path from unittest.mock import MagicMock, patch @@ -13,12 +14,9 @@ from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import get_telemetry_api_url from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker -from codecarbon.telemetry import ( - send_tier1_telemetry, - send_tier2_public_emission, - warn_if_telemetry_not_configured, -) -from tests.testutils import get_custom_mock_open +from codecarbon.output_methods.emissions_data import EmissionsData +from codecarbon.telemetry import send_tier1_at_stop, warn_if_telemetry_not_configured +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": mock_platform_cli_setup = patch( @@ -50,28 +48,58 @@ def test_no_warn_when_config_explicit(self): ) mock_warning.assert_not_called() - def test_minimal_tier1_posts_once_per_session(self): - telemetry_module._TIER1_SENT = False - conf = {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"} - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_client_cls.return_value = mock_client - self.assertTrue(send_tier1_telemetry(conf)) - self.assertFalse(send_tier1_telemetry(conf)) - mock_client_cls.assert_called_once() - payload = mock_client_cls.call_args.kwargs["telemetry"] - self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) - def test_tier1_posts_to_telemetry_endpoint(self): - telemetry_module._TIER1_SENT = False - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: - mock_post.return_value.status_code = 201 - mock_post.return_value.json.return_value = "telemetry-id" - send_tier1_telemetry( - {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"}, - external_conf={"telemetry_api_url": "http://tier1.example"}, - ) + tier1_payload = { + "timestamp": datetime(2020, 1, 1, tzinfo=timezone.utc), + "telemetry_level": "minimal", + "total_emissions_kg": 0.001, + "os": "Linux", + } + tracker = MagicMock() + emissions = EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=10.0, + emissions=0.001, + emissions_rate=0.0001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="cpu", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + with patch( + "codecarbon.telemetry.build_tier1_payload", return_value=tier1_payload + ): + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + send_tier1_at_stop( + tracker, + emissions, + external_conf={"telemetry_api_url": "http://tier1.example"}, + ) mock_post.assert_called_once() self.assertEqual( mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" @@ -80,6 +108,7 @@ def test_tier1_posts_to_telemetry_endpoint(self): mock_post.call_args.kwargs["json"]["telemetry_level"], TelemetryLevel.minimal.value, ) + self.assertIn("total_emissions_kg", mock_post.call_args.kwargs["json"]) def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): with tempfile.TemporaryDirectory() as tmp: @@ -142,8 +171,6 @@ def test_telemetry_api_url_from_config_overrides_default(self): @mock_platform_cli_setup class TestTrackerTelemetryFromConfig(unittest.TestCase): def setUp(self) -> None: - telemetry_module._TIER1_SENT = False - telemetry_module._TIER2_SENT = False telemetry_module._TELEMETRY_CONFIGURE_WARNED = False self._config_patcher = None @@ -157,28 +184,26 @@ def _mock_config(self, conf: str) -> None: ) self._config_patcher.start() - def test_disabled_no_tier1_post_or_tier2_emission(self, mock_cli_setup): + def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): self._mock_config(_conf("disabled")) with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( - measure_power_secs=1, - save_to_api=False, - save_to_file=False, - ) - tracker.start() - tracker.stop() + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() mock_telemetry_cls.assert_not_called() - mock_api_cls.assert_not_called() - def test_minimal_posts_tier1_on_init_not_tier2_on_stop(self, mock_cli_setup): + def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("minimal")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -189,44 +214,50 @@ def test_minimal_posts_tier1_on_init_not_tier2_on_stop(self, mock_cli_setup): tracker.stop() mock_telemetry_cls.assert_called_once() mock_client.add_telemetry.assert_called_once() - mock_api_cls.assert_not_called() + self.assertEqual( + mock_telemetry_cls.call_args.kwargs["telemetry"]["telemetry_level"], + TelemetryLevel.minimal.value, + ) - def test_extensive_posts_tier1_and_tier2_on_stop(self, mock_cli_setup): + def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("extensive")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( - measure_power_secs=1, - save_to_api=False, - save_to_file=False, - ) - tracker.start() - tracker.stop() + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_telemetry = MagicMock() + mock_telemetry.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_telemetry + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() + mock_telemetry.add_telemetry.assert_called_once() + mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() - def test_offline_minimal_with_save_to_api_false_still_posts_tier1( - self, mock_cli_setup - ): + def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): self._mock_config(_conf("minimal")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - OfflineEmissionsTracker( - country_iso_code="CAN", - save_to_api=False, - save_to_file=False, - ) - mock_telemetry_cls.assert_called_once() + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_client.add_telemetry.assert_called_once() def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): self._mock_config("[codecarbon]\n") @@ -271,69 +302,23 @@ def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): ] self.assertEqual(len(configure_warnings), 0) - def test_env_telemetry_disabled_still_minimal_tier1_when_config_minimal( + def test_env_telemetry_disabled_does_not_change_resolved_level( self, mock_cli_setup ): self._mock_config(_conf("minimal")) - with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker(save_to_api=False, save_to_file=False) + with ensure_telemetry_run_duration(): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False + ): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + save_to_api=False, save_to_file=False + ) + tracker.start() + tracker.stop() mock_telemetry_cls.assert_called_once() -class TestTier2ApiSettings(unittest.TestCase): - def test_tier2_uses_resolved_api_url(self): - telemetry_module._TIER2_SENT = False - from codecarbon.output_methods.emissions_data import EmissionsData - - emissions = EmissionsData( - timestamp="2020-01-01T00:00:00", - project_name="test", - run_id="run-1", - experiment_id="exp-1", - duration=1.0, - emissions=0.001, - emissions_rate=0.001, - cpu_power=0.0, - gpu_power=0.0, - ram_power=0.0, - cpu_energy=0.0, - gpu_energy=0.0, - ram_energy=0.0, - energy_consumed=0.01, - water_consumed=0.0, - country_name="", - country_iso_code="", - region="", - cloud_provider="", - cloud_region="", - os="Linux", - python_version="3.11", - codecarbon_version="2.0", - cpu_count=1.0, - cpu_model="", - gpu_count=0.0, - gpu_model="", - longitude=0.0, - latitude=0.0, - ram_total_size=8.0, - tracking_mode="machine", - ) - external_conf = {"telemetry_api_url": "http://custom.example"} - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api - send_tier2_public_emission({}, emissions, external_conf=external_conf) - mock_api_cls.assert_called_once() - self.assertEqual( - mock_api_cls.call_args.kwargs["endpoint_url"], "http://custom.example" - ) - - if __name__ == "__main__": unittest.main() diff --git a/tests/testutils.py b/tests/testutils.py index e3d1dc2d1..57d94fa46 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -1,6 +1,8 @@ import builtins import unittest +from contextlib import contextmanager from pathlib import Path +from unittest.mock import patch from codecarbon.input import DataSource @@ -33,3 +35,22 @@ def conditional_open_func(path, *args, **kwargs): return conditional_open_func return mocked_open + + +@contextmanager +def ensure_telemetry_run_duration(min_seconds: float = 10.0): + """Force tracker stop emissions duration above telemetry's 1s minimum.""" + from codecarbon.emissions_tracker import BaseEmissionsTracker + + original_prepare = BaseEmissionsTracker._prepare_emissions_data + + def prepare_with_min_duration(self): + data = original_prepare(self) + if data is not None and (data.duration is None or data.duration < min_seconds): + data.duration = min_seconds + return data + + with patch.object( + BaseEmissionsTracker, "_prepare_emissions_data", prepare_with_min_duration + ): + yield From 1ecc23775cb55d4e0dda92748b861f77b822724e Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 23:18:37 +0200 Subject: [PATCH 14/29] refactor: update telemetry schema and enhance data collection - Replaced MINIMAL_TELEMETRY_FIELDS with a dynamic frozenset that excludes privacy-related fields. - Introduced EXCLUDED_PRIVACY_TELEMETRY_FIELDS to ensure sensitive data is not collected. - Updated telemetry payload building methods to accept a telemetry level parameter, allowing for more flexible data handling. - Enhanced tests to validate the acceptance of framework versions and rejection of privacy fields in telemetry submissions. - Improved documentation to clarify the changes in telemetry data collection and privacy considerations. --- .../carbonserver/api/schemas_telemetry.py | 68 +++----------- .../tests/api/routers/test_telemetry.py | 21 ++++- codecarbon/core/telemetry_collect.py | 93 +++++++++++++++++-- codecarbon/core/telemetry_schemas.py | 68 +++----------- codecarbon/telemetry.py | 6 +- docs/how-to/telemetry.md | 25 ++--- tests/test_telemetry_client.py | 2 +- tests/test_telemetry_collect.py | 26 +++++- 8 files changed, 171 insertions(+), 138 deletions(-) diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index dec7e0dfa..9c8c860ee 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -144,60 +144,20 @@ def validate_telemetry_level(self): return self -MINIMAL_TELEMETRY_FIELDS = { - "timestamp", - "telemetry_level", - "os", - "country_name", - "country_iso_code", - "region", - "cloud_provider", - "cloud_region", - "on_cloud", - "cpu_count", - "cpu_physical_count", - "cpu_model", - "cpu_architecture", - "gpu_count", - "gpu_model", - "gpu_memory_total_gb", - "ram_total_size_gb", - "cuda_version", - "python_version", - "python_implementation", - "python_env_type", - "python_package_manager", - "codecarbon_version", - "codecarbon_install_method", - "tracking_mode", - "integration_surface", - "offline_mode", - "output_methods", - "save_to_api_enabled", - "task_tracking_used", - "measure_power_interval_secs", - "in_container", - "ci_environment", - "notebook_environment", - "has_torch", - "has_transformers", - "has_tensorflow", - "has_keras", - "has_diffusers", - "has_pytorch_lightning", - "has_fastai", - "ml_framework_primary", - "total_emissions_kg", - "emissions_rate_kg_per_sec", - "energy_consumed_kwh", - "cpu_energy_kwh", - "gpu_energy_kwh", - "ram_energy_kwh", - "duration_seconds", - "cpu_utilization_avg", - "gpu_utilization_avg", - "ram_utilization_avg", -} +EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( + { + "longitude", + "latitude", + "python_executable_hash", + "host_machine_hash", + } +) + +MINIMAL_TELEMETRY_FIELDS = frozenset( + field_name + for field_name in TelemetryBase.model_fields + if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS +) class TelemetryCreate(TelemetryBase): diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index 733269bb0..f8ce065ea 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -52,15 +52,30 @@ def test_add_telemetry(client, custom_test_server): assert response.json() == TELEMETRY_ID -def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): +def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) - telemetry_with_extensive_field = { + repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) + telemetry_with_framework_version = { **MINIMAL_TELEMETRY_TO_CREATE, "torch_version": "2.2.0", } with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_extensive_field) + response = client.post("/telemetry", json=telemetry_with_framework_version) + + assert response.status_code == status.HTTP_201_CREATED + repository_mock.add_telemetry.assert_called_once() + + +def test_minimal_telemetry_rejects_privacy_fields(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + telemetry_with_privacy_field = { + **MINIMAL_TELEMETRY_TO_CREATE, + "longitude": 2.35, + } + + with custom_test_server.container.telemetry_repository.override(repository_mock): + response = client.post("/telemetry", json=telemetry_with_privacy_field) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index bdd3ed676..cd40da6f6 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -129,6 +129,67 @@ def _detect_in_container() -> bool: return False +def _detect_container_runtime() -> Optional[str]: + if os.environ.get("KUBERNETES_SERVICE_HOST"): + return "kubernetes" + if os.path.exists("/.dockerenv"): + return "docker" + return None + + +def _detect_ide() -> Optional[str]: + if os.environ.get("CURSOR_TRACE_ID") or os.environ.get("CURSOR_SESSION"): + return "cursor" + if os.environ.get("VSCODE_PID") or os.environ.get("TERM_PROGRAM") == "vscode": + return "vscode" + if os.environ.get("PYCHARM_HOSTED"): + return "pycharm" + return None + + +def _cudnn_version() -> Optional[str]: + if not _package_installed("torch"): + return None + try: + import torch + + version = torch.backends.cudnn.version() + return str(version) if version is not None else None + except Exception: + return None + + +def _collect_hardware_diagnostics(tracker: Any) -> dict[str, Any]: + from codecarbon.core import cpu + + hardware_tracked: list[str] = [] + for item in getattr(tracker, "_hardware", []) or []: + try: + hardware_tracked.append(item.description()) + except Exception: + pass + + resource_tracker = getattr(tracker, "_resource_tracker", None) + gpu_detection_method: Optional[str] = None + if resource_tracker is not None: + gpu_tracker = getattr(resource_tracker, "gpu_tracker", None) + if gpu_tracker and gpu_tracker != "Unspecified": + gpu_detection_method = gpu_tracker + + rapl_available: Optional[bool] = None + if platform.system() == "Linux": + rapl_available = cpu.is_rapl_available() + + save_to_api = bool(getattr(tracker, "_save_to_api", False)) + return { + "hardware_tracked": hardware_tracked or None, + "hardware_detection_success": bool(hardware_tracked), + "rapl_available": rapl_available, + "gpu_detection_method": gpu_detection_method, + "api_mode": "online" if save_to_api else "offline", + } + + def _detect_integration_surface(tracker: Any) -> str: from codecarbon.emissions_tracker import OfflineEmissionsTracker @@ -266,8 +327,11 @@ def collect_telemetry_context( "task_tracking_used": bool(getattr(tracker, "_tasks", {})), "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), "in_container": _detect_in_container(), + "container_runtime": _detect_container_runtime(), "ci_environment": _detect_ci_environment(), "notebook_environment": _detect_notebook_environment(), + "ide_used": _detect_ide(), + "cudnn_version": _cudnn_version(), "duration_seconds": float(emissions.duration) if emissions.duration else None, "total_emissions_kg": emissions.emissions, "emissions_rate_kg_per_sec": emissions.emissions_rate, @@ -278,36 +342,45 @@ def collect_telemetry_context( "cpu_utilization_avg": emissions.cpu_utilization_percent, "gpu_utilization_avg": emissions.gpu_utilization_percent, "ram_utilization_avg": emissions.ram_utilization_percent, - **_collect_framework_fields(include_versions=False), + **_collect_framework_fields(include_versions=True), + **_collect_hardware_diagnostics(tracker), } - for key in ("gpu_memory_total_gb", "cuda_version"): - if key in gpu_fields: - context[key] = gpu_fields[key] + context.update(gpu_fields) + if context.get("ml_framework_primary"): + context["framework_detected"] = context["ml_framework_primary"] return _strip_none(context) -def project_tier1(context: dict[str, Any]) -> dict[str, Any]: - """Project context to Tier 1 (``telemetry_level=minimal``) fields.""" +def project_tier1( + context: dict[str, Any], + level: TelemetryLevel = TelemetryLevel.minimal, +) -> dict[str, Any]: + """Project context to private ``POST /telemetry`` fields for the resolved tier.""" payload = { key: context[key] for key in MINIMAL_TELEMETRY_FIELDS if key in context } - payload["telemetry_level"] = TelemetryLevel.minimal.value + payload["telemetry_level"] = level.value return _strip_none(payload) -def build_tier1_payload(tracker: Any, emissions: EmissionsData) -> dict[str, Any]: - """Build a Tier 1 payload dict for ``TelemetryCreate``. +def build_tier1_payload( + tracker: Any, + emissions: EmissionsData, + level: TelemetryLevel = TelemetryLevel.minimal, +) -> dict[str, Any]: + """Build a private telemetry payload dict for ``TelemetryCreate``. Args: tracker: Active emissions tracker. emissions: Run emissions data. + level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). Returns: Payload dict for ``POST /telemetry``. """ context = collect_telemetry_context(tracker, emissions) - return project_tier1(context) + return project_tier1(context, level=level) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry_schemas.py index 9dc3e2997..6fcb28446 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry_schemas.py @@ -125,60 +125,20 @@ def validate_telemetry_level(self): return self -MINIMAL_TELEMETRY_FIELDS = { - "timestamp", - "telemetry_level", - "os", - "country_name", - "country_iso_code", - "region", - "cloud_provider", - "cloud_region", - "on_cloud", - "cpu_count", - "cpu_physical_count", - "cpu_model", - "cpu_architecture", - "gpu_count", - "gpu_model", - "gpu_memory_total_gb", - "ram_total_size_gb", - "cuda_version", - "python_version", - "python_implementation", - "python_env_type", - "python_package_manager", - "codecarbon_version", - "codecarbon_install_method", - "tracking_mode", - "integration_surface", - "offline_mode", - "output_methods", - "save_to_api_enabled", - "task_tracking_used", - "measure_power_interval_secs", - "in_container", - "ci_environment", - "notebook_environment", - "has_torch", - "has_transformers", - "has_tensorflow", - "has_keras", - "has_diffusers", - "has_pytorch_lightning", - "has_fastai", - "ml_framework_primary", - "total_emissions_kg", - "emissions_rate_kg_per_sec", - "energy_consumed_kwh", - "cpu_energy_kwh", - "gpu_energy_kwh", - "ram_energy_kwh", - "duration_seconds", - "cpu_utilization_avg", - "gpu_utilization_avg", - "ram_utilization_avg", -} +EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( + { + "longitude", + "latitude", + "python_executable_hash", + "host_machine_hash", + } +) + +MINIMAL_TELEMETRY_FIELDS = frozenset( + field_name + for field_name in TelemetryBase.model_fields + if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS +) class TelemetryCreate(TelemetryBase): diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 397be674b..20cdbeb65 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -64,6 +64,7 @@ def send_tier1_at_stop( tracker: Any, emissions: EmissionsData, external_conf: dict[str, Any] | None = None, + level: TelemetryLevel = TelemetryLevel.minimal, ) -> bool: """Send Tier 1 telemetry: private hardware/usage/run summary via ``POST /telemetry``. @@ -71,6 +72,7 @@ def send_tier1_at_stop( tracker: Active emissions tracker instance. emissions: Total emissions from ``_prepare_emissions_data()``. external_conf: Merged config for telemetry API URL and key resolution. + level: Resolved ``TelemetryLevel`` for the ``telemetry_level`` field. Returns: True if Tier 1 was accepted, False otherwise. @@ -82,7 +84,7 @@ def send_tier1_at_stop( return False settings_conf = external_conf or {} try: - payload = build_tier1_payload(tracker, emissions) + payload = build_tier1_payload(tracker, emissions, level=level) client = TelemetryClient( endpoint_url=get_telemetry_api_url(settings_conf), telemetry=payload, @@ -151,6 +153,6 @@ def send_product_telemetry_at_stop( return settings = external_conf or {} if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_tier1_at_stop(tracker, emissions, settings) + send_tier1_at_stop(tracker, emissions, settings, level=level) if level == TelemetryLevel.extensive: send_tier2_at_stop(tracker, emissions, settings) diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 181605047..82ea56f7d 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -17,8 +17,8 @@ You can use one without the other. | `telemetry_level` | Name | When | Transport | |-------------------|------|------|-----------| | `disabled` | — | — | Nothing | -| `minimal` | **Tier 1** | Each `stop()` | `POST /telemetry` (private) | -| `extensive` | **Tier 2** | Each `stop()` | Tier 1 (`POST /telemetry`) **and** Tier 2 (`ApiClient` → `/emissions`) | +| `minimal` | Private product telemetry | Each `stop()` | `POST /telemetry` (private) | +| `extensive` | Private telemetry + shared run summary | Each `stop()` | Same private `POST /telemetry` **and** `ApiClient` → `/emissions` | Tier is resolved in this order: @@ -30,30 +30,33 @@ Tier is resolved in this order: ```text EmissionsTracker.__init__ → collect hardware/geo (no POST) -EmissionsTracker.stop() → minimal: Tier 1 only | extensive: Tier 1 + Tier 2 +EmissionsTracker.stop() → minimal: private POST only | extensive: private POST + /emissions ``` If the run lasts less than one second, telemetry is not sent. -## Tier 1 (`minimal`) — per run +## Private telemetry (`minimal` and `extensive`) — per run -One private row per tracker run with: +Both levels send the **same private payload** to `POST /telemetry` at each `stop()`. The `telemetry_level` field records which setting was used (`minimal` or `extensive`). -- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region -- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container hints -- **ML stack (presence):** `has_torch`, `has_transformers`, `has_tensorflow`, and related flags +The payload includes: + +- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region, GPU driver/CUDA/cuDNN when available +- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container/IDE hints, hardware diagnostics +- **ML stack:** framework presence flags **and** installed versions when detected - **Run outcome:** duration, emissions, energy (total and per component), utilization averages -Tier 1 does **not** include project names, experiment ids, API keys, file paths, or survey demographics (role, industry, etc.). +Private telemetry does **not** include project names, experiment ids, API keys, file paths, exact coordinates, executable/host hashes, or survey demographics (role, industry, etc.). -## Tier 2 (`extensive`) — per run +## `extensive` — additional public run summary -**Always sends Tier 1 first**, then adds a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. +**Also** posts a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. ## Never collected - Project name, experiment id, run id, API keys - Source code, file paths, hostnames +- Exact GPS coordinates (`longitude` / `latitude`), `python_executable_hash`, `host_machine_hash` - Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) ## Configure telemetry diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index faefc555a..4d2b24b32 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -93,7 +93,7 @@ def test_init_rejects_invalid_telemetry_without_calling_api(self): telemetry={ "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "torch_version": "2.2.0", + "longitude": 2.35, }, ) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 774808d94..1b711ad66 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -6,6 +6,7 @@ collect_telemetry_context, project_tier1, ) +from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData @@ -80,10 +81,9 @@ def test_project_tier1_includes_run_and_framework_flags(self): self.assertEqual(payload["total_emissions_kg"], 0.5) self.assertEqual(payload["duration_seconds"], 10.0) self.assertTrue(payload["has_torch"]) - self.assertNotIn("torch_version", payload) self.assertIn("file", payload["output_methods"]) - def test_build_tier1_payload_excludes_framework_versions(self): + def test_build_tier1_payload_includes_framework_versions(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} tracker._geo = None @@ -110,4 +110,24 @@ def test_build_tier1_payload_excludes_framework_versions(self): self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) - self.assertNotIn("torch_version", payload) + self.assertEqual(payload["torch_version"], "2.0.0") + + def test_build_tier1_payload_uses_resolved_level(self): + tracker = MagicMock() + tracker._conf = {"codecarbon_version": "3.0"} + tracker._save_to_file = False + tracker._save_to_api = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + tracker._hardware = [] + tracker._resource_tracker = None + + emissions = _sample_emissions() + payload = build_tier1_payload( + tracker, emissions, level=TelemetryLevel.extensive + ) + self.assertEqual(payload["telemetry_level"], "extensive") From 6c873eb29551749842acf80d0dc5afa0c2f02885 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 23:24:02 +0200 Subject: [PATCH 15/29] refactor: streamline telemetry schema and enhance privacy measures - Removed sensitive fields from the TelemetryBase schema, including `longitude`, `latitude`, and `python_executable_hash`. - Consolidated privacy-related field management into a single frozenset, `PRIVATE_TELEMETRY_FIELDS`. - Updated telemetry payload building methods to reflect the new schema and ensure compliance with privacy standards. - Refactored tests to validate the rejection of unknown fields and ensure proper handling of telemetry submissions. - Improved documentation to clarify the changes in telemetry data collection and privacy considerations. --- .../carbonserver/api/schemas_telemetry.py | 41 +------------------ .../tests/api/routers/test_telemetry.py | 18 ++++---- codecarbon/core/telemetry_collect.py | 15 +++---- codecarbon/core/telemetry_schemas.py | 40 +----------------- codecarbon/telemetry.py | 32 +++++++-------- docs/how-to/telemetry.md | 2 +- tests/test_telemetry.py | 17 ++++++-- tests/test_telemetry_client.py | 2 +- tests/test_telemetry_collect.py | 16 ++++---- tests/test_telemetry_config.py | 9 ++-- 10 files changed, 62 insertions(+), 130 deletions(-) diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index 9c8c860ee..ced8c13fd 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -42,8 +42,6 @@ class TelemetryBase(BaseModel): cloud_provider: Optional[str] = None cloud_region: Optional[str] = None on_cloud: Optional[bool] = None - longitude: Optional[float] = Field(default=None, ge=-180, le=180) - latitude: Optional[float] = Field(default=None, ge=-90, le=90) cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -59,9 +57,6 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None - python_executable_hash: Optional[str] = Field( - default=None, min_length=64, max_length=64 - ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None @@ -83,7 +78,6 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None - decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) integration_surface: Optional[str] = None offline_mode: Optional[bool] = None @@ -92,15 +86,10 @@ class TelemetryBase(BaseModel): hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None - first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) - tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) - errors_encountered: Optional[List[str]] = None - warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None - python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -121,43 +110,15 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None - host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") - - if self.telemetry_level == TelemetryLevel.minimal: - extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS - submitted_extensive_fields = [ - field - for field in extensive_fields - if getattr(self, field) not in (None, [], {}) - ] - if submitted_extensive_fields: - fields = ", ".join(sorted(submitted_extensive_fields)) - raise ValueError( - f"Minimal telemetry cannot include extensive fields: {fields}" - ) - return self -EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( - { - "longitude", - "latitude", - "python_executable_hash", - "host_machine_hash", - } -) - -MINIMAL_TELEMETRY_FIELDS = frozenset( - field_name - for field_name in TelemetryBase.model_fields - if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS -) +PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) class TelemetryCreate(TelemetryBase): diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index f8ce065ea..a51a19ecd 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -14,7 +14,7 @@ TELEMETRY_ID = "f52fe339-164d-4c2b-a8c0-f562dfce066d" -MINIMAL_TELEMETRY_TO_CREATE = { +SAMPLE_PRIVATE_TELEMETRY = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", "os": "Linux-5.10.0-x86_64", @@ -46,7 +46,7 @@ def test_add_telemetry(client, custom_test_server): repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=MINIMAL_TELEMETRY_TO_CREATE) + response = client.post("/telemetry", json=SAMPLE_PRIVATE_TELEMETRY) assert response.status_code == status.HTTP_201_CREATED assert response.json() == TELEMETRY_ID @@ -56,7 +56,7 @@ def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server repository_mock = mock.Mock(spec=TelemetryRepository) repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) telemetry_with_framework_version = { - **MINIMAL_TELEMETRY_TO_CREATE, + **SAMPLE_PRIVATE_TELEMETRY, "torch_version": "2.2.0", } @@ -67,15 +67,15 @@ def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server repository_mock.add_telemetry.assert_called_once() -def test_minimal_telemetry_rejects_privacy_fields(client, custom_test_server): +def test_telemetry_rejects_unknown_fields(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) - telemetry_with_privacy_field = { - **MINIMAL_TELEMETRY_TO_CREATE, - "longitude": 2.35, + telemetry_with_unknown_field = { + **SAMPLE_PRIVATE_TELEMETRY, + "unknown_field": "value", } with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_privacy_field) + response = client.post("/telemetry", json=telemetry_with_unknown_field) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() @@ -84,7 +84,7 @@ def test_minimal_telemetry_rejects_privacy_fields(client, custom_test_server): def test_disabled_telemetry_is_rejected(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) disabled_telemetry = { - **MINIMAL_TELEMETRY_TO_CREATE, + **SAMPLE_PRIVATE_TELEMETRY, "telemetry_level": "disabled", } diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index cd40da6f6..4122e91dc 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -11,10 +11,7 @@ from codecarbon.core.cloud import get_env_cloud_details from codecarbon.core.gpu import is_nvidia_system -from codecarbon.core.telemetry_schemas import ( - MINIMAL_TELEMETRY_FIELDS, - TelemetryLevel, -) +from codecarbon.core.telemetry_schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData FRAMEWORK_PACKAGES = ( @@ -283,7 +280,7 @@ def collect_telemetry_context( emissions: Total emissions row from ``_prepare_emissions_data()``. Returns: - Flat dictionary for ``project_tier1``. + Flat dictionary for ``project_private_telemetry``. """ conf = getattr(tracker, "_conf", {}) raw_provider, raw_region = _raw_cloud_provider_and_region() @@ -353,21 +350,21 @@ def collect_telemetry_context( return _strip_none(context) -def project_tier1( +def project_private_telemetry( context: dict[str, Any], level: TelemetryLevel = TelemetryLevel.minimal, ) -> dict[str, Any]: """Project context to private ``POST /telemetry`` fields for the resolved tier.""" payload = { key: context[key] - for key in MINIMAL_TELEMETRY_FIELDS + for key in PRIVATE_TELEMETRY_FIELDS if key in context } payload["telemetry_level"] = level.value return _strip_none(payload) -def build_tier1_payload( +def build_telemetry_payload( tracker: Any, emissions: EmissionsData, level: TelemetryLevel = TelemetryLevel.minimal, @@ -383,4 +380,4 @@ def build_tier1_payload( Payload dict for ``POST /telemetry``. """ context = collect_telemetry_context(tracker, emissions) - return project_tier1(context, level=level) + return project_private_telemetry(context, level=level) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry_schemas.py index 6fcb28446..bc6ac2dc2 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry_schemas.py @@ -24,8 +24,6 @@ class TelemetryBase(BaseModel): cloud_provider: Optional[str] = None cloud_region: Optional[str] = None on_cloud: Optional[bool] = None - longitude: Optional[float] = Field(default=None, ge=-180, le=180) - latitude: Optional[float] = Field(default=None, ge=-90, le=90) cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -41,9 +39,6 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None - python_executable_hash: Optional[str] = Field( - default=None, min_length=64, max_length=64 - ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None @@ -65,7 +60,6 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None - decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) integration_surface: Optional[str] = None offline_mode: Optional[bool] = None @@ -74,10 +68,6 @@ class TelemetryBase(BaseModel): hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None - first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) - tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) - errors_encountered: Optional[List[str]] = None - warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None @@ -102,43 +92,15 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None - host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") - - if self.telemetry_level == TelemetryLevel.minimal: - extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS - submitted_extensive_fields = [ - field - for field in extensive_fields - if getattr(self, field) not in (None, [], {}) - ] - if submitted_extensive_fields: - fields = ", ".join(sorted(submitted_extensive_fields)) - raise ValueError( - f"Minimal telemetry cannot include extensive fields: {fields}" - ) - return self -EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( - { - "longitude", - "latitude", - "python_executable_hash", - "host_machine_hash", - } -) - -MINIMAL_TELEMETRY_FIELDS = frozenset( - field_name - for field_name in TelemetryBase.model_fields - if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS -) +PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) class TelemetryCreate(TelemetryBase): diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 20cdbeb65..ed73537b4 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,11 +1,11 @@ -"""Product telemetry sent at tracker stop (Tier 1 private, Tier 2 public emissions).""" +"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" import dataclasses from typing import Any from codecarbon.core.api_client import ApiClient from codecarbon.core.telemetry_client import TelemetryClient -from codecarbon.core.telemetry_collect import build_tier1_payload +from codecarbon.core.telemetry_collect import build_telemetry_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import ( get_telemetry_api_key, @@ -60,13 +60,13 @@ def _run_too_short_for_telemetry(emissions: EmissionsData) -> bool: return emissions.duration is not None and emissions.duration < 1 -def send_tier1_at_stop( +def send_private_telemetry_at_stop( tracker: Any, emissions: EmissionsData, external_conf: dict[str, Any] | None = None, level: TelemetryLevel = TelemetryLevel.minimal, ) -> bool: - """Send Tier 1 telemetry: private hardware/usage/run summary via ``POST /telemetry``. + """Send Tier 1 private telemetry via ``POST /telemetry``. Args: tracker: Active emissions tracker instance. @@ -75,16 +75,16 @@ def send_tier1_at_stop( level: Resolved ``TelemetryLevel`` for the ``telemetry_level`` field. Returns: - True if Tier 1 was accepted, False otherwise. + True if the private telemetry POST was accepted, False otherwise. """ if _run_too_short_for_telemetry(emissions): logger.debug( - "Tier 1 telemetry not sent because run duration is shorter than 1 second." + "Private telemetry not sent because run duration is shorter than 1 second." ) return False settings_conf = external_conf or {} try: - payload = build_tier1_payload(tracker, emissions, level=level) + payload = build_telemetry_payload(tracker, emissions, level=level) client = TelemetryClient( endpoint_url=get_telemetry_api_url(settings_conf), telemetry=payload, @@ -92,16 +92,16 @@ def send_tier1_at_stop( ) return client.add_telemetry() is not None except Exception as error: - logger.error(f"Tier 1 telemetry failed (non-critical): {error}") + logger.error(f"Private telemetry failed (non-critical): {error}") return False -def send_tier2_at_stop( +def send_public_run_summary_at_stop( tracker: Any, emissions: EmissionsData, external_conf: dict[str, Any] | None = None, ) -> bool: - """Send Tier 2 telemetry: run emissions to the shared experiment via ``ApiClient``. + """Send Tier 2 public run summary to the shared telemetry experiment via ``ApiClient``. Args: tracker: Active emissions tracker instance. @@ -109,11 +109,11 @@ def send_tier2_at_stop( external_conf: Merged config for API URL, key, and experiment resolution. Returns: - True if Tier 2 was posted successfully, False otherwise. + True if the run summary was posted successfully, False otherwise. """ if _run_too_short_for_telemetry(emissions): logger.debug( - "Tier 2 telemetry not sent because run duration is shorter than 1 second." + "Public run summary not sent because run duration is shorter than 1 second." ) return False settings_conf = external_conf or {} @@ -128,7 +128,7 @@ def send_tier2_at_stop( ) return bool(api.add_emission(dataclasses.asdict(emissions))) except Exception as error: - logger.error(f"Tier 2 telemetry failed (non-critical): {error}") + logger.error(f"Public run summary failed (non-critical): {error}") return False @@ -141,7 +141,7 @@ def send_product_telemetry_at_stop( """Send product telemetry for the resolved tier at tracker ``stop()``. Tier 1 (``minimal``): private ``POST /telemetry`` only. - Tier 2 (``extensive``): Tier 1 plus ``ApiClient`` run summary. + Tier 2 (``extensive``): same private ``POST /telemetry`` plus public run summary. Args: tracker: Active emissions tracker instance. @@ -153,6 +153,6 @@ def send_product_telemetry_at_stop( return settings = external_conf or {} if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_tier1_at_stop(tracker, emissions, settings, level=level) + send_private_telemetry_at_stop(tracker, emissions, settings, level=level) if level == TelemetryLevel.extensive: - send_tier2_at_stop(tracker, emissions, settings) + send_public_run_summary_at_stop(tracker, emissions, settings) diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 82ea56f7d..192709ea1 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -56,7 +56,7 @@ Private telemetry does **not** include project names, experiment ids, API keys, - Project name, experiment id, run id, API keys - Source code, file paths, hostnames -- Exact GPS coordinates (`longitude` / `latitude`), `python_executable_hash`, `host_machine_hash` +- Exact GPS coordinates, executable/host fingerprints (not in the telemetry schema) - Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) ## Configure telemetry diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 741182d8f..3166addbf 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -7,7 +7,10 @@ from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import send_tier1_at_stop, send_tier2_at_stop +from codecarbon.telemetry import ( + send_private_telemetry_at_stop, + send_public_run_summary_at_stop, +) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -65,7 +68,9 @@ def test_tier1_posts_private_telemetry(self): mock_client = MagicMock() mock_client.add_telemetry.return_value = {"id": "ok"} mock_client_cls.return_value = mock_client - result = send_tier1_at_stop(tracker, emissions, external_conf={}) + result = send_private_telemetry_at_stop( + tracker, emissions, external_conf={} + ) self.assertTrue(result) posted = mock_client_cls.call_args.kwargs["telemetry"] self.assertEqual(posted["telemetry_level"], "minimal") @@ -76,7 +81,9 @@ def test_tier1_skips_short_duration(self): emissions = self._emissions() emissions.duration = 0.5 with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - result = send_tier1_at_stop(tracker, emissions, external_conf={}) + result = send_private_telemetry_at_stop( + tracker, emissions, external_conf={} + ) self.assertFalse(result) mock_client_cls.assert_not_called() @@ -88,7 +95,9 @@ def test_tier2_uses_api_client(self): mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api - result = send_tier2_at_stop(tracker, emissions, external_conf={}) + result = send_public_run_summary_at_stop( + tracker, emissions, external_conf={} + ) self.assertTrue(result) mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 4d2b24b32..48ab743c5 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -93,7 +93,7 @@ def test_init_rejects_invalid_telemetry_without_calling_api(self): telemetry={ "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "longitude": 2.35, + "unknown_field": "value", }, ) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 1b711ad66..1a0cdf8f7 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -2,9 +2,9 @@ from unittest.mock import MagicMock, patch from codecarbon.core.telemetry_collect import ( - build_tier1_payload, + build_telemetry_payload, collect_telemetry_context, - project_tier1, + project_private_telemetry, ) from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData @@ -49,7 +49,7 @@ def _sample_emissions(**overrides): class TestTelemetryCollect(unittest.TestCase): - def test_project_tier1_includes_run_and_framework_flags(self): + def test_project_private_telemetry_includes_run_and_framework_flags(self): tracker = MagicMock() tracker._conf = { "os": "Linux", @@ -75,7 +75,7 @@ def test_project_tier1_includes_run_and_framework_flags(self): side_effect=lambda name: name == "torch", ): context = collect_telemetry_context(tracker, emissions) - payload = project_tier1(context) + payload = project_private_telemetry(context) self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.5) @@ -83,7 +83,7 @@ def test_project_tier1_includes_run_and_framework_flags(self): self.assertTrue(payload["has_torch"]) self.assertIn("file", payload["output_methods"]) - def test_build_tier1_payload_includes_framework_versions(self): + def test_build_telemetry_payload_includes_framework_versions(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} tracker._geo = None @@ -106,13 +106,13 @@ def test_build_tier1_payload_includes_framework_versions(self): "codecarbon.core.telemetry_collect._package_version", return_value="2.0.0", ): - payload = build_tier1_payload(tracker, emissions) + payload = build_telemetry_payload(tracker, emissions) self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) self.assertEqual(payload["torch_version"], "2.0.0") - def test_build_tier1_payload_uses_resolved_level(self): + def test_build_telemetry_payload_uses_resolved_level(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0"} tracker._save_to_file = False @@ -127,7 +127,7 @@ def test_build_tier1_payload_uses_resolved_level(self): tracker._resource_tracker = None emissions = _sample_emissions() - payload = build_tier1_payload( + payload = build_telemetry_payload( tracker, emissions, level=TelemetryLevel.extensive ) self.assertEqual(payload["telemetry_level"], "extensive") diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index e90b5987f..55f8e24f3 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -15,7 +15,10 @@ from codecarbon.core.telemetry_settings import get_telemetry_api_url from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import send_tier1_at_stop, warn_if_telemetry_not_configured +from codecarbon.telemetry import ( + send_private_telemetry_at_stop, + warn_if_telemetry_not_configured, +) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -90,12 +93,12 @@ def test_tier1_posts_to_telemetry_endpoint(self): tracking_mode="machine", ) with patch( - "codecarbon.telemetry.build_tier1_payload", return_value=tier1_payload + "codecarbon.telemetry.build_telemetry_payload", return_value=tier1_payload ): with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: mock_post.return_value.status_code = 201 mock_post.return_value.json.return_value = "telemetry-id" - send_tier1_at_stop( + send_private_telemetry_at_stop( tracker, emissions, external_conf={"telemetry_api_url": "http://tier1.example"}, From e3dca3f129a5dd95407eb20e4339c4a7db444535 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:17:09 +0200 Subject: [PATCH 16/29] refactor: simplify telemetry framework detection and enhance privacy - Removed version tracking for ML frameworks from telemetry payload to improve privacy. - Updated the framework packages list to exclude version fields, focusing solely on presence flags. - Adjusted telemetry context collection to reflect the changes in framework detection. - Modified tests to ensure the telemetry payload omits framework versions as intended. - Updated documentation to clarify the changes in ML stack data collection. --- codecarbon/core/telemetry_collect.py | 45 ++++++--------------------- codecarbon/core/telemetry_settings.py | 4 +-- docs/how-to/telemetry.md | 2 +- tests/test_telemetry_collect.py | 7 ++--- 4 files changed, 15 insertions(+), 43 deletions(-) diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index 4122e91dc..3a057ef77 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -15,13 +15,10 @@ from codecarbon.output_methods.emissions_data import EmissionsData FRAMEWORK_PACKAGES = ( - ("torch", "has_torch", "torch_version"), - ("transformers", "has_transformers", "transformers_version"), - ("tensorflow", "has_tensorflow", "tensorflow_version"), - ("keras", "has_keras", "keras_version"), - ("diffusers", "has_diffusers", "diffusers_version"), - ("pytorch_lightning", "has_pytorch_lightning", "pytorch_lightning_version"), - ("fastai", "has_fastai", "fastai_version"), + ("torch", "has_torch"), + ("transformers", "has_transformers"), + ("diffusers", "has_diffusers"), + ("sklearn", "has_sklearn"), ) def _non_empty(value: Any) -> bool: @@ -42,17 +39,6 @@ def _package_installed(name: str) -> bool: return importlib.util.find_spec(name) is not None -def _package_version(name: str) -> Optional[str]: - if not _package_installed(name): - return None - try: - from importlib.metadata import version - - return version(name) - except Exception: - return None - - def _detect_codecarbon_install_method() -> Optional[str]: try: from importlib.metadata import distribution @@ -233,19 +219,11 @@ def _raw_cloud_provider_and_region() -> tuple[Optional[str], Optional[str]]: return provider, region -def _collect_framework_fields(include_versions: bool) -> dict[str, Any]: - fields: dict[str, Any] = {} - primary: Optional[str] = None - for package, has_field, version_field in FRAMEWORK_PACKAGES: - installed = _package_installed(package) - fields[has_field] = installed - if include_versions and installed: - fields[version_field] = _package_version(package) - if installed and primary is None: - primary = package - if primary: - fields["ml_framework_primary"] = primary - return fields +def _collect_framework_fields() -> dict[str, Any]: + return { + has_field: _package_installed(package) + for package, has_field in FRAMEWORK_PACKAGES + } def _gpu_static_fields() -> dict[str, Any]: @@ -339,14 +317,11 @@ def collect_telemetry_context( "cpu_utilization_avg": emissions.cpu_utilization_percent, "gpu_utilization_avg": emissions.gpu_utilization_percent, "ram_utilization_avg": emissions.ram_utilization_percent, - **_collect_framework_fields(include_versions=True), + **_collect_framework_fields(), **_collect_hardware_diagnostics(tracker), } context.update(gpu_fields) - if context.get("ml_framework_primary"): - context["framework_detected"] = context["ml_framework_primary"] - return _strip_none(context) diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index 1d2f65c87..c12d7f04f 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -7,8 +7,8 @@ from codecarbon.external.logger import logger DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" -DEFAULT_TELEMETRY_API_KEY = "cpt_JZhj-vJdEVG28qErZL5mh1ftiqbnDIBYjWSxwvX3rfI" -DEFAULT_TELEMETRY_EXPERIMENT_ID = "aa69b440-014a-4562-ac06-ba7eecb023f9" +DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 192709ea1..b5bc56057 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -43,7 +43,7 @@ The payload includes: - **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region, GPU driver/CUDA/cuDNN when available - **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container/IDE hints, hardware diagnostics -- **ML stack:** framework presence flags **and** installed versions when detected +- **ML stack:** framework presence flags when detected (no installed package versions) - **Run outcome:** duration, emissions, energy (total and per component), utilization averages Private telemetry does **not** include project names, experiment ids, API keys, file paths, exact coordinates, executable/host hashes, or survey demographics (role, industry, etc.). diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 1a0cdf8f7..85f006ee4 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -83,7 +83,7 @@ def test_project_private_telemetry_includes_run_and_framework_flags(self): self.assertTrue(payload["has_torch"]) self.assertIn("file", payload["output_methods"]) - def test_build_telemetry_payload_includes_framework_versions(self): + def test_build_telemetry_payload_omits_framework_versions(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} tracker._geo = None @@ -102,15 +102,12 @@ def test_build_telemetry_payload_includes_framework_versions(self): with patch( "codecarbon.core.telemetry_collect._package_installed", return_value=True, - ), patch( - "codecarbon.core.telemetry_collect._package_version", - return_value="2.0.0", ): payload = build_telemetry_payload(tracker, emissions) self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) - self.assertEqual(payload["torch_version"], "2.0.0") + self.assertNotIn("torch_version", payload) def test_build_telemetry_payload_uses_resolved_level(self): tracker = MagicMock() From de3fff137bd30673f8f9c6dd255f8897f573689b Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:32:43 +0200 Subject: [PATCH 17/29] refactor: simplify telemetry client and enhance telemetry handling - Replaced the TelemetryClient class with a standalone function, post_private_telemetry, for sending telemetry data. - Updated telemetry payload construction to improve clarity and maintainability. - Refactored telemetry warning handling to ensure it is shown only once during execution. - Enhanced tests to validate the new telemetry sending method and ensure proper payload structure. - Improved error handling and logging for telemetry requests, including handling of connection errors and HTTP response statuses. --- codecarbon/core/telemetry_client.py | 99 +++++++----------- codecarbon/core/telemetry_collect.py | 145 ++++++++++---------------- codecarbon/core/telemetry_settings.py | 96 ++++++++++------- codecarbon/telemetry.py | 54 +++++----- tests/test_telemetry.py | 70 ++++++------- tests/test_telemetry_client.py | 111 +++++--------------- tests/test_telemetry_collect.py | 11 +- tests/test_telemetry_config.py | 56 +++++----- 8 files changed, 266 insertions(+), 376 deletions(-) diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py index 6cf821f28..fbfe76e3d 100644 --- a/codecarbon/core/telemetry_client.py +++ b/codecarbon/core/telemetry_client.py @@ -1,76 +1,47 @@ -import json -from typing import Optional, Union - import requests from codecarbon.core.telemetry_schemas import TelemetryCreate from codecarbon.external.logger import logger -class TelemetryClient: - """ - Client dedicated to sending CodeCarbon telemetry payloads. - """ +def post_private_telemetry(url: str, payload: dict, api_key: str | None) -> bool: + """POST a private telemetry payload to ``/telemetry``. - def __init__( - self, - endpoint_url: str = "https://api.codecarbon.io", - telemetry: Optional[Union[TelemetryCreate, dict]] = None, - api_key: Optional[str] = None, - ): - self.endpoint_url = endpoint_url.rstrip("/") - self.telemetry_url = self.endpoint_url + "/telemetry" - self.api_key = api_key - self.headers = self._build_headers(api_key) - self.telemetry = self._validate_telemetry(telemetry) if telemetry else None + Args: + url: API base URL. + payload: Telemetry fields dict. + api_key: Optional API token. - @staticmethod - def _build_headers(api_key: Optional[str]) -> dict[str, str]: - headers = {"Content-Type": "application/json"} - if api_key: - headers["x-api-token"] = api_key - return headers - - def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None): - telemetry_payload = ( - self._validate_telemetry(telemetry) if telemetry else self.telemetry + Returns: + True if the server accepted the payload (HTTP 201). + """ + headers = {"Content-Type": "application/json"} + if api_key: + headers["x-api-token"] = api_key + body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) + telemetry_url = f"{url.rstrip('/')}/telemetry" + try: + response = requests.post( + url=telemetry_url, + json=body, + headers=headers, + timeout=2, ) - if telemetry_payload is None: - logger.error("TelemetryClient.add_telemetry() needs a telemetry payload") - return None - payload = telemetry_payload.model_dump(mode="json", exclude_none=True) - - try: - response = requests.post( - url=self.telemetry_url, - json=payload, - timeout=2, - headers=self.headers, - ) - if response.status_code == 404: - logger.warning( - "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", - self.telemetry_url, - ) - return None - if response.status_code != 201: - self._log_error(payload, response) - return None - return response.json() - except Exception as e: - logger.error(e, exc_info=True) - return None - - @staticmethod - def _validate_telemetry(telemetry: Union[TelemetryCreate, dict]) -> TelemetryCreate: - if isinstance(telemetry, TelemetryCreate): - return telemetry - return TelemetryCreate(**telemetry) - - def _log_error(self, payload, response): - logger.error( - f"TelemetryClient Error when calling the API on {self.telemetry_url} with : {json.dumps(payload)}" + except Exception: + logger.error("Telemetry request failed.", exc_info=True) + return False + if response.status_code == 201: + return True + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + telemetry_url, ) + else: logger.error( - f"TelemetryClient API return http code {response.status_code} and answer : {response.text}" + "Telemetry API %s: %s", + response.status_code, + response.text, ) + logger.debug("Telemetry request body: %s", body) + return False diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index 3a057ef77..db788ad3c 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -1,4 +1,4 @@ -"""Collect and project private product telemetry (Tier 1 / Tier 2) from tracker state.""" +"""Collect private product telemetry (Tier 1 / Tier 2) from tracker state.""" from __future__ import annotations @@ -21,20 +21,46 @@ ("sklearn", "has_sklearn"), ) +PACKAGE_MANAGER_ENV = ( + ("UV", "uv"), + ("POETRY_ACTIVE", "poetry"), + ("PIP_RUN", "pip"), +) + +CI_ENVIRONMENTS = ( + ("GITHUB_ACTIONS", "github_actions"), + ("GITLAB_CI", "gitlab_ci"), + ("CIRCLECI", "circleci"), + ("JENKINS_URL", "jenkins"), + ("CI", "ci"), +) + +CONTAINER_RUNTIME_ENV = ( + ("KUBERNETES_SERVICE_HOST", "kubernetes"), +) + +OUTPUT_METHOD_ATTRS = ( + ("_save_to_file", "file"), + ("_save_to_api", "api"), + ("_save_to_logger", "logger"), + ("_emissions_endpoint", "http"), + ("_save_to_prometheus", "prometheus"), + ("_save_to_logfire", "logfire"), +) + + def _non_empty(value: Any) -> bool: - if value is None: - return False - if value == [] or value == {}: - return False - if isinstance(value, str) and value == "": - return False - return True + return value not in (None, "", [], {}) def _strip_none(data: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in data.items() if _non_empty(value)} +def _first_env_match(mapping: tuple[tuple[str, str], ...]) -> Optional[str]: + return next((label for var, label in mapping if os.environ.get(var)), None) + + def _package_installed(name: str) -> bool: return importlib.util.find_spec(name) is not None @@ -66,30 +92,6 @@ def _detect_python_env_type() -> Optional[str]: return "system" -def _detect_python_package_manager() -> Optional[str]: - if os.environ.get("UV"): - return "uv" - if os.environ.get("POETRY_ACTIVE"): - return "poetry" - if os.environ.get("PIP_RUN"): - return "pip" - return None - - -def _detect_ci_environment() -> Optional[str]: - if os.environ.get("GITHUB_ACTIONS"): - return "github_actions" - if os.environ.get("GITLAB_CI"): - return "gitlab_ci" - if os.environ.get("CIRCLECI"): - return "circleci" - if os.environ.get("JENKINS_URL"): - return "jenkins" - if os.environ.get("CI"): - return "ci" - return None - - def _detect_notebook_environment() -> Optional[str]: if os.environ.get("COLAB_GPU") is not None or "google.colab" in sys.modules: return "colab" @@ -113,8 +115,9 @@ def _detect_in_container() -> bool: def _detect_container_runtime() -> Optional[str]: - if os.environ.get("KUBERNETES_SERVICE_HOST"): - return "kubernetes" + runtime = _first_env_match(CONTAINER_RUNTIME_ENV) + if runtime: + return runtime if os.path.exists("/.dockerenv"): return "docker" return None @@ -186,18 +189,13 @@ def _detect_integration_surface(tracker: Any) -> str: def _collect_output_methods(tracker: Any) -> list[str]: methods: list[str] = [] - if getattr(tracker, "_save_to_file", False): - methods.append("file") - if getattr(tracker, "_save_to_api", False): - methods.append("api") - if getattr(tracker, "_save_to_logger", False): - methods.append("logger") - if getattr(tracker, "_emissions_endpoint", None): - methods.append("http") - if getattr(tracker, "_save_to_prometheus", False): - methods.append("prometheus") - if getattr(tracker, "_save_to_logfire", False): - methods.append("logfire") + for attr, name in OUTPUT_METHOD_ATTRS: + value = getattr(tracker, attr, False) + if attr == "_emissions_endpoint": + if value: + methods.append(name) + elif value: + methods.append(name) return methods @@ -247,18 +245,20 @@ def _gpu_static_fields() -> dict[str, Any]: return fields -def collect_telemetry_context( +def build_telemetry_payload( tracker: Any, emissions: EmissionsData, + level: TelemetryLevel = TelemetryLevel.minimal, ) -> dict[str, Any]: - """Build a flat telemetry context from tracker state and emissions at stop. + """Build a private telemetry payload dict for ``POST /telemetry``. Args: - tracker: Active ``BaseEmissionsTracker`` instance. - emissions: Total emissions row from ``_prepare_emissions_data()``. + tracker: Active emissions tracker. + emissions: Run emissions data. + level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). Returns: - Flat dictionary for ``project_private_telemetry``. + Payload dict for ``TelemetryCreate``. """ conf = getattr(tracker, "_conf", {}) raw_provider, raw_region = _raw_cloud_provider_and_region() @@ -272,13 +272,13 @@ def collect_telemetry_context( integration_surface = _detect_integration_surface(tracker) gpu_fields = _gpu_static_fields() - context: dict[str, Any] = { + raw: dict[str, Any] = { "timestamp": datetime.now(timezone.utc), "os": conf.get("os") or platform.platform(), "python_version": conf.get("python_version") or platform.python_version(), "python_implementation": platform.python_implementation(), "python_env_type": _detect_python_env_type(), - "python_package_manager": _detect_python_package_manager(), + "python_package_manager": _first_env_match(PACKAGE_MANAGER_ENV), "codecarbon_version": conf.get("codecarbon_version"), "codecarbon_install_method": _detect_codecarbon_install_method(), "country_name": emissions.country_name, @@ -303,7 +303,7 @@ def collect_telemetry_context( "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), "in_container": _detect_in_container(), "container_runtime": _detect_container_runtime(), - "ci_environment": _detect_ci_environment(), + "ci_environment": _first_env_match(CI_ENVIRONMENTS), "notebook_environment": _detect_notebook_environment(), "ide_used": _detect_ide(), "cudnn_version": _cudnn_version(), @@ -317,42 +317,11 @@ def collect_telemetry_context( "cpu_utilization_avg": emissions.cpu_utilization_percent, "gpu_utilization_avg": emissions.gpu_utilization_percent, "ram_utilization_avg": emissions.ram_utilization_percent, + "telemetry_level": level.value, **_collect_framework_fields(), **_collect_hardware_diagnostics(tracker), + **gpu_fields, } - context.update(gpu_fields) - return _strip_none(context) - - -def project_private_telemetry( - context: dict[str, Any], - level: TelemetryLevel = TelemetryLevel.minimal, -) -> dict[str, Any]: - """Project context to private ``POST /telemetry`` fields for the resolved tier.""" - payload = { - key: context[key] - for key in PRIVATE_TELEMETRY_FIELDS - if key in context - } - payload["telemetry_level"] = level.value + payload = {key: raw[key] for key in PRIVATE_TELEMETRY_FIELDS if key in raw} return _strip_none(payload) - - -def build_telemetry_payload( - tracker: Any, - emissions: EmissionsData, - level: TelemetryLevel = TelemetryLevel.minimal, -) -> dict[str, Any]: - """Build a private telemetry payload dict for ``TelemetryCreate``. - - Args: - tracker: Active emissions tracker. - emissions: Run emissions data. - level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). - - Returns: - Payload dict for ``POST /telemetry``. - """ - context = collect_telemetry_context(tracker, emissions) - return project_private_telemetry(context, level=level) diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index c12d7f04f..a0805ed00 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -1,7 +1,7 @@ """Resolve telemetry tier and API settings from config and environment.""" import os -from typing import Any +from typing import Any, Literal from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.external.logger import logger @@ -13,6 +13,8 @@ TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" +TelemetryLevelSource = Literal["override", "external", "file", "default"] + def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: """Parse a telemetry tier from a string or enum value. @@ -36,41 +38,13 @@ def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: ) from error -def is_telemetry_level_explicit( - config_file_conf: dict[str, Any], - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> bool: - """Return whether the user explicitly chose a telemetry tier. - - Explicit sources: tracker ``telemetry_level`` argument, config file - ``telemetry_level``, or environment ``CODECARBON_TELEMETRY_LEVEL``. - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). - override: Value passed to ``EmissionsTracker(telemetry_level=...)``. - external_conf: Merged config from file and environment. - - Returns: - True if any explicit source is set. - """ - if override is not None: - return True - if config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: - return True - if external_conf is None: - return False - return external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None - - -def resolve_telemetry_level( +def resolve_telemetry_level_and_source( config_file_conf: dict[str, Any] | None = None, *, override: str | TelemetryLevel | None = None, external_conf: dict[str, Any] | None = None, -) -> TelemetryLevel: - """Resolve the active telemetry tier. +) -> tuple[TelemetryLevel, TelemetryLevelSource]: + """Resolve the active telemetry tier and where it came from. Precedence: @@ -85,27 +59,77 @@ def resolve_telemetry_level( external_conf: Merged settings from ``get_hierarchical_config()`` (optional). Returns: - Resolved ``TelemetryLevel``. + Resolved tier and its source label. """ if override is not None: raw = override + source: TelemetryLevelSource = "override" elif external_conf is not None and external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: raw = external_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "external" elif config_file_conf is not None and config_file_conf.get( TELEMETRY_LEVEL_CONFIG_KEY ) is not None: raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "file" else: - return DEFAULT_TELEMETRY_LEVEL + return DEFAULT_TELEMETRY_LEVEL, "default" try: - return parse_telemetry_level(raw) + return parse_telemetry_level(raw), source except ValueError: logger.error( "Invalid telemetry_level %r; falling back to %r", raw, DEFAULT_TELEMETRY_LEVEL.value, ) - return DEFAULT_TELEMETRY_LEVEL + return DEFAULT_TELEMETRY_LEVEL, source + + +def resolve_telemetry_level( + config_file_conf: dict[str, Any] | None = None, + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> TelemetryLevel: + """Resolve the active telemetry tier. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (optional). + override: Optional tier from tracker or CLI. + external_conf: Merged settings from ``get_hierarchical_config()`` (optional). + + Returns: + Resolved ``TelemetryLevel``. + """ + return resolve_telemetry_level_and_source( + config_file_conf, + override=override, + external_conf=external_conf, + )[0] + + +def is_telemetry_level_explicit( + config_file_conf: dict[str, Any], + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> bool: + """Return whether the user explicitly chose a telemetry tier. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). + override: Value passed to ``EmissionsTracker(telemetry_level=...)``. + external_conf: Merged config from file and environment. + + Returns: + True if any explicit source is set. + """ + _, source = resolve_telemetry_level_and_source( + config_file_conf, + override=override, + external_conf=external_conf, + ) + return source != "default" def get_telemetry_api_url( diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index ed73537b4..eeec82434 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -4,7 +4,7 @@ from typing import Any from codecarbon.core.api_client import ApiClient -from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_client import post_private_telemetry from codecarbon.core.telemetry_collect import build_telemetry_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import ( @@ -16,8 +16,6 @@ from codecarbon.external.logger import logger from codecarbon.output_methods.emissions_data import EmissionsData -_TELEMETRY_CONFIGURE_WARNED = False - TELEMETRY_NOT_CONFIGURED_MESSAGE = ( "CodeCarbon telemetry_level was not set explicitly; using default %r. " "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " @@ -26,6 +24,14 @@ "codecarbon telemetry set ." ) +_telemetry_default_warning_shown = False + + +def reset_telemetry_warning() -> None: + """Clear the one-shot default-tier warning (for tests).""" + global _telemetry_default_warning_shown + _telemetry_default_warning_shown = False + def warn_if_telemetry_not_configured( config_file_conf: dict[str, Any], @@ -42,22 +48,15 @@ def warn_if_telemetry_not_configured( override: Optional ``telemetry_level`` tracker argument. external_conf: Merged file/env settings for explicit-env detection. """ - global _TELEMETRY_CONFIGURE_WARNED - if _TELEMETRY_CONFIGURE_WARNED: - return + global _telemetry_default_warning_shown if is_telemetry_level_explicit( config_file_conf, override=override, external_conf=external_conf ): return - logger.warning( - TELEMETRY_NOT_CONFIGURED_MESSAGE, - active_level.value, - ) - _TELEMETRY_CONFIGURE_WARNED = True - - -def _run_too_short_for_telemetry(emissions: EmissionsData) -> bool: - return emissions.duration is not None and emissions.duration < 1 + if _telemetry_default_warning_shown: + return + logger.warning(TELEMETRY_NOT_CONFIGURED_MESSAGE, active_level.value) + _telemetry_default_warning_shown = True def send_private_telemetry_at_stop( @@ -68,6 +67,9 @@ def send_private_telemetry_at_stop( ) -> bool: """Send Tier 1 private telemetry via ``POST /telemetry``. + Runs shorter than one second are skipped by ``send_product_telemetry_at_stop``, + not here; direct callers may still post for sub-second runs. + Args: tracker: Active emissions tracker instance. emissions: Total emissions from ``_prepare_emissions_data()``. @@ -77,20 +79,14 @@ def send_private_telemetry_at_stop( Returns: True if the private telemetry POST was accepted, False otherwise. """ - if _run_too_short_for_telemetry(emissions): - logger.debug( - "Private telemetry not sent because run duration is shorter than 1 second." - ) - return False settings_conf = external_conf or {} try: payload = build_telemetry_payload(tracker, emissions, level=level) - client = TelemetryClient( - endpoint_url=get_telemetry_api_url(settings_conf), - telemetry=payload, - api_key=get_telemetry_api_key(settings_conf), + return post_private_telemetry( + get_telemetry_api_url(settings_conf), + payload, + get_telemetry_api_key(settings_conf), ) - return client.add_telemetry() is not None except Exception as error: logger.error(f"Private telemetry failed (non-critical): {error}") return False @@ -111,11 +107,6 @@ def send_public_run_summary_at_stop( Returns: True if the run summary was posted successfully, False otherwise. """ - if _run_too_short_for_telemetry(emissions): - logger.debug( - "Public run summary not sent because run duration is shorter than 1 second." - ) - return False settings_conf = external_conf or {} conf = getattr(tracker, "_conf", {}) try: @@ -151,6 +142,9 @@ def send_product_telemetry_at_stop( """ if level == TelemetryLevel.disabled: return + if emissions.duration is not None and emissions.duration < 1: + logger.debug("Telemetry not sent: run shorter than 1 second.") + return settings = external_conf or {} if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): send_private_telemetry_at_stop(tracker, emissions, settings, level=level) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 3166addbf..757283d7a 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,7 +1,6 @@ import sys import tempfile import unittest -from pathlib import Path from unittest.mock import ANY, MagicMock, patch from codecarbon.core.telemetry_schemas import TelemetryLevel @@ -64,28 +63,32 @@ def _emissions(self) -> EmissionsData: def test_tier1_posts_private_telemetry(self): tracker = MagicMock() emissions = self._emissions() - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = {"id": "ok"} - mock_client_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: result = send_private_telemetry_at_stop( tracker, emissions, external_conf={} ) self.assertTrue(result) - posted = mock_client_cls.call_args.kwargs["telemetry"] - self.assertEqual(posted["telemetry_level"], "minimal") - self.assertEqual(posted["total_emissions_kg"], 0.001) + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertEqual(payload["total_emissions_kg"], 0.001) + + def test_tier1_skips_short_duration_at_dispatcher(self): + from codecarbon.telemetry import send_product_telemetry_at_stop - def test_tier1_skips_short_duration(self): tracker = MagicMock() emissions = self._emissions() emissions.duration = 0.5 - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - result = send_private_telemetry_at_stop( - tracker, emissions, external_conf={} + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + send_product_telemetry_at_stop( + tracker, + emissions, + TelemetryLevel.minimal, + external_conf={}, ) - self.assertFalse(result) - mock_client_cls.assert_not_called() + mock_post.assert_not_called() def test_tier2_uses_api_client(self): tracker = MagicMock() @@ -129,20 +132,19 @@ def _start_config_mock(self, conf: str) -> None: def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) - mock_telemetry_cls.assert_not_called() + mock_post.assert_not_called() def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( self, mock_cli_setup ): self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = {"id": "ok"} - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -151,15 +153,14 @@ def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() - payload = mock_telemetry_cls.call_args.kwargs["telemetry"] + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] self.assertEqual(payload["telemetry_level"], "minimal") self.assertIn("total_emissions_kg", payload) def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): self._start_config_mock(disabled_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -168,15 +169,14 @@ def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_not_called() + mock_post.assert_not_called() def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): self._start_config_mock(extensive_conf) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_telemetry = MagicMock() - mock_telemetry.add_telemetry.return_value = {"id": "ok"} - mock_telemetry_cls.return_value = mock_telemetry + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True @@ -189,18 +189,16 @@ def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_telemetry.add_telemetry.assert_called_once() + mock_post.assert_called_once() mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() def test_offline_tracker_sends_on_stop(self, mock_cli_setup): self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = {"id": "ok"} - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", save_to_api=False, @@ -208,7 +206,7 @@ def test_offline_tracker_sends_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_client.add_telemetry.assert_called_once() + mock_post.assert_called_once() if __name__ == "__main__": diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 48ab743c5..7d8b4be64 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -4,27 +4,12 @@ import requests_mock from pydantic import ValidationError -from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_client import post_private_telemetry from codecarbon.core.telemetry_schemas import TelemetryCreate -class TestTelemetryClient(unittest.TestCase): - def test_init_sets_up_client_without_calling_api(self): - with requests_mock.Mocker() as m: - client = TelemetryClient( - endpoint_url="http://test.com/", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - }, - ) - - self.assertEqual(client.endpoint_url, "http://test.com") - self.assertEqual(client.telemetry_url, "http://test.com/telemetry") - self.assertIsInstance(client.telemetry, TelemetryCreate) - self.assertEqual(m.call_count, 0) - - def test_add_telemetry_posts_configured_payload(self): +class TestPostPrivateTelemetry(unittest.TestCase): + def test_post_private_telemetry_sends_validated_payload(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -37,15 +22,9 @@ def test_add_telemetry_posts_configured_payload(self): json="f52fe339-164d-4c2b-a8c0-f562dfce066d", status_code=201, ) - client = TelemetryClient( - endpoint_url="http://test.com", telemetry=telemetry - ) + result = post_private_telemetry("http://test.com", telemetry, None) - actual_telemetry_id = client.add_telemetry() - - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) + self.assertTrue(result) self.assertEqual(m.call_count, 1) self.assertEqual( m.last_request.json(), @@ -55,56 +34,19 @@ def test_add_telemetry_posts_configured_payload(self): }, ) - def test_add_telemetry_posts_call_payload(self): - telemetry = TelemetryCreate( - timestamp="2026-05-03T12:00:00+00:00", - telemetry_level="minimal", - os="Linux-5.10.0-x86_64", - ) - - with requests_mock.Mocker() as m: - m.post( - "http://test.com/telemetry", - json="f52fe339-164d-4c2b-a8c0-f562dfce066d", - status_code=201, - ) - client = TelemetryClient(endpoint_url="http://test.com") - - actual_telemetry_id = client.add_telemetry(telemetry) - - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) - self.assertEqual(m.call_count, 1) - self.assertEqual( - m.last_request.json(), + def test_post_private_telemetry_rejects_invalid_payload(self): + with self.assertRaises(ValidationError): + post_private_telemetry( + "http://test.com", { - "timestamp": "2026-05-03T12:00:00Z", + "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "os": "Linux-5.10.0-x86_64", + "unknown_field": "value", }, + None, ) - def test_init_rejects_invalid_telemetry_without_calling_api(self): - with requests_mock.Mocker() as m: - with self.assertRaises(ValidationError): - TelemetryClient( - endpoint_url="http://test.com", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - "unknown_field": "value", - }, - ) - - self.assertEqual(m.call_count, 0) - - def test_add_telemetry_returns_none_without_payload(self): - client = TelemetryClient(endpoint_url="http://test.com") - - self.assertIsNone(client.add_telemetry()) - - def test_add_telemetry_logs_warning_on_404(self): + def test_post_private_telemetry_logs_warning_on_404(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -116,14 +58,11 @@ def test_add_telemetry_logs_warning_on_404(self): status_code=404, ) with patch("codecarbon.core.telemetry_client.logger") as mock_logger: - client = TelemetryClient( - endpoint_url="http://test.com", telemetry=telemetry - ) - result = client.add_telemetry() - self.assertIsNone(result) + result = post_private_telemetry("http://test.com", telemetry, None) + self.assertFalse(result) mock_logger.warning.assert_called_once() - def test_add_telemetry_sends_api_key_header_when_configured(self): + def test_post_private_telemetry_sends_api_key_header_when_configured(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -134,10 +73,16 @@ def test_add_telemetry_sends_api_key_header_when_configured(self): json="telemetry-id", status_code=201, ) - client = TelemetryClient( - endpoint_url="http://test.com", - telemetry=telemetry, - api_key="cpt_test_key", - ) - client.add_telemetry() + post_private_telemetry("http://test.com", telemetry, "cpt_test_key") self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") + + def test_post_private_telemetry_returns_false_on_request_error(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + mock_post.side_effect = ConnectionError("network down") + with patch("codecarbon.core.telemetry_client.logger"): + result = post_private_telemetry("http://test.com", telemetry, None) + self.assertFalse(result) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 85f006ee4..be770e460 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -1,11 +1,7 @@ import unittest from unittest.mock import MagicMock, patch -from codecarbon.core.telemetry_collect import ( - build_telemetry_payload, - collect_telemetry_context, - project_private_telemetry, -) +from codecarbon.core.telemetry_collect import build_telemetry_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData @@ -49,7 +45,7 @@ def _sample_emissions(**overrides): class TestTelemetryCollect(unittest.TestCase): - def test_project_private_telemetry_includes_run_and_framework_flags(self): + def test_build_telemetry_payload_includes_run_and_framework_flags(self): tracker = MagicMock() tracker._conf = { "os": "Linux", @@ -74,8 +70,7 @@ def test_project_private_telemetry_includes_run_and_framework_flags(self): "codecarbon.core.telemetry_collect._package_installed", side_effect=lambda name: name == "torch", ): - context = collect_telemetry_context(tracker, emissions) - payload = project_private_telemetry(context) + payload = build_telemetry_payload(tracker, emissions) self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.5) diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 55f8e24f3..5627ae529 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -8,7 +8,6 @@ from pathlib import Path from unittest.mock import MagicMock, patch -import codecarbon.telemetry as telemetry_module from codecarbon.core.config import get_config_file_settings from codecarbon.core.telemetry_settings import resolve_telemetry_level from codecarbon.core.telemetry_schemas import TelemetryLevel @@ -16,6 +15,7 @@ from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData from codecarbon.telemetry import ( + reset_telemetry_warning, send_private_telemetry_at_stop, warn_if_telemetry_not_configured, ) @@ -35,7 +35,7 @@ def _conf(level: str) -> str: class TestTelemetryConfigContract(unittest.TestCase): def setUp(self) -> None: - telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + reset_telemetry_warning() def test_warns_once_when_telemetry_not_explicit(self): with patch("codecarbon.telemetry.logger.warning") as mock_warning: @@ -174,7 +174,7 @@ def test_telemetry_api_url_from_config_overrides_default(self): @mock_platform_cli_setup class TestTrackerTelemetryFromConfig(unittest.TestCase): def setUp(self) -> None: - telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + reset_telemetry_warning() self._config_patcher = None def tearDown(self) -> None: @@ -189,7 +189,7 @@ def _mock_config(self, conf: str) -> None: def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): self._mock_config(_conf("disabled")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -198,15 +198,14 @@ def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_not_called() + mock_post.assert_not_called() def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -215,20 +214,18 @@ def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() + mock_post.assert_called_once() self.assertEqual( - mock_telemetry_cls.call_args.kwargs["telemetry"]["telemetry_level"], + mock_post.call_args[0][1]["telemetry_level"], TelemetryLevel.minimal.value, ) def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("extensive")) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_telemetry = MagicMock() - mock_telemetry.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_telemetry + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True @@ -241,18 +238,16 @@ def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_telemetry.add_telemetry.assert_called_once() + mock_post.assert_called_once() mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", save_to_api=False, @@ -260,7 +255,7 @@ def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_client.add_telemetry.assert_called_once() + mock_post.assert_called_once() def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): self._mock_config("[codecarbon]\n") @@ -276,11 +271,8 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) with patch.dict(os.environ, env_without_telemetry, clear=True): with patch("codecarbon.telemetry.logger.warning") as mock_warning: with patch( - "codecarbon.telemetry.TelemetryClient" - ) as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client + "codecarbon.telemetry.post_private_telemetry", return_value=True + ): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) configure_warnings = [ @@ -293,7 +285,7 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): self._mock_config("[codecarbon]\n") with patch("codecarbon.telemetry.logger.warning") as mock_warning: - with patch("codecarbon.telemetry.TelemetryClient"): + with patch("codecarbon.telemetry.post_private_telemetry"): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker( telemetry_level="disabled", @@ -313,14 +305,16 @@ def test_env_telemetry_disabled_does_not_change_resolved_level( with patch.dict( os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False ): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( save_to_api=False, save_to_file=False ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() + mock_post.assert_called_once() if __name__ == "__main__": From 5a65220b388db7b09c935376c1d909af8a799a97 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:38:28 +0200 Subject: [PATCH 18/29] refactor: overhaul telemetry handling and structure - Replaced the existing telemetry client and settings management with a more modular approach, introducing a dedicated Telemetry class for dispatching telemetry data. - Consolidated telemetry payload construction and validation into a streamlined process, enhancing clarity and maintainability. - Removed deprecated telemetry client and settings files, ensuring a cleaner codebase. - Updated telemetry context collection to improve data accuracy and privacy compliance. - Enhanced tests to validate the new telemetry structure and ensure proper functionality across various scenarios. --- .../tests/api/test_telemetry_schema_drift.py | 4 +- codecarbon/cli/telemetry_cli.py | 14 +- codecarbon/core/telemetry/client.py | 84 ++++++++ .../collect.py} | 140 ++++++++----- codecarbon/core/telemetry/dispatcher.py | 89 +++++++++ .../schemas.py} | 0 codecarbon/core/telemetry/settings.py | 144 ++++++++++++++ codecarbon/core/telemetry_client.py | 47 ----- codecarbon/core/telemetry_settings.py | 185 ------------------ codecarbon/emissions_tracker.py | 42 ++-- codecarbon/telemetry.py | 152 -------------- tests/test_telemetry.py | 75 +++---- tests/test_telemetry_client.py | 54 +++-- tests/test_telemetry_collect.py | 118 ++++++----- tests/test_telemetry_config.py | 128 +++++------- tests/test_telemetry_settings.py | 135 +++++++------ 16 files changed, 685 insertions(+), 726 deletions(-) create mode 100644 codecarbon/core/telemetry/client.py rename codecarbon/core/{telemetry_collect.py => telemetry/collect.py} (71%) create mode 100644 codecarbon/core/telemetry/dispatcher.py rename codecarbon/core/{telemetry_schemas.py => telemetry/schemas.py} (100%) create mode 100644 codecarbon/core/telemetry/settings.py delete mode 100644 codecarbon/core/telemetry_client.py delete mode 100644 codecarbon/core/telemetry_settings.py delete mode 100644 codecarbon/telemetry.py diff --git a/carbonserver/tests/api/test_telemetry_schema_drift.py b/carbonserver/tests/api/test_telemetry_schema_drift.py index abc20d764..388e39963 100644 --- a/carbonserver/tests/api/test_telemetry_schema_drift.py +++ b/carbonserver/tests/api/test_telemetry_schema_drift.py @@ -7,7 +7,9 @@ from carbonserver.api.schemas_telemetry import TelemetryCreate as ServerTelemetryCreate REPO_ROOT = Path(__file__).resolve().parents[3] -CORE_TELEMETRY_SCHEMA_PATH = REPO_ROOT / "codecarbon" / "core" / "telemetry_schemas.py" +CORE_TELEMETRY_SCHEMA_PATH = ( + REPO_ROOT / "codecarbon" / "core" / "telemetry" / "schemas.py" +) def _load_core_telemetry_create(): diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py index 0cf7a8d77..df23a30a5 100644 --- a/codecarbon/cli/telemetry_cli.py +++ b/codecarbon/cli/telemetry_cli.py @@ -14,12 +14,11 @@ overwrite_local_config, ) from codecarbon.core.config import get_config_file_settings, get_hierarchical_config -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import ( +from codecarbon.core.telemetry import ( DEFAULT_TELEMETRY_LEVEL, - is_telemetry_level_explicit, + TelemetryLevel, + TelemetrySettings, parse_telemetry_level, - resolve_telemetry_level, ) telemetry_app = typer.Typer( @@ -142,11 +141,12 @@ def print_telemetry_status(config_path: Optional[Path] = None) -> None: external_conf = get_hierarchical_config() source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" - level = resolve_telemetry_level( - file_settings, + settings = TelemetrySettings.resolve( + config_file_conf=file_settings, external_conf=external_conf or None, ) - explicit = is_telemetry_level_explicit(file_settings, external_conf=external_conf) + level = settings.level + explicit = settings.is_explicit stored = file_settings.get("telemetry_level") print(f"Config source: {source_label}") print(f"telemetry_level in file(s): {stored!r}") diff --git a/codecarbon/core/telemetry/client.py b/codecarbon/core/telemetry/client.py new file mode 100644 index 000000000..99a5a7ed2 --- /dev/null +++ b/codecarbon/core/telemetry/client.py @@ -0,0 +1,84 @@ +"""HTTP and API clients for product telemetry.""" + +from __future__ import annotations + +import dataclasses + +import requests + +from codecarbon.core.api_client import ApiClient +from codecarbon.core.telemetry.schemas import TelemetryCreate +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + + +def post_private(settings: TelemetrySettings, payload: dict) -> bool: + """POST a private telemetry payload to ``/telemetry``. + + Args: + settings: Resolved telemetry API settings. + payload: Telemetry fields dict. + + Returns: + True if the server accepted the payload (HTTP 201). + """ + headers = {"Content-Type": "application/json"} + if settings.api_key: + headers["x-api-token"] = settings.api_key + body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) + telemetry_url = f"{settings.api_url.rstrip('/')}/telemetry" + try: + response = requests.post( + url=telemetry_url, + json=body, + headers=headers, + timeout=2, + ) + except Exception: + logger.error("Telemetry request failed.", exc_info=True) + return False + if response.status_code == 201: + return True + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + telemetry_url, + ) + else: + logger.error( + "Telemetry API %s: %s", + response.status_code, + response.text, + ) + logger.debug("Telemetry request body: %s", body) + return False + + +def post_public_summary( + settings: TelemetrySettings, + conf: dict, + emissions: EmissionsData, +) -> bool: + """Send Tier 2 public run summary via ``ApiClient``. + + Args: + settings: Resolved telemetry API settings. + conf: Tracker configuration dict. + emissions: Run emissions data. + + Returns: + True if the run summary was posted successfully. + """ + try: + api = ApiClient( + endpoint_url=settings.api_url, + experiment_id=settings.experiment_id, + api_key=settings.api_key, + conf=conf, + create_run_automatically=True, + ) + return bool(api.add_emission(dataclasses.asdict(emissions))) + except Exception as error: + logger.error(f"Public run summary failed (non-critical): {error}") + return False diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry/collect.py similarity index 71% rename from codecarbon/core/telemetry_collect.py rename to codecarbon/core/telemetry/collect.py index db788ad3c..49f5263c3 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry/collect.py @@ -6,12 +6,13 @@ import os import platform import sys +from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, Optional from codecarbon.core.cloud import get_env_cloud_details from codecarbon.core.gpu import is_nvidia_system -from codecarbon.core.telemetry_schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel +from codecarbon.core.telemetry.schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData FRAMEWORK_PACKAGES = ( @@ -39,16 +40,64 @@ ("KUBERNETES_SERVICE_HOST", "kubernetes"), ) -OUTPUT_METHOD_ATTRS = ( - ("_save_to_file", "file"), - ("_save_to_api", "api"), - ("_save_to_logger", "logger"), - ("_emissions_endpoint", "http"), - ("_save_to_prometheus", "prometheus"), - ("_save_to_logfire", "logfire"), +OUTPUT_METHOD_FIELDS = ( + ("save_to_file", "file"), + ("save_to_api", "api"), + ("save_to_logger", "logger"), + ("emissions_endpoint", "http"), + ("save_to_prometheus", "prometheus"), + ("save_to_logfire", "logfire"), ) +@dataclass +class TelemetryContext: + """Snapshot of tracker state used to build a telemetry payload.""" + + conf: dict[str, Any] + emissions: EmissionsData + hardware: list[Any] + resource_tracker: Any + save_to_api: bool + save_to_file: bool + save_to_logger: bool + save_to_prometheus: bool + save_to_logfire: bool + emissions_endpoint: str | None + tasks: dict[str, Any] + measure_power_secs: float | None + is_offline: bool + + @classmethod + def from_tracker(cls, tracker: Any, emissions: EmissionsData) -> TelemetryContext: + """Build a context snapshot from an active emissions tracker. + + Args: + tracker: Active emissions tracker instance. + emissions: Run emissions data. + + Returns: + Context for ``build_payload``. + """ + from codecarbon.emissions_tracker import OfflineEmissionsTracker + + return cls( + conf=getattr(tracker, "_conf", {}), + emissions=emissions, + hardware=getattr(tracker, "_hardware", []) or [], + resource_tracker=getattr(tracker, "_resource_tracker", None), + save_to_api=bool(getattr(tracker, "_save_to_api", False)), + save_to_file=bool(getattr(tracker, "_save_to_file", False)), + save_to_logger=bool(getattr(tracker, "_save_to_logger", False)), + save_to_prometheus=bool(getattr(tracker, "_save_to_prometheus", False)), + save_to_logfire=bool(getattr(tracker, "_save_to_logfire", False)), + emissions_endpoint=getattr(tracker, "_emissions_endpoint", None), + tasks=getattr(tracker, "_tasks", {}) or {}, + measure_power_secs=getattr(tracker, "_measure_power_secs", None), + is_offline=isinstance(tracker, OfflineEmissionsTracker), + ) + + def _non_empty(value: Any) -> bool: return value not in (None, "", [], {}) @@ -106,21 +155,12 @@ def _detect_notebook_environment() -> Optional[str]: return None -def _detect_in_container() -> bool: - if os.path.exists("/.dockerenv"): - return True +def _container_info() -> tuple[bool, Optional[str]]: if os.environ.get("KUBERNETES_SERVICE_HOST"): - return True - return False - - -def _detect_container_runtime() -> Optional[str]: - runtime = _first_env_match(CONTAINER_RUNTIME_ENV) - if runtime: - return runtime + return True, "kubernetes" if os.path.exists("/.dockerenv"): - return "docker" - return None + return True, "docker" + return False, None def _detect_ide() -> Optional[str]: @@ -145,20 +185,19 @@ def _cudnn_version() -> Optional[str]: return None -def _collect_hardware_diagnostics(tracker: Any) -> dict[str, Any]: +def _collect_hardware_diagnostics(ctx: TelemetryContext) -> dict[str, Any]: from codecarbon.core import cpu hardware_tracked: list[str] = [] - for item in getattr(tracker, "_hardware", []) or []: + for item in ctx.hardware: try: hardware_tracked.append(item.description()) except Exception: pass - resource_tracker = getattr(tracker, "_resource_tracker", None) gpu_detection_method: Optional[str] = None - if resource_tracker is not None: - gpu_tracker = getattr(resource_tracker, "gpu_tracker", None) + if ctx.resource_tracker is not None: + gpu_tracker = getattr(ctx.resource_tracker, "gpu_tracker", None) if gpu_tracker and gpu_tracker != "Unspecified": gpu_detection_method = gpu_tracker @@ -166,20 +205,17 @@ def _collect_hardware_diagnostics(tracker: Any) -> dict[str, Any]: if platform.system() == "Linux": rapl_available = cpu.is_rapl_available() - save_to_api = bool(getattr(tracker, "_save_to_api", False)) return { "hardware_tracked": hardware_tracked or None, "hardware_detection_success": bool(hardware_tracked), "rapl_available": rapl_available, "gpu_detection_method": gpu_detection_method, - "api_mode": "online" if save_to_api else "offline", + "api_mode": "online" if ctx.save_to_api else "offline", } -def _detect_integration_surface(tracker: Any) -> str: - from codecarbon.emissions_tracker import OfflineEmissionsTracker - - if isinstance(tracker, OfflineEmissionsTracker): +def _detect_integration_surface(ctx: TelemetryContext) -> str: + if ctx.is_offline: return "offline_tracker" argv = " ".join(sys.argv) if "codecarbon" in argv and "monitor" in argv: @@ -187,15 +223,15 @@ def _detect_integration_surface(tracker: Any) -> str: return "library" -def _collect_output_methods(tracker: Any) -> list[str]: +def _collect_output_methods(ctx: TelemetryContext) -> list[str]: methods: list[str] = [] - for attr, name in OUTPUT_METHOD_ATTRS: - value = getattr(tracker, attr, False) - if attr == "_emissions_endpoint": + for field_name, label in OUTPUT_METHOD_FIELDS: + value = getattr(ctx, field_name) + if field_name == "emissions_endpoint": if value: - methods.append(name) + methods.append(label) elif value: - methods.append(name) + methods.append(label) return methods @@ -245,22 +281,21 @@ def _gpu_static_fields() -> dict[str, Any]: return fields -def build_telemetry_payload( - tracker: Any, - emissions: EmissionsData, +def build_payload( + ctx: TelemetryContext, level: TelemetryLevel = TelemetryLevel.minimal, ) -> dict[str, Any]: """Build a private telemetry payload dict for ``POST /telemetry``. Args: - tracker: Active emissions tracker. - emissions: Run emissions data. + ctx: Tracker snapshot from ``TelemetryContext.from_tracker``. level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). Returns: Payload dict for ``TelemetryCreate``. """ - conf = getattr(tracker, "_conf", {}) + emissions = ctx.emissions + conf = ctx.conf raw_provider, raw_region = _raw_cloud_provider_and_region() on_cloud = emissions.on_cloud == "Y" cloud_provider = emissions.cloud_provider or raw_provider @@ -269,7 +304,8 @@ def build_telemetry_payload( if on_cloud and cloud_region: region = region or cloud_region - integration_surface = _detect_integration_surface(tracker) + integration_surface = _detect_integration_surface(ctx) + in_container, container_runtime = _container_info() gpu_fields = _gpu_static_fields() raw: dict[str, Any] = { @@ -297,12 +333,12 @@ def build_telemetry_payload( "tracking_mode": conf.get("tracking_mode"), "integration_surface": integration_surface, "offline_mode": integration_surface == "offline_tracker", - "output_methods": _collect_output_methods(tracker), - "save_to_api_enabled": bool(getattr(tracker, "_save_to_api", False)), - "task_tracking_used": bool(getattr(tracker, "_tasks", {})), - "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), - "in_container": _detect_in_container(), - "container_runtime": _detect_container_runtime(), + "output_methods": _collect_output_methods(ctx), + "save_to_api_enabled": ctx.save_to_api, + "task_tracking_used": bool(ctx.tasks), + "measure_power_interval_secs": ctx.measure_power_secs, + "in_container": in_container, + "container_runtime": container_runtime, "ci_environment": _first_env_match(CI_ENVIRONMENTS), "notebook_environment": _detect_notebook_environment(), "ide_used": _detect_ide(), @@ -319,7 +355,7 @@ def build_telemetry_payload( "ram_utilization_avg": emissions.ram_utilization_percent, "telemetry_level": level.value, **_collect_framework_fields(), - **_collect_hardware_diagnostics(tracker), + **_collect_hardware_diagnostics(ctx), **gpu_fields, } diff --git a/codecarbon/core/telemetry/dispatcher.py b/codecarbon/core/telemetry/dispatcher.py new file mode 100644 index 000000000..24174d94f --- /dev/null +++ b/codecarbon/core/telemetry/dispatcher.py @@ -0,0 +1,89 @@ +"""Per-tracker telemetry dispatcher.""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + +TELEMETRY_NOT_CONFIGURED_MESSAGE = ( + "CodeCarbon telemetry_level was not set explicitly; using default %r. " + "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " + "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " + "to EmissionsTracker / OfflineEmissionsTracker, or run " + "codecarbon telemetry set ." +) + + +class Telemetry: + """Per-tracker telemetry dispatcher.""" + + _default_warning_shown: ClassVar[bool] = False + + def __init__(self, settings: TelemetrySettings) -> None: + self.settings = settings + + @classmethod + def from_tracker(cls, tracker: Any) -> Telemetry: + """Build a dispatcher from tracker config state. + + Args: + tracker: Active emissions tracker with ``_config_file_conf``, + ``_external_conf``, and optional ``_telemetry_override``. + + Returns: + Configured ``Telemetry`` instance. + """ + return cls( + TelemetrySettings.resolve( + config_file_conf=tracker._config_file_conf, + external_conf=tracker._external_conf, + override=getattr(tracker, "_telemetry_override", None), + ) + ) + + def warn_if_implicit(self) -> None: + """Log a one-time warning when telemetry tier was not set explicitly.""" + if self.settings.is_explicit or Telemetry._default_warning_shown: + return + logger.warning( + TELEMETRY_NOT_CONFIGURED_MESSAGE, + self.settings.level.value, + ) + Telemetry._default_warning_shown = True + + def send_at_stop(self, tracker: Any, emissions: EmissionsData) -> None: + """Send product telemetry for the resolved tier at tracker ``stop()``. + + Tier 1 (``minimal``): private ``POST /telemetry`` only. + Tier 2 (``extensive``): Tier 1 plus public run summary. + + Args: + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + """ + if self.settings.level == TelemetryLevel.disabled: + return + if emissions.duration is not None and emissions.duration < 1: + logger.debug("Telemetry not sent: run shorter than 1 second.") + return + ctx = TelemetryContext.from_tracker(tracker, emissions) + payload = build_payload(ctx, level=self.settings.level) + try: + post_private(self.settings, payload) + except Exception as error: + logger.error(f"Private telemetry failed (non-critical): {error}") + if self.settings.level == TelemetryLevel.extensive: + try: + post_public_summary( + self.settings, + getattr(tracker, "_conf", {}), + emissions, + ) + except Exception as error: + logger.error(f"Public run summary failed (non-critical): {error}") diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry/schemas.py similarity index 100% rename from codecarbon/core/telemetry_schemas.py rename to codecarbon/core/telemetry/schemas.py diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py new file mode 100644 index 000000000..8c2c28cb7 --- /dev/null +++ b/codecarbon/core/telemetry/settings.py @@ -0,0 +1,144 @@ +"""Resolve telemetry tier and API settings from config and environment.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Literal + +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.external.logger import logger + +DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" +DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" +DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal + +TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" + +TelemetryLevelSource = Literal["override", "external", "file", "default"] + + +def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: + """Parse a telemetry tier from a string or enum value. + + Args: + raw: Tier name or ``TelemetryLevel`` member. + + Returns: + Parsed ``TelemetryLevel``. + + Raises: + ValueError: If ``raw`` is not a valid tier name. + """ + if isinstance(raw, TelemetryLevel): + return raw + try: + return TelemetryLevel(str(raw).lower()) + except ValueError as error: + raise ValueError( + f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." + ) from error + + +@dataclass(frozen=True) +class TelemetrySettings: + """Resolved telemetry tier and API connection settings.""" + + level: TelemetryLevel + source: TelemetryLevelSource + api_url: str + api_key: str + experiment_id: str + + @property + def is_explicit(self) -> bool: + """Return whether the user explicitly chose a telemetry tier.""" + return self.source != "default" + + @classmethod + def resolve( + cls, + *, + config_file_conf: dict[str, Any] | None = None, + external_conf: dict[str, Any] | None = None, + override: str | TelemetryLevel | None = None, + ) -> TelemetrySettings: + """Resolve telemetry tier and API settings. + + Precedence for tier: + + 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI + 2. ``external_conf`` — merged ``.codecarbon.config`` and env + 3. ``config_file_conf`` — file-only settings + 4. Default: ``minimal`` + + Args: + config_file_conf: Settings from ``get_config_file_settings()``. + external_conf: Merged settings from ``get_hierarchical_config()``. + override: Optional tier from tracker or CLI. + + Returns: + Resolved settings bundle. + """ + merged = external_conf or {} + if override is not None: + raw = override + source: TelemetryLevelSource = "override" + elif merged.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + raw = merged[TELEMETRY_LEVEL_CONFIG_KEY] + source = "external" + elif config_file_conf is not None and config_file_conf.get( + TELEMETRY_LEVEL_CONFIG_KEY + ) is not None: + raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "file" + else: + return cls( + level=DEFAULT_TELEMETRY_LEVEL, + source="default", + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + try: + level = parse_telemetry_level(raw) + except ValueError: + logger.error( + "Invalid telemetry_level %r; falling back to %r", + raw, + DEFAULT_TELEMETRY_LEVEL.value, + ) + level = DEFAULT_TELEMETRY_LEVEL + return cls( + level=level, + source=source, + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + + @staticmethod + def _resolve_api_url(external_conf: dict[str, Any]) -> str: + url = ( + external_conf.get("telemetry_api_url") + or external_conf.get("api_endpoint") + or os.environ.get("CODECARBON_TELEMETRY_API_URL") + ) + return (url or DEFAULT_TELEMETRY_API_URL).rstrip("/") + + @staticmethod + def _resolve_api_key(external_conf: dict[str, Any]) -> str: + key = ( + external_conf.get("telemetry_api_key") + or external_conf.get("api_key") + or os.environ.get("CODECARBON_TELEMETRY_API_KEY") + ) + return key or DEFAULT_TELEMETRY_API_KEY + + @staticmethod + def _resolve_experiment_id(external_conf: dict[str, Any]) -> str: + experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( + "CODECARBON_TELEMETRY_EXPERIMENT_ID" + ) + return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py deleted file mode 100644 index fbfe76e3d..000000000 --- a/codecarbon/core/telemetry_client.py +++ /dev/null @@ -1,47 +0,0 @@ -import requests - -from codecarbon.core.telemetry_schemas import TelemetryCreate -from codecarbon.external.logger import logger - - -def post_private_telemetry(url: str, payload: dict, api_key: str | None) -> bool: - """POST a private telemetry payload to ``/telemetry``. - - Args: - url: API base URL. - payload: Telemetry fields dict. - api_key: Optional API token. - - Returns: - True if the server accepted the payload (HTTP 201). - """ - headers = {"Content-Type": "application/json"} - if api_key: - headers["x-api-token"] = api_key - body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) - telemetry_url = f"{url.rstrip('/')}/telemetry" - try: - response = requests.post( - url=telemetry_url, - json=body, - headers=headers, - timeout=2, - ) - except Exception: - logger.error("Telemetry request failed.", exc_info=True) - return False - if response.status_code == 201: - return True - if response.status_code == 404: - logger.warning( - "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", - telemetry_url, - ) - else: - logger.error( - "Telemetry API %s: %s", - response.status_code, - response.text, - ) - logger.debug("Telemetry request body: %s", body) - return False diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py deleted file mode 100644 index a0805ed00..000000000 --- a/codecarbon/core/telemetry_settings.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Resolve telemetry tier and API settings from config and environment.""" - -import os -from typing import Any, Literal - -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.external.logger import logger - -DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" -DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" -DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" -DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal - -TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" - -TelemetryLevelSource = Literal["override", "external", "file", "default"] - - -def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: - """Parse a telemetry tier from a string or enum value. - - Args: - raw: Tier name or ``TelemetryLevel`` member. - - Returns: - Parsed ``TelemetryLevel``. - - Raises: - ValueError: If ``raw`` is not a valid tier name. - """ - if isinstance(raw, TelemetryLevel): - return raw - try: - return TelemetryLevel(str(raw).lower()) - except ValueError as error: - raise ValueError( - f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." - ) from error - - -def resolve_telemetry_level_and_source( - config_file_conf: dict[str, Any] | None = None, - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> tuple[TelemetryLevel, TelemetryLevelSource]: - """Resolve the active telemetry tier and where it came from. - - Precedence: - - 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI - 2. ``external_conf`` — merged ``.codecarbon.config`` and ``CODECARBON_*`` env - 3. ``config_file_conf`` — file-only settings when ``external_conf`` is omitted - 4. Default: ``minimal`` (Tier 1) - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (optional). - override: Optional tier from tracker or CLI. - external_conf: Merged settings from ``get_hierarchical_config()`` (optional). - - Returns: - Resolved tier and its source label. - """ - if override is not None: - raw = override - source: TelemetryLevelSource = "override" - elif external_conf is not None and external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: - raw = external_conf[TELEMETRY_LEVEL_CONFIG_KEY] - source = "external" - elif config_file_conf is not None and config_file_conf.get( - TELEMETRY_LEVEL_CONFIG_KEY - ) is not None: - raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] - source = "file" - else: - return DEFAULT_TELEMETRY_LEVEL, "default" - try: - return parse_telemetry_level(raw), source - except ValueError: - logger.error( - "Invalid telemetry_level %r; falling back to %r", - raw, - DEFAULT_TELEMETRY_LEVEL.value, - ) - return DEFAULT_TELEMETRY_LEVEL, source - - -def resolve_telemetry_level( - config_file_conf: dict[str, Any] | None = None, - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> TelemetryLevel: - """Resolve the active telemetry tier. - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (optional). - override: Optional tier from tracker or CLI. - external_conf: Merged settings from ``get_hierarchical_config()`` (optional). - - Returns: - Resolved ``TelemetryLevel``. - """ - return resolve_telemetry_level_and_source( - config_file_conf, - override=override, - external_conf=external_conf, - )[0] - - -def is_telemetry_level_explicit( - config_file_conf: dict[str, Any], - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> bool: - """Return whether the user explicitly chose a telemetry tier. - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). - override: Value passed to ``EmissionsTracker(telemetry_level=...)``. - external_conf: Merged config from file and environment. - - Returns: - True if any explicit source is set. - """ - _, source = resolve_telemetry_level_and_source( - config_file_conf, - override=override, - external_conf=external_conf, - ) - return source != "default" - - -def get_telemetry_api_url( - external_conf: dict[str, Any], - default: str = DEFAULT_TELEMETRY_API_URL, -) -> str: - """Return telemetry API base URL from config or environment. - - Args: - external_conf: Merged config from file and environment. - default: URL used when unset. - - Returns: - API base URL without trailing slash. - """ - url = ( - external_conf.get("telemetry_api_url") - or external_conf.get("api_endpoint") - or os.environ.get("CODECARBON_TELEMETRY_API_URL") - ) - return (url or default).rstrip("/") - - -def get_telemetry_api_key(external_conf: dict[str, Any]) -> str: - """Return telemetry API token from config, environment, or public default. - - Args: - external_conf: Merged config from file and environment. - - Returns: - API token string. - """ - key = ( - external_conf.get("telemetry_api_key") - or external_conf.get("api_key") - or os.environ.get("CODECARBON_TELEMETRY_API_KEY") - ) - return key or DEFAULT_TELEMETRY_API_KEY - - -def get_telemetry_experiment_id(external_conf: dict[str, Any]) -> str: - """Return telemetry experiment id from config, environment, or public default. - - Args: - external_conf: Merged config from file and environment. - - Returns: - Experiment UUID string (legacy leaderboard / API helpers). - """ - experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( - "CODECARBON_TELEMETRY_EXPERIMENT_ID" - ) - return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index fa4eeb166..30cb4b571 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -22,12 +22,7 @@ get_hierarchical_config, normalize_gpu_ids, ) -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import resolve_telemetry_level -from codecarbon.telemetry import ( - send_product_telemetry_at_stop, - warn_if_telemetry_not_configured, -) +from codecarbon.core.telemetry import Telemetry, TelemetrySettings from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker from codecarbon.core.units import Energy, Power, Time, Water @@ -289,11 +284,15 @@ def __init__( self._external_conf = get_hierarchical_config() self._config_file_conf = get_config_file_settings() - telemetry_override = None if telemetry_level is _sentinel else telemetry_level - self._telemetry_level = resolve_telemetry_level( - self._config_file_conf, - override=telemetry_override, - external_conf=self._external_conf, + self._telemetry_override = ( + None if telemetry_level is _sentinel else telemetry_level + ) + self._telemetry = Telemetry( + TelemetrySettings.resolve( + config_file_conf=self._config_file_conf, + external_conf=self._external_conf, + override=self._telemetry_override, + ) ) self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", True, bool) if self._allow_multiple_runs: @@ -468,25 +467,12 @@ def __init__( self._data_source, self._electricitymaps_api_token ) - self._apply_init_telemetry(telemetry_override) + self._telemetry.warn_if_implicit() self._init_output_methods(api_key=self._api_key) @suppress(Exception) - def _apply_init_telemetry(self, telemetry_override: str | None) -> None: - warn_if_telemetry_not_configured( - self._config_file_conf, - self._telemetry_level, - override=telemetry_override, - external_conf=self._external_conf, - ) - @suppress(Exception) - def _send_product_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: - send_product_telemetry_at_stop( - self, - emissions_data, - self._telemetry_level, - external_conf=self._external_conf, - ) + def _send_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: + self._telemetry.send_at_stop(self, emissions_data) def _init_output_methods(self, *, api_key: str = None): """ @@ -789,7 +775,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) - self._send_product_telemetry_at_stop(emissions_data) + self._send_telemetry_at_stop(emissions_data) self._persist_data( total_emissions=emissions_data, diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py deleted file mode 100644 index eeec82434..000000000 --- a/codecarbon/telemetry.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" - -import dataclasses -from typing import Any - -from codecarbon.core.api_client import ApiClient -from codecarbon.core.telemetry_client import post_private_telemetry -from codecarbon.core.telemetry_collect import build_telemetry_payload -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import ( - get_telemetry_api_key, - get_telemetry_api_url, - get_telemetry_experiment_id, - is_telemetry_level_explicit, -) -from codecarbon.external.logger import logger -from codecarbon.output_methods.emissions_data import EmissionsData - -TELEMETRY_NOT_CONFIGURED_MESSAGE = ( - "CodeCarbon telemetry_level was not set explicitly; using default %r. " - "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " - "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " - "to EmissionsTracker / OfflineEmissionsTracker, or run " - "codecarbon telemetry set ." -) - -_telemetry_default_warning_shown = False - - -def reset_telemetry_warning() -> None: - """Clear the one-shot default-tier warning (for tests).""" - global _telemetry_default_warning_shown - _telemetry_default_warning_shown = False - - -def warn_if_telemetry_not_configured( - config_file_conf: dict[str, Any], - active_level: TelemetryLevel, - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> None: - """Log a one-time warning when telemetry tier was not set explicitly. - - Args: - config_file_conf: File-only settings from ``get_config_file_settings()``. - active_level: Resolved tier in use for this tracker. - override: Optional ``telemetry_level`` tracker argument. - external_conf: Merged file/env settings for explicit-env detection. - """ - global _telemetry_default_warning_shown - if is_telemetry_level_explicit( - config_file_conf, override=override, external_conf=external_conf - ): - return - if _telemetry_default_warning_shown: - return - logger.warning(TELEMETRY_NOT_CONFIGURED_MESSAGE, active_level.value) - _telemetry_default_warning_shown = True - - -def send_private_telemetry_at_stop( - tracker: Any, - emissions: EmissionsData, - external_conf: dict[str, Any] | None = None, - level: TelemetryLevel = TelemetryLevel.minimal, -) -> bool: - """Send Tier 1 private telemetry via ``POST /telemetry``. - - Runs shorter than one second are skipped by ``send_product_telemetry_at_stop``, - not here; direct callers may still post for sub-second runs. - - Args: - tracker: Active emissions tracker instance. - emissions: Total emissions from ``_prepare_emissions_data()``. - external_conf: Merged config for telemetry API URL and key resolution. - level: Resolved ``TelemetryLevel`` for the ``telemetry_level`` field. - - Returns: - True if the private telemetry POST was accepted, False otherwise. - """ - settings_conf = external_conf or {} - try: - payload = build_telemetry_payload(tracker, emissions, level=level) - return post_private_telemetry( - get_telemetry_api_url(settings_conf), - payload, - get_telemetry_api_key(settings_conf), - ) - except Exception as error: - logger.error(f"Private telemetry failed (non-critical): {error}") - return False - - -def send_public_run_summary_at_stop( - tracker: Any, - emissions: EmissionsData, - external_conf: dict[str, Any] | None = None, -) -> bool: - """Send Tier 2 public run summary to the shared telemetry experiment via ``ApiClient``. - - Args: - tracker: Active emissions tracker instance. - emissions: Total emissions from ``_prepare_emissions_data()``. - external_conf: Merged config for API URL, key, and experiment resolution. - - Returns: - True if the run summary was posted successfully, False otherwise. - """ - settings_conf = external_conf or {} - conf = getattr(tracker, "_conf", {}) - try: - api = ApiClient( - endpoint_url=get_telemetry_api_url(settings_conf), - experiment_id=get_telemetry_experiment_id(settings_conf), - api_key=get_telemetry_api_key(settings_conf), - conf=conf, - create_run_automatically=True, - ) - return bool(api.add_emission(dataclasses.asdict(emissions))) - except Exception as error: - logger.error(f"Public run summary failed (non-critical): {error}") - return False - - -def send_product_telemetry_at_stop( - tracker: Any, - emissions: EmissionsData, - level: TelemetryLevel, - external_conf: dict[str, Any] | None = None, -) -> None: - """Send product telemetry for the resolved tier at tracker ``stop()``. - - Tier 1 (``minimal``): private ``POST /telemetry`` only. - Tier 2 (``extensive``): same private ``POST /telemetry`` plus public run summary. - - Args: - tracker: Active emissions tracker instance. - emissions: Total emissions from ``_prepare_emissions_data()``. - level: Resolved ``TelemetryLevel``. - external_conf: Merged config for API settings. - """ - if level == TelemetryLevel.disabled: - return - if emissions.duration is not None and emissions.duration < 1: - logger.debug("Telemetry not sent: run shorter than 1 second.") - return - settings = external_conf or {} - if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_private_telemetry_at_stop(tracker, emissions, settings, level=level) - if level == TelemetryLevel.extensive: - send_public_run_summary_at_stop(tracker, emissions, settings) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 757283d7a..dd157fb93 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -3,13 +3,9 @@ import unittest from unittest.mock import ANY, MagicMock, patch -from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry import Telemetry from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import ( - send_private_telemetry_at_stop, - send_public_run_summary_at_stop, -) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -62,46 +58,57 @@ def _emissions(self) -> EmissionsData: def test_tier1_posts_private_telemetry(self): tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None + tracker._conf = {"os": "Linux", "tracking_mode": "machine"} + tracker._hardware = [] + tracker._resource_tracker = None + tracker._save_to_api = False + tracker._save_to_file = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 emissions = self._emissions() + telemetry = Telemetry.from_tracker(tracker) with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - result = send_private_telemetry_at_stop( - tracker, emissions, external_conf={} - ) - self.assertTrue(result) + telemetry.send_at_stop(tracker, emissions) mock_post.assert_called_once() payload = mock_post.call_args[0][1] self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.001) def test_tier1_skips_short_duration_at_dispatcher(self): - from codecarbon.telemetry import send_product_telemetry_at_stop - tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None emissions = self._emissions() emissions.duration = 0.5 - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: - send_product_telemetry_at_stop( - tracker, - emissions, - TelemetryLevel.minimal, - external_conf={}, - ) + telemetry = Telemetry.from_tracker(tracker) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + telemetry.send_at_stop(tracker, emissions) mock_post.assert_not_called() def test_tier2_uses_api_client(self): tracker = MagicMock() tracker._conf = {"os": "Linux"} + tracker._config_file_conf = {"telemetry_level": "extensive"} + tracker._external_conf = {} + tracker._telemetry_override = None emissions = self._emissions() - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api - result = send_public_run_summary_at_stop( - tracker, emissions, external_conf={} - ) - self.assertTrue(result) + telemetry = Telemetry.from_tracker(tracker) + with patch("codecarbon.core.telemetry.dispatcher.post_private", return_value=True): + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + telemetry.send_at_stop(tracker, emissions) mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() mock_api_cls.assert_called_with( @@ -118,11 +125,13 @@ class TestTrackerTelemetry(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() self.patcher = None + Telemetry._default_warning_shown = False def tearDown(self) -> None: if self.patcher: self.patcher.stop() self.temp_dir.cleanup() + Telemetry._default_warning_shown = False def _start_config_mock(self, conf: str) -> None: self.patcher = patch( @@ -132,7 +141,7 @@ def _start_config_mock(self, conf: str) -> None: def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) mock_post.assert_not_called() @@ -143,7 +152,7 @@ def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( @@ -160,7 +169,7 @@ def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): self._start_config_mock(disabled_conf) - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -175,9 +184,9 @@ def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): self._start_config_mock(extensive_conf) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api @@ -197,7 +206,7 @@ def test_offline_tracker_sends_on_stop(self, mock_cli_setup): self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 7d8b4be64..8b0128d2a 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -4,12 +4,20 @@ import requests_mock from pydantic import ValidationError -from codecarbon.core.telemetry_client import post_private_telemetry -from codecarbon.core.telemetry_schemas import TelemetryCreate +from codecarbon.core.telemetry import TelemetrySettings, post_private -class TestPostPrivateTelemetry(unittest.TestCase): - def test_post_private_telemetry_sends_validated_payload(self): +class TestPostPrivate(unittest.TestCase): + def _settings(self, api_url: str = "http://test.com", api_key: str | None = None): + return TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url=api_url, + api_key=api_key or TelemetrySettings.resolve().api_key, + experiment_id=TelemetrySettings.resolve().experiment_id, + ) + + def test_post_private_sends_validated_payload(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -22,7 +30,7 @@ def test_post_private_telemetry_sends_validated_payload(self): json="f52fe339-164d-4c2b-a8c0-f562dfce066d", status_code=201, ) - result = post_private_telemetry("http://test.com", telemetry, None) + result = post_private(self._settings(), telemetry) self.assertTrue(result) self.assertEqual(m.call_count, 1) @@ -34,19 +42,18 @@ def test_post_private_telemetry_sends_validated_payload(self): }, ) - def test_post_private_telemetry_rejects_invalid_payload(self): + def test_post_private_rejects_invalid_payload(self): with self.assertRaises(ValidationError): - post_private_telemetry( - "http://test.com", + post_private( + self._settings(), { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", "unknown_field": "value", }, - None, ) - def test_post_private_telemetry_logs_warning_on_404(self): + def test_post_private_logs_warning_on_404(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -57,32 +64,43 @@ def test_post_private_telemetry_logs_warning_on_404(self): text='{"detail":"Not Found"}', status_code=404, ) - with patch("codecarbon.core.telemetry_client.logger") as mock_logger: - result = post_private_telemetry("http://test.com", telemetry, None) + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_private(self._settings(), telemetry) self.assertFalse(result) mock_logger.warning.assert_called_once() - def test_post_private_telemetry_sends_api_key_header_when_configured(self): + def test_post_private_sends_api_key_header_when_configured(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", } + settings = TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id=TelemetrySettings.resolve().experiment_id, + ) with requests_mock.Mocker() as m: m.post( "http://test.com/telemetry", json="telemetry-id", status_code=201, ) - post_private_telemetry("http://test.com", telemetry, "cpt_test_key") + post_private(settings, telemetry) self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") - def test_post_private_telemetry_returns_false_on_request_error(self): + def test_post_private_returns_false_on_request_error(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", } - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: mock_post.side_effect = ConnectionError("network down") - with patch("codecarbon.core.telemetry_client.logger"): - result = post_private_telemetry("http://test.com", telemetry, None) + with patch("codecarbon.core.telemetry.client.logger"): + result = post_private(self._settings(), telemetry) self.assertFalse(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index be770e460..0a5f4f379 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -1,8 +1,7 @@ import unittest from unittest.mock import MagicMock, patch -from codecarbon.core.telemetry_collect import build_telemetry_payload -from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry import TelemetryContext, TelemetryLevel, build_payload from codecarbon.output_methods.emissions_data import EmissionsData @@ -44,33 +43,55 @@ def _sample_emissions(**overrides): return EmissionsData(**base) -class TestTelemetryCollect(unittest.TestCase): - def test_build_telemetry_payload_includes_run_and_framework_flags(self): - tracker = MagicMock() - tracker._conf = { - "os": "Linux", - "codecarbon_version": "3.0", - "cpu_count": 4, - "tracking_mode": "machine", - } - tracker._geo = None - tracker._save_to_file = True - tracker._save_to_api = False - tracker._save_to_logger = False - tracker._emissions_endpoint = None - tracker._save_to_prometheus = False - tracker._save_to_logfire = False - tracker._tasks = {} - tracker._measure_power_secs = 15 - tracker._hardware = [] - tracker._resource_tracker = None +def _tracker_context(**overrides) -> TelemetryContext: + tracker = MagicMock() + tracker._conf = overrides.pop("conf", {"codecarbon_version": "3.0"}) + tracker._hardware = overrides.pop("hardware", []) + tracker._resource_tracker = overrides.pop("resource_tracker", None) + tracker._save_to_file = overrides.pop("save_to_file", False) + tracker._save_to_api = overrides.pop("save_to_api", False) + tracker._save_to_logger = overrides.pop("save_to_logger", False) + tracker._emissions_endpoint = overrides.pop("emissions_endpoint", None) + tracker._save_to_prometheus = overrides.pop("save_to_prometheus", False) + tracker._save_to_logfire = overrides.pop("save_to_logfire", False) + tracker._tasks = overrides.pop("tasks", {}) + tracker._measure_power_secs = overrides.pop("measure_power_secs", 15) + tracker._is_offline = overrides.pop("is_offline", False) + emissions = overrides.pop("emissions", _sample_emissions()) + ctx = TelemetryContext( + conf=tracker._conf, + emissions=emissions, + hardware=tracker._hardware, + resource_tracker=tracker._resource_tracker, + save_to_api=tracker._save_to_api, + save_to_file=tracker._save_to_file, + save_to_logger=tracker._save_to_logger, + save_to_prometheus=tracker._save_to_prometheus, + save_to_logfire=tracker._save_to_logfire, + emissions_endpoint=tracker._emissions_endpoint, + tasks=tracker._tasks, + measure_power_secs=tracker._measure_power_secs, + is_offline=tracker._is_offline, + ) + return ctx + - emissions = _sample_emissions() +class TestTelemetryCollect(unittest.TestCase): + def test_build_payload_includes_run_and_framework_flags(self): + ctx = _tracker_context( + conf={ + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + }, + save_to_file=True, + ) with patch( - "codecarbon.core.telemetry_collect._package_installed", + "codecarbon.core.telemetry.collect._package_installed", side_effect=lambda name: name == "torch", ): - payload = build_telemetry_payload(tracker, emissions) + payload = build_payload(ctx) self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.5) @@ -78,48 +99,23 @@ def test_build_telemetry_payload_includes_run_and_framework_flags(self): self.assertTrue(payload["has_torch"]) self.assertIn("file", payload["output_methods"]) - def test_build_telemetry_payload_omits_framework_versions(self): - tracker = MagicMock() - tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} - tracker._geo = None - tracker._save_to_file = False - tracker._save_to_api = False - tracker._save_to_logger = False - tracker._emissions_endpoint = None - tracker._save_to_prometheus = False - tracker._save_to_logfire = False - tracker._tasks = {} - tracker._measure_power_secs = 15 - tracker._hardware = [] - tracker._resource_tracker = None - - emissions = _sample_emissions() + def test_build_payload_omits_framework_versions(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0", "hardware": ["cpu"]}) with patch( - "codecarbon.core.telemetry_collect._package_installed", + "codecarbon.core.telemetry.collect._package_installed", return_value=True, ): - payload = build_telemetry_payload(tracker, emissions) + payload = build_payload(ctx) self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) self.assertNotIn("torch_version", payload) - def test_build_telemetry_payload_uses_resolved_level(self): - tracker = MagicMock() - tracker._conf = {"codecarbon_version": "3.0"} - tracker._save_to_file = False - tracker._save_to_api = False - tracker._save_to_logger = False - tracker._emissions_endpoint = None - tracker._save_to_prometheus = False - tracker._save_to_logfire = False - tracker._tasks = {} - tracker._measure_power_secs = 15 - tracker._hardware = [] - tracker._resource_tracker = None - - emissions = _sample_emissions() - payload = build_telemetry_payload( - tracker, emissions, level=TelemetryLevel.extensive - ) + def test_build_payload_uses_resolved_level(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.extensive) self.assertEqual(payload["telemetry_level"], "extensive") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 5627ae529..d80ee9ecf 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -9,16 +9,8 @@ from unittest.mock import MagicMock, patch from codecarbon.core.config import get_config_file_settings -from codecarbon.core.telemetry_settings import resolve_telemetry_level -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import get_telemetry_api_url +from codecarbon.core.telemetry import Telemetry, TelemetryLevel, TelemetrySettings, post_private from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker -from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import ( - reset_telemetry_warning, - send_private_telemetry_at_stop, - warn_if_telemetry_not_configured, -) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -35,20 +27,27 @@ def _conf(level: str) -> str: class TestTelemetryConfigContract(unittest.TestCase): def setUp(self) -> None: - reset_telemetry_warning() + Telemetry._default_warning_shown = False + + def tearDown(self) -> None: + Telemetry._default_warning_shown = False def test_warns_once_when_telemetry_not_explicit(self): - with patch("codecarbon.telemetry.logger.warning") as mock_warning: - warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) - warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) + settings = TelemetrySettings.resolve(config_file_conf={}) + telemetry = Telemetry(settings) + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + telemetry.warn_if_implicit() + telemetry.warn_if_implicit() self.assertEqual(mock_warning.call_count, 1) self.assertIn("Tier 1", mock_warning.call_args[0][0]) def test_no_warn_when_config_explicit(self): - with patch("codecarbon.telemetry.logger.warning") as mock_warning: - warn_if_telemetry_not_configured( - {"telemetry_level": "disabled"}, TelemetryLevel.disabled - ) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + telemetry = Telemetry(settings) + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + telemetry.warn_if_implicit() mock_warning.assert_not_called() def test_tier1_posts_to_telemetry_endpoint(self): @@ -58,51 +57,13 @@ def test_tier1_posts_to_telemetry_endpoint(self): "total_emissions_kg": 0.001, "os": "Linux", } - tracker = MagicMock() - emissions = EmissionsData( - timestamp="2020-01-01T00:00:00", - project_name="test", - run_id="run-1", - experiment_id="exp-1", - duration=10.0, - emissions=0.001, - emissions_rate=0.0001, - cpu_power=0.0, - gpu_power=0.0, - ram_power=0.0, - cpu_energy=0.0, - gpu_energy=0.0, - ram_energy=0.0, - energy_consumed=0.01, - water_consumed=0.0, - country_name="France", - country_iso_code="FRA", - region="idf", - cloud_provider="", - cloud_region="", - os="Linux", - python_version="3.11", - codecarbon_version="2.0", - cpu_count=1.0, - cpu_model="cpu", - gpu_count=0.0, - gpu_model="", - longitude=0.0, - latitude=0.0, - ram_total_size=8.0, - tracking_mode="machine", + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://tier1.example"} ) - with patch( - "codecarbon.telemetry.build_telemetry_payload", return_value=tier1_payload - ): - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: - mock_post.return_value.status_code = 201 - mock_post.return_value.json.return_value = "telemetry-id" - send_private_telemetry_at_stop( - tracker, - emissions, - external_conf={"telemetry_api_url": "http://tier1.example"}, - ) + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + post_private(settings, tier1_payload) mock_post.assert_called_once() self.assertEqual( mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" @@ -128,11 +89,11 @@ def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): ): from codecarbon.core.config import get_hierarchical_config - level = resolve_telemetry_level( - get_config_file_settings(), + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), external_conf=get_hierarchical_config(), ) - self.assertEqual(level, TelemetryLevel.minimal) + self.assertEqual(settings.level, TelemetryLevel.minimal) def test_env_codecarbon_telemetry_level_overrides_file(self): with tempfile.TemporaryDirectory() as tmp: @@ -149,11 +110,11 @@ def test_env_codecarbon_telemetry_level_overrides_file(self): ): from codecarbon.core.config import get_hierarchical_config - level = resolve_telemetry_level( - get_config_file_settings(), + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), external_conf=get_hierarchical_config(), ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_telemetry_api_url_env_used_for_tier2_client(self): with patch.dict( @@ -161,25 +122,26 @@ def test_telemetry_api_url_env_used_for_tier2_client(self): {"CODECARBON_TELEMETRY_API_URL": "http://env-telemetry.example"}, clear=False, ): - url = get_telemetry_api_url({}) - self.assertEqual(url, "http://env-telemetry.example") + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env-telemetry.example") def test_telemetry_api_url_from_config_overrides_default(self): - url = get_telemetry_api_url( - {"telemetry_api_url": "http://config-telemetry.example"} + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://config-telemetry.example"} ) - self.assertEqual(url, "http://config-telemetry.example") + self.assertEqual(settings.api_url, "http://config-telemetry.example") @mock_platform_cli_setup class TestTrackerTelemetryFromConfig(unittest.TestCase): def setUp(self) -> None: - reset_telemetry_warning() + Telemetry._default_warning_shown = False self._config_patcher = None def tearDown(self) -> None: if self._config_patcher: self._config_patcher.stop() + Telemetry._default_warning_shown = False def _mock_config(self, conf: str) -> None: self._config_patcher = patch( @@ -189,7 +151,7 @@ def _mock_config(self, conf: str) -> None: def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): self._mock_config(_conf("disabled")) - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -204,7 +166,7 @@ def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( @@ -224,9 +186,9 @@ def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup self._mock_config(_conf("extensive")) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api @@ -246,7 +208,7 @@ def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", @@ -269,9 +231,9 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) ) } with patch.dict(os.environ, env_without_telemetry, clear=True): - with patch("codecarbon.telemetry.logger.warning") as mock_warning: + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) @@ -284,8 +246,8 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): self._mock_config("[codecarbon]\n") - with patch("codecarbon.telemetry.logger.warning") as mock_warning: - with patch("codecarbon.telemetry.post_private_telemetry"): + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch("codecarbon.core.telemetry.dispatcher.post_private"): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker( telemetry_level="disabled", @@ -306,7 +268,7 @@ def test_env_telemetry_disabled_does_not_change_resolved_level( os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False ): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py index c09582c62..e9ad2bb2e 100644 --- a/tests/test_telemetry_settings.py +++ b/tests/test_telemetry_settings.py @@ -2,17 +2,13 @@ import unittest from unittest.mock import patch -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import ( +from codecarbon.core.telemetry import ( DEFAULT_TELEMETRY_API_KEY, DEFAULT_TELEMETRY_API_URL, DEFAULT_TELEMETRY_EXPERIMENT_ID, - get_telemetry_api_key, - get_telemetry_api_url, - get_telemetry_experiment_id, - is_telemetry_level_explicit, + TelemetryLevel, + TelemetrySettings, parse_telemetry_level, - resolve_telemetry_level, ) @@ -30,49 +26,58 @@ def test_parse_rejects_invalid(self): parse_telemetry_level("bogus") -class TestResolveTelemetryLevel(unittest.TestCase): +class TestTelemetrySettingsResolve(unittest.TestCase): def test_default_is_minimal_when_unset(self): - level = resolve_telemetry_level({}) - self.assertEqual(level, TelemetryLevel.minimal) + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertEqual(settings.level, TelemetryLevel.minimal) + self.assertEqual(settings.source, "default") def test_telemetry_level_from_config_file(self): - level = resolve_telemetry_level({"telemetry_level": "disabled"}) - self.assertEqual(level, TelemetryLevel.disabled) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + self.assertEqual(settings.source, "file") def test_telemetry_level_extensive(self): - level = resolve_telemetry_level({"telemetry_level": "extensive"}) - self.assertEqual(level, TelemetryLevel.extensive) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "extensive"} + ) + self.assertEqual(settings.level, TelemetryLevel.extensive) def test_env_telemetry_key_ignored(self): with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): - level = resolve_telemetry_level({"telemetry": "extensive"}) - self.assertEqual(level, TelemetryLevel.minimal) + settings = TelemetrySettings.resolve(config_file_conf={"telemetry": "extensive"}) + self.assertEqual(settings.level, TelemetryLevel.minimal) def test_invalid_level_falls_back_to_minimal(self): - with patch("codecarbon.core.telemetry_settings.logger.error") as mock_error: - level = resolve_telemetry_level({"telemetry_level": "bogus"}) - self.assertEqual(level, TelemetryLevel.minimal) + with patch("codecarbon.core.telemetry.settings.logger.error") as mock_error: + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "bogus"} + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) mock_error.assert_called_once() def test_override_kwarg_takes_precedence_over_config_file(self): - level = resolve_telemetry_level( - {"telemetry_level": "minimal"}, override="disabled" + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, + override="disabled", ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_override_kwarg_takes_precedence_over_external_conf(self): - level = resolve_telemetry_level( + settings = TelemetrySettings.resolve( external_conf={"telemetry_level": "extensive"}, override="disabled", ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_external_conf_env_overrides_file_when_merged(self): - level = resolve_telemetry_level( - {"telemetry_level": "minimal"}, + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, external_conf={"telemetry_level": "disabled"}, ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_env_telemetry_level_via_external_conf(self): with patch.dict( @@ -80,75 +85,87 @@ def test_env_telemetry_level_via_external_conf(self): ): from codecarbon.core.config import get_hierarchical_config - level = resolve_telemetry_level(external_conf=get_hierarchical_config()) - self.assertEqual(level, TelemetryLevel.disabled) + settings = TelemetrySettings.resolve(external_conf=get_hierarchical_config()) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_is_explicit_with_config_file(self): - self.assertTrue(is_telemetry_level_explicit({"telemetry_level": "minimal"})) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"} + ) + self.assertTrue(settings.is_explicit) def test_is_explicit_with_override(self): - self.assertTrue(is_telemetry_level_explicit({}, override="disabled")) + settings = TelemetrySettings.resolve(override="disabled") + self.assertTrue(settings.is_explicit) def test_is_explicit_with_env_telemetry_level(self): - self.assertTrue( - is_telemetry_level_explicit( - {}, external_conf={"telemetry_level": "minimal"} - ) + settings = TelemetrySettings.resolve( + external_conf={"telemetry_level": "minimal"} ) + self.assertTrue(settings.is_explicit) def test_is_not_explicit_when_unset(self): - self.assertFalse(is_telemetry_level_explicit({})) + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertFalse(settings.is_explicit) class TestTelemetryApiSettings(unittest.TestCase): - def test_get_telemetry_api_url_from_conf(self): - url = get_telemetry_api_url({"telemetry_api_url": "http://test.example"}) - self.assertEqual(url, "http://test.example") + def test_api_url_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://test.example"} + ) + self.assertEqual(settings.api_url, "http://test.example") - def test_get_telemetry_api_url_default(self): + def test_api_url_default(self): env = { k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_URL" } with patch.dict(os.environ, env, clear=True): - url = get_telemetry_api_url({}) - self.assertEqual(url, DEFAULT_TELEMETRY_API_URL) + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, DEFAULT_TELEMETRY_API_URL) - def test_get_telemetry_api_url_env_fallback(self): + def test_api_url_env_fallback(self): with patch.dict( os.environ, {"CODECARBON_TELEMETRY_API_URL": "http://env.example"}, clear=False, ): - url = get_telemetry_api_url({}) - self.assertEqual(url, "http://env.example") + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env.example") - def test_get_telemetry_api_key_from_conf(self): - key = get_telemetry_api_key({"telemetry_api_key": "cpt_test"}) - self.assertEqual(key, "cpt_test") + def test_api_key_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_key": "cpt_test"} + ) + self.assertEqual(settings.api_key, "cpt_test") - def test_get_telemetry_api_key_uses_public_default_when_unset(self): + def test_api_key_uses_public_default_when_unset(self): env = { k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_KEY" } with patch.dict(os.environ, env, clear=True): - key = get_telemetry_api_key({}) - self.assertEqual(key, DEFAULT_TELEMETRY_API_KEY) - - def test_get_telemetry_experiment_id_from_conf(self): - experiment_id = get_telemetry_experiment_id( - {"telemetry_experiment_id": "00000000-0000-0000-0000-000000000001"} + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_key, DEFAULT_TELEMETRY_API_KEY) + + def test_experiment_id_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={ + "telemetry_experiment_id": "00000000-0000-0000-0000-000000000001" + } + ) + self.assertEqual( + settings.experiment_id, "00000000-0000-0000-0000-000000000001" ) - self.assertEqual(experiment_id, "00000000-0000-0000-0000-000000000001") - def test_get_telemetry_experiment_id_uses_public_default_when_unset(self): + def test_experiment_id_uses_public_default_when_unset(self): env = { k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_EXPERIMENT_ID" } with patch.dict(os.environ, env, clear=True): - experiment_id = get_telemetry_experiment_id({}) - self.assertEqual(experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + settings = TelemetrySettings.resolve() + self.assertEqual(settings.experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) if __name__ == "__main__": From aefea4c3d8ce0787d7d57e073e15e3cc9f959dd7 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:39:31 +0200 Subject: [PATCH 19/29] refactor: improve telemetry imports and code formatting - Reorganized imports in emissions_tracker.py for clarity and consistency. - Simplified the definition of CONTAINER_RUNTIME_ENV in collect.py for better readability. - Enhanced formatting in settings.py and various test files to maintain consistent code style. - Updated test cases to improve readability and ensure proper mocking of telemetry functions. --- codecarbon/core/telemetry/collect.py | 4 +--- codecarbon/core/telemetry/settings.py | 7 +++--- codecarbon/emissions_tracker.py | 2 +- tests/test_telemetry.py | 8 +++++-- tests/test_telemetry_config.py | 33 ++++++++++++++++++++------- tests/test_telemetry_settings.py | 12 ++++++---- 6 files changed, 44 insertions(+), 22 deletions(-) diff --git a/codecarbon/core/telemetry/collect.py b/codecarbon/core/telemetry/collect.py index 49f5263c3..bbd661525 100644 --- a/codecarbon/core/telemetry/collect.py +++ b/codecarbon/core/telemetry/collect.py @@ -36,9 +36,7 @@ ("CI", "ci"), ) -CONTAINER_RUNTIME_ENV = ( - ("KUBERNETES_SERVICE_HOST", "kubernetes"), -) +CONTAINER_RUNTIME_ENV = (("KUBERNETES_SERVICE_HOST", "kubernetes"),) OUTPUT_METHOD_FIELDS = ( ("save_to_file", "file"), diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py index 8c2c28cb7..a296a9b03 100644 --- a/codecarbon/core/telemetry/settings.py +++ b/codecarbon/core/telemetry/settings.py @@ -88,9 +88,10 @@ def resolve( elif merged.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: raw = merged[TELEMETRY_LEVEL_CONFIG_KEY] source = "external" - elif config_file_conf is not None and config_file_conf.get( - TELEMETRY_LEVEL_CONFIG_KEY - ) is not None: + elif ( + config_file_conf is not None + and config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None + ): raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] source = "file" else: diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 30cb4b571..a26c88df8 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -22,9 +22,9 @@ get_hierarchical_config, normalize_gpu_ids, ) -from codecarbon.core.telemetry import Telemetry, TelemetrySettings from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker +from codecarbon.core.telemetry import Telemetry, TelemetrySettings from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.geography import CloudMetadata, GeoMetadata diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index dd157fb93..78c64359d 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -103,7 +103,9 @@ def test_tier2_uses_api_client(self): tracker._telemetry_override = None emissions = self._emissions() telemetry = Telemetry.from_tracker(tracker) - with patch("codecarbon.core.telemetry.dispatcher.post_private", return_value=True): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ): with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True @@ -186,7 +188,9 @@ def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): with patch( "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index d80ee9ecf..6b230ee83 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -9,7 +9,12 @@ from unittest.mock import MagicMock, patch from codecarbon.core.config import get_config_file_settings -from codecarbon.core.telemetry import Telemetry, TelemetryLevel, TelemetrySettings, post_private +from codecarbon.core.telemetry import ( + Telemetry, + TelemetryLevel, + TelemetrySettings, + post_private, +) from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open @@ -35,7 +40,9 @@ def tearDown(self) -> None: def test_warns_once_when_telemetry_not_explicit(self): settings = TelemetrySettings.resolve(config_file_conf={}) telemetry = Telemetry(settings) - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: telemetry.warn_if_implicit() telemetry.warn_if_implicit() self.assertEqual(mock_warning.call_count, 1) @@ -46,7 +53,9 @@ def test_no_warn_when_config_explicit(self): config_file_conf={"telemetry_level": "disabled"} ) telemetry = Telemetry(settings) - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: telemetry.warn_if_implicit() mock_warning.assert_not_called() @@ -188,7 +197,9 @@ def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup with patch( "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api @@ -231,9 +242,12 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) ) } with patch.dict(os.environ, env_without_telemetry, clear=True): - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: with patch( - "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, ): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) @@ -246,7 +260,9 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): self._mock_config("[codecarbon]\n") - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: with patch("codecarbon.core.telemetry.dispatcher.post_private"): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker( @@ -268,7 +284,8 @@ def test_env_telemetry_disabled_does_not_change_resolved_level( os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False ): with patch( - "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py index e9ad2bb2e..e9a1e09f2 100644 --- a/tests/test_telemetry_settings.py +++ b/tests/test_telemetry_settings.py @@ -47,7 +47,9 @@ def test_telemetry_level_extensive(self): def test_env_telemetry_key_ignored(self): with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): - settings = TelemetrySettings.resolve(config_file_conf={"telemetry": "extensive"}) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry": "extensive"} + ) self.assertEqual(settings.level, TelemetryLevel.minimal) def test_invalid_level_falls_back_to_minimal(self): @@ -85,7 +87,9 @@ def test_env_telemetry_level_via_external_conf(self): ): from codecarbon.core.config import get_hierarchical_config - settings = TelemetrySettings.resolve(external_conf=get_hierarchical_config()) + settings = TelemetrySettings.resolve( + external_conf=get_hierarchical_config() + ) self.assertEqual(settings.level, TelemetryLevel.disabled) def test_is_explicit_with_config_file(self): @@ -153,9 +157,7 @@ def test_experiment_id_from_conf(self): "telemetry_experiment_id": "00000000-0000-0000-0000-000000000001" } ) - self.assertEqual( - settings.experiment_id, "00000000-0000-0000-0000-000000000001" - ) + self.assertEqual(settings.experiment_id, "00000000-0000-0000-0000-000000000001") def test_experiment_id_uses_public_default_when_unset(self): env = { From adb50c3f8c26cf8167570946d8148157a34d155d Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:41:10 +0200 Subject: [PATCH 20/29] feat: introduce telemetry module and enhance telemetry context management - Added a new telemetry module to handle product telemetry at tracker stop, including necessary imports and payload construction. - Updated tests to ensure proper handling of telemetry levels and error management with the Typer library. - Improved clarity and organization of telemetry-related code for better maintainability. --- codecarbon/core/telemetry/__init__.py | 39 +++++++++++++++++++++++++++ tests/cli/test_telemetry_cli.py | 3 ++- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 codecarbon/core/telemetry/__init__.py diff --git a/codecarbon/core/telemetry/__init__.py b/codecarbon/core/telemetry/__init__.py new file mode 100644 index 000000000..0afb851b9 --- /dev/null +++ b/codecarbon/core/telemetry/__init__.py @@ -0,0 +1,39 @@ +"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.dispatcher import Telemetry +from codecarbon.core.telemetry.schemas import ( + PRIVATE_TELEMETRY_FIELDS, + TelemetryCreate, + TelemetryLevel, +) +from codecarbon.core.telemetry.settings import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + DEFAULT_TELEMETRY_LEVEL, + TELEMETRY_LEVEL_CONFIG_KEY, + TelemetryLevelSource, + TelemetrySettings, + parse_telemetry_level, +) + +__all__ = [ + "DEFAULT_TELEMETRY_API_KEY", + "DEFAULT_TELEMETRY_API_URL", + "DEFAULT_TELEMETRY_EXPERIMENT_ID", + "DEFAULT_TELEMETRY_LEVEL", + "PRIVATE_TELEMETRY_FIELDS", + "TELEMETRY_LEVEL_CONFIG_KEY", + "Telemetry", + "TelemetryContext", + "TelemetryCreate", + "TelemetryLevel", + "TelemetryLevelSource", + "TelemetrySettings", + "build_payload", + "parse_telemetry_level", + "post_private", + "post_public_summary", +] diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py index 871bd13c3..88cd9375b 100644 --- a/tests/cli/test_telemetry_cli.py +++ b/tests/cli/test_telemetry_cli.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest +import typer from typer.testing import CliRunner from codecarbon.cli import main as cli_main @@ -17,7 +18,7 @@ def test_normalize_telemetry_level_accepts_valid_values(): def test_normalize_telemetry_level_rejects_invalid(): - with pytest.raises(Exception): + with pytest.raises(typer.BadParameter): normalize_telemetry_level("bogus") From 74eed941a8362374ee989c300af3d984c73fa517 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Mon, 15 Jun 2026 19:47:48 +0200 Subject: [PATCH 21/29] Align SDK telemetry with server schema and simplify collection Match minimal/extensive tier contracts to carbonserver DB fields, split payload building by tier, and trim redundant tracker state and docstrings after ponytail pass. Update docs and tests to drop Tier 1/2 jargon. Co-authored-by: Cursor --- .../carbonserver/api/schemas_telemetry.py | 61 ++- .../tests/api/routers/test_telemetry.py | 31 +- codecarbon/cli/telemetry_cli.py | 12 +- codecarbon/core/telemetry/__init__.py | 4 +- codecarbon/core/telemetry/client.py | 21 +- codecarbon/core/telemetry/collect.py | 387 +++++++++--------- codecarbon/core/telemetry/dispatcher.py | 46 +-- codecarbon/core/telemetry/schemas.py | 66 ++- codecarbon/core/telemetry/settings.py | 45 +- codecarbon/emissions_tracker.py | 10 +- docs/how-to/configuration.md | 2 +- docs/how-to/telemetry.md | 24 +- docs/reference/api.md | 4 +- docs/reference/cli.md | 4 +- tests/cli/test_telemetry_cli.py | 8 +- tests/test_telemetry.py | 4 +- tests/test_telemetry_collect.py | 90 ++-- tests/test_telemetry_config.py | 8 +- 18 files changed, 425 insertions(+), 402 deletions(-) diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index ced8c13fd..5517c2ff4 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -41,7 +41,8 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None - on_cloud: Optional[bool] = None + longitude: Optional[float] = Field(default=None, ge=-180, le=180) + latitude: Optional[float] = Field(default=None, ge=-90, le=90) cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -57,10 +58,12 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None + python_executable_hash: Optional[str] = Field( + default=None, min_length=64, max_length=64 + ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None - python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -78,18 +81,21 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None + decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) - integration_surface: Optional[str] = None - offline_mode: Optional[bool] = None - save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None + first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) + tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) + errors_encountered: Optional[List[str]] = None + warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None + python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -110,15 +116,58 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None + host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") + + if self.telemetry_level == TelemetryLevel.minimal: + extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS + submitted_extensive_fields = [ + field + for field in extensive_fields + if getattr(self, field) not in (None, [], {}) + ] + if submitted_extensive_fields: + fields = ", ".join(sorted(submitted_extensive_fields)) + raise ValueError( + f"Minimal telemetry cannot include extensive fields: {fields}" + ) + return self -PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) +MINIMAL_TELEMETRY_FIELDS = { + "timestamp", + "telemetry_level", + "os", + "country_name", + "country_iso_code", + "region", + "cloud_provider", + "cloud_region", + "longitude", + "latitude", + "cpu_count", + "cpu_physical_count", + "cpu_model", + "cpu_architecture", + "gpu_count", + "gpu_model", + "gpu_driver_version", + "gpu_memory_total_gb", + "ram_total_size_gb", + "cuda_version", + "cudnn_version", + "python_version", + "python_implementation", + "python_executable_hash", + "python_env_type", + "codecarbon_version", + "codecarbon_install_method", +} class TelemetryCreate(TelemetryBase): diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index a51a19ecd..a405b59e8 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -14,7 +14,7 @@ TELEMETRY_ID = "f52fe339-164d-4c2b-a8c0-f562dfce066d" -SAMPLE_PRIVATE_TELEMETRY = { +MINIMAL_TELEMETRY_TO_CREATE = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", "os": "Linux-5.10.0-x86_64", @@ -46,36 +46,21 @@ def test_add_telemetry(client, custom_test_server): repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=SAMPLE_PRIVATE_TELEMETRY) + response = client.post("/telemetry", json=MINIMAL_TELEMETRY_TO_CREATE) assert response.status_code == status.HTTP_201_CREATED assert response.json() == TELEMETRY_ID -def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server): +def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) - repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) - telemetry_with_framework_version = { - **SAMPLE_PRIVATE_TELEMETRY, - "torch_version": "2.2.0", - } - - with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_framework_version) - - assert response.status_code == status.HTTP_201_CREATED - repository_mock.add_telemetry.assert_called_once() - - -def test_telemetry_rejects_unknown_fields(client, custom_test_server): - repository_mock = mock.Mock(spec=TelemetryRepository) - telemetry_with_unknown_field = { - **SAMPLE_PRIVATE_TELEMETRY, - "unknown_field": "value", + telemetry_with_extensive_field = { + **MINIMAL_TELEMETRY_TO_CREATE, + "total_emissions_kg": 0.42, } with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_unknown_field) + response = client.post("/telemetry", json=telemetry_with_extensive_field) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() @@ -84,7 +69,7 @@ def test_telemetry_rejects_unknown_fields(client, custom_test_server): def test_disabled_telemetry_is_rejected(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) disabled_telemetry = { - **SAMPLE_PRIVATE_TELEMETRY, + **MINIMAL_TELEMETRY_TO_CREATE, "telemetry_level": "disabled", } diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py index df23a30a5..279ab1e63 100644 --- a/codecarbon/cli/telemetry_cli.py +++ b/codecarbon/cli/telemetry_cli.py @@ -28,8 +28,8 @@ TIER_DESCRIPTIONS = { "disabled": "No product telemetry.", - "minimal": "Tier 1: private telemetry per run at stop (environment, usage, frameworks, emissions).", - "extensive": "Tier 2: Tier 1 plus run emissions summary to the shared telemetry experiment (ApiClient).", + "minimal": "Environment and hardware only (private POST /telemetry).", + "extensive": "Minimal fields plus run metrics and public run summary.", } @@ -154,7 +154,7 @@ def print_telemetry_status(config_path: Optional[Path] = None) -> None: print(f"Explicitly configured: {explicit}") if not explicit: print( - "[yellow]Tier 1 minimal telemetry will be sent once per session " + "[yellow]Minimal telemetry will be sent on each tracker stop " "until you set telemetry_level.[/yellow]" ) @@ -217,8 +217,8 @@ def telemetry_entry( run_telemetry_interactive(config=config) -@telemetry_app.command("show") -def show( +@telemetry_app.command("status") +def status( config: Annotated[ Optional[Path], typer.Option( @@ -227,7 +227,7 @@ def show( ), ] = None, ) -> None: - """Show the resolved telemetry tier.""" + """Print resolved telemetry tier and configuration source.""" print_telemetry_status(config_path=config) diff --git a/codecarbon/core/telemetry/__init__.py b/codecarbon/core/telemetry/__init__.py index 0afb851b9..efa31dbaa 100644 --- a/codecarbon/core/telemetry/__init__.py +++ b/codecarbon/core/telemetry/__init__.py @@ -4,7 +4,7 @@ from codecarbon.core.telemetry.collect import TelemetryContext, build_payload from codecarbon.core.telemetry.dispatcher import Telemetry from codecarbon.core.telemetry.schemas import ( - PRIVATE_TELEMETRY_FIELDS, + MINIMAL_TELEMETRY_FIELDS, TelemetryCreate, TelemetryLevel, ) @@ -24,7 +24,7 @@ "DEFAULT_TELEMETRY_API_URL", "DEFAULT_TELEMETRY_EXPERIMENT_ID", "DEFAULT_TELEMETRY_LEVEL", - "PRIVATE_TELEMETRY_FIELDS", + "MINIMAL_TELEMETRY_FIELDS", "TELEMETRY_LEVEL_CONFIG_KEY", "Telemetry", "TelemetryContext", diff --git a/codecarbon/core/telemetry/client.py b/codecarbon/core/telemetry/client.py index 99a5a7ed2..984dd5506 100644 --- a/codecarbon/core/telemetry/client.py +++ b/codecarbon/core/telemetry/client.py @@ -14,15 +14,7 @@ def post_private(settings: TelemetrySettings, payload: dict) -> bool: - """POST a private telemetry payload to ``/telemetry``. - - Args: - settings: Resolved telemetry API settings. - payload: Telemetry fields dict. - - Returns: - True if the server accepted the payload (HTTP 201). - """ + """POST private telemetry to ``/telemetry``; return True on HTTP 201.""" headers = {"Content-Type": "application/json"} if settings.api_key: headers["x-api-token"] = settings.api_key @@ -60,16 +52,7 @@ def post_public_summary( conf: dict, emissions: EmissionsData, ) -> bool: - """Send Tier 2 public run summary via ``ApiClient``. - - Args: - settings: Resolved telemetry API settings. - conf: Tracker configuration dict. - emissions: Run emissions data. - - Returns: - True if the run summary was posted successfully. - """ + """Post public run summary via ``ApiClient`` (extensive tier only).""" try: api = ApiClient( endpoint_url=settings.api_url, diff --git a/codecarbon/core/telemetry/collect.py b/codecarbon/core/telemetry/collect.py index bbd661525..2eb9a23ac 100644 --- a/codecarbon/core/telemetry/collect.py +++ b/codecarbon/core/telemetry/collect.py @@ -1,4 +1,4 @@ -"""Collect private product telemetry (Tier 1 / Tier 2) from tracker state.""" +"""Collect private product telemetry from tracker state.""" from __future__ import annotations @@ -8,27 +8,29 @@ import sys from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Optional +from typing import Any from codecarbon.core.cloud import get_env_cloud_details from codecarbon.core.gpu import is_nvidia_system -from codecarbon.core.telemetry.schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.output_methods.base_output import OutputMethod from codecarbon.output_methods.emissions_data import EmissionsData FRAMEWORK_PACKAGES = ( ("torch", "has_torch"), ("transformers", "has_transformers"), ("diffusers", "has_diffusers"), - ("sklearn", "has_sklearn"), ) -PACKAGE_MANAGER_ENV = ( - ("UV", "uv"), - ("POETRY_ACTIVE", "poetry"), - ("PIP_RUN", "pip"), -) +OUTPUT_METHOD_LABELS = { + OutputMethod.CSV: "file", + OutputMethod.API: "api", + OutputMethod.LOGGER: "logger", + OutputMethod.PROMETHEUS: "prometheus", + OutputMethod.LOGFIRE: "logfire", +} -CI_ENVIRONMENTS = ( +CI_ENV_VAR_LABELS = ( ("GITHUB_ACTIONS", "github_actions"), ("GITLAB_CI", "gitlab_ci"), ("CIRCLECI", "circleci"), @@ -36,15 +38,10 @@ ("CI", "ci"), ) -CONTAINER_RUNTIME_ENV = (("KUBERNETES_SERVICE_HOST", "kubernetes"),) - -OUTPUT_METHOD_FIELDS = ( - ("save_to_file", "file"), - ("save_to_api", "api"), - ("save_to_logger", "logger"), - ("emissions_endpoint", "http"), - ("save_to_prometheus", "prometheus"), - ("save_to_logfire", "logfire"), +PACKAGE_MANAGER_ENV = ( + ("UV", "uv"), + ("POETRY_ACTIVE", "poetry"), + ("PIP_RUN", "pip"), ) @@ -56,63 +53,101 @@ class TelemetryContext: emissions: EmissionsData hardware: list[Any] resource_tracker: Any - save_to_api: bool - save_to_file: bool - save_to_logger: bool - save_to_prometheus: bool - save_to_logfire: bool - emissions_endpoint: str | None + output_methods: list[str] tasks: dict[str, Any] measure_power_secs: float | None - is_offline: bool + integration: str @classmethod def from_tracker(cls, tracker: Any, emissions: EmissionsData) -> TelemetryContext: - """Build a context snapshot from an active emissions tracker. - - Args: - tracker: Active emissions tracker instance. - emissions: Run emissions data. - - Returns: - Context for ``build_payload``. - """ + """Build a context snapshot from an active emissions tracker.""" from codecarbon.emissions_tracker import OfflineEmissionsTracker + methods = [ + OUTPUT_METHOD_LABELS[method] + for method in getattr(tracker, "_output_methods", None) or [] + if method in OUTPUT_METHOD_LABELS + ] + if getattr(tracker, "_emissions_endpoint", None): + methods.append("http") + + argv = " ".join(sys.argv) + if isinstance(tracker, OfflineEmissionsTracker): + integration = "offline_tracker" + elif "codecarbon" in argv and "monitor" in argv: + integration = "cli_monitor" + else: + integration = "library" + return cls( conf=getattr(tracker, "_conf", {}), emissions=emissions, hardware=getattr(tracker, "_hardware", []) or [], resource_tracker=getattr(tracker, "_resource_tracker", None), - save_to_api=bool(getattr(tracker, "_save_to_api", False)), - save_to_file=bool(getattr(tracker, "_save_to_file", False)), - save_to_logger=bool(getattr(tracker, "_save_to_logger", False)), - save_to_prometheus=bool(getattr(tracker, "_save_to_prometheus", False)), - save_to_logfire=bool(getattr(tracker, "_save_to_logfire", False)), - emissions_endpoint=getattr(tracker, "_emissions_endpoint", None), + output_methods=methods, tasks=getattr(tracker, "_tasks", {}) or {}, measure_power_secs=getattr(tracker, "_measure_power_secs", None), - is_offline=isinstance(tracker, OfflineEmissionsTracker), + integration=integration, ) -def _non_empty(value: Any) -> bool: - return value not in (None, "", [], {}) - - -def _strip_none(data: dict[str, Any]) -> dict[str, Any]: - return {key: value for key, value in data.items() if _non_empty(value)} +def _strip_empty(data: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in data.items() + if value not in (None, "", [], {}) + } -def _first_env_match(mapping: tuple[tuple[str, str], ...]) -> Optional[str]: - return next((label for var, label in mapping if os.environ.get(var)), None) +def _env_label(env_vars: tuple[tuple[str, str], ...]) -> str | None: + return next((label for var, label in env_vars if os.environ.get(var)), None) def _package_installed(name: str) -> bool: return importlib.util.find_spec(name) is not None -def _detect_codecarbon_install_method() -> Optional[str]: +def _round_coordinate(value: Any) -> float | None: + if value is None: + return None + return round(float(value), 1) + + +def _cloud_region(emissions: EmissionsData) -> tuple[str | None, str | None, str | None]: + details = get_env_cloud_details() + raw_provider = raw_region = None + if details and details.get("metadata"): + provider = (details.get("provider") or "").lower() or None + metadata = details.get("metadata") or {} + if provider == "aws": + raw_region = metadata.get("region") + elif provider == "azure": + raw_region = (metadata.get("compute") or {}).get("location") + elif provider == "gcp": + zone = metadata.get("zone") or "" + parts = zone.split("/") + raw_region = parts[-1].rsplit("-", 1)[0] if parts else None + raw_provider = provider + + cloud_provider = emissions.cloud_provider or raw_provider + cloud_region = emissions.cloud_region or raw_region + region = emissions.region + if emissions.on_cloud == "Y" and cloud_region: + region = region or cloud_region + return cloud_provider, cloud_region, region + + +def _detect_python_env_type() -> str | None: + if os.environ.get("CONDA_DEFAULT_ENV"): + return "conda" + if os.environ.get("VIRTUAL_ENV"): + return "venv" + if sys.prefix != getattr(sys, "base_prefix", sys.prefix): + return "venv" + return "system" + + +def _detect_codecarbon_install_method() -> str | None: try: from importlib.metadata import distribution @@ -129,39 +164,28 @@ def _detect_codecarbon_install_method() -> Optional[str]: return None -def _detect_python_env_type() -> Optional[str]: - if os.environ.get("CONDA_DEFAULT_ENV"): - return "conda" - if os.environ.get("VIRTUAL_ENV"): - return "venv" - if sys.prefix != getattr(sys, "base_prefix", sys.prefix): - return "venv" - return "system" - - -def _detect_notebook_environment() -> Optional[str]: +def _detect_notebook_environment() -> str | None: if os.environ.get("COLAB_GPU") is not None or "google.colab" in sys.modules: return "colab" try: from IPython import get_ipython - shell = get_ipython().__class__.__name__ - if "ZMQInteractiveShell" in shell: + if "ZMQInteractiveShell" in get_ipython().__class__.__name__: return "jupyter" except Exception: pass return None -def _container_info() -> tuple[bool, Optional[str]]: +def _container_info() -> tuple[bool | None, str | None]: if os.environ.get("KUBERNETES_SERVICE_HOST"): return True, "kubernetes" if os.path.exists("/.dockerenv"): return True, "docker" - return False, None + return None, None -def _detect_ide() -> Optional[str]: +def _detect_ide() -> str | None: if os.environ.get("CURSOR_TRACE_ID") or os.environ.get("CURSOR_SESSION"): return "cursor" if os.environ.get("VSCODE_PID") or os.environ.get("TERM_PROGRAM") == "vscode": @@ -171,7 +195,7 @@ def _detect_ide() -> Optional[str]: return None -def _cudnn_version() -> Optional[str]: +def _cudnn_version() -> str | None: if not _package_installed("torch"): return None try: @@ -183,7 +207,28 @@ def _cudnn_version() -> Optional[str]: return None -def _collect_hardware_diagnostics(ctx: TelemetryContext) -> dict[str, Any]: +def _gpu_static_fields() -> dict[str, Any]: + if not is_nvidia_system(): + return {} + try: + import pynvml + + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + mem = pynvml.nvmlDeviceGetMemoryInfo(handle) + cuda_version = pynvml.nvmlSystemGetCudaDriverVersion_v2() + if isinstance(cuda_version, int): + cuda_version = f"{cuda_version // 1000}.{(cuda_version % 1000) // 10}" + return { + "gpu_memory_total_gb": mem.total / (1024**3), + "gpu_driver_version": pynvml.nvmlSystemGetDriverVersion(), + "cuda_version": cuda_version, + } + except Exception: + return {} + + +def _hardware_diagnostics(ctx: TelemetryContext) -> dict[str, Any]: from codecarbon.core import cpu hardware_tracked: list[str] = [] @@ -192,135 +237,40 @@ def _collect_hardware_diagnostics(ctx: TelemetryContext) -> dict[str, Any]: hardware_tracked.append(item.description()) except Exception: pass - - gpu_detection_method: Optional[str] = None + gpu_detection_method = None if ctx.resource_tracker is not None: gpu_tracker = getattr(ctx.resource_tracker, "gpu_tracker", None) if gpu_tracker and gpu_tracker != "Unspecified": gpu_detection_method = gpu_tracker - rapl_available: Optional[bool] = None - if platform.system() == "Linux": - rapl_available = cpu.is_rapl_available() - + rapl_available = cpu.is_rapl_available() if platform.system() == "Linux" else None return { "hardware_tracked": hardware_tracked or None, "hardware_detection_success": bool(hardware_tracked), "rapl_available": rapl_available, "gpu_detection_method": gpu_detection_method, - "api_mode": "online" if ctx.save_to_api else "offline", + "api_mode": "online" if "api" in ctx.output_methods else "offline", } -def _detect_integration_surface(ctx: TelemetryContext) -> str: - if ctx.is_offline: - return "offline_tracker" - argv = " ".join(sys.argv) - if "codecarbon" in argv and "monitor" in argv: - return "cli_monitor" - return "library" - - -def _collect_output_methods(ctx: TelemetryContext) -> list[str]: - methods: list[str] = [] - for field_name, label in OUTPUT_METHOD_FIELDS: - value = getattr(ctx, field_name) - if field_name == "emissions_endpoint": - if value: - methods.append(label) - elif value: - methods.append(label) - return methods - - -def _raw_cloud_provider_and_region() -> tuple[Optional[str], Optional[str]]: - details = get_env_cloud_details() - if not details or not details.get("metadata"): - return None, None - provider = (details.get("provider") or "").lower() or None - metadata = details.get("metadata") or {} - region: Optional[str] = None - if provider == "aws": - region = metadata.get("region") - elif provider == "azure": - region = (metadata.get("compute") or {}).get("location") - elif provider == "gcp": - zone = metadata.get("zone") or "" - parts = zone.split("/") - region = parts[-1].rsplit("-", 1)[0] if parts else None - return provider, region - - -def _collect_framework_fields() -> dict[str, Any]: - return { - has_field: _package_installed(package) - for package, has_field in FRAMEWORK_PACKAGES - } - - -def _gpu_static_fields() -> dict[str, Any]: - fields: dict[str, Any] = {} - if not is_nvidia_system(): - return fields - try: - import pynvml - - pynvml.nvmlInit() - handle = pynvml.nvmlDeviceGetHandleByIndex(0) - mem = pynvml.nvmlDeviceGetMemoryInfo(handle) - fields["gpu_memory_total_gb"] = mem.total / (1024**3) - fields["gpu_driver_version"] = pynvml.nvmlSystemGetDriverVersion() - fields["cuda_version"] = pynvml.nvmlSystemGetCudaDriverVersion_v2() - if isinstance(fields["cuda_version"], int): - v = fields["cuda_version"] - fields["cuda_version"] = f"{v // 1000}.{(v % 1000) // 10}" - except Exception: - pass - return fields - - -def build_payload( - ctx: TelemetryContext, - level: TelemetryLevel = TelemetryLevel.minimal, -) -> dict[str, Any]: - """Build a private telemetry payload dict for ``POST /telemetry``. - - Args: - ctx: Tracker snapshot from ``TelemetryContext.from_tracker``. - level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). - - Returns: - Payload dict for ``TelemetryCreate``. - """ - emissions = ctx.emissions +def _minimal_payload(ctx: TelemetryContext, level: TelemetryLevel) -> dict[str, Any]: + """Fields allowed for ``telemetry_level=minimal`` (matches DB + MINIMAL_TELEMETRY_FIELDS).""" conf = ctx.conf - raw_provider, raw_region = _raw_cloud_provider_and_region() - on_cloud = emissions.on_cloud == "Y" - cloud_provider = emissions.cloud_provider or raw_provider - cloud_region = emissions.cloud_region or raw_region - region = emissions.region or conf.get("region") - if on_cloud and cloud_region: - region = region or cloud_region - - integration_surface = _detect_integration_surface(ctx) - in_container, container_runtime = _container_info() - gpu_fields = _gpu_static_fields() + emissions = ctx.emissions + cloud_provider, cloud_region, region = _cloud_region(emissions) + region = region or conf.get("region") - raw: dict[str, Any] = { + payload = { "timestamp": datetime.now(timezone.utc), + "telemetry_level": level.value, "os": conf.get("os") or platform.platform(), - "python_version": conf.get("python_version") or platform.python_version(), - "python_implementation": platform.python_implementation(), - "python_env_type": _detect_python_env_type(), - "python_package_manager": _first_env_match(PACKAGE_MANAGER_ENV), - "codecarbon_version": conf.get("codecarbon_version"), - "codecarbon_install_method": _detect_codecarbon_install_method(), "country_name": emissions.country_name, "country_iso_code": emissions.country_iso_code, "region": region, "cloud_provider": cloud_provider, "cloud_region": cloud_region, - "on_cloud": on_cloud, + "longitude": _round_coordinate(conf.get("longitude", emissions.longitude)), + "latitude": _round_coordinate(conf.get("latitude", emissions.latitude)), "cpu_count": conf.get("cpu_count"), "cpu_physical_count": conf.get("cpu_physical_count"), "cpu_model": conf.get("cpu_model"), @@ -328,34 +278,63 @@ def build_payload( "gpu_count": conf.get("gpu_count"), "gpu_model": conf.get("gpu_model"), "ram_total_size_gb": conf.get("ram_total_size"), - "tracking_mode": conf.get("tracking_mode"), - "integration_surface": integration_surface, - "offline_mode": integration_surface == "offline_tracker", - "output_methods": _collect_output_methods(ctx), - "save_to_api_enabled": ctx.save_to_api, - "task_tracking_used": bool(ctx.tasks), - "measure_power_interval_secs": ctx.measure_power_secs, - "in_container": in_container, - "container_runtime": container_runtime, - "ci_environment": _first_env_match(CI_ENVIRONMENTS), - "notebook_environment": _detect_notebook_environment(), - "ide_used": _detect_ide(), + "python_version": conf.get("python_version") or platform.python_version(), + "python_implementation": platform.python_implementation(), + "python_env_type": _detect_python_env_type(), + "codecarbon_version": conf.get("codecarbon_version"), + "codecarbon_install_method": _detect_codecarbon_install_method(), "cudnn_version": _cudnn_version(), - "duration_seconds": float(emissions.duration) if emissions.duration else None, - "total_emissions_kg": emissions.emissions, - "emissions_rate_kg_per_sec": emissions.emissions_rate, - "energy_consumed_kwh": emissions.energy_consumed, - "cpu_energy_kwh": emissions.cpu_energy, - "gpu_energy_kwh": emissions.gpu_energy, - "ram_energy_kwh": emissions.ram_energy, - "cpu_utilization_avg": emissions.cpu_utilization_percent, - "gpu_utilization_avg": emissions.gpu_utilization_percent, - "ram_utilization_avg": emissions.ram_utilization_percent, - "telemetry_level": level.value, - **_collect_framework_fields(), - **_collect_hardware_diagnostics(ctx), - **gpu_fields, + **_gpu_static_fields(), + } + return _strip_empty(payload) + + +def _extensive_payload(ctx: TelemetryContext) -> dict[str, Any]: + """Extra fields stored only for ``telemetry_level=extensive``.""" + emissions = ctx.emissions + in_container, container_runtime = _container_info() + framework_fields = { + has_field: _package_installed(package) + for package, has_field in FRAMEWORK_PACKAGES } - payload = {key: raw[key] for key in PRIVATE_TELEMETRY_FIELDS if key in raw} - return _strip_none(payload) + return _strip_empty( + { + "tracking_mode": ctx.conf.get("tracking_mode"), + "decorator_vs_context": ctx.integration, + "output_methods": ctx.output_methods or None, + "task_tracking_used": bool(ctx.tasks), + "measure_power_interval_secs": ctx.measure_power_secs, + "python_package_manager": _env_label(PACKAGE_MANAGER_ENV), + "in_container": in_container, + "container_runtime": container_runtime, + "ci_environment": _env_label(CI_ENV_VAR_LABELS), + "notebook_environment": _detect_notebook_environment(), + "ide_used": _detect_ide(), + "duration_seconds": float(emissions.duration) + if emissions.duration + else None, + "total_emissions_kg": emissions.emissions, + "emissions_rate_kg_per_sec": emissions.emissions_rate, + "energy_consumed_kwh": emissions.energy_consumed, + "cpu_energy_kwh": emissions.cpu_energy, + "gpu_energy_kwh": emissions.gpu_energy, + "ram_energy_kwh": emissions.ram_energy, + "cpu_utilization_avg": emissions.cpu_utilization_percent, + "gpu_utilization_avg": emissions.gpu_utilization_percent, + "ram_utilization_avg": emissions.ram_utilization_percent, + **framework_fields, + **_hardware_diagnostics(ctx), + } + ) + + +def build_payload( + ctx: TelemetryContext, + level: TelemetryLevel = TelemetryLevel.minimal, +) -> dict[str, Any]: + """Build a validated telemetry payload dict for ``POST /telemetry``.""" + payload = _minimal_payload(ctx, level) + if level == TelemetryLevel.extensive: + payload.update(_extensive_payload(ctx)) + return payload diff --git a/codecarbon/core/telemetry/dispatcher.py b/codecarbon/core/telemetry/dispatcher.py index 24174d94f..8692d8bd1 100644 --- a/codecarbon/core/telemetry/dispatcher.py +++ b/codecarbon/core/telemetry/dispatcher.py @@ -12,11 +12,9 @@ from codecarbon.output_methods.emissions_data import EmissionsData TELEMETRY_NOT_CONFIGURED_MESSAGE = ( - "CodeCarbon telemetry_level was not set explicitly; using default %r. " - "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " - "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " - "to EmissionsTracker / OfflineEmissionsTracker, or run " - "codecarbon telemetry set ." + "telemetry_level not set explicitly; default %r. Minimal telemetry sends on each " + "stop. Set telemetry_level in .codecarbon.config, CODECARBON_TELEMETRY_LEVEL, " + "EmissionsTracker(telemetry_level=...), or: codecarbon telemetry set ." ) @@ -30,15 +28,7 @@ def __init__(self, settings: TelemetrySettings) -> None: @classmethod def from_tracker(cls, tracker: Any) -> Telemetry: - """Build a dispatcher from tracker config state. - - Args: - tracker: Active emissions tracker with ``_config_file_conf``, - ``_external_conf``, and optional ``_telemetry_override``. - - Returns: - Configured ``Telemetry`` instance. - """ + """Build a dispatcher from tracker config state.""" return cls( TelemetrySettings.resolve( config_file_conf=tracker._config_file_conf, @@ -58,15 +48,7 @@ def warn_if_implicit(self) -> None: Telemetry._default_warning_shown = True def send_at_stop(self, tracker: Any, emissions: EmissionsData) -> None: - """Send product telemetry for the resolved tier at tracker ``stop()``. - - Tier 1 (``minimal``): private ``POST /telemetry`` only. - Tier 2 (``extensive``): Tier 1 plus public run summary. - - Args: - tracker: Active emissions tracker instance. - emissions: Total emissions from ``_prepare_emissions_data()``. - """ + """Send product telemetry at tracker ``stop()`` for the resolved tier.""" if self.settings.level == TelemetryLevel.disabled: return if emissions.duration is not None and emissions.duration < 1: @@ -74,16 +56,10 @@ def send_at_stop(self, tracker: Any, emissions: EmissionsData) -> None: return ctx = TelemetryContext.from_tracker(tracker, emissions) payload = build_payload(ctx, level=self.settings.level) - try: - post_private(self.settings, payload) - except Exception as error: - logger.error(f"Private telemetry failed (non-critical): {error}") + post_private(self.settings, payload) if self.settings.level == TelemetryLevel.extensive: - try: - post_public_summary( - self.settings, - getattr(tracker, "_conf", {}), - emissions, - ) - except Exception as error: - logger.error(f"Public run summary failed (non-critical): {error}") + post_public_summary( + self.settings, + getattr(tracker, "_conf", {}), + emissions, + ) diff --git a/codecarbon/core/telemetry/schemas.py b/codecarbon/core/telemetry/schemas.py index bc6ac2dc2..81198ddc1 100644 --- a/codecarbon/core/telemetry/schemas.py +++ b/codecarbon/core/telemetry/schemas.py @@ -1,3 +1,8 @@ +"""Telemetry payload schemas aligned with carbonserver ``telemetry_sql_models.Telemetry``. + +ponytail: mirrored in ``carbonserver/api/schemas_telemetry.py`` for drift tests only. +""" + from datetime import datetime from enum import Enum from typing import List, Optional @@ -23,7 +28,8 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None - on_cloud: Optional[bool] = None + longitude: Optional[float] = Field(default=None, ge=-180, le=180) + latitude: Optional[float] = Field(default=None, ge=-90, le=90) cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -39,10 +45,12 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None + python_executable_hash: Optional[str] = Field( + default=None, min_length=64, max_length=64 + ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None - python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -60,18 +68,21 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None + decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) - integration_surface: Optional[str] = None - offline_mode: Optional[bool] = None - save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None + first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) + tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) + errors_encountered: Optional[List[str]] = None + warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None + python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -92,15 +103,58 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None + host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") + + if self.telemetry_level == TelemetryLevel.minimal: + extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS + submitted_extensive_fields = [ + field + for field in extensive_fields + if getattr(self, field) not in (None, [], {}) + ] + if submitted_extensive_fields: + fields = ", ".join(sorted(submitted_extensive_fields)) + raise ValueError( + f"Minimal telemetry cannot include extensive fields: {fields}" + ) + return self -PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) +MINIMAL_TELEMETRY_FIELDS = { + "timestamp", + "telemetry_level", + "os", + "country_name", + "country_iso_code", + "region", + "cloud_provider", + "cloud_region", + "longitude", + "latitude", + "cpu_count", + "cpu_physical_count", + "cpu_model", + "cpu_architecture", + "gpu_count", + "gpu_model", + "gpu_driver_version", + "gpu_memory_total_gb", + "ram_total_size_gb", + "cuda_version", + "cudnn_version", + "python_version", + "python_implementation", + "python_executable_hash", + "python_env_type", + "codecarbon_version", + "codecarbon_install_method", +} class TelemetryCreate(TelemetryBase): diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py index a296a9b03..ca0ee22b9 100644 --- a/codecarbon/core/telemetry/settings.py +++ b/codecarbon/core/telemetry/settings.py @@ -20,17 +20,7 @@ def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: - """Parse a telemetry tier from a string or enum value. - - Args: - raw: Tier name or ``TelemetryLevel`` member. - - Returns: - Parsed ``TelemetryLevel``. - - Raises: - ValueError: If ``raw`` is not a valid tier name. - """ + """Parse a telemetry tier name or enum member.""" if isinstance(raw, TelemetryLevel): return raw try: @@ -64,24 +54,11 @@ def resolve( external_conf: dict[str, Any] | None = None, override: str | TelemetryLevel | None = None, ) -> TelemetrySettings: - """Resolve telemetry tier and API settings. - - Precedence for tier: - - 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI - 2. ``external_conf`` — merged ``.codecarbon.config`` and env - 3. ``config_file_conf`` — file-only settings - 4. Default: ``minimal`` - - Args: - config_file_conf: Settings from ``get_config_file_settings()``. - external_conf: Merged settings from ``get_hierarchical_config()``. - override: Optional tier from tracker or CLI. - - Returns: - Resolved settings bundle. - """ + """Resolve tier (override > external > file > default minimal) and API settings.""" merged = external_conf or {} + api_url = cls._resolve_api_url(merged) + api_key = cls._resolve_api_key(merged) + experiment_id = cls._resolve_experiment_id(merged) if override is not None: raw = override source: TelemetryLevelSource = "override" @@ -98,9 +75,9 @@ def resolve( return cls( level=DEFAULT_TELEMETRY_LEVEL, source="default", - api_url=cls._resolve_api_url(merged), - api_key=cls._resolve_api_key(merged), - experiment_id=cls._resolve_experiment_id(merged), + api_url=api_url, + api_key=api_key, + experiment_id=experiment_id, ) try: level = parse_telemetry_level(raw) @@ -114,9 +91,9 @@ def resolve( return cls( level=level, source=source, - api_url=cls._resolve_api_url(merged), - api_key=cls._resolve_api_key(merged), - experiment_id=cls._resolve_experiment_id(merged), + api_url=api_url, + api_key=api_key, + experiment_id=experiment_id, ) @staticmethod diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 6ad4bf9b9..5e8015023 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -25,7 +25,7 @@ ) from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker -from codecarbon.core.telemetry import Telemetry, TelemetrySettings +from codecarbon.core.telemetry import Telemetry from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.geography import CloudMetadata, GeoMetadata @@ -511,13 +511,7 @@ def __init__( self._telemetry_override = ( None if telemetry_level is _sentinel else telemetry_level ) - self._telemetry = Telemetry( - TelemetrySettings.resolve( - config_file_conf=self._config_file_conf, - external_conf=self._external_conf, - override=self._telemetry_override, - ) - ) + self._telemetry = Telemetry.from_tracker(self) self._set_from_conf( force_carbon_intensity_g_co2e_kwh, "force_carbon_intensity_g_co2e_kwh", diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index b6c093037..c649bd540 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -148,6 +148,6 @@ documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) ## Product telemetry -Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends private product telemetry once per run at `stop()`—see [Product telemetry](telemetry.md). +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). `minimal` sends private product telemetry at each `stop()`—see [Product telemetry](telemetry.md). See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index b5bc56057..e0246a735 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -24,7 +24,7 @@ Tier is resolved in this order: 1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` 2. **Config + environment** — `telemetry_level` in `.codecarbon.config`, then `CODECARBON_TELEMETRY_LEVEL` when both are set -3. **Default:** `minimal` (Tier 1) +3. **Default:** `minimal` ## Lifecycle @@ -35,18 +35,20 @@ EmissionsTracker.stop() → minimal: private POST only | extensive: privat If the run lasts less than one second, telemetry is not sent. -## Private telemetry (`minimal` and `extensive`) — per run +## Private telemetry — per run -Both levels send the **same private payload** to `POST /telemetry` at each `stop()`. The `telemetry_level` field records which setting was used (`minimal` or `extensive`). +Both tiers POST to `/telemetry` at each `stop()`. The server schema defines what each tier may include: -The payload includes: +| Tier | Private `POST /telemetry` payload | +|------|-----------------------------------| +| `minimal` | Environment and hardware only (OS, Python, CPU/GPU/RAM, geo/cloud, CodeCarbon version) | +| `extensive` | Minimal fields **plus** run metrics, output methods, framework flags, usage diagnostics | -- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region, GPU driver/CUDA/cuDNN when available -- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container/IDE hints, hardware diagnostics -- **ML stack:** framework presence flags when detected (no installed package versions) -- **Run outcome:** duration, emissions, energy (total and per component), utilization averages +**Minimal** includes rounded coordinates (1 decimal), cloud region, and hardware metadata. It does **not** include run emissions, energy, duration, or framework flags. -Private telemetry does **not** include project names, experiment ids, API keys, file paths, exact coordinates, executable/host hashes, or survey demographics (role, industry, etc.). +**Extensive** adds run outcome (duration, emissions, energy, utilization), output methods, ML framework presence flags (booleans only, no package versions), CI/notebook/container/IDE hints, and integration context (`decorator_vs_context`: `library`, `cli_monitor`, or `offline_tracker`). + +Private telemetry does **not** include project names, experiment ids, API keys, file paths, executable/host hashes, or survey demographics. ## `extensive` — additional public run summary @@ -72,7 +74,7 @@ telemetry_level = minimal ```bash codecarbon telemetry set minimal -codecarbon telemetry show +codecarbon telemetry status codecarbon monitor --telemetry-level disabled -- python train.py ``` @@ -96,7 +98,7 @@ telemetry_level = disabled ## First run without explicit configuration -If you never set `telemetry_level`, CodeCarbon uses `minimal` (Tier 1) and logs a **one-time warning** per Python session. Set `telemetry_level` explicitly to silence it. +If you never set `telemetry_level`, CodeCarbon uses `minimal` and logs a **one-time warning** per Python session. Set `telemetry_level` explicitly to silence it. ## Related diff --git a/docs/reference/api.md b/docs/reference/api.md index 8b8bea306..fd3a97e5e 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -19,8 +19,8 @@ Optional library telemetry is controlled by **`telemetry_level`** on the tracker | Value | Behavior | |-------|----------| | `disabled` | No product telemetry | -| `minimal` (default) | Tier 1 private telemetry at each `stop()` | -| `extensive` | Tier 1 + Tier 2 (private telemetry and public run summary) at each `stop()` | +| `minimal` (default) | Private telemetry at each `stop()` | +| `extensive` | Private telemetry plus public run summary at each `stop()` | **Resolution order:** tracker argument → `.codecarbon.config` → `CODECARBON_TELEMETRY_LEVEL` → default `minimal`. The tracker argument overrides config and environment. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 6c550f625..56ee7b909 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -98,7 +98,7 @@ Configure **product telemetry** (library usage metadata), separate from `codecar ```bash codecarbon telemetry # interactive wizard codecarbon telemetry set # disabled | minimal | extensive -codecarbon telemetry show # resolved tier and whether it was set explicitly +codecarbon telemetry status # resolved tier and whether it was set explicitly ``` **Options:** @@ -107,7 +107,7 @@ codecarbon telemetry show # resolved tier and whether it was set explici |--------|-------------| | `--config PATH` | Use a specific `.codecarbon.config` (default: local then global) | -See [Product telemetry](../how-to/telemetry.md) for what Tier 1 collects and how to opt out. +See [Product telemetry](../how-to/telemetry.md) for what `minimal` collects and how to opt out. ### `codecarbon monitor --telemetry-level` diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py index 88cd9375b..7e8a9cf88 100644 --- a/tests/cli/test_telemetry_cli.py +++ b/tests/cli/test_telemetry_cli.py @@ -36,21 +36,21 @@ def test_telemetry_set_writes_config(): assert "telemetry_level = disabled" in content -def test_telemetry_show_reports_stored_level(): +def test_telemetry_status_reports_stored_level(): runner = CliRunner() with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / ".codecarbon.config" config_path.write_text("[codecarbon]\ntelemetry_level = extensive\n") result = runner.invoke( telemetry_app, - ["show", "--config", str(config_path)], + ["status", "--config", str(config_path)], ) assert result.exit_code == 0 assert "extensive" in result.output assert "Explicitly configured: True" in result.output -def test_telemetry_show_merged_matches_tracker_precedence(): +def test_telemetry_status_merged_matches_tracker_precedence(): runner = CliRunner() with tempfile.TemporaryDirectory() as tmp: global_path = Path(tmp) / "global.config" @@ -61,7 +61,7 @@ def test_telemetry_show_merged_matches_tracker_precedence(): "codecarbon.core.config._config_file_paths", return_value=(str(global_path), str(local_path)), ): - result = runner.invoke(telemetry_app, ["show"]) + result = runner.invoke(telemetry_app, ["status"]) assert result.exit_code == 0 assert "Resolved tier: disabled" in result.output assert "merged" in result.output diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 78c64359d..9244579df 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -81,7 +81,7 @@ def test_tier1_posts_private_telemetry(self): mock_post.assert_called_once() payload = mock_post.call_args[0][1] self.assertEqual(payload["telemetry_level"], "minimal") - self.assertEqual(payload["total_emissions_kg"], 0.001) + self.assertNotIn("total_emissions_kg", payload) def test_tier1_skips_short_duration_at_dispatcher(self): tracker = MagicMock() @@ -167,7 +167,7 @@ def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( mock_post.assert_called_once() payload = mock_post.call_args[0][1] self.assertEqual(payload["telemetry_level"], "minimal") - self.assertIn("total_emissions_kg", payload) + self.assertNotIn("total_emissions_kg", payload) def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): self._start_config_mock(disabled_conf) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 0a5f4f379..8278cdb8f 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -1,7 +1,8 @@ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import patch from codecarbon.core.telemetry import TelemetryContext, TelemetryLevel, build_payload +from codecarbon.core.telemetry.schemas import MINIMAL_TELEMETRY_FIELDS, TelemetryBase, TelemetryCreate from codecarbon.output_methods.emissions_data import EmissionsData @@ -44,40 +45,23 @@ def _sample_emissions(**overrides): def _tracker_context(**overrides) -> TelemetryContext: - tracker = MagicMock() - tracker._conf = overrides.pop("conf", {"codecarbon_version": "3.0"}) - tracker._hardware = overrides.pop("hardware", []) - tracker._resource_tracker = overrides.pop("resource_tracker", None) - tracker._save_to_file = overrides.pop("save_to_file", False) - tracker._save_to_api = overrides.pop("save_to_api", False) - tracker._save_to_logger = overrides.pop("save_to_logger", False) - tracker._emissions_endpoint = overrides.pop("emissions_endpoint", None) - tracker._save_to_prometheus = overrides.pop("save_to_prometheus", False) - tracker._save_to_logfire = overrides.pop("save_to_logfire", False) - tracker._tasks = overrides.pop("tasks", {}) - tracker._measure_power_secs = overrides.pop("measure_power_secs", 15) - tracker._is_offline = overrides.pop("is_offline", False) + conf = overrides.pop("conf", {"codecarbon_version": "3.0"}) emissions = overrides.pop("emissions", _sample_emissions()) - ctx = TelemetryContext( - conf=tracker._conf, + output_methods = overrides.pop("output_methods", []) + return TelemetryContext( + conf=conf, emissions=emissions, - hardware=tracker._hardware, - resource_tracker=tracker._resource_tracker, - save_to_api=tracker._save_to_api, - save_to_file=tracker._save_to_file, - save_to_logger=tracker._save_to_logger, - save_to_prometheus=tracker._save_to_prometheus, - save_to_logfire=tracker._save_to_logfire, - emissions_endpoint=tracker._emissions_endpoint, - tasks=tracker._tasks, - measure_power_secs=tracker._measure_power_secs, - is_offline=tracker._is_offline, + hardware=overrides.pop("hardware", []), + resource_tracker=overrides.pop("resource_tracker", None), + output_methods=output_methods, + tasks=overrides.pop("tasks", {}), + measure_power_secs=overrides.pop("measure_power_secs", 15), + integration=overrides.pop("integration", "library"), ) - return ctx class TestTelemetryCollect(unittest.TestCase): - def test_build_payload_includes_run_and_framework_flags(self): + def test_build_payload_minimal_omits_run_metrics(self): ctx = _tracker_context( conf={ "os": "Linux", @@ -85,15 +69,36 @@ def test_build_payload_includes_run_and_framework_flags(self): "cpu_count": 4, "tracking_mode": "machine", }, - save_to_file=True, + output_methods=["file"], ) with patch( "codecarbon.core.telemetry.collect._package_installed", side_effect=lambda name: name == "torch", ): - payload = build_payload(ctx) + payload = build_payload(ctx, level=TelemetryLevel.minimal) self.assertEqual(payload["telemetry_level"], "minimal") + self.assertNotIn("total_emissions_kg", payload) + self.assertNotIn("has_torch", payload) + self.assertNotIn("output_methods", payload) + + def test_build_payload_extensive_includes_run_and_framework_flags(self): + ctx = _tracker_context( + conf={ + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + }, + output_methods=["file"], + ) + with patch( + "codecarbon.core.telemetry.collect._package_installed", + side_effect=lambda name: name == "torch", + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + + self.assertEqual(payload["telemetry_level"], "extensive") self.assertEqual(payload["total_emissions_kg"], 0.5) self.assertEqual(payload["duration_seconds"], 10.0) self.assertTrue(payload["has_torch"]) @@ -105,9 +110,9 @@ def test_build_payload_omits_framework_versions(self): "codecarbon.core.telemetry.collect._package_installed", return_value=True, ): - payload = build_payload(ctx) + payload = build_payload(ctx, level=TelemetryLevel.extensive) - self.assertEqual(payload["telemetry_level"], "minimal") + self.assertEqual(payload["telemetry_level"], "extensive") self.assertTrue(payload["has_torch"]) self.assertNotIn("torch_version", payload) @@ -116,6 +121,25 @@ def test_build_payload_uses_resolved_level(self): payload = build_payload(ctx, level=TelemetryLevel.extensive) self.assertEqual(payload["telemetry_level"], "extensive") + def test_minimal_payload_passes_schema_validation(self): + ctx = _tracker_context(conf={"os": "Linux", "codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.minimal) + TelemetryCreate(**payload) + + def test_extensive_payload_passes_schema_validation(self): + ctx = _tracker_context(conf={"os": "Linux", "codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.extensive) + TelemetryCreate(**payload) + + def test_payload_keys_are_schema_fields(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0"}) + for level in (TelemetryLevel.minimal, TelemetryLevel.extensive): + payload = build_payload(ctx, level=level) + self.assertTrue(set(payload).issubset(TelemetryBase.model_fields)) + + def test_minimal_fields_are_schema_subset(self): + self.assertTrue(MINIMAL_TELEMETRY_FIELDS.issubset(TelemetryBase.model_fields)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 6b230ee83..95bdb7246 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -46,7 +46,7 @@ def test_warns_once_when_telemetry_not_explicit(self): telemetry.warn_if_implicit() telemetry.warn_if_implicit() self.assertEqual(mock_warning.call_count, 1) - self.assertIn("Tier 1", mock_warning.call_args[0][0]) + self.assertIn("Minimal telemetry", mock_warning.call_args[0][0]) def test_no_warn_when_config_explicit(self): settings = TelemetrySettings.resolve( @@ -63,8 +63,8 @@ def test_tier1_posts_to_telemetry_endpoint(self): tier1_payload = { "timestamp": datetime(2020, 1, 1, tzinfo=timezone.utc), "telemetry_level": "minimal", - "total_emissions_kg": 0.001, "os": "Linux", + "cpu_count": 4, } settings = TelemetrySettings.resolve( external_conf={"telemetry_api_url": "http://tier1.example"} @@ -81,7 +81,7 @@ def test_tier1_posts_to_telemetry_endpoint(self): mock_post.call_args.kwargs["json"]["telemetry_level"], TelemetryLevel.minimal.value, ) - self.assertIn("total_emissions_kg", mock_post.call_args.kwargs["json"]) + self.assertEqual(mock_post.call_args.kwargs["json"]["cpu_count"], 4) def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): with tempfile.TemporaryDirectory() as tmp: @@ -271,7 +271,7 @@ def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): save_to_file=False, ) configure_warnings = [ - c for c in mock_warning.call_args_list if c[0] and "Tier 1" in str(c[0][0]) + c for c in mock_warning.call_args_list if c[0] and "Minimal telemetry" in str(c[0][0]) ] self.assertEqual(len(configure_warnings), 0) From 29664d7225778f39012b45866793dfc0745ad48a Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 17 Jun 2026 17:41:31 +0200 Subject: [PATCH 22/29] Apply pre-commit formatting for telemetry files Fix isort and black on collect tests after local pre-commit run. Co-authored-by: Cursor --- codecarbon/core/telemetry/collect.py | 14 +++++++------- tests/test_telemetry_collect.py | 6 +++++- tests/test_telemetry_config.py | 4 +++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/codecarbon/core/telemetry/collect.py b/codecarbon/core/telemetry/collect.py index 2eb9a23ac..b3a45e74b 100644 --- a/codecarbon/core/telemetry/collect.py +++ b/codecarbon/core/telemetry/collect.py @@ -93,9 +93,7 @@ def from_tracker(cls, tracker: Any, emissions: EmissionsData) -> TelemetryContex def _strip_empty(data: dict[str, Any]) -> dict[str, Any]: return { - key: value - for key, value in data.items() - if value not in (None, "", [], {}) + key: value for key, value in data.items() if value not in (None, "", [], {}) } @@ -113,7 +111,9 @@ def _round_coordinate(value: Any) -> float | None: return round(float(value), 1) -def _cloud_region(emissions: EmissionsData) -> tuple[str | None, str | None, str | None]: +def _cloud_region( + emissions: EmissionsData, +) -> tuple[str | None, str | None, str | None]: details = get_env_cloud_details() raw_provider = raw_region = None if details and details.get("metadata"): @@ -311,9 +311,9 @@ def _extensive_payload(ctx: TelemetryContext) -> dict[str, Any]: "ci_environment": _env_label(CI_ENV_VAR_LABELS), "notebook_environment": _detect_notebook_environment(), "ide_used": _detect_ide(), - "duration_seconds": float(emissions.duration) - if emissions.duration - else None, + "duration_seconds": ( + float(emissions.duration) if emissions.duration else None + ), "total_emissions_kg": emissions.emissions, "emissions_rate_kg_per_sec": emissions.emissions_rate, "energy_consumed_kwh": emissions.energy_consumed, diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 8278cdb8f..ab6fe021b 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -2,7 +2,11 @@ from unittest.mock import patch from codecarbon.core.telemetry import TelemetryContext, TelemetryLevel, build_payload -from codecarbon.core.telemetry.schemas import MINIMAL_TELEMETRY_FIELDS, TelemetryBase, TelemetryCreate +from codecarbon.core.telemetry.schemas import ( + MINIMAL_TELEMETRY_FIELDS, + TelemetryBase, + TelemetryCreate, +) from codecarbon.output_methods.emissions_data import EmissionsData diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 95bdb7246..e637cb243 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -271,7 +271,9 @@ def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): save_to_file=False, ) configure_warnings = [ - c for c in mock_warning.call_args_list if c[0] and "Minimal telemetry" in str(c[0][0]) + c + for c in mock_warning.call_args_list + if c[0] and "Minimal telemetry" in str(c[0][0]) ] self.assertEqual(len(configure_warnings), 0) From a231750bcb2e94b2ef233cce11c1660f47e7456f Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 17 Jun 2026 17:45:36 +0200 Subject: [PATCH 23/29] Fix CodeQL logging alert and raise telemetry test coverage Resolve api key resolution after tier logging to avoid clear-text secret taint, drop debug logging of telemetry request bodies, and add tests for client error paths, schema validation, and collect environment probes. Co-authored-by: Cursor --- codecarbon/core/telemetry/client.py | 1 - codecarbon/core/telemetry/settings.py | 34 ++++---- tests/test_telemetry_client.py | 90 +++++++++++++++++++- tests/test_telemetry_collect.py | 113 +++++++++++++++++++++++++- tests/test_telemetry_schemas.py | 33 ++++++++ 5 files changed, 251 insertions(+), 20 deletions(-) create mode 100644 tests/test_telemetry_schemas.py diff --git a/codecarbon/core/telemetry/client.py b/codecarbon/core/telemetry/client.py index 984dd5506..571ece940 100644 --- a/codecarbon/core/telemetry/client.py +++ b/codecarbon/core/telemetry/client.py @@ -43,7 +43,6 @@ def post_private(settings: TelemetrySettings, payload: dict) -> bool: response.status_code, response.text, ) - logger.debug("Telemetry request body: %s", body) return False diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py index ca0ee22b9..009f9f08c 100644 --- a/codecarbon/core/telemetry/settings.py +++ b/codecarbon/core/telemetry/settings.py @@ -46,6 +46,21 @@ def is_explicit(self) -> bool: """Return whether the user explicitly chose a telemetry tier.""" return self.source != "default" + @classmethod + def _with_connection( + cls, + merged: dict[str, Any], + level: TelemetryLevel, + source: TelemetryLevelSource, + ) -> TelemetrySettings: + return cls( + level=level, + source=source, + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + @classmethod def resolve( cls, @@ -56,9 +71,6 @@ def resolve( ) -> TelemetrySettings: """Resolve tier (override > external > file > default minimal) and API settings.""" merged = external_conf or {} - api_url = cls._resolve_api_url(merged) - api_key = cls._resolve_api_key(merged) - experiment_id = cls._resolve_experiment_id(merged) if override is not None: raw = override source: TelemetryLevelSource = "override" @@ -72,13 +84,7 @@ def resolve( raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] source = "file" else: - return cls( - level=DEFAULT_TELEMETRY_LEVEL, - source="default", - api_url=api_url, - api_key=api_key, - experiment_id=experiment_id, - ) + return cls._with_connection(merged, DEFAULT_TELEMETRY_LEVEL, "default") try: level = parse_telemetry_level(raw) except ValueError: @@ -88,13 +94,7 @@ def resolve( DEFAULT_TELEMETRY_LEVEL.value, ) level = DEFAULT_TELEMETRY_LEVEL - return cls( - level=level, - source=source, - api_url=api_url, - api_key=api_key, - experiment_id=experiment_id, - ) + return cls._with_connection(merged, level, source) @staticmethod def _resolve_api_url(external_conf: dict[str, Any]) -> str: diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 8b0128d2a..2ca7581f7 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -4,7 +4,12 @@ import requests_mock from pydantic import ValidationError -from codecarbon.core.telemetry import TelemetrySettings, post_private +from codecarbon.core.telemetry import ( + TelemetrySettings, + post_private, + post_public_summary, +) +from codecarbon.output_methods.emissions_data import EmissionsData class TestPostPrivate(unittest.TestCase): @@ -102,5 +107,88 @@ def test_post_private_returns_false_on_request_error(self): self.assertFalse(result) +class TestPostPublicSummary(unittest.TestCase): + def _settings(self) -> TelemetrySettings: + return TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id="00000000-0000-0000-0000-000000000001", + ) + + def _emissions(self) -> EmissionsData: + return EmissionsData( + timestamp="2026-05-03T12:00:00+00:00", + project_name="p", + run_id="r", + experiment_id="e", + duration=10.0, + emissions=0.001, + emissions_rate=0.0001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="cpu", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + + def test_post_public_summary_returns_true_on_success(self): + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_client: + mock_client.return_value.add_emission.return_value = True + result = post_public_summary(self._settings(), {}, self._emissions()) + self.assertTrue(result) + + def test_post_public_summary_returns_false_on_error(self): + with patch( + "codecarbon.core.telemetry.client.ApiClient", + side_effect=RuntimeError("api down"), + ): + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_public_summary(self._settings(), {}, self._emissions()) + self.assertFalse(result) + mock_logger.error.assert_called_once() + + +class TestPostPrivateErrors(unittest.TestCase): + def test_post_private_logs_error_on_unexpected_status(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + settings = TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id=TelemetrySettings.resolve().experiment_id, + ) + with requests_mock.Mocker() as m: + m.post("http://test.com/telemetry", text="server error", status_code=500) + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_private(settings, telemetry) + self.assertFalse(result) + mock_logger.error.assert_called_once() + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index ab6fe021b..5b8459812 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -1,5 +1,7 @@ +import os +import sys import unittest -from unittest.mock import patch +from unittest.mock import MagicMock, patch from codecarbon.core.telemetry import TelemetryContext, TelemetryLevel, build_payload from codecarbon.core.telemetry.schemas import ( @@ -144,6 +146,115 @@ def test_payload_keys_are_schema_fields(self): def test_minimal_fields_are_schema_subset(self): self.assertTrue(MINIMAL_TELEMETRY_FIELDS.issubset(TelemetryBase.model_fields)) + def test_cloud_region_from_aws_metadata(self): + emissions = _sample_emissions(on_cloud="Y", cloud_region="", region="") + ctx = _tracker_context(emissions=emissions) + with patch( + "codecarbon.core.telemetry.collect.get_env_cloud_details", + return_value={"provider": "aws", "metadata": {"region": "eu-west-1"}}, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["cloud_provider"], "aws") + self.assertEqual(payload["cloud_region"], "eu-west-1") + + def test_cloud_region_from_azure_metadata(self): + emissions = _sample_emissions(on_cloud="Y", cloud_region="", region="") + ctx = _tracker_context(emissions=emissions) + with patch( + "codecarbon.core.telemetry.collect.get_env_cloud_details", + return_value={ + "provider": "azure", + "metadata": {"compute": {"location": "westeurope"}}, + }, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["cloud_provider"], "azure") + self.assertEqual(payload["cloud_region"], "westeurope") + + def test_cloud_region_from_gcp_metadata(self): + emissions = _sample_emissions(on_cloud="Y", cloud_region="", region="") + ctx = _tracker_context(emissions=emissions) + with patch( + "codecarbon.core.telemetry.collect.get_env_cloud_details", + return_value={ + "provider": "gcp", + "metadata": {"zone": "projects/p/zones/europe-west1-b"}, + }, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["cloud_provider"], "gcp") + self.assertEqual(payload["cloud_region"], "europe-west1") + + def test_extensive_payload_includes_environment_hints(self): + ctx = _tracker_context(output_methods=["api"]) + with patch.dict( + os.environ, + { + "CONDA_DEFAULT_ENV": "base", + "KUBERNETES_SERVICE_HOST": "10.0.0.1", + "CURSOR_TRACE_ID": "trace-1", + "COLAB_GPU": "0", + }, + clear=False, + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertEqual(payload["python_env_type"], "conda") + self.assertEqual(payload["api_mode"], "online") + self.assertTrue(payload["in_container"]) + self.assertEqual(payload["container_runtime"], "kubernetes") + self.assertEqual(payload["ide_used"], "cursor") + self.assertEqual(payload["notebook_environment"], "colab") + + def test_from_tracker_detects_cli_monitor(self): + tracker = MagicMock() + tracker._conf = {"codecarbon_version": "3.0"} + tracker._hardware = [] + tracker._resource_tracker = None + tracker._output_methods = [] + tracker._emissions_endpoint = None + tracker._tasks = {} + tracker._measure_power_secs = 15 + with patch.object(sys, "argv", ["codecarbon", "monitor", "train.py"]): + ctx = TelemetryContext.from_tracker(tracker, _sample_emissions()) + self.assertEqual(ctx.integration, "cli_monitor") + + def test_hardware_diagnostics_lists_tracked_hardware(self): + hardware = MagicMock() + hardware.description.return_value = "CPU: test" + ctx = _tracker_context( + hardware=[hardware], + resource_tracker=MagicMock(gpu_tracker="pynvml"), + ) + with patch( + "codecarbon.core.telemetry.collect.platform.system", return_value="Linux" + ): + with patch( + "codecarbon.core.cpu.is_rapl_available", + return_value=True, + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertIn("CPU: test", payload["hardware_tracked"]) + self.assertTrue(payload["hardware_detection_success"]) + self.assertTrue(payload["rapl_available"]) + self.assertEqual(payload["gpu_detection_method"], "pynvml") + + def test_gpu_static_fields_when_nvidia_available(self): + ctx = _tracker_context() + mock_mem = MagicMock(total=8 * 1024**3) + mock_pynvml = MagicMock() + mock_pynvml.nvmlDeviceGetMemoryInfo.return_value = mock_mem + mock_pynvml.nvmlSystemGetCudaDriverVersion_v2.return_value = 12040 + mock_pynvml.nvmlSystemGetDriverVersion.return_value = "535.0" + with patch( + "codecarbon.core.telemetry.collect.is_nvidia_system", + return_value=True, + ): + with patch.dict(sys.modules, {"pynvml": mock_pynvml}): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["gpu_memory_total_gb"], 8.0) + self.assertEqual(payload["cuda_version"], "12.4") + self.assertEqual(payload["gpu_driver_version"], "535.0") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_telemetry_schemas.py b/tests/test_telemetry_schemas.py new file mode 100644 index 000000000..cdfa1b853 --- /dev/null +++ b/tests/test_telemetry_schemas.py @@ -0,0 +1,33 @@ +import unittest +from datetime import datetime, timezone + +import pytest +from pydantic import ValidationError + +from codecarbon.core.telemetry.schemas import TelemetryCreate, TelemetryLevel + + +class TestTelemetrySchemaValidation(unittest.TestCase): + def _minimal_payload(self) -> dict: + return { + "timestamp": datetime(2026, 1, 1, tzinfo=timezone.utc), + "telemetry_level": TelemetryLevel.minimal, + "os": "Linux", + } + + def test_rejects_disabled_telemetry_level(self): + with pytest.raises(ValidationError, match="Disabled telemetry must not be submitted"): + TelemetryCreate(**{**self._minimal_payload(), "telemetry_level": "disabled"}) + + def test_rejects_minimal_with_extensive_field(self): + with pytest.raises(ValidationError, match="Minimal telemetry cannot include extensive fields"): + TelemetryCreate( + **{ + **self._minimal_payload(), + "total_emissions_kg": 0.5, + } + ) + + +if __name__ == "__main__": + unittest.main() From edb2d0321d67c16275509e1a1805384524c380a0 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 17 Jun 2026 17:47:38 +0200 Subject: [PATCH 24/29] Apply black formatting to telemetry schema tests Co-authored-by: Cursor --- tests/test_telemetry_schemas.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_telemetry_schemas.py b/tests/test_telemetry_schemas.py index cdfa1b853..085649cee 100644 --- a/tests/test_telemetry_schemas.py +++ b/tests/test_telemetry_schemas.py @@ -16,11 +16,17 @@ def _minimal_payload(self) -> dict: } def test_rejects_disabled_telemetry_level(self): - with pytest.raises(ValidationError, match="Disabled telemetry must not be submitted"): - TelemetryCreate(**{**self._minimal_payload(), "telemetry_level": "disabled"}) + with pytest.raises( + ValidationError, match="Disabled telemetry must not be submitted" + ): + TelemetryCreate( + **{**self._minimal_payload(), "telemetry_level": "disabled"} + ) def test_rejects_minimal_with_extensive_field(self): - with pytest.raises(ValidationError, match="Minimal telemetry cannot include extensive fields"): + with pytest.raises( + ValidationError, match="Minimal telemetry cannot include extensive fields" + ): TelemetryCreate( **{ **self._minimal_payload(), From 13cde15629c3b874d61bd046b51801b1b63978e4 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 17 Jun 2026 17:48:50 +0200 Subject: [PATCH 25/29] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- codecarbon/core/telemetry/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py index 009f9f08c..33ea6d125 100644 --- a/codecarbon/core/telemetry/settings.py +++ b/codecarbon/core/telemetry/settings.py @@ -89,8 +89,7 @@ def resolve( level = parse_telemetry_level(raw) except ValueError: logger.error( - "Invalid telemetry_level %r; falling back to %r", - raw, + "Invalid telemetry_level provided; falling back to %r", DEFAULT_TELEMETRY_LEVEL.value, ) level = DEFAULT_TELEMETRY_LEVEL From 2145b615f07ed15ec3f22df81436ecc7d90f7fd2 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 17 Jun 2026 17:58:20 +0200 Subject: [PATCH 26/29] Raise PR coverage for telemetry CLI and API helpers Add CLI helper and interactive-flow tests, cover coordinate rounding in ApiClient, and extend collect environment probes to satisfy Codecov patch and project thresholds on PR #1200. Co-authored-by: Cursor --- tests/cli/test_telemetry_cli.py | 154 +++++++++++++++++++++++++++++++- tests/test_api_call.py | 7 +- tests/test_telemetry_collect.py | 28 ++++++ 3 files changed, 186 insertions(+), 3 deletions(-) diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py index 7e8a9cf88..ef14b831a 100644 --- a/tests/cli/test_telemetry_cli.py +++ b/tests/cli/test_telemetry_cli.py @@ -2,14 +2,20 @@ import tempfile from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest import typer from typer.testing import CliRunner from codecarbon.cli import main as cli_main -from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app +from codecarbon.cli.telemetry_cli import ( + normalize_telemetry_level, + pick_config_path_interactive, + resolve_config_path, + telemetry_app, + write_telemetry_level, +) def test_normalize_telemetry_level_accepts_valid_values(): @@ -109,3 +115,147 @@ def test_monitor_rejects_invalid_telemetry_level(): ["monitor", "--telemetry-level", "invalid"], ) assert result.exit_code != 0 + + +def test_telemetry_status_missing_config_file(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + missing = Path(tmp) / "missing.config" + result = runner.invoke( + telemetry_app, + ["status", "--config", str(missing)], + ) + assert result.exit_code == 0 + assert "Config file not found" in result.output + assert "not explicit" in result.output + + +def test_telemetry_status_shows_implicit_warning(): + runner = CliRunner() + with patch( + "codecarbon.cli.telemetry_cli.get_config_file_settings", + return_value={}, + ): + with patch( + "codecarbon.cli.telemetry_cli.get_hierarchical_config", + return_value={}, + ): + result = runner.invoke(telemetry_app, ["status"]) + assert "Explicitly configured: False" in result.output + assert "Minimal telemetry will be sent" in result.output + + +def test_resolve_config_path_creates_explicit_file(): + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / "custom.config" + resolved = resolve_config_path(config_path, create=True) + assert resolved == config_path.resolve() + assert config_path.exists() + + +def test_resolve_config_path_prefers_local_config(tmp_path, monkeypatch): + local_path = tmp_path / ".codecarbon.config" + local_path.write_text("[codecarbon]\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + assert resolve_config_path(None) == local_path.resolve() + + +def test_resolve_config_path_uses_global_when_local_missing(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + global_path = home / ".codecarbon.config" + global_path.write_text("[codecarbon]\n", encoding="utf-8") + workdir = tmp_path / "work" + workdir.mkdir() + monkeypatch.chdir(workdir) + monkeypatch.setattr(Path, "home", lambda: home) + assert resolve_config_path(None) == global_path.resolve() + + +def test_write_telemetry_level_creates_missing_file(tmp_path): + config_path = tmp_path / "nested" / ".codecarbon.config" + write_telemetry_level(config_path, "minimal") + assert config_path.exists() + assert "telemetry_level = minimal" in config_path.read_text() + + +def test_pick_config_path_interactive_create_new(): + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = "Create new config file" + with patch( + "codecarbon.cli.telemetry_cli.create_new_config_file", + return_value=Path("/tmp/new.config"), + ) as mock_create: + path = pick_config_path_interactive() + mock_create.assert_called_once() + assert path == Path("/tmp/new.config") + + +def test_resolve_config_path_creates_local_in_cwd(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + monkeypatch.setattr(Path, "home", lambda: home) + resolved = resolve_config_path(None, create=True) + assert resolved == (work / ".codecarbon.config").resolve() + assert resolved.exists() + + +def test_pick_config_path_lists_existing_configs(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + global_path = home / ".codecarbon.config" + global_path.write_text("[codecarbon]\n", encoding="utf-8") + work = tmp_path / "work" + work.mkdir() + local_path = work / ".codecarbon.config" + local_path.write_text("[codecarbon]\n", encoding="utf-8") + monkeypatch.chdir(work) + monkeypatch.setattr(Path, "home", lambda: home) + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = str(local_path) + path = pick_config_path_interactive() + assert path == local_path.resolve() + + +def test_telemetry_default_command_runs_interactive_wizard(tmp_path): + config_path = tmp_path / ".codecarbon.config" + config_path.write_text("[codecarbon]\n", encoding="utf-8") + runner = CliRunner() + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = "disabled" + result = runner.invoke(telemetry_app, ["--config", str(config_path)]) + assert result.exit_code == 0 + assert "telemetry_level = disabled" in config_path.read_text() + + +def test_telemetry_interactive_exits_when_selection_cancelled(tmp_path): + config_path = tmp_path / ".codecarbon.config" + config_path.write_text("[codecarbon]\n", encoding="utf-8") + runner = CliRunner() + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = None + result = runner.invoke(telemetry_app, ["--config", str(config_path)]) + assert result.exit_code == 0 + assert "telemetry_level" not in config_path.read_text() + + +def test_telemetry_interactive_prompts_for_config_path(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + runner = CliRunner() + select = MagicMock() + select.ask.return_value = "minimal" + with patch( + "codecarbon.cli.telemetry_cli.pick_config_path_interactive", + return_value=tmp_path / ".codecarbon.config", + ): + with patch( + "codecarbon.cli.telemetry_cli.questionary.select", return_value=select + ): + result = runner.invoke(telemetry_app, []) + config_path = tmp_path / ".codecarbon.config" + assert result.exit_code == 0 + assert config_path.exists() + assert "telemetry_level = minimal" in config_path.read_text() diff --git a/tests/test_api_call.py b/tests/test_api_call.py index 8d6fa2b80..043f9c5a0 100644 --- a/tests/test_api_call.py +++ b/tests/test_api_call.py @@ -4,7 +4,7 @@ import requests_mock -from codecarbon.core.api_client import ApiClient +from codecarbon.core.api_client import ApiClient, _round_coordinate from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate from codecarbon.output import EmissionsData @@ -26,6 +26,11 @@ class TestApi(unittest.TestCase): + def test_round_coordinate_handles_none_and_values(self): + self.assertEqual(_round_coordinate(None), 0.0) + self.assertEqual(_round_coordinate(-7.61743), -7.6) + self.assertEqual(_round_coordinate(33.58229, decimals=2), 33.58) + def test_get_headers_prefers_api_key_over_access_token(self): api = ApiClient( endpoint_url="http://test.com", diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 5b8459812..64deac26e 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -255,6 +255,34 @@ def test_gpu_static_fields_when_nvidia_available(self): self.assertEqual(payload["cuda_version"], "12.4") self.assertEqual(payload["gpu_driver_version"], "535.0") + def test_extensive_payload_detects_docker_and_jupyter(self): + ctx = _tracker_context() + with patch("os.path.exists", return_value=True): + with patch( + "codecarbon.core.telemetry.collect._detect_notebook_environment", + return_value="jupyter", + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertTrue(payload["in_container"]) + self.assertEqual(payload["container_runtime"], "docker") + self.assertEqual(payload["notebook_environment"], "jupyter") + + def test_minimal_payload_detects_virtualenv(self): + ctx = _tracker_context() + with patch.dict(os.environ, {"VIRTUAL_ENV": "/venv"}, clear=False): + with patch( + "codecarbon.core.telemetry.collect.sys.prefix", + "/venv", + create=True, + ): + with patch( + "codecarbon.core.telemetry.collect.sys.base_prefix", + "/usr", + create=True, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["python_env_type"], "venv") + if __name__ == "__main__": unittest.main() From ea4f275b2312c8cf18546a4af3e1d4be3dba394c Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Sat, 18 Jul 2026 17:28:24 +0200 Subject: [PATCH 27/29] Refactor telemetry schema docstring Remove unnecessary comments in the telemetry payload schemas to streamline the code and improve readability. --- codecarbon/core/telemetry/schemas.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/codecarbon/core/telemetry/schemas.py b/codecarbon/core/telemetry/schemas.py index 81198ddc1..5e948b912 100644 --- a/codecarbon/core/telemetry/schemas.py +++ b/codecarbon/core/telemetry/schemas.py @@ -1,7 +1,4 @@ -"""Telemetry payload schemas aligned with carbonserver ``telemetry_sql_models.Telemetry``. - -ponytail: mirrored in ``carbonserver/api/schemas_telemetry.py`` for drift tests only. -""" +"""Telemetry payload schemas aligned with carbonserver ``telemetry_sql_models.Telemetry``.""" from datetime import datetime from enum import Enum From 8b995d368009e541c4e06039fdb17234a724e481 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Sat, 18 Jul 2026 18:40:06 +0200 Subject: [PATCH 28/29] Update telemetry configuration in emissions tracker Refactor the BaseEmissionsTracker class to utilize default telemetry settings for experiment ID and level, enhancing configurability and maintainability. Remove the previous telemetry override logic for clarity. --- codecarbon/emissions_tracker.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 5e8015023..94c6d0110 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -48,6 +48,7 @@ OutputMethod, PrometheusOutput, ) +from codecarbon.core.telemetry.settings import DEFAULT_TELEMETRY_LEVEL, DEFAULT_TELEMETRY_EXPERIMENT_ID # /!\ Warning: current implementation prevents the user from setting any value to None # from the script call @@ -508,9 +509,6 @@ def __init__( self._external_conf = get_hierarchical_config() self._config_file_conf = get_config_file_settings() - self._telemetry_override = ( - None if telemetry_level is _sentinel else telemetry_level - ) self._telemetry = Telemetry.from_tracker(self) self._set_from_conf( force_carbon_intensity_g_co2e_kwh, @@ -578,8 +576,9 @@ def __init__( self._set_from_conf(rapl_include_dram, "rapl_include_dram", False, bool) self._set_from_conf(rapl_prefer_psys, "rapl_prefer_psys", False, bool) self._set_from_conf( - experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" + experiment_id, "experiment_id", DEFAULT_TELEMETRY_EXPERIMENT_ID ) + self._set_from_conf(telemetry_level, "telemetry_level", DEFAULT_TELEMETRY_LEVEL) if self.force_carbon_intensity_g_co2e_kwh is not None: logger.info( f"Using forced carbon intensity: {self.force_carbon_intensity_g_co2e_kwh} gCO2e/kWh." From 08cc67c4e2580c115912f8b629902233696487e9 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Sat, 18 Jul 2026 18:57:14 +0200 Subject: [PATCH 29/29] Add telemetry experiment ID and default settings integration Enhance the telemetry configuration by introducing a new field for telemetry experiment ID in the Settings class. This change integrates shared telemetry defaults, ensuring consistent access to the experiment ID across the application. Additionally, implement symlinks for shared telemetry defaults and update related tests to validate the new configuration. - Added telemetry_experiment_id to Settings with default value. - Created symlink for telemetry_defaults.py in both carbonserver and core directories. - Updated telemetry API router to utilize the new experiment ID. - Added tests to verify the integration of shared telemetry defaults and their accessibility. --- .../carbonserver/api/routers/telemetry.py | 14 +- carbonserver/carbonserver/config.py | 8 + .../carbonserver/telemetry_defaults.py | 1 + .../integration/test_telemetry_local_api.py | 164 +++++++++++++++++ .../tests/api/routers/test_telemetry.py | 89 +++++++++- .../api/test_telemetry_shared_defaults.py | 99 +++++++++++ codecarbon/core/telemetry/defaults.py | 1 + codecarbon/core/telemetry/settings.py | 25 ++- codecarbon/emissions_tracker.py | 1 + docs/how-to/telemetry.md | 4 +- scripts/seed_telemetry_local.py | 86 +++++++++ shared/telemetry_defaults.py | 5 + tests/cli/test_monitor.py | 39 +++++ tests/test_telemetry_payload_roundtrip.py | 165 ++++++++++++++++++ tests/test_telemetry_settings.py | 30 ++++ 15 files changed, 714 insertions(+), 17 deletions(-) create mode 120000 carbonserver/carbonserver/telemetry_defaults.py create mode 100644 carbonserver/tests/api/integration/test_telemetry_local_api.py create mode 100644 carbonserver/tests/api/test_telemetry_shared_defaults.py create mode 120000 codecarbon/core/telemetry/defaults.py create mode 100644 scripts/seed_telemetry_local.py create mode 100644 shared/telemetry_defaults.py create mode 100644 tests/test_telemetry_payload_roundtrip.py diff --git a/carbonserver/carbonserver/api/routers/telemetry.py b/carbonserver/carbonserver/api/routers/telemetry.py index a2e4da796..565d9688a 100644 --- a/carbonserver/carbonserver/api/routers/telemetry.py +++ b/carbonserver/carbonserver/api/routers/telemetry.py @@ -3,11 +3,14 @@ from uuid import UUID from dependency_injector.wiring import Provide, inject -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Header from starlette import status +from carbonserver.api.schemas import AccessLevel from carbonserver.api.schemas_telemetry import TelemetryCreate +from carbonserver.api.services.project_token_service import ProjectTokenService from carbonserver.api.services.telemetry_service import TelemetryService +from carbonserver.config import settings from carbonserver.container import ServerContainer TELEMETRY_ROUTER_TAGS = ["Telemetry"] @@ -27,5 +30,14 @@ def add_telemetry( telemetry_service: TelemetryService = Depends( Provide[ServerContainer.telemetry_service] ), + project_token_service: ProjectTokenService = Depends( + Provide[ServerContainer.project_token_service] + ), + x_api_token: str = Header(None), ) -> UUID: + project_token_service.project_token_has_access( + AccessLevel.WRITE.value, + experiment_id=settings.telemetry_experiment_id, + project_token=x_api_token, + ) return telemetry_service.add_telemetry(telemetry) diff --git a/carbonserver/carbonserver/config.py b/carbonserver/carbonserver/config.py index da6f2a95c..9bc95cc7b 100644 --- a/carbonserver/carbonserver/config.py +++ b/carbonserver/carbonserver/config.py @@ -1,6 +1,8 @@ from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, SettingsConfigDict +from carbonserver.telemetry_defaults import DEFAULT_TELEMETRY_EXPERIMENT_ID + class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="", extra="ignore") @@ -42,6 +44,12 @@ class Settings(BaseSettings): server_host: str = Field( "0.0.0.0", validation_alias=AliasChoices("SERVER_HOST", "server_host") ) + telemetry_experiment_id: str = Field( + DEFAULT_TELEMETRY_EXPERIMENT_ID, + validation_alias=AliasChoices( + "TELEMETRY_EXPERIMENT_ID", "telemetry_experiment_id" + ), + ) settings = Settings() diff --git a/carbonserver/carbonserver/telemetry_defaults.py b/carbonserver/carbonserver/telemetry_defaults.py new file mode 120000 index 000000000..518dfe9b2 --- /dev/null +++ b/carbonserver/carbonserver/telemetry_defaults.py @@ -0,0 +1 @@ +../../shared/telemetry_defaults.py \ No newline at end of file diff --git a/carbonserver/tests/api/integration/test_telemetry_local_api.py b/carbonserver/tests/api/integration/test_telemetry_local_api.py new file mode 100644 index 000000000..28d8c822d --- /dev/null +++ b/carbonserver/tests/api/integration/test_telemetry_local_api.py @@ -0,0 +1,164 @@ +"""Integration tests for telemetry against a running local carbonserver API.""" + +import os +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import pytest +import requests + +REPO_ROOT = Path(__file__).resolve().parents[4] +sys.path.insert(0, str(REPO_ROOT)) + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.defaults import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_EXPERIMENT_ID, +) +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.output_methods.emissions_data import EmissionsData + +URL = os.getenv("CODECARBON_API_URL") +if URL is None: + pytest.exit("CODECARBON_API_URL is not defined (e.g. http://localhost:8008)") + + +def _api_url(path: str) -> str: + base = URL.rstrip("/") + if not base.endswith("/api"): + base = f"{base}/api" + return f"{base}{path}" + + +def _local_settings() -> TelemetrySettings: + return TelemetrySettings.resolve( + external_conf={ + "telemetry_level": "extensive", + "telemetry_api_url": URL.rstrip("/"), + "telemetry_api_key": DEFAULT_TELEMETRY_API_KEY, + "telemetry_experiment_id": DEFAULT_TELEMETRY_EXPERIMENT_ID, + } + ) + + +def _sample_emissions() -> EmissionsData: + return EmissionsData( + timestamp="2026-01-01T00:00:00", + project_name="telemetry-local", + run_id="local-run", + experiment_id=DEFAULT_TELEMETRY_EXPERIMENT_ID, + duration=10.0, + emissions=0.001, + emissions_rate=0.0001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.12", + codecarbon_version="3.2.8", + cpu_count=4, + cpu_model="test-cpu", + gpu_count=0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=16.0, + tracking_mode="process", + ) + + +def test_local_api_is_up(): + response = requests.get(URL.rstrip("/") + "/", timeout=5) + assert response.status_code == 200 + assert response.json()["status"] == "OK" + + +def test_local_telemetry_post_accepts_sdk_payload(): + settings = _local_settings() + payload = build_payload( + TelemetryContext( + conf={ + "os": "Linux-5.10.0-x86_64", + "codecarbon_version": "3.2.8", + "cpu_count": 4, + "python_version": "3.12", + "tracking_mode": "process", + }, + emissions=_sample_emissions(), + hardware=[], + resource_tracker=None, + output_methods=[], + tasks={}, + measure_power_secs=15, + integration="library", + ), + level=TelemetryLevel.minimal, + ) + assert post_private(settings, payload) is True + + +def test_local_extensive_run_and_emission_flow(): + settings = _local_settings() + run_payload = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "experiment_id": settings.experiment_id, + "os": "Linux", + "python_version": "3.12", + "codecarbon_version": "3.2.8", + "cpu_count": 4, + "tracking_mode": "process", + } + run_response = requests.post( + _api_url("/runs"), + json=run_payload, + headers={"x-api-token": settings.api_key}, + timeout=10, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + emission_payload = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "run_id": run_id, + "duration": 10, + "emissions_sum": 0.001, + "emissions_rate": 0.0001, + "cpu_power": 0.0, + "gpu_power": 0.0, + "ram_power": 0.0, + "cpu_energy": 0.0, + "gpu_energy": 0.0, + "ram_energy": 0.0, + "energy_consumed": 0.01, + "wue": 0, + } + emission_response = requests.post( + _api_url("/emissions"), + json=emission_payload, + headers={"x-api-token": settings.api_key}, + timeout=10, + ) + assert emission_response.status_code == 201, emission_response.text + assert uuid.UUID(emission_response.json()) + + +def test_local_post_public_summary_helper(): + settings = _local_settings() + assert ( + post_public_summary(settings, {"os": "Linux", "tracking_mode": "process"}, _sample_emissions()) + is True + ) diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index a405b59e8..90fa3a0a2 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -6,10 +6,15 @@ from fastapi.testclient import TestClient from starlette import status +from carbonserver.api.infra.repositories.repository_projects_tokens import ( + SqlAlchemyRepository as ProjectTokenRepository, +) from carbonserver.api.infra.repositories.repository_telemetry import ( SqlAlchemyRepository as TelemetryRepository, ) from carbonserver.api.routers import telemetry +from carbonserver.api.schemas import AccessLevel, ProjectToken +from carbonserver.config import settings from carbonserver.container import ServerContainer TELEMETRY_ID = "f52fe339-164d-4c2b-a8c0-f562dfce066d" @@ -44,23 +49,77 @@ def client(custom_test_server): def test_add_telemetry(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) - - with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=MINIMAL_TELEMETRY_TO_CREATE) + project_tokens_repository_mock = mock.Mock(spec=ProjectTokenRepository) + project_tokens_repository_mock.get_project_token_by_experiment_id_and_token.return_value = ProjectToken( + id=UUID("e60afb92-17b7-4720-91a0-1ae91e409ba7"), + project_id=UUID("f52fe339-164d-4c2b-a8c0-f562dfce066d"), + name="Telemetry", + token="token", + access=AccessLevel.WRITE.value, + ) + + with custom_test_server.container.telemetry_repository.override( + repository_mock + ) and custom_test_server.container.project_token_repository.override( + project_tokens_repository_mock + ): + response = client.post( + "/telemetry", + json=MINIMAL_TELEMETRY_TO_CREATE, + headers={"x-api-token": "token"}, + ) assert response.status_code == status.HTTP_201_CREATED assert response.json() == TELEMETRY_ID + project_tokens_repository_mock.get_project_token_by_experiment_id_and_token.assert_called_once_with( + settings.telemetry_experiment_id, "token" + ) + + +def test_add_telemetry_rejects_missing_token(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + project_tokens_repository_mock = mock.Mock(spec=ProjectTokenRepository) + + with custom_test_server.container.telemetry_repository.override( + repository_mock + ) and custom_test_server.container.project_token_repository.override( + project_tokens_repository_mock + ): + response = client.post("/telemetry", json=MINIMAL_TELEMETRY_TO_CREATE) + + assert response.status_code == status.HTTP_403_FORBIDDEN + assert response.json() == { + "detail": "Not allowed to perform this action. Missing project token" + } + repository_mock.add_telemetry.assert_not_called() + project_tokens_repository_mock.get_project_token_by_experiment_id_and_token.assert_not_called() def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) + project_tokens_repository_mock = mock.Mock(spec=ProjectTokenRepository) + project_tokens_repository_mock.get_project_token_by_experiment_id_and_token.return_value = ProjectToken( + id=UUID("e60afb92-17b7-4720-91a0-1ae91e409ba7"), + project_id=UUID("f52fe339-164d-4c2b-a8c0-f562dfce066d"), + name="Telemetry", + token="token", + access=AccessLevel.WRITE.value, + ) telemetry_with_extensive_field = { **MINIMAL_TELEMETRY_TO_CREATE, "total_emissions_kg": 0.42, } - with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_extensive_field) + with custom_test_server.container.telemetry_repository.override( + repository_mock + ) and custom_test_server.container.project_token_repository.override( + project_tokens_repository_mock + ): + response = client.post( + "/telemetry", + json=telemetry_with_extensive_field, + headers={"x-api-token": "token"}, + ) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() @@ -68,13 +127,29 @@ def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): def test_disabled_telemetry_is_rejected(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) + project_tokens_repository_mock = mock.Mock(spec=ProjectTokenRepository) + project_tokens_repository_mock.get_project_token_by_experiment_id_and_token.return_value = ProjectToken( + id=UUID("e60afb92-17b7-4720-91a0-1ae91e409ba7"), + project_id=UUID("f52fe339-164d-4c2b-a8c0-f562dfce066d"), + name="Telemetry", + token="token", + access=AccessLevel.WRITE.value, + ) disabled_telemetry = { **MINIMAL_TELEMETRY_TO_CREATE, "telemetry_level": "disabled", } - with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=disabled_telemetry) + with custom_test_server.container.telemetry_repository.override( + repository_mock + ) and custom_test_server.container.project_token_repository.override( + project_tokens_repository_mock + ): + response = client.post( + "/telemetry", + json=disabled_telemetry, + headers={"x-api-token": "token"}, + ) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() diff --git a/carbonserver/tests/api/test_telemetry_shared_defaults.py b/carbonserver/tests/api/test_telemetry_shared_defaults.py new file mode 100644 index 000000000..274ad65df --- /dev/null +++ b/carbonserver/tests/api/test_telemetry_shared_defaults.py @@ -0,0 +1,99 @@ +"""Verify carbonserver uses the shared telemetry token defaults.""" + +from pathlib import Path +from unittest import mock +from uuid import UUID + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from starlette import status + +from carbonserver.api.infra.repositories.repository_projects_tokens import ( + SqlAlchemyRepository as ProjectTokenRepository, +) +from carbonserver.api.infra.repositories.repository_telemetry import ( + SqlAlchemyRepository as TelemetryRepository, +) +from carbonserver.api.routers import telemetry +from carbonserver.api.schemas import AccessLevel, ProjectToken +from carbonserver.config import settings +from carbonserver.container import ServerContainer +from carbonserver.telemetry_defaults import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_EXPERIMENT_ID, +) + +REPO_ROOT = Path(__file__).resolve().parents[3] +TELEMETRY_ID = UUID("f52fe339-164d-4c2b-a8c0-f562dfce066d") + +MINIMAL_TELEMETRY_TO_CREATE = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + "os": "Linux-5.10.0-x86_64", + "country_name": "France", + "country_iso_code": "FRA", + "cpu_count": 12, + "python_version": "3.11.5", + "codecarbon_version": "3.2.6", +} + + +@pytest.fixture +def custom_test_server(): + container = ServerContainer() + container.wire(modules=[telemetry]) + app = FastAPI() + app.container = container + app.include_router(telemetry.router) + yield app + + +@pytest.fixture +def client(custom_test_server): + yield TestClient(custom_test_server) + + +def test_shared_defaults_are_symlinked(): + client_defaults_path = REPO_ROOT / "codecarbon" / "core" / "telemetry" / "defaults.py" + server_defaults_path = REPO_ROOT / "carbonserver" / "carbonserver" / "telemetry_defaults.py" + shared_defaults_path = REPO_ROOT / "shared" / "telemetry_defaults.py" + + assert client_defaults_path.is_symlink() + assert server_defaults_path.is_symlink() + assert client_defaults_path.resolve() == shared_defaults_path.resolve() + assert server_defaults_path.resolve() == shared_defaults_path.resolve() + assert settings.telemetry_experiment_id == DEFAULT_TELEMETRY_EXPERIMENT_ID + + +def test_add_telemetry_accepts_shared_default_token(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + repository_mock.add_telemetry.return_value = TELEMETRY_ID + project_tokens_repository_mock = mock.Mock(spec=ProjectTokenRepository) + project_tokens_repository_mock.get_project_token_by_experiment_id_and_token.return_value = ProjectToken( + id=UUID("e60afb92-17b7-4720-91a0-1ae91e409ba7"), + project_id=UUID("f52fe339-164d-4c2b-a8c0-f562dfce066d"), + name="Telemetry", + token=DEFAULT_TELEMETRY_API_KEY, + access=AccessLevel.WRITE.value, + ) + + with custom_test_server.container.telemetry_repository.override( + repository_mock + ) and custom_test_server.container.project_token_repository.override( + project_tokens_repository_mock + ): + response = client.post( + "/telemetry", + json=MINIMAL_TELEMETRY_TO_CREATE, + headers={"x-api-token": DEFAULT_TELEMETRY_API_KEY}, + ) + + assert response.status_code == status.HTTP_201_CREATED + project_tokens_repository_mock.get_project_token_by_experiment_id_and_token.assert_called_once_with( + DEFAULT_TELEMETRY_EXPERIMENT_ID, + DEFAULT_TELEMETRY_API_KEY, + ) + saved_telemetry = repository_mock.add_telemetry.call_args.args[0] + assert saved_telemetry.os == MINIMAL_TELEMETRY_TO_CREATE["os"] + assert saved_telemetry.country_name == MINIMAL_TELEMETRY_TO_CREATE["country_name"] diff --git a/codecarbon/core/telemetry/defaults.py b/codecarbon/core/telemetry/defaults.py new file mode 120000 index 000000000..1e6888f5d --- /dev/null +++ b/codecarbon/core/telemetry/defaults.py @@ -0,0 +1 @@ +../../../shared/telemetry_defaults.py \ No newline at end of file diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py index 33ea6d125..8cd81c13c 100644 --- a/codecarbon/core/telemetry/settings.py +++ b/codecarbon/core/telemetry/settings.py @@ -6,14 +6,27 @@ from dataclasses import dataclass from typing import Any, Literal +from codecarbon.core.telemetry.defaults import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, +) from codecarbon.core.telemetry.schemas import TelemetryLevel from codecarbon.external.logger import logger -DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" -DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" -DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal +__all__ = [ + "DEFAULT_TELEMETRY_API_KEY", + "DEFAULT_TELEMETRY_API_URL", + "DEFAULT_TELEMETRY_EXPERIMENT_ID", + "DEFAULT_TELEMETRY_LEVEL", + "TELEMETRY_LEVEL_CONFIG_KEY", + "TelemetryLevelSource", + "TelemetrySettings", + "parse_telemetry_level", +] + TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" TelemetryLevelSource = Literal["override", "external", "file", "default"] @@ -106,10 +119,8 @@ def _resolve_api_url(external_conf: dict[str, Any]) -> str: @staticmethod def _resolve_api_key(external_conf: dict[str, Any]) -> str: - key = ( - external_conf.get("telemetry_api_key") - or external_conf.get("api_key") - or os.environ.get("CODECARBON_TELEMETRY_API_KEY") + key = external_conf.get("telemetry_api_key") or os.environ.get( + "CODECARBON_TELEMETRY_API_KEY" ) return key or DEFAULT_TELEMETRY_API_KEY diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 94c6d0110..ede23f173 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -505,6 +505,7 @@ def __init__( CPU TDP and may be unreliable on older systems. :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). Overrides config file and ``CODECARBON_TELEMETRY_LEVEL`` when set. + Defaults to ``minimal``. """ self._external_conf = get_hierarchical_config() diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index e0246a735..23b728b57 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -17,7 +17,7 @@ You can use one without the other. | `telemetry_level` | Name | When | Transport | |-------------------|------|------|-----------| | `disabled` | — | — | Nothing | -| `minimal` | Private product telemetry | Each `stop()` | `POST /telemetry` (private) | +| `minimal` (default) | Private product telemetry | Each `stop()` | `POST /telemetry` (private) | | `extensive` | Private telemetry + shared run summary | Each `stop()` | Same private `POST /telemetry` **and** `ApiClient` → `/emissions` | Tier is resolved in this order: @@ -52,7 +52,7 @@ Private telemetry does **not** include project names, experiment ids, API keys, ## `extensive` — additional public run summary -**Also** posts a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. +**Also** posts a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), then the built-in defaults. Dashboard `api_key` / `experiment_id` are **not** used for telemetry. ## Never collected diff --git a/scripts/seed_telemetry_local.py b/scripts/seed_telemetry_local.py new file mode 100644 index 000000000..382c0f595 --- /dev/null +++ b/scripts/seed_telemetry_local.py @@ -0,0 +1,86 @@ +"""Seed local Postgres with the shared telemetry project, experiment, and API token.""" + +from __future__ import annotations + +import sys +import uuid +from datetime import datetime +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +CARBONSERVER_ROOT = REPO_ROOT / "carbonserver" +sys.path[:0] = [str(CARBONSERVER_ROOT), str(REPO_ROOT)] + +from carbonserver.api.infra.api_key_utils import generate_lookup_value, get_api_key_hash +from carbonserver.api.infra.database.sql_models import ( + Experiment, + Organization, + Project, + ProjectToken, +) +from carbonserver.api.schemas import AccessLevel +from carbonserver.database.database import SessionLocal +from carbonserver.telemetry_defaults import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_EXPERIMENT_ID, +) + +TELEMETRY_ORG_ID = uuid.UUID("11111111-1111-1111-1111-111111111111") +TELEMETRY_PROJECT_ID = uuid.UUID("22222222-2222-2222-2222-222222222222") +TELEMETRY_TOKEN_ID = uuid.UUID("33333333-3333-3333-3333-333333333333") + + +def seed_telemetry_local() -> None: + """Insert telemetry org/project/experiment/token if missing.""" + experiment_id = uuid.UUID(DEFAULT_TELEMETRY_EXPERIMENT_ID) + hashed_token = get_api_key_hash(DEFAULT_TELEMETRY_API_KEY) + if isinstance(hashed_token, bytes): + hashed_token = hashed_token.decode() + + with SessionLocal() as session: + if session.get(Experiment, experiment_id) is not None: + print("Telemetry experiment already seeded.") + return + + session.merge( + Organization( + id=TELEMETRY_ORG_ID, + name="CodeCarbon Telemetry", + description="Local telemetry seed data", + ) + ) + session.merge( + Project( + id=TELEMETRY_PROJECT_ID, + name="Telemetry", + description="Product telemetry", + public=True, + organization_id=TELEMETRY_ORG_ID, + ) + ) + session.merge( + Experiment( + id=experiment_id, + timestamp=datetime.utcnow(), + name="Telemetry", + description="Product telemetry experiment", + project_id=TELEMETRY_PROJECT_ID, + ) + ) + session.merge( + ProjectToken( + id=TELEMETRY_TOKEN_ID, + project_id=TELEMETRY_PROJECT_ID, + name="Telemetry default token", + hashed_token=hashed_token, + lookup_value=generate_lookup_value(DEFAULT_TELEMETRY_API_KEY), + revoked=False, + access=AccessLevel.READ_WRITE.value, + ) + ) + session.commit() + print("Seeded telemetry org/project/experiment/token for local API testing.") + + +if __name__ == "__main__": + seed_telemetry_local() diff --git a/shared/telemetry_defaults.py b/shared/telemetry_defaults.py new file mode 100644 index 000000000..5d02db1da --- /dev/null +++ b/shared/telemetry_defaults.py @@ -0,0 +1,5 @@ +"""Shared defaults for CodeCarbon product telemetry (client + server).""" + +DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" +DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" diff --git a/tests/cli/test_monitor.py b/tests/cli/test_monitor.py index d4dd718a2..49e63557b 100644 --- a/tests/cli/test_monitor.py +++ b/tests/cli/test_monitor.py @@ -2,7 +2,9 @@ import pytest import typer +from typer.testing import CliRunner +from codecarbon.cli import main as cli_main from codecarbon.cli import monitor as monitor_module @@ -150,3 +152,40 @@ def kill(self): assert exc_info.value.exit_code == 130 assert process_info["terminated"] == 1 assert process_info["killed"] == 1 + + +def test_run_and_monitor_passes_telemetry_level_override(monkeypatch): + captured = {} + + class CapturingTracker(FakeTracker): + def __init__(self, **kwargs): + captured.update(kwargs) + super().__init__() + + class FakePopen: + def __init__(self, command, text=True): + pass + + def wait(self): + return 0 + + monkeypatch.setattr(monitor_module, "EmissionsTracker", CapturingTracker) + monkeypatch.setattr(monitor_module.subprocess, "Popen", FakePopen) + monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) + + with pytest.raises(typer.Exit) as exc_info: + monitor_module.run_and_monitor( + SimpleNamespace(args=["echo", "ok"]), + telemetry_level="disabled", + ) + + assert exc_info.value.exit_code == 0 + assert captured.get("telemetry_level") == "disabled" + + +def test_monitor_cli_rejects_invalid_telemetry_level(): + result = CliRunner().invoke( + cli_main.codecarbon, + ["monitor", "--telemetry-level", "invalid"], + ) + assert result.exit_code != 0 diff --git a/tests/test_telemetry_payload_roundtrip.py b/tests/test_telemetry_payload_roundtrip.py new file mode 100644 index 000000000..ad380ff87 --- /dev/null +++ b/tests/test_telemetry_payload_roundtrip.py @@ -0,0 +1,165 @@ +"""Verify SDK telemetry payloads match the server schema and shared defaults.""" + +import importlib.util +import unittest +from copy import deepcopy +from pathlib import Path +from unittest.mock import patch + +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.defaults import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_EXPERIMENT_ID, +) +from codecarbon.core.telemetry.schemas import TelemetryCreate as CoreTelemetryCreate +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.core.telemetry.client import post_private +from codecarbon.output_methods.emissions_data import EmissionsData + +REPO_ROOT = Path(__file__).resolve().parents[1] +SERVER_TELEMETRY_SCHEMA_PATH = ( + REPO_ROOT / "carbonserver" / "carbonserver" / "api" / "schemas_telemetry.py" +) + + +def _load_server_telemetry_create(): + spec = importlib.util.spec_from_file_location( + "server_telemetry_schemas", + SERVER_TELEMETRY_SCHEMA_PATH, + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.TelemetryCreate + + +def _sample_emissions() -> EmissionsData: + return EmissionsData( + timestamp="2026-01-01T00:00:00", + project_name="p", + run_id="r", + experiment_id="e", + duration=10.0, + emissions=0.5, + emissions_rate=0.05, + cpu_power=1.0, + gpu_power=2.0, + ram_power=0.5, + cpu_energy=0.01, + gpu_energy=0.02, + ram_energy=0.001, + energy_consumed=0.031, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux-5.10.0-x86_64", + python_version="3.11.5", + codecarbon_version="3.2.6", + cpu_count=12, + cpu_model="Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz", + gpu_count=0, + gpu_model="", + longitude=2.3, + latitude=48.8, + ram_total_size=16.0, + tracking_mode="machine", + ) + + +def _tracker_context(**overrides) -> TelemetryContext: + conf = overrides.pop( + "conf", + { + "os": "Linux-5.10.0-x86_64", + "codecarbon_version": "3.2.6", + "cpu_count": 12, + "cpu_model": "Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz", + "python_version": "3.11.5", + "tracking_mode": "machine", + }, + ) + return TelemetryContext( + conf=conf, + emissions=overrides.pop("emissions", _sample_emissions()), + hardware=overrides.pop("hardware", []), + resource_tracker=overrides.pop("resource_tracker", None), + output_methods=overrides.pop("output_methods", []), + tasks=overrides.pop("tasks", {}), + measure_power_secs=overrides.pop("measure_power_secs", 15), + integration=overrides.pop("integration", "library"), + ) + + +def _sdk_request_body(level: TelemetryLevel = TelemetryLevel.minimal) -> dict: + payload = build_payload(_tracker_context(), level=level) + return CoreTelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) + + +class TestTelemetryPayloadContract(unittest.TestCase): + def test_shared_defaults_are_symlinked(self): + client_defaults_path = REPO_ROOT / "codecarbon" / "core" / "telemetry" / "defaults.py" + server_defaults_path = ( + REPO_ROOT / "carbonserver" / "carbonserver" / "telemetry_defaults.py" + ) + shared_defaults_path = REPO_ROOT / "shared" / "telemetry_defaults.py" + + self.assertTrue(client_defaults_path.is_symlink()) + self.assertTrue(server_defaults_path.is_symlink()) + self.assertEqual(client_defaults_path.resolve(), shared_defaults_path.resolve()) + self.assertEqual(server_defaults_path.resolve(), shared_defaults_path.resolve()) + + def test_default_settings_use_shared_token_and_experiment(self): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_key, DEFAULT_TELEMETRY_API_KEY) + self.assertEqual(settings.experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + + def test_minimal_sdk_payload_is_accepted_by_server_schema(self): + ServerTelemetryCreate = _load_server_telemetry_create() + request_body = _sdk_request_body() + + parsed = ServerTelemetryCreate.model_validate(request_body) + + self.assertEqual(parsed.telemetry_level, "minimal") + self.assertEqual(parsed.os, request_body["os"]) + self.assertEqual(parsed.country_name, request_body["country_name"]) + self.assertIsNone(parsed.total_emissions_kg) + + def test_extensive_sdk_payload_is_accepted_by_server_schema(self): + ServerTelemetryCreate = _load_server_telemetry_create() + request_body = _sdk_request_body(level=TelemetryLevel.extensive) + + parsed = ServerTelemetryCreate.model_validate(request_body) + + self.assertEqual(parsed.telemetry_level, "extensive") + self.assertEqual(parsed.total_emissions_kg, request_body["total_emissions_kg"]) + self.assertEqual(parsed.duration_seconds, request_body["duration_seconds"]) + self.assertEqual( + parsed.decorator_vs_context, request_body["decorator_vs_context"] + ) + + def test_post_private_sends_shared_token_and_round_tripped_body(self): + ServerTelemetryCreate = _load_server_telemetry_create() + request_body = _sdk_request_body() + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://test.example"} + ) + + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + self.assertTrue(post_private(settings, deepcopy(request_body))) + + self.assertEqual( + mock_post.call_args.kwargs["headers"]["x-api-token"], + DEFAULT_TELEMETRY_API_KEY, + ) + sent_body = mock_post.call_args.kwargs["json"] + ServerTelemetryCreate.model_validate(sent_body) + self.assertEqual(sent_body["telemetry_level"], "minimal") + self.assertEqual(sent_body["os"], request_body["os"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py index e9a1e09f2..f45e5f3e2 100644 --- a/tests/test_telemetry_settings.py +++ b/tests/test_telemetry_settings.py @@ -169,6 +169,36 @@ def test_experiment_id_uses_public_default_when_unset(self): settings = TelemetrySettings.resolve() self.assertEqual(settings.experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + def test_api_key_from_env(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_KEY": "cpt_env"}, + clear=False, + ): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_key, "cpt_env") + + def test_experiment_id_from_env(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_EXPERIMENT_ID": "00000000-0000-0000-0000-000000000003"}, + clear=False, + ): + settings = TelemetrySettings.resolve() + self.assertEqual( + settings.experiment_id, "00000000-0000-0000-0000-000000000003" + ) + + def test_dashboard_api_credentials_are_not_used_for_telemetry(self): + settings = TelemetrySettings.resolve( + external_conf={ + "api_key": "cpt_user", + "experiment_id": "00000000-0000-0000-0000-000000000002", + } + ) + self.assertEqual(settings.api_key, DEFAULT_TELEMETRY_API_KEY) + self.assertEqual(settings.experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + if __name__ == "__main__": unittest.main()