feat(metrics): add opt-in BEAM runtime metrics collector

lib/sentry/application.ex

@@ -45,6 +45,13 @@ defmodule Sentry.Application do
         []
       end
 
+    maybe_runtime_metrics =
+      if Config.metrics()[:runtime][:enabled] do
+        [{Sentry.Metrics.Runtime, Config.metrics()[:runtime]}]
+      else
+        []
+      end
+
     maybe_span_storage =
       if Config.tracing?() do
         [Sentry.OpenTelemetry.SpanStorage]
@@ -79,7 +86,8 @@ defmodule Sentry.Application do
         maybe_span_storage ++
         telemetry_processor ++
         maybe_rate_limiter() ++
-        [Sentry.Transport.SenderPool]
+        [Sentry.Transport.SenderPool] ++
+        maybe_runtime_metrics
 
     cache_loaded_applications()
 

lib/sentry/config.ex

@@ -448,6 +448,55 @@ defmodule Sentry.Config do
       *Available since 13.0.0*.
       """
     ],
+    metrics: [
+      type: :keyword_list,
+      default: [],
+      doc: """
+      Configuration for metrics the SDK collects on its own, without you calling
+      `Sentry.Metrics` yourself. Every collector is opt-in, as required by the
+      [Sentry Metrics Protocol](https://develop.sentry.dev/sdk/telemetry/metrics/).
+      *Available since 14.0.0*.
+      """,
+      keys: [
+        runtime: [
+          type: :keyword_list,
+          default: [],
+          doc: """
+          Configuration for the BEAM runtime metrics collector, which periodically
+          reports scheduler, memory, run queue and system limit measurements.
+          *Available since 14.0.0*.
+          """,
+          keys: [
+            enabled: [
+              type: :boolean,
+              default: false,
+              doc: """
+              Whether to start the runtime metrics collector.
+              *Available since 14.0.0*.
+              """
+            ],
+            interval: [
+              type: :pos_integer,
+              default: 30_000,
+              doc: """
+              How often, in milliseconds, to collect and report runtime measurements.
+              Values below `1000` are raised to `1000`.
+              *Available since 14.0.0*.
+              """
+            ],
+            version_attributes: [
+              type: :boolean,
+              default: true,
+              doc: """
+              Whether to attach `elixir_version` and `otp_release` attributes to every
+              reported measurement, so metrics can be grouped by runtime version.
+              *Available since 14.0.0*.
+              """
+            ]
+          ]
+        ]
+      ]
+    ],
     logs: [
       type: :keyword_list,
       default: [],
@@ -1129,6 +1178,9 @@ defmodule Sentry.Config do
   @spec enable_metrics?() :: boolean()
   def enable_metrics?, do: fetch!(:enable_metrics)
 
+  @spec metrics() :: keyword()
+  def metrics, do: fetch!(:metrics)
+
   @spec logs() :: keyword()
   def logs, do: fetch!(:logs)
 

lib/sentry/metrics.ex

@@ -25,6 +25,26 @@ defmodule Sentry.Metrics do
       # Record a distribution
       Sentry.Metrics.distribution("response.time", 42.5, unit: "millisecond")
 
+  ## Automatically Collected Metrics
+
+  The SDK can also collect BEAM runtime health metrics on its own, without any
+  calls to the functions above. This is opt-in:
+
+      config :sentry,
+        metrics: [
+          runtime: [enabled: true]
+        ]
+
+  Once enabled, a supervised collector reports these gauges every 30 seconds:
+
+    * `elixir.runtime.mem.total`, `elixir.runtime.mem.processes`,
+      `elixir.runtime.mem.binary`, `elixir.runtime.mem.ets`,
+      `elixir.runtime.mem.atom` — in bytes
+
+  Every reported metric carries `elixir_version` and `otp_release` attributes so
+  measurements can be grouped by runtime version. See the `:metrics` option in
+  the `Sentry` module documentation for the full configuration.
+
   ## Configuration
 
   Metrics can be disabled globally via configuration:

lib/sentry/metrics/runtime.ex

@@ -0,0 +1,77 @@
+defmodule Sentry.Metrics.Runtime do
+  @moduledoc false
+
+  use GenServer
+
+  alias Sentry.LoggerUtils
+  alias Sentry.Metrics
+  alias Sentry.TelemetryProcessor
+
+  @memory_gauges [:total, :processes, :binary, :ets, :atom]
+  @min_interval 1_000
+
+  defstruct [:interval, :attributes]
+
+  @spec start_link(keyword()) :: GenServer.on_start()
+  def start_link(opts) when is_list(opts) do
+    GenServer.start_link(__MODULE__, opts, name: Keyword.get(opts, :name, __MODULE__))
+  end
+
+  @impl true
+  def init(opts) do
+    interval = normalize_interval(Keyword.fetch!(opts, :interval))
+    attributes = version_attributes(Keyword.get(opts, :version_attributes, true))
+
+    schedule_tick(interval)
+
+    {:ok, %__MODULE__{interval: interval, attributes: attributes}}
+  end
+
+  @impl true
+  def handle_info(:tick, %__MODULE__{} = state) do
+    collect_and_emit(state)
+    schedule_tick(state.interval)
+    {:noreply, state}
+  end
+
+  defp collect_and_emit(%__MODULE__{} = state) do
+    memory = :erlang.memory()
+
+    Enum.each(@memory_gauges, fn key ->
+      gauge(state, "elixir.runtime.mem.#{key}", Keyword.fetch!(memory, key), unit: "byte")
+    end)
+
+    # A snapshot is a burst of a dozen metrics every interval, far below the
+    # metric buffer's batch size, so without an explicit flush it would sit
+    # buffered until unrelated telemetry happened to signal the scheduler.
+    TelemetryProcessor.flush()
+  end
+
+  defp gauge(%__MODULE__{} = state, name, value, opts) do
+    Metrics.gauge(name, value, Keyword.put(opts, :attributes, state.attributes))
+  end
+
+  defp normalize_interval(interval) when is_integer(interval) and interval >= @min_interval do
+    interval
+  end
+
+  defp normalize_interval(interval) do
+    LoggerUtils.warning(
+      "[Sentry] runtime metrics collection interval of #{inspect(interval)}ms is below the " <>
+        "supported minimum, falling back to #{@min_interval}ms"
+    )
+
+    @min_interval
+  end
+
+  defp version_attributes(false), do: %{}
+
+  defp version_attributes(true) do
+    %{
+      elixir_version: System.version(),
+      otp_release: List.to_string(:erlang.system_info(:otp_release))
+    }
+  end
+
+  defp schedule_tick(interval), do: Process.send_after(self(), :tick, interval)
+end

test/sentry/application_test.exs

@@ -243,6 +243,20 @@ defmodule Sentry.ApplicationTest do
     end
   end
 
+  describe "runtime metrics collector" do
+    test "is not started by default" do
+      restart_sentry_with([])
+
+      refute Process.whereis(Sentry.Metrics.Runtime)
+    end
+
+    test "is started when runtime metrics are enabled" do
+      restart_sentry_with(metrics: [runtime: [enabled: true]])
+
+      assert is_pid(Process.whereis(Sentry.Metrics.Runtime))
+    end
+  end
+
   defp restart_sentry_with(config) do
     Application.stop(:sentry)
 

test/sentry/metrics/runtime_test.exs

@@ -0,0 +1,101 @@
+defmodule Sentry.Metrics.RuntimeTest do
+  use Sentry.Case, async: false
+
+  import ExUnit.CaptureLog
+  import Sentry.TestHelpers
+
+  alias Sentry.Metrics.Runtime
+
+  setup do
+    %{ref: ref} = Sentry.Test.setup_sentry(collect_envelopes: true)
+    %{ref: ref}
+  end
+
+  describe "memory metrics" do
+    test "reports memory gauges without any user code recording metrics", %{ref: ref} do
+      collect_once()
+
+      names = Enum.map(snapshot(ref), & &1["name"])
+
+      assert "elixir.runtime.mem.total" in names
+      assert "elixir.runtime.mem.processes" in names
+      assert "elixir.runtime.mem.binary" in names
+      assert "elixir.runtime.mem.ets" in names
+      assert "elixir.runtime.mem.atom" in names
+    end
+
+    test "reports memory in bytes", %{ref: ref} do
+      collect_once()
+
+      assert metric = find_metric(ref, "elixir.runtime.mem.total")
+      assert metric["unit"] == "byte"
+      assert metric["type"] == "gauge"
+      assert metric["value"] > 0
+    end
+  end
+
+  describe "version attributes" do
+    test "attaches the Elixir and OTP versions to every metric", %{ref: ref} do
+      collect_once()
+
+      for metric <- snapshot(ref) do
+        assert metric["attributes"]["elixir_version"]["value"] == System.version()
+
+        assert metric["attributes"]["otp_release"]["value"] ==
+                 List.to_string(:erlang.system_info(:otp_release))
+      end
+    end
+
+    test "omits the version attributes when they are disabled", %{ref: ref} do
+      collect_once(version_attributes: false)
+
+      for metric <- snapshot(ref) do
+        refute Map.has_key?(metric["attributes"], "elixir_version")
+        refute Map.has_key?(metric["attributes"], "otp_release")
+      end
+    end
+  end
+
+  describe "delivery" do
+    test "delivers a whole snapshot from a single collection", %{ref: ref} do
+      collect_once()
+
+      assert length(snapshot(ref)) == 5
+    end
+  end
+
+  describe "collection interval" do
+    test "clamps an interval below the supported minimum" do
+      log =
+        capture_log(fn ->
+          name = :"test_runtime_metrics_#{System.unique_integer([:positive])}"
+          start_supervised!({Runtime, name: name, interval: 100}, id: name)
+        end)
+
+      assert log =~ "collection interval"
+      assert log =~ "1000"
+    end
+  end
+
+  defp collect_once(opts \\ []) do
+    name = :"test_runtime_metrics_#{System.unique_integer([:positive])}"
+    opts = Keyword.merge([name: name, interval: :timer.hours(1)], opts)
+
+    pid = start_supervised!({Runtime, opts}, id: name)
+    Sentry.Test.allow_sentry_reports(self(), pid)
+
+    send(name, :tick)
+    _ = :sys.get_state(name)
+
+    :ok
+  end
+
+  defp snapshot(ref) do
+    assert [%{"items" => items}] = collect_sentry_metric_items(ref, 1, timeout: 2000)
+    items
+  end
+
+  defp find_metric(ref, name) do
+    Enum.find(snapshot(ref), &(&1["name"] == name))
+  end
+end