chore: disable unneeded telemetry logging

This method of sending telemetry was regularly overloading the Splunk
log handler and causing more important logs to be dropped (in particular
DeviceMonitor, which we rely on for alerting us to screen outages). We
have not actually used these log spans to investigate anything in a long
time, so we'll remove them until we may need them again.

Author: digitalcora

config/config.exs

@@ -38,7 +38,7 @@ config :ex_aws, json_codec: Jason
 config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase
 
 config :hackney, mod_metrics: :hackney_telemetry
-config :hackney_telemetry, report_interval: 5_000
+config :hackney_telemetry, report_interval: 10_000
 
 config :screens,
   redirect_http?: true,

lib/screens/alerts/alert.ex

@@ -159,29 +159,27 @@ defmodule Screens.Alerts.Alert do
 
   @callback fetch(options()) :: result()
   def fetch(opts \\ [], get_json_fn \\ &V3Api.get_json/2) do
-    Screens.Telemetry.span([:screens, :alerts, :alert, :fetch], fn ->
-      includes =
-        if Keyword.get(opts, :include_all?, false),
-          do: @all_includes,
-          else: @base_includes
-
-      params =
-        opts
-        |> Enum.flat_map(&format_query_param/1)
-        |> Map.new()
-        |> Map.put("include", Enum.join(includes, ","))
-
-      case get_json_fn.("alerts", params) do
-        {:ok, response} ->
-          {:ok,
-           response
-           |> V3Api.Parser.parse()
-           |> normalize_informed_entities_for_direction_id()}
-
-        _ ->
-          :error
-      end
-    end)
+    includes =
+      if Keyword.get(opts, :include_all?, false),
+        do: @all_includes,
+        else: @base_includes
+
+    params =
+      opts
+      |> Enum.flat_map(&format_query_param/1)
+      |> Map.new()
+      |> Map.put("include", Enum.join(includes, ","))
+
+    case get_json_fn.("alerts", params) do
+      {:ok, response} ->
+        {:ok,
+         response
+         |> V3Api.Parser.parse()
+         |> normalize_informed_entities_for_direction_id()}
+
+      _ ->
+        :error
+    end
   end
 
   @doc """

lib/screens/stops/stop.ex

@@ -83,12 +83,10 @@ defmodule Screens.Stops.Stop do
 
   @callback fetch_stop_name(id()) :: String.t() | nil
   def fetch_stop_name(stop_id) do
-    Screens.Telemetry.span(~w[screens stops stop fetch_stop_name]a, %{stop_id: stop_id}, fn ->
-      case fetch(%{ids: [stop_id]}) do
-        {:ok, [%__MODULE__{name: name}]} -> name
-        _ -> nil
-      end
-    end)
+    case fetch(%{ids: [stop_id]}) do
+      {:ok, [%__MODULE__{name: name}]} -> name
+      _ -> nil
+    end
   end
 
   @spec fetch_subway_platforms_for_stop(id()) :: [t()]

lib/screens/telemetry.ex

@@ -11,31 +11,11 @@ defmodule Screens.Telemetry do
   @impl Supervisor
   def init(_) do
     handlers = [
-      # V3 API
-      log_span(~w[screens v3_api get_json]a, metadata: ~w[path query cache]a),
-      # Stops
-      log_span(~w[screens stops stop fetch_stop_name]a, metadata: ~w[stop_id]),
-      # Location Context
-      log_span(~w[screens location_context fetch]a, metadata: ~w[app stop_id]),
-      # Alerts
-      log_span(~w[screens alerts alert fetch]a),
-      # Routes
-      log_span(~w[screens routes route fetch_routes_by_stop]a,
-        metadata: ~w[stop_id type_filters]a
-      ),
-      # DUP Candidate Generator
-      log_span(~w[screens v2 candidate_generator dup]a),
-      log_span(~w[screens v2 candidate_generator dup departures_instances]a),
-      log_span(~w[screens v2 candidate_generator dup departures get_section_data]a),
-      log_span(~w[screens v2 candidate_generator dup departures get_sections_data]a),
-      log_span(~w[screens v2 candidate_generator dup header_instances]a),
-      log_span(~w[screens v2 candidate_generator dup alerts_instances]a),
-      log_span(~w[screens v2 candidate_generator dup evergreen_content_instances]a),
-      # New DUP Candidate Generator
-      log_span(~w[screens v2 candidate_generator dup_new]a),
-      log_span(~w[screens v2 candidate_generator dup_new departures_instances]a),
-      log_span(~w[screens v2 candidate_generator dup_new evergreen_instances]a),
-      log_span(~w[screens v2 candidate_generator dup_new header_instances]a),
+      # example span to avoid unused function warnings; nothing emits this
+      log_span(~w[screens example_event]a),
+
+      # enable V3 API span logging (WARNING: high log volume)
+      # log_span(~w[screens v3_api get_json]a, metadata: ~w[path query cache]a),
 
       # events
       log_event(~w[hackney_pool]a,
@@ -59,7 +39,8 @@ defmodule Screens.Telemetry do
         measurements: [
           {Screens.V3Api.Cache.Realtime, :dispatch_stats, []},
           {Screens.V3Api.Cache.Static, :dispatch_stats, []}
-        ]
+        ],
+        period: 10_000
       }
     ]
 

lib/screens/v2/candidate_generator/dup.ex

@@ -77,28 +77,16 @@ defmodule Screens.V2.CandidateGenerator.Dup do
         departures_instances_fn \\ &DeparturesInstances.departures_instances/2,
         alerts_instances_fn \\ &AlertsInstances.alert_instances/2
       ) do
-    Screens.Telemetry.span([:screens, :v2, :candidate_generator, :dup], fn ->
-      ctx = Screens.Telemetry.context()
-
-      [
-        span_thunk(:header_instances, ctx, fn -> header_instances_fn.(config, now) end),
-        span_thunk(:alerts_instances, ctx, fn -> alerts_instances_fn.(config, now) end),
-        span_thunk(:departures_instances, ctx, fn -> departures_instances_fn.(config, now) end),
-        span_thunk(:evergreen_content_instances, ctx, fn ->
-          evergreen_content_instances_fn.(config, now)
-        end)
-      ]
-      |> Task.async_stream(& &1.(), timeout: 30_000)
-      |> Enum.flat_map(fn {:ok, instances} -> instances end)
-    end)
+    [
+      fn -> header_instances_fn.(config, now) end,
+      fn -> alerts_instances_fn.(config, now) end,
+      fn -> departures_instances_fn.(config, now) end,
+      fn -> evergreen_content_instances_fn.(config, now) end
+    ]
+    |> Task.async_stream(& &1.(), timeout: 30_000)
+    |> Enum.flat_map(fn {:ok, instances} -> instances end)
   end
 
   @impl CandidateGenerator
   def audio_only_instances(_widgets, _config), do: []
-
-  defp span_thunk(name, meta, fun) when is_atom(name) and is_function(fun, 0) do
-    fn ->
-      Screens.Telemetry.span([:screens, :v2, :candidate_generator, :dup, name], meta, fun)
-    end
-  end
 end

lib/screens/v2/candidate_generator/dup/departures.ex

@@ -294,87 +294,61 @@ defmodule Screens.V2.CandidateGenerator.Dup.Departures do
          fetch_routes_fn,
          now
        ) do
-    Screens.Telemetry.span(
-      [:screens, :v2, :candidate_generator, :dup, :departures, :get_sections_data],
-      fn ->
-        ctx = Screens.Telemetry.context()
-
-        sections
-        |> Task.async_stream(
-          &get_section_data(
-            &1,
-            fetch_departures_fn,
-            fetch_alerts_fn,
-            fetch_routes_fn,
-            now,
-            ctx
-          ),
-          on_timeout: :kill_task
-        )
-        |> Enum.map(fn
-          {:ok, data} ->
-            data
-
-          {:exit, reason} ->
-            ctx =
-              Screens.Telemetry.context()
-              |> to_log()
-
-            Logger.error(["event=get_section_data.exit reason=#{reason} ", ctx])
-            raise "Failed to get section data"
-        end)
-      end
+    sections
+    |> Task.async_stream(
+      &get_section_data(&1, fetch_departures_fn, fetch_alerts_fn, fetch_routes_fn, now),
+      on_timeout: :kill_task
     )
+    |> Enum.map(fn
+      {:ok, data} ->
+        data
+
+      {:exit, reason} ->
+        Logger.error("get_section_data.exit reason=#{reason}")
+        raise "Failed to get section data"
+    end)
   end
 
   defp get_section_data(
          %Section{query: %Query{params: %Params{stop_ids: stop_ids} = params}} = section,
          fetch_departures_fn,
          fetch_alerts_fn,
          fetch_routes_fn,
-         now,
-         ctx
+         now
        ) do
-    Screens.Telemetry.span(
-      [:screens, :v2, :candidate_generator, :dup, :departures, :get_section_data],
-      ctx,
-      fn ->
-        routes = get_routes_serving_section(params, fetch_routes_fn)
-        # DUP sections will always show no more than one mode.
-        # For subway, each route will have its own section.
-        # If the stop is served by two different subway/light rail routes, route_ids must be populated for each section
-        # Otherwise, we only need the first route in the list of routes serving the stop.
-        primary_route_for_section = List.first(routes)
-
-        disabled_modes = Screens.Config.Cache.disabled_modes()
-
-        # If we know the predictions are unreliable, don't even bother fetching them.
-        if is_nil(primary_route_for_section) or
-             primary_route_for_section.type in disabled_modes do
-          %{type: :no_data_section, route: primary_route_for_section}
-        else
-          section_departures =
-            case Widgets.Departures.fetch_section_departures(
-                   section,
-                   disabled_modes,
-                   fetch_departures_fn
-                 ) do
-              {:ok, departures} -> departures
-              :error -> []
-            end
+    routes = get_routes_serving_section(params, fetch_routes_fn)
+    # DUP sections will always show no more than one mode.
+    # For subway, each route will have its own section.
+    # If the stop is served by two different subway/light rail routes, route_ids must be populated for each section
+    # Otherwise, we only need the first route in the list of routes serving the stop.
+    primary_route_for_section = List.first(routes)
+
+    disabled_modes = Screens.Config.Cache.disabled_modes()
+
+    # If we know the predictions are unreliable, don't even bother fetching them.
+    if is_nil(primary_route_for_section) or primary_route_for_section.type in disabled_modes do
+      %{type: :no_data_section, route: primary_route_for_section}
+    else
+      section_departures =
+        case Widgets.Departures.fetch_section_departures(
+               section,
+               disabled_modes,
+               fetch_departures_fn
+             ) do
+          {:ok, departures} -> departures
+          :error -> []
+        end
 
-          alert_informed_entities = get_section_entities(params, fetch_alerts_fn, now)
+      alert_informed_entities = get_section_entities(params, fetch_alerts_fn, now)
 
-          %{
-            departures: section_departures,
-            alert_informed_entities: alert_informed_entities,
-            stop_ids: stop_ids,
-            routes: routes,
-            params: params
-          }
-        end
-      end
-    )
+      %{
+        departures: section_departures,
+        alert_informed_entities: alert_informed_entities,
+        stop_ids: stop_ids,
+        routes: routes,
+        params: params
+      }
+    end
   end
 
   defp get_section_route_from_entities(
@@ -742,10 +716,6 @@ defmodule Screens.V2.CandidateGenerator.Dup.Departures do
     |> Enum.uniq()
   end
 
-  defp to_log(map) do
-    Enum.map_join(map, " ", fn {k, v} -> "#{k}=#{v}" end)
-  end
-
   defp headway_time_range(stop_ids, routes, now) do
     all_headways =
       for stop_id <- stop_ids, %{id: route_id} <- routes do

lib/screens/v2/candidate_generator/dup_new.ex

@@ -1,36 +1,27 @@
 defmodule Screens.V2.CandidateGenerator.DupNew do
   @moduledoc false
 
-  alias Screens.Telemetry
   alias Screens.V2.CandidateGenerator
   alias Screens.V2.CandidateGenerator.Widgets
 
   alias __MODULE__.Departures
 
   @behaviour CandidateGenerator
 
-  @telemetry_name ~w[screens v2 candidate_generator dup_new]a
   @instance_generators [
-                         header_instances: &Widgets.Header.instances/2,
-                         departures_instances: &Departures.instances/2,
-                         evergreen_instances: &Widgets.Evergreen.evergreen_content_instances/2
-                       ]
-                       |> Enum.map(fn {name, func} -> {@telemetry_name ++ [name], func} end)
+    &Widgets.Header.instances/2,
+    &Departures.instances/2,
+    &Widgets.Evergreen.evergreen_content_instances/2
+  ]
 
   @impl CandidateGenerator
   defdelegate screen_template(screen), to: Screens.V2.CandidateGenerator.Dup
 
   @impl CandidateGenerator
   def candidate_instances(config, now \\ DateTime.utc_now()) do
-    Telemetry.span(@telemetry_name, fn ->
-      context = Telemetry.context()
-
-      @instance_generators
-      |> Task.async_stream(fn {name, func} ->
-        Telemetry.span(name, context, fn -> func.(config, now) end)
-      end)
-      |> Enum.flat_map(fn {:ok, instances} -> instances end)
-    end)
+    @instance_generators
+    |> Task.async_stream(& &1.(config, now))
+    |> Enum.flat_map(fn {:ok, instances} -> instances end)
   end
 
   @impl CandidateGenerator