fix(severity): Add option to route severity requests to GPU (#95992)

as part of [migration
plan](https://www.notion.so/sentry/Moving-severity-and-fixability-services-to-GPU-2348b10e4b5d80688a64e3235a653744?source=copy_link#2348b10e4b5d800aa55dccb0695ed2ba)
to serve severity on `group-ingest` pods

need to wait for getsentry/ops#16321 before
increasing option value from 0

Author: kddubey

src/sentry/event_manager.py

@@ -108,6 +108,7 @@
 from sentry.models.releaseprojectenvironment import ReleaseProjectEnvironment
 from sentry.models.releases.release_project import ReleaseProject
 from sentry.net.http import connection_from_url
+from sentry.options.rollout import in_random_rollout
 from sentry.performance_issues.performance_detection import detect_performance_problems
 from sentry.performance_issues.performance_problem import PerformanceProblem
 from sentry.plugins.base import plugins
@@ -1947,6 +1948,12 @@ def _process_existing_aggregate(
     timeout=settings.SEER_SEVERITY_TIMEOUT,  # Defaults to 300 milliseconds
 )
 
+severity_connection_pool_gpu = connection_from_url(
+    settings.SEER_GROUPING_URL,
+    retries=settings.SEER_SEVERITY_RETRIES,
+    timeout=settings.SEER_SEVERITY_TIMEOUT,  # Defaults to 300 milliseconds
+)
+
 
 def _get_severity_metadata_for_group(
     event: Event, project_id: int, group_type: int | None
@@ -2168,8 +2175,14 @@ def _get_severity_score(event: Event) -> tuple[float, str]:
                     "issues.severity.seer-timout",
                     settings.SEER_SEVERITY_TIMEOUT / 1000,
                 )
+
+                if in_random_rollout("issues.severity.gpu-rollout-rate"):
+                    connection_pool = severity_connection_pool_gpu
+                else:
+                    connection_pool = severity_connection_pool
+
                 response = make_signed_seer_api_request(
-                    severity_connection_pool,
+                    connection_pool,
                     "/v0/issues/severity-score",
                     body=orjson.dumps(payload),
                     timeout=timeout,

src/sentry/options/defaults.py

@@ -928,6 +928,13 @@
     flags=FLAG_ALLOW_EMPTY | FLAG_AUTOMATOR_MODIFIABLE,
 )
 
+register(
+    "issues.severity.gpu-rollout-rate",
+    type=Float,
+    default=0.0,
+    flags=FLAG_AUTOMATOR_MODIFIABLE,
+)
+
 register(
     "issues.priority.projects-allowlist",
     type=Sequence,