fix(): business logic fixes and code refactoring

src/backend/apps/guardrails/services.py

@@ -4,7 +4,7 @@ from typing import Any
 
 from django.core.exceptions import ValidationError
 from django.db import transaction
-from django.db.models import QuerySet
+from django.db.models import Case, QuerySet, Value, When
 from django.utils import timezone
 
 from apps.experiments.models import (
@@ -21,23 +21,38 @@ from apps.guardrails.models import (
     GuardrailAction,
     GuardrailTrigger,
 )
-from apps.metrics.models import MetricDefinition
+from apps.metrics.models import MetricDefinition, MetricDirection
 from apps.notifications.services import (
     NotificationPayload,
     notification_enqueue,
 )
 from apps.reports.services import calculate_metric_value
 
+_ACTION_SEVERITY = {
+    GuardrailAction.ROLLBACK: 0,
+    GuardrailAction.PAUSE: 1,
+}
+
 
 @transaction.atomic
 def guardrail_create(
     *,
     experiment: Experiment,
     metric: MetricDefinition,
-    threshold: Any,
+    threshold: Decimal,
     observation_window_minutes: int = 60,
     action: str = GuardrailAction.PAUSE,
 ) -> Guardrail:
+    if experiment.status in STARTED_STATUSES:
+        raise ValidationError(
+            {
+                "experiment": (
+                    "Guardrails cannot be added after the experiment "
+                    "has been started "
+                    f"(status: '{experiment.status}')."
+                )
+            }
+        )
     guardrail = Guardrail(
         experiment=experiment,
         metric=metric,
@@ -135,9 +150,22 @@ def _calculate_guardrail_metric(
     if not values:
         return None
 
+    direction = guardrail.metric.direction
+    if direction == MetricDirection.HIGHER_IS_BETTER:
+        return min(values)
     return max(values)
 
 
+def _is_threshold_breached(
+    actual_value: Decimal,
+    threshold: Decimal,
+    direction: str,
+) -> bool:
+    if direction == MetricDirection.HIGHER_IS_BETTER:
+        return actual_value < threshold
+    return actual_value > threshold
+
+
 @transaction.atomic
 def _execute_guardrail_action(
     guardrail: Guardrail,
@@ -146,6 +174,12 @@ def _execute_guardrail_action(
 ) -> GuardrailTrigger:
     now = timezone.now()
 
+    experiment = Experiment.objects.select_for_update().get(pk=experiment.pk)
+    if experiment.status != ExperimentStatus.RUNNING:
+        return None
+
+    from_status = experiment.status
+
     trigger = GuardrailTrigger.objects.create(
         guardrail=guardrail,
         experiment=experiment,
@@ -175,7 +209,7 @@ def _execute_guardrail_action(
                 "threshold": str(guardrail.threshold),
                 "actual_value": str(actual_value),
                 "action": guardrail.action,
-                "from_status": ExperimentStatus.RUNNING,
+                "from_status": from_status,
                 "to_status": ExperimentStatus.PAUSED,
             },
         )
@@ -213,7 +247,7 @@ def _execute_guardrail_action(
                 "threshold": str(guardrail.threshold),
                 "actual_value": str(actual_value),
                 "action": guardrail.action,
-                "from_status": ExperimentStatus.RUNNING,
+                "from_status": from_status,
                 "to_status": ExperimentStatus.COMPLETED,
                 "control_variant_id": (
                     str(control_variant.pk) if control_variant else None
@@ -252,10 +286,21 @@ def check_experiment_guardrails(
     if experiment.status != ExperimentStatus.RUNNING:
         return []
 
-    guardrails = Guardrail.objects.filter(
-        experiment=experiment,
-        is_active=True,
-    ).select_related("metric")
+    guardrails = (
+        Guardrail.objects.filter(
+            experiment=experiment,
+            is_active=True,
+        )
+        .select_related("metric")
+        .annotate(
+            action_order=Case(
+                When(action=GuardrailAction.ROLLBACK, then=Value(0)),
+                When(action=GuardrailAction.PAUSE, then=Value(1)),
+                default=Value(2),
+            ),
+        )
+        .order_by("action_order", "-created_at")
+    )
 
     triggers: list[GuardrailTrigger] = []
 
@@ -264,16 +309,19 @@ def check_experiment_guardrails(
         if actual_value is None:
             continue
 
-        if actual_value > guardrail.threshold:
-            experiment.refresh_from_db()
-            if experiment.status != ExperimentStatus.RUNNING:
-                break
-
+        if _is_threshold_breached(
+            actual_value,
+            guardrail.threshold,
+            guardrail.metric.direction,
+        ):
             trigger = _execute_guardrail_action(
                 guardrail,
                 experiment,
                 actual_value,
             )
+            if trigger is None:
+                break
+
             triggers.append(trigger)
 
             if guardrail.action in {