From d87671e49a74db6b2e4ff797f3e0a0bea18c28d9 Mon Sep 17 00:00:00 2001 From: ITQ Date: Mon, 23 Feb 2026 10:54:04 +0300 Subject: [PATCH] feat(experiments): added experiments business logic --- src/backend/apps/experiments/__init__.py | 0 src/backend/apps/experiments/apps.py | 5 + .../experiments/migrations/0001_initial.py | 95 +++ .../experiments/migrations/0002_initial.py | 86 +++ .../apps/experiments/migrations/__init__.py | 0 src/backend/apps/experiments/models.py | 557 +++++++++++++++++ src/backend/apps/experiments/selectors.py | 69 +++ src/backend/apps/experiments/services.py | 495 +++++++++++++++ .../apps/experiments/tests/__init__.py | 0 src/backend/apps/experiments/tests/helpers.py | 65 ++ .../apps/experiments/tests/test_models.py | 179 ++++++ .../apps/experiments/tests/test_services.py | 585 ++++++++++++++++++ 12 files changed, 2136 insertions(+) create mode 100644 src/backend/apps/experiments/__init__.py create mode 100644 src/backend/apps/experiments/apps.py create mode 100644 src/backend/apps/experiments/migrations/0001_initial.py create mode 100644 src/backend/apps/experiments/migrations/0002_initial.py create mode 100644 src/backend/apps/experiments/migrations/__init__.py create mode 100644 src/backend/apps/experiments/models.py create mode 100644 src/backend/apps/experiments/selectors.py create mode 100644 src/backend/apps/experiments/services.py create mode 100644 src/backend/apps/experiments/tests/__init__.py create mode 100644 src/backend/apps/experiments/tests/helpers.py create mode 100644 src/backend/apps/experiments/tests/test_models.py create mode 100644 src/backend/apps/experiments/tests/test_services.py diff --git a/src/backend/apps/experiments/__init__.py b/src/backend/apps/experiments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/apps/experiments/apps.py b/src/backend/apps/experiments/apps.py new file mode 100644 index 0000000..f64a0a9 --- /dev/null +++ b/src/backend/apps/experiments/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ExperimentsConfig(AppConfig): + name = "apps.experiments" diff --git a/src/backend/apps/experiments/migrations/0001_initial.py b/src/backend/apps/experiments/migrations/0001_initial.py new file mode 100644 index 0000000..32cc3d2 --- /dev/null +++ b/src/backend/apps/experiments/migrations/0001_initial.py @@ -0,0 +1,95 @@ +# Generated by Django 5.2.11 on 2026-02-14 09:55 + +import django.core.validators +import uuid +from decimal import Decimal +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Approval', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('comment', models.TextField(blank=True, verbose_name='comment')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ], + options={ + 'verbose_name': 'approval', + 'verbose_name_plural': 'approvals', + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='Experiment', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=200, verbose_name='name')), + ('description', models.TextField(blank=True, verbose_name='description')), + ('hypothesis', models.TextField(blank=True, verbose_name='hypothesis')), + ('status', models.CharField(choices=[('draft', 'Draft'), ('in_review', 'In review'), ('approved', 'Approved'), ('running', 'Running'), ('paused', 'Paused'), ('completed', 'Completed'), ('archived', 'Archived'), ('rejected', 'Rejected')], db_index=True, default='draft', max_length=20, verbose_name='status')), + ('version', models.PositiveIntegerField(default=1, verbose_name='version')), + ('traffic_allocation', models.DecimalField(decimal_places=2, default=Decimal('100.00'), help_text='Percentage of eligible users included in experiment', max_digits=5, validators=[django.core.validators.MinValueValidator(Decimal('0.01')), django.core.validators.MaxValueValidator(Decimal('100.00'))], verbose_name='traffic allocation %')), + ('targeting_rules', models.TextField(blank=True, help_text='DSL expression for targeting', verbose_name='targeting rules')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='updated at')), + ], + options={ + 'verbose_name': 'experiment', + 'verbose_name_plural': 'experiments', + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='ExperimentLog', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('log_type', models.CharField(choices=[('status_change', 'Status change'), ('approval', 'Approval'), ('rejection', 'Rejection'), ('review_requested', 'Review requested'), ('guardrail_triggered', 'Guardrail triggered'), ('completed', 'Completed'), ('comment', 'Comment')], max_length=30, verbose_name='log type')), + ('comment', models.TextField(blank=True, verbose_name='comment')), + ('metadata', models.JSONField(blank=True, default=dict, verbose_name='metadata')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ], + options={ + 'verbose_name': 'experiment log', + 'verbose_name_plural': 'experiment logs', + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='ExperimentOutcome', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('outcome', models.CharField(choices=[('rollout', 'Rollout winner'), ('rollback', 'Rollback'), ('no_effect', 'No effect')], max_length=20, verbose_name='outcome')), + ('rationale', models.TextField(verbose_name='rationale')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ], + options={ + 'verbose_name': 'experiment outcome', + 'verbose_name_plural': 'experiment outcomes', + }, + ), + migrations.CreateModel( + name='Variant', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=100, verbose_name='name')), + ('value', models.CharField(help_text='Value returned when this variant is selected', max_length=500, verbose_name='value')), + ('weight', models.DecimalField(decimal_places=2, max_digits=5, validators=[django.core.validators.MinValueValidator(Decimal('0.00')), django.core.validators.MaxValueValidator(Decimal('100.00'))], verbose_name='weight %')), + ('is_control', models.BooleanField(default=False, verbose_name='is control')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='updated at')), + ], + options={ + 'verbose_name': 'variant', + 'verbose_name_plural': 'variants', + 'ordering': ['name'], + }, + ), + ] diff --git a/src/backend/apps/experiments/migrations/0002_initial.py b/src/backend/apps/experiments/migrations/0002_initial.py new file mode 100644 index 0000000..7941318 --- /dev/null +++ b/src/backend/apps/experiments/migrations/0002_initial.py @@ -0,0 +1,86 @@ +# Generated by Django 5.2.11 on 2026-02-14 09:55 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('experiments', '0001_initial'), + ('flags', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='approval', + name='approver', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='experiment_approvals', to=settings.AUTH_USER_MODEL, verbose_name='approver'), + ), + migrations.AddField( + model_name='experiment', + name='flag', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='experiments', to='flags.featureflag', verbose_name='feature flag'), + ), + migrations.AddField( + model_name='experiment', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='owned_experiments', to=settings.AUTH_USER_MODEL, verbose_name='owner'), + ), + migrations.AddField( + model_name='experiment', + name='previous_version', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='next_versions', to='experiments.experiment', verbose_name='previous version'), + ), + migrations.AddField( + model_name='approval', + name='experiment', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='approvals', to='experiments.experiment', verbose_name='experiment'), + ), + migrations.AddField( + model_name='experimentlog', + name='experiment', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='logs', to='experiments.experiment', verbose_name='experiment'), + ), + migrations.AddField( + model_name='experimentlog', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='experiment_logs', to=settings.AUTH_USER_MODEL, verbose_name='user'), + ), + migrations.AddField( + model_name='experimentoutcome', + name='decided_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='experiment_decisions', to=settings.AUTH_USER_MODEL, verbose_name='decided by'), + ), + migrations.AddField( + model_name='experimentoutcome', + name='experiment', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='outcome', to='experiments.experiment', verbose_name='experiment'), + ), + migrations.AddField( + model_name='variant', + name='experiment', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='variants', to='experiments.experiment', verbose_name='experiment'), + ), + migrations.AddField( + model_name='experimentoutcome', + name='winning_variant', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='wins', to='experiments.variant', verbose_name='winning variant'), + ), + migrations.AddIndex( + model_name='experiment', + index=models.Index(fields=['flag', 'status'], name='idx_experiment_flag_status'), + ), + migrations.AddConstraint( + model_name='experiment', + constraint=models.UniqueConstraint(condition=models.Q(('status__in', ('running', 'paused'))), fields=('flag',), name='unique_active_experiment_per_flag'), + ), + migrations.AddConstraint( + model_name='approval', + constraint=models.UniqueConstraint(fields=('experiment', 'approver'), name='unique_approval_per_user'), + ), + ] diff --git a/src/backend/apps/experiments/migrations/__init__.py b/src/backend/apps/experiments/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/apps/experiments/models.py b/src/backend/apps/experiments/models.py new file mode 100644 index 0000000..56a456c --- /dev/null +++ b/src/backend/apps/experiments/models.py @@ -0,0 +1,557 @@ +from decimal import Decimal +from typing import Any, override + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.core.validators import MaxValueValidator, MinValueValidator +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from apps.core.models import BaseModel + + +class ExperimentStatus(models.TextChoices): + DRAFT = "draft", _("Draft") + IN_REVIEW = "in_review", _("In review") + APPROVED = "approved", _("Approved") + RUNNING = "running", _("Running") + PAUSED = "paused", _("Paused") + COMPLETED = "completed", _("Completed") + ARCHIVED = "archived", _("Archived") + REJECTED = "rejected", _("Rejected") + + +ALLOWED_TRANSITIONS: dict[str, tuple[str, ...]] = { + ExperimentStatus.DRAFT: (ExperimentStatus.IN_REVIEW,), + ExperimentStatus.IN_REVIEW: ( + ExperimentStatus.APPROVED, + ExperimentStatus.DRAFT, + ExperimentStatus.REJECTED, + ), + ExperimentStatus.APPROVED: (ExperimentStatus.RUNNING,), + ExperimentStatus.RUNNING: ( + ExperimentStatus.PAUSED, + ExperimentStatus.COMPLETED, + ), + ExperimentStatus.PAUSED: ( + ExperimentStatus.RUNNING, + ExperimentStatus.COMPLETED, + ), + ExperimentStatus.COMPLETED: (ExperimentStatus.ARCHIVED,), + ExperimentStatus.ARCHIVED: (), + ExperimentStatus.REJECTED: (ExperimentStatus.DRAFT,), +} + +ACTIVE_STATUSES = (ExperimentStatus.RUNNING, ExperimentStatus.PAUSED) + +STARTED_STATUSES = ( + ExperimentStatus.RUNNING, + ExperimentStatus.PAUSED, + ExperimentStatus.COMPLETED, + ExperimentStatus.ARCHIVED, +) + +FROZEN_AFTER_START: tuple[str, ...] = ( + "traffic_allocation", + "targeting_rules", + "flag", +) + +EDITABLE_IN_DRAFT: tuple[str, ...] = ( + "name", + "description", + "hypothesis", + "traffic_allocation", + "targeting_rules", + "flag", +) + + +class ConflictPolicy(models.TextChoices): + MUTUAL_EXCLUSION = "mutual_exclusion", _("Mutual exclusion") + PRIORITY = "priority", _("Priority tiers") + + +class OutcomeType(models.TextChoices): + ROLLOUT = "rollout", _("Rollout winner") + ROLLBACK = "rollback", _("Rollback") + NO_EFFECT = "no_effect", _("No effect") + + +class TransitionValidator: + def __init__(self, instance: "Experiment") -> None: + self.instance = instance + + def __call__(self) -> None: + if not self.instance.pk: + return + try: + old = Experiment.objects.get(pk=self.instance.pk) + except Experiment.DoesNotExist: + return + if old.status == self.instance.status: + return + allowed = ALLOWED_TRANSITIONS.get(old.status, ()) + if self.instance.status not in allowed: + raise ValidationError( + { + "status": ( + f"Transition from '{old.status}' to " + f"'{self.instance.status}' is not allowed. " + f"Allowed: {', '.join(allowed) or 'none'}." + ) + } + ) + + +class FrozenFieldsValidator: + def __init__(self, instance: "Experiment") -> None: + self.instance = instance + + def __call__(self) -> None: + if not self.instance.pk: + return + try: + old = Experiment.objects.get(pk=self.instance.pk) + except Experiment.DoesNotExist: + return + if old.status not in STARTED_STATUSES: + return + errors: dict[str, str] = {} + for field_name in FROZEN_AFTER_START: + old_value = getattr(old, field_name) + new_value = getattr(self.instance, field_name) + if hasattr(old_value, "pk"): + changed = getattr(old_value, "pk", None) != getattr( + new_value, "pk", None + ) + else: + changed = old_value != new_value + if changed: + errors[field_name] = ( + f"Field '{field_name}' cannot be modified after " + f"the experiment has been started " + f"(current status: '{old.status}')." + ) + if errors: + raise ValidationError(errors) + + +class UniqueActiveFlagValidator: + def __init__(self, instance: "Experiment") -> None: + self.instance = instance + + def __call__(self) -> None: + if self.instance.status not in ACTIVE_STATUSES: + return + conflict = ( + Experiment.objects.filter( + flag=self.instance.flag, + status__in=ACTIVE_STATUSES, + ) + .exclude(pk=self.instance.pk) + .first() + ) + if conflict: + raise ValidationError( + { + "flag": ( + f"Flag '{self.instance.flag.key}' already has an " + f"active experiment: '{conflict.name}' " + f"(status: {conflict.status})." + ) + } + ) + + +class Experiment(BaseModel): + VALIDATORS = ( + TransitionValidator, + FrozenFieldsValidator, + UniqueActiveFlagValidator, + ) + + flag = models.ForeignKey( + "flags.FeatureFlag", + on_delete=models.PROTECT, + related_name="experiments", + verbose_name=_("feature flag"), + ) + name = models.CharField( + max_length=200, + verbose_name=_("name"), + ) + description = models.TextField( + blank=True, + verbose_name=_("description"), + ) + hypothesis = models.TextField( + blank=True, + verbose_name=_("hypothesis"), + ) + status = models.CharField( + max_length=20, + choices=ExperimentStatus.choices, + default=ExperimentStatus.DRAFT, + db_index=True, + verbose_name=_("status"), + ) + version = models.PositiveIntegerField( + default=1, + verbose_name=_("version"), + ) + previous_version = models.ForeignKey( + "self", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="next_versions", + verbose_name=_("previous version"), + ) + traffic_allocation = models.DecimalField( + max_digits=5, + decimal_places=2, + default=Decimal("100.00"), + validators=[ + MinValueValidator(Decimal("0.01")), + MaxValueValidator(Decimal("100.00")), + ], + verbose_name=_("traffic allocation %"), + help_text=_("Percentage of eligible users included in experiment"), + ) + targeting_rules = models.TextField( + blank=True, + verbose_name=_("targeting rules"), + help_text=_("DSL expression for targeting"), + ) + owner = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.PROTECT, + related_name="owned_experiments", + verbose_name=_("owner"), + ) + created_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("created at"), + ) + updated_at = models.DateTimeField( + auto_now=True, + verbose_name=_("updated at"), + ) + + class Meta: + verbose_name = _("experiment") + verbose_name_plural = _("experiments") + ordering = ["-created_at"] + indexes = [ + models.Index( + fields=["flag", "status"], + name="idx_experiment_flag_status", + ), + ] + constraints = [ + models.UniqueConstraint( + fields=["flag"], + condition=models.Q(status__in=ACTIVE_STATUSES), + name="unique_active_experiment_per_flag", + ), + ] + + @override + def __str__(self) -> str: + return f"{self.name} (v{self.version}, {self.status})" + + @override + def save(self, *args: Any, **kwargs: Any) -> None: + for validator_cls in self.VALIDATORS: + validator_cls(self)() + super().save(*args, **kwargs) + + +class Variant(BaseModel): + experiment = models.ForeignKey( + Experiment, + on_delete=models.CASCADE, + related_name="variants", + verbose_name=_("experiment"), + ) + name = models.CharField( + max_length=100, + verbose_name=_("name"), + ) + value = models.CharField( + max_length=500, + verbose_name=_("value"), + help_text=_("Value returned when this variant is selected"), + ) + weight = models.DecimalField( + max_digits=5, + decimal_places=2, + validators=[ + MinValueValidator(Decimal("0.00")), + MaxValueValidator(Decimal("100.00")), + ], + verbose_name=_("weight %"), + ) + is_control = models.BooleanField( + default=False, + verbose_name=_("is control"), + ) + created_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("created at"), + ) + updated_at = models.DateTimeField( + auto_now=True, + verbose_name=_("updated at"), + ) + + class Meta: + verbose_name = _("variant") + verbose_name_plural = _("variants") + ordering = ["name"] + + @override + def __str__(self) -> str: + control = " [control]" if self.is_control else "" + return f"{self.name}: {self.weight}%{control}" + + @override + def save(self, *args: Any, **kwargs: Any) -> None: + VariantFrozenValidator(self)() + super().save(*args, **kwargs) + + +class VariantFrozenValidator: + def __init__(self, instance: Variant) -> None: + self.instance = instance + + def __call__(self) -> None: + experiment = self.instance.experiment + if experiment.status in STARTED_STATUSES: + raise ValidationError( + { + "experiment": ( + "Variants cannot be modified after the experiment " + f"has been started (status: '{experiment.status}')." + ) + } + ) + + +class LogType(models.TextChoices): + STATUS_CHANGE = "status_change", _("Status change") + APPROVAL = "approval", _("Approval") + REJECTION = "rejection", _("Rejection") + REVIEW_REQUESTED = "review_requested", _("Review requested") + GUARDRAIL_TRIGGERED = "guardrail_triggered", _("Guardrail triggered") + COMPLETED = "completed", _("Completed") + COMMENT = "comment", _("Comment") + + +class ExperimentLog(BaseModel): + experiment = models.ForeignKey( + Experiment, + on_delete=models.CASCADE, + related_name="logs", + verbose_name=_("experiment"), + ) + log_type = models.CharField( + max_length=30, + choices=LogType.choices, + verbose_name=_("log type"), + ) + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="experiment_logs", + verbose_name=_("user"), + ) + comment = models.TextField( + blank=True, + verbose_name=_("comment"), + ) + metadata = models.JSONField( + default=dict, + blank=True, + verbose_name=_("metadata"), + ) + created_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("created at"), + ) + + class Meta: + verbose_name = _("experiment log") + verbose_name_plural = _("experiment logs") + ordering = ["-created_at"] + + @override + def __str__(self) -> str: + return f"[{self.log_type}] {self.experiment.name}" + + +class Approval(BaseModel): + experiment = models.ForeignKey( + Experiment, + on_delete=models.CASCADE, + related_name="approvals", + verbose_name=_("experiment"), + ) + approver = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.PROTECT, + related_name="experiment_approvals", + verbose_name=_("approver"), + ) + comment = models.TextField( + blank=True, + verbose_name=_("comment"), + ) + created_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("created at"), + ) + + class Meta: + verbose_name = _("approval") + verbose_name_plural = _("approvals") + ordering = ["-created_at"] + constraints = [ + models.UniqueConstraint( + fields=["experiment", "approver"], + name="unique_approval_per_user", + ), + ] + + @override + def __str__(self) -> str: + return f"Approval by {self.approver} for {self.experiment}" + + +class ConflictDomain(BaseModel): + name = models.CharField( + max_length=200, + unique=True, + verbose_name=_("name"), + ) + description = models.TextField( + blank=True, + verbose_name=_("description"), + ) + policy = models.CharField( + max_length=30, + choices=ConflictPolicy.choices, + default=ConflictPolicy.MUTUAL_EXCLUSION, + verbose_name=_("conflict policy"), + ) + max_concurrent = models.PositiveIntegerField( + default=1, + verbose_name=_("max concurrent experiments"), + ) + created_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("created at"), + ) + updated_at = models.DateTimeField( + auto_now=True, + verbose_name=_("updated at"), + ) + + class Meta: + verbose_name = _("conflict domain") + verbose_name_plural = _("conflict domains") + ordering = ["name"] + + @override + def __str__(self) -> str: + return f"{self.name} ({self.policy})" + + +class ExperimentConflictDomain(BaseModel): + experiment = models.ForeignKey( + Experiment, + on_delete=models.CASCADE, + related_name="conflict_memberships", + verbose_name=_("experiment"), + ) + conflict_domain = models.ForeignKey( + ConflictDomain, + on_delete=models.CASCADE, + related_name="experiment_memberships", + verbose_name=_("conflict domain"), + ) + priority = models.IntegerField( + default=0, + verbose_name=_("priority"), + help_text=_("Higher value wins in priority-based resolution"), + ) + created_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("created at"), + ) + + class Meta: + verbose_name = _("experiment conflict domain") + verbose_name_plural = _("experiment conflict domains") + ordering = ["-priority"] + constraints = [ + models.UniqueConstraint( + fields=["experiment", "conflict_domain"], + name="unique_experiment_conflict_domain", + ), + ] + + @override + def __str__(self) -> str: + return ( + f"{self.experiment.name} in {self.conflict_domain.name} " + f"(priority={self.priority})" + ) + + +class ExperimentOutcome(BaseModel): + experiment = models.OneToOneField( + Experiment, + on_delete=models.CASCADE, + related_name="outcome", + verbose_name=_("experiment"), + ) + outcome = models.CharField( + max_length=20, + choices=OutcomeType.choices, + verbose_name=_("outcome"), + ) + winning_variant = models.ForeignKey( + Variant, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="wins", + verbose_name=_("winning variant"), + ) + rationale = models.TextField( + verbose_name=_("rationale"), + ) + decided_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="experiment_decisions", + verbose_name=_("decided by"), + ) + created_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("created at"), + ) + + class Meta: + verbose_name = _("experiment outcome") + verbose_name_plural = _("experiment outcomes") + + @override + def __str__(self) -> str: + return f"{self.experiment.name}: {self.outcome}" diff --git a/src/backend/apps/experiments/selectors.py b/src/backend/apps/experiments/selectors.py new file mode 100644 index 0000000..5223bd4 --- /dev/null +++ b/src/backend/apps/experiments/selectors.py @@ -0,0 +1,69 @@ +from uuid import UUID + +from django.db.models import QuerySet + +from apps.experiments.models import ( + Approval, + Experiment, + ExperimentLog, + ExperimentStatus, + Variant, +) + + +def experiment_list( + *, + status: str | None = None, + flag_id: UUID | None = None, + owner_id: UUID | None = None, + search: str | None = None, +) -> QuerySet[Experiment]: + qs = Experiment.objects.select_related("flag", "owner").prefetch_related( + "variants" + ) + if status: + qs = qs.filter(status=status) + if flag_id: + qs = qs.filter(flag_id=flag_id) + if owner_id: + qs = qs.filter(owner_id=owner_id) + if search: + qs = qs.filter(name__icontains=search) + return qs + + +def experiment_get(experiment_id: UUID) -> Experiment | None: + return ( + Experiment.objects.select_related("flag", "owner") + .prefetch_related("variants", "approvals__approver", "logs") + .filter(pk=experiment_id) + .first() + ) + + +def experiment_logs(experiment_id: UUID) -> QuerySet[ExperimentLog]: + return ExperimentLog.objects.filter( + experiment_id=experiment_id, + ).select_related("user") + + +def experiment_approvals(experiment_id: UUID) -> QuerySet[Approval]: + return Approval.objects.filter( + experiment_id=experiment_id, + ).select_related("approver") + + +def variant_list(experiment_id: UUID) -> QuerySet[Variant]: + return Variant.objects.filter(experiment_id=experiment_id) + + +def active_experiment_for_flag(flag_id: UUID) -> Experiment | None: + return ( + Experiment.objects.filter( + flag_id=flag_id, + status=ExperimentStatus.RUNNING, + ) + .select_related("flag") + .prefetch_related("variants") + .first() + ) diff --git a/src/backend/apps/experiments/services.py b/src/backend/apps/experiments/services.py new file mode 100644 index 0000000..8d7f1f2 --- /dev/null +++ b/src/backend/apps/experiments/services.py @@ -0,0 +1,495 @@ +from decimal import Decimal +from typing import Any + +from django.core.exceptions import ValidationError +from django.db import transaction + +from apps.experiments.models import ( + ACTIVE_STATUSES, + ALLOWED_TRANSITIONS, + Approval, + Experiment, + ExperimentLog, + ExperimentOutcome, + ExperimentStatus, + LogType, + OutcomeType, + Variant, +) +from apps.flags.models import FeatureFlag, validate_value_for_type +from apps.notifications.services import ( + NotificationPayload, + notification_enqueue, +) +from apps.reviews.selectors import ( + can_user_approve_experimenter, + get_min_approvals_for_experimenter, +) +from apps.users.models import User +from config.errors import ForbiddenError + + +def _notify(event_type: str, experiment: Experiment, extra: dict[str, Any] | None = None) -> None: + notification_enqueue( + event_type, + NotificationPayload( + title=f"{event_type.replace('_', ' ').title()}", + body=f"Experiment '{experiment.name}' — {event_type.replace('_', ' ')}.", + event_type=event_type, + experiment_id=str(experiment.pk), + experiment_name=experiment.name, + extra=extra or {}, + ), + ) + + +def ensure_owner_or_admin(experiment: Experiment, user: User) -> None: + if user.is_admin_role: + return + if experiment.owner_id != user.pk: + raise ForbiddenError( + "Only the experiment owner or an admin can perform this action." + ) + + +@transaction.atomic +def experiment_create( + *, + flag: FeatureFlag, + name: str, + owner: User, + description: str = "", + hypothesis: str = "", + traffic_allocation: Decimal = Decimal("100.00"), + targeting_rules: str = "", +) -> Experiment: + experiment = Experiment( + flag=flag, + name=name, + owner=owner, + description=description, + hypothesis=hypothesis, + traffic_allocation=traffic_allocation, + targeting_rules=targeting_rules, + status=ExperimentStatus.DRAFT, + version=1, + ) + experiment.save() + ExperimentLog.objects.create( + experiment=experiment, + log_type=LogType.STATUS_CHANGE, + user=owner, + metadata={"to_status": ExperimentStatus.DRAFT}, + ) + return experiment + + +@transaction.atomic +def experiment_update( + *, + experiment: Experiment, + user: User, + **fields: Any, +) -> Experiment: + ensure_owner_or_admin(experiment, user) + allowed_fields = { + "name", + "description", + "hypothesis", + "traffic_allocation", + "targeting_rules", + } + for key in fields: + if key not in allowed_fields: + raise ValidationError( + {key: f"Field '{key}' cannot be updated via this endpoint."} + ) + for key, value in fields.items(): + if value is not None: + setattr(experiment, key, value) + experiment.save() + return experiment + + +@transaction.atomic +def variant_create( + *, + experiment: Experiment, + user: User, + name: str, + value: str, + weight: Decimal, + is_control: bool = False, +) -> Variant: + ensure_owner_or_admin(experiment, user) + validate_value_for_type(value, experiment.flag.value_type) + variant = Variant( + experiment=experiment, + name=name, + value=value, + weight=weight, + is_control=is_control, + ) + variant.save() + _validate_variant_invariants(experiment) + return variant + + +@transaction.atomic +def variant_update( + *, + variant: Variant, + user: User, + name: str | None = None, + value: str | None = None, + weight: Decimal | None = None, + is_control: bool | None = None, +) -> Variant: + ensure_owner_or_admin(variant.experiment, user) + if name is not None: + variant.name = name + if value is not None: + validate_value_for_type(value, variant.experiment.flag.value_type) + variant.value = value + if weight is not None: + variant.weight = weight + if is_control is not None: + variant.is_control = is_control + variant.save() + _validate_variant_invariants(variant.experiment) + return variant + + +@transaction.atomic +def variant_delete(*, variant: Variant, user: User) -> None: + ensure_owner_or_admin(variant.experiment, user) + experiment = variant.experiment + if experiment.status != ExperimentStatus.DRAFT: + raise ValidationError( + { + "experiment": ( + "Variants can only be deleted while the experiment " + f"is in '{ExperimentStatus.DRAFT}' status." + ) + } + ) + variant.delete() + + +def _validate_variant_invariants(experiment: Experiment) -> None: + variants = experiment.variants.all() + if not variants.exists(): + return + + control_count = variants.filter(is_control=True).count() + if control_count > 1: + raise ValidationError( + {"is_control": "Exactly one variant must be the control."} + ) + + total_weight = sum(v.weight for v in variants) + if total_weight > experiment.traffic_allocation: + raise ValidationError( + { + "weight": ( + f"Sum of variant weights ({total_weight}%) exceeds " + f"traffic allocation ({experiment.traffic_allocation}%)." + ) + } + ) + + +def _validate_experiment_ready_for_review(experiment: Experiment) -> None: + variants = experiment.variants.all() + if variants.count() < 2: + raise ValidationError( + {"variants": "At least two variants are required."} + ) + control_count = variants.filter(is_control=True).count() + if control_count != 1: + raise ValidationError( + {"is_control": "Exactly one variant must be the control."} + ) + total_weight = sum(v.weight for v in variants) + if total_weight != experiment.traffic_allocation: + raise ValidationError( + { + "weight": ( + f"Sum of variant weights ({total_weight}%) must equal " + f"traffic allocation ({experiment.traffic_allocation}%)." + ) + } + ) + + +def _validate_no_active_flag_conflict(experiment: Experiment) -> None: + conflict = ( + Experiment.objects.filter( + flag=experiment.flag, + status__in=ACTIVE_STATUSES, + ) + .exclude(pk=experiment.pk) + .first() + ) + if conflict: + raise ValidationError( + { + "flag": ( + f"Flag '{experiment.flag.key}' already has an active " + f"experiment: '{conflict.name}'." + ) + } + ) + + +def _transition( + experiment: Experiment, + new_status: str, + user: User, + log_type: str = LogType.STATUS_CHANGE, + comment: str = "", + metadata: dict[str, Any] | None = None, +) -> Experiment: + old_status = experiment.status + allowed = ALLOWED_TRANSITIONS.get(old_status, ()) + if new_status not in allowed: + raise ValidationError( + { + "status": ( + f"Transition from '{old_status}' to '{new_status}' " + f"is not allowed." + ) + } + ) + experiment.status = new_status + experiment.save(update_fields=["status", "updated_at"]) + ExperimentLog.objects.create( + experiment=experiment, + log_type=log_type, + user=user, + comment=comment, + metadata={ + "from_status": old_status, + "to_status": new_status, + **(metadata or {}), + }, + ) + return experiment + + +@transaction.atomic +def experiment_submit_for_review( + *, experiment: Experiment, user: User +) -> Experiment: + ensure_owner_or_admin(experiment, user) + _validate_experiment_ready_for_review(experiment) + experiment = _transition( + experiment, + ExperimentStatus.IN_REVIEW, + user, + log_type=LogType.REVIEW_REQUESTED, + ) + _notify("review_requested", experiment) + return experiment + + +@transaction.atomic +def experiment_approve( + *, experiment: Experiment, approver: User, comment: str = "" +) -> Experiment: + if experiment.status != ExperimentStatus.IN_REVIEW: + raise ValidationError( + {"status": "Experiment must be in 'in_review' status to approve."} + ) + if not can_user_approve_experimenter(approver, experiment.owner): + raise ValidationError( + { + "approver": ( + f"User '{approver.username}' is not authorized to " + f"approve experiments for '{experiment.owner.username}'." + ) + } + ) + if experiment.approvals.filter(approver=approver).exists(): + raise ValidationError( + {"approver": "You have already approved this experiment."} + ) + Approval.objects.create( + experiment=experiment, + approver=approver, + comment=comment, + ) + ExperimentLog.objects.create( + experiment=experiment, + log_type=LogType.APPROVAL, + user=approver, + comment=comment, + metadata={"approver": str(approver.pk)}, + ) + approval_count = experiment.approvals.count() + min_approvals = get_min_approvals_for_experimenter(experiment.owner) + if approval_count >= min_approvals: + experiment = _transition( + experiment, + ExperimentStatus.APPROVED, + approver, + metadata={"approval_count": approval_count}, + ) + _notify("review_approved", experiment) + return experiment + return experiment + + +@transaction.atomic +def experiment_reject( + *, experiment: Experiment, user: User, comment: str = "" +) -> Experiment: + experiment = _transition( + experiment, + ExperimentStatus.REJECTED, + user, + log_type=LogType.REJECTION, + comment=comment, + ) + _notify("review_rejected", experiment) + return experiment + + +@transaction.atomic +def experiment_request_changes( + *, experiment: Experiment, user: User, comment: str = "" +) -> Experiment: + experiment.approvals.all().delete() + return _transition( + experiment, + ExperimentStatus.DRAFT, + user, + comment=comment, + metadata={"action": "request_changes"}, + ) + + +@transaction.atomic +def experiment_start(*, experiment: Experiment, user: User) -> Experiment: + ensure_owner_or_admin(experiment, user) + _validate_no_active_flag_conflict(experiment) + experiment = _transition( + experiment, + ExperimentStatus.RUNNING, + user, + ) + _notify("experiment_started", experiment) + return experiment + + +@transaction.atomic +def experiment_pause( + *, experiment: Experiment, user: User, comment: str = "" +) -> Experiment: + ensure_owner_or_admin(experiment, user) + experiment = _transition( + experiment, + ExperimentStatus.PAUSED, + user, + comment=comment, + ) + _notify("experiment_paused", experiment, extra={"comment": comment}) + return experiment + + +@transaction.atomic +def experiment_resume(*, experiment: Experiment, user: User) -> Experiment: + ensure_owner_or_admin(experiment, user) + _validate_no_active_flag_conflict(experiment) + return _transition( + experiment, + ExperimentStatus.RUNNING, + user, + ) + + +@transaction.atomic +def experiment_complete( + *, + experiment: Experiment, + user: User, + outcome: str, + rationale: str, + winning_variant_id: str | None = None, +) -> Experiment: + ensure_owner_or_admin(experiment, user) + valid_outcomes = {c[0] for c in OutcomeType.choices} + if outcome not in valid_outcomes: + raise ValidationError( + { + "outcome": ( + f"Invalid outcome '{outcome}'. " + f"Must be one of: {', '.join(sorted(valid_outcomes))}." + ) + } + ) + if not rationale.strip(): + raise ValidationError( + {"rationale": "Rationale is required when completing."} + ) + winning_variant = None + if outcome == OutcomeType.ROLLOUT: + if not winning_variant_id: + raise ValidationError( + {"winning_variant_id": "Winner must be specified for rollout."} + ) + winning_variant = experiment.variants.filter( + pk=winning_variant_id + ).first() + if not winning_variant: + raise ValidationError( + {"winning_variant_id": "Variant not found in this experiment."} + ) + experiment = _transition( + experiment, + ExperimentStatus.COMPLETED, + user, + log_type=LogType.COMPLETED, + comment=rationale, + metadata={ + "outcome": outcome, + "winning_variant_id": str(winning_variant.pk) + if winning_variant + else None, + }, + ) + ExperimentOutcome.objects.create( + experiment=experiment, + outcome=outcome, + winning_variant=winning_variant, + rationale=rationale, + decided_by=user, + ) + _notify( + "experiment_completed", + experiment, + extra={"outcome": outcome, "rationale": rationale}, + ) + return experiment + + +@transaction.atomic +def experiment_archive(*, experiment: Experiment, user: User) -> Experiment: + ensure_owner_or_admin(experiment, user) + return _transition( + experiment, + ExperimentStatus.ARCHIVED, + user, + ) + + +@transaction.atomic +def experiment_reopen(*, experiment: Experiment, user: User) -> Experiment: + ensure_owner_or_admin(experiment, user) + experiment.approvals.all().delete() + return _transition( + experiment, + ExperimentStatus.DRAFT, + user, + metadata={"action": "reopen_from_rejected"}, + ) diff --git a/src/backend/apps/experiments/tests/__init__.py b/src/backend/apps/experiments/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/apps/experiments/tests/helpers.py b/src/backend/apps/experiments/tests/helpers.py new file mode 100644 index 0000000..659457a --- /dev/null +++ b/src/backend/apps/experiments/tests/helpers.py @@ -0,0 +1,65 @@ +from decimal import Decimal + +from apps.experiments.models import Experiment, Variant +from apps.experiments.services import ( + experiment_create, + variant_create, +) +from apps.flags.models import FeatureFlagType +from apps.flags.services import feature_flag_create +from apps.users.tests.helpers import make_user + + +def make_flag(suffix="", value_type=FeatureFlagType.STRING, default="a"): + return feature_flag_create( + key=f"flag{suffix}", + name=f"Flag {suffix}", + value_type=value_type, + default_value=default, + ) + + +def make_experiment( + flag=None, + owner=None, + suffix="", + name=None, + **kwargs, +) -> Experiment: + if not flag: + flag = make_flag(suffix=suffix) + if not owner: + owner = make_user( + username=f"owner{suffix}", + email=f"owner{suffix}@lotty.local", + ) + return experiment_create( + flag=flag, + name=name or f"Exp{suffix}", + owner=owner, + **kwargs, + ) + + +def add_two_variants( + experiment: Experiment, + weight_a: Decimal = Decimal("50.00"), + weight_b: Decimal = Decimal("50.00"), +) -> tuple[Variant, Variant]: + v_control = variant_create( + experiment=experiment, + user=experiment.owner, + name="control", + value="a", + weight=weight_a, + is_control=True, + ) + v_treatment = variant_create( + experiment=experiment, + user=experiment.owner, + name="treatment", + value="b", + weight=weight_b, + is_control=False, + ) + return v_control, v_treatment diff --git a/src/backend/apps/experiments/tests/test_models.py b/src/backend/apps/experiments/tests/test_models.py new file mode 100644 index 0000000..6ce6c77 --- /dev/null +++ b/src/backend/apps/experiments/tests/test_models.py @@ -0,0 +1,179 @@ +from decimal import Decimal + +from django.core.exceptions import ValidationError +from django.test import TestCase + +from apps.experiments.models import ( + ALLOWED_TRANSITIONS, + FROZEN_AFTER_START, + STARTED_STATUSES, + Experiment, + ExperimentStatus, + Variant, +) +from apps.experiments.tests.helpers import ( + add_two_variants, + make_experiment, + make_flag, +) +from apps.users.tests.helpers import make_user + + +class TransitionValidatorTest(TestCase): + def test_all_allowed_transitions_succeed(self) -> None: + exp = make_experiment(suffix="_tr") + for from_status, targets in ALLOWED_TRANSITIONS.items(): + for target in targets: + Experiment.objects.filter(pk=exp.pk).update(status=from_status) + exp.refresh_from_db() + exp.status = target + exp.save() + exp.refresh_from_db() + self.assertEqual(exp.status, target) + + def test_forbidden_transition_raises(self) -> None: + exp = make_experiment(suffix="_fb") + Experiment.objects.filter(pk=exp.pk).update( + status=ExperimentStatus.DRAFT + ) + exp.refresh_from_db() + exp.status = ExperimentStatus.RUNNING + with self.assertRaises(ValidationError) as ctx: + exp.save() + self.assertIn("status", ctx.exception.message_dict) + + def test_same_status_save_allowed(self) -> None: + exp = make_experiment(suffix="_ss") + exp.name = "Updated name" + exp.save() + exp.refresh_from_db() + self.assertEqual(exp.name, "Updated name") + + +class FrozenFieldsValidatorTest(TestCase): + def test_frozen_fields_after_start(self) -> None: + exp = make_experiment( + suffix="_ff", + traffic_allocation=Decimal("100.00"), + ) + add_two_variants(exp) + Experiment.objects.filter(pk=exp.pk).update( + status=ExperimentStatus.RUNNING + ) + exp.refresh_from_db() + + for field_name in FROZEN_AFTER_START: + exp_copy = Experiment.objects.get(pk=exp.pk) + if field_name == "traffic_allocation": + exp_copy.traffic_allocation = Decimal("10.00") + elif field_name == "targeting_rules": + exp_copy.targeting_rules = "changed" + elif field_name == "flag": + exp_copy.flag = make_flag(suffix=f"_alt_{field_name}") + with self.assertRaises( + ValidationError, msg=f"Expected freeze on {field_name}" + ): + exp_copy.save() + + def test_non_frozen_fields_editable_after_start(self) -> None: + exp = make_experiment(suffix="_nf") + Experiment.objects.filter(pk=exp.pk).update( + status=ExperimentStatus.RUNNING + ) + exp.refresh_from_db() + exp.name = "New Name" + exp.description = "New Desc" + exp.save() + exp.refresh_from_db() + self.assertEqual(exp.name, "New Name") + + def test_frozen_fields_editable_in_draft(self) -> None: + exp = make_experiment(suffix="_dft") + exp.traffic_allocation = Decimal("25.00") + exp.targeting_rules = "age > 18" + exp.save() + exp.refresh_from_db() + self.assertEqual(exp.traffic_allocation, Decimal("25.00")) + + def test_frozen_in_all_started_statuses(self) -> None: + for status in STARTED_STATUSES: + exp = make_experiment(suffix=f"_s{status}") + Experiment.objects.filter(pk=exp.pk).update(status=status) + exp.refresh_from_db() + exp.traffic_allocation = Decimal("1.00") + with self.assertRaises( + ValidationError, msg=f"Expected freeze in {status}" + ): + exp.save() + + +class UniqueActiveFlagValidatorTest(TestCase): + def test_two_running_experiments_same_flag_rejected(self) -> None: + flag = make_flag(suffix="_uaf") + owner = make_user(username="uaf_owner", email="uaf_owner@lotty.local") + exp1 = make_experiment(flag=flag, owner=owner, suffix="_u1", name="E1") + Experiment.objects.filter(pk=exp1.pk).update( + status=ExperimentStatus.RUNNING + ) + + exp2 = make_experiment(flag=flag, owner=owner, suffix="_u2", name="E2") + Experiment.objects.filter(pk=exp2.pk).update( + status=ExperimentStatus.APPROVED + ) + exp2.refresh_from_db() + exp2.status = ExperimentStatus.RUNNING + with self.assertRaises(ValidationError) as ctx: + exp2.save() + self.assertIn("flag", ctx.exception.message_dict) + + def test_different_flags_both_running_allowed(self) -> None: + owner = make_user(username="dfb_owner", email="dfb_owner@lotty.local") + exp1 = make_experiment(owner=owner, suffix="_df1") + exp2 = make_experiment(owner=owner, suffix="_df2") + Experiment.objects.filter(pk=exp1.pk).update( + status=ExperimentStatus.RUNNING + ) + Experiment.objects.filter(pk=exp2.pk).update( + status=ExperimentStatus.APPROVED + ) + exp2.refresh_from_db() + exp2.status = ExperimentStatus.RUNNING + exp2.save() + exp2.refresh_from_db() + self.assertEqual(exp2.status, ExperimentStatus.RUNNING) + + +class VariantFrozenValidatorTest(TestCase): + def test_variant_not_editable_after_start(self) -> None: + exp = make_experiment(suffix="_vf") + _vc, vt = add_two_variants(exp) + Experiment.objects.filter(pk=exp.pk).update( + status=ExperimentStatus.RUNNING + ) + vt.refresh_from_db() + vt.name = "changed" + with self.assertRaises(ValidationError): + vt.save() + + def test_variant_editable_in_draft(self) -> None: + exp = make_experiment(suffix="_ved") + _vc, vt = add_two_variants(exp) + vt.name = "renamed" + vt.save() + vt.refresh_from_db() + self.assertEqual(vt.name, "renamed") + + def test_new_variant_rejected_after_start(self) -> None: + exp = make_experiment(suffix="_vnr") + add_two_variants(exp) + Experiment.objects.filter(pk=exp.pk).update( + status=ExperimentStatus.RUNNING + ) + exp.refresh_from_db() + with self.assertRaises(ValidationError): + Variant( + experiment=exp, + name="extra", + value="c", + weight=Decimal("10.00"), + ).save() diff --git a/src/backend/apps/experiments/tests/test_services.py b/src/backend/apps/experiments/tests/test_services.py new file mode 100644 index 0000000..ac263e9 --- /dev/null +++ b/src/backend/apps/experiments/tests/test_services.py @@ -0,0 +1,585 @@ +from decimal import Decimal +from typing import override + +from django.core.exceptions import ValidationError +from django.test import TestCase + +from apps.experiments.models import ( + Approval, + Experiment, + ExperimentLog, + ExperimentOutcome, + ExperimentStatus, + LogType, + OutcomeType, +) +from apps.experiments.services import ( + ensure_owner_or_admin, + experiment_approve, + experiment_archive, + experiment_complete, + experiment_create, + experiment_pause, + experiment_reject, + experiment_reopen, + experiment_request_changes, + experiment_resume, + experiment_start, + experiment_submit_for_review, + experiment_update, + variant_create, + variant_delete, + variant_update, +) +from apps.experiments.tests.helpers import ( + add_two_variants, + make_experiment, + make_flag, +) +from apps.reviews.services import approver_group_create, review_settings_update +from apps.reviews.tests.helpers import ( + make_admin, + make_approver, + make_experimenter, + make_viewer, +) +from config.errors import ForbiddenError + + +class ExperimentCreateTest(TestCase): + def test_creates_with_defaults(self) -> None: + exp = make_experiment(suffix="_cw") + self.assertEqual(exp.status, ExperimentStatus.DRAFT) + self.assertEqual(exp.version, 1) + self.assertEqual(exp.traffic_allocation, Decimal("100.00")) + + def test_creates_log_entry(self) -> None: + exp = make_experiment(suffix="_cl") + logs = ExperimentLog.objects.filter(experiment=exp) + self.assertEqual(logs.count(), 1) + self.assertEqual(logs.first().log_type, LogType.STATUS_CHANGE) + + +class ExperimentUpdateTest(TestCase): + def test_updates_allowed_fields(self) -> None: + exp = make_experiment(suffix="_uf") + exp = experiment_update( + experiment=exp, + user=exp.owner, + name="Updated", + description="desc", + ) + exp.refresh_from_db() + self.assertEqual(exp.name, "Updated") + self.assertEqual(exp.description, "desc") + + def test_rejects_disallowed_field(self) -> None: + exp = make_experiment(suffix="_rd") + with self.assertRaises(ValidationError): + experiment_update( + experiment=exp, + user=exp.owner, + status="running", + ) + + +class VariantCrudTest(TestCase): + def test_create_variants(self) -> None: + exp = make_experiment(suffix="_vc") + vc, vt = add_two_variants(exp) + self.assertTrue(vc.is_control) + self.assertFalse(vt.is_control) + + def test_total_weight_exceeds_allocation_raises(self) -> None: + exp = make_experiment( + suffix="_twe", traffic_allocation=Decimal("50.00") + ) + variant_create( + experiment=exp, + user=exp.owner, + name="control", + value="a", + weight=Decimal("30.00"), + is_control=True, + ) + with self.assertRaises(ValidationError) as ctx: + variant_create( + experiment=exp, + user=exp.owner, + name="treatment", + value="b", + weight=Decimal("30.00"), + ) + self.assertIn("weight", ctx.exception.message_dict) + + def test_two_controls_raises(self) -> None: + exp = make_experiment(suffix="_2c") + variant_create( + experiment=exp, + user=exp.owner, + name="c1", + value="a", + weight=Decimal("20.00"), + is_control=True, + ) + with self.assertRaises(ValidationError) as ctx: + variant_create( + experiment=exp, + user=exp.owner, + name="c2", + value="b", + weight=Decimal("20.00"), + is_control=True, + ) + self.assertIn("is_control", ctx.exception.message_dict) + + def test_update_variant(self) -> None: + exp = make_experiment(suffix="_vu") + _vc, vt = add_two_variants(exp) + vt = variant_update(variant=vt, user=exp.owner, name="renamed") + self.assertEqual(vt.name, "renamed") + + def test_delete_variant_in_draft(self) -> None: + exp = make_experiment(suffix="_vd") + _vc, vt = add_two_variants(exp) + variant_delete(variant=vt, user=exp.owner) + self.assertEqual(exp.variants.count(), 1) + + def test_delete_variant_not_in_draft_raises(self) -> None: + exp = make_experiment(suffix="_vnd") + _vc, vt = add_two_variants(exp) + Experiment.objects.filter(pk=exp.pk).update( + status=ExperimentStatus.IN_REVIEW + ) + vt.refresh_from_db() + with self.assertRaises(ValidationError): + variant_delete(variant=vt, user=exp.owner) + + +class SubmitForReviewTest(TestCase): + def test_submit_with_valid_variants(self) -> None: + exp = make_experiment(suffix="_sr") + add_two_variants(exp) + exp = experiment_submit_for_review(experiment=exp, user=exp.owner) + self.assertEqual(exp.status, ExperimentStatus.IN_REVIEW) + + def test_submit_without_variants_raises(self) -> None: + exp = make_experiment(suffix="_snv") + with self.assertRaises(ValidationError) as ctx: + experiment_submit_for_review(experiment=exp, user=exp.owner) + self.assertIn("variants", ctx.exception.message_dict) + + def test_submit_without_control_raises(self) -> None: + exp = make_experiment(suffix="_snc") + variant_create( + experiment=exp, + user=exp.owner, + name="v1", + value="a", + weight=Decimal("50.00"), + is_control=False, + ) + variant_create( + experiment=exp, + user=exp.owner, + name="v2", + value="b", + weight=Decimal("50.00"), + is_control=False, + ) + with self.assertRaises(ValidationError) as ctx: + experiment_submit_for_review(experiment=exp, user=exp.owner) + self.assertIn("is_control", ctx.exception.message_dict) + + def test_submit_weights_not_equal_allocation_raises(self) -> None: + exp = make_experiment(suffix="_sne") + variant_create( + experiment=exp, + user=exp.owner, + name="c", + value="a", + weight=Decimal("30.00"), + is_control=True, + ) + variant_create( + experiment=exp, + user=exp.owner, + name="t", + value="b", + weight=Decimal("20.00"), + ) + with self.assertRaises(ValidationError) as ctx: + experiment_submit_for_review(experiment=exp, user=exp.owner) + self.assertIn("weight", ctx.exception.message_dict) + + def test_submit_one_variant_raises(self) -> None: + exp = make_experiment(suffix="_s1v") + variant_create( + experiment=exp, + user=exp.owner, + name="c", + value="a", + weight=Decimal("100.00"), + is_control=True, + ) + with self.assertRaises(ValidationError) as ctx: + experiment_submit_for_review(experiment=exp, user=exp.owner) + self.assertIn("variants", ctx.exception.message_dict) + + +class ApprovalFlowTest(TestCase): + @override + def setUp(self) -> None: + self.experimenter = make_experimenter("_af") + self.approver1 = make_approver("_af1") + self.approver2 = make_approver("_af2") + self.flag = make_flag(suffix="_af") + approver_group_create( + experimenter=self.experimenter, + approver_ids=[ + str(self.approver1.pk), + str(self.approver2.pk), + ], + min_approvals=2, + ) + self.exp = experiment_create( + flag=self.flag, + name="AF Exp", + owner=self.experimenter, + ) + add_two_variants(self.exp) + self.exp = experiment_submit_for_review( + experiment=self.exp, user=self.experimenter + ) + + def test_first_approval_stays_in_review(self) -> None: + exp = experiment_approve( + experiment=self.exp, + approver=self.approver1, + comment="lgtm", + ) + self.assertEqual(exp.status, ExperimentStatus.IN_REVIEW) + self.assertEqual(Approval.objects.filter(experiment=exp).count(), 1) + + def test_second_approval_transitions_to_approved(self) -> None: + experiment_approve( + experiment=self.exp, + approver=self.approver1, + ) + exp = experiment_approve( + experiment=self.exp, + approver=self.approver2, + ) + self.assertEqual(exp.status, ExperimentStatus.APPROVED) + + def test_duplicate_approval_raises(self) -> None: + experiment_approve( + experiment=self.exp, + approver=self.approver1, + ) + with self.assertRaises(ValidationError) as ctx: + experiment_approve( + experiment=self.exp, + approver=self.approver1, + ) + self.assertIn("approver", ctx.exception.message_dict) + + def test_unauthorized_approver_raises(self) -> None: + viewer = make_viewer("_unauth") + with self.assertRaises(ValidationError) as ctx: + experiment_approve( + experiment=self.exp, + approver=viewer, + ) + self.assertIn("approver", ctx.exception.message_dict) + + +class RejectAndReqChangesTest(TestCase): + @override + def setUp(self) -> None: + self.experimenter = make_experimenter("_rr") + self.approver1 = make_approver("_rr1") + self.approver2 = make_approver("_rr2") + self.flag = make_flag(suffix="_rr") + approver_group_create( + experimenter=self.experimenter, + approver_ids=[ + str(self.approver1.pk), + str(self.approver2.pk), + ], + min_approvals=2, + ) + self.exp = experiment_create( + flag=self.flag, + name="RR Exp", + owner=self.experimenter, + ) + add_two_variants(self.exp) + self.exp = experiment_submit_for_review( + experiment=self.exp, user=self.experimenter + ) + + def test_reject(self) -> None: + exp = experiment_reject( + experiment=self.exp, + user=self.approver1, + comment="not ready", + ) + self.assertEqual(exp.status, ExperimentStatus.REJECTED) + + def test_request_changes_clears_approvals(self) -> None: + experiment_approve( + experiment=self.exp, + approver=self.approver1, + ) + self.exp.refresh_from_db() + exp = experiment_request_changes( + experiment=self.exp, user=self.approver2, comment="fix please" + ) + self.assertEqual(exp.status, ExperimentStatus.DRAFT) + self.assertEqual(Approval.objects.filter(experiment=exp).count(), 0) + + def test_reopen_from_rejected(self) -> None: + exp = experiment_reject( + experiment=self.exp, + user=self.approver1, + ) + exp = experiment_reopen(experiment=exp, user=self.experimenter) + self.assertEqual(exp.status, ExperimentStatus.DRAFT) + + +class LifecycleFlowTest(TestCase): + @override + def setUp(self) -> None: + self.experimenter = make_experimenter("_lf") + self.approver = make_approver("_lf") + self.flag = make_flag(suffix="_lf") + review_settings_update( + default_min_approvals=1, allow_any_approver=True + ) + self.exp = experiment_create( + flag=self.flag, + name="LF Exp", + owner=self.experimenter, + ) + self.vc, self.vt = add_two_variants(self.exp) + + def _approve_and_start(self) -> Experiment: + exp = experiment_submit_for_review( + experiment=self.exp, user=self.experimenter + ) + exp = experiment_approve(experiment=exp, approver=self.approver) + return experiment_start(experiment=exp, user=self.experimenter) + + def test_full_lifecycle_to_complete(self) -> None: + exp = self._approve_and_start() + self.assertEqual(exp.status, ExperimentStatus.RUNNING) + + exp = experiment_pause(experiment=exp, user=self.experimenter) + self.assertEqual(exp.status, ExperimentStatus.PAUSED) + + exp = experiment_resume(experiment=exp, user=self.experimenter) + self.assertEqual(exp.status, ExperimentStatus.RUNNING) + + exp = experiment_complete( + experiment=exp, + user=self.experimenter, + outcome=OutcomeType.ROLLOUT, + rationale="Treatment won", + winning_variant_id=str(self.vt.pk), + ) + self.assertEqual(exp.status, ExperimentStatus.COMPLETED) + outcome = ExperimentOutcome.objects.get(experiment=exp) + self.assertEqual(outcome.outcome, OutcomeType.ROLLOUT) + self.assertEqual(outcome.winning_variant, self.vt) + + def test_complete_rollback(self) -> None: + exp = self._approve_and_start() + exp = experiment_complete( + experiment=exp, + user=self.experimenter, + outcome=OutcomeType.ROLLBACK, + rationale="No improvement", + ) + self.assertEqual(exp.status, ExperimentStatus.COMPLETED) + + def test_complete_rollout_without_winner_raises(self) -> None: + exp = self._approve_and_start() + with self.assertRaises(ValidationError) as ctx: + experiment_complete( + experiment=exp, + user=self.experimenter, + outcome=OutcomeType.ROLLOUT, + rationale="Treatment won", + ) + self.assertIn("winning_variant_id", ctx.exception.message_dict) + + def test_complete_empty_rationale_raises(self) -> None: + exp = self._approve_and_start() + with self.assertRaises(ValidationError) as ctx: + experiment_complete( + experiment=exp, + user=self.experimenter, + outcome=OutcomeType.ROLLBACK, + rationale=" ", + ) + self.assertIn("rationale", ctx.exception.message_dict) + + def test_archive_after_complete(self) -> None: + exp = self._approve_and_start() + exp = experiment_complete( + experiment=exp, + user=self.experimenter, + outcome=OutcomeType.NO_EFFECT, + rationale="No effect observed", + ) + exp = experiment_archive(experiment=exp, user=self.experimenter) + self.assertEqual(exp.status, ExperimentStatus.ARCHIVED) + + def test_start_with_competing_active_raises(self) -> None: + exp1 = self._approve_and_start() + self.assertEqual(exp1.status, ExperimentStatus.RUNNING) + + exp2 = experiment_create( + flag=self.flag, + name="LF Exp 2", + owner=self.experimenter, + ) + add_two_variants(exp2) + exp2 = experiment_submit_for_review( + experiment=exp2, user=self.experimenter + ) + exp2 = experiment_approve(experiment=exp2, approver=self.approver) + with self.assertRaises(ValidationError) as ctx: + experiment_start(experiment=exp2, user=self.experimenter) + self.assertIn("flag", ctx.exception.message_dict) + + def test_start_from_draft_raises(self) -> None: + with self.assertRaises(ValidationError): + experiment_start(experiment=self.exp, user=self.experimenter) + + def test_audit_log_tracks_all_transitions(self) -> None: + exp = self._approve_and_start() + exp = experiment_complete( + experiment=exp, + user=self.experimenter, + outcome=OutcomeType.ROLLBACK, + rationale="done", + ) + logs = ExperimentLog.objects.filter(experiment=exp).order_by( + "created_at" + ) + log_types = list(logs.values_list("log_type", flat=True)) + self.assertIn(LogType.STATUS_CHANGE, log_types) + self.assertIn(LogType.REVIEW_REQUESTED, log_types) + self.assertIn(LogType.APPROVAL, log_types) + self.assertIn(LogType.COMPLETED, log_types) + + +class OwnershipPermissionTest(TestCase): + @override + def setUp(self) -> None: + self.owner = make_experimenter("_own") + self.other = make_experimenter("_oth") + self.admin = make_admin("_adm") + self.flag = make_flag(suffix="_op") + self.exp = experiment_create( + flag=self.flag, + name="Ownership Exp", + owner=self.owner, + ) + add_two_variants(self.exp) + + def test_owner_can_update(self) -> None: + exp = experiment_update( + experiment=self.exp, user=self.owner, name="Renamed" + ) + self.assertEqual(exp.name, "Renamed") + + def test_admin_can_update(self) -> None: + exp = experiment_update( + experiment=self.exp, user=self.admin, name="Admin Renamed" + ) + self.assertEqual(exp.name, "Admin Renamed") + + def test_other_experimenter_cannot_update(self) -> None: + with self.assertRaises(ForbiddenError): + experiment_update( + experiment=self.exp, user=self.other, name="Hijack" + ) + + def test_other_experimenter_cannot_create_variant(self) -> None: + with self.assertRaises(ForbiddenError): + variant_create( + experiment=self.exp, + user=self.other, + name="v3", + value="c", + weight=Decimal("10.00"), + ) + + def test_other_experimenter_cannot_update_variant(self) -> None: + _vc, vt = self.exp.variants.all()[0], self.exp.variants.all()[1] + with self.assertRaises(ForbiddenError): + variant_update(variant=vt, user=self.other, name="hijack") + + def test_other_experimenter_cannot_delete_variant(self) -> None: + vt = self.exp.variants.filter(is_control=False).first() + with self.assertRaises(ForbiddenError): + variant_delete(variant=vt, user=self.other) + + def test_other_experimenter_cannot_submit_for_review(self) -> None: + with self.assertRaises(ForbiddenError): + experiment_submit_for_review( + experiment=self.exp, user=self.other + ) + + def test_admin_can_submit_for_review(self) -> None: + exp = experiment_submit_for_review( + experiment=self.exp, user=self.admin + ) + self.assertEqual(exp.status, ExperimentStatus.IN_REVIEW) + + def test_other_experimenter_cannot_start(self) -> None: + review_settings_update( + default_min_approvals=1, allow_any_approver=True + ) + approver = make_approver("_ops") + exp = experiment_submit_for_review( + experiment=self.exp, user=self.owner + ) + exp = experiment_approve( + experiment=exp, approver=approver, comment="ok" + ) + with self.assertRaises(ForbiddenError): + experiment_start(experiment=exp, user=self.other) + + def test_other_experimenter_cannot_pause(self) -> None: + review_settings_update( + default_min_approvals=1, allow_any_approver=True + ) + approver = make_approver("_opp") + exp = experiment_submit_for_review( + experiment=self.exp, user=self.owner + ) + exp = experiment_approve( + experiment=exp, approver=approver, comment="ok" + ) + exp = experiment_start(experiment=exp, user=self.owner) + with self.assertRaises(ForbiddenError): + experiment_pause(experiment=exp, user=self.other) + + def test_admin_can_pause(self) -> None: + review_settings_update( + default_min_approvals=1, allow_any_approver=True + ) + approver = make_approver("_opa") + exp = experiment_submit_for_review( + experiment=self.exp, user=self.owner + ) + exp = experiment_approve( + experiment=exp, approver=approver, comment="ok" + ) + exp = experiment_start(experiment=exp, user=self.owner) + exp = experiment_pause(experiment=exp, user=self.admin) + self.assertEqual(exp.status, ExperimentStatus.PAUSED)