feat(experiments): added experiments business logic
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
from decimal import Decimal
|
||||
|
||||
from apps.experiments.models import Experiment, Variant
|
||||
from apps.experiments.services import (
|
||||
experiment_create,
|
||||
variant_create,
|
||||
)
|
||||
from apps.flags.models import FeatureFlagType
|
||||
from apps.flags.services import feature_flag_create
|
||||
from apps.users.tests.helpers import make_user
|
||||
|
||||
|
||||
def make_flag(suffix="", value_type=FeatureFlagType.STRING, default="a"):
|
||||
return feature_flag_create(
|
||||
key=f"flag{suffix}",
|
||||
name=f"Flag {suffix}",
|
||||
value_type=value_type,
|
||||
default_value=default,
|
||||
)
|
||||
|
||||
|
||||
def make_experiment(
|
||||
flag=None,
|
||||
owner=None,
|
||||
suffix="",
|
||||
name=None,
|
||||
**kwargs,
|
||||
) -> Experiment:
|
||||
if not flag:
|
||||
flag = make_flag(suffix=suffix)
|
||||
if not owner:
|
||||
owner = make_user(
|
||||
username=f"owner{suffix}",
|
||||
email=f"owner{suffix}@lotty.local",
|
||||
)
|
||||
return experiment_create(
|
||||
flag=flag,
|
||||
name=name or f"Exp{suffix}",
|
||||
owner=owner,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def add_two_variants(
|
||||
experiment: Experiment,
|
||||
weight_a: Decimal = Decimal("50.00"),
|
||||
weight_b: Decimal = Decimal("50.00"),
|
||||
) -> tuple[Variant, Variant]:
|
||||
v_control = variant_create(
|
||||
experiment=experiment,
|
||||
user=experiment.owner,
|
||||
name="control",
|
||||
value="a",
|
||||
weight=weight_a,
|
||||
is_control=True,
|
||||
)
|
||||
v_treatment = variant_create(
|
||||
experiment=experiment,
|
||||
user=experiment.owner,
|
||||
name="treatment",
|
||||
value="b",
|
||||
weight=weight_b,
|
||||
is_control=False,
|
||||
)
|
||||
return v_control, v_treatment
|
||||
@@ -0,0 +1,179 @@
|
||||
from decimal import Decimal
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.test import TestCase
|
||||
|
||||
from apps.experiments.models import (
|
||||
ALLOWED_TRANSITIONS,
|
||||
FROZEN_AFTER_START,
|
||||
STARTED_STATUSES,
|
||||
Experiment,
|
||||
ExperimentStatus,
|
||||
Variant,
|
||||
)
|
||||
from apps.experiments.tests.helpers import (
|
||||
add_two_variants,
|
||||
make_experiment,
|
||||
make_flag,
|
||||
)
|
||||
from apps.users.tests.helpers import make_user
|
||||
|
||||
|
||||
class TransitionValidatorTest(TestCase):
|
||||
def test_all_allowed_transitions_succeed(self) -> None:
|
||||
exp = make_experiment(suffix="_tr")
|
||||
for from_status, targets in ALLOWED_TRANSITIONS.items():
|
||||
for target in targets:
|
||||
Experiment.objects.filter(pk=exp.pk).update(status=from_status)
|
||||
exp.refresh_from_db()
|
||||
exp.status = target
|
||||
exp.save()
|
||||
exp.refresh_from_db()
|
||||
self.assertEqual(exp.status, target)
|
||||
|
||||
def test_forbidden_transition_raises(self) -> None:
|
||||
exp = make_experiment(suffix="_fb")
|
||||
Experiment.objects.filter(pk=exp.pk).update(
|
||||
status=ExperimentStatus.DRAFT
|
||||
)
|
||||
exp.refresh_from_db()
|
||||
exp.status = ExperimentStatus.RUNNING
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
exp.save()
|
||||
self.assertIn("status", ctx.exception.message_dict)
|
||||
|
||||
def test_same_status_save_allowed(self) -> None:
|
||||
exp = make_experiment(suffix="_ss")
|
||||
exp.name = "Updated name"
|
||||
exp.save()
|
||||
exp.refresh_from_db()
|
||||
self.assertEqual(exp.name, "Updated name")
|
||||
|
||||
|
||||
class FrozenFieldsValidatorTest(TestCase):
|
||||
def test_frozen_fields_after_start(self) -> None:
|
||||
exp = make_experiment(
|
||||
suffix="_ff",
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(exp)
|
||||
Experiment.objects.filter(pk=exp.pk).update(
|
||||
status=ExperimentStatus.RUNNING
|
||||
)
|
||||
exp.refresh_from_db()
|
||||
|
||||
for field_name in FROZEN_AFTER_START:
|
||||
exp_copy = Experiment.objects.get(pk=exp.pk)
|
||||
if field_name == "traffic_allocation":
|
||||
exp_copy.traffic_allocation = Decimal("10.00")
|
||||
elif field_name == "targeting_rules":
|
||||
exp_copy.targeting_rules = "changed"
|
||||
elif field_name == "flag":
|
||||
exp_copy.flag = make_flag(suffix=f"_alt_{field_name}")
|
||||
with self.assertRaises(
|
||||
ValidationError, msg=f"Expected freeze on {field_name}"
|
||||
):
|
||||
exp_copy.save()
|
||||
|
||||
def test_non_frozen_fields_editable_after_start(self) -> None:
|
||||
exp = make_experiment(suffix="_nf")
|
||||
Experiment.objects.filter(pk=exp.pk).update(
|
||||
status=ExperimentStatus.RUNNING
|
||||
)
|
||||
exp.refresh_from_db()
|
||||
exp.name = "New Name"
|
||||
exp.description = "New Desc"
|
||||
exp.save()
|
||||
exp.refresh_from_db()
|
||||
self.assertEqual(exp.name, "New Name")
|
||||
|
||||
def test_frozen_fields_editable_in_draft(self) -> None:
|
||||
exp = make_experiment(suffix="_dft")
|
||||
exp.traffic_allocation = Decimal("25.00")
|
||||
exp.targeting_rules = "age > 18"
|
||||
exp.save()
|
||||
exp.refresh_from_db()
|
||||
self.assertEqual(exp.traffic_allocation, Decimal("25.00"))
|
||||
|
||||
def test_frozen_in_all_started_statuses(self) -> None:
|
||||
for status in STARTED_STATUSES:
|
||||
exp = make_experiment(suffix=f"_s{status}")
|
||||
Experiment.objects.filter(pk=exp.pk).update(status=status)
|
||||
exp.refresh_from_db()
|
||||
exp.traffic_allocation = Decimal("1.00")
|
||||
with self.assertRaises(
|
||||
ValidationError, msg=f"Expected freeze in {status}"
|
||||
):
|
||||
exp.save()
|
||||
|
||||
|
||||
class UniqueActiveFlagValidatorTest(TestCase):
|
||||
def test_two_running_experiments_same_flag_rejected(self) -> None:
|
||||
flag = make_flag(suffix="_uaf")
|
||||
owner = make_user(username="uaf_owner", email="uaf_owner@lotty.local")
|
||||
exp1 = make_experiment(flag=flag, owner=owner, suffix="_u1", name="E1")
|
||||
Experiment.objects.filter(pk=exp1.pk).update(
|
||||
status=ExperimentStatus.RUNNING
|
||||
)
|
||||
|
||||
exp2 = make_experiment(flag=flag, owner=owner, suffix="_u2", name="E2")
|
||||
Experiment.objects.filter(pk=exp2.pk).update(
|
||||
status=ExperimentStatus.APPROVED
|
||||
)
|
||||
exp2.refresh_from_db()
|
||||
exp2.status = ExperimentStatus.RUNNING
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
exp2.save()
|
||||
self.assertIn("flag", ctx.exception.message_dict)
|
||||
|
||||
def test_different_flags_both_running_allowed(self) -> None:
|
||||
owner = make_user(username="dfb_owner", email="dfb_owner@lotty.local")
|
||||
exp1 = make_experiment(owner=owner, suffix="_df1")
|
||||
exp2 = make_experiment(owner=owner, suffix="_df2")
|
||||
Experiment.objects.filter(pk=exp1.pk).update(
|
||||
status=ExperimentStatus.RUNNING
|
||||
)
|
||||
Experiment.objects.filter(pk=exp2.pk).update(
|
||||
status=ExperimentStatus.APPROVED
|
||||
)
|
||||
exp2.refresh_from_db()
|
||||
exp2.status = ExperimentStatus.RUNNING
|
||||
exp2.save()
|
||||
exp2.refresh_from_db()
|
||||
self.assertEqual(exp2.status, ExperimentStatus.RUNNING)
|
||||
|
||||
|
||||
class VariantFrozenValidatorTest(TestCase):
|
||||
def test_variant_not_editable_after_start(self) -> None:
|
||||
exp = make_experiment(suffix="_vf")
|
||||
_vc, vt = add_two_variants(exp)
|
||||
Experiment.objects.filter(pk=exp.pk).update(
|
||||
status=ExperimentStatus.RUNNING
|
||||
)
|
||||
vt.refresh_from_db()
|
||||
vt.name = "changed"
|
||||
with self.assertRaises(ValidationError):
|
||||
vt.save()
|
||||
|
||||
def test_variant_editable_in_draft(self) -> None:
|
||||
exp = make_experiment(suffix="_ved")
|
||||
_vc, vt = add_two_variants(exp)
|
||||
vt.name = "renamed"
|
||||
vt.save()
|
||||
vt.refresh_from_db()
|
||||
self.assertEqual(vt.name, "renamed")
|
||||
|
||||
def test_new_variant_rejected_after_start(self) -> None:
|
||||
exp = make_experiment(suffix="_vnr")
|
||||
add_two_variants(exp)
|
||||
Experiment.objects.filter(pk=exp.pk).update(
|
||||
status=ExperimentStatus.RUNNING
|
||||
)
|
||||
exp.refresh_from_db()
|
||||
with self.assertRaises(ValidationError):
|
||||
Variant(
|
||||
experiment=exp,
|
||||
name="extra",
|
||||
value="c",
|
||||
weight=Decimal("10.00"),
|
||||
).save()
|
||||
@@ -0,0 +1,585 @@
|
||||
from decimal import Decimal
|
||||
from typing import override
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.test import TestCase
|
||||
|
||||
from apps.experiments.models import (
|
||||
Approval,
|
||||
Experiment,
|
||||
ExperimentLog,
|
||||
ExperimentOutcome,
|
||||
ExperimentStatus,
|
||||
LogType,
|
||||
OutcomeType,
|
||||
)
|
||||
from apps.experiments.services import (
|
||||
ensure_owner_or_admin,
|
||||
experiment_approve,
|
||||
experiment_archive,
|
||||
experiment_complete,
|
||||
experiment_create,
|
||||
experiment_pause,
|
||||
experiment_reject,
|
||||
experiment_reopen,
|
||||
experiment_request_changes,
|
||||
experiment_resume,
|
||||
experiment_start,
|
||||
experiment_submit_for_review,
|
||||
experiment_update,
|
||||
variant_create,
|
||||
variant_delete,
|
||||
variant_update,
|
||||
)
|
||||
from apps.experiments.tests.helpers import (
|
||||
add_two_variants,
|
||||
make_experiment,
|
||||
make_flag,
|
||||
)
|
||||
from apps.reviews.services import approver_group_create, review_settings_update
|
||||
from apps.reviews.tests.helpers import (
|
||||
make_admin,
|
||||
make_approver,
|
||||
make_experimenter,
|
||||
make_viewer,
|
||||
)
|
||||
from config.errors import ForbiddenError
|
||||
|
||||
|
||||
class ExperimentCreateTest(TestCase):
|
||||
def test_creates_with_defaults(self) -> None:
|
||||
exp = make_experiment(suffix="_cw")
|
||||
self.assertEqual(exp.status, ExperimentStatus.DRAFT)
|
||||
self.assertEqual(exp.version, 1)
|
||||
self.assertEqual(exp.traffic_allocation, Decimal("100.00"))
|
||||
|
||||
def test_creates_log_entry(self) -> None:
|
||||
exp = make_experiment(suffix="_cl")
|
||||
logs = ExperimentLog.objects.filter(experiment=exp)
|
||||
self.assertEqual(logs.count(), 1)
|
||||
self.assertEqual(logs.first().log_type, LogType.STATUS_CHANGE)
|
||||
|
||||
|
||||
class ExperimentUpdateTest(TestCase):
|
||||
def test_updates_allowed_fields(self) -> None:
|
||||
exp = make_experiment(suffix="_uf")
|
||||
exp = experiment_update(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="Updated",
|
||||
description="desc",
|
||||
)
|
||||
exp.refresh_from_db()
|
||||
self.assertEqual(exp.name, "Updated")
|
||||
self.assertEqual(exp.description, "desc")
|
||||
|
||||
def test_rejects_disallowed_field(self) -> None:
|
||||
exp = make_experiment(suffix="_rd")
|
||||
with self.assertRaises(ValidationError):
|
||||
experiment_update(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
status="running",
|
||||
)
|
||||
|
||||
|
||||
class VariantCrudTest(TestCase):
|
||||
def test_create_variants(self) -> None:
|
||||
exp = make_experiment(suffix="_vc")
|
||||
vc, vt = add_two_variants(exp)
|
||||
self.assertTrue(vc.is_control)
|
||||
self.assertFalse(vt.is_control)
|
||||
|
||||
def test_total_weight_exceeds_allocation_raises(self) -> None:
|
||||
exp = make_experiment(
|
||||
suffix="_twe", traffic_allocation=Decimal("50.00")
|
||||
)
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="control",
|
||||
value="a",
|
||||
weight=Decimal("30.00"),
|
||||
is_control=True,
|
||||
)
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="treatment",
|
||||
value="b",
|
||||
weight=Decimal("30.00"),
|
||||
)
|
||||
self.assertIn("weight", ctx.exception.message_dict)
|
||||
|
||||
def test_two_controls_raises(self) -> None:
|
||||
exp = make_experiment(suffix="_2c")
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="c1",
|
||||
value="a",
|
||||
weight=Decimal("20.00"),
|
||||
is_control=True,
|
||||
)
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="c2",
|
||||
value="b",
|
||||
weight=Decimal("20.00"),
|
||||
is_control=True,
|
||||
)
|
||||
self.assertIn("is_control", ctx.exception.message_dict)
|
||||
|
||||
def test_update_variant(self) -> None:
|
||||
exp = make_experiment(suffix="_vu")
|
||||
_vc, vt = add_two_variants(exp)
|
||||
vt = variant_update(variant=vt, user=exp.owner, name="renamed")
|
||||
self.assertEqual(vt.name, "renamed")
|
||||
|
||||
def test_delete_variant_in_draft(self) -> None:
|
||||
exp = make_experiment(suffix="_vd")
|
||||
_vc, vt = add_two_variants(exp)
|
||||
variant_delete(variant=vt, user=exp.owner)
|
||||
self.assertEqual(exp.variants.count(), 1)
|
||||
|
||||
def test_delete_variant_not_in_draft_raises(self) -> None:
|
||||
exp = make_experiment(suffix="_vnd")
|
||||
_vc, vt = add_two_variants(exp)
|
||||
Experiment.objects.filter(pk=exp.pk).update(
|
||||
status=ExperimentStatus.IN_REVIEW
|
||||
)
|
||||
vt.refresh_from_db()
|
||||
with self.assertRaises(ValidationError):
|
||||
variant_delete(variant=vt, user=exp.owner)
|
||||
|
||||
|
||||
class SubmitForReviewTest(TestCase):
|
||||
def test_submit_with_valid_variants(self) -> None:
|
||||
exp = make_experiment(suffix="_sr")
|
||||
add_two_variants(exp)
|
||||
exp = experiment_submit_for_review(experiment=exp, user=exp.owner)
|
||||
self.assertEqual(exp.status, ExperimentStatus.IN_REVIEW)
|
||||
|
||||
def test_submit_without_variants_raises(self) -> None:
|
||||
exp = make_experiment(suffix="_snv")
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_submit_for_review(experiment=exp, user=exp.owner)
|
||||
self.assertIn("variants", ctx.exception.message_dict)
|
||||
|
||||
def test_submit_without_control_raises(self) -> None:
|
||||
exp = make_experiment(suffix="_snc")
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="v1",
|
||||
value="a",
|
||||
weight=Decimal("50.00"),
|
||||
is_control=False,
|
||||
)
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="v2",
|
||||
value="b",
|
||||
weight=Decimal("50.00"),
|
||||
is_control=False,
|
||||
)
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_submit_for_review(experiment=exp, user=exp.owner)
|
||||
self.assertIn("is_control", ctx.exception.message_dict)
|
||||
|
||||
def test_submit_weights_not_equal_allocation_raises(self) -> None:
|
||||
exp = make_experiment(suffix="_sne")
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="c",
|
||||
value="a",
|
||||
weight=Decimal("30.00"),
|
||||
is_control=True,
|
||||
)
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="t",
|
||||
value="b",
|
||||
weight=Decimal("20.00"),
|
||||
)
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_submit_for_review(experiment=exp, user=exp.owner)
|
||||
self.assertIn("weight", ctx.exception.message_dict)
|
||||
|
||||
def test_submit_one_variant_raises(self) -> None:
|
||||
exp = make_experiment(suffix="_s1v")
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=exp.owner,
|
||||
name="c",
|
||||
value="a",
|
||||
weight=Decimal("100.00"),
|
||||
is_control=True,
|
||||
)
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_submit_for_review(experiment=exp, user=exp.owner)
|
||||
self.assertIn("variants", ctx.exception.message_dict)
|
||||
|
||||
|
||||
class ApprovalFlowTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
self.experimenter = make_experimenter("_af")
|
||||
self.approver1 = make_approver("_af1")
|
||||
self.approver2 = make_approver("_af2")
|
||||
self.flag = make_flag(suffix="_af")
|
||||
approver_group_create(
|
||||
experimenter=self.experimenter,
|
||||
approver_ids=[
|
||||
str(self.approver1.pk),
|
||||
str(self.approver2.pk),
|
||||
],
|
||||
min_approvals=2,
|
||||
)
|
||||
self.exp = experiment_create(
|
||||
flag=self.flag,
|
||||
name="AF Exp",
|
||||
owner=self.experimenter,
|
||||
)
|
||||
add_two_variants(self.exp)
|
||||
self.exp = experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.experimenter
|
||||
)
|
||||
|
||||
def test_first_approval_stays_in_review(self) -> None:
|
||||
exp = experiment_approve(
|
||||
experiment=self.exp,
|
||||
approver=self.approver1,
|
||||
comment="lgtm",
|
||||
)
|
||||
self.assertEqual(exp.status, ExperimentStatus.IN_REVIEW)
|
||||
self.assertEqual(Approval.objects.filter(experiment=exp).count(), 1)
|
||||
|
||||
def test_second_approval_transitions_to_approved(self) -> None:
|
||||
experiment_approve(
|
||||
experiment=self.exp,
|
||||
approver=self.approver1,
|
||||
)
|
||||
exp = experiment_approve(
|
||||
experiment=self.exp,
|
||||
approver=self.approver2,
|
||||
)
|
||||
self.assertEqual(exp.status, ExperimentStatus.APPROVED)
|
||||
|
||||
def test_duplicate_approval_raises(self) -> None:
|
||||
experiment_approve(
|
||||
experiment=self.exp,
|
||||
approver=self.approver1,
|
||||
)
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_approve(
|
||||
experiment=self.exp,
|
||||
approver=self.approver1,
|
||||
)
|
||||
self.assertIn("approver", ctx.exception.message_dict)
|
||||
|
||||
def test_unauthorized_approver_raises(self) -> None:
|
||||
viewer = make_viewer("_unauth")
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_approve(
|
||||
experiment=self.exp,
|
||||
approver=viewer,
|
||||
)
|
||||
self.assertIn("approver", ctx.exception.message_dict)
|
||||
|
||||
|
||||
class RejectAndReqChangesTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
self.experimenter = make_experimenter("_rr")
|
||||
self.approver1 = make_approver("_rr1")
|
||||
self.approver2 = make_approver("_rr2")
|
||||
self.flag = make_flag(suffix="_rr")
|
||||
approver_group_create(
|
||||
experimenter=self.experimenter,
|
||||
approver_ids=[
|
||||
str(self.approver1.pk),
|
||||
str(self.approver2.pk),
|
||||
],
|
||||
min_approvals=2,
|
||||
)
|
||||
self.exp = experiment_create(
|
||||
flag=self.flag,
|
||||
name="RR Exp",
|
||||
owner=self.experimenter,
|
||||
)
|
||||
add_two_variants(self.exp)
|
||||
self.exp = experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.experimenter
|
||||
)
|
||||
|
||||
def test_reject(self) -> None:
|
||||
exp = experiment_reject(
|
||||
experiment=self.exp,
|
||||
user=self.approver1,
|
||||
comment="not ready",
|
||||
)
|
||||
self.assertEqual(exp.status, ExperimentStatus.REJECTED)
|
||||
|
||||
def test_request_changes_clears_approvals(self) -> None:
|
||||
experiment_approve(
|
||||
experiment=self.exp,
|
||||
approver=self.approver1,
|
||||
)
|
||||
self.exp.refresh_from_db()
|
||||
exp = experiment_request_changes(
|
||||
experiment=self.exp, user=self.approver2, comment="fix please"
|
||||
)
|
||||
self.assertEqual(exp.status, ExperimentStatus.DRAFT)
|
||||
self.assertEqual(Approval.objects.filter(experiment=exp).count(), 0)
|
||||
|
||||
def test_reopen_from_rejected(self) -> None:
|
||||
exp = experiment_reject(
|
||||
experiment=self.exp,
|
||||
user=self.approver1,
|
||||
)
|
||||
exp = experiment_reopen(experiment=exp, user=self.experimenter)
|
||||
self.assertEqual(exp.status, ExperimentStatus.DRAFT)
|
||||
|
||||
|
||||
class LifecycleFlowTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
self.experimenter = make_experimenter("_lf")
|
||||
self.approver = make_approver("_lf")
|
||||
self.flag = make_flag(suffix="_lf")
|
||||
review_settings_update(
|
||||
default_min_approvals=1, allow_any_approver=True
|
||||
)
|
||||
self.exp = experiment_create(
|
||||
flag=self.flag,
|
||||
name="LF Exp",
|
||||
owner=self.experimenter,
|
||||
)
|
||||
self.vc, self.vt = add_two_variants(self.exp)
|
||||
|
||||
def _approve_and_start(self) -> Experiment:
|
||||
exp = experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.experimenter
|
||||
)
|
||||
exp = experiment_approve(experiment=exp, approver=self.approver)
|
||||
return experiment_start(experiment=exp, user=self.experimenter)
|
||||
|
||||
def test_full_lifecycle_to_complete(self) -> None:
|
||||
exp = self._approve_and_start()
|
||||
self.assertEqual(exp.status, ExperimentStatus.RUNNING)
|
||||
|
||||
exp = experiment_pause(experiment=exp, user=self.experimenter)
|
||||
self.assertEqual(exp.status, ExperimentStatus.PAUSED)
|
||||
|
||||
exp = experiment_resume(experiment=exp, user=self.experimenter)
|
||||
self.assertEqual(exp.status, ExperimentStatus.RUNNING)
|
||||
|
||||
exp = experiment_complete(
|
||||
experiment=exp,
|
||||
user=self.experimenter,
|
||||
outcome=OutcomeType.ROLLOUT,
|
||||
rationale="Treatment won",
|
||||
winning_variant_id=str(self.vt.pk),
|
||||
)
|
||||
self.assertEqual(exp.status, ExperimentStatus.COMPLETED)
|
||||
outcome = ExperimentOutcome.objects.get(experiment=exp)
|
||||
self.assertEqual(outcome.outcome, OutcomeType.ROLLOUT)
|
||||
self.assertEqual(outcome.winning_variant, self.vt)
|
||||
|
||||
def test_complete_rollback(self) -> None:
|
||||
exp = self._approve_and_start()
|
||||
exp = experiment_complete(
|
||||
experiment=exp,
|
||||
user=self.experimenter,
|
||||
outcome=OutcomeType.ROLLBACK,
|
||||
rationale="No improvement",
|
||||
)
|
||||
self.assertEqual(exp.status, ExperimentStatus.COMPLETED)
|
||||
|
||||
def test_complete_rollout_without_winner_raises(self) -> None:
|
||||
exp = self._approve_and_start()
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_complete(
|
||||
experiment=exp,
|
||||
user=self.experimenter,
|
||||
outcome=OutcomeType.ROLLOUT,
|
||||
rationale="Treatment won",
|
||||
)
|
||||
self.assertIn("winning_variant_id", ctx.exception.message_dict)
|
||||
|
||||
def test_complete_empty_rationale_raises(self) -> None:
|
||||
exp = self._approve_and_start()
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_complete(
|
||||
experiment=exp,
|
||||
user=self.experimenter,
|
||||
outcome=OutcomeType.ROLLBACK,
|
||||
rationale=" ",
|
||||
)
|
||||
self.assertIn("rationale", ctx.exception.message_dict)
|
||||
|
||||
def test_archive_after_complete(self) -> None:
|
||||
exp = self._approve_and_start()
|
||||
exp = experiment_complete(
|
||||
experiment=exp,
|
||||
user=self.experimenter,
|
||||
outcome=OutcomeType.NO_EFFECT,
|
||||
rationale="No effect observed",
|
||||
)
|
||||
exp = experiment_archive(experiment=exp, user=self.experimenter)
|
||||
self.assertEqual(exp.status, ExperimentStatus.ARCHIVED)
|
||||
|
||||
def test_start_with_competing_active_raises(self) -> None:
|
||||
exp1 = self._approve_and_start()
|
||||
self.assertEqual(exp1.status, ExperimentStatus.RUNNING)
|
||||
|
||||
exp2 = experiment_create(
|
||||
flag=self.flag,
|
||||
name="LF Exp 2",
|
||||
owner=self.experimenter,
|
||||
)
|
||||
add_two_variants(exp2)
|
||||
exp2 = experiment_submit_for_review(
|
||||
experiment=exp2, user=self.experimenter
|
||||
)
|
||||
exp2 = experiment_approve(experiment=exp2, approver=self.approver)
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_start(experiment=exp2, user=self.experimenter)
|
||||
self.assertIn("flag", ctx.exception.message_dict)
|
||||
|
||||
def test_start_from_draft_raises(self) -> None:
|
||||
with self.assertRaises(ValidationError):
|
||||
experiment_start(experiment=self.exp, user=self.experimenter)
|
||||
|
||||
def test_audit_log_tracks_all_transitions(self) -> None:
|
||||
exp = self._approve_and_start()
|
||||
exp = experiment_complete(
|
||||
experiment=exp,
|
||||
user=self.experimenter,
|
||||
outcome=OutcomeType.ROLLBACK,
|
||||
rationale="done",
|
||||
)
|
||||
logs = ExperimentLog.objects.filter(experiment=exp).order_by(
|
||||
"created_at"
|
||||
)
|
||||
log_types = list(logs.values_list("log_type", flat=True))
|
||||
self.assertIn(LogType.STATUS_CHANGE, log_types)
|
||||
self.assertIn(LogType.REVIEW_REQUESTED, log_types)
|
||||
self.assertIn(LogType.APPROVAL, log_types)
|
||||
self.assertIn(LogType.COMPLETED, log_types)
|
||||
|
||||
|
||||
class OwnershipPermissionTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
self.owner = make_experimenter("_own")
|
||||
self.other = make_experimenter("_oth")
|
||||
self.admin = make_admin("_adm")
|
||||
self.flag = make_flag(suffix="_op")
|
||||
self.exp = experiment_create(
|
||||
flag=self.flag,
|
||||
name="Ownership Exp",
|
||||
owner=self.owner,
|
||||
)
|
||||
add_two_variants(self.exp)
|
||||
|
||||
def test_owner_can_update(self) -> None:
|
||||
exp = experiment_update(
|
||||
experiment=self.exp, user=self.owner, name="Renamed"
|
||||
)
|
||||
self.assertEqual(exp.name, "Renamed")
|
||||
|
||||
def test_admin_can_update(self) -> None:
|
||||
exp = experiment_update(
|
||||
experiment=self.exp, user=self.admin, name="Admin Renamed"
|
||||
)
|
||||
self.assertEqual(exp.name, "Admin Renamed")
|
||||
|
||||
def test_other_experimenter_cannot_update(self) -> None:
|
||||
with self.assertRaises(ForbiddenError):
|
||||
experiment_update(
|
||||
experiment=self.exp, user=self.other, name="Hijack"
|
||||
)
|
||||
|
||||
def test_other_experimenter_cannot_create_variant(self) -> None:
|
||||
with self.assertRaises(ForbiddenError):
|
||||
variant_create(
|
||||
experiment=self.exp,
|
||||
user=self.other,
|
||||
name="v3",
|
||||
value="c",
|
||||
weight=Decimal("10.00"),
|
||||
)
|
||||
|
||||
def test_other_experimenter_cannot_update_variant(self) -> None:
|
||||
_vc, vt = self.exp.variants.all()[0], self.exp.variants.all()[1]
|
||||
with self.assertRaises(ForbiddenError):
|
||||
variant_update(variant=vt, user=self.other, name="hijack")
|
||||
|
||||
def test_other_experimenter_cannot_delete_variant(self) -> None:
|
||||
vt = self.exp.variants.filter(is_control=False).first()
|
||||
with self.assertRaises(ForbiddenError):
|
||||
variant_delete(variant=vt, user=self.other)
|
||||
|
||||
def test_other_experimenter_cannot_submit_for_review(self) -> None:
|
||||
with self.assertRaises(ForbiddenError):
|
||||
experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.other
|
||||
)
|
||||
|
||||
def test_admin_can_submit_for_review(self) -> None:
|
||||
exp = experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.admin
|
||||
)
|
||||
self.assertEqual(exp.status, ExperimentStatus.IN_REVIEW)
|
||||
|
||||
def test_other_experimenter_cannot_start(self) -> None:
|
||||
review_settings_update(
|
||||
default_min_approvals=1, allow_any_approver=True
|
||||
)
|
||||
approver = make_approver("_ops")
|
||||
exp = experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.owner
|
||||
)
|
||||
exp = experiment_approve(
|
||||
experiment=exp, approver=approver, comment="ok"
|
||||
)
|
||||
with self.assertRaises(ForbiddenError):
|
||||
experiment_start(experiment=exp, user=self.other)
|
||||
|
||||
def test_other_experimenter_cannot_pause(self) -> None:
|
||||
review_settings_update(
|
||||
default_min_approvals=1, allow_any_approver=True
|
||||
)
|
||||
approver = make_approver("_opp")
|
||||
exp = experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.owner
|
||||
)
|
||||
exp = experiment_approve(
|
||||
experiment=exp, approver=approver, comment="ok"
|
||||
)
|
||||
exp = experiment_start(experiment=exp, user=self.owner)
|
||||
with self.assertRaises(ForbiddenError):
|
||||
experiment_pause(experiment=exp, user=self.other)
|
||||
|
||||
def test_admin_can_pause(self) -> None:
|
||||
review_settings_update(
|
||||
default_min_approvals=1, allow_any_approver=True
|
||||
)
|
||||
approver = make_approver("_opa")
|
||||
exp = experiment_submit_for_review(
|
||||
experiment=self.exp, user=self.owner
|
||||
)
|
||||
exp = experiment_approve(
|
||||
experiment=exp, approver=approver, comment="ok"
|
||||
)
|
||||
exp = experiment_start(experiment=exp, user=self.owner)
|
||||
exp = experiment_pause(experiment=exp, user=self.admin)
|
||||
self.assertEqual(exp.status, ExperimentStatus.PAUSED)
|
||||
Reference in New Issue
Block a user