390 lines
12 KiB
Python
390 lines
12 KiB
Python
from decimal import Decimal
|
|
from typing import override
|
|
|
|
from django.core.cache import cache
|
|
from django.core.exceptions import ValidationError
|
|
from django.test import TestCase
|
|
from django.utils import timezone
|
|
|
|
from apps.events.services import (
|
|
decision_create,
|
|
event_type_create,
|
|
process_events_batch,
|
|
)
|
|
from apps.experiments.models import ExperimentStatus
|
|
from apps.experiments.services import (
|
|
experiment_approve,
|
|
experiment_complete,
|
|
experiment_create,
|
|
experiment_start,
|
|
experiment_submit_for_review,
|
|
variant_create,
|
|
)
|
|
from apps.experiments.tests.helpers import add_two_variants, make_flag
|
|
from apps.reviews.services import (
|
|
approver_group_create,
|
|
review_settings_update,
|
|
)
|
|
from apps.reviews.tests.helpers import (
|
|
make_approver,
|
|
make_experimenter,
|
|
make_viewer,
|
|
)
|
|
from config.errors import ForbiddenError
|
|
|
|
|
|
class InvalidLifecycleTransitionsTest(TestCase):
|
|
@override
|
|
def setUp(self) -> None:
|
|
cache.clear()
|
|
review_settings_update(
|
|
default_min_approvals=1,
|
|
allow_any_approver=True,
|
|
)
|
|
self.owner = make_experimenter("_ilt")
|
|
self.approver = make_approver("_ilt")
|
|
|
|
def test_cannot_start_draft_experiment(self) -> None:
|
|
flag = make_flag(suffix="_nsd", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="Draft Start",
|
|
owner=self.owner,
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(experiment)
|
|
|
|
with self.assertRaises(ValidationError) as ctx:
|
|
experiment_start(experiment=experiment, user=self.owner)
|
|
self.assertIn("status", ctx.exception.message_dict)
|
|
|
|
def test_cannot_start_without_enough_approvals(self) -> None:
|
|
flag = make_flag(suffix="_nsa", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="No Approval Start",
|
|
owner=self.owner,
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(experiment)
|
|
|
|
review_settings_update(
|
|
default_min_approvals=2,
|
|
allow_any_approver=True,
|
|
)
|
|
experiment = experiment_submit_for_review(
|
|
experiment=experiment, user=self.owner
|
|
)
|
|
experiment = experiment_approve(
|
|
experiment=experiment, approver=self.approver
|
|
)
|
|
self.assertEqual(experiment.status, ExperimentStatus.IN_REVIEW)
|
|
|
|
with self.assertRaises(ValidationError):
|
|
experiment_start(experiment=experiment, user=self.owner)
|
|
|
|
def test_cannot_submit_without_variants(self) -> None:
|
|
flag = make_flag(suffix="_nsv", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="No Variants",
|
|
owner=self.owner,
|
|
)
|
|
|
|
with self.assertRaises(ValidationError) as ctx:
|
|
experiment_submit_for_review(
|
|
experiment=experiment, user=self.owner
|
|
)
|
|
self.assertIn("variants", ctx.exception.message_dict)
|
|
|
|
def test_cannot_complete_without_rationale(self) -> None:
|
|
flag = make_flag(suffix="_ncr", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="No Rationale",
|
|
owner=self.owner,
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(experiment)
|
|
experiment = experiment_submit_for_review(
|
|
experiment=experiment, user=self.owner
|
|
)
|
|
experiment = experiment_approve(
|
|
experiment=experiment, approver=self.approver
|
|
)
|
|
experiment = experiment_start(experiment=experiment, user=self.owner)
|
|
|
|
with self.assertRaises(ValidationError) as ctx:
|
|
experiment_complete(
|
|
experiment=experiment,
|
|
user=self.owner,
|
|
outcome="no_effect",
|
|
rationale="",
|
|
)
|
|
self.assertIn("rationale", ctx.exception.message_dict)
|
|
|
|
def test_cannot_run_two_experiments_on_same_flag(self) -> None:
|
|
flag = make_flag(suffix="_dup", default="d")
|
|
|
|
exp1 = experiment_create(
|
|
flag=flag,
|
|
name="Exp1",
|
|
owner=self.owner,
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(exp1)
|
|
exp1 = experiment_submit_for_review(experiment=exp1, user=self.owner)
|
|
exp1 = experiment_approve(experiment=exp1, approver=self.approver)
|
|
exp1 = experiment_start(experiment=exp1, user=self.owner)
|
|
|
|
exp2 = experiment_create(
|
|
flag=flag,
|
|
name="Exp2",
|
|
owner=self.owner,
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(exp2)
|
|
exp2 = experiment_submit_for_review(experiment=exp2, user=self.owner)
|
|
exp2 = experiment_approve(experiment=exp2, approver=self.approver)
|
|
|
|
with self.assertRaises(ValidationError) as ctx:
|
|
experiment_start(experiment=exp2, user=self.owner)
|
|
self.assertIn("flag", ctx.exception.message_dict)
|
|
|
|
|
|
class ReviewPolicyEnforcementTest(TestCase):
|
|
@override
|
|
def setUp(self) -> None:
|
|
cache.clear()
|
|
|
|
def test_approver_group_restricts_who_can_approve(self) -> None:
|
|
review_settings_update(
|
|
default_min_approvals=1,
|
|
allow_any_approver=False,
|
|
)
|
|
|
|
owner = make_experimenter("_rpea")
|
|
approved_approver = make_approver("_rpea1")
|
|
unauthorized_approver = make_approver("_rpea2")
|
|
|
|
approver_group_create(
|
|
experimenter=owner,
|
|
approver_ids=[approved_approver.pk],
|
|
min_approvals=1,
|
|
)
|
|
|
|
flag = make_flag(suffix="_rpea", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="Restricted Approval",
|
|
owner=owner,
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(experiment)
|
|
|
|
experiment = experiment_submit_for_review(
|
|
experiment=experiment, user=owner
|
|
)
|
|
|
|
with self.assertRaises(ValidationError):
|
|
experiment_approve(
|
|
experiment=experiment,
|
|
approver=unauthorized_approver,
|
|
)
|
|
|
|
experiment = experiment_approve(
|
|
experiment=experiment,
|
|
approver=approved_approver,
|
|
)
|
|
self.assertEqual(experiment.status, ExperimentStatus.APPROVED)
|
|
|
|
def test_non_owner_cannot_submit_for_review(self) -> None:
|
|
viewer = make_viewer("_voe")
|
|
flag = make_flag(suffix="_voe", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="Viewer Experiment",
|
|
owner=viewer,
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
other_user = make_experimenter("_voe2")
|
|
|
|
with self.assertRaises(ForbiddenError):
|
|
experiment_submit_for_review(
|
|
experiment=experiment, user=other_user
|
|
)
|
|
|
|
|
|
class EventValidationIntegrationTest(TestCase):
|
|
@override
|
|
def setUp(self) -> None:
|
|
event_type_create(
|
|
name="neg_exposure",
|
|
display_name="Exposure",
|
|
is_exposure=True,
|
|
)
|
|
event_type_create(
|
|
name="neg_click",
|
|
display_name="Click",
|
|
requires_exposure=True,
|
|
required_fields=["screen"],
|
|
)
|
|
self.now = timezone.now().isoformat()
|
|
|
|
def test_batch_with_mixed_valid_and_invalid_events(self) -> None:
|
|
decision_create(
|
|
decision_id="neg_dec_1",
|
|
flag_key="test_flag",
|
|
subject_id="u1",
|
|
value="v",
|
|
reason="test",
|
|
)
|
|
|
|
process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "neg_exp_1",
|
|
"event_type": "neg_exposure",
|
|
"decision_id": "neg_dec_1",
|
|
"subject_id": "u1",
|
|
"timestamp": self.now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
|
|
result = process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "neg_valid_click",
|
|
"event_type": "neg_click",
|
|
"decision_id": "neg_dec_1",
|
|
"subject_id": "u1",
|
|
"timestamp": self.now,
|
|
"properties": {"screen": "home"},
|
|
},
|
|
{
|
|
"event_id": "neg_invalid_type",
|
|
"event_type": "nonexistent_type",
|
|
"decision_id": "neg_dec_1",
|
|
"subject_id": "u1",
|
|
"timestamp": self.now,
|
|
"properties": {},
|
|
},
|
|
{
|
|
"event_id": "neg_missing_field",
|
|
"event_type": "neg_click",
|
|
"decision_id": "neg_dec_1",
|
|
"subject_id": "u1",
|
|
"timestamp": self.now,
|
|
"properties": {},
|
|
},
|
|
{
|
|
"event_id": "neg_missing_decision",
|
|
"event_type": "neg_click",
|
|
"decision_id": "",
|
|
"subject_id": "u1",
|
|
"timestamp": self.now,
|
|
"properties": {"screen": "home"},
|
|
},
|
|
{
|
|
"event_id": "neg_bad_ts",
|
|
"event_type": "neg_click",
|
|
"decision_id": "neg_dec_1",
|
|
"subject_id": "u1",
|
|
"timestamp": 12345,
|
|
"properties": {"screen": "home"},
|
|
},
|
|
{
|
|
"event_id": "neg_exp_1",
|
|
"event_type": "neg_exposure",
|
|
"decision_id": "neg_dec_1",
|
|
"subject_id": "u1",
|
|
"timestamp": self.now,
|
|
"properties": {},
|
|
},
|
|
]
|
|
)
|
|
self.assertEqual(result.accepted, 1)
|
|
self.assertEqual(result.duplicates, 1)
|
|
self.assertGreaterEqual(result.rejected, 3)
|
|
|
|
def test_non_string_event_type_rejected(self) -> None:
|
|
result = process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "neg_bad_type",
|
|
"event_type": 999,
|
|
"decision_id": "dec",
|
|
"subject_id": "u1",
|
|
"timestamp": self.now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
self.assertEqual(result.rejected, 1)
|
|
self.assertEqual(result.accepted, 0)
|
|
|
|
|
|
class VariantWeightValidationTest(TestCase):
|
|
def test_weights_matching_allocation_allows_submit(self) -> None:
|
|
owner = make_experimenter("_vwv")
|
|
flag = make_flag(suffix="_vwv", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="Weight Validation",
|
|
owner=owner,
|
|
traffic_allocation=Decimal("30.00"),
|
|
)
|
|
variant_create(
|
|
experiment=experiment,
|
|
user=owner,
|
|
name="control",
|
|
value="a",
|
|
weight=Decimal("15.00"),
|
|
is_control=True,
|
|
)
|
|
variant_create(
|
|
experiment=experiment,
|
|
user=owner,
|
|
name="treatment",
|
|
value="b",
|
|
weight=Decimal("15.00"),
|
|
)
|
|
|
|
review_settings_update(
|
|
default_min_approvals=1,
|
|
allow_any_approver=True,
|
|
)
|
|
experiment = experiment_submit_for_review(
|
|
experiment=experiment, user=owner
|
|
)
|
|
self.assertEqual(experiment.status, ExperimentStatus.IN_REVIEW)
|
|
|
|
def test_weights_exceeding_allocation_rejected(self) -> None:
|
|
owner = make_experimenter("_vwe")
|
|
flag = make_flag(suffix="_vwe", default="d")
|
|
experiment = experiment_create(
|
|
flag=flag,
|
|
name="Weight Exceeds",
|
|
owner=owner,
|
|
traffic_allocation=Decimal("30.00"),
|
|
)
|
|
variant_create(
|
|
experiment=experiment,
|
|
user=owner,
|
|
name="control",
|
|
value="a",
|
|
weight=Decimal("15.00"),
|
|
is_control=True,
|
|
)
|
|
|
|
with self.assertRaises(ValidationError):
|
|
variant_create(
|
|
experiment=experiment,
|
|
user=owner,
|
|
name="treatment",
|
|
value="b",
|
|
weight=Decimal("20.00"),
|
|
)
|