test(integration): added integration tests
This commit is contained in:
@@ -0,0 +1,389 @@
|
||||
from decimal import Decimal
|
||||
from typing import override
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.events.services import (
|
||||
decision_create,
|
||||
event_type_create,
|
||||
process_events_batch,
|
||||
)
|
||||
from apps.experiments.models import ExperimentStatus
|
||||
from apps.experiments.services import (
|
||||
experiment_approve,
|
||||
experiment_complete,
|
||||
experiment_create,
|
||||
experiment_start,
|
||||
experiment_submit_for_review,
|
||||
variant_create,
|
||||
)
|
||||
from apps.experiments.tests.helpers import add_two_variants, make_flag
|
||||
from apps.reviews.services import (
|
||||
approver_group_create,
|
||||
review_settings_update,
|
||||
)
|
||||
from apps.reviews.tests.helpers import (
|
||||
make_approver,
|
||||
make_experimenter,
|
||||
make_viewer,
|
||||
)
|
||||
from config.errors import ForbiddenError
|
||||
|
||||
|
||||
class InvalidLifecycleTransitionsTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
cache.clear()
|
||||
review_settings_update(
|
||||
default_min_approvals=1,
|
||||
allow_any_approver=True,
|
||||
)
|
||||
self.owner = make_experimenter("_ilt")
|
||||
self.approver = make_approver("_ilt")
|
||||
|
||||
def test_cannot_start_draft_experiment(self) -> None:
|
||||
flag = make_flag(suffix="_nsd", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="Draft Start",
|
||||
owner=self.owner,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(experiment)
|
||||
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_start(experiment=experiment, user=self.owner)
|
||||
self.assertIn("status", ctx.exception.message_dict)
|
||||
|
||||
def test_cannot_start_without_enough_approvals(self) -> None:
|
||||
flag = make_flag(suffix="_nsa", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="No Approval Start",
|
||||
owner=self.owner,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(experiment)
|
||||
|
||||
review_settings_update(
|
||||
default_min_approvals=2,
|
||||
allow_any_approver=True,
|
||||
)
|
||||
experiment = experiment_submit_for_review(
|
||||
experiment=experiment, user=self.owner
|
||||
)
|
||||
experiment = experiment_approve(
|
||||
experiment=experiment, approver=self.approver
|
||||
)
|
||||
self.assertEqual(experiment.status, ExperimentStatus.IN_REVIEW)
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
experiment_start(experiment=experiment, user=self.owner)
|
||||
|
||||
def test_cannot_submit_without_variants(self) -> None:
|
||||
flag = make_flag(suffix="_nsv", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="No Variants",
|
||||
owner=self.owner,
|
||||
)
|
||||
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_submit_for_review(
|
||||
experiment=experiment, user=self.owner
|
||||
)
|
||||
self.assertIn("variants", ctx.exception.message_dict)
|
||||
|
||||
def test_cannot_complete_without_rationale(self) -> None:
|
||||
flag = make_flag(suffix="_ncr", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="No Rationale",
|
||||
owner=self.owner,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(experiment)
|
||||
experiment = experiment_submit_for_review(
|
||||
experiment=experiment, user=self.owner
|
||||
)
|
||||
experiment = experiment_approve(
|
||||
experiment=experiment, approver=self.approver
|
||||
)
|
||||
experiment = experiment_start(experiment=experiment, user=self.owner)
|
||||
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_complete(
|
||||
experiment=experiment,
|
||||
user=self.owner,
|
||||
outcome="no_effect",
|
||||
rationale="",
|
||||
)
|
||||
self.assertIn("rationale", ctx.exception.message_dict)
|
||||
|
||||
def test_cannot_run_two_experiments_on_same_flag(self) -> None:
|
||||
flag = make_flag(suffix="_dup", default="d")
|
||||
|
||||
exp1 = experiment_create(
|
||||
flag=flag,
|
||||
name="Exp1",
|
||||
owner=self.owner,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(exp1)
|
||||
exp1 = experiment_submit_for_review(experiment=exp1, user=self.owner)
|
||||
exp1 = experiment_approve(experiment=exp1, approver=self.approver)
|
||||
exp1 = experiment_start(experiment=exp1, user=self.owner)
|
||||
|
||||
exp2 = experiment_create(
|
||||
flag=flag,
|
||||
name="Exp2",
|
||||
owner=self.owner,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(exp2)
|
||||
exp2 = experiment_submit_for_review(experiment=exp2, user=self.owner)
|
||||
exp2 = experiment_approve(experiment=exp2, approver=self.approver)
|
||||
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
experiment_start(experiment=exp2, user=self.owner)
|
||||
self.assertIn("flag", ctx.exception.message_dict)
|
||||
|
||||
|
||||
class ReviewPolicyEnforcementTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
cache.clear()
|
||||
|
||||
def test_approver_group_restricts_who_can_approve(self) -> None:
|
||||
review_settings_update(
|
||||
default_min_approvals=1,
|
||||
allow_any_approver=False,
|
||||
)
|
||||
|
||||
owner = make_experimenter("_rpea")
|
||||
approved_approver = make_approver("_rpea1")
|
||||
unauthorized_approver = make_approver("_rpea2")
|
||||
|
||||
approver_group_create(
|
||||
experimenter=owner,
|
||||
approver_ids=[approved_approver.pk],
|
||||
min_approvals=1,
|
||||
)
|
||||
|
||||
flag = make_flag(suffix="_rpea", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="Restricted Approval",
|
||||
owner=owner,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(experiment)
|
||||
|
||||
experiment = experiment_submit_for_review(
|
||||
experiment=experiment, user=owner
|
||||
)
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
experiment_approve(
|
||||
experiment=experiment,
|
||||
approver=unauthorized_approver,
|
||||
)
|
||||
|
||||
experiment = experiment_approve(
|
||||
experiment=experiment,
|
||||
approver=approved_approver,
|
||||
)
|
||||
self.assertEqual(experiment.status, ExperimentStatus.APPROVED)
|
||||
|
||||
def test_non_owner_cannot_submit_for_review(self) -> None:
|
||||
viewer = make_viewer("_voe")
|
||||
flag = make_flag(suffix="_voe", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="Viewer Experiment",
|
||||
owner=viewer,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
other_user = make_experimenter("_voe2")
|
||||
|
||||
with self.assertRaises(ForbiddenError):
|
||||
experiment_submit_for_review(
|
||||
experiment=experiment, user=other_user
|
||||
)
|
||||
|
||||
|
||||
class EventValidationIntegrationTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
event_type_create(
|
||||
name="neg_exposure",
|
||||
display_name="Exposure",
|
||||
is_exposure=True,
|
||||
)
|
||||
event_type_create(
|
||||
name="neg_click",
|
||||
display_name="Click",
|
||||
requires_exposure=True,
|
||||
required_fields=["screen"],
|
||||
)
|
||||
self.now = timezone.now().isoformat()
|
||||
|
||||
def test_batch_with_mixed_valid_and_invalid_events(self) -> None:
|
||||
decision_create(
|
||||
decision_id="neg_dec_1",
|
||||
flag_key="test_flag",
|
||||
subject_id="u1",
|
||||
value="v",
|
||||
reason="test",
|
||||
)
|
||||
|
||||
process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "neg_exp_1",
|
||||
"event_type": "neg_exposure",
|
||||
"decision_id": "neg_dec_1",
|
||||
"subject_id": "u1",
|
||||
"timestamp": self.now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
result = process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "neg_valid_click",
|
||||
"event_type": "neg_click",
|
||||
"decision_id": "neg_dec_1",
|
||||
"subject_id": "u1",
|
||||
"timestamp": self.now,
|
||||
"properties": {"screen": "home"},
|
||||
},
|
||||
{
|
||||
"event_id": "neg_invalid_type",
|
||||
"event_type": "nonexistent_type",
|
||||
"decision_id": "neg_dec_1",
|
||||
"subject_id": "u1",
|
||||
"timestamp": self.now,
|
||||
"properties": {},
|
||||
},
|
||||
{
|
||||
"event_id": "neg_missing_field",
|
||||
"event_type": "neg_click",
|
||||
"decision_id": "neg_dec_1",
|
||||
"subject_id": "u1",
|
||||
"timestamp": self.now,
|
||||
"properties": {},
|
||||
},
|
||||
{
|
||||
"event_id": "neg_missing_decision",
|
||||
"event_type": "neg_click",
|
||||
"decision_id": "",
|
||||
"subject_id": "u1",
|
||||
"timestamp": self.now,
|
||||
"properties": {"screen": "home"},
|
||||
},
|
||||
{
|
||||
"event_id": "neg_bad_ts",
|
||||
"event_type": "neg_click",
|
||||
"decision_id": "neg_dec_1",
|
||||
"subject_id": "u1",
|
||||
"timestamp": 12345,
|
||||
"properties": {"screen": "home"},
|
||||
},
|
||||
{
|
||||
"event_id": "neg_exp_1",
|
||||
"event_type": "neg_exposure",
|
||||
"decision_id": "neg_dec_1",
|
||||
"subject_id": "u1",
|
||||
"timestamp": self.now,
|
||||
"properties": {},
|
||||
},
|
||||
]
|
||||
)
|
||||
self.assertEqual(result.accepted, 1)
|
||||
self.assertEqual(result.duplicates, 1)
|
||||
self.assertGreaterEqual(result.rejected, 3)
|
||||
|
||||
def test_non_string_event_type_rejected(self) -> None:
|
||||
result = process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "neg_bad_type",
|
||||
"event_type": 999,
|
||||
"decision_id": "dec",
|
||||
"subject_id": "u1",
|
||||
"timestamp": self.now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
self.assertEqual(result.rejected, 1)
|
||||
self.assertEqual(result.accepted, 0)
|
||||
|
||||
|
||||
class VariantWeightValidationTest(TestCase):
|
||||
def test_weights_matching_allocation_allows_submit(self) -> None:
|
||||
owner = make_experimenter("_vwv")
|
||||
flag = make_flag(suffix="_vwv", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="Weight Validation",
|
||||
owner=owner,
|
||||
traffic_allocation=Decimal("30.00"),
|
||||
)
|
||||
variant_create(
|
||||
experiment=experiment,
|
||||
user=owner,
|
||||
name="control",
|
||||
value="a",
|
||||
weight=Decimal("15.00"),
|
||||
is_control=True,
|
||||
)
|
||||
variant_create(
|
||||
experiment=experiment,
|
||||
user=owner,
|
||||
name="treatment",
|
||||
value="b",
|
||||
weight=Decimal("15.00"),
|
||||
)
|
||||
|
||||
review_settings_update(
|
||||
default_min_approvals=1,
|
||||
allow_any_approver=True,
|
||||
)
|
||||
experiment = experiment_submit_for_review(
|
||||
experiment=experiment, user=owner
|
||||
)
|
||||
self.assertEqual(experiment.status, ExperimentStatus.IN_REVIEW)
|
||||
|
||||
def test_weights_exceeding_allocation_rejected(self) -> None:
|
||||
owner = make_experimenter("_vwe")
|
||||
flag = make_flag(suffix="_vwe", default="d")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="Weight Exceeds",
|
||||
owner=owner,
|
||||
traffic_allocation=Decimal("30.00"),
|
||||
)
|
||||
variant_create(
|
||||
experiment=experiment,
|
||||
user=owner,
|
||||
name="control",
|
||||
value="a",
|
||||
weight=Decimal("15.00"),
|
||||
is_control=True,
|
||||
)
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
variant_create(
|
||||
experiment=experiment,
|
||||
user=owner,
|
||||
name="treatment",
|
||||
value="b",
|
||||
weight=Decimal("20.00"),
|
||||
)
|
||||
Reference in New Issue
Block a user