test(integration): added integration tests
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
from decimal import Decimal
|
||||
from typing import override
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.decision.services import decide_for_flag
|
||||
from apps.events.services import process_events_batch
|
||||
from apps.events.tests.helpers import make_event_type, make_exposure_type
|
||||
from apps.experiments.models import ExperimentStatus
|
||||
from apps.experiments.services import (
|
||||
experiment_approve,
|
||||
experiment_complete,
|
||||
experiment_create,
|
||||
experiment_start,
|
||||
experiment_submit_for_review,
|
||||
variant_create,
|
||||
)
|
||||
from apps.experiments.tests.helpers import (
|
||||
add_two_variants,
|
||||
make_experiment,
|
||||
make_flag,
|
||||
)
|
||||
from apps.metrics.services import (
|
||||
experiment_metric_add,
|
||||
metric_definition_create,
|
||||
)
|
||||
from apps.reports.services import build_experiment_report
|
||||
from apps.reviews.services import review_settings_update
|
||||
from apps.reviews.tests.helpers import make_approver, make_experimenter
|
||||
|
||||
|
||||
class FullHappyPathTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
cache.clear()
|
||||
review_settings_update(
|
||||
default_min_approvals=1,
|
||||
allow_any_approver=True,
|
||||
)
|
||||
|
||||
self.owner = make_experimenter("_hp")
|
||||
self.approver = make_approver("_hp")
|
||||
|
||||
self.experiment = make_experiment(
|
||||
owner=self.owner,
|
||||
suffix="_hp",
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
self.v_control, self.v_treatment = add_two_variants(self.experiment)
|
||||
|
||||
self.metric = metric_definition_create(
|
||||
key="ctr_hp",
|
||||
name="CTR",
|
||||
metric_type="ratio",
|
||||
direction="higher_is_better",
|
||||
calculation_rule={
|
||||
"numerator_event": "hp_click",
|
||||
"denominator_event": "hp_exposure",
|
||||
},
|
||||
)
|
||||
experiment_metric_add(
|
||||
experiment=self.experiment,
|
||||
metric=self.metric,
|
||||
is_primary=True,
|
||||
)
|
||||
|
||||
make_exposure_type(name="hp_exposure")
|
||||
make_event_type(
|
||||
name="hp_click",
|
||||
display_name="Click",
|
||||
requires_exposure=True,
|
||||
)
|
||||
|
||||
self.experiment = experiment_submit_for_review(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
self.experiment = experiment_approve(
|
||||
experiment=self.experiment, approver=self.approver
|
||||
)
|
||||
self.experiment = experiment_start(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
|
||||
def test_full_decide_event_report_flow(self) -> None:
|
||||
decisions = []
|
||||
for i in range(10):
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_hp", f"user_{i}", {"country": "US"})
|
||||
self.assertEqual(d["reason"], "experiment_assigned")
|
||||
self.assertIsNotNone(d["variant_id"])
|
||||
decisions.append(d)
|
||||
|
||||
now = timezone.now().isoformat()
|
||||
exposure_events = [
|
||||
{
|
||||
"event_id": f"hp_exp_{i}",
|
||||
"event_type": "hp_exposure",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": f"user_{i}",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
for i, d in enumerate(decisions)
|
||||
]
|
||||
result = process_events_batch(exposure_events)
|
||||
self.assertEqual(result.accepted, 10)
|
||||
|
||||
click_events = [
|
||||
{
|
||||
"event_id": f"hp_click_{i}",
|
||||
"event_type": "hp_click",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": f"user_{i}",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
for i, d in enumerate(decisions[:5])
|
||||
]
|
||||
result = process_events_batch(click_events)
|
||||
self.assertEqual(result.accepted, 5)
|
||||
|
||||
report = build_experiment_report(self.experiment)
|
||||
self.assertEqual(str(report["experiment_id"]), str(self.experiment.pk))
|
||||
total_exposures = sum(v["exposures"] for v in report["variants"])
|
||||
self.assertEqual(total_exposures, 10)
|
||||
|
||||
def test_lifecycle_with_rollout_outcome(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_hp", "subject_1", {})
|
||||
self.assertEqual(d["reason"], "experiment_assigned")
|
||||
|
||||
self.experiment = experiment_complete(
|
||||
experiment=self.experiment,
|
||||
user=self.owner,
|
||||
outcome="rollout",
|
||||
rationale="Treatment wins",
|
||||
winning_variant_id=str(self.v_treatment.pk),
|
||||
)
|
||||
self.assertEqual(self.experiment.status, ExperimentStatus.COMPLETED)
|
||||
|
||||
def test_decide_returns_default_after_complete(self) -> None:
|
||||
self.experiment = experiment_complete(
|
||||
experiment=self.experiment,
|
||||
user=self.owner,
|
||||
outcome="no_effect",
|
||||
rationale="No significant difference",
|
||||
)
|
||||
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_hp", "subject_2", {})
|
||||
self.assertEqual(d["reason"], "no_active_experiment")
|
||||
self.assertEqual(d["value"], "a")
|
||||
|
||||
def test_targeting_mismatch_returns_default(self) -> None:
|
||||
owner = make_experimenter("_tm")
|
||||
approver = make_approver("_tm")
|
||||
|
||||
flag = make_flag(suffix="_tm", default="a")
|
||||
|
||||
exp = experiment_create(
|
||||
flag=flag,
|
||||
name="Targeting Test",
|
||||
owner=owner,
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
targeting_rules='country IN ["DE"]',
|
||||
)
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=owner,
|
||||
name="control",
|
||||
value="a",
|
||||
weight=Decimal("50.00"),
|
||||
is_control=True,
|
||||
)
|
||||
variant_create(
|
||||
experiment=exp,
|
||||
user=owner,
|
||||
name="treatment",
|
||||
value="b",
|
||||
weight=Decimal("50.00"),
|
||||
)
|
||||
exp = experiment_submit_for_review(experiment=exp, user=owner)
|
||||
exp = experiment_approve(experiment=exp, approver=approver)
|
||||
exp = experiment_start(experiment=exp, user=owner)
|
||||
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_tm", "subject_3", {"country": "US"})
|
||||
self.assertEqual(d["reason"], "targeting_mismatch")
|
||||
self.assertEqual(d["value"], "a")
|
||||
|
||||
def test_report_with_period_filter(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_hp", "user_rp", {})
|
||||
now = timezone.now()
|
||||
|
||||
process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "hp_rp_exp",
|
||||
"event_type": "hp_exposure",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_rp",
|
||||
"timestamp": now.isoformat(),
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
future = now + timezone.timedelta(hours=1)
|
||||
report = build_experiment_report(
|
||||
self.experiment,
|
||||
start_date=future,
|
||||
end_date=future + timezone.timedelta(hours=1),
|
||||
)
|
||||
total_exposures = sum(v["exposures"] for v in report["variants"])
|
||||
self.assertEqual(total_exposures, 0)
|
||||
Reference in New Issue
Block a user