324 lines
9.9 KiB
Python
324 lines
9.9 KiB
Python
from decimal import Decimal
|
|
from typing import override
|
|
|
|
from django.core.cache import cache
|
|
from django.test import TestCase
|
|
from django.utils import timezone
|
|
|
|
from apps.decision.services import decide_for_flag
|
|
from apps.events.models import Exposure, PendingEvent
|
|
from apps.events.services import process_events_batch
|
|
from apps.events.tests.helpers import make_event_type, make_exposure_type
|
|
from apps.experiments.services import (
|
|
experiment_approve,
|
|
experiment_start,
|
|
experiment_submit_for_review,
|
|
)
|
|
from apps.experiments.tests.helpers import add_two_variants, make_experiment
|
|
from apps.metrics.services import (
|
|
experiment_metric_add,
|
|
metric_definition_create,
|
|
)
|
|
from apps.reports.services import build_experiment_report
|
|
from apps.reviews.services import review_settings_update
|
|
from apps.reviews.tests.helpers import make_approver, make_experimenter
|
|
|
|
|
|
class OutOfOrderAttributionTest(TestCase):
|
|
@override
|
|
def setUp(self) -> None:
|
|
cache.clear()
|
|
review_settings_update(
|
|
default_min_approvals=1,
|
|
allow_any_approver=True,
|
|
)
|
|
self.owner = make_experimenter("_ooa")
|
|
self.approver = make_approver("_ooa")
|
|
|
|
self.experiment = make_experiment(
|
|
owner=self.owner,
|
|
suffix="_ooa",
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(self.experiment)
|
|
|
|
self.metric = metric_definition_create(
|
|
key="conv_ooa",
|
|
name="Conversion",
|
|
metric_type="count",
|
|
direction="higher_is_better",
|
|
calculation_rule={"event": "ooa_purchase"},
|
|
)
|
|
experiment_metric_add(
|
|
experiment=self.experiment,
|
|
metric=self.metric,
|
|
is_primary=True,
|
|
)
|
|
|
|
make_exposure_type(name="ooa_exposure")
|
|
make_event_type(
|
|
name="ooa_purchase",
|
|
display_name="Purchase",
|
|
requires_exposure=True,
|
|
)
|
|
|
|
self.experiment = experiment_submit_for_review(
|
|
experiment=self.experiment, user=self.owner
|
|
)
|
|
self.experiment = experiment_approve(
|
|
experiment=self.experiment, approver=self.approver
|
|
)
|
|
self.experiment = experiment_start(
|
|
experiment=self.experiment, user=self.owner
|
|
)
|
|
|
|
def test_conversion_before_exposure_goes_to_pending(self) -> None:
|
|
cache.clear()
|
|
d = decide_for_flag("flag_ooa", "user_ooa_1", {})
|
|
now = timezone.now().isoformat()
|
|
|
|
result = process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "ooa_conv_1",
|
|
"event_type": "ooa_purchase",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_ooa_1",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
self.assertEqual(result.accepted, 1)
|
|
self.assertTrue(
|
|
PendingEvent.objects.filter(event_id="ooa_conv_1").exists()
|
|
)
|
|
|
|
def test_pending_event_promoted_on_exposure_arrival(self) -> None:
|
|
cache.clear()
|
|
d = decide_for_flag("flag_ooa", "user_ooa_2", {})
|
|
now = timezone.now().isoformat()
|
|
|
|
process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "ooa_conv_2",
|
|
"event_type": "ooa_purchase",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_ooa_2",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
|
|
self.assertTrue(
|
|
PendingEvent.objects.filter(event_id="ooa_conv_2").exists()
|
|
)
|
|
|
|
result = process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "ooa_exp_2",
|
|
"event_type": "ooa_exposure",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_ooa_2",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
self.assertEqual(result.accepted, 1)
|
|
|
|
self.assertFalse(
|
|
PendingEvent.objects.filter(event_id="ooa_conv_2").exists()
|
|
)
|
|
|
|
def test_promoted_event_appears_in_report(self) -> None:
|
|
cache.clear()
|
|
d = decide_for_flag("flag_ooa", "user_ooa_3", {})
|
|
now = timezone.now().isoformat()
|
|
|
|
process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "ooa_conv_3",
|
|
"event_type": "ooa_purchase",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_ooa_3",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "ooa_exp_3",
|
|
"event_type": "ooa_exposure",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_ooa_3",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
|
|
report = build_experiment_report(self.experiment)
|
|
total_exposures = sum(v["exposures"] for v in report["variants"])
|
|
self.assertEqual(total_exposures, 1)
|
|
|
|
|
|
class EventDeduplicationTest(TestCase):
|
|
@override
|
|
def setUp(self) -> None:
|
|
cache.clear()
|
|
review_settings_update(
|
|
default_min_approvals=1,
|
|
allow_any_approver=True,
|
|
)
|
|
self.owner = make_experimenter("_ded")
|
|
self.approver = make_approver("_ded")
|
|
|
|
self.experiment = make_experiment(
|
|
owner=self.owner,
|
|
suffix="_ded",
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(self.experiment)
|
|
|
|
make_exposure_type(name="ded_exposure")
|
|
|
|
self.experiment = experiment_submit_for_review(
|
|
experiment=self.experiment, user=self.owner
|
|
)
|
|
self.experiment = experiment_approve(
|
|
experiment=self.experiment, approver=self.approver
|
|
)
|
|
self.experiment = experiment_start(
|
|
experiment=self.experiment, user=self.owner
|
|
)
|
|
|
|
def test_duplicate_exposure_not_counted_twice(self) -> None:
|
|
cache.clear()
|
|
d = decide_for_flag("flag_ded", "user_dup", {})
|
|
now = timezone.now().isoformat()
|
|
|
|
event = {
|
|
"event_id": "ded_same_id",
|
|
"event_type": "ded_exposure",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_dup",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
|
|
r1 = process_events_batch([event])
|
|
self.assertEqual(r1.accepted, 1)
|
|
|
|
r2 = process_events_batch([event])
|
|
self.assertEqual(r2.duplicates, 1)
|
|
self.assertEqual(r2.accepted, 0)
|
|
|
|
exposures = Exposure.objects.filter(
|
|
decision_id=d["decision_id"],
|
|
)
|
|
self.assertEqual(exposures.count(), 1)
|
|
|
|
def test_deduplication_prevents_metric_inflation(self) -> None:
|
|
cache.clear()
|
|
d = decide_for_flag("flag_ded", "user_infl", {})
|
|
now = timezone.now().isoformat()
|
|
|
|
event = {
|
|
"event_id": "ded_infl_id",
|
|
"event_type": "ded_exposure",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_infl",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
|
|
process_events_batch([event])
|
|
process_events_batch([event])
|
|
process_events_batch([event])
|
|
|
|
report = build_experiment_report(self.experiment)
|
|
total_exposures = sum(v["exposures"] for v in report["variants"])
|
|
self.assertEqual(total_exposures, 1)
|
|
|
|
|
|
class ConversionWithoutExposureTest(TestCase):
|
|
@override
|
|
def setUp(self) -> None:
|
|
cache.clear()
|
|
review_settings_update(
|
|
default_min_approvals=1,
|
|
allow_any_approver=True,
|
|
)
|
|
self.owner = make_experimenter("_cwe")
|
|
self.approver = make_approver("_cwe")
|
|
|
|
self.experiment = make_experiment(
|
|
owner=self.owner,
|
|
suffix="_cwe",
|
|
traffic_allocation=Decimal("100.00"),
|
|
)
|
|
add_two_variants(self.experiment)
|
|
|
|
self.metric = metric_definition_create(
|
|
key="conv_cwe",
|
|
name="Conversion",
|
|
metric_type="count",
|
|
direction="higher_is_better",
|
|
calculation_rule={"event": "cwe_purchase"},
|
|
)
|
|
experiment_metric_add(
|
|
experiment=self.experiment,
|
|
metric=self.metric,
|
|
is_primary=True,
|
|
)
|
|
|
|
make_exposure_type(name="cwe_exposure")
|
|
make_event_type(
|
|
name="cwe_purchase",
|
|
display_name="Purchase",
|
|
requires_exposure=True,
|
|
)
|
|
|
|
self.experiment = experiment_submit_for_review(
|
|
experiment=self.experiment, user=self.owner
|
|
)
|
|
self.experiment = experiment_approve(
|
|
experiment=self.experiment, approver=self.approver
|
|
)
|
|
self.experiment = experiment_start(
|
|
experiment=self.experiment, user=self.owner
|
|
)
|
|
|
|
def test_unresolved_pending_event_not_in_report(self) -> None:
|
|
cache.clear()
|
|
d = decide_for_flag("flag_cwe", "user_no_exp", {})
|
|
now = timezone.now().isoformat()
|
|
|
|
process_events_batch(
|
|
[
|
|
{
|
|
"event_id": "cwe_conv_only",
|
|
"event_type": "cwe_purchase",
|
|
"decision_id": d["decision_id"],
|
|
"subject_id": "user_no_exp",
|
|
"timestamp": now,
|
|
"properties": {},
|
|
}
|
|
]
|
|
)
|
|
|
|
self.assertTrue(
|
|
PendingEvent.objects.filter(event_id="cwe_conv_only").exists()
|
|
)
|
|
|
|
report = build_experiment_report(self.experiment)
|
|
total_exposures = sum(v["exposures"] for v in report["variants"])
|
|
self.assertEqual(total_exposures, 0)
|