test(integration): added integration tests
This commit is contained in:
@@ -0,0 +1,323 @@
|
||||
from decimal import Decimal
|
||||
from typing import override
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.decision.services import decide_for_flag
|
||||
from apps.events.models import Exposure, PendingEvent
|
||||
from apps.events.services import process_events_batch
|
||||
from apps.events.tests.helpers import make_event_type, make_exposure_type
|
||||
from apps.experiments.services import (
|
||||
experiment_approve,
|
||||
experiment_start,
|
||||
experiment_submit_for_review,
|
||||
)
|
||||
from apps.experiments.tests.helpers import add_two_variants, make_experiment
|
||||
from apps.metrics.services import (
|
||||
experiment_metric_add,
|
||||
metric_definition_create,
|
||||
)
|
||||
from apps.reports.services import build_experiment_report
|
||||
from apps.reviews.services import review_settings_update
|
||||
from apps.reviews.tests.helpers import make_approver, make_experimenter
|
||||
|
||||
|
||||
class OutOfOrderAttributionTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
cache.clear()
|
||||
review_settings_update(
|
||||
default_min_approvals=1,
|
||||
allow_any_approver=True,
|
||||
)
|
||||
self.owner = make_experimenter("_ooa")
|
||||
self.approver = make_approver("_ooa")
|
||||
|
||||
self.experiment = make_experiment(
|
||||
owner=self.owner,
|
||||
suffix="_ooa",
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(self.experiment)
|
||||
|
||||
self.metric = metric_definition_create(
|
||||
key="conv_ooa",
|
||||
name="Conversion",
|
||||
metric_type="count",
|
||||
direction="higher_is_better",
|
||||
calculation_rule={"event": "ooa_purchase"},
|
||||
)
|
||||
experiment_metric_add(
|
||||
experiment=self.experiment,
|
||||
metric=self.metric,
|
||||
is_primary=True,
|
||||
)
|
||||
|
||||
make_exposure_type(name="ooa_exposure")
|
||||
make_event_type(
|
||||
name="ooa_purchase",
|
||||
display_name="Purchase",
|
||||
requires_exposure=True,
|
||||
)
|
||||
|
||||
self.experiment = experiment_submit_for_review(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
self.experiment = experiment_approve(
|
||||
experiment=self.experiment, approver=self.approver
|
||||
)
|
||||
self.experiment = experiment_start(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
|
||||
def test_conversion_before_exposure_goes_to_pending(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_ooa", "user_ooa_1", {})
|
||||
now = timezone.now().isoformat()
|
||||
|
||||
result = process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "ooa_conv_1",
|
||||
"event_type": "ooa_purchase",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_ooa_1",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
self.assertEqual(result.accepted, 1)
|
||||
self.assertTrue(
|
||||
PendingEvent.objects.filter(event_id="ooa_conv_1").exists()
|
||||
)
|
||||
|
||||
def test_pending_event_promoted_on_exposure_arrival(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_ooa", "user_ooa_2", {})
|
||||
now = timezone.now().isoformat()
|
||||
|
||||
process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "ooa_conv_2",
|
||||
"event_type": "ooa_purchase",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_ooa_2",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
self.assertTrue(
|
||||
PendingEvent.objects.filter(event_id="ooa_conv_2").exists()
|
||||
)
|
||||
|
||||
result = process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "ooa_exp_2",
|
||||
"event_type": "ooa_exposure",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_ooa_2",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
self.assertEqual(result.accepted, 1)
|
||||
|
||||
self.assertFalse(
|
||||
PendingEvent.objects.filter(event_id="ooa_conv_2").exists()
|
||||
)
|
||||
|
||||
def test_promoted_event_appears_in_report(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_ooa", "user_ooa_3", {})
|
||||
now = timezone.now().isoformat()
|
||||
|
||||
process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "ooa_conv_3",
|
||||
"event_type": "ooa_purchase",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_ooa_3",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "ooa_exp_3",
|
||||
"event_type": "ooa_exposure",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_ooa_3",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
report = build_experiment_report(self.experiment)
|
||||
total_exposures = sum(v["exposures"] for v in report["variants"])
|
||||
self.assertEqual(total_exposures, 1)
|
||||
|
||||
|
||||
class EventDeduplicationTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
cache.clear()
|
||||
review_settings_update(
|
||||
default_min_approvals=1,
|
||||
allow_any_approver=True,
|
||||
)
|
||||
self.owner = make_experimenter("_ded")
|
||||
self.approver = make_approver("_ded")
|
||||
|
||||
self.experiment = make_experiment(
|
||||
owner=self.owner,
|
||||
suffix="_ded",
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(self.experiment)
|
||||
|
||||
make_exposure_type(name="ded_exposure")
|
||||
|
||||
self.experiment = experiment_submit_for_review(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
self.experiment = experiment_approve(
|
||||
experiment=self.experiment, approver=self.approver
|
||||
)
|
||||
self.experiment = experiment_start(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
|
||||
def test_duplicate_exposure_not_counted_twice(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_ded", "user_dup", {})
|
||||
now = timezone.now().isoformat()
|
||||
|
||||
event = {
|
||||
"event_id": "ded_same_id",
|
||||
"event_type": "ded_exposure",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_dup",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
|
||||
r1 = process_events_batch([event])
|
||||
self.assertEqual(r1.accepted, 1)
|
||||
|
||||
r2 = process_events_batch([event])
|
||||
self.assertEqual(r2.duplicates, 1)
|
||||
self.assertEqual(r2.accepted, 0)
|
||||
|
||||
exposures = Exposure.objects.filter(
|
||||
decision_id=d["decision_id"],
|
||||
)
|
||||
self.assertEqual(exposures.count(), 1)
|
||||
|
||||
def test_deduplication_prevents_metric_inflation(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_ded", "user_infl", {})
|
||||
now = timezone.now().isoformat()
|
||||
|
||||
event = {
|
||||
"event_id": "ded_infl_id",
|
||||
"event_type": "ded_exposure",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_infl",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
|
||||
process_events_batch([event])
|
||||
process_events_batch([event])
|
||||
process_events_batch([event])
|
||||
|
||||
report = build_experiment_report(self.experiment)
|
||||
total_exposures = sum(v["exposures"] for v in report["variants"])
|
||||
self.assertEqual(total_exposures, 1)
|
||||
|
||||
|
||||
class ConversionWithoutExposureTest(TestCase):
|
||||
@override
|
||||
def setUp(self) -> None:
|
||||
cache.clear()
|
||||
review_settings_update(
|
||||
default_min_approvals=1,
|
||||
allow_any_approver=True,
|
||||
)
|
||||
self.owner = make_experimenter("_cwe")
|
||||
self.approver = make_approver("_cwe")
|
||||
|
||||
self.experiment = make_experiment(
|
||||
owner=self.owner,
|
||||
suffix="_cwe",
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
)
|
||||
add_two_variants(self.experiment)
|
||||
|
||||
self.metric = metric_definition_create(
|
||||
key="conv_cwe",
|
||||
name="Conversion",
|
||||
metric_type="count",
|
||||
direction="higher_is_better",
|
||||
calculation_rule={"event": "cwe_purchase"},
|
||||
)
|
||||
experiment_metric_add(
|
||||
experiment=self.experiment,
|
||||
metric=self.metric,
|
||||
is_primary=True,
|
||||
)
|
||||
|
||||
make_exposure_type(name="cwe_exposure")
|
||||
make_event_type(
|
||||
name="cwe_purchase",
|
||||
display_name="Purchase",
|
||||
requires_exposure=True,
|
||||
)
|
||||
|
||||
self.experiment = experiment_submit_for_review(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
self.experiment = experiment_approve(
|
||||
experiment=self.experiment, approver=self.approver
|
||||
)
|
||||
self.experiment = experiment_start(
|
||||
experiment=self.experiment, user=self.owner
|
||||
)
|
||||
|
||||
def test_unresolved_pending_event_not_in_report(self) -> None:
|
||||
cache.clear()
|
||||
d = decide_for_flag("flag_cwe", "user_no_exp", {})
|
||||
now = timezone.now().isoformat()
|
||||
|
||||
process_events_batch(
|
||||
[
|
||||
{
|
||||
"event_id": "cwe_conv_only",
|
||||
"event_type": "cwe_purchase",
|
||||
"decision_id": d["decision_id"],
|
||||
"subject_id": "user_no_exp",
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
self.assertTrue(
|
||||
PendingEvent.objects.filter(event_id="cwe_conv_only").exists()
|
||||
)
|
||||
|
||||
report = build_experiment_report(self.experiment)
|
||||
total_exposures = sum(v["exposures"] for v in report["variants"])
|
||||
self.assertEqual(total_exposures, 0)
|
||||
Reference in New Issue
Block a user