from decimal import Decimal from typing import override from django.core.cache import cache from django.test import TestCase from django.utils import timezone from apps.decision.services import decide_for_flag from apps.events.models import Exposure, PendingEvent from apps.events.services import process_events_batch from apps.events.tests.helpers import make_event_type, make_exposure_type from apps.experiments.services import ( experiment_approve, experiment_start, experiment_submit_for_review, ) from apps.experiments.tests.helpers import add_two_variants, make_experiment from apps.metrics.services import ( experiment_metric_add, metric_definition_create, ) from apps.reports.services import build_experiment_report from apps.reviews.services import review_settings_update from apps.reviews.tests.helpers import make_approver, make_experimenter class OutOfOrderAttributionTest(TestCase): @override def setUp(self) -> None: cache.clear() review_settings_update( default_min_approvals=1, allow_any_approver=True, ) self.owner = make_experimenter("_ooa") self.approver = make_approver("_ooa") self.experiment = make_experiment( owner=self.owner, suffix="_ooa", traffic_allocation=Decimal("100.00"), ) add_two_variants(self.experiment) self.metric = metric_definition_create( key="conv_ooa", name="Conversion", metric_type="count", direction="higher_is_better", calculation_rule={"event": "ooa_purchase"}, ) experiment_metric_add( experiment=self.experiment, metric=self.metric, is_primary=True, ) make_exposure_type(name="ooa_exposure") make_event_type( name="ooa_purchase", display_name="Purchase", requires_exposure=True, ) self.experiment = experiment_submit_for_review( experiment=self.experiment, user=self.owner ) self.experiment = experiment_approve( experiment=self.experiment, approver=self.approver ) self.experiment = experiment_start( experiment=self.experiment, user=self.owner ) def test_conversion_before_exposure_goes_to_pending(self) -> None: cache.clear() d = decide_for_flag("flag_ooa", "user_ooa_1", {}) now = timezone.now().isoformat() result = process_events_batch( [ { "event_id": "ooa_conv_1", "event_type": "ooa_purchase", "decision_id": d["decision_id"], "subject_id": "user_ooa_1", "timestamp": now, "properties": {}, } ] ) self.assertEqual(result.accepted, 1) self.assertTrue( PendingEvent.objects.filter(event_id="ooa_conv_1").exists() ) def test_pending_event_promoted_on_exposure_arrival(self) -> None: cache.clear() d = decide_for_flag("flag_ooa", "user_ooa_2", {}) now = timezone.now().isoformat() process_events_batch( [ { "event_id": "ooa_conv_2", "event_type": "ooa_purchase", "decision_id": d["decision_id"], "subject_id": "user_ooa_2", "timestamp": now, "properties": {}, } ] ) self.assertTrue( PendingEvent.objects.filter(event_id="ooa_conv_2").exists() ) result = process_events_batch( [ { "event_id": "ooa_exp_2", "event_type": "ooa_exposure", "decision_id": d["decision_id"], "subject_id": "user_ooa_2", "timestamp": now, "properties": {}, } ] ) self.assertEqual(result.accepted, 1) self.assertFalse( PendingEvent.objects.filter(event_id="ooa_conv_2").exists() ) def test_promoted_event_appears_in_report(self) -> None: cache.clear() d = decide_for_flag("flag_ooa", "user_ooa_3", {}) now = timezone.now().isoformat() process_events_batch( [ { "event_id": "ooa_conv_3", "event_type": "ooa_purchase", "decision_id": d["decision_id"], "subject_id": "user_ooa_3", "timestamp": now, "properties": {}, } ] ) process_events_batch( [ { "event_id": "ooa_exp_3", "event_type": "ooa_exposure", "decision_id": d["decision_id"], "subject_id": "user_ooa_3", "timestamp": now, "properties": {}, } ] ) report = build_experiment_report(self.experiment) total_exposures = sum(v["exposures"] for v in report["variants"]) self.assertEqual(total_exposures, 1) class EventDeduplicationTest(TestCase): @override def setUp(self) -> None: cache.clear() review_settings_update( default_min_approvals=1, allow_any_approver=True, ) self.owner = make_experimenter("_ded") self.approver = make_approver("_ded") self.experiment = make_experiment( owner=self.owner, suffix="_ded", traffic_allocation=Decimal("100.00"), ) add_two_variants(self.experiment) make_exposure_type(name="ded_exposure") self.experiment = experiment_submit_for_review( experiment=self.experiment, user=self.owner ) self.experiment = experiment_approve( experiment=self.experiment, approver=self.approver ) self.experiment = experiment_start( experiment=self.experiment, user=self.owner ) def test_duplicate_exposure_not_counted_twice(self) -> None: cache.clear() d = decide_for_flag("flag_ded", "user_dup", {}) now = timezone.now().isoformat() event = { "event_id": "ded_same_id", "event_type": "ded_exposure", "decision_id": d["decision_id"], "subject_id": "user_dup", "timestamp": now, "properties": {}, } r1 = process_events_batch([event]) self.assertEqual(r1.accepted, 1) r2 = process_events_batch([event]) self.assertEqual(r2.duplicates, 1) self.assertEqual(r2.accepted, 0) exposures = Exposure.objects.filter( decision_id=d["decision_id"], ) self.assertEqual(exposures.count(), 1) def test_deduplication_prevents_metric_inflation(self) -> None: cache.clear() d = decide_for_flag("flag_ded", "user_infl", {}) now = timezone.now().isoformat() event = { "event_id": "ded_infl_id", "event_type": "ded_exposure", "decision_id": d["decision_id"], "subject_id": "user_infl", "timestamp": now, "properties": {}, } process_events_batch([event]) process_events_batch([event]) process_events_batch([event]) report = build_experiment_report(self.experiment) total_exposures = sum(v["exposures"] for v in report["variants"]) self.assertEqual(total_exposures, 1) class ConversionWithoutExposureTest(TestCase): @override def setUp(self) -> None: cache.clear() review_settings_update( default_min_approvals=1, allow_any_approver=True, ) self.owner = make_experimenter("_cwe") self.approver = make_approver("_cwe") self.experiment = make_experiment( owner=self.owner, suffix="_cwe", traffic_allocation=Decimal("100.00"), ) add_two_variants(self.experiment) self.metric = metric_definition_create( key="conv_cwe", name="Conversion", metric_type="count", direction="higher_is_better", calculation_rule={"event": "cwe_purchase"}, ) experiment_metric_add( experiment=self.experiment, metric=self.metric, is_primary=True, ) make_exposure_type(name="cwe_exposure") make_event_type( name="cwe_purchase", display_name="Purchase", requires_exposure=True, ) self.experiment = experiment_submit_for_review( experiment=self.experiment, user=self.owner ) self.experiment = experiment_approve( experiment=self.experiment, approver=self.approver ) self.experiment = experiment_start( experiment=self.experiment, user=self.owner ) def test_unresolved_pending_event_not_in_report(self) -> None: cache.clear() d = decide_for_flag("flag_cwe", "user_no_exp", {}) now = timezone.now().isoformat() process_events_batch( [ { "event_id": "cwe_conv_only", "event_type": "cwe_purchase", "decision_id": d["decision_id"], "subject_id": "user_no_exp", "timestamp": now, "properties": {}, } ] ) self.assertTrue( PendingEvent.objects.filter(event_id="cwe_conv_only").exists() ) report = build_experiment_report(self.experiment) total_exposures = sum(v["exposures"] for v in report["variants"]) self.assertEqual(total_exposures, 0)