feat(): added demo e2e command
This commit is contained in:
@@ -0,0 +1,262 @@
|
||||
import json
|
||||
import uuid
|
||||
from decimal import Decimal
|
||||
from typing import override
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.decision.services import decide_for_flag
|
||||
from apps.events.models import EventType
|
||||
from apps.events.services import process_events_batch
|
||||
from apps.experiments.services import (
|
||||
experiment_approve,
|
||||
experiment_create,
|
||||
experiment_start,
|
||||
experiment_submit_for_review,
|
||||
variant_create,
|
||||
)
|
||||
from apps.flags.services import feature_flag_create
|
||||
from apps.metrics.models import ExperimentMetric, MetricDefinition
|
||||
from apps.reports.services import build_experiment_report
|
||||
from apps.reviews.models import ApproverGroup
|
||||
from apps.users.models import User, UserRole
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Run a complete E2E demo scenario: "
|
||||
"flag -> experiment -> decide -> events -> report"
|
||||
)
|
||||
|
||||
@override
|
||||
def handle(self, *args, **options) -> None:
|
||||
self._step("1. Create users")
|
||||
owner = self._get_or_create_user(
|
||||
"demo_experimenter", UserRole.EXPERIMENTER
|
||||
)
|
||||
approver = self._get_or_create_user("demo_approver", UserRole.APPROVER)
|
||||
self._ensure_review_policy(owner, approver)
|
||||
|
||||
self._step("2. Create feature flag")
|
||||
flag = feature_flag_create(
|
||||
key=f"demo_button_color_{uuid.uuid4().hex[:6]}",
|
||||
name="Demo Button Color",
|
||||
value_type="string",
|
||||
default_value="green",
|
||||
)
|
||||
self._ok(f"Flag created: key={flag.key}, default={flag.default_value}")
|
||||
|
||||
self._step("3. Create experiment with variants")
|
||||
experiment = experiment_create(
|
||||
flag=flag,
|
||||
name="Button Color A/B Test",
|
||||
owner=owner,
|
||||
description="Testing blue vs red button colors",
|
||||
hypothesis="Red button increases click-through rate",
|
||||
traffic_allocation=Decimal("100.00"),
|
||||
targeting_rules='country == "US"',
|
||||
)
|
||||
variant_create(
|
||||
experiment=experiment,
|
||||
user=owner,
|
||||
name="control",
|
||||
value="blue",
|
||||
weight=Decimal("50.00"),
|
||||
is_control=True,
|
||||
)
|
||||
variant_create(
|
||||
experiment=experiment,
|
||||
user=owner,
|
||||
name="treatment",
|
||||
value="red",
|
||||
weight=Decimal("50.00"),
|
||||
)
|
||||
self._ok(
|
||||
f"Experiment created: "
|
||||
f"id={experiment.pk}, status={experiment.status}"
|
||||
)
|
||||
|
||||
self._step("4. Lifecycle: draft → in_review → approved → running")
|
||||
experiment = experiment_submit_for_review(
|
||||
experiment=experiment, user=owner
|
||||
)
|
||||
self._ok(f"Status after submit: {experiment.status}")
|
||||
|
||||
experiment = experiment_approve(
|
||||
experiment=experiment, approver=approver, comment="Looks good"
|
||||
)
|
||||
self._ok(f"Status after approve: {experiment.status}")
|
||||
|
||||
experiment = experiment_start(experiment=experiment, user=owner)
|
||||
self._ok(f"Status after start: {experiment.status}")
|
||||
|
||||
self._step("5. Create event types")
|
||||
self._get_or_create_event_type(
|
||||
"exposure",
|
||||
"Exposure",
|
||||
is_exposure=True,
|
||||
requires_exposure=False,
|
||||
)
|
||||
self._get_or_create_event_type(
|
||||
"click",
|
||||
"Click",
|
||||
requires_exposure=True,
|
||||
)
|
||||
|
||||
self._step("6. Create metrics and attach to experiment")
|
||||
click_metric = self._get_or_create_metric(
|
||||
"click_rate",
|
||||
"Click Rate",
|
||||
)
|
||||
ExperimentMetric.objects.get_or_create(
|
||||
experiment=experiment,
|
||||
metric=click_metric,
|
||||
defaults={"is_primary": True},
|
||||
)
|
||||
|
||||
self._step("7. Make decisions for multiple subjects")
|
||||
decisions = {}
|
||||
subjects = [f"user_{i}" for i in range(1, 11)]
|
||||
for subj in subjects:
|
||||
result = decide_for_flag(flag.key, subj, {"country": "US"})
|
||||
decisions[subj] = result
|
||||
self._ok(
|
||||
f" {subj}: value={result['value']}, "
|
||||
f"reason={result['reason']}, "
|
||||
f"variant_id={result.get('variant_id')}"
|
||||
)
|
||||
|
||||
self._step("7b. Decide for non-targeted user")
|
||||
result_outside = decide_for_flag(
|
||||
flag.key, "user_de", {"country": "DE"}
|
||||
)
|
||||
self._ok(
|
||||
f" user_de: value={result_outside['value']}, "
|
||||
f"reason={result_outside['reason']} (expected: targeting_mismatch)"
|
||||
)
|
||||
|
||||
self._step("8. Send exposure events")
|
||||
now = timezone.now().isoformat()
|
||||
exposure_events = []
|
||||
for subj, dec in decisions.items():
|
||||
if dec["reason"] != "experiment_assigned":
|
||||
continue
|
||||
exposure_events.append(
|
||||
{
|
||||
"event_id": str(uuid.uuid4()),
|
||||
"event_type": "exposure",
|
||||
"decision_id": dec["decision_id"],
|
||||
"subject_id": subj,
|
||||
"timestamp": now,
|
||||
"properties": {},
|
||||
}
|
||||
)
|
||||
batch_result = process_events_batch(exposure_events)
|
||||
self._ok(
|
||||
f"Exposures: accepted={batch_result.accepted}, "
|
||||
f"duplicates={batch_result.duplicates}, "
|
||||
f"rejected={batch_result.rejected}"
|
||||
)
|
||||
|
||||
self._step("9. Send conversion (click) events for half the subjects")
|
||||
assigned = [
|
||||
s
|
||||
for s, d in decisions.items()
|
||||
if d["reason"] == "experiment_assigned"
|
||||
]
|
||||
click_events = [
|
||||
{
|
||||
"event_id": str(uuid.uuid4()),
|
||||
"event_type": "click",
|
||||
"decision_id": decisions[subj]["decision_id"],
|
||||
"subject_id": subj,
|
||||
"timestamp": now,
|
||||
"properties": {"element": "buy_button"},
|
||||
}
|
||||
for subj in assigned[: len(assigned) // 2]
|
||||
]
|
||||
batch_result = process_events_batch(click_events)
|
||||
self._ok(
|
||||
f"Clicks: accepted={batch_result.accepted}, "
|
||||
f"duplicates={batch_result.duplicates}, "
|
||||
f"rejected={batch_result.rejected}"
|
||||
)
|
||||
|
||||
self._step("10. Build experiment report")
|
||||
report = build_experiment_report(experiment)
|
||||
self.stdout.write(json.dumps(report, indent=2, default=str))
|
||||
|
||||
self._step("E2E demo completed successfully!")
|
||||
|
||||
def _step(self, msg: str) -> None:
|
||||
self.stdout.write(self.style.MIGRATE_HEADING(f"\n=== {msg}"))
|
||||
|
||||
def _ok(self, msg: str) -> None:
|
||||
self.stdout.write(self.style.SUCCESS(msg))
|
||||
|
||||
def _get_or_create_user(
|
||||
self,
|
||||
username: str,
|
||||
role: str,
|
||||
) -> User:
|
||||
user, created = User.objects.get_or_create(
|
||||
username=username,
|
||||
defaults={
|
||||
"email": f"{username}@lotty.local",
|
||||
"role": role,
|
||||
},
|
||||
password="password123",
|
||||
)
|
||||
if created:
|
||||
user.set_password("password123")
|
||||
user.save()
|
||||
status = "created" if created else "exists"
|
||||
self._ok(
|
||||
f"User {status}: {username} ({role})",
|
||||
)
|
||||
return user
|
||||
|
||||
def _ensure_review_policy(self, owner: User, approver: User) -> None:
|
||||
group, _ = ApproverGroup.objects.get_or_create(
|
||||
experimenter=owner,
|
||||
defaults={"min_approvals": 1},
|
||||
)
|
||||
group.approvers.add(approver)
|
||||
|
||||
def _get_or_create_event_type(
|
||||
self,
|
||||
name: str,
|
||||
display_name: str,
|
||||
*,
|
||||
is_exposure: bool = False,
|
||||
requires_exposure: bool,
|
||||
):
|
||||
et, created = EventType.objects.get_or_create(
|
||||
name=name,
|
||||
defaults={
|
||||
"display_name": display_name,
|
||||
"is_exposure": is_exposure,
|
||||
"requires_exposure": requires_exposure,
|
||||
},
|
||||
)
|
||||
et_status = "created" if created else "exists"
|
||||
self._ok(f"EventType {et_status}: {name}")
|
||||
return et
|
||||
|
||||
def _get_or_create_metric(self, key: str, name: str) -> MetricDefinition:
|
||||
metric, created = MetricDefinition.objects.get_or_create(
|
||||
key=key,
|
||||
defaults={
|
||||
"name": name,
|
||||
"metric_type": "ratio",
|
||||
"direction": "higher_is_better",
|
||||
"calculation_rule": {
|
||||
"numerator_event": "click",
|
||||
"denominator_event": "exposure",
|
||||
},
|
||||
},
|
||||
)
|
||||
m_status = "created" if created else "exists"
|
||||
self._ok(f"Metric {m_status}: {key}")
|
||||
return metric
|
||||
Reference in New Issue
Block a user