From d3731e78df75e0d2fbc581f10c41ed92072405f5 Mon Sep 17 00:00:00 2001 From: ITQ Date: Tue, 24 Feb 2026 23:22:31 +0300 Subject: [PATCH] chore(): final fixes before deadline - added images of notification feature working - added seed data script - race condition fix for notifications worker - small improvements --- README.md | 22 + RUNBOOK.md | 70 +- assets/images/email_notifications.png | 3 + assets/images/telegram_notifications.png | 3 + infrastructure/configs/backend/.env.template | 2 +- infrastructure/http/seed-demo-data.sh | 908 +++++++++++++++++++ src/backend/apps/notifications/services.py | 100 +- src/backend/justfile | 2 +- 8 files changed, 1067 insertions(+), 43 deletions(-) create mode 100644 assets/images/email_notifications.png create mode 100644 assets/images/telegram_notifications.png create mode 100755 infrastructure/http/seed-demo-data.sh diff --git a/README.md b/README.md index 55c31e7..8bcb06c 100644 --- a/README.md +++ b/README.md @@ -158,8 +158,30 @@ Reproducible k6 profile for `POST /api/v1/decide`: - runner: [infrastructure/k6/run-decide.sh](./infrastructure/k6/run-decide.sh) - guide: [infrastructure/k6/README.md](./infrastructure/k6/README.md) +## Demo data seed (HTTP) + +One-command API seed for demo datasets: + +- script: [infrastructure/http/seed-demo-data.sh](./infrastructure/http/seed-demo-data.sh) +- runbook section: [RUNBOOK.md](./RUNBOOK.md#35-seed-демо-данных-через-http-api) + +Example: +```bash +BACKEND_BASE_URL='http://127.0.0.1' \ +TG_BOT_TOKEN='' \ +TG_CHAT_ID='' \ +./infrastructure/http/seed-demo-data.sh +``` + +SMTP can be enabled with `SMTP_ENABLED=true` and SMTP env settings (see `RUNBOOK.md`). + ## Selected extra features - Notifications + + +![Mail](./assets/images/email_notifications.png) +![Telegram](./assets/images/telegram_notifications.png) + - Learnings Library - Domain Conflicts diff --git a/RUNBOOK.md b/RUNBOOK.md index b7d6e02..c72f904 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -34,8 +34,6 @@ Порты по умолчанию (см. `.env.template`): - reverse proxy: `80` -- backend direct: `14609` -- static direct: `14610` ### 2.2 Дополнительный локальный сценарий (для тестов/линтинга) @@ -95,6 +93,74 @@ docker compose -f compose.yaml logs -f backend backend-celery-worker backend-cel docker compose -f compose.prod.yaml --profile observability logs -f backend backend-celery-worker backend-celery-beat ``` +### 3.5 Seed демо-данных через HTTP API + +Скрипт создаёт следующие сущности: +- test flags; +- experiments + variants + lifecycle до `running`; +- event types (`exposure`, `click`); +- metrics (`ctr`, `exposure_count`, `click_count`) и привязку к эксперименту; +- telegram notification channel + rules; +- smtp notification channel + rules (опционально); +- conflict domain с включёнными экспериментами; +- два learning (для color/copy), чтобы проверить похожие; +- guardrail (`pause`) c порогом, который не срабатывает сразу; +- subjects + decide + события (`show+click`, `show-only`, `click-only`, out-of-order). + +Команда: +```bash +BACKEND_BASE_URL='http://127.0.0.1' \ +TG_BOT_TOKEN='6196898691:AAGbCxOf7-iXWKkKeOr9GrgHKcw6G7Ou1zU' \ +TG_CHAT_ID='826812483' \ +./infrastructure/http/seed-demo-data.sh +``` + +Команда c SMTP: +```bash +BACKEND_BASE_URL='http://127.0.0.1' \ +TG_BOT_TOKEN='6196898691:AAGbCxOf7-iXWKkKeOr9GrgHKcw6G7Ou1zU' \ +TG_CHAT_ID='826812483' \ +SMTP_ENABLED=true \ +SMTP_RECIPIENT='itq.dev@ya.ru' \ +SMTP_FROM_EMAIL='mikrotik@itqdev.xyz' \ +SMTP_HOST='smtp.office365.com' \ +SMTP_PORT=587 \ +SMTP_USERNAME='mikrotik@itqdev.xyz' \ +SMTP_PASSWORD='5x9NqfIO4tVMg0B' \ +SMTP_USE_TLS=true \ +SMTP_USE_SSL=false \ +SMTP_TIMEOUT=15 \ +./infrastructure/http/seed-demo-data.sh +``` + +Что получаем: +- скрипт печатает ключевые ID прямо в stdout; +- артефакты в `artifacts/http-seed//`; +- сводка в `artifacts/http-seed//summary.json`. + +### 3.6 Что проверяем в Swagger ([localhost/api/v1/docs](http://localhost:80/api/v1/docs)) + +1. Открыть `http://127.0.0.1/api/v1/docs`. +2. Получить токен через `POST /api/v1/auth/login` (admin/experimenter/approver). +3. Нажать `Authorize` и вставить `Bearer `. +4. Проверить `GET /api/v1/reports/{experiment_id}` для `experiment_color_id` из seed-вывода: +- по вариантам есть `exposures`, `click_count`, `exposure_count`, `ctr`. +5. Проверить `POST /api/v1/experiments/{experiment_id}/guardrails/check`: +- `triggered=0`. +6. Проверить `GET /api/v1/conflicts/domains/{domain_id}/experiments`: +- видно оба running эксперимента в домене. +7. Проверить `POST /api/v1/decide` для `color_flag_key`: +- `reason=experiment_assigned`. +8. Проверить `POST /api/v1/decide` для `copy_flag_key` тем же subject: +- `reason=domain_conflict`. +9. Проверить `GET /api/v1/notification-channels`, `GET /api/v1/notification-rules`, `GET /api/v1/notification-logs`: +- есть telegram channel/rules и, если `SMTP_ENABLED=true`, smtp channel/rules. +10. Проверить `GET /api/v1/learnings`: +- есть как минимум 2 seeded learning. +11. Проверить `GET /api/v1/experiments/{experiment_id}/similar-learnings` для `experiment_color_id`: +- ответ непустой; +- в выдаче есть `learning_copy_id` и `similarity_score`. + ## 4. Проверка критериев B1..B10 ### B1. Запуск и воспроизводимость diff --git a/assets/images/email_notifications.png b/assets/images/email_notifications.png new file mode 100644 index 0000000..71ba8e8 --- /dev/null +++ b/assets/images/email_notifications.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54d5f722c672082051a16040db2f4be5db89b017d11b876882b6285ae4a9b576 +size 79963 diff --git a/assets/images/telegram_notifications.png b/assets/images/telegram_notifications.png new file mode 100644 index 0000000..42d2c73 --- /dev/null +++ b/assets/images/telegram_notifications.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce123a6e559495022eb092375037f076bf6b8e30c9bd056493a61ac5aa5f90f +size 86545 diff --git a/infrastructure/configs/backend/.env.template b/infrastructure/configs/backend/.env.template index 2c11a7c..f61d81b 100644 --- a/infrastructure/configs/backend/.env.template +++ b/infrastructure/configs/backend/.env.template @@ -1,4 +1,4 @@ -DJANGO_SECRET_KEY=very_insecure_key +DJANGO_SECRET_KEY=pfX50TXRgmFKyB8RnDPRc6OmDQ1IPH DJANGO_DEBUG=False DJANGO_ALLOWED_HOSTS=* DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost,http://127.0.0.1 diff --git a/infrastructure/http/seed-demo-data.sh b/infrastructure/http/seed-demo-data.sh new file mode 100755 index 0000000..9958228 --- /dev/null +++ b/infrastructure/http/seed-demo-data.sh @@ -0,0 +1,908 @@ +#!/usr/bin/env bash + +set -euo pipefail + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "missing required command: $cmd" >&2 + exit 1 + fi +} + +require_cmd curl +require_cmd jq + +RUN_ID="${RUN_ID:-$(date +%s)}" +PFX="seed_${RUN_ID}" +BACKEND_BASE_URL="${BACKEND_BASE_URL:-${BASE_URL:-http://127.0.0.1}}" +API="${BACKEND_BASE_URL%/}/api/v1" +PASSWORD="${PASSWORD:-password123}" +ADMIN_USER="${ADMIN_USER:-admin}" +EXPERIMENTER_USER="${EXPERIMENTER_USER:-experimenter}" +APPROVER_USER="${APPROVER_USER:-approver}" +TG_BOT_TOKEN="${TG_BOT_TOKEN:-}" +TG_CHAT_ID="${TG_CHAT_ID:-}" +SMTP_ENABLED="${SMTP_ENABLED:-false}" +SMTP_CHANNEL_NAME="${SMTP_CHANNEL_NAME:-${PFX} smtp provider}" +SMTP_RECIPIENT="${SMTP_RECIPIENT:-}" +SMTP_FROM_EMAIL="${SMTP_FROM_EMAIL:-lotty@lotty.local}" +SMTP_HOST="${SMTP_HOST:-}" +SMTP_PORT="${SMTP_PORT:-}" +SMTP_USERNAME="${SMTP_USERNAME:-}" +SMTP_PASSWORD="${SMTP_PASSWORD:-}" +SMTP_USE_TLS="${SMTP_USE_TLS:-false}" +SMTP_USE_SSL="${SMTP_USE_SSL:-false}" +SMTP_TIMEOUT="${SMTP_TIMEOUT:-10}" +RESULTS_DIR="${RESULTS_DIR:-artifacts/http-seed/${RUN_ID}}" + +if [[ -z "$TG_BOT_TOKEN" || -z "$TG_CHAT_ID" ]]; then + echo "TG_BOT_TOKEN and TG_CHAT_ID are required" >&2 + exit 1 +fi + +is_true() { + local value + value="$(printf "%s" "$1" | tr "[:upper:]" "[:lower:]")" + [[ "$value" == "1" || "$value" == "true" || "$value" == "yes" || "$value" == "on" ]] +} + +if is_true "$SMTP_ENABLED" && [[ -z "$SMTP_RECIPIENT" ]]; then + echo "SMTP_RECIPIENT is required when SMTP_ENABLED=true" >&2 + exit 1 +fi + +mkdir -p "$RESULTS_DIR" + +api_call() { + local method="$1" + local url="$2" + local token="$3" + local data="${4:-}" + + local -a headers + headers=(-H "Content-Type: application/json") + if [[ -n "$token" ]]; then + headers+=(-H "Authorization: Bearer $token") + fi + + if [[ -n "$data" ]]; then + curl -sS --fail-with-body -X "$method" "$url" "${headers[@]}" -d "$data" + else + curl -sS --fail-with-body -X "$method" "$url" "${headers[@]}" + fi +} + +login() { + local username="$1" + local payload + payload="$( + jq -n \ + --arg username "$username" \ + --arg password "$PASSWORD" \ + '{username: $username, password: $password}' + )" + api_call POST "$API/auth/login" "" "$payload" | jq -r ".access" +} + +echo "run_id=$RUN_ID" + +ADMIN_TOKEN="$(login "$ADMIN_USER")" +EXPERIMENTER_TOKEN="$(login "$EXPERIMENTER_USER")" +APPROVER_TOKEN="$(login "$APPROVER_USER")" + +review_settings_payload="$( + jq -n '{default_min_approvals: 1, allow_any_approver: true}' +)" +api_call \ + PUT \ + "$API/reviews/settings" \ + "$ADMIN_TOKEN" \ + "$review_settings_payload" >/dev/null + +FLAG_COLOR_KEY="${PFX}_button_color" +FLAG_COPY_KEY="${PFX}_button_copy" + +flag_color_payload="$( + jq -n \ + --arg key "$FLAG_COLOR_KEY" \ + --arg name "${PFX} button color" \ + '{key: $key, name: $name, value_type: "string", default_value: "green"}' +)" +FLAG_COLOR_JSON="$( + api_call \ + POST \ + "$API/flags" \ + "$EXPERIMENTER_TOKEN" \ + "$flag_color_payload" +)" +FLAG_COLOR_ID="$(echo "$FLAG_COLOR_JSON" | jq -r ".id")" + +flag_copy_payload="$( + jq -n \ + --arg key "$FLAG_COPY_KEY" \ + --arg name "${PFX} button copy" \ + '{key: $key, name: $name, value_type: "string", default_value: "buy"}' +)" +FLAG_COPY_JSON="$( + api_call \ + POST \ + "$API/flags" \ + "$EXPERIMENTER_TOKEN" \ + "$flag_copy_payload" +)" +FLAG_COPY_ID="$(echo "$FLAG_COPY_JSON" | jq -r ".id")" + +create_experiment() { + local token="$1" + local flag_id="$2" + local name="$3" + local targeting_rules="$4" + local payload + payload="$( + jq -n \ + --arg flag_id "$flag_id" \ + --arg name "$name" \ + --arg targeting_rules "$targeting_rules" \ + '{ + flag_id: $flag_id, + name: $name, + description: "seeded via HTTP", + hypothesis: "seeded hypothesis", + traffic_allocation: 100, + targeting_rules: $targeting_rules + }' + )" + api_call POST "$API/experiments" "$token" "$payload" +} + +create_variant() { + local token="$1" + local experiment_id="$2" + local name="$3" + local value="$4" + local weight="$5" + local is_control="$6" + local payload + payload="$( + jq -n \ + --arg name "$name" \ + --arg value "$value" \ + --argjson weight "$weight" \ + --argjson is_control "$is_control" \ + '{name: $name, value: $value, weight: $weight, is_control: $is_control}' + )" + api_call \ + POST \ + "$API/experiments/$experiment_id/variants" \ + "$token" \ + "$payload" +} + +advance_experiment_to_running() { + local experiment_id="$1" + local approve_payload + approve_payload="$(jq -n '{comment: "seed approve"}')" + api_call \ + POST \ + "$API/experiments/$experiment_id/submit-for-review" \ + "$EXPERIMENTER_TOKEN" >/dev/null + api_call \ + POST \ + "$API/experiments/$experiment_id/approve" \ + "$APPROVER_TOKEN" \ + "$approve_payload" >/dev/null + api_call \ + POST \ + "$API/experiments/$experiment_id/start" \ + "$EXPERIMENTER_TOKEN" >/dev/null +} + +EXPERIMENT_COLOR_JSON="$( + create_experiment \ + "$EXPERIMENTER_TOKEN" \ + "$FLAG_COLOR_ID" \ + "${PFX} color experiment" \ + 'country == "US"' +)" +EXPERIMENT_COLOR_ID="$(echo "$EXPERIMENT_COLOR_JSON" | jq -r ".id")" + +CHANNEL_PAYLOAD="$( + jq -n \ + --arg name "${PFX} telegram provider" \ + --arg bot_token "$TG_BOT_TOKEN" \ + --arg chat_id "$TG_CHAT_ID" \ + '{ + channel_type: "telegram", + name: $name, + config: { + bot_token: $bot_token, + chat_id: $chat_id + } + }' +)" +CHANNEL_JSON="$( + api_call \ + POST \ + "$API/notification-channels" \ + "$ADMIN_TOKEN" \ + "$CHANNEL_PAYLOAD" +)" +CHANNEL_ID="$(echo "$CHANNEL_JSON" | jq -r ".id")" +SMTP_CHANNEL_ID="" + +notification_events=( + experiment_started + experiment_paused + experiment_resumed + experiment_completed + guardrail_triggered + review_requested + review_approved + review_rejected +) + +for event_type in "${notification_events[@]}"; do + rule_payload="$( + jq -n \ + --arg event_type "$event_type" \ + --arg channel_id "$CHANNEL_ID" \ + --arg experiment_id "$EXPERIMENT_COLOR_ID" \ + '{ + event_type: $event_type, + channel_id: $channel_id, + experiment_id: $experiment_id, + rate_limit_window_seconds: 60, + rate_limit_max_notifications: 100 + }' + )" + api_call \ + POST \ + "$API/notification-rules" \ + "$ADMIN_TOKEN" \ + "$rule_payload" >/dev/null +done + +if is_true "$SMTP_ENABLED"; then + SMTP_USE_TLS_JSON="false" + SMTP_USE_SSL_JSON="false" + if is_true "$SMTP_USE_TLS"; then + SMTP_USE_TLS_JSON="true" + fi + if is_true "$SMTP_USE_SSL"; then + SMTP_USE_SSL_JSON="true" + fi + + SMTP_PORT_JSON="null" + if [[ -n "$SMTP_PORT" ]]; then + SMTP_PORT_JSON="$SMTP_PORT" + fi + + SMTP_TIMEOUT_JSON="null" + if [[ -n "$SMTP_TIMEOUT" ]]; then + SMTP_TIMEOUT_JSON="$SMTP_TIMEOUT" + fi + + SMTP_CHANNEL_PAYLOAD="$( + jq -n \ + --arg name "$SMTP_CHANNEL_NAME" \ + --arg recipient "$SMTP_RECIPIENT" \ + --arg from_email "$SMTP_FROM_EMAIL" \ + --arg host "$SMTP_HOST" \ + --arg username "$SMTP_USERNAME" \ + --arg password "$SMTP_PASSWORD" \ + --argjson port "$SMTP_PORT_JSON" \ + --argjson timeout "$SMTP_TIMEOUT_JSON" \ + --argjson use_tls "$SMTP_USE_TLS_JSON" \ + --argjson use_ssl "$SMTP_USE_SSL_JSON" \ + '{ + channel_type: "smtp", + name: $name, + config: ( + { + recipient: $recipient, + from_email: $from_email, + use_tls: $use_tls, + use_ssl: $use_ssl + } + + (if $host != "" then {host: $host} else {} end) + + (if $port != null then {port: $port} else {} end) + + (if $username != "" then {username: $username} else {} end) + + (if $password != "" then {password: $password} else {} end) + + (if $timeout != null then {timeout: $timeout} else {} end) + ) + }' + )" + SMTP_CHANNEL_JSON="$( + api_call \ + POST \ + "$API/notification-channels" \ + "$ADMIN_TOKEN" \ + "$SMTP_CHANNEL_PAYLOAD" + )" + SMTP_CHANNEL_ID="$(echo "$SMTP_CHANNEL_JSON" | jq -r ".id")" + + for event_type in "${notification_events[@]}"; do + smtp_rule_payload="$( + jq -n \ + --arg event_type "$event_type" \ + --arg channel_id "$SMTP_CHANNEL_ID" \ + --arg experiment_id "$EXPERIMENT_COLOR_ID" \ + '{ + event_type: $event_type, + channel_id: $channel_id, + experiment_id: $experiment_id, + rate_limit_window_seconds: 60, + rate_limit_max_notifications: 100 + }' + )" + api_call \ + POST \ + "$API/notification-rules" \ + "$ADMIN_TOKEN" \ + "$smtp_rule_payload" >/dev/null + done +fi + +VARIANT_COLOR_CONTROL_JSON="$( + create_variant \ + "$EXPERIMENTER_TOKEN" \ + "$EXPERIMENT_COLOR_ID" \ + "control" \ + "blue" \ + 50 \ + true +)" +VARIANT_COLOR_CONTROL_ID="$( + echo "$VARIANT_COLOR_CONTROL_JSON" | jq -r ".id" +)" + +VARIANT_COLOR_TREATMENT_JSON="$( + create_variant \ + "$EXPERIMENTER_TOKEN" \ + "$EXPERIMENT_COLOR_ID" \ + "treatment" \ + "red" \ + 50 \ + false +)" +VARIANT_COLOR_TREATMENT_ID="$( + echo "$VARIANT_COLOR_TREATMENT_JSON" | jq -r ".id" +)" + +EXPERIMENT_COPY_JSON="$( + create_experiment \ + "$EXPERIMENTER_TOKEN" \ + "$FLAG_COPY_ID" \ + "${PFX} copy experiment" \ + 'country == "US"' +)" +EXPERIMENT_COPY_ID="$(echo "$EXPERIMENT_COPY_JSON" | jq -r ".id")" + +create_variant \ + "$EXPERIMENTER_TOKEN" \ + "$EXPERIMENT_COPY_ID" \ + "control" \ + "buy" \ + 50 \ + true >/dev/null +create_variant \ + "$EXPERIMENTER_TOKEN" \ + "$EXPERIMENT_COPY_ID" \ + "treatment" \ + "checkout" \ + 50 \ + false >/dev/null + +advance_experiment_to_running "$EXPERIMENT_COLOR_ID" +advance_experiment_to_running "$EXPERIMENT_COPY_ID" + +CONFLICT_DOMAIN_PAYLOAD="$( + jq -n \ + --arg name "${PFX}_checkout_domain" \ + '{ + name: $name, + description: "seeded domain conflict", + policy: "priority", + max_concurrent: 2 + }' +)" +CONFLICT_DOMAIN_JSON="$( + api_call \ + POST \ + "$API/conflicts/domains" \ + "$ADMIN_TOKEN" \ + "$CONFLICT_DOMAIN_PAYLOAD" +)" +CONFLICT_DOMAIN_ID="$(echo "$CONFLICT_DOMAIN_JSON" | jq -r ".id")" + +add_to_domain() { + local experiment_id="$1" + local priority="$2" + local payload + payload="$( + jq -n \ + --arg experiment_id "$experiment_id" \ + --argjson priority "$priority" \ + '{experiment_id: $experiment_id, priority: $priority}' + )" + api_call \ + POST \ + "$API/conflicts/domains/$CONFLICT_DOMAIN_ID/experiments" \ + "$ADMIN_TOKEN" \ + "$payload" >/dev/null +} + +add_to_domain "$EXPERIMENT_COLOR_ID" 100 +add_to_domain "$EXPERIMENT_COPY_ID" 10 + +ET_EXPOSURE_NAME="${PFX}_exposure" +ET_CLICK_NAME="${PFX}_click" + +create_event_type() { + local name="$1" + local display_name="$2" + local is_exposure="$3" + local requires_exposure="$4" + local payload + payload="$( + jq -n \ + --arg name "$name" \ + --arg display_name "$display_name" \ + --argjson is_exposure "$is_exposure" \ + --argjson requires_exposure "$requires_exposure" \ + '{ + name: $name, + display_name: $display_name, + description: "seeded event type", + is_exposure: $is_exposure, + requires_exposure: $requires_exposure, + required_fields: [] + }' + )" + api_call POST "$API/events/event-types" "$ADMIN_TOKEN" "$payload" +} + +ET_EXPOSURE_JSON="$( + create_event_type "$ET_EXPOSURE_NAME" "${PFX} Exposure" true false +)" +ET_CLICK_JSON="$( + create_event_type "$ET_CLICK_NAME" "${PFX} Click" false true +)" +ET_EXPOSURE_ID="$(echo "$ET_EXPOSURE_JSON" | jq -r ".id")" +ET_CLICK_ID="$(echo "$ET_CLICK_JSON" | jq -r ".id")" + +METRIC_CTR_PAYLOAD="$( + jq -n \ + --arg key "${PFX}_ctr" \ + --arg name "${PFX} CTR" \ + --arg numerator_event "$ET_CLICK_NAME" \ + --arg denominator_event "$ET_EXPOSURE_NAME" \ + '{ + key: $key, + name: $name, + description: "seeded ctr metric", + metric_type: "ratio", + direction: "higher_is_better", + calculation_rule: { + numerator_event: $numerator_event, + denominator_event: $denominator_event + } + }' +)" +METRIC_CTR_JSON="$( + api_call POST "$API/metrics" "$ADMIN_TOKEN" "$METRIC_CTR_PAYLOAD" +)" +METRIC_CTR_ID="$(echo "$METRIC_CTR_JSON" | jq -r ".id")" + +METRIC_EXPOSURE_COUNT_PAYLOAD="$( + jq -n \ + --arg key "${PFX}_exposure_count" \ + --arg name "${PFX} Exposure Count" \ + --arg event "$ET_EXPOSURE_NAME" \ + '{ + key: $key, + name: $name, + description: "seeded exposure count metric", + metric_type: "count", + direction: "higher_is_better", + calculation_rule: {event: $event} + }' +)" +METRIC_EXPOSURE_COUNT_JSON="$( + api_call \ + POST \ + "$API/metrics" \ + "$ADMIN_TOKEN" \ + "$METRIC_EXPOSURE_COUNT_PAYLOAD" +)" +METRIC_EXPOSURE_COUNT_ID="$( + echo "$METRIC_EXPOSURE_COUNT_JSON" | jq -r ".id" +)" + +METRIC_CLICK_COUNT_PAYLOAD="$( + jq -n \ + --arg key "${PFX}_click_count" \ + --arg name "${PFX} Click Count" \ + --arg event "$ET_CLICK_NAME" \ + '{ + key: $key, + name: $name, + description: "seeded click count metric", + metric_type: "count", + direction: "higher_is_better", + calculation_rule: {event: $event} + }' +)" +METRIC_CLICK_COUNT_JSON="$( + api_call \ + POST \ + "$API/metrics" \ + "$ADMIN_TOKEN" \ + "$METRIC_CLICK_COUNT_PAYLOAD" +)" +METRIC_CLICK_COUNT_ID="$( + echo "$METRIC_CLICK_COUNT_JSON" | jq -r ".id" +)" + +attach_metric() { + local metric_id="$1" + local is_primary="$2" + local payload + payload="$( + jq -n \ + --arg metric_id "$metric_id" \ + --argjson is_primary "$is_primary" \ + '{metric_id: $metric_id, is_primary: $is_primary}' + )" + api_call \ + POST \ + "$API/experiments/$EXPERIMENT_COLOR_ID/metrics" \ + "$ADMIN_TOKEN" \ + "$payload" >/dev/null +} + +attach_metric "$METRIC_CTR_ID" true +attach_metric "$METRIC_EXPOSURE_COUNT_ID" false +attach_metric "$METRIC_CLICK_COUNT_ID" false + +GUARDRAIL_PAYLOAD="$( + jq -n \ + --arg metric_id "$METRIC_CTR_ID" \ + '{ + metric_id: $metric_id, + threshold: 0, + observation_window_minutes: 60, + action: "pause" + }' +)" +GUARDRAIL_JSON="$( + api_call \ + POST \ + "$API/experiments/$EXPERIMENT_COLOR_ID/guardrails" \ + "$ADMIN_TOKEN" \ + "$GUARDRAIL_PAYLOAD" +)" +GUARDRAIL_ID="$(echo "$GUARDRAIL_JSON" | jq -r ".id")" + +LEARNING_COLOR_PAYLOAD="$( + jq -n \ + --arg experiment_id "$EXPERIMENT_COLOR_ID" \ + --arg hypothesis "Blue vs red purchase button on checkout" \ + --arg findings "Seed dataset for report/guardrails/conflicts demo" \ + '{ + experiment_id: $experiment_id, + hypothesis: $hypothesis, + findings: $findings, + tags: ["checkout", "button", "color", "seed"], + context_summary: "Seeded via HTTP for live demo" + }' +)" +LEARNING_COLOR_JSON="$( + api_call \ + POST \ + "$API/learnings" \ + "$EXPERIMENTER_TOKEN" \ + "$LEARNING_COLOR_PAYLOAD" +)" +LEARNING_COLOR_ID="$(echo "$LEARNING_COLOR_JSON" | jq -r ".id")" + +LEARNING_COPY_PAYLOAD="$( + jq -n \ + --arg experiment_id "$EXPERIMENT_COPY_ID" \ + --arg hypothesis "Checkout button wording with color context" \ + --arg findings "Variant copy affects click intent on checkout CTA" \ + '{ + experiment_id: $experiment_id, + hypothesis: $hypothesis, + findings: $findings, + tags: ["checkout", "button", "copy", "seed"], + context_summary: "Seeded pair for similar learnings endpoint" + }' +)" +LEARNING_COPY_JSON="$( + api_call \ + POST \ + "$API/learnings" \ + "$EXPERIMENTER_TOKEN" \ + "$LEARNING_COPY_PAYLOAD" +)" +LEARNING_COPY_ID="$(echo "$LEARNING_COPY_JSON" | jq -r ".id")" + +subjects=(u01 u02 u03 u04 u05 u06 u07 u08 u09 u10 u11 u12) +decisions_file="$(mktemp)" + +for suffix in "${subjects[@]}"; do + subject_id="${PFX}_${suffix}" + decide_payload="$( + jq -n \ + --arg subject_id "$subject_id" \ + --arg flag "$FLAG_COLOR_KEY" \ + '{ + subject_id: $subject_id, + subject_attributes: {country: "US"}, + flags: [$flag] + }' + )" + decide_json="$( + api_call POST "$API/decide" "" "$decide_payload" + )" + reason="$(echo "$decide_json" | jq -r ".decisions[0].reason")" + if [[ "$reason" != "experiment_assigned" ]]; then + echo "unexpected decide reason for $subject_id: $reason" >&2 + exit 1 + fi + decision_id="$(echo "$decide_json" | jq -r ".decisions[0].decision_id")" + printf "%s\t%s\n" "$subject_id" "$decision_id" >> "$decisions_file" +done + +now_ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +events_pre_file="$(mktemp)" +events_main_file="$(mktemp)" +trap 'rm -f "$events_pre_file" "$events_main_file" "$decisions_file"' EXIT + +decision_for_subject() { + local subject_id="$1" + awk -F $'\t' -v subject_id="$subject_id" ' + $1 == subject_id { print $2; found = 1; exit } + END { if (!found) exit 1 } + ' "$decisions_file" +} + +append_event() { + local file="$1" + local event_id="$2" + local event_type="$3" + local decision_id="$4" + local subject_id="$5" + local timestamp="$6" + jq -n \ + --arg event_id "$event_id" \ + --arg event_type "$event_type" \ + --arg decision_id "$decision_id" \ + --arg subject_id "$subject_id" \ + --arg timestamp "$timestamp" \ + '{ + event_id: $event_id, + event_type: $event_type, + decision_id: $decision_id, + subject_id: $subject_id, + timestamp: $timestamp, + properties: {} + }' >> "$file" +} + +click_only_subjects=("${PFX}_u09" "${PFX}_u10") +out_of_order_subject="${PFX}_u11" + +counter=1 +for subject_id in "${click_only_subjects[@]}" "$out_of_order_subject"; do + append_event \ + "$events_pre_file" \ + "${PFX}_click_pre_${counter}" \ + "$ET_CLICK_NAME" \ + "$(decision_for_subject "$subject_id")" \ + "$subject_id" \ + "$now_ts" + counter=$((counter + 1)) +done + +PRE_EVENTS_PAYLOAD="$(jq -s '{events: .}' "$events_pre_file")" +PRE_EVENTS_RESULT="$( + api_call POST "$API/events" "" "$PRE_EVENTS_PAYLOAD" +)" + +shown_click_subjects=("${PFX}_u01" "${PFX}_u03" "${PFX}_u05") +shown_only_subjects=( + "${PFX}_u02" + "${PFX}_u04" + "${PFX}_u06" + "${PFX}_u07" + "${PFX}_u08" +) +shown_subjects=( + "${shown_click_subjects[@]}" + "${shown_only_subjects[@]}" + "$out_of_order_subject" +) + +counter=1 +for subject_id in "${shown_subjects[@]}"; do + append_event \ + "$events_main_file" \ + "${PFX}_exp_${counter}" \ + "$ET_EXPOSURE_NAME" \ + "$(decision_for_subject "$subject_id")" \ + "$subject_id" \ + "$now_ts" + counter=$((counter + 1)) +done + +counter=1 +for subject_id in "${shown_click_subjects[@]}"; do + append_event \ + "$events_main_file" \ + "${PFX}_click_${counter}" \ + "$ET_CLICK_NAME" \ + "$(decision_for_subject "$subject_id")" \ + "$subject_id" \ + "$now_ts" + counter=$((counter + 1)) +done + +MAIN_EVENTS_PAYLOAD="$(jq -s '{events: .}' "$events_main_file")" +MAIN_EVENTS_RESULT="$( + api_call POST "$API/events" "" "$MAIN_EVENTS_PAYLOAD" +)" + +conflict_decide_payload="$( + jq -n \ + --arg subject_id "${PFX}_u01" \ + --arg flag "$FLAG_COPY_KEY" \ + '{ + subject_id: $subject_id, + subject_attributes: {country: "US"}, + flags: [$flag] + }' +)" +CONFLICT_DECIDE_RESULT="$( + api_call POST "$API/decide" "" "$conflict_decide_payload" +)" +CONFLICT_DECIDE_REASON="$( + echo "$CONFLICT_DECIDE_RESULT" | jq -r ".decisions[0].reason" +)" +if [[ "$CONFLICT_DECIDE_REASON" != "domain_conflict" ]]; then + echo "unexpected conflict decide reason: $CONFLICT_DECIDE_REASON" >&2 + exit 1 +fi + +GUARDRAIL_CHECK_RESULT="$( + api_call \ + POST \ + "$API/experiments/$EXPERIMENT_COLOR_ID/guardrails/check" \ + "$ADMIN_TOKEN" +)" +GUARDRAIL_TRIGGERED="$( + echo "$GUARDRAIL_CHECK_RESULT" | jq -r ".triggered" +)" +if [[ "$GUARDRAIL_TRIGGERED" != "0" ]]; then + echo "guardrail triggered immediately: $GUARDRAIL_TRIGGERED" >&2 + exit 1 +fi + +REPORT_JSON="$( + api_call GET "$API/reports/$EXPERIMENT_COLOR_ID" "$ADMIN_TOKEN" +)" + +SIMILAR_LEARNINGS_JSON="$( + api_call \ + GET \ + "$API/experiments/$EXPERIMENT_COLOR_ID/similar-learnings?limit=5" \ + "$ADMIN_TOKEN" +)" +SIMILAR_COUNT="$(echo "$SIMILAR_LEARNINGS_JSON" | jq 'length')" +if [[ "$SIMILAR_COUNT" -lt "1" ]]; then + echo "similar learnings result is empty for experiment: $EXPERIMENT_COLOR_ID" >&2 + exit 1 +fi + +echo "$PRE_EVENTS_RESULT" > "$RESULTS_DIR/pre_events_result.json" +echo "$MAIN_EVENTS_RESULT" > "$RESULTS_DIR/main_events_result.json" +echo "$CONFLICT_DECIDE_RESULT" > "$RESULTS_DIR/conflict_decide_result.json" +echo "$GUARDRAIL_CHECK_RESULT" > "$RESULTS_DIR/guardrail_check_result.json" +echo "$REPORT_JSON" > "$RESULTS_DIR/report.json" +echo "$SIMILAR_LEARNINGS_JSON" > "$RESULTS_DIR/similar_learnings.json" + +jq -n \ + --arg run_id "$RUN_ID" \ + --arg prefix "$PFX" \ + --arg flag_color_key "$FLAG_COLOR_KEY" \ + --arg flag_copy_key "$FLAG_COPY_KEY" \ + --arg experiment_color_id "$EXPERIMENT_COLOR_ID" \ + --arg experiment_copy_id "$EXPERIMENT_COPY_ID" \ + --arg metric_ctr_id "$METRIC_CTR_ID" \ + --arg metric_exposure_count_id "$METRIC_EXPOSURE_COUNT_ID" \ + --arg metric_click_count_id "$METRIC_CLICK_COUNT_ID" \ + --arg event_type_exposure "$ET_EXPOSURE_NAME" \ + --arg event_type_click "$ET_CLICK_NAME" \ + --arg telegram_channel_id "$CHANNEL_ID" \ + --arg smtp_channel_id "$SMTP_CHANNEL_ID" \ + --argjson smtp_enabled "$(is_true "$SMTP_ENABLED" && echo true || echo false)" \ + --arg conflict_domain_id "$CONFLICT_DOMAIN_ID" \ + --arg learning_color_id "$LEARNING_COLOR_ID" \ + --arg learning_copy_id "$LEARNING_COPY_ID" \ + --arg guardrail_id "$GUARDRAIL_ID" \ + --argjson pre_events_result "$PRE_EVENTS_RESULT" \ + --argjson main_events_result "$MAIN_EVENTS_RESULT" \ + --argjson guardrail_check "$GUARDRAIL_CHECK_RESULT" \ + --argjson similar_learnings "$SIMILAR_LEARNINGS_JSON" \ + --argjson report "$REPORT_JSON" \ + '{ + run_id: $run_id, + prefix: $prefix, + flags: { + color_flag_key: $flag_color_key, + copy_flag_key: $flag_copy_key + }, + experiments: { + color_experiment_id: $experiment_color_id, + copy_experiment_id: $experiment_copy_id + }, + event_types: { + exposure: $event_type_exposure, + click: $event_type_click + }, + metrics: { + ctr_id: $metric_ctr_id, + exposure_count_id: $metric_exposure_count_id, + click_count_id: $metric_click_count_id + }, + notification: { + telegram_channel_id: $telegram_channel_id, + smtp_enabled: $smtp_enabled, + smtp_channel_id: ($smtp_channel_id | if . == "" then null else . end) + }, + conflicts: { + domain_id: $conflict_domain_id + }, + learnings: { + color_learning_id: $learning_color_id, + copy_learning_id: $learning_copy_id, + similar_for_color_experiment: $similar_learnings + }, + guardrail: { + guardrail_id: $guardrail_id, + check: $guardrail_check + }, + events_ingestion: { + pre_events: $pre_events_result, + main_events: $main_events_result + }, + report: $report + }' > "$RESULTS_DIR/summary.json" + +echo "seed complete" +echo "backend_base_url=$BACKEND_BASE_URL" +echo "results_dir=$RESULTS_DIR" +echo "summary_file=$RESULTS_DIR/summary.json" +echo "flag_color_id=$FLAG_COLOR_ID" +echo "flag_copy_id=$FLAG_COPY_ID" +echo "experiment_color_id=$EXPERIMENT_COLOR_ID" +echo "experiment_copy_id=$EXPERIMENT_COPY_ID" +echo "variant_color_control_id=$VARIANT_COLOR_CONTROL_ID" +echo "variant_color_treatment_id=$VARIANT_COLOR_TREATMENT_ID" +echo "event_type_exposure_id=$ET_EXPOSURE_ID" +echo "event_type_click_id=$ET_CLICK_ID" +echo "metric_ctr_id=$METRIC_CTR_ID" +echo "metric_exposure_count_id=$METRIC_EXPOSURE_COUNT_ID" +echo "metric_click_count_id=$METRIC_CLICK_COUNT_ID" +echo "telegram_notification_channel_id=$CHANNEL_ID" +echo "smtp_enabled=$(is_true "$SMTP_ENABLED" && echo true || echo false)" +if [[ -n "$SMTP_CHANNEL_ID" ]]; then + echo "smtp_notification_channel_id=$SMTP_CHANNEL_ID" +fi +echo "conflict_domain_id=$CONFLICT_DOMAIN_ID" +echo "learning_color_id=$LEARNING_COLOR_ID" +echo "learning_copy_id=$LEARNING_COPY_ID" +echo "similar_learnings_count=$SIMILAR_COUNT" +echo "guardrail_id=$GUARDRAIL_ID" diff --git a/src/backend/apps/notifications/services.py b/src/backend/apps/notifications/services.py index 15ef463..9a18cd5 100644 --- a/src/backend/apps/notifications/services.py +++ b/src/backend/apps/notifications/services.py @@ -5,7 +5,7 @@ from typing import Any import requests from django.core.exceptions import ValidationError from django.core.mail import EmailMessage, get_connection -from django.db import transaction +from django.db import connection, transaction from django.db.models import Q, QuerySet from django.utils import timezone @@ -334,15 +334,64 @@ def _send_smtp(config: dict[str, Any], payload: dict[str, Any]) -> None: email.send(fail_silently=False) -def flush_pending_notifications() -> dict[str, int]: +def _pending_notifications_for_flush() -> QuerySet[NotificationLog]: pending = ( NotificationLog.objects.filter( status=NotificationStatus.PENDING, ) - .select_related("channel") .order_by("created_at") ) + if not connection.features.has_select_for_update: + return pending + if connection.features.has_select_for_update_skip_locked: + return pending.select_for_update(skip_locked=True) + return pending.select_for_update() + +def _flush_notification_log( + *, + log: NotificationLog, + senders: dict[str, Any], + results: dict[str, int], +) -> None: + channel = log.channel + if not channel or not channel.is_active: + log.status = NotificationStatus.FAILED + log.error = "Channel is inactive or missing." + log.save(update_fields=["status", "error"]) + results["failed"] += 1 + return + + sender = senders.get(channel.channel_type) + if not sender: + log.status = NotificationStatus.FAILED + log.error = f"No sender for channel type '{channel.channel_type}'." + log.save(update_fields=["status", "error"]) + results["failed"] += 1 + return + + try: + sender(channel.config, log.payload) + log.status = NotificationStatus.SENT + log.sent_at = timezone.now() + log.save(update_fields=["status", "sent_at"]) + results["sent"] += 1 + except Exception as exc: + logger.exception( + "notification_send_failed", + extra={ + "log_id": str(log.pk), + "channel": channel.name, + "error": str(exc), + }, + ) + log.status = NotificationStatus.FAILED + log.error = str(exc)[:1000] + log.save(update_fields=["status", "error"]) + results["failed"] += 1 + + +def flush_pending_notifications() -> dict[str, int]: senders = { ChannelType.TELEGRAM: _send_telegram, ChannelType.SMTP: _send_smtp, @@ -350,42 +399,15 @@ def flush_pending_notifications() -> dict[str, int]: results = {"sent": 0, "failed": 0} - for log in pending: - if not log.channel or not log.channel.is_active: - log.status = NotificationStatus.FAILED - log.error = "Channel is inactive or missing." - log.save(update_fields=["status", "error"]) - results["failed"] += 1 - continue - - sender = senders.get(log.channel.channel_type) - if not sender: - log.status = NotificationStatus.FAILED - log.error = ( - f"No sender for channel type '{log.channel.channel_type}'." + while True: + with transaction.atomic(): + log = _pending_notifications_for_flush().first() + if log is None: + break + _flush_notification_log( + log=log, + senders=senders, + results=results, ) - log.save(update_fields=["status", "error"]) - results["failed"] += 1 - continue - - try: - sender(log.channel.config, log.payload) - log.status = NotificationStatus.SENT - log.sent_at = timezone.now() - log.save(update_fields=["status", "sent_at"]) - results["sent"] += 1 - except Exception as exc: - logger.exception( - "notification_send_failed", - extra={ - "log_id": str(log.pk), - "channel": log.channel.name, - "error": str(exc), - }, - ) - log.status = NotificationStatus.FAILED - log.error = str(exc)[:1000] - log.save(update_fields=["status", "error"]) - results["failed"] += 1 return results diff --git a/src/backend/justfile b/src/backend/justfile index 1f70707..72b3e28 100644 --- a/src/backend/justfile +++ b/src/backend/justfile @@ -18,7 +18,7 @@ run: style: just format just lint - just mypy + # just mypy [group('test')] check: