chore(): final fixes before deadline
- added images of notification feature working - added seed data script - race condition fix for notifications worker - small improvements
This commit is contained in:
Executable
+908
@@ -0,0 +1,908 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "missing required command: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_cmd curl
|
||||
require_cmd jq
|
||||
|
||||
RUN_ID="${RUN_ID:-$(date +%s)}"
|
||||
PFX="seed_${RUN_ID}"
|
||||
BACKEND_BASE_URL="${BACKEND_BASE_URL:-${BASE_URL:-http://127.0.0.1}}"
|
||||
API="${BACKEND_BASE_URL%/}/api/v1"
|
||||
PASSWORD="${PASSWORD:-password123}"
|
||||
ADMIN_USER="${ADMIN_USER:-admin}"
|
||||
EXPERIMENTER_USER="${EXPERIMENTER_USER:-experimenter}"
|
||||
APPROVER_USER="${APPROVER_USER:-approver}"
|
||||
TG_BOT_TOKEN="${TG_BOT_TOKEN:-}"
|
||||
TG_CHAT_ID="${TG_CHAT_ID:-}"
|
||||
SMTP_ENABLED="${SMTP_ENABLED:-false}"
|
||||
SMTP_CHANNEL_NAME="${SMTP_CHANNEL_NAME:-${PFX} smtp provider}"
|
||||
SMTP_RECIPIENT="${SMTP_RECIPIENT:-}"
|
||||
SMTP_FROM_EMAIL="${SMTP_FROM_EMAIL:-lotty@lotty.local}"
|
||||
SMTP_HOST="${SMTP_HOST:-}"
|
||||
SMTP_PORT="${SMTP_PORT:-}"
|
||||
SMTP_USERNAME="${SMTP_USERNAME:-}"
|
||||
SMTP_PASSWORD="${SMTP_PASSWORD:-}"
|
||||
SMTP_USE_TLS="${SMTP_USE_TLS:-false}"
|
||||
SMTP_USE_SSL="${SMTP_USE_SSL:-false}"
|
||||
SMTP_TIMEOUT="${SMTP_TIMEOUT:-10}"
|
||||
RESULTS_DIR="${RESULTS_DIR:-artifacts/http-seed/${RUN_ID}}"
|
||||
|
||||
if [[ -z "$TG_BOT_TOKEN" || -z "$TG_CHAT_ID" ]]; then
|
||||
echo "TG_BOT_TOKEN and TG_CHAT_ID are required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
is_true() {
|
||||
local value
|
||||
value="$(printf "%s" "$1" | tr "[:upper:]" "[:lower:]")"
|
||||
[[ "$value" == "1" || "$value" == "true" || "$value" == "yes" || "$value" == "on" ]]
|
||||
}
|
||||
|
||||
if is_true "$SMTP_ENABLED" && [[ -z "$SMTP_RECIPIENT" ]]; then
|
||||
echo "SMTP_RECIPIENT is required when SMTP_ENABLED=true" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
api_call() {
|
||||
local method="$1"
|
||||
local url="$2"
|
||||
local token="$3"
|
||||
local data="${4:-}"
|
||||
|
||||
local -a headers
|
||||
headers=(-H "Content-Type: application/json")
|
||||
if [[ -n "$token" ]]; then
|
||||
headers+=(-H "Authorization: Bearer $token")
|
||||
fi
|
||||
|
||||
if [[ -n "$data" ]]; then
|
||||
curl -sS --fail-with-body -X "$method" "$url" "${headers[@]}" -d "$data"
|
||||
else
|
||||
curl -sS --fail-with-body -X "$method" "$url" "${headers[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
login() {
|
||||
local username="$1"
|
||||
local payload
|
||||
payload="$(
|
||||
jq -n \
|
||||
--arg username "$username" \
|
||||
--arg password "$PASSWORD" \
|
||||
'{username: $username, password: $password}'
|
||||
)"
|
||||
api_call POST "$API/auth/login" "" "$payload" | jq -r ".access"
|
||||
}
|
||||
|
||||
echo "run_id=$RUN_ID"
|
||||
|
||||
ADMIN_TOKEN="$(login "$ADMIN_USER")"
|
||||
EXPERIMENTER_TOKEN="$(login "$EXPERIMENTER_USER")"
|
||||
APPROVER_TOKEN="$(login "$APPROVER_USER")"
|
||||
|
||||
review_settings_payload="$(
|
||||
jq -n '{default_min_approvals: 1, allow_any_approver: true}'
|
||||
)"
|
||||
api_call \
|
||||
PUT \
|
||||
"$API/reviews/settings" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$review_settings_payload" >/dev/null
|
||||
|
||||
FLAG_COLOR_KEY="${PFX}_button_color"
|
||||
FLAG_COPY_KEY="${PFX}_button_copy"
|
||||
|
||||
flag_color_payload="$(
|
||||
jq -n \
|
||||
--arg key "$FLAG_COLOR_KEY" \
|
||||
--arg name "${PFX} button color" \
|
||||
'{key: $key, name: $name, value_type: "string", default_value: "green"}'
|
||||
)"
|
||||
FLAG_COLOR_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/flags" \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$flag_color_payload"
|
||||
)"
|
||||
FLAG_COLOR_ID="$(echo "$FLAG_COLOR_JSON" | jq -r ".id")"
|
||||
|
||||
flag_copy_payload="$(
|
||||
jq -n \
|
||||
--arg key "$FLAG_COPY_KEY" \
|
||||
--arg name "${PFX} button copy" \
|
||||
'{key: $key, name: $name, value_type: "string", default_value: "buy"}'
|
||||
)"
|
||||
FLAG_COPY_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/flags" \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$flag_copy_payload"
|
||||
)"
|
||||
FLAG_COPY_ID="$(echo "$FLAG_COPY_JSON" | jq -r ".id")"
|
||||
|
||||
create_experiment() {
|
||||
local token="$1"
|
||||
local flag_id="$2"
|
||||
local name="$3"
|
||||
local targeting_rules="$4"
|
||||
local payload
|
||||
payload="$(
|
||||
jq -n \
|
||||
--arg flag_id "$flag_id" \
|
||||
--arg name "$name" \
|
||||
--arg targeting_rules "$targeting_rules" \
|
||||
'{
|
||||
flag_id: $flag_id,
|
||||
name: $name,
|
||||
description: "seeded via HTTP",
|
||||
hypothesis: "seeded hypothesis",
|
||||
traffic_allocation: 100,
|
||||
targeting_rules: $targeting_rules
|
||||
}'
|
||||
)"
|
||||
api_call POST "$API/experiments" "$token" "$payload"
|
||||
}
|
||||
|
||||
create_variant() {
|
||||
local token="$1"
|
||||
local experiment_id="$2"
|
||||
local name="$3"
|
||||
local value="$4"
|
||||
local weight="$5"
|
||||
local is_control="$6"
|
||||
local payload
|
||||
payload="$(
|
||||
jq -n \
|
||||
--arg name "$name" \
|
||||
--arg value "$value" \
|
||||
--argjson weight "$weight" \
|
||||
--argjson is_control "$is_control" \
|
||||
'{name: $name, value: $value, weight: $weight, is_control: $is_control}'
|
||||
)"
|
||||
api_call \
|
||||
POST \
|
||||
"$API/experiments/$experiment_id/variants" \
|
||||
"$token" \
|
||||
"$payload"
|
||||
}
|
||||
|
||||
advance_experiment_to_running() {
|
||||
local experiment_id="$1"
|
||||
local approve_payload
|
||||
approve_payload="$(jq -n '{comment: "seed approve"}')"
|
||||
api_call \
|
||||
POST \
|
||||
"$API/experiments/$experiment_id/submit-for-review" \
|
||||
"$EXPERIMENTER_TOKEN" >/dev/null
|
||||
api_call \
|
||||
POST \
|
||||
"$API/experiments/$experiment_id/approve" \
|
||||
"$APPROVER_TOKEN" \
|
||||
"$approve_payload" >/dev/null
|
||||
api_call \
|
||||
POST \
|
||||
"$API/experiments/$experiment_id/start" \
|
||||
"$EXPERIMENTER_TOKEN" >/dev/null
|
||||
}
|
||||
|
||||
EXPERIMENT_COLOR_JSON="$(
|
||||
create_experiment \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$FLAG_COLOR_ID" \
|
||||
"${PFX} color experiment" \
|
||||
'country == "US"'
|
||||
)"
|
||||
EXPERIMENT_COLOR_ID="$(echo "$EXPERIMENT_COLOR_JSON" | jq -r ".id")"
|
||||
|
||||
CHANNEL_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg name "${PFX} telegram provider" \
|
||||
--arg bot_token "$TG_BOT_TOKEN" \
|
||||
--arg chat_id "$TG_CHAT_ID" \
|
||||
'{
|
||||
channel_type: "telegram",
|
||||
name: $name,
|
||||
config: {
|
||||
bot_token: $bot_token,
|
||||
chat_id: $chat_id
|
||||
}
|
||||
}'
|
||||
)"
|
||||
CHANNEL_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/notification-channels" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$CHANNEL_PAYLOAD"
|
||||
)"
|
||||
CHANNEL_ID="$(echo "$CHANNEL_JSON" | jq -r ".id")"
|
||||
SMTP_CHANNEL_ID=""
|
||||
|
||||
notification_events=(
|
||||
experiment_started
|
||||
experiment_paused
|
||||
experiment_resumed
|
||||
experiment_completed
|
||||
guardrail_triggered
|
||||
review_requested
|
||||
review_approved
|
||||
review_rejected
|
||||
)
|
||||
|
||||
for event_type in "${notification_events[@]}"; do
|
||||
rule_payload="$(
|
||||
jq -n \
|
||||
--arg event_type "$event_type" \
|
||||
--arg channel_id "$CHANNEL_ID" \
|
||||
--arg experiment_id "$EXPERIMENT_COLOR_ID" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
channel_id: $channel_id,
|
||||
experiment_id: $experiment_id,
|
||||
rate_limit_window_seconds: 60,
|
||||
rate_limit_max_notifications: 100
|
||||
}'
|
||||
)"
|
||||
api_call \
|
||||
POST \
|
||||
"$API/notification-rules" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$rule_payload" >/dev/null
|
||||
done
|
||||
|
||||
if is_true "$SMTP_ENABLED"; then
|
||||
SMTP_USE_TLS_JSON="false"
|
||||
SMTP_USE_SSL_JSON="false"
|
||||
if is_true "$SMTP_USE_TLS"; then
|
||||
SMTP_USE_TLS_JSON="true"
|
||||
fi
|
||||
if is_true "$SMTP_USE_SSL"; then
|
||||
SMTP_USE_SSL_JSON="true"
|
||||
fi
|
||||
|
||||
SMTP_PORT_JSON="null"
|
||||
if [[ -n "$SMTP_PORT" ]]; then
|
||||
SMTP_PORT_JSON="$SMTP_PORT"
|
||||
fi
|
||||
|
||||
SMTP_TIMEOUT_JSON="null"
|
||||
if [[ -n "$SMTP_TIMEOUT" ]]; then
|
||||
SMTP_TIMEOUT_JSON="$SMTP_TIMEOUT"
|
||||
fi
|
||||
|
||||
SMTP_CHANNEL_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg name "$SMTP_CHANNEL_NAME" \
|
||||
--arg recipient "$SMTP_RECIPIENT" \
|
||||
--arg from_email "$SMTP_FROM_EMAIL" \
|
||||
--arg host "$SMTP_HOST" \
|
||||
--arg username "$SMTP_USERNAME" \
|
||||
--arg password "$SMTP_PASSWORD" \
|
||||
--argjson port "$SMTP_PORT_JSON" \
|
||||
--argjson timeout "$SMTP_TIMEOUT_JSON" \
|
||||
--argjson use_tls "$SMTP_USE_TLS_JSON" \
|
||||
--argjson use_ssl "$SMTP_USE_SSL_JSON" \
|
||||
'{
|
||||
channel_type: "smtp",
|
||||
name: $name,
|
||||
config: (
|
||||
{
|
||||
recipient: $recipient,
|
||||
from_email: $from_email,
|
||||
use_tls: $use_tls,
|
||||
use_ssl: $use_ssl
|
||||
}
|
||||
+ (if $host != "" then {host: $host} else {} end)
|
||||
+ (if $port != null then {port: $port} else {} end)
|
||||
+ (if $username != "" then {username: $username} else {} end)
|
||||
+ (if $password != "" then {password: $password} else {} end)
|
||||
+ (if $timeout != null then {timeout: $timeout} else {} end)
|
||||
)
|
||||
}'
|
||||
)"
|
||||
SMTP_CHANNEL_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/notification-channels" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$SMTP_CHANNEL_PAYLOAD"
|
||||
)"
|
||||
SMTP_CHANNEL_ID="$(echo "$SMTP_CHANNEL_JSON" | jq -r ".id")"
|
||||
|
||||
for event_type in "${notification_events[@]}"; do
|
||||
smtp_rule_payload="$(
|
||||
jq -n \
|
||||
--arg event_type "$event_type" \
|
||||
--arg channel_id "$SMTP_CHANNEL_ID" \
|
||||
--arg experiment_id "$EXPERIMENT_COLOR_ID" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
channel_id: $channel_id,
|
||||
experiment_id: $experiment_id,
|
||||
rate_limit_window_seconds: 60,
|
||||
rate_limit_max_notifications: 100
|
||||
}'
|
||||
)"
|
||||
api_call \
|
||||
POST \
|
||||
"$API/notification-rules" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$smtp_rule_payload" >/dev/null
|
||||
done
|
||||
fi
|
||||
|
||||
VARIANT_COLOR_CONTROL_JSON="$(
|
||||
create_variant \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$EXPERIMENT_COLOR_ID" \
|
||||
"control" \
|
||||
"blue" \
|
||||
50 \
|
||||
true
|
||||
)"
|
||||
VARIANT_COLOR_CONTROL_ID="$(
|
||||
echo "$VARIANT_COLOR_CONTROL_JSON" | jq -r ".id"
|
||||
)"
|
||||
|
||||
VARIANT_COLOR_TREATMENT_JSON="$(
|
||||
create_variant \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$EXPERIMENT_COLOR_ID" \
|
||||
"treatment" \
|
||||
"red" \
|
||||
50 \
|
||||
false
|
||||
)"
|
||||
VARIANT_COLOR_TREATMENT_ID="$(
|
||||
echo "$VARIANT_COLOR_TREATMENT_JSON" | jq -r ".id"
|
||||
)"
|
||||
|
||||
EXPERIMENT_COPY_JSON="$(
|
||||
create_experiment \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$FLAG_COPY_ID" \
|
||||
"${PFX} copy experiment" \
|
||||
'country == "US"'
|
||||
)"
|
||||
EXPERIMENT_COPY_ID="$(echo "$EXPERIMENT_COPY_JSON" | jq -r ".id")"
|
||||
|
||||
create_variant \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$EXPERIMENT_COPY_ID" \
|
||||
"control" \
|
||||
"buy" \
|
||||
50 \
|
||||
true >/dev/null
|
||||
create_variant \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$EXPERIMENT_COPY_ID" \
|
||||
"treatment" \
|
||||
"checkout" \
|
||||
50 \
|
||||
false >/dev/null
|
||||
|
||||
advance_experiment_to_running "$EXPERIMENT_COLOR_ID"
|
||||
advance_experiment_to_running "$EXPERIMENT_COPY_ID"
|
||||
|
||||
CONFLICT_DOMAIN_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg name "${PFX}_checkout_domain" \
|
||||
'{
|
||||
name: $name,
|
||||
description: "seeded domain conflict",
|
||||
policy: "priority",
|
||||
max_concurrent: 2
|
||||
}'
|
||||
)"
|
||||
CONFLICT_DOMAIN_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/conflicts/domains" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$CONFLICT_DOMAIN_PAYLOAD"
|
||||
)"
|
||||
CONFLICT_DOMAIN_ID="$(echo "$CONFLICT_DOMAIN_JSON" | jq -r ".id")"
|
||||
|
||||
add_to_domain() {
|
||||
local experiment_id="$1"
|
||||
local priority="$2"
|
||||
local payload
|
||||
payload="$(
|
||||
jq -n \
|
||||
--arg experiment_id "$experiment_id" \
|
||||
--argjson priority "$priority" \
|
||||
'{experiment_id: $experiment_id, priority: $priority}'
|
||||
)"
|
||||
api_call \
|
||||
POST \
|
||||
"$API/conflicts/domains/$CONFLICT_DOMAIN_ID/experiments" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$payload" >/dev/null
|
||||
}
|
||||
|
||||
add_to_domain "$EXPERIMENT_COLOR_ID" 100
|
||||
add_to_domain "$EXPERIMENT_COPY_ID" 10
|
||||
|
||||
ET_EXPOSURE_NAME="${PFX}_exposure"
|
||||
ET_CLICK_NAME="${PFX}_click"
|
||||
|
||||
create_event_type() {
|
||||
local name="$1"
|
||||
local display_name="$2"
|
||||
local is_exposure="$3"
|
||||
local requires_exposure="$4"
|
||||
local payload
|
||||
payload="$(
|
||||
jq -n \
|
||||
--arg name "$name" \
|
||||
--arg display_name "$display_name" \
|
||||
--argjson is_exposure "$is_exposure" \
|
||||
--argjson requires_exposure "$requires_exposure" \
|
||||
'{
|
||||
name: $name,
|
||||
display_name: $display_name,
|
||||
description: "seeded event type",
|
||||
is_exposure: $is_exposure,
|
||||
requires_exposure: $requires_exposure,
|
||||
required_fields: []
|
||||
}'
|
||||
)"
|
||||
api_call POST "$API/events/event-types" "$ADMIN_TOKEN" "$payload"
|
||||
}
|
||||
|
||||
ET_EXPOSURE_JSON="$(
|
||||
create_event_type "$ET_EXPOSURE_NAME" "${PFX} Exposure" true false
|
||||
)"
|
||||
ET_CLICK_JSON="$(
|
||||
create_event_type "$ET_CLICK_NAME" "${PFX} Click" false true
|
||||
)"
|
||||
ET_EXPOSURE_ID="$(echo "$ET_EXPOSURE_JSON" | jq -r ".id")"
|
||||
ET_CLICK_ID="$(echo "$ET_CLICK_JSON" | jq -r ".id")"
|
||||
|
||||
METRIC_CTR_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg key "${PFX}_ctr" \
|
||||
--arg name "${PFX} CTR" \
|
||||
--arg numerator_event "$ET_CLICK_NAME" \
|
||||
--arg denominator_event "$ET_EXPOSURE_NAME" \
|
||||
'{
|
||||
key: $key,
|
||||
name: $name,
|
||||
description: "seeded ctr metric",
|
||||
metric_type: "ratio",
|
||||
direction: "higher_is_better",
|
||||
calculation_rule: {
|
||||
numerator_event: $numerator_event,
|
||||
denominator_event: $denominator_event
|
||||
}
|
||||
}'
|
||||
)"
|
||||
METRIC_CTR_JSON="$(
|
||||
api_call POST "$API/metrics" "$ADMIN_TOKEN" "$METRIC_CTR_PAYLOAD"
|
||||
)"
|
||||
METRIC_CTR_ID="$(echo "$METRIC_CTR_JSON" | jq -r ".id")"
|
||||
|
||||
METRIC_EXPOSURE_COUNT_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg key "${PFX}_exposure_count" \
|
||||
--arg name "${PFX} Exposure Count" \
|
||||
--arg event "$ET_EXPOSURE_NAME" \
|
||||
'{
|
||||
key: $key,
|
||||
name: $name,
|
||||
description: "seeded exposure count metric",
|
||||
metric_type: "count",
|
||||
direction: "higher_is_better",
|
||||
calculation_rule: {event: $event}
|
||||
}'
|
||||
)"
|
||||
METRIC_EXPOSURE_COUNT_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/metrics" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$METRIC_EXPOSURE_COUNT_PAYLOAD"
|
||||
)"
|
||||
METRIC_EXPOSURE_COUNT_ID="$(
|
||||
echo "$METRIC_EXPOSURE_COUNT_JSON" | jq -r ".id"
|
||||
)"
|
||||
|
||||
METRIC_CLICK_COUNT_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg key "${PFX}_click_count" \
|
||||
--arg name "${PFX} Click Count" \
|
||||
--arg event "$ET_CLICK_NAME" \
|
||||
'{
|
||||
key: $key,
|
||||
name: $name,
|
||||
description: "seeded click count metric",
|
||||
metric_type: "count",
|
||||
direction: "higher_is_better",
|
||||
calculation_rule: {event: $event}
|
||||
}'
|
||||
)"
|
||||
METRIC_CLICK_COUNT_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/metrics" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$METRIC_CLICK_COUNT_PAYLOAD"
|
||||
)"
|
||||
METRIC_CLICK_COUNT_ID="$(
|
||||
echo "$METRIC_CLICK_COUNT_JSON" | jq -r ".id"
|
||||
)"
|
||||
|
||||
attach_metric() {
|
||||
local metric_id="$1"
|
||||
local is_primary="$2"
|
||||
local payload
|
||||
payload="$(
|
||||
jq -n \
|
||||
--arg metric_id "$metric_id" \
|
||||
--argjson is_primary "$is_primary" \
|
||||
'{metric_id: $metric_id, is_primary: $is_primary}'
|
||||
)"
|
||||
api_call \
|
||||
POST \
|
||||
"$API/experiments/$EXPERIMENT_COLOR_ID/metrics" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$payload" >/dev/null
|
||||
}
|
||||
|
||||
attach_metric "$METRIC_CTR_ID" true
|
||||
attach_metric "$METRIC_EXPOSURE_COUNT_ID" false
|
||||
attach_metric "$METRIC_CLICK_COUNT_ID" false
|
||||
|
||||
GUARDRAIL_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg metric_id "$METRIC_CTR_ID" \
|
||||
'{
|
||||
metric_id: $metric_id,
|
||||
threshold: 0,
|
||||
observation_window_minutes: 60,
|
||||
action: "pause"
|
||||
}'
|
||||
)"
|
||||
GUARDRAIL_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/experiments/$EXPERIMENT_COLOR_ID/guardrails" \
|
||||
"$ADMIN_TOKEN" \
|
||||
"$GUARDRAIL_PAYLOAD"
|
||||
)"
|
||||
GUARDRAIL_ID="$(echo "$GUARDRAIL_JSON" | jq -r ".id")"
|
||||
|
||||
LEARNING_COLOR_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg experiment_id "$EXPERIMENT_COLOR_ID" \
|
||||
--arg hypothesis "Blue vs red purchase button on checkout" \
|
||||
--arg findings "Seed dataset for report/guardrails/conflicts demo" \
|
||||
'{
|
||||
experiment_id: $experiment_id,
|
||||
hypothesis: $hypothesis,
|
||||
findings: $findings,
|
||||
tags: ["checkout", "button", "color", "seed"],
|
||||
context_summary: "Seeded via HTTP for live demo"
|
||||
}'
|
||||
)"
|
||||
LEARNING_COLOR_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/learnings" \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$LEARNING_COLOR_PAYLOAD"
|
||||
)"
|
||||
LEARNING_COLOR_ID="$(echo "$LEARNING_COLOR_JSON" | jq -r ".id")"
|
||||
|
||||
LEARNING_COPY_PAYLOAD="$(
|
||||
jq -n \
|
||||
--arg experiment_id "$EXPERIMENT_COPY_ID" \
|
||||
--arg hypothesis "Checkout button wording with color context" \
|
||||
--arg findings "Variant copy affects click intent on checkout CTA" \
|
||||
'{
|
||||
experiment_id: $experiment_id,
|
||||
hypothesis: $hypothesis,
|
||||
findings: $findings,
|
||||
tags: ["checkout", "button", "copy", "seed"],
|
||||
context_summary: "Seeded pair for similar learnings endpoint"
|
||||
}'
|
||||
)"
|
||||
LEARNING_COPY_JSON="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/learnings" \
|
||||
"$EXPERIMENTER_TOKEN" \
|
||||
"$LEARNING_COPY_PAYLOAD"
|
||||
)"
|
||||
LEARNING_COPY_ID="$(echo "$LEARNING_COPY_JSON" | jq -r ".id")"
|
||||
|
||||
subjects=(u01 u02 u03 u04 u05 u06 u07 u08 u09 u10 u11 u12)
|
||||
decisions_file="$(mktemp)"
|
||||
|
||||
for suffix in "${subjects[@]}"; do
|
||||
subject_id="${PFX}_${suffix}"
|
||||
decide_payload="$(
|
||||
jq -n \
|
||||
--arg subject_id "$subject_id" \
|
||||
--arg flag "$FLAG_COLOR_KEY" \
|
||||
'{
|
||||
subject_id: $subject_id,
|
||||
subject_attributes: {country: "US"},
|
||||
flags: [$flag]
|
||||
}'
|
||||
)"
|
||||
decide_json="$(
|
||||
api_call POST "$API/decide" "" "$decide_payload"
|
||||
)"
|
||||
reason="$(echo "$decide_json" | jq -r ".decisions[0].reason")"
|
||||
if [[ "$reason" != "experiment_assigned" ]]; then
|
||||
echo "unexpected decide reason for $subject_id: $reason" >&2
|
||||
exit 1
|
||||
fi
|
||||
decision_id="$(echo "$decide_json" | jq -r ".decisions[0].decision_id")"
|
||||
printf "%s\t%s\n" "$subject_id" "$decision_id" >> "$decisions_file"
|
||||
done
|
||||
|
||||
now_ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
events_pre_file="$(mktemp)"
|
||||
events_main_file="$(mktemp)"
|
||||
trap 'rm -f "$events_pre_file" "$events_main_file" "$decisions_file"' EXIT
|
||||
|
||||
decision_for_subject() {
|
||||
local subject_id="$1"
|
||||
awk -F $'\t' -v subject_id="$subject_id" '
|
||||
$1 == subject_id { print $2; found = 1; exit }
|
||||
END { if (!found) exit 1 }
|
||||
' "$decisions_file"
|
||||
}
|
||||
|
||||
append_event() {
|
||||
local file="$1"
|
||||
local event_id="$2"
|
||||
local event_type="$3"
|
||||
local decision_id="$4"
|
||||
local subject_id="$5"
|
||||
local timestamp="$6"
|
||||
jq -n \
|
||||
--arg event_id "$event_id" \
|
||||
--arg event_type "$event_type" \
|
||||
--arg decision_id "$decision_id" \
|
||||
--arg subject_id "$subject_id" \
|
||||
--arg timestamp "$timestamp" \
|
||||
'{
|
||||
event_id: $event_id,
|
||||
event_type: $event_type,
|
||||
decision_id: $decision_id,
|
||||
subject_id: $subject_id,
|
||||
timestamp: $timestamp,
|
||||
properties: {}
|
||||
}' >> "$file"
|
||||
}
|
||||
|
||||
click_only_subjects=("${PFX}_u09" "${PFX}_u10")
|
||||
out_of_order_subject="${PFX}_u11"
|
||||
|
||||
counter=1
|
||||
for subject_id in "${click_only_subjects[@]}" "$out_of_order_subject"; do
|
||||
append_event \
|
||||
"$events_pre_file" \
|
||||
"${PFX}_click_pre_${counter}" \
|
||||
"$ET_CLICK_NAME" \
|
||||
"$(decision_for_subject "$subject_id")" \
|
||||
"$subject_id" \
|
||||
"$now_ts"
|
||||
counter=$((counter + 1))
|
||||
done
|
||||
|
||||
PRE_EVENTS_PAYLOAD="$(jq -s '{events: .}' "$events_pre_file")"
|
||||
PRE_EVENTS_RESULT="$(
|
||||
api_call POST "$API/events" "" "$PRE_EVENTS_PAYLOAD"
|
||||
)"
|
||||
|
||||
shown_click_subjects=("${PFX}_u01" "${PFX}_u03" "${PFX}_u05")
|
||||
shown_only_subjects=(
|
||||
"${PFX}_u02"
|
||||
"${PFX}_u04"
|
||||
"${PFX}_u06"
|
||||
"${PFX}_u07"
|
||||
"${PFX}_u08"
|
||||
)
|
||||
shown_subjects=(
|
||||
"${shown_click_subjects[@]}"
|
||||
"${shown_only_subjects[@]}"
|
||||
"$out_of_order_subject"
|
||||
)
|
||||
|
||||
counter=1
|
||||
for subject_id in "${shown_subjects[@]}"; do
|
||||
append_event \
|
||||
"$events_main_file" \
|
||||
"${PFX}_exp_${counter}" \
|
||||
"$ET_EXPOSURE_NAME" \
|
||||
"$(decision_for_subject "$subject_id")" \
|
||||
"$subject_id" \
|
||||
"$now_ts"
|
||||
counter=$((counter + 1))
|
||||
done
|
||||
|
||||
counter=1
|
||||
for subject_id in "${shown_click_subjects[@]}"; do
|
||||
append_event \
|
||||
"$events_main_file" \
|
||||
"${PFX}_click_${counter}" \
|
||||
"$ET_CLICK_NAME" \
|
||||
"$(decision_for_subject "$subject_id")" \
|
||||
"$subject_id" \
|
||||
"$now_ts"
|
||||
counter=$((counter + 1))
|
||||
done
|
||||
|
||||
MAIN_EVENTS_PAYLOAD="$(jq -s '{events: .}' "$events_main_file")"
|
||||
MAIN_EVENTS_RESULT="$(
|
||||
api_call POST "$API/events" "" "$MAIN_EVENTS_PAYLOAD"
|
||||
)"
|
||||
|
||||
conflict_decide_payload="$(
|
||||
jq -n \
|
||||
--arg subject_id "${PFX}_u01" \
|
||||
--arg flag "$FLAG_COPY_KEY" \
|
||||
'{
|
||||
subject_id: $subject_id,
|
||||
subject_attributes: {country: "US"},
|
||||
flags: [$flag]
|
||||
}'
|
||||
)"
|
||||
CONFLICT_DECIDE_RESULT="$(
|
||||
api_call POST "$API/decide" "" "$conflict_decide_payload"
|
||||
)"
|
||||
CONFLICT_DECIDE_REASON="$(
|
||||
echo "$CONFLICT_DECIDE_RESULT" | jq -r ".decisions[0].reason"
|
||||
)"
|
||||
if [[ "$CONFLICT_DECIDE_REASON" != "domain_conflict" ]]; then
|
||||
echo "unexpected conflict decide reason: $CONFLICT_DECIDE_REASON" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GUARDRAIL_CHECK_RESULT="$(
|
||||
api_call \
|
||||
POST \
|
||||
"$API/experiments/$EXPERIMENT_COLOR_ID/guardrails/check" \
|
||||
"$ADMIN_TOKEN"
|
||||
)"
|
||||
GUARDRAIL_TRIGGERED="$(
|
||||
echo "$GUARDRAIL_CHECK_RESULT" | jq -r ".triggered"
|
||||
)"
|
||||
if [[ "$GUARDRAIL_TRIGGERED" != "0" ]]; then
|
||||
echo "guardrail triggered immediately: $GUARDRAIL_TRIGGERED" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPORT_JSON="$(
|
||||
api_call GET "$API/reports/$EXPERIMENT_COLOR_ID" "$ADMIN_TOKEN"
|
||||
)"
|
||||
|
||||
SIMILAR_LEARNINGS_JSON="$(
|
||||
api_call \
|
||||
GET \
|
||||
"$API/experiments/$EXPERIMENT_COLOR_ID/similar-learnings?limit=5" \
|
||||
"$ADMIN_TOKEN"
|
||||
)"
|
||||
SIMILAR_COUNT="$(echo "$SIMILAR_LEARNINGS_JSON" | jq 'length')"
|
||||
if [[ "$SIMILAR_COUNT" -lt "1" ]]; then
|
||||
echo "similar learnings result is empty for experiment: $EXPERIMENT_COLOR_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$PRE_EVENTS_RESULT" > "$RESULTS_DIR/pre_events_result.json"
|
||||
echo "$MAIN_EVENTS_RESULT" > "$RESULTS_DIR/main_events_result.json"
|
||||
echo "$CONFLICT_DECIDE_RESULT" > "$RESULTS_DIR/conflict_decide_result.json"
|
||||
echo "$GUARDRAIL_CHECK_RESULT" > "$RESULTS_DIR/guardrail_check_result.json"
|
||||
echo "$REPORT_JSON" > "$RESULTS_DIR/report.json"
|
||||
echo "$SIMILAR_LEARNINGS_JSON" > "$RESULTS_DIR/similar_learnings.json"
|
||||
|
||||
jq -n \
|
||||
--arg run_id "$RUN_ID" \
|
||||
--arg prefix "$PFX" \
|
||||
--arg flag_color_key "$FLAG_COLOR_KEY" \
|
||||
--arg flag_copy_key "$FLAG_COPY_KEY" \
|
||||
--arg experiment_color_id "$EXPERIMENT_COLOR_ID" \
|
||||
--arg experiment_copy_id "$EXPERIMENT_COPY_ID" \
|
||||
--arg metric_ctr_id "$METRIC_CTR_ID" \
|
||||
--arg metric_exposure_count_id "$METRIC_EXPOSURE_COUNT_ID" \
|
||||
--arg metric_click_count_id "$METRIC_CLICK_COUNT_ID" \
|
||||
--arg event_type_exposure "$ET_EXPOSURE_NAME" \
|
||||
--arg event_type_click "$ET_CLICK_NAME" \
|
||||
--arg telegram_channel_id "$CHANNEL_ID" \
|
||||
--arg smtp_channel_id "$SMTP_CHANNEL_ID" \
|
||||
--argjson smtp_enabled "$(is_true "$SMTP_ENABLED" && echo true || echo false)" \
|
||||
--arg conflict_domain_id "$CONFLICT_DOMAIN_ID" \
|
||||
--arg learning_color_id "$LEARNING_COLOR_ID" \
|
||||
--arg learning_copy_id "$LEARNING_COPY_ID" \
|
||||
--arg guardrail_id "$GUARDRAIL_ID" \
|
||||
--argjson pre_events_result "$PRE_EVENTS_RESULT" \
|
||||
--argjson main_events_result "$MAIN_EVENTS_RESULT" \
|
||||
--argjson guardrail_check "$GUARDRAIL_CHECK_RESULT" \
|
||||
--argjson similar_learnings "$SIMILAR_LEARNINGS_JSON" \
|
||||
--argjson report "$REPORT_JSON" \
|
||||
'{
|
||||
run_id: $run_id,
|
||||
prefix: $prefix,
|
||||
flags: {
|
||||
color_flag_key: $flag_color_key,
|
||||
copy_flag_key: $flag_copy_key
|
||||
},
|
||||
experiments: {
|
||||
color_experiment_id: $experiment_color_id,
|
||||
copy_experiment_id: $experiment_copy_id
|
||||
},
|
||||
event_types: {
|
||||
exposure: $event_type_exposure,
|
||||
click: $event_type_click
|
||||
},
|
||||
metrics: {
|
||||
ctr_id: $metric_ctr_id,
|
||||
exposure_count_id: $metric_exposure_count_id,
|
||||
click_count_id: $metric_click_count_id
|
||||
},
|
||||
notification: {
|
||||
telegram_channel_id: $telegram_channel_id,
|
||||
smtp_enabled: $smtp_enabled,
|
||||
smtp_channel_id: ($smtp_channel_id | if . == "" then null else . end)
|
||||
},
|
||||
conflicts: {
|
||||
domain_id: $conflict_domain_id
|
||||
},
|
||||
learnings: {
|
||||
color_learning_id: $learning_color_id,
|
||||
copy_learning_id: $learning_copy_id,
|
||||
similar_for_color_experiment: $similar_learnings
|
||||
},
|
||||
guardrail: {
|
||||
guardrail_id: $guardrail_id,
|
||||
check: $guardrail_check
|
||||
},
|
||||
events_ingestion: {
|
||||
pre_events: $pre_events_result,
|
||||
main_events: $main_events_result
|
||||
},
|
||||
report: $report
|
||||
}' > "$RESULTS_DIR/summary.json"
|
||||
|
||||
echo "seed complete"
|
||||
echo "backend_base_url=$BACKEND_BASE_URL"
|
||||
echo "results_dir=$RESULTS_DIR"
|
||||
echo "summary_file=$RESULTS_DIR/summary.json"
|
||||
echo "flag_color_id=$FLAG_COLOR_ID"
|
||||
echo "flag_copy_id=$FLAG_COPY_ID"
|
||||
echo "experiment_color_id=$EXPERIMENT_COLOR_ID"
|
||||
echo "experiment_copy_id=$EXPERIMENT_COPY_ID"
|
||||
echo "variant_color_control_id=$VARIANT_COLOR_CONTROL_ID"
|
||||
echo "variant_color_treatment_id=$VARIANT_COLOR_TREATMENT_ID"
|
||||
echo "event_type_exposure_id=$ET_EXPOSURE_ID"
|
||||
echo "event_type_click_id=$ET_CLICK_ID"
|
||||
echo "metric_ctr_id=$METRIC_CTR_ID"
|
||||
echo "metric_exposure_count_id=$METRIC_EXPOSURE_COUNT_ID"
|
||||
echo "metric_click_count_id=$METRIC_CLICK_COUNT_ID"
|
||||
echo "telegram_notification_channel_id=$CHANNEL_ID"
|
||||
echo "smtp_enabled=$(is_true "$SMTP_ENABLED" && echo true || echo false)"
|
||||
if [[ -n "$SMTP_CHANNEL_ID" ]]; then
|
||||
echo "smtp_notification_channel_id=$SMTP_CHANNEL_ID"
|
||||
fi
|
||||
echo "conflict_domain_id=$CONFLICT_DOMAIN_ID"
|
||||
echo "learning_color_id=$LEARNING_COLOR_ID"
|
||||
echo "learning_copy_id=$LEARNING_COPY_ID"
|
||||
echo "similar_learnings_count=$SIMILAR_COUNT"
|
||||
echo "guardrail_id=$GUARDRAIL_ID"
|
||||
Reference in New Issue
Block a user