feat(loadtest): added loadtesting with k6

2026-02-24 19:47:49 +03:00
parent ade94d35fd
commit f56a26836d
7 changed files with 618 additions and 3 deletions
@@ -0,0 +1,83 @@
+# k6 Load Testing
+
+Reproducible load test profile for `POST /api/v1/decide`.
+
+## Prerequisites
+
+- Docker + Docker Compose
+- `jq`
+- Running stack (`docker compose -f compose.yaml up -d`)
+
+## One-command run
+
+```bash
+./infrastructure/k6/run-decide.sh
+```
+
+This command:
+
+1. Prepares deterministic fixture via `prepare_k6_fixture`.
+2. Runs `grafana/k6` in a pinned container image.
+3. Saves artifacts to `artifacts/k6/<RUN_ID>/`.
+
+Artifacts:
+
+- `fixture.json`
+- `run.env`
+- `summary.json`
+
+## Reproducible rerun
+
+Use the same `RUN_ID` and k6 profile parameters.
+
+```bash
+RUN_ID=baseline_20260224 \
+START_RPS=20 \
+RAMP_UP_RPS=200 \
+HOLD_RPS=200 \
+HOLD_DURATION=2m \
+./infrastructure/k6/run-decide.sh
+```
+
+## Target URL
+
+Default target for k6 container:
+
+- `K6_BASE_URL=http://host.docker.internal`
+
+Override if needed:
+
+```bash
+K6_BASE_URL=http://host.docker.internal:14609 ./infrastructure/k6/run-decide.sh
+```
+
+## Profile knobs
+
+- `START_RPS`
+- `RAMP_UP_RPS`
+- `HOLD_RPS`
+- `RAMP_UP_DURATION`
+- `HOLD_DURATION`
+- `RAMP_DOWN_DURATION`
+- `PRE_ALLOCATED_VUS`
+- `MAX_VUS`
+- `THRESHOLD_ERROR_RATE`
+- `THRESHOLD_P95_MS`
+- `THRESHOLD_P99_MS`
+- `K6_IMAGE`
+
+## Compare two runs
+
+```bash
+BASE=artifacts/k6/baseline_20260224/summary.json
+CAND=artifacts/k6/candidate_20260224/summary.json
+
+jq -n --argfile b "$BASE" --argfile c "$CAND" '{
+  baseline_p95_ms: $b.metrics.http_req_duration["p(95)"],
+  candidate_p95_ms: $c.metrics.http_req_duration["p(95)"],
+  baseline_req_per_s: $b.metrics.http_reqs.rate,
+  candidate_req_per_s: $c.metrics.http_reqs.rate,
+  baseline_error_rate: $b.metrics.http_req_failed.value,
+  candidate_error_rate: $c.metrics.http_req_failed.value
+}'
+```
@@ -0,0 +1,121 @@
+import http from "k6/http";
+import { check, sleep } from "k6";
+import { Counter, Rate, Trend } from "k6/metrics";
+
+const BASE_URL = (__ENV.BASE_URL || "http://host.docker.internal").replace(
+  /\/$/,
+  "",
+);
+const API_URL = `${BASE_URL}/api/v1`;
+const FLAG_KEY = __ENV.FLAG_KEY || "";
+const SUBJECT_PREFIX = __ENV.SUBJECT_PREFIX || "k6_subject";
+const SUBJECT_COUNTRY = __ENV.SUBJECT_COUNTRY || "US";
+const SUBJECT_POOL = Number(__ENV.SUBJECT_POOL || "20000");
+const THINK_TIME_SECONDS = Number(__ENV.THINK_TIME_SECONDS || "0");
+
+const START_RATE = Number(__ENV.START_RPS || "20");
+const RAMP_UP_RATE = Number(__ENV.RAMP_UP_RPS || "200");
+const HOLD_RATE = Number(__ENV.HOLD_RPS || "200");
+const PRE_ALLOCATED_VUS = Number(__ENV.PRE_ALLOCATED_VUS || "100");
+const MAX_VUS = Number(__ENV.MAX_VUS || "600");
+
+const RAMP_UP_DURATION = __ENV.RAMP_UP_DURATION || "30s";
+const HOLD_DURATION = __ENV.HOLD_DURATION || "2m";
+const RAMP_DOWN_DURATION = __ENV.RAMP_DOWN_DURATION || "20s";
+
+const THRESHOLD_ERROR_RATE = __ENV.THRESHOLD_ERROR_RATE || "0.01";
+const THRESHOLD_P95_MS = __ENV.THRESHOLD_P95_MS || "250";
+const THRESHOLD_P99_MS = __ENV.THRESHOLD_P99_MS || "500";
+
+if (!FLAG_KEY) {
+  throw new Error("FLAG_KEY is required");
+}
+
+export const options = {
+  scenarios: {
+    decide_hot_path: {
+      executor: "ramping-arrival-rate",
+      startRate: START_RATE,
+      timeUnit: "1s",
+      preAllocatedVUs: PRE_ALLOCATED_VUS,
+      maxVUs: MAX_VUS,
+      stages: [
+        { target: RAMP_UP_RATE, duration: RAMP_UP_DURATION },
+        { target: HOLD_RATE, duration: HOLD_DURATION },
+        { target: 0, duration: RAMP_DOWN_DURATION },
+      ],
+    },
+  },
+  thresholds: {
+    http_req_failed: [`rate<${THRESHOLD_ERROR_RATE}`],
+    http_req_duration: [
+      `p(95)<${THRESHOLD_P95_MS}`,
+      `p(99)<${THRESHOLD_P99_MS}`,
+    ],
+    decide_status_200_rate: ["rate>0.99"],
+  },
+  summaryTrendStats: [
+    "avg",
+    "min",
+    "med",
+    "p(90)",
+    "p(95)",
+    "p(99)",
+    "max",
+  ],
+};
+
+const decideStatus200Rate = new Rate("decide_status_200_rate");
+const decideAssignedRate = new Rate("decide_experiment_assigned_rate");
+const decideRequests = new Counter("decide_requests_total");
+const decideDuration = new Trend("decide_request_duration_ms", true);
+
+function buildSubjectId() {
+  const idx = ((__ITER * 104729 + __VU * 8191) % SUBJECT_POOL) + 1;
+  return `${SUBJECT_PREFIX}_${idx}`;
+}
+
+function buildPayload() {
+  return JSON.stringify({
+    subject_id: buildSubjectId(),
+    subject_attributes: { country: SUBJECT_COUNTRY },
+    flags: [FLAG_KEY],
+  });
+}
+
+export default function () {
+  const response = http.post(`${API_URL}/decide`, buildPayload(), {
+    headers: { "Content-Type": "application/json" },
+    tags: { endpoint: "decide" },
+  });
+
+  decideRequests.add(1);
+  decideDuration.add(response.timings.duration);
+  decideStatus200Rate.add(response.status === 200);
+
+  let reason = "";
+  if (response.status === 200) {
+    const body = response.json();
+    if (body && body.decisions && body.decisions.length > 0) {
+      reason = String(body.decisions[0].reason || "");
+    }
+  }
+  decideAssignedRate.add(reason === "experiment_assigned");
+
+  check(response, {
+    "status is 200": (r) => r.status === 200,
+    "has one decision": (r) => {
+      const body = r.json();
+      return (
+        body !== null &&
+        typeof body === "object" &&
+        Array.isArray(body.decisions) &&
+        body.decisions.length === 1
+      );
+    },
+  });
+
+  if (THINK_TIME_SECONDS > 0) {
+    sleep(THINK_TIME_SECONDS);
+  }
+}
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+RUN_ID="${RUN_ID:-$(date -u +%Y%m%d%H%M%S)}"
+K6_IMAGE="${K6_IMAGE:-grafana/k6:0.50.0}"
+K6_BASE_URL="${K6_BASE_URL:-http://host.docker.internal}"
+
+START_RPS="${START_RPS:-20}"
+RAMP_UP_RPS="${RAMP_UP_RPS:-200}"
+HOLD_RPS="${HOLD_RPS:-200}"
+PRE_ALLOCATED_VUS="${PRE_ALLOCATED_VUS:-100}"
+MAX_VUS="${MAX_VUS:-600}"
+RAMP_UP_DURATION="${RAMP_UP_DURATION:-30s}"
+HOLD_DURATION="${HOLD_DURATION:-2m}"
+RAMP_DOWN_DURATION="${RAMP_DOWN_DURATION:-20s}"
+
+THRESHOLD_ERROR_RATE="${THRESHOLD_ERROR_RATE:-0.01}"
+THRESHOLD_P95_MS="${THRESHOLD_P95_MS:-250}"
+THRESHOLD_P99_MS="${THRESHOLD_P99_MS:-500}"
+
+RESULTS_DIR="${RESULTS_DIR:-$ROOT_DIR/artifacts/k6/$RUN_ID}"
+mkdir -p "$RESULTS_DIR"
+
+prepare_fixture() {
+  local output=""
+
+  if (
+    cd "$ROOT_DIR" &&
+      docker compose exec -T backend true >/dev/null 2>&1
+  ); then
+    if output="$(
+      cd "$ROOT_DIR" &&
+        docker compose exec -T backend python manage.py prepare_k6_fixture \
+          --run-id "$RUN_ID" \
+          --json
+    )"; then
+      echo "$output"
+      return
+    fi
+  fi
+
+  output="$(
+    cd "$ROOT_DIR/src/backend"
+    uv run python manage.py prepare_k6_fixture \
+      --run-id "$RUN_ID" \
+      --json
+  )"
+  echo "$output"
+}
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo "jq is required" >&2
+  exit 1
+fi
+
+FIXTURE_JSON="$(prepare_fixture)"
+echo "$FIXTURE_JSON" >"$RESULTS_DIR/fixture.json"
+
+FLAG_KEY="$(echo "$FIXTURE_JSON" | jq -r '.flag_key')"
+SUBJECT_COUNTRY="$(
+  echo "$FIXTURE_JSON" | jq -r '.subject_attributes.country // "US"'
+)"
+
+if [[ -z "$FLAG_KEY" || "$FLAG_KEY" == "null" ]]; then
+  echo "failed to resolve FLAG_KEY from fixture" >&2
+  exit 1
+fi
+
+cat >"$RESULTS_DIR/run.env" <<EOF
+RUN_ID=$RUN_ID
+K6_IMAGE=$K6_IMAGE
+K6_BASE_URL=$K6_BASE_URL
+FLAG_KEY=$FLAG_KEY
+SUBJECT_COUNTRY=$SUBJECT_COUNTRY
+START_RPS=$START_RPS
+RAMP_UP_RPS=$RAMP_UP_RPS
+HOLD_RPS=$HOLD_RPS
+PRE_ALLOCATED_VUS=$PRE_ALLOCATED_VUS
+MAX_VUS=$MAX_VUS
+RAMP_UP_DURATION=$RAMP_UP_DURATION
+HOLD_DURATION=$HOLD_DURATION
+RAMP_DOWN_DURATION=$RAMP_DOWN_DURATION
+THRESHOLD_ERROR_RATE=$THRESHOLD_ERROR_RATE
+THRESHOLD_P95_MS=$THRESHOLD_P95_MS
+THRESHOLD_P99_MS=$THRESHOLD_P99_MS
+EOF
+
+docker run --rm -i \
+  --add-host host.docker.internal:host-gateway \
+  -v "$ROOT_DIR/infrastructure/k6:/k6:ro" \
+  -v "$RESULTS_DIR:/results" \
+  -e BASE_URL="$K6_BASE_URL" \
+  -e FLAG_KEY="$FLAG_KEY" \
+  -e SUBJECT_COUNTRY="$SUBJECT_COUNTRY" \
+  -e START_RPS="$START_RPS" \
+  -e RAMP_UP_RPS="$RAMP_UP_RPS" \
+  -e HOLD_RPS="$HOLD_RPS" \
+  -e PRE_ALLOCATED_VUS="$PRE_ALLOCATED_VUS" \
+  -e MAX_VUS="$MAX_VUS" \
+  -e RAMP_UP_DURATION="$RAMP_UP_DURATION" \
+  -e HOLD_DURATION="$HOLD_DURATION" \
+  -e RAMP_DOWN_DURATION="$RAMP_DOWN_DURATION" \
+  -e THRESHOLD_ERROR_RATE="$THRESHOLD_ERROR_RATE" \
+  -e THRESHOLD_P95_MS="$THRESHOLD_P95_MS" \
+  -e THRESHOLD_P99_MS="$THRESHOLD_P99_MS" \
+  "$K6_IMAGE" run \
+  --summary-export /results/summary.json \
+  /k6/decide.js
+
+echo "results: $RESULTS_DIR"