feat: added zipkin and loadtest services to compose.yaml

compose.yaml

@@ -366,6 +366,53 @@ services:
         source: minio_data
         target: /data
 
+  zipkin:
+    image: docker.io/openzipkin/zipkin:3
+    healthcheck:
+      test: ["CMD", "wget", "-O", "-", "http://localhost:9411/health"]
+      interval: 1m30s
+      timeout: 5s
+      start_period: 5s
+      start_interval: 2s
+      retries: 5
+    ports:
+      - name: web
+        target: 9411
+        published: 13247
+        host_ip: 127.0.0.1
+        protocol: tcp
+        app_protocol: http
+      - name: api
+        target: 9411
+        published: 13247
+        host_ip: 127.0.0.1
+        protocol: tcp
+        app_protocol: http
+    restart: unless-stopped
+
+  loadtest:
+    build:
+      context: ./services/loadtest
+      dockerfile: Dockerfile
+    depends_on:
+      backend:
+        restart: false
+        condition: service_healthy
+        required: true
+    env_file:
+      - path: ./infrastructure/loadtest/.env.template
+        required: true
+      - path: ./infrastructure/loadtest/.env
+        required: false
+    ports:
+      - name: web
+        target: 5001
+        published: 13248
+        host_ip: 127.0.0.1
+        protocol: tcp
+        app_protocol: http
+    restart: unless-stopped
+
 volumes:
   redis_data:
   postgres_data:

infrastructure/backend/.env.template

@@ -1,4 +1,4 @@
-DJANGO_SECRET_KEY=secretees
+DJANGO_SECRET_KEY=very_insecure_key
 DJANGO_DEBUG=False
 DJANGO_ALLOWED_HOSTS=*
 DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost,http://127.0.0.1
@@ -21,3 +21,7 @@ MINIO_ENDPOINT=minio:9000
 MINIO_CUSTOM_ENDPOINT_URL=http://127.0.0.1:13244
 MINIO_ACCESS_KEY=admin
 MINIO_SECRET_KEY=password
+
+OTEL_METRICS_EXPORTER=none
+OTEL_EXPORTER_ZIPKIN_ENDPOINT=http://zipkin:9411/api/v2/spans
+OTEL_TRACES_EXPORTER=zipkin_json

services/backend/.gitignore

@@ -171,3 +171,6 @@ cython_debug/
 
 # Collected static files
 static
+
+# Profile files
+*.prof

services/backend/Dockerfile

@@ -32,11 +32,12 @@ USER app
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONOPTIMIZE=2 \
-    PATH="/opt/venv/bin:$PATH"
+    PATH="/opt/venv/bin:$PATH" \
+    DJANGO_SETTINGS_MODULE=config.settings
 
 EXPOSE 8080
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --start-interval=2s --retries=3 \
     CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:8080/health?format=json || exit 1
 
-CMD [ "gunicorn", "config.wsgi", "--workers=8", "-b", "0.0.0.0:8080", "--access-logfile", "-", "--error-logfile", "-" ]
+CMD [ "opentelemetry-instrument", "--service_name", "backend-django", "--traces_exporter", "zipkin_json", "gunicorn", "config.wsgi", "--workers=8", "-b", "0.0.0.0:8080", "--access-logfile", "-", "--error-logfile", "-" ]

services/backend/api/v1/ads/views.py

@@ -4,6 +4,7 @@ from uuid import UUID
 from django.http import Http404, HttpRequest
 from django.shortcuts import get_object_or_404
 from ninja import Router
+from silk.profiling.profiler import silk_profile
 
 from api.v1 import schemas as global_schemas
 from api.v1.ads import schemas
@@ -21,6 +22,7 @@ router = Router(tags=["ads"])
         status.NOT_FOUND: global_schemas.NotFoundError,
     },
 )
+@silk_profile("Get Advertisment")
 def get_advertisment(
     request: HttpRequest, client_id: UUID
 ) -> tuple[status, Campaign]:

services/backend/config/settings.py

@@ -4,6 +4,7 @@ import contextlib
 import logging
 from collections.abc import Callable
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import django_stubs_ext
 import environ
@@ -12,6 +13,10 @@ from health_check.plugins import plugin_dir
 
 from integrations.yandexai.healthcheck import YandexAIHealthCheck
 
+if TYPE_CHECKING:
+    from django.contrib.auth.models import User
+
+
 BASE_DIR = Path(__file__).resolve().parent.parent
 
 env = environ.Env()
@@ -53,7 +58,7 @@ REDIS_URI = env("REDIS_URI", default="redis://localhost:6379")
 
 CACHES = {
     "default": {
-        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "BACKEND": "django_prometheus.cache.backends.redis.RedisCache",
         "LOCATION": REDIS_URI,
         "TIMEOUT": None,
         "KEY_PREFIX": "backend",
@@ -80,6 +85,9 @@ CELERY_TASK_TRACK_STARTED = True
 # Database
 
 DB_URI = env.db_url("DJANGO_DB_URI", default="sqlite:///db.sqlite3")
+DB_URI["ENGINE"] = DB_URI["ENGINE"].replace(
+    "django.db.backends", "django_prometheus.db.backends"
+)
 
 DATABASES = {"default": {**DB_URI, "CONN_MAX_AGE": 50}}
 
@@ -284,12 +292,15 @@ INTERNAL_IPS = env(
 )
 
 MIDDLEWARE = [
+    "silk.middleware.SilkyMiddleware",
+    "django_prometheus.middleware.PrometheusBeforeMiddleware",
     "django_guid.middleware.guid_middleware",
     "corsheaders.middleware.CorsMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
+    "django_prometheus.middleware.PrometheusAfterMiddleware",
 ]
 
 SIGNING_BACKEND = "django.core.signing.TimestampSigner"
@@ -442,8 +453,10 @@ INSTALLED_APPS = [
     "corsheaders",
     "django_extensions",
     "django_guid",
+    "django_prometheus",
     "ninja",
     "minio_storage",
+    "silk",
     # Internal apps
     "apps.core",
     "apps.advertiser",
@@ -516,6 +529,13 @@ SECRET_KEY = env("DJANGO_SECRET_KEY", default="very_insecure_key")
 SECRET_KEY_FALLBACKS: list[str] = []
 
 
+# Auth
+
+LOGIN_REDIRECT_URL = "/admin/"
+
+LOGIN_URL = "/admin/"
+
+
 # Sessions
 
 SESSION_CACHE_ALIAS = "default"
@@ -594,3 +614,63 @@ DEBUG_TOOLBAR_CONFIG = {"SHOW_COLLAPSED": True, "UPDATE_ON_FETCH": True}
 if DEBUG and DEBUG_TOOLBAR_ENABLED:
     INSTALLED_APPS.append("debug_toolbar")
     MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
+
+
+# Prometheus
+
+PROMETHEUS_LATENCY_BUCKETS = (
+    0.005,
+    0.01,
+    0.025,
+    0.05,
+    0.075,
+    0.1,
+    0.25,
+    0.5,
+    0.75,
+    1.0,
+    2.5,
+    5.0,
+    7.5,
+    10.0,
+    25.0,
+    50.0,
+    75.0,
+    float("inf"),
+)
+
+
+# django-silk
+
+SILKY_PYTHON_PROFILER = True
+
+SILKY_PYTHON_PROFILER_BINARY = True
+
+SILKY_PYTHON_PROFILER_RESULT_PATH = "./profiles"
+
+SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True
+
+SILKY_AUTHENTICATION = True
+
+SILKY_AUTHORISATION = True
+
+
+def is_allowed_to_use_profiling(user: "User") -> bool:
+    return user.is_staff
+
+
+SILKY_PERMISSIONS = is_allowed_to_use_profiling
+
+SILKY_MAX_RECORDED_REQUESTS = 10**3
+
+SILKY_MAX_RECORDED_REQUESTS_CHECK_PERCENT = 10
+
+SILKY_MAX_REQUEST_BODY_SIZE = 128
+
+SILKY_INTERCEPT_PERCENT = 25
+
+SILKY_META = True
+
+SILKY_DYNAMIC_PROFILING = [
+    {"module": "api.v1.ads.views", "function": "get_advertisment"}
+]

services/backend/config/urls.py

@@ -16,6 +16,10 @@ urlpatterns = [
     path("admin/", admin.site.urls),
     # API urls
     path("", include("api.urls")),
+    # Prometheus urls
+    path("", include("django_prometheus.urls")),
+    # Django-silk
+    path("silk/", include("silk.urls", namespace="silk")),
 ]
 
 

services/backend/pyproject.toml

@@ -9,9 +9,29 @@ dependencies = [
  "django-health-check>=3.18.3,<4.0.0",
  "django-minio-storage>=0.5.7,<0.6.0",
  "django-ninja>=1.3.0,<2.0.0",
+ "django-prometheus>=2.4.1,<3.0.0",
+ "django-redis>=6.0.0,<7.0.0",
+ "django-silk[formatting]>=5.4.0,<6.0.0",
  "django-stubs-ext>=5.1.3,<6.0.0",
  "gunicorn>=23.0.0,<24.0.0",
  "httpx>=0.28.1,<0.29.0",
+ "opentelemetry-api>=1.35.0",
+ "opentelemetry-distro>=0.56b0",
+ "opentelemetry-exporter-otlp>=1.35.0",
+ "opentelemetry-exporter-zipkin-proto-http>=1.11.1",
+ "opentelemetry-instrumentation-asyncio>=0.56b0",
+ "opentelemetry-instrumentation-celery>=0.56b0",
+ "opentelemetry-instrumentation-dbapi>=0.56b0",
+ "opentelemetry-instrumentation-django>=0.56b0",
+ "opentelemetry-instrumentation-httpx>=0.56b0",
+ "opentelemetry-instrumentation-psycopg2>=0.56b0",
+ "opentelemetry-instrumentation-requests>=0.56b0",
+ "opentelemetry-instrumentation-sqlite3>=0.56b0",
+ "opentelemetry-instrumentation-threading>=0.56b0",
+ "opentelemetry-instrumentation-urllib>=0.56b0",
+ "opentelemetry-instrumentation-urllib3>=0.56b0",
+ "opentelemetry-instrumentation-wsgi>=0.56b0",
+ "opentelemetry-sdk>=1.35.0",
  "pillow>=11.1.0,<12.0.0",
  "psycopg2-binary>=2.9.10,<3.0.0",
  "pydantic>=2.10.5,<3.0.0",

services/backend/scripts/initdb

@@ -11,3 +11,4 @@ if [ "$DJANGO_CREATE_SUPERUSER" = "True" ]; then
 fi
 
 python manage.py init_cache
+python manage.py silk_clear_request_log