diff --git a/README.md b/README.md index 78235e6..66dae0d 100644 --- a/README.md +++ b/README.md @@ -105,10 +105,10 @@ To run the compose configuration, use the following command: ```bash docker compose -f compose.yaml up # OR -docker compose -f compose.prod.yaml up +docker compose -f compose.prod.yaml --profile observability up ``` -Thats it, project is already preconfigured for running, so no changes before running this are required. +That's it, project is already preconfigured for running, so no changes before running this are required. ## Linting && formatting @@ -134,6 +134,10 @@ Example run: System metrics (gc, requests, etc.) and several business metrics (`lotty_decide_requests_total`, `lotty_events_ingested_total`). +### OTEL + +You can enable full observability for backend by setting `OTEL_ENABLED=True` in `.env`, also you need to run `compose.prod.yaml` with `observability` profile in order to deploy full observability stack + ## Load testing (k6) Reproducible k6 profile for `POST /api/v1/decide`: diff --git a/deploy/compose/compose.backend.yaml b/deploy/compose/compose.backend.yaml index 904d327..cc79771 100644 --- a/deploy/compose/compose.backend.yaml +++ b/deploy/compose/compose.backend.yaml @@ -106,7 +106,10 @@ services: - lotty-backend:latest pull: true entrypoint: ["/bin/sh", "-c"] - command: ["celery -A config worker -l INFO"] + command: + [ + "celery -A config worker -l INFO --concurrency=${CELERY_WORKER_CONCURRENCY:-8}", + ] depends_on: valkey: restart: false diff --git a/infrastructure/configs/backend/.env.template b/infrastructure/configs/backend/.env.template index 05c947d..2c11a7c 100644 --- a/infrastructure/configs/backend/.env.template +++ b/infrastructure/configs/backend/.env.template @@ -8,7 +8,8 @@ DJANGO_LANGUAGE_CODE=en-us DJANGO_STATIC_URL=static/ REDIS_URI=redis://default:valkey@valkey:6379 DJANGO_DB_URI=postgresql://postgres:postgres@postgresql/postgres -DJANGO_CONN_MAX_AGE=300 +DJANGO_CONN_MAX_AGE=60 +DJANGO_CONN_HEALTH_CHECKS=True DJANGO_SILKY_PYTHON_PROFILER= DJANGO_CREATE_SUPERUSER=True @@ -37,3 +38,5 @@ GUNICORN_BIND=0.0.0.0:8080 GUNICORN_WORKER_CLASS=uvicorn_worker.UvicornWorker GUNICORN_ACCESS_LOG=- GUNICORN_ERROR_LOG=- + +CELERY_WORKER_CONCURRENCY=8 diff --git a/infrastructure/configs/postgresql/postgresql.conf b/infrastructure/configs/postgresql/postgresql.conf index d3e5b61..49442be 100644 --- a/infrastructure/configs/postgresql/postgresql.conf +++ b/infrastructure/configs/postgresql/postgresql.conf @@ -62,7 +62,7 @@ listen_addresses = '*' # what IP address(es) to listen on; # defaults to 'localhost'; use '*' for all # (change requires restart) #port = 5432 # (change requires restart) -max_connections = 400 # (change requires restart) +max_connections = 500 # (change requires restart) #reserved_connections = 0 # (change requires restart) #superuser_reserved_connections = 3 # (change requires restart) #unix_socket_directories = '/tmp' # comma-separated list of directories diff --git a/infrastructure/configs/victoriametrics/vmagent.yaml b/infrastructure/configs/victoriametrics/vmagent.yaml index 63f4fa0..322cb76 100644 --- a/infrastructure/configs/victoriametrics/vmagent.yaml +++ b/infrastructure/configs/victoriametrics/vmagent.yaml @@ -26,3 +26,7 @@ scrape_configs: static_configs: - targets: - caddy:8404 + - job_name: backend + static_configs: + - targets: + - backend:8080 diff --git a/src/backend/.env.template b/src/backend/.env.template index 38a5d58..d8b468b 100644 --- a/src/backend/.env.template +++ b/src/backend/.env.template @@ -11,7 +11,8 @@ DJANGO_LANGUAGE_CODE=en-us DJANGO_STATIC_URL=static/ REDIS_URI= DJANGO_DB_URI=sqlite:///db.sqlite3 -DJANGO_CONN_MAX_AGE=300 +DJANGO_CONN_MAX_AGE=60 +DJANGO_CONN_HEALTH_CHECKS=True DECISION_RESULT_CACHE_TTL_SECONDS=60 DECISION_WRITE_MODE=sync DJANGO_SILKY_ENABLED=False @@ -47,6 +48,7 @@ GUNICORN_BIND=0.0.0.0:8080 GUNICORN_WORKER_CLASS=uvicorn_worker.UvicornWorker GUNICORN_ACCESS_LOG=- GUNICORN_ERROR_LOG=- +CELERY_WORKER_CONCURRENCY=8 RUN_MIGRATIONS=False COLLECT_STATIC=False diff --git a/src/backend/config/settings/base.py b/src/backend/config/settings/base.py index 7dd1dcf..1a71fdf 100644 --- a/src/backend/config/settings/base.py +++ b/src/backend/config/settings/base.py @@ -104,7 +104,10 @@ DB_URI["ENGINE"] = DB_URI["ENGINE"].replace( DATABASES = { "default": { **DB_URI, - "CONN_MAX_AGE": env.int("DJANGO_CONN_MAX_AGE", default=300), + "CONN_MAX_AGE": env.int("DJANGO_CONN_MAX_AGE", default=60), + "CONN_HEALTH_CHECKS": env.bool( + "DJANGO_CONN_HEALTH_CHECKS", default=True + ), } }