From cb4da51cf73478e57e3621564e0d4aae0df64314 Mon Sep 17 00:00:00 2001 From: ITQ Date: Mon, 16 Feb 2026 14:13:05 +0300 Subject: [PATCH] chore(): small infrastructure refactoring, improves and fixes --- .gitignore | 6 +++- README.md | 2 +- deploy/compose/compose.backend.yaml | 5 +-- deploy/compose/compose.observability.yaml | 8 ++--- deploy/compose/compose.yaml | 2 +- infrastructure/configs/backend/.env.template | 3 +- infrastructure/configs/caddy/Caddyfile | 4 +-- infrastructure/configs/grafana/.env.template | 2 +- src/backend/.env.template | 5 +-- src/backend/api/urls.py | 1 + src/backend/api/v1/handlers.py | 2 +- src/backend/api/v1/users/endpoints.py | 16 ++++----- src/backend/api/v1/users/schemas.py | 2 -- .../v1/users/tests/test_crud_read_update.py | 4 +-- src/backend/config/gunicorn.py | 35 +++++++++++++++++++ src/backend/justfile | 2 +- src/backend/pyproject.toml | 2 ++ src/backend/scripts/entrypoint.sh | 19 +++++----- src/backend/uv.lock | 33 +++++++++++++++++ 19 files changed, 111 insertions(+), 42 deletions(-) create mode 100644 src/backend/config/gunicorn.py diff --git a/.gitignore b/.gitignore index a4557fb..55e51e6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,8 @@ __MACOSX/ .AppleDouble .LSOverride -Icon[ ] +Icon[ +] # Thumbnails ._* @@ -23,3 +24,6 @@ Icon[ ] Network Trash Folder Temporary Items .apdisk + +# Env files +.env diff --git a/README.md b/README.md index 4a4f0c9..46f79af 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ x-defaults: &defaults #### 2. Choosing compose configuration -- [compose.yaml](./compose.yaml) - default configuration for with base services included. +- [compose.yaml](./compose.yaml) - default configuration with base services included. - [compose.prod.yaml](./compose.prod.yaml) - configuration for production environment with full observability stack. #### 3. Running compose configuration diff --git a/deploy/compose/compose.backend.yaml b/deploy/compose/compose.backend.yaml index 29346cb..e9e00c5 100644 --- a/deploy/compose/compose.backend.yaml +++ b/deploy/compose/compose.backend.yaml @@ -79,7 +79,7 @@ services: - path: ./infrastructure/configs/backend/.env required: false healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:1080"] + test: ["CMD", "wget", "-qO-", "http://localhost:80"] interval: 30s timeout: 5s start_period: 5s @@ -105,7 +105,8 @@ services: tags: - lotty-backend:latest pull: true - command: celery -A config worker -l INFO + entrypoint: ["/bin/sh", "-c"] + command: ["celery -A config worker -l INFO"] depends_on: valkey: restart: false diff --git a/deploy/compose/compose.observability.yaml b/deploy/compose/compose.observability.yaml index 2b6f162..9f41ae9 100644 --- a/deploy/compose/compose.observability.yaml +++ b/deploy/compose/compose.observability.yaml @@ -10,7 +10,7 @@ services: required: false healthcheck: test: ["CMD", "wget", "-O", "-", "http://localhost:3000/api/health"] - interval: 1m30s + interval: 30s timeout: 5s start_period: 5s start_interval: 2s @@ -161,7 +161,7 @@ services: - -httpListenAddr=:8428 healthcheck: test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/-/healthy"] - interval: 1m30s + interval: 30s timeout: 5s start_period: 5s start_interval: 2s @@ -199,7 +199,7 @@ services: required: false healthcheck: test: ["CMD", "wget", "-O", "-", "http://localhost:9121/metrics"] - interval: 1m30s + interval: 30s timeout: 5s start_period: 5s start_interval: 2s @@ -225,7 +225,7 @@ services: required: false healthcheck: test: ["CMD", "wget", "-O", "-", "http://localhost:9187/metrics"] - interval: 1m30s + interval: 30s timeout: 5s start_period: 5s start_interval: 2s diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 2c79d7d..d32f4e8 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -11,7 +11,7 @@ services: required: false healthcheck: test: ["CMD", "sh", "-c", "pg_isready -U postgres -d postgres"] - interval: 1m30s + interval: 30s timeout: 5s start_period: 5s start_interval: 2s diff --git a/infrastructure/configs/backend/.env.template b/infrastructure/configs/backend/.env.template index 9a63365..d94008b 100644 --- a/infrastructure/configs/backend/.env.template +++ b/infrastructure/configs/backend/.env.template @@ -16,12 +16,13 @@ DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=admin@mail.com DJANGO_SUPERUSER_PASSWORD=admin -OTEL_ENABLED=False +OTEL_ENABLED=True OTEL_SERVICE_NAME=backend-django OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 OTEL_TRACES_EXPORTER=otlp OTEL_METRICS_EXPORTER=otlp OTEL_LOGS_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc AWS_ACCESS_KEY_ID=access AWS_SECRET_ACCESS_KEY=storage-secret diff --git a/infrastructure/configs/caddy/Caddyfile b/infrastructure/configs/caddy/Caddyfile index f9a613e..99bd6a5 100644 --- a/infrastructure/configs/caddy/Caddyfile +++ b/infrastructure/configs/caddy/Caddyfile @@ -17,11 +17,11 @@ } } - handle_path /grafana/* { + handle /grafana/* { reverse_proxy grafana:3000 { fail_duration 30s max_fails 10 - health_uri /grafana + health_uri /api/health health_interval 10s health_timeout 2s } diff --git a/infrastructure/configs/grafana/.env.template b/infrastructure/configs/grafana/.env.template index 383c4c2..6d940ee 100644 --- a/infrastructure/configs/grafana/.env.template +++ b/infrastructure/configs/grafana/.env.template @@ -1,3 +1,3 @@ GF_SECURITY_ADMIN_PASSWORD=prooooood -GF_SERVER_ROOT_URL=http://localhost:80/grafana +GF_SERVER_ROOT_URL=http://localhost:80/grafana/ GF_SERVER_SERVE_FROM_SUB_PATH=true diff --git a/src/backend/.env.template b/src/backend/.env.template index a3b4e4a..ba1f251 100644 --- a/src/backend/.env.template +++ b/src/backend/.env.template @@ -9,10 +9,10 @@ DJANGO_CORS_ALLOWED_ORIGINS=* DJANGO_INTERNAL_IPS=127.0.0.1 DJANGO_LANGUAGE_CODE=en-us DJANGO_STATIC_URL=static/ -REDIS_URI=redis://localhost:6379 +REDIS_URI= DJANGO_DB_URI=sqlite:///db.sqlite3 DJANGO_CONN_MAX_AGE=300 -DJANGO_SILKY_ENABLED=True +DJANGO_SILKY_ENABLED=False DJANGO_SILKY_PYTHON_PROFILER=False @@ -24,6 +24,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 OTEL_TRACES_EXPORTER=otlp OTEL_METRICS_EXPORTER=otlp OTEL_LOGS_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc # Storages (S3) diff --git a/src/backend/api/urls.py b/src/backend/api/urls.py index 4789ef4..11c509b 100644 --- a/src/backend/api/urls.py +++ b/src/backend/api/urls.py @@ -21,6 +21,7 @@ urlpatterns = [ "health_check.Cache", "health_check.Database", "health_check.Storage", + "health_check.contrib.celery.Ping", ], ), name="readiness", diff --git a/src/backend/api/v1/handlers.py b/src/backend/api/v1/handlers.py index 9182d76..6b8982a 100644 --- a/src/backend/api/v1/handlers.py +++ b/src/backend/api/v1/handlers.py @@ -187,7 +187,7 @@ def handle_unknown_exception( exc: Exception, router: NinjaAPI, ) -> HttpResponse: - logger.error("Internal server error: %s", exc, exc_info=True) # noqa: LOG014 + logger.error("Internal server error: %s", exc) return create_error_response( request, diff --git a/src/backend/api/v1/users/endpoints.py b/src/backend/api/v1/users/endpoints.py index 9ece296..02a60a5 100644 --- a/src/backend/api/v1/users/endpoints.py +++ b/src/backend/api/v1/users/endpoints.py @@ -23,13 +23,12 @@ from apps.users.services import ( user_update, ) -router = Router(tags=["users"]) +router = Router(tags=["users"], auth=jwt_bearer) @router.get( "", response={HTTPStatus.OK: UserListOut}, - auth=jwt_bearer, summary="List users", description=( "Return a filtered, paginated list of platform users. Admin only." @@ -56,7 +55,6 @@ def list_users( @router.post( "", response={HTTPStatus.CREATED: UserOut}, - auth=jwt_bearer, summary="Create user", description=( "Create a new platform user with the specified role. Admin only." @@ -75,7 +73,6 @@ def create_user( @router.get( "/{user_id}", response={HTTPStatus.OK: UserOut}, - auth=jwt_bearer, summary="Get user", description="Retrieve a single user by their UUID. Admin only.", ) @@ -91,7 +88,6 @@ def get_user( @router.patch( "/{user_id}", response={HTTPStatus.OK: UserOut}, - auth=jwt_bearer, summary="Update user", description=( "Partially update an existing user. " @@ -113,7 +109,6 @@ def update_user( @router.delete( "/{user_id}", response={HTTPStatus.NO_CONTENT: None}, - auth=jwt_bearer, summary="Delete user", description="Permanently delete a user. Admin only.", ) @@ -124,9 +119,7 @@ def delete_user( ) -> tuple[HTTPStatus, None]: user = get_object_or_404(User, pk=user_id) - current_user = getattr(request, "auth", None) - if not isinstance(current_user, User): - raise ValidationError({"user": "Authentication required."}) + current_user = request.user if str(user.pk) == str(current_user.pk): raise ValidationError({"user": "You cannot delete your own account."}) @@ -138,7 +131,6 @@ def delete_user( @router.post( "/{user_id}/role", response={HTTPStatus.OK: UserOut}, - auth=jwt_bearer, summary="Assign role", description="Change the platform role of an existing user. Admin only.", ) @@ -150,5 +142,9 @@ def assign_role( ) -> tuple[HTTPStatus, UserOut]: user = get_object_or_404(User, pk=user_id) + current_user = request.user + if str(user.pk) == str(current_user.pk): + raise ValidationError({"user": "You cannot change your own role."}) + updated_user = user_assign_role(user=user, role=payload.role) return HTTPStatus.OK, UserOut.model_validate(updated_user) diff --git a/src/backend/api/v1/users/schemas.py b/src/backend/api/v1/users/schemas.py index 6b1d879..a58452c 100644 --- a/src/backend/api/v1/users/schemas.py +++ b/src/backend/api/v1/users/schemas.py @@ -33,7 +33,6 @@ class UserCreateIn(ModelSchema): class UserUpdateIn(ModelSchema): - username: str | None = None email: str | None = None first_name: str | None = None last_name: str | None = None @@ -41,7 +40,6 @@ class UserUpdateIn(ModelSchema): class Meta: model = User fields: ClassVar[tuple[str, ...]] = ( - User.username.field.name, User.email.field.name, User.first_name.field.name, User.last_name.field.name, diff --git a/src/backend/api/v1/users/tests/test_crud_read_update.py b/src/backend/api/v1/users/tests/test_crud_read_update.py index 59c77f1..d388a85 100644 --- a/src/backend/api/v1/users/tests/test_crud_read_update.py +++ b/src/backend/api/v1/users/tests/test_crud_read_update.py @@ -34,13 +34,13 @@ class UsersAPIReadUpdateTest(BaseUsersAPITest): reverse( "api-1:update_user", kwargs={"user_id": str(self.viewer.pk)} ), - data=json.dumps({"username": "renamed_viewer"}), + data=json.dumps({"email": "updated@email.com"}), content_type="application/json", HTTP_AUTHORIZATION=self.admin_auth, ) self.assertEqual(resp.status_code, 200) data = resp.json() - self.assertEqual(data["username"], "renamed_viewer") + self.assertEqual(data["email"], "updated@email.com") def test_update_user_partial(self) -> None: original_role = self.viewer.role diff --git a/src/backend/config/gunicorn.py b/src/backend/config/gunicorn.py new file mode 100644 index 0000000..5deedbd --- /dev/null +++ b/src/backend/config/gunicorn.py @@ -0,0 +1,35 @@ +import os + +accesslog = os.environ.get("GUNICORN_ACCESS_LOG", "-") +errorlog = os.environ.get("GUNICORN_ERROR_LOG", "-") + +access_log_format = '%(h)s "%({X-Request-Id}i)s" %(m)s %(U)s %(s)s %(D)s' + +logconfig_dict = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "json": { + "()": "pythonjsonlogger.jsonlogger.JsonFormatter", + "format": "%(levelname)s %(asctime)s %(name)s %(message)s", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "json", + }, + }, + "loggers": { + "gunicorn.error": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "gunicorn.access": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + }, +} diff --git a/src/backend/justfile b/src/backend/justfile index beeb376..7afb740 100644 --- a/src/backend/justfile +++ b/src/backend/justfile @@ -63,7 +63,7 @@ show-coverage: # generates migrations [group('generate')] -generate-migrations: +make-migrations: @ uv run python manage.py makemigrations # applies migrations diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 57dd1d8..b584849 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "opentelemetry-distro>=0.56b0", "opentelemetry-exporter-otlp>=1.35.0", "opentelemetry-sdk>=1.35.0", + "opentelemetry-instrumentation-asgi>=0.56b0", "opentelemetry-instrumentation-asyncio>=0.56b0", "opentelemetry-instrumentation-celery>=0.56b0", "opentelemetry-instrumentation-dbapi>=0.56b0", @@ -33,6 +34,7 @@ dependencies = [ "opentelemetry-instrumentation-urllib>=0.56b0", "opentelemetry-instrumentation-urllib3>=0.56b0", "opentelemetry-instrumentation-wsgi>=0.56b0", + "opentelemetry-instrumentation-logging>=0.60b1", "gunicorn>=23.0.0,<24.0.0", "uvicorn[standard]>=0.34.0,<1.0.0", "uvicorn-worker>=0.2.0,<1.0.0", diff --git a/src/backend/scripts/entrypoint.sh b/src/backend/scripts/entrypoint.sh index 8b6a663..0480c69 100755 --- a/src/backend/scripts/entrypoint.sh +++ b/src/backend/scripts/entrypoint.sh @@ -2,40 +2,37 @@ set -e -if [ "$RUN_MIGRATIONS" = "true" ]; then +if [ "$RUN_MIGRATIONS" = "True" ]; then echo "Running migrations..." python manage.py migrate --noinput fi -if [ "$COLLECT_STATIC" = "true" ]; then +if [ "$COLLECT_STATIC" = "True" ]; then echo "Collecting static files..." python manage.py collectstatic --noinput fi -if [ "$OTEL_ENABLED" = "true" ]; then +if [ "$OTEL_ENABLED" = "True" ]; then echo "Starting with OpenTelemetry instrumentation..." export OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED="true" export OTEL_TRACES_EXPORTER="${OTEL_TRACES_EXPORTER:-otlp}" export OTEL_METRICS_EXPORTER="${OTEL_METRICS_EXPORTER:-otlp}" export OTEL_LOGS_EXPORTER="${OTEL_LOGS_EXPORTER:-otlp}" export OTEL_EXPORTER_OTLP_ENDPOINT="${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4317}" + export OTEL_EXPORTER_OTLP_PROTOCOL="${OTEL_EXPORTER_OTLP_PROTOCOL:-grpc}" exec opentelemetry-instrument \ --service_name "${OTEL_SERVICE_NAME:-backend-django}" \ gunicorn config.asgi:application \ + --config config/gunicorn.py \ --workers "${GUNICORN_WORKERS:-4}" \ --worker-class "${GUNICORN_WORKER_CLASS:-uvicorn_worker.UvicornWorker}" \ - --bind "${GUNICORN_BIND:-0.0.0.0:8080}" \ - --access-logfile "${GUNICORN_ACCESS_LOG:--}" \ - --error-logfile "${GUNICORN_ERROR_LOG:--}" \ - --access-logformat '{"remote_ip": "%(h)s", "request_id": "%({X-Request-Id}i)s", "response_code": "%(s)s", "request_method": "%(m)s", "request_path": "%(U)s", "request_timetaken": "%(D)s"}' + --bind "${GUNICORN_BIND:-0.0.0.0:8080}" else echo "Starting without OpenTelemetry instrumentation..." exec gunicorn config.asgi:application \ + --config config/gunicorn.py \ --workers "${GUNICORN_WORKERS:-4}" \ --worker-class "${GUNICORN_WORKER_CLASS:-uvicorn_worker.UvicornWorker}" \ - --bind "${GUNICORN_BIND:-0.0.0.0:8080}" \ - --access-logfile "${GUNICORN_ACCESS_LOG:--}" \ - --error-logfile "${GUNICORN_ERROR_LOG:--}" \ - --access-logformat '{"remote_ip": "%(h)s", "request_id": "%({X-Request-Id}i)s", "response_code": "%(s)s", "request_method": "%(m)s", "request_path": "%(U)s", "request_timetaken": "%(D)s"}' + --bind "${GUNICORN_BIND:-0.0.0.0:8080}" fi diff --git a/src/backend/uv.lock b/src/backend/uv.lock index f474682..61ae8f9 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -706,10 +706,12 @@ dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-distro" }, { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-instrumentation-asgi" }, { name = "opentelemetry-instrumentation-asyncio" }, { name = "opentelemetry-instrumentation-celery" }, { name = "opentelemetry-instrumentation-dbapi" }, { name = "opentelemetry-instrumentation-django" }, + { name = "opentelemetry-instrumentation-logging" }, { name = "opentelemetry-instrumentation-psycopg2" }, { name = "opentelemetry-instrumentation-requests" }, { name = "opentelemetry-instrumentation-threading" }, @@ -755,10 +757,12 @@ requires-dist = [ { name = "opentelemetry-api", specifier = ">=1.35.0" }, { name = "opentelemetry-distro", specifier = ">=0.56b0" }, { name = "opentelemetry-exporter-otlp", specifier = ">=1.35.0" }, + { name = "opentelemetry-instrumentation-asgi", specifier = ">=0.56b0" }, { name = "opentelemetry-instrumentation-asyncio", specifier = ">=0.56b0" }, { name = "opentelemetry-instrumentation-celery", specifier = ">=0.56b0" }, { name = "opentelemetry-instrumentation-dbapi", specifier = ">=0.56b0" }, { name = "opentelemetry-instrumentation-django", specifier = ">=0.56b0" }, + { name = "opentelemetry-instrumentation-logging", specifier = ">=0.60b1" }, { name = "opentelemetry-instrumentation-psycopg2", specifier = ">=0.56b0" }, { name = "opentelemetry-instrumentation-requests", specifier = ">=0.56b0" }, { name = "opentelemetry-instrumentation-threading", specifier = ">=0.56b0" }, @@ -924,6 +928,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/d2/6788e83c5c86a2690101681aeef27eeb2a6bf22df52d3f263a22cee20915/opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d", size = 33096, upload-time = "2025-12-11T13:35:33.067Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-asgi" +version = "0.60b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/db/851fa88db7441da82d50bd80f2de5ee55213782e25dc858e04d0c9961d60/opentelemetry_instrumentation_asgi-0.60b1.tar.gz", hash = "sha256:16bfbe595cd24cda309a957456d0fc2523f41bc7b076d1f2d7e98a1ad9876d6f", size = 26107, upload-time = "2025-12-11T13:36:47.015Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/76/1fb94367cef64420d2171157a6b9509582873bd09a6afe08a78a8d1f59d9/opentelemetry_instrumentation_asgi-0.60b1-py3-none-any.whl", hash = "sha256:d48def2dbed10294c99cfcf41ebbd0c414d390a11773a41f472d20000fcddc25", size = 16933, upload-time = "2025-12-11T13:35:40.462Z" }, +] + [[package]] name = "opentelemetry-instrumentation-asyncio" version = "0.60b1" @@ -984,6 +1004,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/05/6b348ea989f7a9e1e6311fa653e113bd39f4506771323e27a639c2a1ea54/opentelemetry_instrumentation_django-0.60b1-py3-none-any.whl", hash = "sha256:3f6b4ba201eee35406dab965b254eed0c64fa1ef42e4a7b0296ad1b30e8e3f81", size = 21172, upload-time = "2025-12-11T13:35:57.365Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-logging" +version = "0.60b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/a6/4515895b383113677fd2ad21813df5e56108a2df14ebb7916c962c9a0234/opentelemetry_instrumentation_logging-0.60b1.tar.gz", hash = "sha256:98f4b9c7aeb9314a30feee7c002c7ea9abea07c90df5f97fb058b850bc45b89a", size = 9968, upload-time = "2025-12-11T13:37:03.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/f9/8a4ce3901bc52277794e4b18c4ac43dc5929806eff01d22812364132f45f/opentelemetry_instrumentation_logging-0.60b1-py3-none-any.whl", hash = "sha256:f2e18cbc7e1dd3628c80e30d243897fdc93c5b7e0c8ae60abd2b9b6a99f82343", size = 12577, upload-time = "2025-12-11T13:36:08.123Z" }, +] + [[package]] name = "opentelemetry-instrumentation-psycopg2" version = "0.60b1"