diff --git a/solution/assets/images/backend-coverage.png b/solution/assets/images/backend-coverage.png new file mode 100644 index 0000000..71bfd39 Binary files /dev/null and b/solution/assets/images/backend-coverage.png differ diff --git a/solution/infrastructure/.gitignore b/solution/infrastructure/.gitignore index 7d35adf..dad66cc 100644 --- a/solution/infrastructure/.gitignore +++ b/solution/infrastructure/.gitignore @@ -1,4 +1,4 @@ -# Custom evnironment files +# Custom environment files .env # Password files diff --git a/solution/infrastructure/backend/.env.template b/solution/infrastructure/backend/.env.template index e219fb4..2fac0be 100644 --- a/solution/infrastructure/backend/.env.template +++ b/solution/infrastructure/backend/.env.template @@ -8,15 +8,16 @@ DJANGO_LANGUAGE_CODE=en-us DJANGO_STATIC_URL=http://localhost:13241/ REDIS_URI=redis://redis:6379 DJANGO_DB_URI=postgresql://postgres:postgres@postgres/postgres + DJANGO_CREATE_SUPERUSER=True DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=admin@mail.com DJANGO_SUPERUSER_PASSWORD=admin + YANDEX_CLOUD_FOLDER_ID=b1grd7nb04jbfvgm9121 YANDEX_CLOUD_API_KEY=AQVN19lztDgUcBrHeW0HtJ0bbDGda6i6e3InXsDl + MINIO_ENDPOINT=minio:9000 MINIO_CUSTOM_ENDPOINT_URL=http://127.0.0.1:13244 MINIO_ACCESS_KEY=admin MINIO_SECRET_KEY=password -DJANGO_NOTIFIER_TELEGRAM_BOT_TOKEN=6196898691:AAFucgj7ieEuYMvWG_MZAn0Ao3UBuHpvaVY -DJANGO_NOTIFIER_TELEGRAM_CHAT_ID=-1002304409222 diff --git a/solution/services/backend/.env.template b/solution/services/backend/.env.template index 378245d..5988a69 100644 --- a/solution/services/backend/.env.template +++ b/solution/services/backend/.env.template @@ -31,12 +31,3 @@ DJANGO_CREATE_SUPERUSER=False DJANGO_SUPERUSER_USERNAME= DJANGO_SUPERUSER_EMAIL= DJANGO_SUPERUSER_PASSWORD= - - -# Notifiers settings (only with DJANGO_DEBUG=False) - -# Telegram - -DJANGO_NOTIFIER_TELEGRAM_BOT_TOKEN= -DJANGO_NOTIFIER_TELEGRAM_CHAT_ID= -DJANGO_NOTIFIER_TELEGRAM_THREAD_ID= diff --git a/solution/services/backend/Dockerfile.staticfiles b/solution/services/backend/Dockerfile.staticfiles index fb3fef2..5150bf5 100644 --- a/solution/services/backend/Dockerfile.staticfiles +++ b/solution/services/backend/Dockerfile.staticfiles @@ -19,7 +19,7 @@ COPY . . RUN uv run python manage.py collectstatic --noinput -# Stage 2: Start nginx to serve staticfiles +# Stage 2: Start nginx and serve staticfiles FROM docker.io/nginx:latest COPY --from=builder /app/static /usr/share/nginx/html diff --git a/solution/services/backend/api/v1/clients/tests.py b/solution/services/backend/api/v1/clients/tests.py index 03a323f..a61f8ad 100644 --- a/solution/services/backend/api/v1/clients/tests.py +++ b/solution/services/backend/api/v1/clients/tests.py @@ -1,6 +1,5 @@ from http import HTTPStatus as status from django.test import TestCase -from django.urls import reverse import json from uuid import uuid4 from apps.client.models import Client diff --git a/solution/services/backend/apps/advertiser/tests/test_advertiser_model.py b/solution/services/backend/apps/advertiser/tests/test_advertiser_model.py index 08b5806..c0e5e91 100644 --- a/solution/services/backend/apps/advertiser/tests/test_advertiser_model.py +++ b/solution/services/backend/apps/advertiser/tests/test_advertiser_model.py @@ -22,6 +22,7 @@ class AdvertiserModelTest(TestCase): new_id = uuid4() self.advertiser.advertiser_id = new_id + self.assertEqual(self.advertiser.id, new_id) @override_settings( diff --git a/solution/services/backend/apps/campaign/models.py b/solution/services/backend/apps/campaign/models.py index a60d5b8..c8ba5a8 100644 --- a/solution/services/backend/apps/campaign/models.py +++ b/solution/services/backend/apps/campaign/models.py @@ -6,7 +6,6 @@ from uuid import UUID from django.conf import settings from django.core.cache import cache -from django.core.exceptions import ValidationError from django.core.validators import ( MaxValueValidator, MinLengthValidator, @@ -20,6 +19,7 @@ from apps.campaign.validators import ( CampaignDurationValidator, CampaignLimitsValidator, CampaignReportMessageValidator, + CampaignStartDateValidator, CampaignTargetingLocationValidator, ) from apps.client.models import Client @@ -100,21 +100,7 @@ class Campaign(BaseModel): CampaignAgeValidator()(self) CampaignDurationValidator()(self) CampaignLimitsValidator()(self) - - current_date = cache.get("current_date", default=0) - - err = "start_date must be greater than the current date." - - try: - original = Campaign.objects.get(id=self.id or "") - if ( - original.start_date != self.start_date - and self.start_date < current_date - ): - raise ValidationError(err) - except Campaign.DoesNotExist: - if self.start_date < current_date: - raise ValidationError(err) from None + CampaignStartDateValidator()(self) def save(self, *args: Any, **kwargs: Any) -> None: created = self.pk is None @@ -134,6 +120,20 @@ class Campaign(BaseModel): ) cache.set(f"campaign_{self.id}_clicks_count", self.clicks.count()) + def inc_views(self) -> None: + try: + cache.incr(f"campaign_{self.id}_impressions_count", 1) + except ValueError: + self.setup_cache() + logger.warning("Seems that %s missing caches", self.campaign_id) + + def inc_clicks(self) -> None: + try: + cache.incr(f"campaign_{self.id}_clicks_count", 1) + except ValueError: + self.setup_cache() + logger.warning("Seems that %s missing caches", self.campaign_id) + @property def ad_id(self) -> UUID: return self.id @@ -175,13 +175,7 @@ class Campaign(BaseModel): price=self.cost_per_impression, date=cache.get("current_date", default=0), ) - try: - cache.incr(f"campaign_{self.id}_impressions_count", 1) - except ValueError: - self.setup_cache() - logger.warning( - "Seems that %s missing caches", self.campaign_id - ) + self.inc_views() except ConflictError: pass @@ -198,13 +192,7 @@ class Campaign(BaseModel): price=self.cost_per_click, date=cache.get("current_date", default=0), ) - try: - cache.incr(f"campaign_{self.id}_clicks_count", 1) - except ValueError: - self.setup_cache() - logger.warning( - "Seems that %s missing caches", self.campaign_id - ) + self.inc_clicks() except ConflictError: pass @@ -235,7 +223,6 @@ class Campaign(BaseModel): total=models.Count("id"), spent=models.Sum("price", default=0.0), ) - clicks = self.clicks.values("date").annotate( total=models.Count("id"), spent=models.Sum("price", default=0.0), diff --git a/solution/services/backend/apps/campaign/validators.py b/solution/services/backend/apps/campaign/validators.py index 98426a9..ff45719 100644 --- a/solution/services/backend/apps/campaign/validators.py +++ b/solution/services/backend/apps/campaign/validators.py @@ -1,48 +1,83 @@ +from typing import TYPE_CHECKING + +from django.core.cache import cache from django.core.exceptions import ValidationError +from django.db.models.fields import Field + +if TYPE_CHECKING: + from apps.campaign.models import Campaign, CampaignReport + + +class CampaignTargetingLocationValidator: + def __call__(self, instance: "Campaign") -> None: + if instance.location == "": + err = { + "targeting": { + type( + instance + ).location.field.name: Field.default_error_messages[ + "blank" + ] + } + } + raise ValidationError(err) class CampaignAgeValidator: - def __call__(self, instance) -> None: # noqa: ANN001 + def __call__(self, instance: "Campaign") -> None: if ( isinstance(instance.age_from, int) and isinstance(instance.age_to, int) and instance.age_from > instance.age_to ): - err = "age_from can't be greater than age_to" + err = "targeting.age_from can't be greater than targeting.age_to." raise ValidationError(err) class CampaignDurationValidator: - def __call__(self, instance) -> None: # noqa: ANN001 + def __call__(self, instance: "Campaign") -> None: if ( isinstance(instance.start_date, int) and isinstance(instance.end_date, int) and instance.start_date > instance.end_date ): - err = "start_date can't be greater than end_date" + err = "start_date can't be greater than end_date." raise ValidationError(err) class CampaignLimitsValidator: - def __call__(self, instance) -> None: # noqa: ANN001 + def __call__(self, instance: "Campaign") -> None: if ( isinstance(instance.impressions_limit, int) and isinstance(instance.clicks_limit, int) and instance.impressions_limit < instance.clicks_limit ): - err = "clicks_limit can't be greater than impressions_limit" + err = "clicks_limit can't be greater than impressions_limit." raise ValidationError(err) -class CampaignTargetingLocationValidator: - def __call__(self, instance) -> None: # noqa: ANN001 - if instance.location == "": - err = "targeting.location cannot be blank" - raise ValidationError(err) +class CampaignStartDateValidator: + def __call__(self, instance: "Campaign") -> None: + current_date = cache.get("current_date", default=0) + err = "start_date must be greater or equal than the current_date." + try: + original = type(instance).objects.get(id=instance.id or "") + if ( + original.start_date != instance.start_date + and instance.start_date < current_date + ): + raise ValidationError(err) + except type(instance).DoesNotExist: + if instance.start_date < current_date: + raise ValidationError(err) from None class CampaignReportMessageValidator: - def __call__(self, instance) -> None: # noqa: ANN001 + def __call__(self, instance: "CampaignReport") -> None: if instance.message == "": - err = "message cannot be blank" + err = { + instance.message.field.name: Field.default_error_messages[ + "blank" + ] + } raise ValidationError(err) diff --git a/solution/services/backend/config/notifiers/__init__.py b/solution/services/backend/config/notifiers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/solution/services/backend/config/notifiers/telegram.py b/solution/services/backend/config/notifiers/telegram.py deleted file mode 100644 index 6ca3a65..0000000 --- a/solution/services/backend/config/notifiers/telegram.py +++ /dev/null @@ -1,133 +0,0 @@ -import datetime -import logging -import time -import traceback -from concurrent.futures import ThreadPoolExecutor - -import httpx -from django.utils.timezone import get_current_timezone - -TELEGRAM_LOG_HANDLER = logging.getLogger("telegram_log_handler") - -LEVEL_EMOJIS = { - "DEBUG": "🐞", - "INFO": "ℹ️", - "WARNING": "⚠️", - "ERROR": "❌", - "CRITICAL": "🚨", -} - - -class LoggingHandler(logging.Handler): - _executor = ThreadPoolExecutor(max_workers=5) - - def __init__( - self, - token: str, - chat_id: int, - thread_id: int | None = None, - retries: int | None = 3, - delay: int | None = 2, - timeout: int | None = 5, - ) -> None: - super().__init__() - - self.token = token - self.chat_id = chat_id - self.thread_id = thread_id - self.retries = retries - self.delay = delay - self.timeout = timeout - self.api_url = f"https://api.telegram.org/bot{self.token}/sendMessage" - - self.template = ( - "{levelname}\n" - "\tGuid: {correlation_id}\n" - "\tTimestamp: {asctime}\n" - "\tLogger: {name}\n" - "\tFile: {pathname} " - "(Line: {lineno})\n\n" - '
{message}
\n' - ) - - def emit(self, record: logging.LogRecord) -> None: - try: - formatted_record = self.format(record) - self._executor.submit(self._send_message, formatted_record) - except Exception as e: # noqa: BLE001 - self.handleError(record) - TELEGRAM_LOG_HANDLER.exception(e) - - def _send_message(self, formatted_record: str) -> None: - payload = { - "chat_id": self.chat_id, - "text": formatted_record, - "parse_mode": "HTML", - } - if self.thread_id: - payload["reply_to_message_id"] = self.thread_id - - for attempt in range(1, self.retries + 1): - response = httpx.post( - self.api_url, - data=payload, - timeout=self.timeout, - ) - if response.status_code != httpx.codes.OK: - if attempt == self.retries: - TELEGRAM_LOG_HANDLER.exception( - "Failed to send to Telegram after %d attempts: %s", - self.retries, - response.text, - ) - else: - time.sleep(self.delay) - else: - return - - def format(self, record: logging.LogRecord) -> str: - try: - asctime = datetime.datetime.fromtimestamp( - record.created, - tz=get_current_timezone(), - ).strftime("%Y-%m-%d %H:%M:%S %Z") - level_emoji = LEVEL_EMOJIS.get(record.levelname, "") - - formatted_message = self.template.format( - levelname=f"{level_emoji} {record.levelname}", - correlation_id=getattr(record, "correlation_id", "N/A"), - asctime=asctime, - name=record.name, - pathname=record.pathname, - lineno=record.lineno, - message=record.getMessage(), - ) - - if record.exc_info: - formatted_message += self._format_exception(record.exc_info) - - formatted_message += ( - f"\n#{record.levelname.lower()} " - f"#{record.name.replace('.', '_')}" - ) - if hasattr(record, "correlation_id"): - formatted_message += f" #{record.correlation_id}" - except Exception as format_error: # noqa: BLE001 - TELEGRAM_LOG_HANDLER.exception( - "Error formatting log record: %s", - format_error, - ) - return f"Error formatting log record: {format_error}" - else: - return formatted_message - - @staticmethod - def _format_exception(exc_info: Exception) -> str: - exc_text = "".join(traceback.format_exception(*exc_info)) - return ( - f"\n
{exc_text}
" - ) - - @classmethod - def shutdown_executor(cls) -> None: - cls._executor.shutdown(wait=True) diff --git a/solution/services/backend/config/settings.py b/solution/services/backend/config/settings.py index 122b846..a1d1102 100644 --- a/solution/services/backend/config/settings.py +++ b/solution/services/backend/config/settings.py @@ -41,7 +41,8 @@ YANDEX_CLOUD_INTEGRATION_ENABLED = ( YANDEX_CLOUD_FOLDER_ID and YANDEX_CLOUD_API_KEY ) -# Register healthcheck + +# Register healthchecks plugin_dir.register(YandexAIHealthCheck) @@ -300,26 +301,6 @@ USE_X_FORWARDED_PORT = False WSGI_APPLICATION = "config.wsgi.application" -# Notifiers - -# Telegram - -NOTIFIER_TELEGRAM_BOT_TOKEN = env( - "DJANGO_NOTIFIER_TELEGRAM_BOT_TOKEN", - default=None, -) - -NOTIFIER_TELEGRAM_CHAT_ID = env( - "DJANGO_NOTIFIER_TELEGRAM_CHAT_ID", - default=None, -) - -NOTIFIER_TELEGRAM_THREAD_ID = env( - "DJANGO_NOTIFIER_TELEGRAM_THREAD_ID", - default=None, -) - - # Logging LOGGER_NAME = "adnova" @@ -435,24 +416,6 @@ LOGGING = { LOGGING_CONFIG = "logging.config.dictConfig" -if NOTIFIER_TELEGRAM_BOT_TOKEN and NOTIFIER_TELEGRAM_CHAT_ID: - LOGGING_HANDLERS["telegram"] = { - "class": "config.notifiers.telegram.LoggingHandler", - "level": "INFO", - "filters": ["require_debug_false"], - "token": NOTIFIER_TELEGRAM_BOT_TOKEN, - "chat_id": NOTIFIER_TELEGRAM_CHAT_ID, - "thread_id": NOTIFIER_TELEGRAM_THREAD_ID, - "retries": 5, - "delay": 2, - "timeout": 5, - } - LOGGING_LOGGERS["django"]["handlers"].append("telegram") - LOGGING_LOGGERS["django.request"]["handlers"].append("telegram") - LOGGING_LOGGERS["health-check"]["handlers"].append("telegram") - LOGGING_LOGGERS[LOGGER_NAME]["handlers"].append("telegram") - - # Models ABSOLUTE_URL_OVERRIDES: dict[str, Callable] = {} diff --git a/solution/services/backend/integrations/yandexai/healthcheck.py b/solution/services/backend/integrations/yandexai/healthcheck.py index 3f4659b..48fe99a 100644 --- a/solution/services/backend/integrations/yandexai/healthcheck.py +++ b/solution/services/backend/integrations/yandexai/healthcheck.py @@ -16,8 +16,10 @@ class YandexAIHealthCheck(BaseHealthCheckBackend): result = sdk.models.completions( "yandexgpt-lite", model_version="latest" ).tokenize("ping") + if not result: self.add_error("YandexAI API is unaccessible") + except YCloudMLError: self.add_error("YandexAI API is unaccessible") diff --git a/solution/tests/README.md b/solution/tests/README.md index d338986..04d8c23 100644 --- a/solution/tests/README.md +++ b/solution/tests/README.md @@ -1,6 +1,6 @@ # AdNova Tests -There is `unit` tests and `e2e` tests available, unit tests are placed all around `backend` serivce folder and `e2e` tests placed [here](./e2e/) +There is `unit` and `e2e` tests available, unit tests are placed all around `backend` serivce folder and `e2e` tests placed [here](./e2e/) ## Running unit tests @@ -12,7 +12,7 @@ See [services/backend/README.md](../services/backend/README.md#testing) ### Backend service --image- +![backend coverage](../assets/images/backend-coverage.png) ## Running e2e tests diff --git a/solution/tests/e2e/README.md b/solution/tests/e2e/README.md index dd4bbf5..6ed5ed4 100644 --- a/solution/tests/e2e/README.md +++ b/solution/tests/e2e/README.md @@ -31,7 +31,7 @@ cd devitq/solution/tests/e2e uv sync --no-dev ``` -## Customize environment +## Customize environment (optional) ```bash cp .env.template .env diff --git a/solution/tests/e2e/pyproject.toml b/solution/tests/e2e/pyproject.toml index cdaf1a1..2e81273 100644 --- a/solution/tests/e2e/pyproject.toml +++ b/solution/tests/e2e/pyproject.toml @@ -107,4 +107,3 @@ unfixable = [] [tool.ruff.lint.pylint] max-args = 6 - diff --git a/solution/tests/e2e/tests/test_ad_text_generation.py b/solution/tests/e2e/tests/test_ad_text_generation.py index dcd499a..98e836b 100644 --- a/solution/tests/e2e/tests/test_ad_text_generation.py +++ b/solution/tests/e2e/tests/test_ad_text_generation.py @@ -8,8 +8,15 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -# Tests integration between: backend, redis, yandexgpt and celery def test_generate_ad_text(client: Client) -> None: + """ + Tests integration between: + - backend + - redis + - yandexgpt + - celery + """ + payload = { "advertiser_name": "Центральный Университет", "ad_title": "Всероссийский кейс-чемпионат DEADLINE", @@ -25,10 +32,7 @@ def test_generate_ad_text(client: Client) -> None: while True: result_response = client.get(f"/generate/ad_text/{task_id}/result") - assert ( - result_response.status_code == status.OK - or result_response.status_code == status.NOT_FOUND - ) + assert result_response.status_code in (status.OK, status.NOT_FOUND) result_data = result_response.json() if ( diff --git a/solution/tests/e2e/tests/test_backend_health.py b/solution/tests/e2e/tests/test_backend_health.py index b00804e..7a9a33e 100644 --- a/solution/tests/e2e/tests/test_backend_health.py +++ b/solution/tests/e2e/tests/test_backend_health.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def test_healthcheck(client: Client) -> None: """ - Checks that backend can use theese services: + Tests integration between: - redis - celery - postgres