PROD.2025/AdNova
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: global codebase refactoring

Browse Source
This commit is contained in:
ITQ
2025-02-21 18:00:42 +03:00
parent ea8f5cfd49
commit cebb87fa73
18 changed files with 89 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
solution/assets/images/backend-coverage.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 67 KiB

solution/infrastructure/.gitignore
+1 -1
View File
@@ -1,4 +1,4 @@
# Custom evnironment files
# Custom environment files
.env
# Password files
solution/infrastructure/backend/.env.template
+3 -2
View File
@@ -8,15 +8,16 @@ DJANGO_LANGUAGE_CODE=en-us
DJANGO_STATIC_URL=http://localhost:13241/
REDIS_URI=redis://redis:6379
DJANGO_DB_URI=postgresql://postgres:postgres@postgres/postgres
DJANGO_CREATE_SUPERUSER=True
DJANGO_SUPERUSER_USERNAME=admin
DJANGO_SUPERUSER_EMAIL=admin@mail.com
DJANGO_SUPERUSER_PASSWORD=admin
YANDEX_CLOUD_FOLDER_ID=b1grd7nb04jbfvgm9121
YANDEX_CLOUD_API_KEY=AQVN19lztDgUcBrHeW0HtJ0bbDGda6i6e3InXsDl
MINIO_ENDPOINT=minio:9000
MINIO_CUSTOM_ENDPOINT_URL=http://127.0.0.1:13244
MINIO_ACCESS_KEY=admin
MINIO_SECRET_KEY=password
DJANGO_NOTIFIER_TELEGRAM_BOT_TOKEN=6196898691:AAFucgj7ieEuYMvWG_MZAn0Ao3UBuHpvaVY
DJANGO_NOTIFIER_TELEGRAM_CHAT_ID=-1002304409222
solution/services/backend/.env.template
-9
View File
@@ -31,12 +31,3 @@ DJANGO_CREATE_SUPERUSER=False
DJANGO_SUPERUSER_USERNAME=
DJANGO_SUPERUSER_EMAIL=
DJANGO_SUPERUSER_PASSWORD=
# Notifiers settings (only with DJANGO_DEBUG=False)
# Telegram
DJANGO_NOTIFIER_TELEGRAM_BOT_TOKEN=
DJANGO_NOTIFIER_TELEGRAM_CHAT_ID=
DJANGO_NOTIFIER_TELEGRAM_THREAD_ID=
solution/services/backend/Dockerfile.staticfiles
+1 -1
View File
@@ -19,7 +19,7 @@ COPY . .
RUN uv run python manage.py collectstatic --noinput
# Stage 2: Start nginx to serve staticfiles
# Stage 2: Start nginx and serve staticfiles
FROM docker.io/nginx:latest
COPY --from=builder /app/static /usr/share/nginx/html
solution/services/backend/api/v1/clients/tests.py
-1
View File
@@ -1,6 +1,5 @@
from http import HTTPStatus as status
from django.test import TestCase
from django.urls import reverse
import json
from uuid import uuid4
from apps.client.models import Client
solution/services/backend/apps/advertiser/tests/test_advertiser_model.py
+1
View File
@@ -22,6 +22,7 @@ class AdvertiserModelTest(TestCase):
new_id = uuid4()
self.advertiser.advertiser_id = new_id
self.assertEqual(self.advertiser.id, new_id)
@override_settings(
solution/services/backend/apps/campaign/models.py
+18 -31
View File
@@ -6,7 +6,6 @@ from uuid import UUID
from django.conf import settings
from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.core.validators import (
MaxValueValidator,
MinLengthValidator,
@@ -20,6 +19,7 @@ from apps.campaign.validators import (
CampaignDurationValidator,
CampaignLimitsValidator,
CampaignReportMessageValidator,
CampaignStartDateValidator,
CampaignTargetingLocationValidator,
)
from apps.client.models import Client
@@ -100,21 +100,7 @@ class Campaign(BaseModel):
CampaignAgeValidator()(self)
CampaignDurationValidator()(self)
CampaignLimitsValidator()(self)
current_date = cache.get("current_date", default=0)
err = "start_date must be greater than the current date."
try:
original = Campaign.objects.get(id=self.id or "")
if (
original.start_date != self.start_date
and self.start_date < current_date
):
raise ValidationError(err)
except Campaign.DoesNotExist:
if self.start_date < current_date:
raise ValidationError(err) from None
CampaignStartDateValidator()(self)
def save(self, *args: Any, **kwargs: Any) -> None:
created = self.pk is None
@@ -134,6 +120,20 @@ class Campaign(BaseModel):
)
cache.set(f"campaign_{self.id}_clicks_count", self.clicks.count())
def inc_views(self) -> None:
try:
cache.incr(f"campaign_{self.id}_impressions_count", 1)
except ValueError:
self.setup_cache()
logger.warning("Seems that %s missing caches", self.campaign_id)
def inc_clicks(self) -> None:
try:
cache.incr(f"campaign_{self.id}_clicks_count", 1)
except ValueError:
self.setup_cache()
logger.warning("Seems that %s missing caches", self.campaign_id)
@property
def ad_id(self) -> UUID:
return self.id
@@ -175,13 +175,7 @@ class Campaign(BaseModel):
price=self.cost_per_impression,
date=cache.get("current_date", default=0),
)
try:
cache.incr(f"campaign_{self.id}_impressions_count", 1)
except ValueError:
self.setup_cache()
logger.warning(
"Seems that %s missing caches", self.campaign_id
)
self.inc_views()
except ConflictError:
pass
@@ -198,13 +192,7 @@ class Campaign(BaseModel):
price=self.cost_per_click,
date=cache.get("current_date", default=0),
)
try:
cache.incr(f"campaign_{self.id}_clicks_count", 1)
except ValueError:
self.setup_cache()
logger.warning(
"Seems that %s missing caches", self.campaign_id
)
self.inc_clicks()
except ConflictError:
pass
@@ -235,7 +223,6 @@ class Campaign(BaseModel):
total=models.Count("id"),
spent=models.Sum("price", default=0.0),
)
clicks = self.clicks.values("date").annotate(
total=models.Count("id"),
spent=models.Sum("price", default=0.0),
solution/services/backend/apps/campaign/validators.py
+47 -12
View File
@@ -1,48 +1,83 @@
from typing import TYPE_CHECKING
from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.db.models.fields import Field
if TYPE_CHECKING:
from apps.campaign.models import Campaign, CampaignReport
class CampaignTargetingLocationValidator:
def __call__(self, instance: "Campaign") -> None:
if instance.location == "":
err = {
"targeting": {
type(
instance
).location.field.name: Field.default_error_messages[
"blank"
]
}
}
raise ValidationError(err)
class CampaignAgeValidator:
def __call__(self, instance) -> None: # noqa: ANN001
def __call__(self, instance: "Campaign") -> None:
if (
isinstance(instance.age_from, int)
and isinstance(instance.age_to, int)
and instance.age_from > instance.age_to
):
err = "age_from can't be greater than age_to"
err = "targeting.age_from can't be greater than targeting.age_to."
raise ValidationError(err)
class CampaignDurationValidator:
def __call__(self, instance) -> None: # noqa: ANN001
def __call__(self, instance: "Campaign") -> None:
if (
isinstance(instance.start_date, int)
and isinstance(instance.end_date, int)
and instance.start_date > instance.end_date
):
err = "start_date can't be greater than end_date"
err = "start_date can't be greater than end_date."
raise ValidationError(err)
class CampaignLimitsValidator:
def __call__(self, instance) -> None: # noqa: ANN001
def __call__(self, instance: "Campaign") -> None:
if (
isinstance(instance.impressions_limit, int)
and isinstance(instance.clicks_limit, int)
and instance.impressions_limit < instance.clicks_limit
):
err = "clicks_limit can't be greater than impressions_limit"
err = "clicks_limit can't be greater than impressions_limit."
raise ValidationError(err)
class CampaignTargetingLocationValidator:
def __call__(self, instance) -> None: # noqa: ANN001
if instance.location == "":
err = "targeting.location cannot be blank"
class CampaignStartDateValidator:
def __call__(self, instance: "Campaign") -> None:
current_date = cache.get("current_date", default=0)
err = "start_date must be greater or equal than the current_date."
try:
original = type(instance).objects.get(id=instance.id or "")
if (
original.start_date != instance.start_date
and instance.start_date < current_date
):
raise ValidationError(err)
except type(instance).DoesNotExist:
if instance.start_date < current_date:
raise ValidationError(err) from None
class CampaignReportMessageValidator:
def __call__(self, instance) -> None: # noqa: ANN001
def __call__(self, instance: "CampaignReport") -> None:
if instance.message == "":
err = "message cannot be blank"
err = {
instance.message.field.name: Field.default_error_messages[
"blank"
]
}
raise ValidationError(err)
solution/services/backend/config/notifiers/__init__.py
View File
solution/services/backend/config/notifiers/telegram.py
-133
View File
@@ -1,133 +0,0 @@
import datetime
import logging
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
import httpx
from django.utils.timezone import get_current_timezone
TELEGRAM_LOG_HANDLER = logging.getLogger("telegram_log_handler")
LEVEL_EMOJIS = {
"DEBUG": "🐞",
"INFO": "ℹ️",
"WARNING": "⚠️",
"ERROR": "❌",
"CRITICAL": "🚨",
}
class LoggingHandler(logging.Handler):
_executor = ThreadPoolExecutor(max_workers=5)
def __init__(
self,
token: str,
chat_id: int,
thread_id: int | None = None,
retries: int | None = 3,
delay: int | None = 2,
timeout: int | None = 5,
) -> None:
super().__init__()
self.token = token
self.chat_id = chat_id
self.thread_id = thread_id
self.retries = retries
self.delay = delay
self.timeout = timeout
self.api_url = f"https://api.telegram.org/bot{self.token}/sendMessage"
self.template = (
"<b>{levelname}</b>\n"
"\t<b>Guid:</b> <code>{correlation_id}</code>\n"
"\t<b>Timestamp:</b> <code>{asctime}</code>\n"
"\t<b>Logger:</b> <code>{name}</code>\n"
"\t<b>File:</b> <code>{pathname}</code> "
"(Line: <code>{lineno}</code>)\n\n"
'<pre><code class="language-message">{message}</code></pre>\n'
)
def emit(self, record: logging.LogRecord) -> None:
try:
formatted_record = self.format(record)
self._executor.submit(self._send_message, formatted_record)
except Exception as e: # noqa: BLE001
self.handleError(record)
TELEGRAM_LOG_HANDLER.exception(e)
def _send_message(self, formatted_record: str) -> None:
payload = {
"chat_id": self.chat_id,
"text": formatted_record,
"parse_mode": "HTML",
}
if self.thread_id:
payload["reply_to_message_id"] = self.thread_id
for attempt in range(1, self.retries + 1):
response = httpx.post(
self.api_url,
data=payload,
timeout=self.timeout,
)
if response.status_code != httpx.codes.OK:
if attempt == self.retries:
TELEGRAM_LOG_HANDLER.exception(
"Failed to send to Telegram after %d attempts: %s",
self.retries,
response.text,
)
else:
time.sleep(self.delay)
else:
return
def format(self, record: logging.LogRecord) -> str:
try:
asctime = datetime.datetime.fromtimestamp(
record.created,
tz=get_current_timezone(),
).strftime("%Y-%m-%d %H:%M:%S %Z")
level_emoji = LEVEL_EMOJIS.get(record.levelname, "")
formatted_message = self.template.format(
levelname=f"{level_emoji} {record.levelname}",
correlation_id=getattr(record, "correlation_id", "N/A"),
asctime=asctime,
name=record.name,
pathname=record.pathname,
lineno=record.lineno,
message=record.getMessage(),
)
if record.exc_info:
formatted_message += self._format_exception(record.exc_info)
formatted_message += (
f"\n#{record.levelname.lower()} "
f"#{record.name.replace('.', '_')}"
)
if hasattr(record, "correlation_id"):
formatted_message += f" #{record.correlation_id}"
except Exception as format_error: # noqa: BLE001
TELEGRAM_LOG_HANDLER.exception(
"Error formatting log record: %s",
format_error,
)
return f"Error formatting log record: {format_error}"
else:
return formatted_message
@staticmethod
def _format_exception(exc_info: Exception) -> str:
exc_text = "".join(traceback.format_exception(*exc_info))
return (
f"\n<pre><code class='language-traceback'>{exc_text}</code></pre>"
)
@classmethod
def shutdown_executor(cls) -> None:
cls._executor.shutdown(wait=True)
solution/services/backend/config/settings.py
+2 -39
View File
@@ -41,7 +41,8 @@ YANDEX_CLOUD_INTEGRATION_ENABLED = (
YANDEX_CLOUD_FOLDER_ID and YANDEX_CLOUD_API_KEY
)
# Register healthcheck
# Register healthchecks
plugin_dir.register(YandexAIHealthCheck)
@@ -300,26 +301,6 @@ USE_X_FORWARDED_PORT = False
WSGI_APPLICATION = "config.wsgi.application"
# Notifiers
# Telegram
NOTIFIER_TELEGRAM_BOT_TOKEN = env(
"DJANGO_NOTIFIER_TELEGRAM_BOT_TOKEN",
default=None,
)
NOTIFIER_TELEGRAM_CHAT_ID = env(
"DJANGO_NOTIFIER_TELEGRAM_CHAT_ID",
default=None,
)
NOTIFIER_TELEGRAM_THREAD_ID = env(
"DJANGO_NOTIFIER_TELEGRAM_THREAD_ID",
default=None,
)
# Logging
LOGGER_NAME = "adnova"
@@ -435,24 +416,6 @@ LOGGING = {
LOGGING_CONFIG = "logging.config.dictConfig"
if NOTIFIER_TELEGRAM_BOT_TOKEN and NOTIFIER_TELEGRAM_CHAT_ID:
LOGGING_HANDLERS["telegram"] = {
"class": "config.notifiers.telegram.LoggingHandler",
"level": "INFO",
"filters": ["require_debug_false"],
"token": NOTIFIER_TELEGRAM_BOT_TOKEN,
"chat_id": NOTIFIER_TELEGRAM_CHAT_ID,
"thread_id": NOTIFIER_TELEGRAM_THREAD_ID,
"retries": 5,
"delay": 2,
"timeout": 5,
}
LOGGING_LOGGERS["django"]["handlers"].append("telegram")
LOGGING_LOGGERS["django.request"]["handlers"].append("telegram")
LOGGING_LOGGERS["health-check"]["handlers"].append("telegram")
LOGGING_LOGGERS[LOGGER_NAME]["handlers"].append("telegram")
# Models
ABSOLUTE_URL_OVERRIDES: dict[str, Callable] = {}
solution/services/backend/integrations/yandexai/healthcheck.py
+2
View File
@@ -16,8 +16,10 @@ class YandexAIHealthCheck(BaseHealthCheckBackend):
result = sdk.models.completions(
"yandexgpt-lite", model_version="latest"
).tokenize("ping")
if not result:
self.add_error("YandexAI API is unaccessible")
except YCloudMLError:
self.add_error("YandexAI API is unaccessible")
solution/tests/README.md
+2 -2
View File
@@ -1,6 +1,6 @@
# AdNova Tests
There is `unit` tests and `e2e` tests available, unit tests are placed all around `backend` serivce folder and `e2e` tests placed [here](./e2e/)
There is `unit` and `e2e` tests available, unit tests are placed all around `backend` serivce folder and `e2e` tests placed [here](./e2e/)
## Running unit tests
@@ -12,7 +12,7 @@ See [services/backend/README.md](../services/backend/README.md#testing)
### Backend service
-image-
![backend coverage](../assets/images/backend-coverage.png)
## Running e2e tests
solution/tests/e2e/README.md
+1 -1
View File
@@ -31,7 +31,7 @@ cd devitq/solution/tests/e2e
uv sync --no-dev
```
## Customize environment
## Customize environment (optional)
```bash
cp .env.template .env
solution/tests/e2e/pyproject.toml
-1
View File
@@ -107,4 +107,3 @@ unfixable = []
[tool.ruff.lint.pylint]
max-args = 6
solution/tests/e2e/tests/test_ad_text_generation.py
+9 -5
View File
@@ -8,8 +8,15 @@ logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Tests integration between: backend, redis, yandexgpt and celery
def test_generate_ad_text(client: Client) -> None:
"""
Tests integration between:
- backend
- redis
- yandexgpt
- celery
"""
payload = {
"advertiser_name": "Центральный Университет",
"ad_title": "Всероссийский кейс-чемпионат DEADLINE",
@@ -25,10 +32,7 @@ def test_generate_ad_text(client: Client) -> None:
while True:
result_response = client.get(f"/generate/ad_text/{task_id}/result")
assert (
result_response.status_code == status.OK
or result_response.status_code == status.NOT_FOUND
)
assert result_response.status_code in (status.OK, status.NOT_FOUND)
result_data = result_response.json()
if (
solution/tests/e2e/tests/test_backend_health.py
+1 -1
View File
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def test_healthcheck(client: Client) -> None:
"""
Checks that backend can use theese services:
Tests integration between:
- redis
- celery
- postgres