mirror of
https://gitlab.com/megazordpobeda/DataRush.git
synced 2026-05-23 15:37:10 +00:00
Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -54,6 +54,24 @@ build_backend-staticfiles:
|
|||||||
DOCKERFILE_PATH: "Dockerfile.staticfiles"
|
DOCKERFILE_PATH: "Dockerfile.staticfiles"
|
||||||
IMAGE_NAME: "$CI_REGISTRY_IMAGE/backend-staticfiles"
|
IMAGE_NAME: "$CI_REGISTRY_IMAGE/backend-staticfiles"
|
||||||
|
|
||||||
|
build_checker:
|
||||||
|
<<: *build-template
|
||||||
|
rules:
|
||||||
|
- if: '$CI_COMMIT_REF_NAME == "master"'
|
||||||
|
variables:
|
||||||
|
CONTEXT: "${CI_PROJECT_DIR}/services/checker"
|
||||||
|
DOCKERFILE_PATH: "Dockerfile"
|
||||||
|
IMAGE_NAME: "$CI_REGISTRY_IMAGE/checker"
|
||||||
|
|
||||||
|
build_custom-python:
|
||||||
|
<<: *build-template
|
||||||
|
rules:
|
||||||
|
- if: '$CI_COMMIT_REF_NAME == "master"'
|
||||||
|
variables:
|
||||||
|
CONTEXT: "${CI_PROJECT_DIR}/services/checker"
|
||||||
|
DOCKERFILE_PATH: "Dockerfile.checker"
|
||||||
|
IMAGE_NAME: "$CI_REGISTRY_IMAGE/custom-python"
|
||||||
|
|
||||||
build_docs:
|
build_docs:
|
||||||
<<: *build-template
|
<<: *build-template
|
||||||
rules:
|
rules:
|
||||||
|
|||||||
@@ -370,6 +370,23 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
shm_size: 4mb
|
shm_size: 4mb
|
||||||
|
|
||||||
|
checker:
|
||||||
|
image: gitlab.prodcontest.ru:5050/team-15/project/checker:latest
|
||||||
|
build:
|
||||||
|
context: ./services/checker
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- name: web
|
||||||
|
target: 8000
|
||||||
|
published: 8009
|
||||||
|
host_ip: 0.0.0.0
|
||||||
|
protocol: tcp
|
||||||
|
volumes:
|
||||||
|
- type: bind
|
||||||
|
source: /var/run/docker.sock
|
||||||
|
target: /var/run/docker.sock
|
||||||
|
|
||||||
proxy:
|
proxy:
|
||||||
image: docker.io/nginx:1.27-alpine3.21
|
image: docker.io/nginx:1.27-alpine3.21
|
||||||
configs:
|
configs:
|
||||||
|
|||||||
@@ -18,3 +18,5 @@ MINIO_ENDPOINT=minio:9000
|
|||||||
MINIO_CUSTOM_ENDPOINT_URL=https://prod-team-15-minio-2pc0i3lc.final.prodcontest.ru
|
MINIO_CUSTOM_ENDPOINT_URL=https://prod-team-15-minio-2pc0i3lc.final.prodcontest.ru
|
||||||
MINIO_ACCESS_KEY=admin
|
MINIO_ACCESS_KEY=admin
|
||||||
MINIO_SECRET_KEY=password
|
MINIO_SECRET_KEY=password
|
||||||
|
|
||||||
|
CHECKER_API_ENDPOINT=http://checker:8000
|
||||||
|
|||||||
@@ -24,4 +24,6 @@ FROM docker.io/nginx:latest
|
|||||||
|
|
||||||
COPY --from=builder /app/static /usr/share/nginx/html
|
COPY --from=builder /app/static /usr/share/nginx/html
|
||||||
|
|
||||||
|
COPY ../checker/checker_requirements.txt .
|
||||||
|
|
||||||
CMD ["nginx", "-g", "daemon off;"]
|
CMD ["nginx", "-g", "daemon off;"]
|
||||||
|
|||||||
@@ -155,5 +155,5 @@ def get_submissions_history(request, competition_id: UUID, task_id: UUID):
|
|||||||
def get_task_attachments(request, competition_id: UUID, task_id: UUID):
|
def get_task_attachments(request, competition_id: UUID, task_id: UUID):
|
||||||
task = get_object_or_404(CompetitionTask, id=task_id)
|
task = get_object_or_404(CompetitionTask, id=task_id)
|
||||||
return status.OK, CompetitionTaskAttachment.objects.filter(
|
return status.OK, CompetitionTaskAttachment.objects.filter(
|
||||||
competition_id=competition_id, task=task, user=request.auth
|
task=task
|
||||||
)
|
).all()
|
||||||
|
|||||||
@@ -1,155 +1,50 @@
|
|||||||
import ast
|
import requests
|
||||||
import contextlib
|
from celery import shared_task
|
||||||
import hashlib
|
from django.core.files.base import ContentFile
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from io import StringIO
|
|
||||||
|
|
||||||
from apps.task.models import CompetitionTaskSubmission
|
from django.conf import settings
|
||||||
from config.celery import app
|
|
||||||
|
|
||||||
ALLOWED_MODULES = {
|
|
||||||
"pandas",
|
|
||||||
"numpy",
|
|
||||||
"matplotlib",
|
|
||||||
"seaborn",
|
|
||||||
"scipy",
|
|
||||||
"sklearn",
|
|
||||||
"datetime",
|
|
||||||
"json",
|
|
||||||
"csv",
|
|
||||||
"math",
|
|
||||||
"statistics",
|
|
||||||
"statsmodels",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class SecurityException(Exception):
|
@shared_task(bind=True, max_retries=3)
|
||||||
pass
|
def analyze_data_task(self, submission_id):
|
||||||
|
from .models import CompetitionTaskSubmission
|
||||||
|
|
||||||
|
submission = CompetitionTaskSubmission.objects.get(id=submission_id)
|
||||||
def validate_code(code_str):
|
|
||||||
try:
|
try:
|
||||||
tree = ast.parse(code_str)
|
code = submission.content.read().decode()
|
||||||
except SyntaxError as e:
|
files = [
|
||||||
raise SecurityException(f"Syntax error: {e!s}")
|
(f.name, f.file.open("rb"))
|
||||||
|
for f in submission.task.attachments.filter(public=True)
|
||||||
|
]
|
||||||
|
|
||||||
class ImportVisitor(ast.NodeVisitor):
|
response = requests.post(
|
||||||
def visit_Import(self, node):
|
f"{settings.CHECKER_API_ENDPOINT}/execute",
|
||||||
for alias in node.names:
|
files=[("files", (f.name, f)) for f in files]
|
||||||
module = alias.name.split(".")[0]
|
+ [
|
||||||
if module not in ALLOWED_MODULES:
|
("code", code),
|
||||||
raise SecurityException(f"Disallowed import: {module}")
|
("expected_hash", submission.task.correct_answer_hash),
|
||||||
|
],
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
result = response.json()
|
||||||
|
|
||||||
def visit_ImportFrom(self, node):
|
submission.stdout.save("output.txt", ContentFile(result["output"]))
|
||||||
if node.module:
|
submission.result = {
|
||||||
module = node.module.split(".")[0]
|
"correct": result["hash_match"],
|
||||||
if module not in ALLOWED_MODULES:
|
"result_hash": result["result_hash"],
|
||||||
raise SecurityException(
|
"error": result.get("error"),
|
||||||
f"Disallowed import from: {module}"
|
|
||||||
)
|
|
||||||
|
|
||||||
class SecurityVisitor(ast.NodeVisitor):
|
|
||||||
def generic_visit(self, node):
|
|
||||||
if isinstance(node, (ast.Call, ast.Attribute)):
|
|
||||||
if "system" in getattr(node, "attr", ""):
|
|
||||||
raise SecurityException("Dangerous system call detected")
|
|
||||||
super().generic_visit(node)
|
|
||||||
|
|
||||||
try:
|
|
||||||
ImportVisitor().visit(tree)
|
|
||||||
SecurityVisitor().visit(tree)
|
|
||||||
except SecurityException:
|
|
||||||
raise
|
|
||||||
except Exception as e:
|
|
||||||
raise SecurityException(f"Security check failed: {e!s}")
|
|
||||||
|
|
||||||
|
|
||||||
def secure_exec(code_str, result_path, input_files=None):
|
|
||||||
original_dir = os.getcwd()
|
|
||||||
original_stdout = sys.stdout
|
|
||||||
sys.stdout = captured_stdout = StringIO()
|
|
||||||
result_content = None
|
|
||||||
|
|
||||||
if input_files is None:
|
|
||||||
input_files = []
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
|
||||||
try:
|
|
||||||
os.chdir(temp_dir)
|
|
||||||
|
|
||||||
for file in input_files:
|
|
||||||
file_path = os.path.join(temp_dir, file["bind_at"])
|
|
||||||
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
|
||||||
with open(file_path, "wb") as f:
|
|
||||||
f.write(file["content"])
|
|
||||||
|
|
||||||
restricted_globals = {
|
|
||||||
"__builtins__": {
|
|
||||||
"open": open,
|
|
||||||
"print": print,
|
|
||||||
"str": str,
|
|
||||||
"int": int,
|
|
||||||
"float": float,
|
|
||||||
"bool": bool,
|
|
||||||
"list": list,
|
|
||||||
"dict": dict,
|
|
||||||
"tuple": tuple,
|
|
||||||
"set": set,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
exec(code_str, restricted_globals)
|
|
||||||
|
|
||||||
if result_path == "stdout":
|
|
||||||
result_content = captured_stdout.getvalue().encode("utf-8")
|
|
||||||
else:
|
|
||||||
with open(result_path, "rb") as f:
|
|
||||||
result_content = f.read()
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
raise RuntimeError(f"Execution error: {e!s}")
|
|
||||||
finally:
|
|
||||||
os.chdir(original_dir)
|
|
||||||
sys.stdout = original_stdout
|
|
||||||
|
|
||||||
return result_content
|
|
||||||
|
|
||||||
|
|
||||||
@app.task(bind=True)
|
|
||||||
def analyze_data_task(
|
|
||||||
self,
|
|
||||||
code_str,
|
|
||||||
result_path,
|
|
||||||
expected_file_link,
|
|
||||||
submission_id,
|
|
||||||
input_files=[],
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
validate_code(code_str)
|
|
||||||
|
|
||||||
result_content = secure_exec(code_str, result_path, input_files)
|
|
||||||
|
|
||||||
result_hash = hashlib.sha256(result_content).hexdigest()
|
|
||||||
expected_hash = hashlib.sha256(expected_bytes).hexdigest()
|
|
||||||
|
|
||||||
with contextlib.suppress(CompetitionTaskSubmission.DoesNotExist):
|
|
||||||
submission = CompetitionTaskSubmission.objects.get(
|
|
||||||
id=submission_id
|
|
||||||
)
|
|
||||||
submission.result = {"correct": True}
|
|
||||||
|
|
||||||
return {
|
|
||||||
"success": True,
|
|
||||||
"match": result_hash == expected_hash,
|
|
||||||
"result_hash": result_hash,
|
|
||||||
"expected_hash": expected_hash,
|
|
||||||
}
|
}
|
||||||
|
submission.earned_points = (
|
||||||
|
submission.task.points if result["hash_match"] else 0
|
||||||
|
)
|
||||||
|
submission.status = CompetitionTaskSubmission.StatusChoices.CHECKED
|
||||||
|
|
||||||
except SecurityException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
return {"success": False, "error": f"Security violation: {e!s}"}
|
self.retry(countdown=2**self.request.retries)
|
||||||
except RuntimeError as e:
|
|
||||||
return {"success": False, "error": f"Execution error: {e!s}"}
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"success": False, "error": f"Unexpected error: {e!s}"}
|
submission.result = {"error": str(e)}
|
||||||
|
submission.status = CompetitionTaskSubmission.StatusChoices.CHECKED
|
||||||
|
submission.earned_points = 0
|
||||||
|
finally:
|
||||||
|
submission.save()
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ from pathlib import Path
|
|||||||
|
|
||||||
import django_stubs_ext
|
import django_stubs_ext
|
||||||
import environ
|
import environ
|
||||||
|
from health_check.plugins import plugin_dir
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
from integrations.checker.healthcheck import CheckerHealthCheck
|
||||||
|
|
||||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||||
|
|
||||||
@@ -30,18 +32,12 @@ ALLOWED_HOSTS = env(
|
|||||||
|
|
||||||
# Integrations
|
# Integrations
|
||||||
|
|
||||||
YANDEX_CLOUD_FOLDER_ID = env("YANDEX_CLOUD_FOLDER_ID", default=None)
|
CHECKER_API_ENDPOINT = env("CHECKER_API_ENDPOINT", default=None)
|
||||||
|
|
||||||
YANDEX_CLOUD_API_KEY = env("YANDEX_CLOUD_API_KEY", default=None)
|
|
||||||
|
|
||||||
YANDEX_CLOUD_INTEGRATION_ENABLED = (
|
|
||||||
YANDEX_CLOUD_FOLDER_ID and YANDEX_CLOUD_API_KEY
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# Register healthchecks
|
# Register healthchecks
|
||||||
|
|
||||||
# plugin_dir.register(SomeHealthCheckClass)
|
plugin_dir.register(CheckerHealthCheck)
|
||||||
|
|
||||||
|
|
||||||
# Caching
|
# Caching
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
from http import HTTPStatus as status
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from django.conf import settings
|
||||||
|
from health_check.backends import BaseHealthCheckBackend
|
||||||
|
|
||||||
|
|
||||||
|
class CheckerHealthCheck(BaseHealthCheckBackend):
|
||||||
|
critical_service = False
|
||||||
|
|
||||||
|
def check_status(self) -> None:
|
||||||
|
try:
|
||||||
|
response = httpx.get(
|
||||||
|
f"{settings.CHECKER_API_ENDPOINT}/ping", timeout=1
|
||||||
|
)
|
||||||
|
if response.status_code >= status.INTERNAL_SERVER_ERROR:
|
||||||
|
self.add_error("Checker service is unaccessible")
|
||||||
|
except httpx.HTTPError:
|
||||||
|
self.add_error("Checker service is unaccessible")
|
||||||
|
|
||||||
|
def identifier(self) -> str:
|
||||||
|
return self.__class__.__name__
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
.python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
Pipfile.lock
|
||||||
|
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
uv.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
poetry.lock
|
||||||
|
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
#pdm.lock
|
||||||
|
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||||
|
# in version control.
|
||||||
|
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||||
|
.pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
.idea/
|
||||||
|
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
|
||||||
|
# Ruff files
|
||||||
|
.ruff_cache
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# Stage 1: Install dependencies
|
||||||
|
FROM docker.io/python:3.11-alpine3.20 AS builder
|
||||||
|
|
||||||
|
COPY --from=ghcr.io/astral-sh/uv:0.4.30 /uv /uvx /bin/
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONOPTIMIZE=2 \
|
||||||
|
UV_COMPILE_BYTECODE=1 \
|
||||||
|
UV_PROJECT_ENVIRONMENT=/opt/venv
|
||||||
|
|
||||||
|
COPY pyproject.toml .
|
||||||
|
|
||||||
|
RUN uv sync --no-dev --no-install-project --no-cache
|
||||||
|
|
||||||
|
|
||||||
|
# Stage 2: Start the application
|
||||||
|
FROM docker.io/python:3.11-alpine3.20
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY --from=builder /opt/venv /opt/venv
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN adduser -D -g '' app && chown -R app:app ./
|
||||||
|
|
||||||
|
USER app
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONOPTIMIZE=2 \
|
||||||
|
PATH="/opt/venv/bin:$PATH"
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --start-interval=2s --retries=3 \
|
||||||
|
CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:8000/ping || exit 1
|
||||||
|
|
||||||
|
CMD uvicorn main:app --host 0.0.0.0 --port 8000
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
FROM docker.io/python:3.11-slim
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
PIP_NO_CACHE_DIR=1
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY checker_requirements.txt .
|
||||||
|
|
||||||
|
RUN pip install --no-cache-dir -r checker_requirements.txt
|
||||||
|
|
||||||
|
CMD ["python"]
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
# DataRush Checker
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
Ensure you have the following installed on your system:
|
||||||
|
|
||||||
|
- [Python](https://www.python.org/) (>=3.10,<3.12)
|
||||||
|
- [uv](https://docs.astral.sh/uv/)
|
||||||
|
- [Docker](https://www.docker.com/) (for containerized setup)
|
||||||
|
|
||||||
|
## Basic setup
|
||||||
|
|
||||||
|
### Installation
|
||||||
|
|
||||||
|
#### Clone the project
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone git@gitlab.prodcontest.ru:team-15/project.git
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Go to the project directory
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd project/services/checker
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Install dependencies
|
||||||
|
|
||||||
|
##### For dev environment
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv sync --all-extras
|
||||||
|
```
|
||||||
|
|
||||||
|
##### For prod environment
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv sync --no-dev
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Running
|
||||||
|
|
||||||
|
##### Apply migrations
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run python manage.py migrate
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Start celery worker
|
||||||
|
|
||||||
|
```bash
|
||||||
|
celery -A config worker -l INFO
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Start server
|
||||||
|
|
||||||
|
In dev mode:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run python manage.py runserver
|
||||||
|
```
|
||||||
|
|
||||||
|
In prod mode:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run gunicorn config.wsgi
|
||||||
|
```
|
||||||
|
|
||||||
|
## Containerized setup
|
||||||
|
|
||||||
|
### Clone the project
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone git@gitlab.prodcontest.ru:team-15/project.git
|
||||||
|
```
|
||||||
|
|
||||||
|
### Go to the project directory
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd project/services/checker
|
||||||
|
```
|
||||||
|
|
||||||
|
### Build docker image
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t datarush-checker .
|
||||||
|
```
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
pandas==2.2.3
|
||||||
|
numpy==2.2.3
|
||||||
|
matplotlib==3.10.1
|
||||||
|
scipy==1.15.2
|
||||||
|
scikit-learn==1.6.1
|
||||||
|
seaborn==0.13.2
|
||||||
|
statsmodels==0.14.4
|
||||||
@@ -0,0 +1,289 @@
|
|||||||
|
from fastapi import FastAPI, HTTPException, status
|
||||||
|
from pydantic import BaseModel, Field, HttpUrl, constr
|
||||||
|
import aiohttp
|
||||||
|
import asyncio
|
||||||
|
import docker
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import base64
|
||||||
|
import tempfile
|
||||||
|
import logging
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
import re
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
docker_client = docker.from_env()
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
|
DOCKER_IMAGE = "gitlab.python:3-slim"
|
||||||
|
CONTAINER_TIMEOUT = 60
|
||||||
|
MAX_FILE_SIZE = 4 * 1024 * 1024
|
||||||
|
ALLOWED_FILENAME_CHARS = r"[^a-zA-Z0-9_\-.]"
|
||||||
|
|
||||||
|
|
||||||
|
class FileDetails(BaseModel):
|
||||||
|
url: HttpUrl = Field(
|
||||||
|
..., description="URL to download the file from (supports HTTP/HTTPS)"
|
||||||
|
)
|
||||||
|
bind_path: str = Field(
|
||||||
|
...,
|
||||||
|
description="Container path to bind the file (absolute)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionRequest(BaseModel):
|
||||||
|
code: str = Field(..., description="Base64 encoded Python code to execute")
|
||||||
|
answer_file_path: str = Field(
|
||||||
|
"stdout", description="Base64 encoded path to result file or 'stdout'"
|
||||||
|
)
|
||||||
|
expected_hash: str | None = Field(
|
||||||
|
None, description="Optional SHA-256 hash of expected output"
|
||||||
|
)
|
||||||
|
files: list[FileDetails] = Field(
|
||||||
|
[], description="List of files to mount in container"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionResponse(BaseModel):
|
||||||
|
success: bool = Field(..., description="Execution success status")
|
||||||
|
hash_match: bool | None = Field(
|
||||||
|
None, description="Output hash matches expected (if provided)"
|
||||||
|
)
|
||||||
|
output: str = Field(..., description="Captured stdout or file contents")
|
||||||
|
result_hash: str = Field(..., description="SHA-256 hash of output")
|
||||||
|
error: str = Field(..., description="Execution errors or stderr")
|
||||||
|
|
||||||
|
|
||||||
|
class HealthCheckResponse(BaseModel):
|
||||||
|
status: str = Field(..., description="Service health status")
|
||||||
|
docker: str = Field(..., description="Docker daemon status")
|
||||||
|
|
||||||
|
|
||||||
|
def decode_base64(encoded_str: str, field_name: str) -> str:
|
||||||
|
try:
|
||||||
|
return base64.b64decode(encoded_str).decode("utf-8")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Base64 decode failed for {field_name}: {str(e)}")
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=f"Invalid Base64 in {field_name}",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def sanitize_filename(url: str) -> str:
|
||||||
|
parsed = urlparse(url)
|
||||||
|
base_name = os.path.basename(parsed.path)
|
||||||
|
|
||||||
|
if not base_name:
|
||||||
|
base_name = "file"
|
||||||
|
|
||||||
|
clean = re.sub(ALLOWED_FILENAME_CHARS, "", base_name)[:255]
|
||||||
|
return clean or "file"
|
||||||
|
|
||||||
|
|
||||||
|
async def download_file(
|
||||||
|
session: aiohttp.ClientSession, url: str, dest_path: str
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
async with session.get(
|
||||||
|
url, timeout=aiohttp.ClientTimeout(total=30)
|
||||||
|
) as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=f"Failed to download {url} - Status {resp.status}",
|
||||||
|
)
|
||||||
|
|
||||||
|
content = b""
|
||||||
|
async for chunk in resp.content.iter_chunked(8192):
|
||||||
|
content += chunk
|
||||||
|
if len(content) > MAX_FILE_SIZE:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||||
|
detail="File size exceeds 4MB limit",
|
||||||
|
)
|
||||||
|
|
||||||
|
with open(dest_path, "wb") as f:
|
||||||
|
f.write(content)
|
||||||
|
logger.info(f"Downloaded {url} to {dest_path}")
|
||||||
|
|
||||||
|
except aiohttp.ClientError as e:
|
||||||
|
logger.error(f"Download error for {url}: {str(e)}")
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=f"Download failed: {str(e)}",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run_container_safely(
|
||||||
|
tmp_dir: str,
|
||||||
|
command: list[str],
|
||||||
|
bound_files: dict[str, str],
|
||||||
|
timeout: int = CONTAINER_TIMEOUT,
|
||||||
|
) -> dict:
|
||||||
|
container = None
|
||||||
|
try:
|
||||||
|
volumes = {tmp_dir: {"bind": "/execution", "mode": "rw"}}
|
||||||
|
for host_path, container_path in bound_files.items():
|
||||||
|
volumes[host_path] = {"bind": container_path, "mode": "ro"}
|
||||||
|
|
||||||
|
container = docker_client.containers.run(
|
||||||
|
image=DOCKER_IMAGE,
|
||||||
|
command=command,
|
||||||
|
volumes=volumes,
|
||||||
|
working_dir="/execution",
|
||||||
|
stdout=True,
|
||||||
|
stderr=True,
|
||||||
|
detach=True,
|
||||||
|
mem_limit="100m",
|
||||||
|
network_mode="none",
|
||||||
|
cpu_period=100000,
|
||||||
|
cpu_quota=50000,
|
||||||
|
user="root",
|
||||||
|
security_opt=["no-new-privileges"],
|
||||||
|
)
|
||||||
|
|
||||||
|
exit_code = container.wait(timeout=timeout)["StatusCode"]
|
||||||
|
stdout = container.logs(stdout=True, stderr=False).decode().strip()
|
||||||
|
stderr = container.logs(stdout=False, stderr=True).decode().strip()
|
||||||
|
|
||||||
|
return {"stdout": stdout, "stderr": stderr, "status": exit_code}
|
||||||
|
|
||||||
|
except docker.errors.DockerException as e:
|
||||||
|
logger.error(f"Docker error: {str(e)}")
|
||||||
|
return {
|
||||||
|
"stdout": "",
|
||||||
|
"stderr": f"Container error: {str(e)}",
|
||||||
|
"status": -1,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
if container:
|
||||||
|
try:
|
||||||
|
container.remove(force=True)
|
||||||
|
except docker.errors.DockerException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/execute", response_model=ExecutionResponse)
|
||||||
|
async def execute_code(request: ExecutionRequest) -> ExecutionResponse:
|
||||||
|
try:
|
||||||
|
code = decode_base64(request.code, "code")
|
||||||
|
answer_path = (
|
||||||
|
decode_base64(request.answer_file_path, "answer_file_path")
|
||||||
|
if request.answer_file_path != "stdout"
|
||||||
|
else "stdout"
|
||||||
|
)
|
||||||
|
except HTTPException as e:
|
||||||
|
return ExecutionResponse(
|
||||||
|
success=False,
|
||||||
|
output="",
|
||||||
|
result_hash="",
|
||||||
|
error=e.detail,
|
||||||
|
hash_match=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
if answer_path != "stdout":
|
||||||
|
if os.path.isabs(answer_path) or not validate_file_path(answer_path):
|
||||||
|
return ExecutionResponse(
|
||||||
|
success=False,
|
||||||
|
output="",
|
||||||
|
result_hash="",
|
||||||
|
error="Invalid answer file path",
|
||||||
|
hash_match=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||||
|
bound_files = {}
|
||||||
|
if request.files:
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
download_tasks = []
|
||||||
|
for file in request.files:
|
||||||
|
filename = sanitize_filename(str(file.url))
|
||||||
|
dest_path = os.path.join(tmp_dir, filename)
|
||||||
|
bound_files[dest_path] = file.bind_path
|
||||||
|
download_tasks.append(
|
||||||
|
download_file(session, str(file.url), dest_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
await asyncio.gather(*download_tasks)
|
||||||
|
except HTTPException as e:
|
||||||
|
return ExecutionResponse(
|
||||||
|
success=False,
|
||||||
|
output="",
|
||||||
|
result_hash="",
|
||||||
|
error=e.detail,
|
||||||
|
hash_match=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
code_path = os.path.join(tmp_dir, "submission.py")
|
||||||
|
with open(code_path, "w") as f:
|
||||||
|
f.write(code)
|
||||||
|
os.chmod(code_path, 0o444)
|
||||||
|
|
||||||
|
if answer_path == "stdout":
|
||||||
|
cmd = ["python", "submission.py"]
|
||||||
|
else:
|
||||||
|
cmd = [
|
||||||
|
"sh",
|
||||||
|
"-c",
|
||||||
|
f"python submission.py && cat {answer_path} || echo 'EXECUTION_FAILED'",
|
||||||
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = await asyncio.to_thread(
|
||||||
|
run_container_safely,
|
||||||
|
tmp_dir,
|
||||||
|
cmd,
|
||||||
|
bound_files,
|
||||||
|
CONTAINER_TIMEOUT,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Container execution failed: {str(e)}")
|
||||||
|
return ExecutionResponse(
|
||||||
|
success=False,
|
||||||
|
output="",
|
||||||
|
result_hash="",
|
||||||
|
error=f"Execution failed: {str(e)}",
|
||||||
|
hash_match=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
output = result["stdout"]
|
||||||
|
error = result["stderr"]
|
||||||
|
success = result["status"] == 0
|
||||||
|
|
||||||
|
if answer_path != "stdout" and not output:
|
||||||
|
error += "\nNo output captured - check answer file path"
|
||||||
|
|
||||||
|
result_hash = hashlib.sha256(output.encode()).hexdigest()
|
||||||
|
|
||||||
|
return ExecutionResponse(
|
||||||
|
success=success,
|
||||||
|
hash_match=(
|
||||||
|
result_hash == request.expected_hash
|
||||||
|
if request.expected_hash
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
output=output[:5000],
|
||||||
|
result_hash=result_hash,
|
||||||
|
error=error[:5000],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health", response_model=HealthCheckResponse)
|
||||||
|
async def health_check() -> HealthCheckResponse:
|
||||||
|
try:
|
||||||
|
docker_client.ping()
|
||||||
|
return HealthCheckResponse(status="healthy", docker="connected")
|
||||||
|
except docker.errors.DockerException:
|
||||||
|
return HealthCheckResponse(status="degraded", docker="unavailable")
|
||||||
|
|
||||||
|
|
||||||
|
def validate_file_path(path: str) -> bool:
|
||||||
|
return (
|
||||||
|
not os.path.isabs(path)
|
||||||
|
and os.path.basename(path) == path
|
||||||
|
and all(c.isalnum() or c in {"_", "-", "."} for c in path)
|
||||||
|
)
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[project]
|
||||||
|
name = "checker"
|
||||||
|
version = "0.1.0"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.11"
|
||||||
|
dependencies = [
|
||||||
|
"aiohttp>=3.11.13",
|
||||||
|
"docker>=7.1.0",
|
||||||
|
"fastapi>=0.115.11",
|
||||||
|
"python-multipart>=0.0.20",
|
||||||
|
"regex>=2024.11.6",
|
||||||
|
"uvicorn>=0.34.0",
|
||||||
|
]
|
||||||
Executable
+8
@@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
GREEN='\033[1;32m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
uvx ruff format .
|
||||||
|
uvx ruff check . --fix
|
||||||
|
printf "${GREEN}Linters/formatters runned${NC}\n"
|
||||||
@@ -45,6 +45,7 @@ const CompetitionSession = () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const handleSubmit = () => {
|
const handleSubmit = () => {
|
||||||
|
console.log(currentTask, competitionId, answer)
|
||||||
if (!currentTask || !competitionId || !answer.trim()) return;
|
if (!currentTask || !competitionId || !answer.trim()) return;
|
||||||
submitMutation.mutate();
|
submitMutation.mutate();
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -20,11 +20,10 @@ export const submitTaskSolution = async (
|
|||||||
solution: string | File
|
solution: string | File
|
||||||
) => {
|
) => {
|
||||||
const endpoint = `/competitions/${competitionId}/tasks/${taskId}/submit`;
|
const endpoint = `/competitions/${competitionId}/tasks/${taskId}/submit`;
|
||||||
|
|
||||||
if (typeof solution === 'string') {
|
if (typeof solution === 'string') {
|
||||||
return await userFetch(endpoint, {
|
return await userFetch(endpoint, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
body: { answer: solution }
|
body: { content: solution }
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
const formData = new FormData();
|
const formData = new FormData();
|
||||||
|
|||||||
Reference in New Issue
Block a user