diff --git a/Dockerfile b/Dockerfile index ceecf93..9442b8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,4 +11,4 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . # Запуск приложения (замени app.py на свой файл) -CMD ["python", "main.py"] +CMD ["python", "aiws.py"] diff --git a/main.py b/aiws.py similarity index 100% rename from main.py rename to aiws.py diff --git a/api/endpoints/assets_router.py b/api/endpoints/assets_router.py index 4c25cd5..9b6d49b 100644 --- a/api/endpoints/assets_router.py +++ b/api/endpoints/assets_router.py @@ -1,17 +1,21 @@ -from typing import List, Optional +from typing import List, Optional, Dict, Any from aiogram.types import BufferedInputFile +from bson import ObjectId from fastapi import APIRouter, UploadFile, File, Form, Depends from fastapi.openapi.models import MediaType +from motor.motor_asyncio import AsyncIOMotorClient +from pymongo import MongoClient from starlette import status from starlette.exceptions import HTTPException from starlette.requests import Request from starlette.responses import Response, JSONResponse +from adapters.s3_adapter import S3Adapter from api.models.AssetDTO import AssetsResponse, AssetResponse from models.Asset import Asset, AssetType, AssetContentType from repos.dao import DAO -from api.dependency import get_dao +from api.dependency import get_dao, get_mongo_client, get_s3_adapter import asyncio import logging @@ -51,6 +55,119 @@ async def get_asset( return Response(content=content, media_type=media_type, headers=headers) +@router.delete("/orphans", dependencies=[Depends(get_current_user)]) +async def delete_orphan_assets_from_minio( + mongo: AsyncIOMotorClient = Depends(get_mongo_client), + minio_client: S3Adapter = Depends(get_s3_adapter), + *, + assets_collection: str = "assets", + generations_collection: str = "generations", + asset_type: Optional[str] = "generated", + project_id: Optional[str] = None, + dry_run: bool = True, + mark_assets_deleted: bool = False, + batch_size: int = 500, +) -> Dict[str, Any]: + db = mongo['bot_db'] # БД уже выбрана в get_mongo_client + assets = db[assets_collection] + + match_assets: Dict[str, Any] = {} + if asset_type is not None: + match_assets["type"] = asset_type + if project_id is not None: + match_assets["project_id"] = project_id + + pipeline: List[Dict[str, Any]] = [ + {"$match": match_assets} if match_assets else {"$match": {}}, + { + "$lookup": { + "from": generations_collection, + "let": {"assetIdStr": {"$toString": "$_id"}}, + "pipeline": [ + # считаем "живыми" те, где is_deleted != True (т.е. false или поля нет) + {"$match": {"is_deleted": {"$ne": True}}}, + { + "$match": { + "$expr": { + "$in": [ + "$$assetIdStr", + {"$ifNull": ["$result_list", []]}, + ] + } + } + }, + {"$limit": 1}, + ], + "as": "alive_generations", + } + }, + { + "$match": { + "$expr": {"$eq": [{"$size": "$alive_generations"}, 0]} + } + }, + { + "$project": { + "_id": 1, + "minio_object_name": 1, + "minio_thumbnail_object_name": 1, + } + }, + ] + print(pipeline) + cursor = assets.aggregate(pipeline, allowDiskUse=True, batchSize=batch_size) + + deleted_objects = 0 + deleted_assets = 0 + errors: List[Dict[str, Any]] = [] + orphan_asset_ids: List[ObjectId] = [] + + async for asset in cursor: + aid = asset["_id"] + obj = asset.get("minio_object_name") + thumb = asset.get("minio_thumbnail_object_name") + + orphan_asset_ids.append(aid) + + if dry_run: + print(f"[DRY RUN] orphan asset={aid} obj={obj} thumb={thumb}") + continue + + try: + if obj: + await minio_client.delete_file(obj) + deleted_objects += 1 + + if thumb: + await minio_client.delete_file(thumb) + deleted_objects += 1 + + deleted_assets += 1 + + except Exception as e: + errors.append({"asset_id": str(aid), "error": str(e)}) + + if (not dry_run) and mark_assets_deleted and orphan_asset_ids: + res = await assets.update_many( + {"_id": {"$in": orphan_asset_ids}}, + {"$set": {"is_deleted": True}}, + ) + marked = res.modified_count + else: + marked = 0 + + return { + "dry_run": dry_run, + "filter": { + "asset_type": asset_type, + "project_id": project_id, + }, + "orphans_found": len(orphan_asset_ids), + "deleted_assets": deleted_assets, + "deleted_objects": deleted_objects, + "marked_assets_deleted": marked, + "errors": errors, + } @router.delete("/{asset_id}", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(get_current_user)]) async def delete_asset( @@ -189,4 +306,5 @@ async def migrate_to_minio(dao: DAO = Depends(get_dao)): logger.info("Starting migration to MinIO") result = await dao.assets.migrate_to_minio() logger.info(f"Migration result: {result}") - return result \ No newline at end of file + return result + diff --git a/tests/verify_albums_manual.py b/tests/verify_albums_manual.py index c87a933..a4137d8 100644 --- a/tests/verify_albums_manual.py +++ b/tests/verify_albums_manual.py @@ -12,7 +12,7 @@ from models.Generation import Generation, GenerationStatus from models.enums import AspectRatios, Quality # Mock config -# Use the same host as main.py but different DB +# Use the same host as aiws.py but different DB MONGO_HOST = os.getenv("MONGO_HOST", "mongodb://admin:super_secure_password@31.59.58.220:27017") DB_NAME = "bot_db_test_albums"