From c93e577bcf19b446fd394c367085ff223b726b67 Mon Sep 17 00:00:00 2001 From: xds Date: Tue, 17 Feb 2026 12:51:40 +0300 Subject: [PATCH] feat: Implement asset soft deletion with S3 file purging, enhance type safety, and improve error handling in generation and adapter services. --- .../google_adapter.cpython-313.pyc | Bin 8694 -> 8804 bytes .../__pycache__/s3_adapter.cpython-313.pyc | Bin 6690 -> 6690 bytes adapters/google_adapter.py | 12 ++-- adapters/s3_adapter.py | 2 +- aiws.py | 11 +++- api/__pycache__/dependency.cpython-313.pyc | Bin 2727 -> 3002 bytes api/dependency.py | 6 +- .../__pycache__/admin.cpython-313.pyc | Bin 5109 -> 5109 bytes api/endpoints/admin.py | 2 +- api/endpoints/album_router.py | 5 +- .../generation_service.cpython-313.pyc | Bin 28544 -> 30146 bytes api/service/generation_service.py | 47 +++++++++++--- models/Asset.py | 1 + models/__pycache__/Asset.cpython-313.pyc | Bin 3305 -> 3353 bytes repos/__pycache__/assets_repo.cpython-313.pyc | Bin 15403 -> 17866 bytes .../generation_repo.cpython-313.pyc | Bin 7404 -> 9050 bytes repos/assets_repo.py | 60 +++++++++++++++++- repos/generation_repo.py | 42 +++++++++++- 18 files changed, 162 insertions(+), 26 deletions(-) diff --git a/adapters/__pycache__/google_adapter.cpython-313.pyc b/adapters/__pycache__/google_adapter.cpython-313.pyc index 5179ccb3f704156a5d48194b0bddd048ddc6f12b..21119cc9ae806f2ca637c180a91825b065612d6c 100644 GIT binary patch delta 588 zcmX|;&u`>Dap7$JlAYHErd*ruw zRa)_E3?j64fvsevx-UatX+W~RI(SIar@}swgiA0?vO>_|T^Uq;9;6CnTet{*vM;=X z0J-Ylfg$qFFG7Pf{kI@U+yRmHvA}%@=`RB>KoW0jWvyDMl`3Vqs@0xW%UV$`SL9-$ zR?vq-B@T*YS5#qHKNi=4Zkq@Tqu(O2c4WL8y>8yfb)tDw&UdG#KQDe-JZN;L7DzTc zJD{L&{B%rz6TS&c3i>*ebfhNnA(s@=3Obr}re*%>geK#!b66j#&Mo=I6~h>w}8IWj5-pMRNaPcK*~!b|4z z-JguTB&-8JvZq>z@!)Kq&3d1vRx%$~+S<0cQ*UekLJOgJyo4!IILmMctN0hg8t(l^ z3g2UAdHjH313$!89HB>evu`)qXdZ7dm!hC<9zUUL*fZ$mdPf$(lfy9b1F>EW(+~aw D%Z#H( delta 509 zcmaFj^39p=GcPX}0}wEM?#)~*y^$}6k#Wc5YDR;}8yOWQUuV?V%*v$0#v&>`!D6#1 z2PZpY#AbW`bNXeEZDx4f zQP*5vwWw-?#$|Qu$!leu+2k2ywZExt{wC|l=q<`{+{)3~M}_I6sgsDWD8m^QE?*(m zGkOxfysT&0*nuKvd0BxXXN8zSY*8*hF6OiHYJRND=U7>R>~mbqe!L9lcxC)FCpXET zWqdF>KtY+~1A_po`h~!-$qfqP^&c4IS!GTHe+5w=ofzX7StS<)e_#O7J9r@UiE;>i zAvNYv$`lHc3ulT3Q87&MAS#h5 w1;R^_2Pq9_N(FIUffSHe&zJ@@2IK&sJ)0dBMHu;`nHVKMF@VS-Szvqv0NbvOxc~qF diff --git a/adapters/__pycache__/s3_adapter.cpython-313.pyc b/adapters/__pycache__/s3_adapter.cpython-313.pyc index fbe35de50a77f27075483c748def3c9935f3a626..3d604c1aa26e36dccf7cb459ddfcb65e55b55bf9 100644 GIT binary patch delta 23 dcmZ2vvdDz*GcPX}0}#BZn4HPQzmZQw3II`11~dQw delta 23 dcmZ2vvdDz*GcPX}0}y1L?#+D2w~ tuple: + def _prepare_contents(self, prompt: str, images_list: List[bytes] | None = None) -> tuple: """Вспомогательный метод для подготовки контента (текст + картинки). Returns (contents, opened_images) — caller MUST close opened_images after use.""" - contents = [prompt] + contents : list [Any]= [prompt] opened_images = [] if images_list: logger.info(f"Preparing content with {len(images_list)} images") @@ -41,7 +41,7 @@ class GoogleAdapter: logger.info("Preparing content with no images") return contents, opened_images - def generate_text(self, prompt: str, images_list: List[bytes] = None) -> str: + def generate_text(self, prompt: str, images_list: List[bytes] | None = None) -> str: """ Генерация текста (Чат или Vision). Возвращает строку с ответом. @@ -74,7 +74,7 @@ class GoogleAdapter: for img in opened_images: img.close() - def generate_image(self, prompt: str, aspect_ratio: AspectRatios, quality: Quality, images_list: List[bytes] = None, ) -> Tuple[List[io.BytesIO], Dict[str, Any]]: + def generate_image(self, prompt: str, aspect_ratio: AspectRatios, quality: Quality, images_list: List[bytes] | None = None, ) -> Tuple[List[io.BytesIO], Dict[str, Any]]: """ Генерация изображений (Text-to-Image или Image-to-Image). Возвращает список байтовых потоков (готовых к отправке). @@ -130,7 +130,9 @@ class GoogleAdapter: try: # 1. Берем сырые байты raw_data = part.inline_data.data - byte_arr = io.BytesIO(raw_data) + if raw_data is None: + raise GoogleGenerationException("Generation returned no data") + byte_arr : io.BytesIO = io.BytesIO(raw_data) # 2. Нейминг (формально, для TG) timestamp = datetime.now().timestamp() diff --git a/adapters/s3_adapter.py b/adapters/s3_adapter.py index 957cbec..28ba113 100644 --- a/adapters/s3_adapter.py +++ b/adapters/s3_adapter.py @@ -18,7 +18,7 @@ class S3Adapter: @asynccontextmanager async def _get_client(self): - async with self.session.client( + async with self.session.client( # type: ignore[reportGeneralTypeIssues] "s3", endpoint_url=self.endpoint_url, aws_access_key_id=self.aws_access_key_id, diff --git a/aiws.py b/aiws.py index e08bdda..f755f37 100644 --- a/aiws.py +++ b/aiws.py @@ -64,6 +64,8 @@ def setup_logging(): # --- ИНИЦИАЛИЗАЦИЯ ЗАВИСИМОСТЕЙ --- +if BOT_TOKEN is None: + raise ValueError("BOT_TOKEN is not set") bot = Bot(token=BOT_TOKEN, default=DefaultBotProperties(parse_mode=ParseMode.HTML)) # Клиент БД создаем глобально, чтобы он был доступен и боту (Storage), и API @@ -83,8 +85,12 @@ s3_adapter = S3Adapter( ) dao = DAO(mongo_client, s3_adapter) # Главный DAO для бота +if GEMINI_API_KEY is None: + raise ValueError("GEMINI_API_KEY is not set") gemini = GoogleAdapter(api_key=GEMINI_API_KEY) -generation_service = GenerationService(dao, gemini, bot) +if bot is None: + raise ValueError("bot is not set") +generation_service = GenerationService(dao=dao, gemini=gemini, s3_adapter=s3_adapter, bot=bot) album_service = AlbumService(dao) # Dispatcher @@ -126,11 +132,12 @@ async def start_scheduler(service: GenerationService): try: logger.info("Running scheduler for stacked generation killing") await service.cleanup_stale_generations() + await service.cleanup_old_data(days=2) except asyncio.CancelledError: break except Exception as e: logger.error(f"Scheduler error: {e}") - await asyncio.sleep(60) # Check every 10 minutes + await asyncio.sleep(60) # Check every 60 seconds # --- LIFESPAN (Запуск FastAPI + Bot) --- @asynccontextmanager diff --git a/api/__pycache__/dependency.cpython-313.pyc b/api/__pycache__/dependency.cpython-313.pyc index c32ebf65f0d33940009ec4b5fe23d148b5f8783f..94bb23c5dc69317f62275bbbb0926d8b4d706f7b 100644 GIT binary patch delta 1429 zcmZWo&u<$=6rS;JY{$FaIClJNy-rps;5dLwTho@dszgta6sgjvmv$wyjTdXv#E#y& zAP`a@TsTtgrAKZwCk|W?XZ`>qi;LAtT$Uvd(w%qhIfDJNrcRXSd^6C&rRV$sQ4Q5dGpw38xFq1znF z(!CtEcy0eANOHv!2~`xFj97A#D3cRK$3TsAJ0nW5Mny1D-mGqzVA@go@<}Y0v6^Gz zb}O-e4gIT|Y{LQz{jF6dOi>fo_4g8OF6vI+snhu|Nj$^Q2Zj(iEls^L3onw#N3`S|jTta4{CD`t=w8jh3M zs8p3oe;q4bn^DRH=Q?iJ-5Yu^vpXp`Wj}?276}#_ziEw;+5i(7C=slNzs{Y%P4Y00 z&`iWM&VvMk9iKQwo9wy+c%4YHE+Blx{?VDf@a>mlZEd8kN!^KpZj)qG9asr(CNJ(w z{S*c?Rd@pjXb`LttRMs`f{kBP-BMKDz6b8mAM`sxbaT)|FZzQ;geZ0Wfq1w+)-H|I zOENj$w)SDybK}e2irz!0r{1c_J*;EoA(wkNzR&PRs${j0KS$D8Dg8(*eHV-@C)wKRho6lb7oOv?{(a#Y4L@K1j|Q3G EB}g1E4FCWD delta 1118 zcmZXT&ubGw6vt)ewOU>lhFbBgKPCP$^MY38E5GO3YPe?C=y4Q4cFUZfmp5 zj$Go@hcPvak7rd)OxuQ-vE3@{YF0f35uwr!hlH{x#&$J}$_1X?iH$XE6tztevpt9j zi?|&`h_7mjgv&?@CX(x^H4CibsQo`)m8)9Qwvdu?{b}@D>ufCp8T4mnnMjL_$X?UX zox_D@>GLw!#2sR3-Yw0w(&d7TYPNi)?enm#==Y~m=#I+Q`per99F=v5;tOIJhM`t- ztG(T>ZEUui+kHr&=Fm13AxY^-tJ$ypZ^@#bKv-J#3?H}TzW(s~1Xh34orHx6hKX^r z0+K96x-jLEZcdO)XSr5y)O-CVOsq};PvlP{FiXH#=}VwAFiK#d0&xmOX?QL_AUyan zlr>kzv7=!qx?k+2MK`*VkZm4UGWmR$DU zvJ{+LWv<}kgOa=6WbQobOGF4M!q-4@*EtMi2SygFKYQ&8Rezw;3b{_4vD(ERbUV$B zey!brMa*B3cYHamoR@!nFYbEl?QRR|TM)p^Fy@|yQ#rBoLCSdgPkQa`7SIZS?i|ov z0Xk-&qfuK5CHcc&5M`RvnqdLMP)Ul}x=v&?DHXd&_Z@?-hXw}X-#-BRp e{5zXFVoTrH;t^XtV(E`{bBUXO80Yw#q45~g9^o_q diff --git a/api/dependency.py b/api/dependency.py index 51674c5..42fba46 100644 --- a/api/dependency.py +++ b/api/dependency.py @@ -5,6 +5,7 @@ from motor.motor_asyncio import AsyncIOMotorClient from adapters.google_adapter import GoogleAdapter from api.service.generation_service import GenerationService from repos.dao import DAO +from api.service.album_service import AlbumService # ... ваши импорты ... @@ -53,4 +54,7 @@ def get_idea_service(dao: DAO = Depends(get_dao)) -> IdeaService: from fastapi import Header async def get_project_id(x_project_id: Optional[str] = Header(None, alias="X-Project-ID")) -> Optional[str]: - return x_project_id \ No newline at end of file + return x_project_id + +async def get_album_service(dao: DAO = Depends(get_dao)) -> AlbumService: + return AlbumService(dao) \ No newline at end of file diff --git a/api/endpoints/__pycache__/admin.cpython-313.pyc b/api/endpoints/__pycache__/admin.cpython-313.pyc index 4654462d2c40513c957016252704327b7699fc8a..5e1f99e1b626e86f871345ab52cae2893440fb8a 100644 GIT binary patch delta 39 ucmeyW{#Bj#GcPX}0}xE9oSd1zk#`Rpx4hc?>Y3FmOfSnDZoa~%$O`}r$_==lbO~@A0bhlIL)Nvf^kU3u6OGuqOy~Kr_+hsKiF~8w9T|Vcfn4k zKe8j8`<-*XbMAL{udepcCwS&>*!Wp`x*ox`?|?M4wD&b*pU(0+M(8;dLLwHBNC{Y^ zg<|Omou{l=(1!GaE|eD1vrr;XWg4*hkiLc!(sI!-GmJ%pP%Ici=^>+ZMVFs#5=sP9 zNVCc}hs;7o$Sh%fK}kl9QOI7c$rP;6R}Z`_(zk{Ya!5wB2&KTyj_aXcwwmDt+iHzf zC>89X93k^w_uP9(AF|bGg}ha!{TrECz;vuKbA|j+b`2#s)!Fk_Jy*NxIzS*v@~W4= zD%lBQ3)F~9aOWbC7rcTeWCmU#iNXyj5Q;?i0Oehi$VltU#Q#c6@F#Xl>3u{7zfRot~n_ZG#y zW!`$#y*=sJt~j>;Qxc2UnzBfbFJtrY0FD;6o22RDI&6|YEneGg!m|_|V9_kAZ#U4h z=BhRmb%DaIEO&wSv*65Waa$TUr&F;J18L6WYO7@D$|-^?Fu|2Jz!!B?y9r-RBbDQ+RpC2MWVOu2Nvbb>Rdx8pm7(gtsteJ!#+ zh`+c~h0ONPkY1O$Xv$Vh1>jiW-m8C-nOX&xqqfs#|eSb}^?nnkg9(d@_N#Ch2kj}3;g|A6H_r1%07oi|0HTx`7?=S@H zI9EDs$in&35|f41gj7*x&(Z`nrJ$c&VIP!eb~vT!+FWfo4;*Gk0in|*sO-p+;y#U( z@~yT?Xv#7%US%7Atp(OCV12vXiq*Qr!SdtG%Dhrh^^TRGbgVi%b1fRR28~&0ixZ*I zoS=1O9-Je+X19PWt$|1P^9ZGp-EK{!sz0FAiN?T!9;v!+-rlT5o7v7ksP+-n>cBFc z8xxlLHVS_z?QST;H>IhDU*@r$S~c~8Y8kLp=Nn*g=bsb1H~gL{O_{=vhR4HVcQ_V{ zj&~15#5TAeDct}#YDnAyzF3~v-ux+L)Qi#B#0a6qQaCKN2X=99LOS+e>7~Fgwq`k! zSw4k)>KD9aui9R=z3O_|mGrhK-j?(0lYx*D2wnB=NqY7?v+d&?&ly9qbUl0)Y~G}; zLa|j$JCcy@*TY*8;wl#X$O+9TN+2urAfQF(|oKr0=2V zYis?!Js$d_JiN!redN#(>`CvT;`B!qEo^ra{mDAqUC(_|t0B0Fg_1>z?(v{S7K7|X zt)+*j7acS-Uvy!Dc@=w#!B@+&3YM0C^&c-BewS>zwOKT{CVab5Y7=kN_ z;H5ecS%H@th}NYhVs9V}>$ySedmHH+=|#PD^o=T(#OqiRX@n^gZTJ6$!n+c_?dy$L zO!V*loWqwBd!mog_+#nX;CuNgCK5gv>!V=j-Mz>^@=^GHc2W9Xe@Af;3wS;5fIr+p z8lgw=b95(bKod~0-xH(+y5UFgn=h5ZI}0{3LCS~(Hi*G;U1@h42I+OMZO8?2O||B( z2fs@C-l?$}A!*T)BVBe`q;97r?_SOS&pgUVryg*v(*_ynrmMck#;>5jOd53)v7O?@T91&bSnLv(;z8T)ZRYNB1JAJ@_S*qb)7 z1g-S)Pm~|X=aj?B5&1XrG_Qhb<&bhrd4iXJElQ{9OLifr>nZ8u0HyNHjb;vA==$>cvJ-D@MSi@<_k#70KPe?f#l%v4qe{Q5>w7K=p8Te7zz( z8tL9oM6MjyF>#9Ueh0?OiGG5~?I8#HXm=khc08rKd(_8LoLU(XYk(Ky!0_rN)&WeJ zSM`D@ZKRv5Amx@`9n6grk;J_TjbkaGk~ffLgHhLv7gx^xq_;`&HYL3eE8d6Y?mii; zHryoWB*{K{#U@{#*-hFbi>)=hT4{2TsLL#FMVAifik9$5q$Pq;YKc5gaJMj~BO8p;vHAc?-9(<3_X;dVn&8yG|P$?2@hwZ%UspR5o@lBLmCJyZ0olk2GPSip`W;gk?m=z5VjQu(J0N)mujBThtB; z27%g1-6GyD@PG%QK5BycyNU<#9*T|N={0wd3YME$>8B$;`)#+5ZO6B)7Pc9mss0Bd z<1(>{!i zMVc~!K-(XPj(1ZF>VWc3rxgb5U?3_GitEsgu8W&dr=iSAB@kso^4bmxnp&Pa0*L$L zd4FH`KF@o1(k*`g&sD*E*JRQYycT@)@XXAQ&4+a2w}6nBNrvP>B6&uFJS(-(ujra* zS|n{oFX=LdjGo6^Bz;C76C{I&9N~^Y-YB(5M)XXoXUb^)FJ<0NnWd0q`L9=(u}Jk9 z3;m4VPQ7|#TV2d7*|(GRk^{TxF>6(JjB%1vaq>2)74uHFAtS1~Ai1{huuH9yJL8~B zdZ*19(@G87y6b=X0W0d>ZQU*TGGdI88r6M0+sv=B2JEbO0hM{TXYt__K~-s#nmi=$ zmzwi|1*W}CZZ{mV!aezGzSqI>3Lyb9$sIgHNF$jbwA?AuS$|4?&94LS%0CI51^5O1 zY|}IJSB;JG2TgA<&_TZ)Y5@~1hE8g<^(m-vJRNCmGu;fwE8*dCc=(KP)ZwsmKWP-NHPC454#m{M@BiO#o2+!5lKfcy7)NS*sG(dn$4 z7JJzetT8;j(B9m6Gq|G?jF*G)QzI|!TlX}6C;ZKD#S<-iq7~2XvS;_NZEK!nMNF2( zBz@2xU9#a09~Yid&dVp`@Iqp6#7ve27%_6ongoZWXaF>dMo_TXlk07E6SKiR{VB8-B&LBTz|g8Nw9ZbQE?e{?>dqZc|nx<~5? zX~viQGOfjfv^K7#fA2UAzof^z-1ODX*I`CM;G)3=8-1nOpXr|(yIc0p`=3f^>6|t)q24un@At;e z96yyiJ@?Ywxv}T(uVjqEs0)S z-f~6LgXC=i(!02~HHk(fzikbnamC1I1LR6QWPRKfE2m&b9QAkbi5P`)wP5mm4w2`GgyN zI}vYma+uxN0ZNUjwO_4xoki4N^% zSj7=g^r)EQHCqgWc#8uC^IT5%D7&RoCf>3rQ*K$+Y8Mpj13b@GK!4E9ZgqqXitH^5 zkNPc@(&+>#0*c#w`cz*z-4)J{#h1o(AmRp(Th*_K#<;jdRmiL z{w?_lI(=-OE+6Xyf!;qhtgR6vA6S*9;N*%p>&8j!F;6RIY=}79Q>%`9#KK str: + async def ask_prompt_assistant(self, prompt: str, assets: list[str] | None = None) -> str: future_prompt = """You are an prompt-assistant. You improving user-entered prompts for image generation. User may upload reference image too. I will provide sources prompt entered by user. Understand user needs and generate best variation of prompt. ANSWER ONLY PROMPT STRING!!! USER_ENTERED_PROMPT: """ @@ -157,8 +157,9 @@ class GenerationService: # если генерация уже пошла и упала — пометим FAILED try: db_gen = await self.dao.generations.get_generation(gen.id) - db_gen.status = GenerationStatus.FAILED - await self.dao.generations.update_generation(db_gen) + if db_gen is not None: + db_gen.status = GenerationStatus.FAILED + await self.dao.generations.update_generation(db_gen) except Exception: logger.exception("Failed to mark generation as FAILED") logger.exception("create_generation task failed") @@ -172,8 +173,9 @@ class GenerationService: if gen_id is not None: try: gen = await self.dao.generations.get_generation(gen_id) - gen.status = GenerationStatus.FAILED - await self.dao.generations.update_generation(gen) + if gen is not None: + gen.status = GenerationStatus.FAILED + await self.dao.generations.update_generation(gen) except Exception: logger.exception("Failed to mark generation as FAILED in create_generation_task") raise @@ -201,9 +203,10 @@ class GenerationService: if char_info is None: raise Exception(f"Character ID {generation.linked_character_id} not found") if generation.use_profile_image: - avatar_asset = await self.dao.assets.get_asset(char_info.avatar_asset_id) - if avatar_asset: - media_group_bytes.append(avatar_asset.data) + if char_info.avatar_asset_id is not None: + avatar_asset = await self.dao.assets.get_asset(char_info.avatar_asset_id) + if avatar_asset and avatar_asset.data: + media_group_bytes.append(avatar_asset.data) # generation_prompt = generation_prompt.replace("$char_bio_inserted", f"1. CHARACTER BIO (Must be strictly followed): {char_info.character_bio}") reference_assets = await self.dao.assets.get_assets_by_ids(generation.assets_list) @@ -304,7 +307,9 @@ class GenerationService: # 5. (Опционально) Обновляем запись генерации ссылками на результаты # Предполагаем, что у модели Generation есть поле result_asset_ids - result_ids = [a.id for a in created_assets] + result_ids = [] + for a in created_assets: + result_ids.append(a.id) generation.result_list = result_ids generation.status = GenerationStatus.DONE @@ -479,4 +484,26 @@ class GenerationService: if count > 0: logger.info(f"Cleaned up {count} stale generations (timeout)") except Exception as e: - logger.error(f"Error cleaning up stale generations: {e}") \ No newline at end of file + logger.error(f"Error cleaning up stale generations: {e}") + + async def cleanup_old_data(self, days: int = 2): + """ + Очистка старых данных: + 1. Мягко удаляет генерации старше N дней + 2. Мягко удаляет связанные ассеты + жёстко удаляет файлы из S3 + """ + try: + # 1. Мягко удаляем генерации и собираем asset IDs + gen_count, asset_ids = await self.dao.generations.soft_delete_old_generations(days=days) + + if gen_count > 0: + logger.info(f"Soft-deleted {gen_count} generations older than {days} days. " + f"Found {len(asset_ids)} associated asset IDs.") + + # 2. Мягко удаляем ассеты + жёстко удаляем файлы из S3 + if asset_ids: + purged = await self.dao.assets.soft_delete_and_purge_assets(asset_ids) + logger.info(f"Purged {purged} assets (soft-deleted + S3 files removed).") + + except Exception as e: + logger.error(f"Error during old data cleanup: {e}") \ No newline at end of file diff --git a/models/Asset.py b/models/Asset.py index ff4eeef..6c4d3dc 100644 --- a/models/Asset.py +++ b/models/Asset.py @@ -30,6 +30,7 @@ class Asset(BaseModel): tags: List[str] = [] created_by: Optional[str] = None project_id: Optional[str] = None + is_deleted: bool = False created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) diff --git a/models/__pycache__/Asset.cpython-313.pyc b/models/__pycache__/Asset.cpython-313.pyc index 90efbf52318691d8bc0dee63592fa724fe5ad889..e5c7d04787c943eaf495c2ce6900a0453b63020f 100644 GIT binary patch delta 365 zcmaDUIa7-FGcPX}0}zztPtNS!$g9cBcw@6Ub0A|_uuze7j7$uBu&_E%I!0EBAy`C; zAz0Lsu}CgP9w;sb7Z-<$D}co%z~YKP^^(C-mW(k}+lP|D{Dyqb& z0{LtvOu@2J3|TCuAp3!!NG({dNIjiNQ+_fV>nR?$TU?pN@hPb}sU@kC&$IS1Dou`K z>(Vd>sk8tQLLfp4L`Z`OD-dA~B$Po6o6Q33_Kb|Gn*%uF7#STWuje#k)R_E;bDAT_ zx?3zs`T03T+#r>9Ac7f0IDiNqAaRR3IVZ8WI5)K2xlC}jA%pk&k@;WYQZch*sfXQ=sbQr@YpX71mH)9m)@chaEq`>w90P{^hX#fBK diff --git a/repos/__pycache__/assets_repo.cpython-313.pyc b/repos/__pycache__/assets_repo.cpython-313.pyc index 2dc1d3b62b912b992a943781b527d5b31b3d2313..acfb67240ef7c9f6da7a49034fc9f8567502ad81 100644 GIT binary patch delta 4798 zcma)9du*J=5#RkD_wwEOakg_lzs|1eQc)d(paju&Wb~l7IS8~*_h|9ERv&eH(?5gJQ$!&O?Ik^w(H?;b zJt7nRNE3P`Cc#(im3w3+_gI*P)1_W(kB!-QU+%T{IGBU?Exma?PUhr&Yp<)v&D^|i z>&@>eUhY zgglkn@9hdL(}IpAv?#OH5fb{^MMCO`A5ViSf+Dui7Miez=uN30W1pmhUPN?2EOC?9*;_wQBz6JfYvJz2x?2 zxB4r!ldgD!Gj{~JZzxZ3p@i*)0K6&g%zF3FX*$2paKzPcG&T}T#iG~5`b2CiTLLT$ zPmDDnv>|jOY(zjom;VXP5X14PA;hCEkYE8T0kR9OQj;tNxGO%!ZbPy)l?;!- zF|$=T633EbEDw6DG>zDVMB8vI6&{L@#LPXH;{;LLGoO zyG#y}nVQB4c_MYWFp3FIE6l93WDh)?4gU;Rs=3= z-aA2?>6HPTLSGn;F_wHVHi(zWw7;@qei`0T75DJZHCCp*UQ$ovnGZ|$%4t6`)*#^3 zW_1Yl2&ncosUEIKu#~HB*t;GYjPV=GN|A$W6~G=w2g7(~c7z}TkEf;ByBVPofm_s! zy%q#G>TQH^yH?_uN0VcG3^T#JYrbYJKg~7ByB1*`!g>T;f1{Pn6-(&B%(jY0g|++) z29a?mKv3jcu^1B2H9+8T#yyH0st5r70$+Kn*V5w|rTQIeo+)0DpkSKZi+#%Ogb(-E zb;oQsx3~kRc;w!Ny?%s+vku`94?Ji!6&OJRXU9M{B#2;@2Q#~ukbI(Afx2*vFV?iv z=QDq)IY750aN+!w0R6Zx^9PgTiBuS*jE}|=DK#ixx6M2^yBij(a!xeZe5$U6{wnk5 zx(rS8Fc9nG3G=vlf>;PgSKsutzY>qYwDiP$l3*7lN7RgG=by z%*VmSG>x)y4d2*D;Q#6e5CvvVU&!N8NEX91L_OqgK^2;bDtO545atPCdu~1Hzh2rL zwU&g0TxQ0bf-MSsTl|orDcZx^0@Izs7PvEc@T4Tfgb)(v7nur>eo2cg^=Pw8i#uSp zg({*V0m@B+Vu3$vlaPCV6Ouv`ax+O-LQ(-KB8qJ%7`BH#CHBcq0*{nMd%#m%ith{t zK~76Q36Xw>R+wLI-A~{Z>UU{ZmKOVR(>Z-2aQUlDR?vs$c zQboU#2bObn-9&MQY;Mvf<&^T1YEV64>#+6hH@`!4h^!Z@$q)sxoBq4l&59eua;?8n zZ0bg_4id7=`}THleD0(t+@7>zSA>uSYYQb`krbO^+wRSo)$hyo7QVG)JH7d=3jJ)&py*Y)%nfA$gmWqm?_11P`F z9`frG*@v?aV>kPRUq9{FU&$T?${}F>ciz+4$NbqxVfAaU9@+I*{YqPtL;sz3#!_??NfevI7X; zMu-BW*)r_50vLI7@8oDCv5#4hA|{f18NLufyP4)O;~HWFMDN3%WpJE9k!Co@#@TQz z8s?9K;esq2hEmNG$Fs0OV=6x6m?n8+Ww^dVO(%ySlk&1L9D(wM3kvfDs~U?J>ggu+ z8yMdOS?XtS1Xy!@<{$-SGoIp)npa;cQGU7ag!r4=r`xyuu6@hfgVXK(+USpV9#fwl zePQ2WhhCzjKgdVsSqmvDeWvkrFwynr8*EBuZzhYSYJupB~wYxm49V?d{dQ zy|B7?k-2R3v}aA$vj&&##${h$rDR*TUaAUBdqcW6G;1TSvSXlZ<@Yv!@yR+;x|Lq> zfqqxqq--<&`}^hfR|$p9-}hD>cRrhT>Wep=ZPPby%igwK-!PyL+^>fp(C?4v4Lhd2 zgIVw3m$M?WU%O%>Wp~f7-#xb;mUo>U)O&`r-8=QJxE|l7kBsWO5_&^&+B=r@j`8KY z>9yH%;`4tNAkMNG@O1Oh=INrwY*Ax+rm*x_$#X8<+oTsX|EbtF6Ie3SxNK(Cy7vQ3 zzZiLbxUiuvRuM`wG-%*ODQa#Y^1Dw>X z$96+PX#lg#+m020nN&FKW*8dQ@x(5;q6uCa8*+R!G8{AONI!DdAk-r8+##BRc%jT? zJO)$y%Q_>eh<`Lb%pzFrYu|6pYhvGl8Q2}l>A7l}_v&z1k5i&qW1BU%Hq|<^iRYGu zs*Puv0P^F@b6pY7tM(Yuc-4+MXdwj!J#3M-zpZlh(@4hhjhA<(7r4c8YTsBaY}$B3 z)qVunwcoZavD}C2)@xt2RlsjD-||*LrkTFwof0k9wAJ+m&*LaUVh$+L$t<_b|jM{Ov3Rtga;6Y5FSJrLr5V&fkW6KgeL)@?v4+p z*l{hqro5^Tsr=^@CPC)T0TdfVJ&`%K=5d-{Ay=I4JnMOF(=37R+jXDfXEs4MP}y-p z`kdg`l!42GC58nY6Hg2?5v6hc&>F)!k{ljJx&`T&%E1H`e$?@oLK+{3RE~++@Qfx? zN!G+M97(JD5`*z1L$?^t(IjMOwTWkA?iIrmiAKiY7qZ%U=J!6dh z6u0Eh3perWW*fG45Ia5+`xZL|eE5E7UaoL^Gg^7v8|2wx2=)y_*&Hp<8 z`JZR{;4Oapa@}8%NSM(l-ysWcZoOI8GJS)%Tg+r8A7^7+@G(tjN$}dZE_C|V$Bi+c z@F|%w?jH+?fTDfl!Lg7CDcV0C9;*>GiVlp|jzvTy$)U-fS zZJ}RhtcxWnus&7{O!19KfIc~cv6q@n-a|BLf>Xw_W|k^%YEwZ|hfYxjs43sFMJ_~V zPnbTXXHx;wuk`*EJ$&btRLVe${IVw|Ka3>wfEko`BA?2M9ijH+Qcu;GVVph{B~Tx; zrh2{ZGoGrQ6i^7%U!(wifCGRLKw6$|h%Yn}=^AGX#aY*IY*8jWUnq%v5C_V3c8(e) z`oZZXyCi}{i5OrLWWl^u&YmsIS>6rXpujb*KUXSQ#o0`~P?dcaihY3nfMI}&WFtax z{p6x-3D?MpydwsoZGc>#wP!>d*p%#uCa5(-(e4IB?JmG>zyJW}*Mv_Jw-(N1 zU+=oelGhwIbfJ2u{S9qERs3a~o*~HS>V9z=0u-ABYJgPzVBzQlR2{0!)x-e*sQPu{ z0rxtn!Cn(-`Vym{xfln%rl}$CiZQ760VV*40jr%o3KpHn1iS$95bOhBf?b0q(@{ez zOvdV&6Cmko(;8BP{FCb8)Kz}CfHP{=G3pebDscUbT`88czQToE-ttCc%@$$yXv&nM&C5O(>2+cAts}W|E_tDl7dfo!?xZDzEs|&H zZ(bWnQVgrRqo^d#V6)PxcMYEySQ(ZO0|BU4S*4*$dfez|(*%U<+U?K_;fg z#I2dB2zsrvw4^yVyfjNhi`QX^eyM&wR*w!}`PGt@^-A#M^19rT14F(3<2W`T&keQH zLwkMb8SQ&k{bp#y;L&%7(+xMERgKjrCnqzlvVVBz8BD!E-vs@pG0_ZAkD{8kA+X5N zHO|=foY28f0ZsvO01I#)U<1m4D+IJ*g_*K=TRt3KzX6Z2Xa&RpHOo*-gma@BK5(6n z4s&1lbK`G@Z_gHHnMT)7i5H4_?{>uM2IlO1o5j}Cd(vFFgSL4pUp=r z&3ve9rb(mpwW5=z6|HR5G^AyuY=No_#y|AG_XJw9w^j5Xq(#~!Gg_fdleTl-O9)G| zKXxtue9k@Ro_p@O=bY=mZq@g?pSWBif${MvB{g4t(d{do0qz4L5sB#~5r#4mma-9! zasixUd-(`Y`G`P;h=p2=Ij+|lu~C~r^S$0DRl^u=FYsW`=;TSJ605}~ZbP)@2> z#+XpAMY3(M64K5~_8M}Gr}lbMPb9x24v9ln0gOlvpd1F}9CAu7_`09TJkO|-AyN4g z)6f}Mab?*vz5lhgjU8kflnZR-hM-gndk8}nL(U0oSGmA%^i_c-T?L?XG%3p*P=P>zRx7joPnf8Z7yh=xi&EQYDl-z# zW|9YBwx^9ed8%QhevJ*3q*Uus$#ODc?OBXde=bq%;2<(k7k=WlwqMswCF=UuBk zj;;#kj*ICkQtnAbm+ z%o(w#T{veX!onf;DVs`s7BjR;e~F)|R@({3#x8M*oLqycWlr3zZ2yzA>bjF z`!AT6FodFInIXWO5p*-+J~(}REC~gX$fcCt%FRrC`fR12Er(d9b8yQ>0vIkGQl_iI zJ;>{>(L^c@`3-30$egBbrPVI?RjekF^Wxw0cv$70}MBg-_Kf^)r~cuAK|O z^mX-fAkU~%+V|8Mbta&l)y@HPMw?Wx2ciM>QF9c_Ij|Z{tnd67C`@IAR$AI2YR|?+yJ}@sJFlZh{3{5 zbtV)RLk!J=_j_p+kd6SrTVfRRQu$y-hgjVK?^8aVGy1;nFigt^6Qyt9Vv2AG-~be# z*$_r?rN^i%I)>tvs zn*=&e)2QGaO{H^58h<07q^ET|_*Q`CkQ$M7%W$5`SxP0;#fGAn6lJOee7^rbnJNnruean_pSzI@!!Xav~dENwI zUyG#5e<}T5`byh0P2OWE7JNmSaKD9oo zdi&>U1DDsSHQ~or-X~5vi*6DKPTJ>6D=+yk`e#emYNcy0r*C(uuN~Cd58W?)ebVyB zLbd-XVeMPrT|MWloORa03-qD%(ZL@by!@T1-S<~@K5%YXS?~wvmd64t{LotzNoDn= z{)_#y6)jpt%be5qRf%^l*!bsQ^C#Q)2UoMTEj6 z_qCVQ1+H+Hdp>sGuWeCnzBy0XtfxWqG^nfFR8N~~Yx{@X3Gq4CwXwUAxzoz;Vc0tz z%dI)ga(QX+A!UNVOnf%ce$7a;Z#?;#)(z5h4gNgu@7NgmC~} z$mJ)}N%|JjmGQWog9{&qcO*wsNAkI(9FH6Snk5{Oi;CBkOw};MG$otxO535 zXHCq=x*Ob%?8lT%g2Aj2OoZCFjk# zamVm+$>&J^oZnFSAK2Xs?nwd!G4n zwV1C}P4J?9(y_1C>fq;6S_H>SFK+d4*yrUZ`E#0O%kCKUG;g^3h0*8GXb|BP!VrgT zo>sE`h<~;+2WG)Ci}`{DY1}`7@GOV40-fxU_i$!_E4JIJ+r@ z8L4eXC-Jm%Bc1{e!oIuNn~Ih~!Vp2gEHqROJOO(jPsJz7EAB{qRay%lZ*D35l-^_= z{!47xpPiDk@I0POQH`Be$LIpH z)JgWUnhLtcReE2h<387Yw=}Lcr9iJRg*jRgd&^Dzxlkz=9TqT(cf5$u zE?&vBoV>wHxaO2`hgXKZAJXJ@TG*{noHO|*^fJ|(Lijg&4sZO&)zdJowA7AIS*jO# z&Gx#(saXQuKE+|L8MtDHxiJe!aW)YN)9L0Lk$d#o88jV1IE!$g?w)=8$8di)h=cs4 zc3!8^=s;C*@|==_#wAxJ2WCW-DzMb?e7OL-p%2u@Au7?zVR{pFfkxM^X>5ZaF+d$ zSRGfTkvp&5HNLvoB3xXXoL-xGXMM8bXb(;d+!_3$uO;%G2NRdprY?Vfv8_~>*RHIr zSFdu7eVL4gxj{=HC1FAM8;x5r_DeF`^^gRmR12Za2dQM!huCZ?$v#Mh{W!q3>n)yr z@UqoZlq&4&REql8MyhMh$p%K=j^OdAY|rRot+-S%Ux2IJ1Yev6Hd;%hlx~vbCh1!z ceOn~CL82REV2g}wlHrFke List[Asset]: - filter = {} + filter: dict[str, Any]= {"is_deleted": {"$ne": True}} if asset_type: filter["type"] = asset_type args = {} @@ -202,6 +203,61 @@ class AssetsRepo: res = await self.collection.delete_one({"_id": ObjectId(asset_id)}) return res.deleted_count > 0 + async def soft_delete_and_purge_assets(self, asset_ids: List[str]) -> int: + """ + Мягко удаляет ассеты и жёстко удаляет их файлы из S3. + Возвращает количество обработанных ассетов. + """ + if not asset_ids: + return 0 + + object_ids = [ObjectId(aid) for aid in asset_ids if ObjectId.is_valid(aid)] + if not object_ids: + return 0 + + # Находим ассеты, которые ещё не удалены + cursor = self.collection.find( + {"_id": {"$in": object_ids}, "is_deleted": {"$ne": True}}, + {"minio_object_name": 1, "minio_thumbnail_object_name": 1} + ) + + purged_count = 0 + ids_to_update = [] + + async for doc in cursor: + ids_to_update.append(doc["_id"]) + + # Жёсткое удаление файлов из S3 + if self.s3: + if doc.get("minio_object_name"): + try: + await self.s3.delete_file(doc["minio_object_name"]) + except Exception as e: + logger.error(f"Failed to delete S3 object {doc['minio_object_name']}: {e}") + if doc.get("minio_thumbnail_object_name"): + try: + await self.s3.delete_file(doc["minio_thumbnail_object_name"]) + except Exception as e: + logger.error(f"Failed to delete S3 thumbnail {doc['minio_thumbnail_object_name']}: {e}") + + purged_count += 1 + + # Мягкое удаление + очистка ссылок на S3 + if ids_to_update: + await self.collection.update_many( + {"_id": {"$in": ids_to_update}}, + { + "$set": { + "is_deleted": True, + "minio_object_name": None, + "minio_thumbnail_object_name": None, + "updated_at": datetime.now(UTC) + } + } + ) + + return purged_count + async def migrate_to_minio(self) -> dict: """Переносит данные и thumbnails из Mongo в MinIO.""" if not self.s3: diff --git a/repos/generation_repo.py b/repos/generation_repo.py index f77ceee..5132eaa 100644 --- a/repos/generation_repo.py +++ b/repos/generation_repo.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import Any, Optional, List from datetime import datetime, timedelta, UTC from PIL.ImageChops import offset @@ -17,7 +17,7 @@ class GenerationRepo: res = await self.collection.insert_one(generation.model_dump()) return str(res.inserted_id) - async def get_generation(self, generation_id: str) -> Optional[Generation]: + async def get_generation(self, generation_id: str) -> Generation | None: res = await self.collection.find_one({"_id": ObjectId(generation_id)}) if res is None: return None @@ -28,7 +28,7 @@ class GenerationRepo: async def get_generations(self, character_id: Optional[str] = None, status: Optional[GenerationStatus] = None, limit: int = 10, offset: int = 0, created_by: Optional[str] = None, project_id: Optional[str] = None, idea_id: Optional[str] = None) -> List[Generation]: - filter = {"is_deleted": False} + filter: dict[str, Any] = {"is_deleted": False} if character_id is not None: filter["linked_character_id"] = character_id if status is not None: @@ -69,6 +69,8 @@ class GenerationRepo: args["project_id"] = project_id if idea_id is not None: args["idea_id"] = idea_id + if album_id is not None: + args["album_id"] = album_id return await self.collection.count_documents(args) async def get_generations_by_ids(self, generation_ids: List[str]) -> List[Generation]: @@ -114,3 +116,37 @@ class GenerationRepo: } ) return res.modified_count + + async def soft_delete_old_generations(self, days: int = 2) -> tuple[int, List[str]]: + """ + Мягко удаляет генерации старше N дней. + Возвращает (количество удалённых, список asset IDs для очистки). + """ + cutoff_time = datetime.now(UTC) - timedelta(days=days) + filter_query = { + "is_deleted": False, + "status": {"$in": [GenerationStatus.DONE, GenerationStatus.FAILED]}, + "created_at": {"$lt": cutoff_time} + } + + # Сначала собираем asset IDs из удаляемых генераций + asset_ids: List[str] = [] + cursor = self.collection.find(filter_query, {"result_list": 1, "assets_list": 1}) + async for doc in cursor: + asset_ids.extend(doc.get("result_list", [])) + asset_ids.extend(doc.get("assets_list", [])) + + # Мягкое удаление + res = await self.collection.update_many( + filter_query, + { + "$set": { + "is_deleted": True, + "updated_at": datetime.now(UTC) + } + } + ) + + # Убираем дубликаты + unique_asset_ids = list(set(asset_ids)) + return res.modified_count, unique_asset_ids