From cdb09e84fc3b3e082c44870d1adc41a21b85ad12 Mon Sep 17 00:00:00 2001 From: xds Date: Fri, 6 Feb 2026 14:07:10 +0300 Subject: [PATCH] + s3 --- .env | 6 +- .gitignore | 1 + __pycache__/main.cpython-313.pyc | Bin 7176 -> 7688 bytes .../__pycache__/s3_adapter.cpython-313.pyc | Bin 0 -> 5532 bytes adapters/s3_adapter.py | 81 +++++++ api/__pycache__/dependency.cpython-313.pyc | Bin 1741 -> 2155 bytes api/dependency.py | 13 +- .../__pycache__/assets_router.cpython-313.pyc | Bin 7980 -> 8590 bytes api/endpoints/assets_router.py | 13 +- docker-compose.yml | 24 +- main.py | 15 +- models/Asset.py | 3 + models/__pycache__/Asset.cpython-313.pyc | Bin 2121 -> 2302 bytes repos/__pycache__/assets_repo.cpython-313.pyc | Bin 6303 -> 14364 bytes repos/__pycache__/dao.cpython-313.pyc | Bin 1020 -> 1171 bytes repos/assets_repo.py | 208 ++++++++++++++++-- repos/dao.py | 7 +- requirements.txt | 1 + tests/test_s3_connection.py | 44 ++++ tests/verify_minio_integration.py | 84 +++++++ 20 files changed, 470 insertions(+), 30 deletions(-) create mode 100644 .gitignore create mode 100644 adapters/__pycache__/s3_adapter.cpython-313.pyc create mode 100644 adapters/s3_adapter.py create mode 100644 tests/test_s3_connection.py create mode 100644 tests/verify_minio_integration.py diff --git a/.env b/.env index 081a7fd..12c478b 100644 --- a/.env +++ b/.env @@ -2,4 +2,8 @@ BOT_TOKEN=8495170789:AAHyjjhHwwVtd9_ROnjHqPHRdnmyVr1aeaY # BOT_TOKEN=8011562605:AAF3kyzrZJgii0Jx-H8Sum5Njbo0BdbsiAo GEMINI_API_KEY=AIzaSyAHzDYhgjOqZZnvOnOFRGaSkKu4OAN3kZE MONGO_HOST=mongodb://admin:super_secure_password@31.59.58.220:27017/ -ADMIN_ID=567047 \ No newline at end of file +ADMIN_ID=567047 +MINIO_ENDPOINT=http://localhost:9000 +MINIO_ACCESS_KEY=admin +MINIO_SECRET_KEY=SuperSecretPassword123! +MINIO_BUCKET=ai-char \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f4f6ff9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +minio_backup.tar.gz diff --git a/__pycache__/main.cpython-313.pyc b/__pycache__/main.cpython-313.pyc index fee451a6a9f688220cb52fded01727339ac80a84..1e738a12194de671c8c45e7ee652bfeb4eccaef2 100644 GIT binary patch delta 2459 zcmbtVO>9#~5Ps{w#DBqwW5+g-IDaG#AqgY_9Ac7STVV1>ynui#JU#3OdBw4#eNJf+ zQs@Dd18r&ckf^OxAyq0pRDjY0YKwY+LtCjRQmQJo`rlJ4ArYmgO07EU_$8)r!_wRR z=9`(_*_qk(wY}GRn^*02ivYjtUtU$JY6RhTE@(e<0jx$!?lyao(&iVifZc=bhdb!b z0^S9r16OsUHJ0{i zB>Nj#js2SBz}|I}0gZHHAKu)pD|j9t;kxxDXYdv*XsUJ32;S-wXq1@1u#Z?_w8yU( z!YP_?br9j2ppG6SKCl^B_di5hK|ZuD$4JXl@?qi!8BQgzw@pq5_TySD1}$a>uEX`u zk`2$2Gq@3N(=2gQ&_D-C8{du$YAhWh)xh-Q<{+ZOqz>>9872q2^@W^_kP_f3^IivV zAgIGFU^z-+WE5|20wWRu)S5@bfOh23X#6Iqn9<;1KkIFqN;hgo$H?fc!DS!QQUFEd z1bf41LwWG7z5wp)Z8M|s4~o)SfbkEX=$EI`(BOHW*j6HxCf&y z1K#&k7$*q#*=A}Ag9QZH`;_)1IfX+#Cj>!){(*M`4UW@s65Z$qgsKSDG?-SVZTrV%sC;XuMy6nKp$H+{i!$O&V_p=Ib_@p&C$)~ z%=IY1J~P+PN?OTCNtiS(j)vpWqj{}u&Hli#@L=@lr4mWmk(iKZz>u@a>Eu*KNoU1r zn#%1imoBQJlt{>`DqfIZ6_sSp#VJ)zP&r$q$yv`%CoTY$mL_HPlR0eK2S=g3tjV$q z4Y5;}-%uI*!cw!nL~H-?!s`ghEo$&_ho+BL@MqLAUa1l6S8FAE%ZibgiB2E;$GXS5 zVeDglwtr!Z{dTt<{s)AdQO!zOnTD8T_gF1zrzj<*DInM@_6X`{_wCi*7H(IXQqIZh zl$54pAZFoH|6qUFM{PfBZoCqHJ@#7cTZ4y%%jU-#A`R$4lQG;UJZMGXkm11&1IIgU zoDU(6`)VS!`iIpzU>^Dr$F+LEmn;LSDrZ&yz>!!$mosLSY)YnM3$2@0ksmrtPwEM!P++sGDmvISRzm)9Uy zKB**=DfyyAWwlL8ot>VfF80Lbo^f;SrebYU%FwDjOXaDI+QyqpaPA z(9z)`dXUQzE?am2YCf) zc|W}g9PbZxj=ksUg@@rg&mE;yD66`0dCA^z#qzz)JKwrwYr0}yF#)?=?ws?^dKWiu zU)&K{DvvI^J#)&evgqG+XK2Yiun4_ zGTjx+N|{h%xz;*wSu)ixeC%yO?iSQEU$QE|w5k_$t|vM`D`erZ&x|U*2~~}_g-34F LNR9qcHG=U!?EfWY delta 2009 zcmb7FOKe+36rHghe_}hf96SERjg6Bwwwor+$7!688{#x+^8>vpZ3cA##j+At zO>$Ww+lZTZA}ZN)lD-(Ih1?dAQq=u5Kd4dv}S2fUQRH*E=n*R(9gFRETXKC=1eUR)eRe%hz36DqB**PSiVXE=a< zsX_m2MdW`KgR(-0U{VOee^C1z3(_a`l_`i%Y8;_+q5|Y>6gLE3l0H>5^YJ!nIt9Cap97iiB@iS)p+B9ut? z>$xe2ddY=8Tzh+?8mJ+G8u~T99q1KZy{FNJIy_r3)}UCDKCAmGh+1J~wWFe_$wg`- z{R9tb$biVCrIyMo6vFUBZFd|bkm%s4L~qcWWN365p(LD4xHa(nGFztc`ssuFAOhdu ztqw$y^gw+5Dte2@bnVX9bSMemfgkzM8eT;cY9Cs~p<%wI`>nL5{}DHZVaY>$ytxwx z`5Vohe8d>n_5;TT_+8@w4)6!YU-&nsHe-vDn`DIv97bEsc5rM?V<-R1?%{XMml~8V zeq{c8%v^5TVwqgAY|7?yYx%^=W+t60vJMDPZYmU$#WWif*u0j`W^x&MysUmRU*vkr zG#=)QmJWAFNc5YT)pTJinPW3xZopf3%I{fT;y+pTqn2q4KC(5%!|2F~<3Y`lOCzw? zB={f}IP9A?sUI3tz&tc!flcaZ3;){c#51MutS8vNC}N6x)6Xh18Vc+AJBju5<`yHs z-Y-3|jo`B{17y;_r7PJ~0`g%+b_0eE_=x=izRp+ePjQS-J47PyI(n>?u$pW#lVd56 z9rAA-xAm08M98jExGjWv*6GKs{5_|0!wziOn$4tAo9R1AmM-)rw~On{A!th@ol53e zTZLumt$d+(Wj)Cf0y1XIuwiu>AOkal2AwGMi2$*z3Y2#a$k6D4HL*T^z6eNDrGtNKqeATT*EzbAf zIW~9gHa|AG_j`^_-MfazTEkw?vDUe(R#557_ArL>cJJ#>5OgPMq;mYB0`%vrrQf{< WZ2P|7x!8;zHES1J)sJi#+J685w|_MN diff --git a/adapters/__pycache__/s3_adapter.cpython-313.pyc b/adapters/__pycache__/s3_adapter.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..563d84724a525da71365a8979fbf3fa3d1a4e033 GIT binary patch literal 5532 zcmc(jO>7&-6@X`Uc9*~UF-3`_X-O+uKc;Mpwj|rJDkoj$50(QZ@tUa%t6*tzMKhsE zb(Tsj7X^y+2LXz<$-!+3q(BSwNAAhJ84b{Mg&xwImWlv{Oqa+A)-T^t*5m{NrMQFMT?+?hi8a=B8TvRoScD;QqO zWjIP)vP45KE5p%?BlV%l4fjBO zWM;!1t4BvS+~GT+!CGkSz0lbEq4;a!5B*!`arBUFkKy(uE$1evDs>Dw&5YcvAaYH zdN)zP$aec21m>=~i7ue@5U->dlVw+-1%w;MW%Wp4>upU%amh^1v z4xC#Tp#Pm2Hb9P%l9*zxk;#Dn#c7R>fg8*c@Y{Q|7Cw8X2)kyAl#O%G_m%f0~k?F-yx zBs>lw8l5B<-t2;lO`*-)s=0TkYyTEup2$D^`__YpSJl<2^+007f9y_A_+G;W6`vVz zz~jD3lZWVchmwr^nyLro-!RZ_4y$oyc2IRhHakVs13oaD{Zt(j%>fbh7`8JHQq)HP zK4y;DdR$HtmzkjIq~AQ|M|~DUnvbFX8bN)Hd6Oe-t)nxku(ha!c11$V2m{@cAsa9A zQnnoz91$DOD{Xjw0SIY3G$kP=JSagt3_w`)1pse%43=UV!vuOBpJ~t0S^)O8GGO0? zzcyTVMODZ!rU%G8^OJB!;Pf?-rC$hHx{d61@@gV0Ucya-eQ07zyboDg5>+t|&xYk5 zWToqepD4dla-hv9IQx}FUcL&*Dw$GFaR#c%^Thm3fydxR{2+>95Z9v)vbkmugtpm! zMv2Q!W-;HiF>8rj&*W@!IWD!p_71n0#Vj}1jiqu4HZF0v+}X`IthzCt4Qktdq-8sf z4RG2!l**oP{XQy(#%e>8)uGAT!?l^Y>df5wbC=eKCfA1+YSD$)-Sz&FUo2MpkJkDp ztNoMj_fOYC(|08YN!Y)95MB6S#{e*{hvN4>l}UGWiwK_Z9fUD)i>?~$fvFAu^qrp2 zy@mu8pBcY|rvex!>34(44)P{d56eGepuIJqPO(2w^{D*ze$dQbswQN!kD(s%g9GAs zTsB8V)WmF+rmc7%3#+OQG#VB^37`Jc4q zwPu99d9BU-C1_xL3q*^U$o<4%9viRrjo%usO(v_8$#;G0edFtW7i!%X*1IpX5*BPA zHxZN%c1*wrSe;ZQcp^3(M^>L-eSAG|Y{P%tPE|k$q^cjm6965(K!F5SJKm$7Z zkdA=-wg}ogKCqcyss=^VC!*ekjm&O}dVqq98MO62Sv~494^VZ`Z}$69k7GRZi0yyW z){l9U0cOs0COg=gTY@-i9THg90t|G^t6y6x@C?3HXBj+R>)6g!Gye+~kY^s0t2E)E z*@{g-e$&L$Y;X1{iJyl@O0@IOEC3RQ)BE4#p07aLhaw`iAp%?A2OuJ;lAV2^ybqDY z(|SpV9@c@8C0O(e`I4SfQsBia8b;V&S?}SCFc7=tc)Z9q+HC^=dwgY}c*x6J%sYz~Q>=C>*<|mFx#-R;Y45Y2K zkKBiOXjeoly7T6BU*(g~-1rFyhduAZvel#2p-1aMr9L`d8$DSaJ$d_^)#$nUz@dhV z#2#x9F$UX5@j&2pcMHXap9BxQ_UvYdy?cbJRt4@Jow^p3PRg3T_53Y>@{tYy>9&;h zV|X-xGKx^1L@0X^%9G@+m>Ok&q-schM+D7eJ}{b{RE>ycmxOu{TVW~#Jw(wbV(U>^ zopPB&R2}x4F+b`PBKl0){!X}C~xlfl@h9@P4X9N83HQe=yP z)e6c7zYDgB6y|BZ}% ZNV+~Ehd(5qyW$+B;eWyhRvNaS?mw3ai}e5i literal 0 HcmV?d00001 diff --git a/adapters/s3_adapter.py b/adapters/s3_adapter.py new file mode 100644 index 0000000..611229e --- /dev/null +++ b/adapters/s3_adapter.py @@ -0,0 +1,81 @@ +from contextlib import asynccontextmanager +from typing import Optional, BinaryIO +import aioboto3 +from botocore.exceptions import ClientError +import os + +class S3Adapter: + def __init__(self, + endpoint_url: str, + aws_access_key_id: str, + aws_secret_access_key: str, + bucket_name: str): + self.endpoint_url = endpoint_url + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + self.bucket_name = bucket_name + self.session = aioboto3.Session() + + @asynccontextmanager + async def _get_client(self): + async with self.session.client( + "s3", + endpoint_url=self.endpoint_url, + aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + ) as client: + yield client + + async def upload_file(self, object_name: str, data: bytes, content_type: Optional[str] = None): + """Uploads bytes data to S3.""" + try: + extra_args = {} + if content_type: + extra_args["ContentType"] = content_type + + async with self._get_client() as client: + await client.put_object( + Bucket=self.bucket_name, + Key=object_name, + Body=data, + **extra_args + ) + return True + except ClientError as e: + # logging.error(e) + print(f"Error uploading to S3: {e}") + return False + + async def get_file(self, object_name: str) -> Optional[bytes]: + """Downloads a file from S3 and returns bytes.""" + try: + async with self._get_client() as client: + response = await client.get_object(Bucket=self.bucket_name, Key=object_name) + return await response['Body'].read() + except ClientError as e: + print(f"Error downloading from S3: {e}") + return None + + async def delete_file(self, object_name: str): + """Deletes a file from S3.""" + try: + async with self._get_client() as client: + await client.delete_object(Bucket=self.bucket_name, Key=object_name) + return True + except ClientError as e: + print(f"Error deleting from S3: {e}") + return False + + async def get_presigned_url(self, object_name: str, expiration: int = 3600) -> Optional[str]: + """Generate a presigned URL to share an S3 object.""" + try: + async with self._get_client() as client: + response = await client.generate_presigned_url( + 'get_object', + Params={'Bucket': self.bucket_name, 'Key': object_name}, + ExpiresIn=expiration + ) + return response + except ClientError as e: + print(f"Error generating presigned URL: {e}") + return None diff --git a/api/__pycache__/dependency.cpython-313.pyc b/api/__pycache__/dependency.cpython-313.pyc index 191971852965bec3ce04b0d789f354d8ccfe9fb2..515c6ceb918a869adb29828641cf83114e4c85f0 100644 GIT binary patch delta 1154 zcmZWn&1(}u6rcUb=6kdGPOR0~#+tO+fK8D$equprq0$aQ(S!Pw?nPNXS856ufv4J*2eWeY4rL1_$QNyx+XvdvAU-AHBaM@xbeKAs8<| zJS#5=+hUBJX0}rc4TSA;-dT}|34{@L%=u;|BEjA%GuS2bSsuGjpf#TOEkppsVUs?bW3f->u_RkGDC^1!*iWfz3<;az z;XZPp&kOdvEQbTK3kPKZhhz`rabz7hY~_iRBI)Q)T0hMi_C@8vhN5bQo?#Z_X~uBk zi5n}0x~33t9JlLQxmGPy^e4#Ju5BC-MdB>YQPR zY6VRr$Jgt-53sro9h1lqfb@zgHJj+GJFs=>O>R5)DVS^plW!|~Gi~=&lb<5cYaEdf zM3DrTA`7NBcs;|%8cWQi1tt-p@(?GOG3IT_g~e<(zf!1?BzUJ`sy;BA=+J{a zz88<%?o^Xc(Xz&v2TdCY3-S4|lD$~qd zYoELRW#qzcwLNlW-zG5W&w*rzgY~-<+ljZN;SL2m0ur2m?6BJp>Cc6mJeB`16`k(~ delta 729 zcmZWmOKTHR6h3!mGMUUf>BA&VbXr3(q0|SFiWWg!*p(oIE_9&QG=`M6fjf04MAU^V zrCbF42l@xxxpz~TkljEwLUG}yv34nV&Lk0v1NS_>`<-vjeDl*iuY(gB}8@$2?l zYTwRFzhM8uy>2>Sz7{BT#m^LBOW~G3LR3Wb1hL16iHMOPP9QBnU|nS-2vBFf87Qct zBf}!b7UOQhaVMOlvrlynb&H9B?Aw9vJAvxEfr&y|UGuYJp=>Kxof1!3LwslNm?^mQ zFg1fyj_64*mmI9BGIufYEQZ)>HFw&(?e)fXyScl^rw|KiPmfQFu6%|S#DP@K)OtoO zQfhowe3c*H_)kQB(m6d!%66-{w~iMR=@No7xJ04r^$agkJU)vE&tYhcoxu`79-rep z7VqKFzq|V|@uLEqcHkKdRhCFhmPRB_)0;OpI{Ykl1w0|KU%+n_Oy|w>V&5o5N-24L zEF&*gM<(z&ahSSRNu<*_fkvIf#61dg7(&|3^T?Z9YH4u+J0gVSB+BbxmOuEP&i9Sw zNLfzw?^bi-mHPGpr(eKn7fwDTn{k@|3?mz+v5vLXozCX7?dDBh!~W{mxS z$^fn$!sP+14xn(N1Ivq#>Fe`D4KA%jD|h-A*M=aoa#UP8qWsN2p)|BbMZ2w#*M9*J Cjg9UA diff --git a/api/dependency.py b/api/dependency.py index 5cd2d5f..3dabd86 100644 --- a/api/dependency.py +++ b/api/dependency.py @@ -11,6 +11,9 @@ from repos.dao import DAO from aiogram import Bot +from adapters.s3_adapter import S3Adapter +from typing import Optional + # Провайдеры "сырых" клиентов из состояния приложения def get_mongo_client(request: Request) -> AsyncIOMotorClient: return request.app.state.mongo_client @@ -21,11 +24,17 @@ def get_gemini_client(request: Request) -> GoogleAdapter: def get_bot_client(request: Request) -> Bot: return request.app.state.bot +def get_s3_adapter(request: Request) -> Optional[S3Adapter]: + return getattr(request.app.state, "s3_adapter", None) + # Провайдер DAO (собирается из mongo_client) -def get_dao(mongo_client: AsyncIOMotorClient = Depends(get_mongo_client)) -> DAO: +def get_dao( + mongo_client: AsyncIOMotorClient = Depends(get_mongo_client), + s3_adapter: Optional[S3Adapter] = Depends(get_s3_adapter) +) -> DAO: # FastAPI кэширует результат Depends в рамках одного запроса, # так что DAO создастся один раз за запрос. - return DAO(mongo_client) + return DAO(mongo_client, s3_adapter) # Провайдер сервиса (собирается из DAO и Gemini) def get_generation_service( diff --git a/api/endpoints/__pycache__/assets_router.cpython-313.pyc b/api/endpoints/__pycache__/assets_router.cpython-313.pyc index b406e970a089b97d98cb72a1a539ce4eb1506503..467c20944cc66bf07342301b9f3f9490818b96cb 100644 GIT binary patch delta 608 zcmZ2u*XPXlnU|M~0SF!*Y0aD@F_BM#Nr`!*#w{j^7}cP7kOV3S)|mW(QB2N+F<2}{ z)`TfoQ;H#rC5zD%s;bC1omW$9GaK_(aXu*qaitBc7rDJIuy}n`oqSSiwRV-Der{%Z zQDRAId`W(MZf0I)eljDH1uQ^zGZ25?z%W^XOM}Z4DyhIQc>%B3SJip(J0yH#5)EUyrLwI=CdUs3bElT>;|O%=|pKv|p7F#2AoZ zQEG8%PKlL*CQFem}LMqw| z!WMWyRdNe9_o?q(`J9!gN$DCjNXh3g1<0;=+DdyOzIz)K+F$Lj0{X1?b40X7kCveGb-Lx(_N8% cSp@K2EMGcPX}0}wns+L{^1vXM`diOGs_vL(~RdNx*|a147empYIRW-8*3VU2-_ zaYMxfki>YPVuC<1Zm3>fkeEJ0kx+~hmvRF)kMu0$(WUyF~R63KU_+)G5Fk8Xg%)HF}`23`- z)Z~)*yu{qpTRaeL+pVrCAq(#aE;=P^o6)@SL`&;}W;10n>$ zmVgK;5TOSm^nrvlh+(k#A&UwlqugXRHX}yq$(C&KdLXA3X@Ce#5WxW=xIu&wh+qa0 zCLjV7=0%p1x3j4+UYmT6EnHh&`vQyTgo*{s6KgNCXx~s$Yw($%F~fMG?qwFW8-}JA uSmYN-tWaJkdzr;_vI+YtM(fG<+0_{xC$n=H^Gh%ab$EVd08&MgKyv~5!)BBK delta 304 zcmew-cv67xGcPX}0}%Y%){=RZc_W`D6VpG2$(BqL>$zEh!ZGZ@TC(^uDb)lKf*?W?L`Z=M zZ4jXYBqTr#-ObUgDvXTMlN;EK7$qieW{a2S0Eu&h2z?O23?d9c1jye-W|JM*)fiVz kPG=9F{EB@gqs8PY9O{gAllO2K^NTVHb$EVd08(K40P%P_cmMzZ diff --git a/repos/__pycache__/assets_repo.cpython-313.pyc b/repos/__pycache__/assets_repo.cpython-313.pyc index 3ad0ca821402074cde6928395210a1a92b0ca5c7..fa07fdbe83d34f4920b21aafecee2b0398555bb6 100644 GIT binary patch literal 14364 zcmd6OdsG`&nrD?%r56GT&;yBw#mgWtFM~sj9|R281{>RyWs`tMgF(i!Wr>u8?YQ@> zdonL_X8RbXI}1LOIVOMX3E8uAnx50WBTvsva2zL0_x@2eaz~nKx5s_9xA$bv{sD1v z(&_zU_WN$BN-`GdbnhRt*Vb3}-ny^)?stFp`@Va5nxAi?ApGltKMrm*Qq+IPgc6u^ z5~YjGvx*we-$To3zMzDvqBy;q;tYE<={(D{#%ufp6JCwP91BKclHtTm zEEow-goeE|B+g$7Ova8*LAVwEIA)w3*RfEfgW*^=b&0t|^LkD@&~POr<>Xv^3CzY0$wW9B9SO`tKFNb_h9%u3;kIOow)-d2JXYig z3Q#zkTKGM!2l6I0T1-YqBlS@spmG@XgwnD=QPVKt(C`FKUi4j!WSWeGLNL=X^O7do z?qwt<8VFtB%}|xk2O{aG&ilg?R{~M&*#kl9Zt+C}e6;2IRJ3I(a5WH$%(TqB=9%Fm zC^ONPiD1*@#RCTUd=9$+dgWn$v2V;J}RGj(dW+fB5j@p5IlS;Wg zw_8@~MR)rg^B*j!qOx!4pTY}DVo8gVjt|6U`S4^~tnn44gWn&*?`andxSH|+V|>&I zzS$CxX&4UE6L*1O^hkCq@+(RyGx1}J8biL40A;b4En&pCB?FRs(pk!43~QjSvNW=a zkKjjd2?YQVJ`GT;Q?TVFBnzl^HmuG4Z6k%xmmh7*^@R!~90drv2*Pxgep5TlnyE13 zW7MzrQq*W$Nd3NT`m{aYr;iSyD3ibek28XJqIygPu8~ruOy#o;-(=p{ubwwf*RBh*+i)@0%$>^hjoZ$g&ZkqqA7}1V$fEFNn?AqJ%&8Li z8)r`cHD`_*@hIcUJV&ZCaU}-r)hhAKbL-lAPJ;DgbxLWUener3k_AsiifSkWpuq4q z4Ak&4{3<%N7ih0xSfX1b+JD12FcAy|raZBTXLcqOnV9kr@bTO*V%G6s_~?mW8(Lx) zXRn-(w*1=UC$Z*BGt)PAJXe#v3qyOJ3lZMqwev+V42hYVh)qb^U^oVN%m*f7fhqq) zOftrT0K2h?D>D)kdu=8lu~#QTvjK@gIF|C{!T2x0;H1JU0H_gvgzVY`uBE*5vy+zt zF*#wD-^*E&E)sxKpIGgSrbW{ud&!bL z>GFy$Z_?E+y4n-2Jqh~@^O{uU&Uf5zyOWh|Vr84)*}Yo1+Pm7Z`fA+%p8KvlZWb!~ z=kuOa*1qF>+nKEF5Gy-Y$3AMf*N~_@o-92+Z+Ki>nrhzr(AJppHhtImt~2S~FM9XS z_dm4lOjXv-_rE##3!8iK*upWPx+}ity{@}m@m*^{q4MNI+o=s5Rp?l>ELyf7ou~F4 z)7z$`r&aW{3Om|XOX5{=Tl`df_PrZ-Z^T3E4MMqZ-t?rd@ttdLUrW|?iFIA^u8(fs zyP2pvo2)!LU+}oNGPU>6L;DM8Y+aBctkYeLn_5ACnO`0TF56ALGjwr0`RyfT|?=@whM6D>UnTW_+UcTW4r zQZ(Pb*uBu5uv80%>Q7s!vSE7DO;wH3n?;o65dCHJMVOY?d!0v0>Gxgq5gYTq`%oDq z9=t#waWW4&d$o}GsEa=0Vm{jEhWNUZMhaz1LF)_!uLNU~E^^@lC@_-oS}=CekE>@A zwG$<%D!?ycOP5uER@~6E3IV{$yZ`NjRHQo)CM1|zE!tRr6!D51+M2Hk8PpCIv-n19r<4r71ocT21fz_pt=bt zdC-zh=|_3`^qd|!F{W-qoB_U7-$%bdhYcAHPP{|7h48f{C_J9{k5Ixel4EO8OLT45 z#)xxEqmazqmNM?=IAQb|)qX%8MwddWiUlpXpKo6+{o9TzM_b#D%H%Vt{otrpLh82a z(~azSepEh#(w2uh{=X?I4-{IT4mYgfU%_t{e!IwaWgO;Ff8{+$nvhfh`Q%?ANLqkn zNfVrUm-1R=6lX33!&9K@NB9`f%fT6bAC#8#v4}qe#vtE@DeX*ThDX^YyA+QDmkiUe zrQnt$chrn;u{SU&W@ZB6DQ}*P`q&)5AISlH%S`aoQ685*k76ikry`SNXCTc;EZGC1 zDC-2~Wg>&U340_+AC>G`P@l=h!A5J-K%cS{&M}WIC8^S~xxRVF8^<=PsLCpcmWt-` zC$4a@}($*>3I^*WG^9kG0q-|KV4XI8o9#cW9n@Q8kn-8=61; zxTI{QxC8#Ds_WkAe7iGQ-7Z$Qul6T9d}4<$QGF&^erDeHxTJj5{?^SW z=Av82-#ETh`?1-xxr4Gf;j68Vx%|yK%EBY`mLH=xO;p8M1%P{6``hSyyXpQ$=3aX* z1Bv^cbbpiX{);^(NIcNc{Vlo&S}(?%XrwJjB?FlWkf+IPP*ef;qzbqk1-6Ms5%wv7 zNREbonr6+mHcNJkFr^qcZMM~=EQMCur&Yl9xrLu)w@iUdOjBbj{Q8(&V9Tl2gwj52 z3=9E56?{1s0iS>%==RE2r9n^y7J1A_)e;DTY%c3fEg{2Ajy7`HaJk!3#{C>GvMm9G zn?RNg$9@fH{`#ZJ0rT6AO7GLFUyq}j&De-OmO$Gt0M>pVeog{yIvLu$nk;a837E|H zAn66-HOsJOKv+Z7l*cWZpcsD`W6z=(e+2V9NK^n*)YTK1OEAlnc4&ZWL`E*uWaV1N zAeKWfHL92-*>WM(I9B^95Y$LhcGsKyE6eMhQbK1GSH;j+0_675jcELQy{xZ+D zWV_?M<4jp>Ny|?7pDM4sbN2SxWOzaic`swQ}gWO;*ypAw_2d^oO!d7vYbJ9DLG7U8YtHY zzzdpXukY>KN&k@U^Duwc%R=-%P4`tX_cc8xNPJ+X`)ZgE@@*KeqLJ1h&5bDk0H~1m z;%L6gi!%jY03WT&6-@YOQ3!@2hZ6;3D!PEWM}zM~Rlou?sD0V$W4eyg7WjvtW1l9M z&qpaU@t;6t%J@e#Iek2;GNz0whkpn#GZ$KnpfR&05YLbnDAviLh!Ot~ll$97(2m*? zMtgI8;dVHyemO)&d$U$>PG076Qu{O^l_TIA(uC9*g75h%pU$8CoA4UUhp*x_$EJNc zIV343w?}e)<8S0PaBOrW{%czi=k;*NT3Q5IJ0)_!Kf=c0*kjuBi>9Tw$sxi_^ zb9$eymELZY9@uS(C&dcbJqABh+LLlahC|QMK2JCj1K-DNc&f|8<0SLtNGgz^`pdrr zyrU;HX-Yw9xR66Ni9Uzq`N_MHzeI1qPrvEej(xLoG(gM)xFd3c7v=>3YG1vZL1@zeQ`Qo`(DG{hIpB< zV_?2C<*v9>ce^g>ZW7&1E0@vnkZ=zqZ392C4WwH4J#;js8e6{G`EF;j@u1jva6a(R zQ4gBueBe8m0P1huymfQ?{f^iPHSvix_9N3hQ~ckpcL^0|=H1VZ;L>|7cUuzf z!K7{QC$_;cxWkmD=x@#Oo7n zhXilm+STYYD@(FQceczTUEz z{vO@Q{IzZ`4gNehE}EExBSd6!Hh>zS=pVxG=}WL~etRbSfRV^jpiB$Rv3s?-d^K6V z7+l(00+<@mH-?LY0%fm9F*hA)GE7C8>gmepA>+Td7FhSP40(|jL1WZCM$ z7*Vo~vTPl~3}WB+V{8Bkn&7ggeH3HJMg>d#m_drekkHp1nT!%A06zrj=fTvjD$!B@ z`PYGUlGG z#|VjMjxup25b3`Jk>6JOvy@?I2bJDd`UKs&t-KP$iy?BN8RjRTM-U0JOIjvjzmEA~ z;%2YF;eQnJ38vu8Mp1SYG_j}?<0n-lcdkj3-~A<4-UTFW3(3>fA=*1WemNk#FtBbG z_xOa%R}%Jc(h?R7VbKy!Lq|h9{a@%d=C9axxP3tOT#)4GGn`X#qb7qJbTM^C6Vi@Ktf(fM>7i`orff{G`QOoxoWm(zj6Vk;)VDvoMA@C$1$pa-$n1ntyN zWX|>SR&qmI6AWiiA$?x69B$^jknsg#LMNZ^C})Z(S>+-?_MS{bNV{T`;c&`yo|*s1ggRmit%wSG$D9 zgF>$uk)h|@M4F6Ci@3~C>Tv{Hf@<wpuveqj zvfW5MXolzNmr-i`&iyjKq2HWWNOJDu+o*QR2%qD=)~C&7@^ft0A5j!R<$r>;dJ}&A z@N<#X3NEpifKLOmkwctV2*lvTBJ1-M#O39RegR1f3ros_quK!2I^?|AluqC&ZoV7` zod99p%VZQr1zpiukG^~L3Q*eYNjE|u_!+Y&`^A68iU}YD=)nqUx!sa6r-(@BR{ba+?1 zLdRjT;Yh-9G-*95SdXfu_a&|SMeF|ON}m!NMiLG#Y2^efr#jp4By*Mh{`ECOqVuTG zGMKOpB@2eY#cwG}T0Ek~v)m?HY6L^grjxRtq&Lk};UN8UEBEt)p)aGT{Jh@Kr-u#c zP&e~eJvNBG56&){d0(>+;|J(IE%W}tCWzl}r2AOr{w^%@0h9r%C6q-vscuEkB#JoM z_!(G+U$y)dai1bvJO`FT4K6p7g)JfHmPIxyj>YYY(>1GGU>NNDum*lwA5FF`=GmJ; z72pf!$R#{fa7S_|&74sYcknvoE@9&G6hA^(zfBo4m#>yFY*VH{?J4`n4~+LLf1EKF z*eHGxrTz*0YLm|ZULK3j1k~zd3DtuqO&`t!zHZ?uCUZf+=zmp44U7F5-YN4n0fMv5 zJmpaJIfIXd87drI%{c;4SGEQ?elBX(W|WYv<3J$gikrX;32Tmj3S-fxQ`+NSQm7hy z2ES4GIV!0!#1Inn)C4Hk?ay^k%h|S>DfK85xN7&AfvYNP!+6vY>;RsH%{}JB z^l>mogk9_mZK;hlDuOZ=1MQ*~@ zZNB&aX9+LaD&d@~-4HJP`ZDe-^jV*g@@F`x%K4GsMb4q^+-ohjq`RM!^k>EbvQ?0l z?tJGe108@?Ql zTniHqti%SdOu)smDlC(6?xJ@J{7|F~;)?Y^#wWPn-Ej56-5t-B;50vhSCBpALK?Xn zcSF>-ROXN_<0ppph}NUJc5hUiR&KH7Y!7ZW=ga+o_k-NG%Ojmd(1qY3YhFOk~2qr6$R3ky#O!mHcF?JS7BNF7hw}|LgQr_e&4_7bbp5X>Iv95s; z{5P;9x1q;PO$k@aLwoC5&%EYI<<4dO@~dzKgjwlXxw!HQT=28SbwWk= zd|t{`e#d#+nRM+EUAtDs60YuKaraM(yU~A5bXWAfr(6}w2BE~8a_w0CM)LHSczP^( zdRja^ExbA_R$diKu03V7rH(Hst=+Lfh=H>2Sgu?+@#lk2Y&$@wwAK9i&}KD!?WT>g zy+m(npyaPUE1+!UPbp1)J20^=c_ZEPct_!QL+nItIWgy6YD*-OV^e!uJ9{UaJTZv zY8TvIjKwdk4GQHa=1tEs|CP91X8*_AvwzJkoSR6Vn-b4WCC|->=Vmr2>LpE|W|N|- z`ZbVn4QMiRv8l`js`WSpPi&s$9?@3&?PI@sR9uFvuYOeMTr68C6Fdjw(f6+1y|y;{ z!+$Rv99};!oEi~Maze%FhlRdWVfk{QSXi}KMwnksn7AnNH59*8qlbmWk4ovm3f+&&Oc>vZ zz5S@3amZudNg=++BKN|}~p>y1g`4S{}lFpOwLxN7ZXgTa{e?o0)G;BHC z(cPfn8Na@66uU=0$M}X}2V1kkZcy-8t^N!jo4vG`bu3SAQ1FOr$z#2bJV3gI^V8zF zi=X4W%`>!}wXZ-4c*Gg@flXyu$kFf90)vCJ3hn5M%ha=Yg%ExpztYk4*BJ^N;=rvg5ha-hHyS5 zJ>E};ESS0|T)siij8uFD|NN|(P`C~Ve~}i95D}2f2!T9!rG;P!r48VS{Fk*7t}Mqk!PMB^wS(wG z^~4@>G_8={RlwSbNc$3`ZDi^`uzhMKO^T*Tvt?2ou`f7Ur%uD_Bccr}T1hw2RS(g%eG=*w;nna63n*;b+H2HZ@Tl5O?RkpcEVMU?`_!|T(xkNrU zmB?luSU|I(#Zt*cIv2rVr~FKW{I45hR^3?jXYu5`TaWJ=SBe@;A>p1HzpS**IWZ- z*TB-;+McoUp0UcFV8wN8&2g+K-*+}%AHFfRFjjH;OJ@J3N}R{(lbjQdU)>$F(YrKg zQIn9qatp&Z+SIR`1Q@w@=AZ$-vd< ztE<7i#}3n#A$sg3Wo0;MfrVifiO9e7wjKBeMEt~ujfHOgNdrn+o|gJx2S*eVP^Z(f zF89JpP)aXf)I9uXgNv$sx?zalXy_`a`=E#5sLj*RC-BHQOl&c96NF3|r~9c0SicCE zIrIPz1{4;6$xtx+4JH3}HeopOvk`Dw>lryQQDh331ao*|UVy$vApy`yu9E)TBK za`Qyd`lZda-ql^4xPG~8?_A%}3)H)1yPqF*wT0GqA9yI!=0lGObsmC3aJAf&Z<}tJ z7S)d(6<2RjE{@MzK6kq2-@Bfhf3IZrZkj;wNe(mNYETZ2&=r*ihm;jziJ-hf9`N8x^&LM|K8fHLa?T6P}m-6FXb7nl_%Prg?C({fB9Y;_oXv= z-}%g3IzH+Xj{A?&1Dy?=h7L3=C$7X2S!_ksBCyV7p>z^Yz32t0z*b`rzQgYDv@xI# z`_>teS4Iom%xxZ5VE_fs0tojxUDvK$yYhLXx70bbd~kVZsr_)J@kq&jWZlttEi<24 zbM%!ReM{T_(0Z@0GI+cc82{|@>fG97q&yj^Oh!wu%v2n)HFKA%|l zN(&`A~i)bKDO;;x}O)e{e_13xUV>f{~N&5L>F_$Al7*-oViQzy~+V! zG~`| z{x?orO{#m@wH*6>+XeyhUjDNae|+~_BsUr*bvvGdX?f%^PE}j*^`U^((BO-ZOwU#w zgZA^Y2M&g+Rz7dJVSf2iCO(%+9AR&P92ib=9RM3LrSzYq>ub{X1sVF9bUxNjQo8-$ I1W{b}KO9t>C;$Ke diff --git a/repos/__pycache__/dao.cpython-313.pyc b/repos/__pycache__/dao.cpython-313.pyc index 94cbfc8d4e1f862b20854f85420833de5af28ecd..8e0d2f5132594fb3bf5e8a7b517c964b1938421c 100644 GIT binary patch delta 603 zcmXw0J#W)c6ur;y#UF7Upi!f!rK*KQiw96>+5y6cA|WAK(IOrdARf4`5&fRSZ}}Obl#=2)Z!zKD#{0=broSz3151|FqaBj4dkd9f450yna1aV7f^v=12IH)eYp6M_E-{{)hHbV@ zgvM2&w-rZE=(YO#&k3xqCF*9-@w-uy535brJDYj2BYhd92*qc-g)oE%1Vei672^$J zvkkgT<40tc6yqVeZnK1iey}Ah>=6ZoMDtqBuDj!hQq_+Ck~^v_hE9A-DP4GXcdvAL zm%TQ>%{Xu8_hv5cvM+jhj1AyWaRP%lE@4t$DEPjuUexqjjuF36YNjqrhJ;f5Rhhj} zmUK#j`5)wGhBm@0_@&TX?bF=(eZcts@kitH)!PS>rxzj}mUC;)vA9Z~K1+Eu2T#r-YCZEdGEqBXCAg V8Num4Dv*0-rbWl{AKhT*uTYc~wjb(visxRzrW_rMZT7?+SI%pn73Mc@WLF$ol(HuEzOej>3D z$dR%h;dkV+vJkFI_5U(2c+l-P2OaB>Wko|C?Q9542tvq*y*JqLLHA+Thpm~apyC$@ G%=JH<)?CQ| diff --git a/repos/assets_repo.py b/repos/assets_repo.py index acbd33f..5eadc2b 100644 --- a/repos/assets_repo.py +++ b/repos/assets_repo.py @@ -1,52 +1,145 @@ from typing import List, Optional - +import logging from bson import ObjectId from motor.motor_asyncio import AsyncIOMotorClient from models.Asset import Asset +from adapters.s3_adapter import S3Adapter +logger = logging.getLogger(__name__) class AssetsRepo: - def __init__(self, client: AsyncIOMotorClient, db_name="bot_db"): + def __init__(self, client: AsyncIOMotorClient, s3_adapter: Optional[S3Adapter] = None, db_name="bot_db"): self.collection = client[db_name]["assets"] + self.s3 = s3_adapter async def create_asset(self, asset: Asset) -> str: - res = await self.collection.insert_one(asset.model_dump()) + # Если есть S3 и данные - грузим в S3 + if self.s3: + # Main data + if asset.data: + ts = int(asset.created_at.timestamp()) + object_name = f"{asset.type.value}/{ts}_{asset.name}" + + uploaded = await self.s3.upload_file(object_name, asset.data) + if uploaded: + asset.minio_object_name = object_name + asset.minio_bucket = self.s3.bucket_name + asset.data = None # Clear data + else: + logger.error(f"Failed to upload asset {asset.name} to MinIO") + # Thumbnail + if asset.thumbnail: + ts = int(asset.created_at.timestamp()) + thumb_name = f"{asset.type.value}/thumbs/{ts}_{asset.name}_thumb.jpg" + + uploaded_thumb = await self.s3.upload_file(thumb_name, asset.thumbnail) + if uploaded_thumb: + asset.minio_thumbnail_object_name = thumb_name + asset.minio_bucket = self.s3.bucket_name # Assumes same bucket + asset.thumbnail = None # Clear thumbnail data + else: + logger.error(f"Failed to upload thumbnail for {asset.name} to MinIO") + + + res = await self.collection.insert_one(asset.model_dump()) return str(res.inserted_id) async def get_assets(self, limit: int = 10, offset: int = 0, with_data: bool = False) -> List[Asset]: args = {} if not with_data: args["data"] = 0 + # We assume thumbnails are fetched only if needed or kept sparse. + # If they are in MinIO, we don't fetch them by default list unless specifically asked? + # User requirement "Get bytes ... from minio" usually refers to full asset. used in detail view. + # In list view, we might want thumbnails. + # If thumbnails are in MinIO, list view will be slow if we fetch all. + # Usually we return a URL. But this bot might serve bytes. + # Let's assuming list view needs thumbnails if they are small. + # But if we moved them to S3, we probably don't want to fetch 10x S3 requests for list. + # For now: If minio_thumbnail_object_name is present, user might need to fetch separately + # or we fetch if `with_data` is True? + # Standard pattern: return URL or ID. + # Let's keep existing logic: args["thumbnail"] = 0 if not with_data. + # EXCEPT if we want to show thumbnails in list. + # Original code: + # if not with_data: args["data"] = 0; args["thumbnail"] = 0 + # So list DOES NOT return thumbnails by default. args["thumbnail"] = 0 + res = await self.collection.find({}, args).sort("created_at", -1).skip(offset).limit(limit).to_list(None) assets = [] for doc in res: - # Конвертируем ObjectId в строку и кладем в поле id doc["id"] = str(doc.pop("_id")) - - # Создаем объект - assets.append(Asset(**doc)) + asset = Asset(**doc) + + if with_data and self.s3: + # Fetch data + if asset.minio_object_name: + data = await self.s3.get_file(asset.minio_object_name) + if data: asset.data = data + + # Fetch thumbnail + if asset.minio_thumbnail_object_name: + thumb = await self.s3.get_file(asset.minio_thumbnail_object_name) + if thumb: asset.thumbnail = thumb + + assets.append(asset) return assets - async def get_asset(self, asset_id: str, with_data: bool = True) -> Asset: - projection = {"_id": 1, "name": 1, "type": 1, "tg_doc_file_id": 1} - if with_data: - projection["data"] = 1 - projection["thumbnail"] = 1 + projection = None + if not with_data: + projection = {"data": 0, "thumbnail": 0} - res = await self.collection.find_one({"_id": ObjectId(asset_id)}, - projection) + res = await self.collection.find_one({"_id": ObjectId(asset_id)}, projection) + if not res: + return None + res["id"] = str(res.pop("_id")) - return Asset(**res) + asset = Asset(**res) + + if with_data and self.s3: + if asset.minio_object_name: + data = await self.s3.get_file(asset.minio_object_name) + if data: asset.data = data + + if asset.minio_thumbnail_object_name: + thumb = await self.s3.get_file(asset.minio_thumbnail_object_name) + if thumb: asset.thumbnail = thumb + + return asset async def update_asset(self, asset_id: str, asset: Asset): if not asset.id: - raise Exception(f"Asset ID not found: {asset_id}") - await self.collection.update_one({"_id": ObjectId(asset_id)}, {"$set": asset.model_dump()}) + if asset_id: asset.id = asset_id + else: raise Exception(f"Asset ID not found: {asset_id}") + + # NOTE: simplistic update. If asset has data/thumbnail bytes, we might need to upload? + # Assuming for now we just save what's given. + # If user wants to update data, they should probably use a specialized method or we handle it here. + # Let's handle it: If data/thumbnail is present AND we have S3, upload it. + + if self.s3: + if asset.data: + ts = int(asset.created_at.timestamp()) + object_name = f"{asset.type.value}/{ts}_{asset.name}" + if await self.s3.upload_file(object_name, asset.data): + asset.minio_object_name = object_name + asset.minio_bucket = self.s3.bucket_name + asset.data = None + + if asset.thumbnail: + ts = int(asset.created_at.timestamp()) + thumb_name = f"{asset.type.value}/thumbs/{ts}_{asset.name}_thumb.jpg" + if await self.s3.upload_file(thumb_name, asset.thumbnail): + asset.minio_thumbnail_object_name = thumb_name + asset.thumbnail = None + + model_dump = asset.model_dump() + await self.collection.update_one({"_id": ObjectId(asset_id)}, {"$set": model_dump}) async def set_tg_photo_file_id(self, asset_id: str, tg_photo_file_id: str): await self.collection.update_one({"_id": ObjectId(asset_id)}, {"$set": {"tg_photo_file_id": tg_photo_file_id}}) @@ -64,11 +157,10 @@ class AssetsRepo: async def get_asset_count(self, character_id: Optional[str] = None) -> int: return await self.collection.count_documents({"linked_char_id": character_id} if character_id else {}) - - async def get_assets_by_ids(self, asset_ids: List[str]) -> List[Asset]: object_ids = [ObjectId(asset_id) for asset_id in asset_ids] - res = self.collection.find({"_id": {"$in": object_ids}}, {"thumbnail": 0}) + res = self.collection.find({"_id": {"$in": object_ids}}, {"data": 0}) # Exclude data but maybe allow thumbnail if small? + # Original excluded thumbnail too. assets = [] async for doc in res: doc["id"] = str(doc.pop("_id")) @@ -76,5 +168,81 @@ class AssetsRepo: return assets async def delete_asset(self, asset_id: str) -> bool: + asset_doc = await self.collection.find_one({"_id": ObjectId(asset_id)}) + if not asset_doc: + return False + + if self.s3: + if asset_doc.get("minio_object_name"): + await self.s3.delete_file(asset_doc["minio_object_name"]) + if asset_doc.get("minio_thumbnail_object_name"): + await self.s3.delete_file(asset_doc["minio_thumbnail_object_name"]) + res = await self.collection.delete_one({"_id": ObjectId(asset_id)}) return res.deleted_count > 0 + + async def migrate_to_minio(self) -> dict: + """Переносит данные и thumbnails из Mongo в MinIO.""" + if not self.s3: + return {"error": "MinIO adapter not initialized"} + + # 1. Migrate Data + cursor_data = self.collection.find({"data": {"$ne": None}, "minio_object_name": {"$eq": None}}) + count_data = 0 + errors_data = 0 + + async for doc in cursor_data: + try: + asset_id = doc["_id"] + data = doc.get("data") + name = doc.get("name", "unknown") + type_ = doc.get("type", "image") + created_at = doc.get("created_at") + ts = int(created_at.timestamp()) if created_at else 0 + + object_name = f"{type_}/{ts}_{asset_id}_{name}" + if await self.s3.upload_file(object_name, data): + await self.collection.update_one( + {"_id": asset_id}, + {"$set": {"minio_object_name": object_name, "minio_bucket": self.s3.bucket_name, "data": None}} + ) + count_data += 1 + else: + errors_data += 1 + except Exception as e: + logger.error(f"Data migration error for {doc.get('_id')}: {e}") + errors_data += 1 + + # 2. Migrate Thumbnails + cursor_thumb = self.collection.find({"thumbnail": {"$ne": None}, "minio_thumbnail_object_name": {"$eq": None}}) + count_thumb = 0 + errors_thumb = 0 + + async for doc in cursor_thumb: + try: + asset_id = doc["_id"] + thumb = doc.get("thumbnail") + name = doc.get("name", "unknown") + type_ = doc.get("type", "image") + created_at = doc.get("created_at") + ts = int(created_at.timestamp()) if created_at else 0 + + thumb_name = f"{type_}/thumbs/{ts}_{asset_id}_{name}_thumb.jpg" + if await self.s3.upload_file(thumb_name, thumb): + await self.collection.update_one( + {"_id": asset_id}, + {"$set": {"minio_thumbnail_object_name": thumb_name, "minio_bucket": self.s3.bucket_name, "thumbnail": None}} + ) + count_thumb += 1 + else: + errors_thumb += 1 + except Exception as e: + logger.error(f"Thumbnail migration error for {doc.get('_id')}: {e}") + errors_thumb += 1 + + return { + "migrated_data": count_data, + "errors_data": errors_data, + "migrated_thumbnails": count_thumb, + "errors_thumbnails": errors_thumb + } diff --git a/repos/dao.py b/repos/dao.py index e8c03f3..f43a5e3 100644 --- a/repos/dao.py +++ b/repos/dao.py @@ -6,8 +6,11 @@ from repos.generation_repo import GenerationRepo from repos.user_repo import UsersRepo +from typing import Optional +from adapters.s3_adapter import S3Adapter + class DAO: - def __init__(self, client: AsyncIOMotorClient, db_name="bot_db"): + def __init__(self, client: AsyncIOMotorClient, s3_adapter: Optional[S3Adapter] = None, db_name="bot_db"): self.chars = CharacterRepo(client, db_name) - self.assets = AssetsRepo(client, db_name) + self.assets = AssetsRepo(client, s3_adapter, db_name) self.generations = GenerationRepo(client, db_name) diff --git a/requirements.txt b/requirements.txt index 696b62e..7a19a3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,3 +45,4 @@ urllib3==2.6.3 uvicorn==0.40.0 websockets==15.0.1 yarl==1.22.0 +aioboto3==13.3.0 diff --git a/tests/test_s3_connection.py b/tests/test_s3_connection.py new file mode 100644 index 0000000..deabd41 --- /dev/null +++ b/tests/test_s3_connection.py @@ -0,0 +1,44 @@ +import asyncio +import os +from dotenv import load_dotenv +from adapters.s3_adapter import S3Adapter + +async def test_s3(): + load_dotenv() + + endpoint = os.getenv("MINIO_ENDPOINT", "http://localhost:9000") + access_key = os.getenv("MINIO_ACCESS_KEY") + secret_key = os.getenv("MINIO_SECRET_KEY") + bucket = os.getenv("MINIO_BUCKET") + + print(f"Connecting to {endpoint}, bucket: {bucket}") + + s3 = S3Adapter(endpoint, access_key, secret_key, bucket) + + test_filename = "test_connection.txt" + test_data = b"Hello MinIO!" + + print("Uploading...") + success = await s3.upload_file(test_filename, test_data) + if success: + print("Upload successful!") + else: + print("Upload failed!") + return + + print("Downloading...") + data = await s3.get_file(test_filename) + if data == test_data: + print("Download successful and data matches!") + else: + print(f"Download mismatch: {data}") + + print("Deleting...") + deleted = await s3.delete_file(test_filename) + if deleted: + print("Delete successful!") + else: + print("Delete failed!") + +if __name__ == "__main__": + asyncio.run(test_s3()) diff --git a/tests/verify_minio_integration.py b/tests/verify_minio_integration.py new file mode 100644 index 0000000..12c762d --- /dev/null +++ b/tests/verify_minio_integration.py @@ -0,0 +1,84 @@ +import asyncio +import os +from datetime import datetime +from dotenv import load_dotenv +from motor.motor_asyncio import AsyncIOMotorClient + +from models.Asset import Asset, AssetType +from repos.assets_repo import AssetsRepo +from adapters.s3_adapter import S3Adapter + +# Load env to get credentials +load_dotenv() + +async def test_integration(): + print("🚀 Starting integration test...") + + # 1. Setup Dependencies + mongo_uri = os.getenv("MONGO_HOST", "mongodb://localhost:27017") + client = AsyncIOMotorClient(mongo_uri) + db_name = os.getenv("DB_NAME", "bot_db_test") + + s3_adapter = S3Adapter( + endpoint_url=os.getenv("MINIO_ENDPOINT", "http://localhost:9000"), + aws_access_key_id=os.getenv("MINIO_ACCESS_KEY", "admin"), + aws_secret_access_key=os.getenv("MINIO_SECRET_KEY", "SuperSecretPassword123!"), + bucket_name=os.getenv("MINIO_BUCKET", "ai-char") + ) + + repo = AssetsRepo(client, s3_adapter, db_name=db_name) + + # 2. Create Asset with Data and Thumbnail + print("📝 Creating asset...") + dummy_data = b"image_data_bytes" + dummy_thumb = b"thumbnail_bytes" + + asset = Asset( + name="test_asset_with_thumb.png", + type=AssetType.IMAGE, + data=dummy_data, + thumbnail=dummy_thumb + ) + + asset_id = await repo.create_asset(asset) + print(f"✅ Asset created with ID: {asset_id}") + + # 3. Verify object names in Mongo (Raw check) + print("🔍 Verifying Mongo metadata...") + # Used repo to fetch is better + fetched_asset = await repo.get_asset(asset_id, with_data=False) + + if fetched_asset.minio_object_name: + print(f"✅ minio_object_name set: {fetched_asset.minio_object_name}") + else: + print("❌ minio_object_name NOT set!") + + if fetched_asset.minio_thumbnail_object_name: + print(f"✅ minio_thumbnail_object_name set: {fetched_asset.minio_thumbnail_object_name}") + else: + print("❌ minio_thumbnail_object_name NOT set!") + + # 4. Fetch Data from S3 via Repo + print("📥 Fetching data from MinIO...") + full_asset = await repo.get_asset(asset_id, with_data=True) + + if full_asset.data == dummy_data: + print("✅ Data matches!") + else: + print(f"❌ Data mismatch! Got: {full_asset.data}") + + if full_asset.thumbnail == dummy_thumb: + print("✅ Thumbnail matches!") + else: + print(f"❌ Thumbnail mismatch! Got: {full_asset.thumbnail}") + + # 5. Clean up + print("🧹 Cleaning up...") + deleted = await repo.delete_asset(asset_id) + if deleted: + print("✅ Asset deleted") + else: + print("❌ Failed to delete asset") + +if __name__ == "__main__": + asyncio.run(test_integration())