From 2f8ec83cd85f3d48056d59675ba9656ea6570088 Mon Sep 17 00:00:00 2001 From: wander Date: Thu, 1 Jan 2026 00:39:28 -0500 Subject: [PATCH] Implement video transcoding, add dashboard UI, and cleanup repository --- .gitignore | 3 + Dockerfile | 4 +- docker-compose.yml | 8 +- ta-organizerr.tar.gz | Bin 10317 -> 0 bytes ta_symlink.py | 642 ++++++++++++++++++++++++++++++------- templates/dashboard.html | 305 ++++++++++++++++++ templates/transcoding.html | 213 ++++++++++++ 7 files changed, 1060 insertions(+), 115 deletions(-) delete mode 100644 ta-organizerr.tar.gz create mode 100644 templates/dashboard.html create mode 100644 templates/transcoding.html diff --git a/.gitignore b/.gitignore index b6cf5f0..d5f0dc8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ __pycache__/ *.pyc .env +*.tar +*.gz +ta-organizerr.tar.gz diff --git a/Dockerfile b/Dockerfile index 93bc449..196c0d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM python:3.11-slim WORKDIR /app -COPY ta_symlink.py . +COPY . . +RUN apt-get update && apt-get install -y ffmpeg && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir requests flask +RUN mkdir -p /app/data EXPOSE 5000 CMD ["python", "ta_symlink.py"] diff --git a/docker-compose.yml b/docker-compose.yml index 4594ddd..ff9eaea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,8 +3,12 @@ services: build: /mnt/user/appdata/dockerbuildings container_name: ta-organizer volumes: - - /mnt/user/appdata/dockerbuildings/source:/app/source:ro + - /mnt/user/appdata/dockerbuildings/source:/app/source - /mnt/user/appdata/dockerbuildings/target:/app/target + - /mnt/user/appdata/dockerbuildings/data:/app/data + ports: + - "8002:5000" environment: - - SCAN_INTERVAL=${SCAN_INTERVAL:-60} + - SCAN_INTERVAL=60 + - ALLOWED_IPS=127.0.0.1,192.168.1.0/24,10.0.0.0/8,172.16.0.0/12 env_file: /mnt/user/appdata/dockerbuildings/.env diff --git a/ta-organizerr.tar.gz b/ta-organizerr.tar.gz deleted file mode 100644 index 59e03c1635f25c845f57caee6f578a6e384008cf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10317 zcmV-TD6-cdiwFP!000001MGYUToc*zIA^_!ie0gsu0f>;NvI-Hq$*NGihz``Bnzx0 z*|@u*2#6g$JBVFTQPERW>=pGyP7ga4JiDSIb`-IJ|Mzy2kWkd`?{{~<-~aQwclVLx z?Yx(LD49QAUa|~L=z+yk4qYd`_}XSPaBUKRQnPkgBKuf^sScH&cGD8FSz`z7CAPtNVltQ9e0NA1l5|Q9C48diZ z-LNs;_RJq1MU%MP&`9e!9?gQFGBF|}XcZtz(i9|ZXrOOkq>T}_I4sEoiAN=hI^1#z zP8%B70|w2E8o(12WF998StuA94hAuUIas0ylhcShF2Q^V+MQ6y#BL->kSs+xiOXn1 zZV*b!a9NBkBN-$bz%rahLujrMhvVeyAq@2M0!RUw7>ou`hFoPQcn&`WjPi8zX$Zh{ zLlm(XD#l0(KqPaW6m%>>;uBCBCuCe(B$^vQ#9=ZdSv?|^o66y^%z=ksZ2@-(c#5F3 zEei%Hr$W49EE-{q045X52{=HG(y>Z`Do6<+7|m6WNQMtnxf*ki=7z~kO-&IMkz_C? z6#+e5R>o>IfsMo{IU%FqlBjeJ&ejl&F+r3wv{(K$L19xE|16KGD;104A6H5(8vgJKmY)O zNdT>;v1rpIF&nAxcNmHNt{fIPcmP#PL4+tdVciqTm(rqQXXg+*hDbf~R4m`$VH z_z*ITqrr(A^9Lv@vPc69>iEtz^pARNy0M9T^bq?Z=PD#TdbNlmm1^_}o4j z{#$P!YVAO%WUM%qx~?rW0Kz^6ddL7ku%e`|9K>wn|Ff1>rB{UiIK`SVaAbz=lsZQ*dT#ksB?{FVzY_q|~ufwaAkO? zvB&M=ET5KB3w;YYEjqMCmgVjkM`azB2Kt3OzQz~dh0;u%+h&!haR-Zv6=aP&(kXE&Y=BTvo07GzCdDemLCW%L~i4>z2BG=68BO`NE}_PPI=4 zb$q`(eu@2_n!j=pmbYh2?Y4Kg!bh;(EDc${R@6BPaLso>} zi12%*Yd=_K-O6{zjj`+6w=jC?J~?gPw(jNqhsJ$=I5XWW`PqceAsrlbJKY&FXC30# z>BOn32ODcGhE|4u*~vFOQM4v&`MgLelKsb^qxrvrMN+pM3Ou(=W_3*zA;fYU;84)z$kq6dj1@`6>9z^rQLt$Ibig zE4PT*bTMl#HF#~O@f}+gtT8&BGSB^nYqquLaGMcd_J%z3Nm@`i^?B4@^JTgDKO0Q> zxrOfUiROFDx^CL*zG?MSx9z{^2J{}fcY`kONWN*g-81H0|C0CTede#ukM2F7_SC!s zB^6H{6XU(Us8Ki^nr zf0Fylv>AdQTJH*LU6Pk}J9VO{=5ulJ)boXB=We?)RK6^I!T6`vF&RVR2fi#VD0_Cf zW`OYcAmaSTpHenjrQCkL$L!8bS3%CkgwA`wfOwu_@DbM&7Kh%Gy2or#kJwV z=~-Pz_-s70?Bj$hvB>DWS1umIj$i4zZGP9X`H2a=|z`6M4I28{$=k$d+^VTJ9l1mti5wH zcioY{CRD#*E2*RgrK1KN4J zB)a+@DZZUjO*}K(hjqXF=-{ldKFwy%nK7l^!;J;UbqDT!NzMN-^TW>jHa#Y6Tk3nt zx2w>v^|4u{5AJ>lGtqr?_S(^Ioqk3UQ6K$}o*fTNDW@l`?Kka`&Wy>A=Pm77v|_uk z-@EJAEUgi9j?L=AG(EwYj5z~uI=vizy!@BuJDid@RX4j?JbHDZO|<{Br-+TW^KUn% z*jMZKTG4fx!At|e+KNB?BqPh#39eOdPkUqa$!PwSBDV#4_bddfyYZkse;iPkzf6De#+Yaf+OonbREnhce)&t+S-fuds8QP;~PUJYGdy$6EiAsMKWL5eL03=k3~l zT>kF<$u_p}WiAw_)NW}@57*q_K?8eBj>|e(_x89sxyMi2lDeQH-DVWFw$z(*vd!Ix znZpWeT8G}K_TF)!Af>YWRk^9chl~3du>#roGsH;H&d9S8yuUKMgXhihg&wWLtCoGULdd$Gs|| zE`=PM(|lL(@;_`Y+|qUNAOCE))8^Sx{A6Qp;MzsEElWMtJxc5HK7X!DXn^^Go~=C7 zk7smPGj8er)w^tZ+^Ow4EZ4r>J)hwoy$2qf9@0DG=I#u!fAX7(LxZ;$6n#lTl4cBZ zt~%Q6l;POqRkz={=g$6OwtUW)9w!$~8B_Dg_<6s%sMiV8t6mQx;znP&Q#*9%>}G|7 zcmw50CPTfG*WHYmBkwZrht`wxqFO|ae>h{NCrDW+48>g{w}|64-frAU!13RWWh?E1fk$r^B+;?P74Bl$)8z~GPX~l zV{LKTmuq)+@^5zRGG%(U!;sAJ7k?@}+oE9Jp5eSAnZ!}HYx|CeJ3Xs=Qgf`@c5itp z=jyb1NdD+YpPmwi&!)8M_{_qKp0nWmxi=epdo8TI+AS&Gt0LpYl_P_0ttuG#*;DWC zr)zgUKc3&EPtQlBF;$jTPM7gIdjeK$-dVU2j7vj4*Apj z_P(Q>*I9GjBfPo<|9auk+sH4=s>-fkzju1cCt=&#_j@ak9L+CT7L#hqnf4Ox<{yz+ zd%^$n)u^Q{`rY*GTTruBd^oNq!Fp($$4B$dzIZgDCT&hs$UTWzl)lfz|4_``+3(U{ z?tWUCJgC)g>$0Ls+GqWc^6BHt7dkVItGE0c%Y1E%)0}4gzVNV9WCn_NkGU3KcIilFV)Dkkvv0E=PdpO1^7ED> zZz`_TT&el|NnClXnYgR`2fu@-cngD;U&L}t#^oP+*0Jx6pM0E#{%ozN>0cxtHGg$> z@7`nQuUeDV|*C*Kne#?pV^UZ@^n-@8_ z&f2n3HhbaIj9Xp2E{4v~_aBQF@Sb-X@6!K7^kthb)2|<`-du2~dP(8A2=ZKt&fnzo3*Pn_;-)?=mHPeGNJ zohFRSJR<8lX?<_q0!3g&igQAQt8l&bVshKVXXk;N^6u+QC--pZ7&y*+kAJzR$HU4pC8oW;b(ICikPZsCks6hXP zWW?F$JQMrOgu~ljo{F1nFu(1){zc;st}K+C3K4bdPcOj__)N8wTu2|_&>^?eFBMbL zw{GeFb5*zE06*`YvROZEb$$?Zc(G!G^Kj#>RK$(_0Y=;RZ5esmc&zQLw1=$+7i=(I zoYiCV3Vw0b?@n26e>NMq@6DB(vOO2p{I>B@cqhXXTQ_=S>2qri_}LuFC|S}!y<%QA z-^(Q;ypPqxA39ib=bgo-&5ZJU;6ArUMfBLbJCxO^wYJ8Rzu>Wxv-K z-oUVjV#`bMvz^C?uK!u;IHr0-QQ*&OBL>>0?jH8CMOn?kvID`(I=*ZtPOgax)wO!t zC5JcD&i!rQKX+^n`wc737?nGCC0Ey(tF!L-IRpKO&Lz(iUj--Yg;7r)$6L}5PPqqE z-rU?PZLI$DSt~adTL$bzI|WVOd;Rk2c28dH?zi-;Vet$BlF-97_SkgWjoFji?4R(% zBCif&yZf8{#?LNXYdUSrii8;RYwlAnZ(MIZ`i0&zlbKS!XUR{6p)2|%Ug$MB#c9tY zs~cFq&R&kEtxs2vAh-U_q5;iJSw zGoBarA7MBAtVL<_5Z7I!u8R)j-P*nEcicfd=<15ppCeC9O5T6W?vEE!7T(tho0B=Q z>w)5e;H2``j}9F*;=Kth$aq2Y%jo4*;2UZlZ`(tmH{hh}ko*ZR(%oLRT=HVX@tiJy zZaBu9>b@jGP-w@$Ul4H3R_$bP^-VK~Q?7^jOE{}Wu z647#mb!5${@i~uvn|hqK;3Zgwp;vtUHs4x8^^U@l&Tt0r4D>%5bF(B)FZ#n`2|v@M z?Xu{Q{&^v1?3<_O+%^7b*-FdaUV7dSxc!##y6ZIS5o2+B(Cb+#qy2jcX1^ULOd5D- z^mgQ4>Fqfmg>Pc@ek|xcDWLl1sg&*O=^Qu7R_n3CEC!m#>u9^hqCdC2v#66@xvz z38!*zY%yKFdhwmff&Q=D4*x3mI+C)uT0CuY=lR30^xB+XTk|ySee$~=y4)oTZ}KST zftNi6Q&-KCR4ty}2R##PwRy;SS;wdEN)E2r{Mq=~6=8|X&!=rVkGwBETHDR5Dvx{W zezW3LT%Dwq>1QkQ%Iy!n>n||o_v}$LqWkK!W-ralVr+lkw|;uL;&bsC+s!o<#G(~* zXB<3teN~}9!LOMaxpUv3nz-$sdz=4h75H$Qkw=#*fjm0-z`H}4FQWQh?s9gLz`ruj z<8Irx9R~U3M(R51l}C^3l)C-?&_g!&PSPW@r>Cr$EuKlOElAiJd&Sc|KhAhv29nu7 ztEvJ^64_Ycnf0{I>1!i(9uNdVSz5^PDrOXG`5jTRnuA z6oa$(2^Pe@49Htt(BEsxxs-yqDIp=&kA5j!;`(CwvI%-|vhj0}Z0j@Pp;r_;^rqak z?&^5jsyU6rTizV@` zO3C17ddA_ox)&?;A4cr#vU>bLN2{(8Ji>_l_J-Uv(^gkIqJa4JvMdIxL!#aOBIAs%~jt^7mYSy7kSMzBzUdLni#; z{Gnu$LycvmOX7=wPuDM=eq1@Q`O~1GHnz1xwjAz+&WNx4+@sAcU;DS!c6;;hweD5f z?_#S}=DP}WMt+P-Pn&o4-GcZN^WtBVy;m;|9e}7TOS-BZaO0+=RJxYrfL!n zKKYcbYoDKcCrvT*Y1Nm}BVV^!e*5Z)RTt%_e!tZ*!Md+q?>~=hJF;unn`5_=E+m&f zlgJzs6Vp=C;$FA5yp?XBbLh>n@}5K6->&Sn95b)7Sl;(fou@l_n*Telt8{E{@v*y( z?_Zt&;qm?4oLs%#cvA6SjauElRsXd#Q|A9H2M@L~hxtDX>woLN|4h?I0-TUhs{Aj9 z^C~-Y3gV=o2`NhBBBtQ0$_`UVfFcTWB1N%yJWkPwm`ISpVlai^GMYeQ6;f1YN@A!O zjgnwUG$9dVBtp@oLPRS_Fj_&u+60v(mEf{CiYY3<*aI4!{d z&OoaEWbRK|Mlsb?Z_r;tDyiQ-~L zJOmvJM922q`}*3I{r3#-{-! zs)95^RAnShMi=D*BC=>aMnN*Q8V-jQDS{%AuxcL3md$Y)0;Y|Su+~RJv!5|JVawq} zMn=M82kNPb>&oW#NrZxgr&xqEtb+@I+?cZAD5mZRR)mPhND?e5rK-f+!^wcTNC=?i zu>?ii+E`i*<^vR*F&?gNqrL*es0QRlb9M`MsU#Ulp)r8C84q!h5HbwR0H>HKu;&+m zs>HxqGn)pd*?C2wR4fOG6%jE*8WFTVDv@I(n2lD!h6f-xA~B-b$8u)QL)^pwA(UUF znBmM+#I7+9Y{5vV=}aXMet^Daly->0#?he6=iFbk^8TVp;D5Ct@d1Ve z_D?|&r0%J?z}(7IUfHtKLmi>2;as(p`o>Xp$7%J_aQGHf(cOU^I0qhk< z#R8QF+m6HWVAY-yk+_^jD6lJJG%&If;;0J&SqZj~471~%})iJ5& zBv!0_xqm+5g^K_~K*HW00gPP~6G%cV&~>o)po$?0sEjs2#sQa$#NhE@jlmuUDvC!l zPZjV80?pIJyKD|Ly8s4QFlVsPG%)!{sCp|jYLV1Wji6@JkX^{Tw(nAUr*R5d3=qJp z>^ZytkCumd{D!vSucq;h_)lPEsXqT_VJR>N@t@UTtAFdi|45@SG9GMO#;R*N5%!=3 zV_avaRf3nthTT^R6s%oqd8p7`o3!gbA`9HnH zUu`@-3?c9s8PM&&i^jM1zraFJ@BPox!p!2|{r^uip-`SALV<@D3i%vk9{9rf->{LeqqIME~_Mc~_*1dl?LXa0S` z*M>I%>faFn?^4Q7RFb-rj)fT@_!gt;z@->VBwB(}aViq-@QVp^NEwe#G6BeBARrfl z_X@cLyfFyjH2_(RiHazEm-m7f4@67`0bvkocyS~iC2^Q{qP~98-UQ(``QFbps(bU- z9fTa2EXd63BwGN^%J5JiBOgEGc?k=;qfXva|7ZIpT9K*<$U ztbG8fz*u_hi?kGl@=9g{NO*=Dd^2OO&cwo;7lf}n+T7sIh34sR`(*g9^58c!{rXoM zzUzvGa?$Ib-&Y>+CsCZjgwX^kWbaedyBYBvmnZ7q-%%bPhxZGVN4XJ#WWH)18gF=s zCkmL8xCXsi9WbBDE`<_v)7bQZq?4E&FD?@^X#o#c9^yfR-4qi`BT*P)kVGh8E>nn$ z4IqaG@IwPUfE;_FfKBi~>2m7^a*YrbOY9@2uK^Mu6*zwuS3_Q5gRfLzpJR;l0pnaKr{9BHg$Dv zI0X_-5He`ArY&bn2$)z?dJwD;;i+Narch!FwKWrjt1kJdI*kwqd&m`Y$4Ax4BQfCf zM8MGTAuZEJ2sEaEtXT`My6pv)351SK&`KLhWhHwz&hR6omqr zGL}VCvT&;@6R#9a$RPj-w17iq46z|tZm|StFB(O}W6btL$tavAQD7`3xD=-y8=>aG zEK`iqf+JB1TAQc|B1My;U`p43LQn)2|A2L3oF8x;Ac|=WsWZR=uu2HPqypGJ3B<`( z2nmB`j){3l070V?RrQbpZUBY%&fpGJZz!YGM!a|(O@MTe z%6F?;{hN19-FMUE@&2kg2y^d^We$cE@;Z*$%WzHe!AKuvM?|Bzgi;9`$c~C76cRDJ z&Hz0Kc*rD2#GUcB2`0cC!S!ICg8hcZ6TpuF^}s$;SAmbnoL@>yo7Tt!rsCEDB z{>tffH3qhf3e(DfB*sISi?a=ctz3-8LMLUYrUqu?G?4^DWeT~FkcfrK8kH(g<8oOq z;V(jE6rvIL!U#bLtIASBfg21G2y6uc16H^E(P&Hpi?Wbirrse%M)`P!z|8`_B&kM1 zjRay)6?qLAf1|Ey!`d#~Sw=i9meet1?g6$+3+XGT!eba_IAZ91i`z!yOa>PSyj9on z1WYfi7|9#-s;>P(TT~|{9$PmuG}64R3s->MWco-DDv1L+BR7TBKBg~P!<&n7k|{aW z;R6aqf?iF8N}?KLcsM1j2gFRLvA{>`DmkkbN~&528IaqzE^Wh51w%oo4KK1bjE5R@KqGJlVNw9R*u!qM`O* zJce9jo-wx(gsNj7s#jqK1&=xH(WugCHb&rH1Q7@Hj?ohl&;l5?)dz&Dr6vFXnD;6b zouF9MC$&Qhooj}U37_SV?pY}jkYmHZ}1d<_Fn8V6TXj>0!T zirHKy?2#QI5!Zw*sH37v@zk3E`;OlT>#UNi<{iyVX5bCfZ|nx)j7h3TAhn@OtqZs8 zwdBFqB+>M!|H`aU5v@QatQOmYt(Zs~Hq>M=C6(%=LY+2Y-maU}A`P7Xp)$c#KSNi@ zoJ>@w($!}G>P|pn6hBJdNEU}d!RF;zlVD#TD_@gA9*IeSHHwFUiD8}TXoyz~f7nAQ zrn+?kE5l{JC2QiO6w%RmVj~OOWOM_6A4El-$1>{7Kl2iIhDp=wRXKbozce{4FS`zp5ouGBQ8<>M9QwY}kY zq<`6Abv;duQLZQ$!Tjb`)2pZ~GkhcFSEh#`E)5(F@=-9iq7euCszlCaqUx`}R})fN z`9&Wrf$xqd0z^=^D4<$uJ!wp_w8oB6ZW7f{@S32tZ5l(M^?O)M^%SNw6%vfpu2aG{ zfUFF}85PCFkj#s@8W5q_`vDpWCwMeB3DG|4ma3{PHztF|>Xqu`)}m=J;${%5h(hwH ztnr)`J68K0Lj3@q8{knA43isb*UF7yF&w^*Fy&_i%n_Eshk>obV?M(&iV-6r_?u~o zD(NWY*Z4UDLjwc0$Y&QzOC=7OLGzLfX~$lN{9C+r^sI z9|L;o9>a*q$o~CXNd|KTvXeE?+4Uu7lRbu!~aAG%S)N$uWB_rHGQ^G}l3? z&g}gb%8t(hX%ON=N}7C`JJf9f)hN(e2ceMEeMug`zD}q!6cjQ*yQX6&!5xy5<9X`s zpUMQ3pofhlQxQY>MQK!0Q;onxNC$t`6xcF=O1qGnYR3o2G--pig5)(w+VNHFfw`(B z4rZWUHn5T=N?z?7>~4;>o~R^kqFf5fK9uV_%0B=`(i$(gDivb#;UF|=RA*sHMf)fA zlp3e;`*!dz+}l8|B%lhCYnL0m`U4mV?;a zBU~YP&&Fj!A=kDp&jHVEsXYUbWmKKVU=MMzISu9`JdvY1*u@KAJ{hvXwY?e#2+UAS z3bdupz_Vl6UuxCNvoP~j$e7s9o-5!n|ABo0Plnlp%^~>r$NK)M#^bZM)?g&|chUIP z`EPUc`uBe=EzAY~p8xzOn!Dd9Z$!>q4Yh?=Lrp0OF69J`^7CX8TwteiTt@kZAX=w{ z5gx}c&6K?b}HTfG`X|(HoVfYD|vPI^6#V;sAnzu2hi!)`dVJyZoh5d3MI_ z{zuvFzvG-(&i%hc_Wu=oL7i&Xu9RVvZs;rXl9OarwXBxB*QzmZ$F@EO)Z!_KT9BZ2 f(NLeCtr_*#9{>OV0000006#SWR~0ng0C)fZ(GNGW diff --git a/ta_symlink.py b/ta_symlink.py index fd53f1a..a122db9 100644 --- a/ta_symlink.py +++ b/ta_symlink.py @@ -1,4 +1,3 @@ - from pathlib import Path import os import requests @@ -6,29 +5,266 @@ import re import sys import threading import time -from flask import Flask, jsonify, render_template_string, request +import ipaddress +from flask import Flask, jsonify, render_template, request, abort # Load config from environment variables API_URL = os.getenv("API_URL", "http://localhost:8457/api") VIDEO_URL = os.getenv("VIDEO_URL", "http://localhost:8457/video/") API_TOKEN = os.getenv("API_TOKEN", "") SCAN_INTERVAL = int(os.getenv("SCAN_INTERVAL", 60)) # Default 60 minutes +ALLOWED_IPS = [ip.strip() for ip in os.getenv("ALLOWED_IPS", "127.0.0.1").split(",")] SOURCE_DIR = Path("/app/source") TARGET_DIR = Path("/app/target") HEADERS = {"Authorization": f"Token {API_TOKEN}"} app = Flask(__name__) +# Database setup +import sqlite3 +from contextlib import contextmanager + +DB_PATH = Path("/app/data/videos.db") +DB_PATH.parent.mkdir(parents=True, exist_ok=True) + +@contextmanager +def get_db(): + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + try: + yield conn + finally: + conn.close() + +def init_db(): + with get_db() as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS videos ( + video_id TEXT PRIMARY KEY, + title TEXT, + channel TEXT, + published TEXT, + symlink TEXT, + status TEXT, + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.commit() + +init_db() + +# Global State processed_videos = [] +log_buffer = [] +log_lock = threading.Lock() +transcode_log_buffer = [] +transcode_log_lock = threading.Lock() # Utility functions +def log(msg): + """Logs a message to stdout and the in-memory buffer.""" + print(msg, flush=True) + with log_lock: + log_buffer.append(msg) + if len(log_buffer) > 1000: + log_buffer.pop(0) + +def tlog(msg): + """Logs a message to the transcode log buffer.""" + print(f"[TRANSCODE] {msg}", flush=True) + with transcode_log_lock: + transcode_log_buffer.append(msg) + if len(transcode_log_buffer) > 500: + transcode_log_buffer.pop(0) + +def detect_encoder(): + """Detect best available hardware encoder.""" + import subprocess + try: + result = subprocess.run(['ffmpeg', '-hide_banner', '-encoders'], + capture_output=True, text=True) + encoders = result.stdout + + if 'h264_nvenc' in encoders: + return 'h264_nvenc' + elif 'h264_vaapi' in encoders: + return 'h264_vaapi' + elif 'h264_videotoolbox' in encoders: + return 'h264_videotoolbox' + else: + return 'libx264' + except: + return 'libx264' + +def probe_codecs(filepath): + """Probe video and audio codecs using ffprobe.""" + import subprocess + try: + # Get video codec + v_result = subprocess.run([ + 'ffprobe', '-v', 'error', '-select_streams', 'v:0', + '-show_entries', 'stream=codec_name', '-of', 'csv=p=0', filepath + ], capture_output=True, text=True) + video_codec = v_result.stdout.strip() + + # Get audio codec + a_result = subprocess.run([ + 'ffprobe', '-v', 'error', '-select_streams', 'a:0', + '-show_entries', 'stream=codec_name', '-of', 'csv=p=0', filepath + ], capture_output=True, text=True) + audio_codec = a_result.stdout.strip() + + return video_codec, audio_codec + except Exception as e: + tlog(f"Error probing {filepath}: {e}") + return None, None + +def transcode_video(filepath, encoder='libx264'): + """Transcode a video file to H.264/AAC.""" + import subprocess + + original_path = Path(filepath) + + # Try to resolve symlink first (don't check if it exists, broken symlinks still exist as links) + if original_path.is_symlink(): + try: + actual_file = Path(os.readlink(original_path)).resolve() + tlog(f"Following symlink: {filepath} -> {actual_file}") + + # Translate host path to container path + # Host: /mnt/user/tubearchives/bp/... → Container: /app/source/... + actual_file_str = str(actual_file) + if actual_file_str.startswith("/mnt/user/tubearchives/bp"): + container_path = actual_file_str.replace("/mnt/user/tubearchives/bp", "/app/source", 1) + tlog(f"Translated path: {actual_file} -> {container_path}") + filepath = container_path + else: + filepath = str(actual_file) + except Exception as e: + tlog(f"Error resolving symlink: {e}") + return False + elif not original_path.exists(): + tlog(f"File not found: {filepath}") + return False + + # Now check if the actual file exists + if not Path(filepath).exists(): + tlog(f"Source file not found: {filepath}") + return False + + video_codec, audio_codec = probe_codecs(filepath) + + if video_codec == 'h264' and audio_codec == 'aac': + tlog(f"Already H.264/AAC: {filepath}") + return True + + temp_file = f"{filepath}.temp.mp4" + + try: + # Determine transcode strategy + if video_codec == 'h264': + tlog(f"Audio-only transcode: {filepath}") + cmd = [ + 'ffmpeg', '-v', 'error', '-stats', '-i', filepath, + '-c:v', 'copy', + '-c:a', 'aac', '-b:a', '192k', + '-movflags', '+faststart', + '-y', temp_file + ] + else: + tlog(f"Full transcode using {encoder}: {filepath}") + if encoder == 'h264_nvenc': + cmd = [ + 'ffmpeg', '-v', 'error', '-stats', '-i', filepath, + '-c:v', 'h264_nvenc', '-preset', 'fast', '-cq', '23', + '-c:a', 'aac', '-b:a', '192k', + '-movflags', '+faststart', + '-y', temp_file + ] + elif encoder == 'h264_vaapi': + cmd = [ + 'ffmpeg', '-v', 'error', '-stats', + '-hwaccel', 'vaapi', '-hwaccel_output_format', 'vaapi', + '-i', filepath, + '-vf', 'format=nv12,hwupload', + '-c:v', 'h264_vaapi', '-b:v', '5M', + '-c:a', 'aac', '-b:a', '192k', + '-movflags', '+faststart', + '-y', temp_file + ] + else: # libx264 + cmd = [ + 'ffmpeg', '-v', 'error', '-stats', '-i', filepath, + '-c:v', 'libx264', '-crf', '23', '-preset', 'medium', + '-c:a', 'aac', '-b:a', '192k', + '-movflags', '+faststart', + '-y', temp_file + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + # Replace original + Path(filepath).unlink() + Path(temp_file).rename(filepath) + tlog(f"✅ Success: {filepath}") + return True + else: + # Check if it's a GPU error and retry with CPU + if encoder in ['h264_nvenc', 'h264_vaapi', 'h264_videotoolbox'] and 'libcuda' in result.stderr or 'Cannot load' in result.stderr: + tlog(f"⚠️ GPU encoding failed, retrying with CPU (libx264)...") + + # Retry with libx264 + if video_codec == 'h264': + cpu_cmd = [ + 'ffmpeg', '-v', 'error', '-stats', '-i', filepath, + '-c:v', 'copy', + '-c:a', 'aac', '-b:a', '192k', + '-movflags', '+faststart', + '-y', temp_file + ] + else: + cpu_cmd = [ + 'ffmpeg', '-v', 'error', '-stats', '-i', filepath, + '-c:v', 'libx264', '-crf', '23', '-preset', 'medium', + '-c:a', 'aac', '-b:a', '192k', + '-movflags', '+faststart', + '-y', temp_file + ] + + cpu_result = subprocess.run(cpu_cmd, capture_output=True, text=True) + + if cpu_result.returncode == 0: + Path(filepath).unlink() + Path(temp_file).rename(filepath) + tlog(f"✅ Success (CPU): {filepath}") + return True + else: + tlog(f"❌ Failed (CPU): {filepath}") + tlog(f"Error: {cpu_result.stderr}") + if Path(temp_file).exists(): + Path(temp_file).unlink() + return False + else: + tlog(f"❌ Failed: {filepath}") + tlog(f"Error: {result.stderr}") + if Path(temp_file).exists(): + Path(temp_file).unlink() + return False + + except Exception as e: + tlog(f"❌ Exception: {e}") + if Path(temp_file).exists(): + Path(temp_file).unlink() + return False + def sanitize(text): text = text.encode("ascii", "ignore").decode() text = re.sub(r'[\/:*?"<>|]', "_", text) return text.strip() def fetch_all_metadata(): - print("📥 Fetching all video metadata...", flush=True) + log("📥 Fetching all video metadata...") video_map = {} page = 1 while True: @@ -67,20 +303,16 @@ def fetch_all_metadata(): if current is not None and last is not None and current >= last: break else: - # Fallback if no pagination info, just stop if empty data (handled above) or arbitrary limit? - # If we got data but no pagination, maybe it's a single page result? - # But we loop until no data. pass - print(f" - Page {page} fetched. Total videos so far: {len(video_map)}", flush=True) + log(f" - Page {page} fetched. Total videos so far: {len(video_map)}") page += 1 except Exception as e: - print(f"❌ Error fetching page {page}: {e}", flush=True) - # If a page fails, maybe we should stop or retry? For now, let's stop to avoid infinite loops on auth error + log(f"❌ Error fetching page {page}: {e}") break - print(f"✅ Metadata fetch complete. Found {len(video_map)} videos.", flush=True) + log(f"✅ Metadata fetch complete. Found {len(video_map)} videos.") return video_map def cleanup_old_folders(): @@ -88,7 +320,7 @@ def cleanup_old_folders(): Scans TARGET_DIR for folders containing '+00:00'. Safely deletes them ONLY if they contain no real files (only symlinks or empty). """ - print("🧹 Starting cleanup. Scanning ONLY for folders containing '+00:00'...", flush=True) + log("🧹 Starting cleanup. Scanning ONLY for folders containing '+00:00'...") cleaned_count = 0 skipped_count = 0 @@ -123,15 +355,86 @@ def cleanup_old_folders(): item.unlink() # Remove directory video_dir.rmdir() - print(f" [DELETED] {video_dir.name}", flush=True) + log(f" [DELETED] {video_dir.name}") cleaned_count += 1 except Exception as e: - print(f" ❌ Failed to delete {video_dir.name}: {e}", flush=True) + log(f" ❌ Failed to delete {video_dir.name}: {e}") else: - print(f" ⚠️ SKIPPING {video_dir.name} - {reason}", flush=True) + log(f" ⚠️ SKIPPING {video_dir.name} - {reason}") skipped_count += 1 - print(f"🧹 Cleanup complete. Removed: {cleaned_count}, Skipped: {skipped_count}", flush=True) + log(f"🧹 Cleanup complete. Removed: {cleaned_count}, Skipped: {skipped_count}") + +def check_orphaned_links(): + """ + Scans TARGET_DIR for video.mp4 symlinks and checks if they point to valid files. + For orphaned links, parses the folder structure to extract metadata. + Stores results in database. + """ + log("🔍 Checking for orphaned symlinks...") + orphaned = [] + total_checked = 0 + + if not TARGET_DIR.exists(): + log("⚠️ Target directory does not exist") + return orphaned + + with get_db() as conn: + for channel_dir in TARGET_DIR.iterdir(): + if not channel_dir.is_dir(): + continue + + channel_name = channel_dir.name + + for video_dir in channel_dir.iterdir(): + if not video_dir.is_dir(): + continue + + folder_name = video_dir.name + + # Look for video files + for video_file in video_dir.glob("video.*"): + total_checked += 1 + + if video_file.is_symlink(): + try: + # Check if the symlink target exists + target = Path(os.readlink(video_file)) + + if not target.exists(): + # Parse folder name: "YYYY-MM-DD - Title" + parts = folder_name.split(" - ", 1) + published = parts[0] if len(parts) > 0 else "unknown" + title = parts[1] if len(parts) > 1 else folder_name + + # Try to extract video ID from symlink target path + video_id = target.stem if target.stem else "unknown" + + orphaned.append({ + "video_id": video_id, + "path": str(video_file), + "target": str(target), + "folder": folder_name, + "channel": channel_name, + "title": title, + "published": published + }) + + # Store in DB + conn.execute(""" + INSERT OR REPLACE INTO videos + (video_id, title, channel, published, symlink, status) + VALUES (?, ?, ?, ?, ?, 'missing') + """, (video_id, title, channel_name, published, str(video_file))) + + log(f" ⚠️ BROKEN: {folder_name} -> {target}") + except Exception as e: + log(f" ❌ ERROR: {folder_name}: {e}") + + conn.commit() + + log(f"✅ Check complete. Scanned {total_checked} files, found {len(orphaned)} orphaned symlinks.") + return orphaned # Main logic @@ -149,116 +452,231 @@ def process_videos(): new_links = 0 verified_links = 0 - try: - for channel_path in SOURCE_DIR.iterdir(): - if not channel_path.is_dir(): - continue - for video_file in channel_path.glob("*.*"): - video_id = video_file.stem - - # 2. Lookup in local map - meta = video_map.get(video_id) - if not meta: - continue - sanitized_channel_name = sanitize(meta["channel_name"]) - channel_dir = TARGET_DIR / sanitized_channel_name - channel_dir.mkdir(parents=True, exist_ok=True) - sanitized_title = sanitize(meta["title"]) - folder_name = f"{meta['published']} - {sanitized_title}" - video_dir = channel_dir / folder_name - video_dir.mkdir(parents=True, exist_ok=True) - actual_file = next(channel_path.glob(f"{video_id}.*"), None) - if not actual_file: - continue - host_path_root = Path("/mnt/user/tubearchives/bp") - host_source_path = host_path_root / actual_file.relative_to(SOURCE_DIR) - dest_file = video_dir / f"video{actual_file.suffix}" - try: - if dest_file.exists(): - if dest_file.is_symlink(): - current_target = Path(os.readlink(dest_file)) - if current_target.resolve() != host_source_path.resolve(): - dest_file.unlink() - os.symlink(host_source_path, dest_file) - print(f" [FIX] Relinked: {folder_name}", flush=True) - new_links += 1 - else: - verified_links += 1 - else: - os.symlink(host_source_path, dest_file) - print(f" [NEW] Linked: {folder_name}", flush=True) - new_links += 1 - except Exception: - pass - processed_videos.append({ - "video_id": video_id, - "title": meta["title"], - "channel": meta["channel_name"], - "published": meta["published"], - "symlink": str(dest_file) - }) - except Exception as e: - return str(e) + with get_db() as conn: + # Clear existing "linked" videos (we'll repopulate) + conn.execute("DELETE FROM videos WHERE status = 'linked'") - print(f"✅ Scan complete. Processed {len(processed_videos)} videos.", flush=True) - print(f" - New/Fixed Links: {new_links}", flush=True) - print(f" - Verified Links: {verified_links}", flush=True) + try: + for channel_path in SOURCE_DIR.iterdir(): + if not channel_path.is_dir(): + continue + for video_file in channel_path.glob("*.*"): + video_id = video_file.stem + + # Lookup in local map + meta = video_map.get(video_id) + if not meta: + continue + sanitized_channel_name = sanitize(meta["channel_name"]) + channel_dir = TARGET_DIR / sanitized_channel_name + channel_dir.mkdir(parents=True, exist_ok=True) + sanitized_title = sanitize(meta["title"]) + folder_name = f"{meta['published']} - {sanitized_title}" + video_dir = channel_dir / folder_name + video_dir.mkdir(parents=True, exist_ok=True) + actual_file = next(channel_path.glob(f"{video_id}.*"), None) + if not actual_file: + continue + host_path_root = Path("/mnt/user/tubearchives/bp") + host_source_path = host_path_root / actual_file.relative_to(SOURCE_DIR) + dest_file = video_dir / f"video{actual_file.suffix}" + try: + if dest_file.exists(): + if dest_file.is_symlink(): + current_target = Path(os.readlink(dest_file)) + if current_target.resolve() != host_source_path.resolve(): + dest_file.unlink() + os.symlink(host_source_path, dest_file) + log(f" [FIX] Relinked: {folder_name}") + new_links += 1 + else: + verified_links += 1 + else: + os.symlink(host_source_path, dest_file) + log(f" [NEW] Linked: {folder_name}") + new_links += 1 + except Exception: + pass + + # Store in database + conn.execute(""" + INSERT OR REPLACE INTO videos + (video_id, title, channel, published, symlink, status) + VALUES (?, ?, ?, ?, ?, 'linked') + """, (video_id, meta["title"], meta["channel_name"], + meta["published"], str(dest_file))) + + processed_videos.append({ + "video_id": video_id, + "title": meta["title"], + "channel": meta["channel_name"], + "published": meta["published"], + "symlink": str(dest_file) + }) + except Exception as e: + conn.rollback() + return str(e) + + conn.commit() + + log(f"✅ Scan complete. Processed {len(processed_videos)} videos.") + log(f" - New/Fixed Links: {new_links}") + log(f" - Verified Links: {verified_links}") return None def scheduler(): - print(f"🕒 Background scheduler started. Scanning every {SCAN_INTERVAL} minutes.", flush=True) + log(f"🕒 Background scheduler started. Scanning every {SCAN_INTERVAL} minutes.") while True: - print("🔄 Running scheduled scan...", flush=True) + log("🔄 Running scheduled scan...") process_videos() time.sleep(SCAN_INTERVAL * 60) # Flask routes + +@app.before_request +def limit_remote_addr(): + # Skip check for local requests if needed, but generally good to enforce + client_ip = request.remote_addr + try: + ip_obj = ipaddress.ip_address(client_ip) + allowed = False + for allowed_ip in ALLOWED_IPS: + if not allowed_ip: continue + if "/" in allowed_ip: + if ip_obj in ipaddress.ip_network(allowed_ip, strict=False): + allowed = True + break + else: + if ip_obj == ipaddress.ip_address(allowed_ip): + allowed = True + break + if not allowed: + log(f"⛔ Access denied for IP: {client_ip}") + abort(403) + except ValueError as e: + log(f"⛔ Invalid IP format: {client_ip}, Error: {e}") + abort(403) + @app.route("/") def index(): - return render_template_string(''' - - TA Organizerr - -

TA Organizerr

-
- -
-

Processed Videos

-
    - {% for v in videos %} -
  • {{v.published}} - {{v.title}} ({{v.channel}})
    Symlink: {{v.symlink}}
    • - {% endfor %} -
- - - ''', videos=processed_videos) + return render_template('dashboard.html') -@app.route("/process", methods=["POST"]) -def process(): - error = process_videos() - if error: - return f"Error: {error}", 500 - return render_template_string(''' - - TA Organizerr - -

TA Organizerr

-
- -
-

Processed Videos

-
    - {% for v in videos %} -
  • {{v.published}} - {{v.title}} ({{v.channel}})
    Symlink: {{v.symlink}}
    • - {% endfor %} -
- - - ''', videos=processed_videos) +@app.route("/api/status") +def api_status(): + with get_db() as conn: + # Get all videos from DB + videos = [] + for row in conn.execute("SELECT * FROM videos ORDER BY channel, published DESC"): + videos.append({ + "video_id": row["video_id"], + "title": row["title"], + "channel": row["channel"], + "published": row["published"], + "symlink": row["symlink"], + "status": row["status"] + }) + + # Calculate stats + total = len(videos) + linked = sum(1 for v in videos if v["status"] == "linked") + missing = sum(1 for v in videos if v["status"] == "missing") + + return jsonify({ + "total_videos": total, + "verified_links": linked, + "missing_count": missing, + "videos": videos + }) -@app.route("/api/videos") -def api_videos(): - return jsonify(processed_videos) +@app.route("/api/logs") +def api_logs(): + start = request.args.get('start', 0, type=int) + with log_lock: + return jsonify({ + "logs": log_buffer[start:], + "next_index": len(log_buffer) + }) + +@app.route("/api/scan", methods=["POST"]) +def api_scan(): + # Run in background to avoid blocking + threading.Thread(target=process_videos).start() + return jsonify({"status": "started"}) + +@app.route("/api/cleanup", methods=["POST"]) +def api_cleanup(): + threading.Thread(target=cleanup_old_folders).start() + return jsonify({"status": "started"}) + +@app.route("/api/check-orphans", methods=["POST"]) +def api_check_orphans(): + orphaned = check_orphaned_links() + return jsonify({"status": "complete", "orphaned": orphaned, "count": len(orphaned)}) + +@app.route("/transcode") +def transcode_page(): + return render_template('transcoding.html') + +@app.route("/api/transcode/videos") +def api_transcode_videos(): + """Get all videos that need transcoding.""" + page = request.args.get('page', 1, type=int) + per_page = request.args.get('per_page', 100, type=int) + offset = (page - 1) * per_page + + with get_db() as conn: + # Get total count + total = conn.execute("SELECT COUNT(*) as count FROM videos WHERE status = 'missing'").fetchone()['count'] + + videos = [] + for row in conn.execute( + "SELECT * FROM videos WHERE status = 'missing' LIMIT ? OFFSET ?", + (per_page, offset) + ): + videos.append({ + "video_id": row["video_id"], + "title": row["title"], + "channel": row["channel"], + "published": row["published"], + "symlink": row["symlink"] + }) + + return jsonify({ + "videos": videos, + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page + }) + +@app.route("/api/transcode/start", methods=["POST"]) +def api_transcode_start(): + """Start transcoding a video.""" + data = request.get_json() + filepath = data.get('filepath') + + if not filepath: + return jsonify({"error": "No filepath provided"}), 400 + + encoder = detect_encoder() + tlog(f"🖥️ Selected encoder: {encoder}") + + # Run in background + def run_transcode(): + transcode_video(filepath, encoder) + + threading.Thread(target=run_transcode).start() + return jsonify({"message": "Transcode started", "encoder": encoder}) + +@app.route("/api/transcode/logs") +def api_transcode_logs(): + """Get transcode logs.""" + start = request.args.get('start', 0, type=int) + with transcode_log_lock: + return jsonify({ + "logs": transcode_log_buffer[start:], + "next_index": len(transcode_log_buffer) + }) if __name__ == "__main__": # Start scheduler in background thread diff --git a/templates/dashboard.html b/templates/dashboard.html new file mode 100644 index 0000000..75e0db9 --- /dev/null +++ b/templates/dashboard.html @@ -0,0 +1,305 @@ + + + + + + + TA Organizerr Dashboard + + + + + + +
+
+

TA Organizerr

+
+ + Transcode + + Connecting... +
+
+ + +
+
+
+
Total Videos
+
+

0

+
+
+
+
+
+
Linked & Verified
+
+

0

+
+
+
+
+
+
New / Fixed
+
+

0

+
+
+
+
+
+
Missing / Error
+
+

0

+
+
+
+
+ + +
+
+
+
Control Panel
+
+ + + +
+ Next scheduled scan in: -- min +
+
+
+
+
+
+
+ Live Logs + +
+
+
+
Waiting for logs...
+
+
+
+
+
+ +
+
+ Video Matrix +
+ + + +
+
+
+ + + + + + + + + + + + + + +
StatusPublishedChannelTitleVideo IDSymlink Path
+
+
+
+ + + + + + \ No newline at end of file diff --git a/templates/transcoding.html b/templates/transcoding.html new file mode 100644 index 0000000..b13d6f3 --- /dev/null +++ b/templates/transcoding.html @@ -0,0 +1,213 @@ + + + + + + + Transcode Manager - TA Organizerr + + + + + + +
+
+

Transcode Manager

+ Back to Dashboard +
+ +
+
+ ℹ️ Info: This page shows videos with broken/missing source files. Click "Find Missing + Videos" to scan for orphaned symlinks, then transcode them to restore compatibility. +
+ +
+ +
+
+
Transcode Queue
+
+
+
+ + + + + + + + + + + + + + + +
ChannelPublishedTitleSymlink PathActions
+ Loading... +
+ +
+
+ +
+
Transcoding Log
+
+
+
Waiting for transcode jobs...
+
+
+
+
+ + + + + + \ No newline at end of file