From bc8b25ea0219ad63f9f939ae0be0090dc6bdd9c9 Mon Sep 17 00:00:00 2001 From: efrazer-oai Date: Thu, 6 Aug 2026 02:55:59 +0000 Subject: [PATCH] Add durable user-message queue dispatch (#37204) ## What changed - Add a storage-neutral queue extension for listing, adding, editing, reordering, deleting, and explicitly starting queued user messages. - Dispatch queued messages in FIFO order when a thread becomes idle after a completed turn, while leaving them queued after interruptions or failures. - Wait for user input to be persisted before removing its queue item, and report hook rejection, task termination, and persistence failures distinctly. - Snapshot local image and audio attachments into validated data URLs before storing them, enforce input limits, and emit `ThreadQueueChanged` events after mutations. ## Testing - Cover queue mutations, automatic and explicit dispatch, interruption and failure behavior, hook rejection, invalid records, input limits, and local attachment snapshotting. GitOrigin-RevId: 14f39d7c8cbaccb767a6da84b82cced0edfcb4b9 --- codex-rs/Cargo.lock | 22 + codex-rs/Cargo.toml | 2 + .../app-server-exports-stable.json.zst | Bin 130446 -> 130503 bytes codex-rs/app-server/Cargo.toml | 1 + .../app-server/src/bespoke_event_handling.rs | 1 + codex-rs/app-server/src/extensions.rs | 18 +- codex-rs/app-server/src/mcp_refresh.rs | 2 +- codex-rs/app-server/src/message_processor.rs | 13 +- .../thread_goal_processor.rs | 9 +- .../request_processors/thread_lifecycle.rs | 7 +- .../request_processors/thread_processor.rs | 6 +- codex-rs/core/src/codex_thread.rs | 91 ++- codex-rs/core/src/guardian/review.rs | 5 +- codex-rs/core/src/hook_runtime.rs | 85 +- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/session/inject.rs | 12 + codex-rs/core/src/session/input_queue.rs | 4 +- codex-rs/core/src/session/mod.rs | 15 +- codex-rs/core/src/session/tests.rs | 87 +- codex-rs/core/src/session/turn.rs | 16 +- codex-rs/core/src/state/turn.rs | 3 + codex-rs/core/src/tasks/lifecycle.rs | 4 +- codex-rs/core/src/tasks/mod.rs | 85 +- codex-rs/core/src/user_message_admission.rs | 199 ++++- .../tests/suite/user_message_admission.rs | 76 +- .../ext/extension-api/src/contributors.rs | 1 + .../src/contributors/thread_lifecycle.rs | 13 + codex-rs/ext/extension-api/src/lib.rs | 1 + codex-rs/ext/queue/BUILD.bazel | 6 + codex-rs/ext/queue/Cargo.toml | 34 + codex-rs/ext/queue/src/lib.rs | 19 + codex-rs/ext/queue/src/service.rs | 394 +++++++++ codex-rs/ext/queue/tests/queue_service.rs | 756 ++++++++++++++++++ codex-rs/mcp-server/src/codex_tool_runner.rs | 4 +- codex-rs/protocol/src/lib.rs | 1 + codex-rs/protocol/src/local_media.rs | 93 +++ codex-rs/protocol/src/local_media_tests.rs | 128 +++ codex-rs/protocol/src/models.rs | 20 +- codex-rs/protocol/src/protocol.rs | 12 +- codex-rs/rollout-trace/src/protocol_event.rs | 2 + codex-rs/rollout/src/policy.rs | 1 + codex-rs/utils/audio/src/lib.rs | 5 +- 42 files changed, 2114 insertions(+), 140 deletions(-) create mode 100644 codex-rs/ext/queue/BUILD.bazel create mode 100644 codex-rs/ext/queue/Cargo.toml create mode 100644 codex-rs/ext/queue/src/lib.rs create mode 100644 codex-rs/ext/queue/src/service.rs create mode 100644 codex-rs/ext/queue/tests/queue_service.rs create mode 100644 codex-rs/protocol/src/local_media.rs create mode 100644 codex-rs/protocol/src/local_media_tests.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 70880f31a7..58b932b68e 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2035,6 +2035,7 @@ dependencies = [ "codex-otel", "codex-plugin", "codex-protocol", + "codex-queue-extension", "codex-rmcp-client", "codex-rollout", "codex-sandboxing", @@ -3880,6 +3881,27 @@ dependencies = [ "wildmatch", ] +[[package]] +name = "codex-queue-extension" +version = "0.0.0" +dependencies = [ + "anyhow", + "codex-core", + "codex-extension-api", + "codex-protocol", + "codex-state", + "codex-thread-store", + "codex-utils-absolute-path", + "core_test_support", + "pretty_assertions", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "codex-response-debug-context" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 9f971f7e96..21e9e91f11 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -61,6 +61,7 @@ members = [ "ext/items", "ext/memories", "ext/mcp", + "ext/queue", "ext/skills", "ext/web-search", "external-agent-migration", @@ -226,6 +227,7 @@ codex-model-provider = { path = "model-provider" } codex-process-hardening = { path = "process-hardening" } codex-protocol = { path = "protocol" } codex-prompts = { path = "prompts" } +codex-queue-extension = { path = "ext/queue" } codex-responses-api-proxy = { path = "responses-api-proxy" } codex-response-debug-context = { path = "response-debug-context" } codex-rmcp-client = { path = "rmcp-client" } diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index 3f5b1db01e625a55c77530ec7ca3645bd9c7b046..9ec0af06a19839bc173da5e754e4eb05d130a29e 100644 GIT binary patch delta 15151 zcmV+~JJ7_A`v=GS2Y|Ezqr!jG^8jWncp^Y=b2g3a675XXK>nL-ceas~vnr^H>6eHE z_6}W7hGC2eYXEQnZ~##sNC*t&^VD}b11SQPIy$r~Rq8rntIO%9gWIq{miWy$QrwUK z`Z&3^tTe1Or=oWE%?Q@rF0rkp?prvc2=Q)^XypsrpdNPkk zV_J@v;3E`w*FRLXz*KG=-gH3dN>z!C%R*$GP}T}b(TGWPEUUO%UdDCRstU& z9Bce@<6cLpt8$Fg$(5FCFwFAXJX;2G-p=~Q91dbLwGB>z2CFnrI}{?2gU6YZ!@5B! z6|1s#K*y)io8o=C*#wcoyCM>^Z$`NJs5=~K+90su%9aEJU$KAM3H}u?DAs0daZ=XR z4yR&WD|OvOCp@5Kujhc8`jo3NTMSf*$$Nu%H_o|^H7~6Y;;+uO#LDZ@a{mxHhn_Fj zcs!uxbKofjkaTH0TE;fPo4PirPnzJD zU4e>;p^P(%V)GAFe7M$J##EhJi4(n3tD*7~KF02dG4OwP%8Q$9ArB@n0<={-$ahA| zVBBZuMx{DdW{nogqu7dCg4)@={lzv1mrSC~vcj@WR9Rb|J!;2%)cayY!%K{5LAi7P z9*;J6&9MoQeQ-Staxky$2{+=gw*tExlh$MT!BJAsOfLmNn4k<8!3j?KK;%DGCqr=X z0Sow$pniXVS*gJ%Z2!EjDdHn(cZ|+b4iCxIuZ|4_YawcVeqc)X1bnsZIy)ZJ7GWQ{Y2^=S$GJouD`eo!<&voz}Ms zd1HUPI4?azos<<|U)OSnbY15Z*(+lj3F~7!eo*TR=*tF!6rVytv{fRi(~QKwcH?__ z)%&7_(~|RCqKWFWFZ~=tPFh#vz>#A5!ReUbihIq|VWDgsLe$LdAxgqo8&~r__F!%O zu4q7RY%^i%Fc=nVl4|FdNw6bT}YS4c>9E3BR>1U6Cp%Xr%E!bQze2TcpOy^q&j%7-u>j%w#fC}B04KGYMS1BgOJZnuB5 zvXX|OyCf(rVeT20cG${yYlhKEVwio8AL*wTMHVPN=l(=m9L9kSrfmLga z8b1#Qp8}B)cvgK@@F@K{#EoR0&!ReSy~9>66WuQ zgZb!Y1*P*uK7%eUA5 z_$O2K8zK_{*Q>W`wou7>;BF8x?Bw=-+lLL`JFzRuS%H0%#ef)51oMASoMh#qwdKDR zOw3-up0I9(IVlxXPj9^_RU|S`0Kt| zllUSd@X!3A8)W;IAFqGvRWMiK_MD&%syo_+@%xB`mh%WbGGm6gi`;QYU9cBwf+ziD z@i5X4s6LkzUr}RMkt}+=B?ImTW1Qq4F16~>r|wdt0&*}AalWdlmDDfV^3-g1P|}p* z%(CP-)dQIb{Ya@^kT4mN^RaW9>qL#FmxNXsP zE`SsmWN7>b{w05kNiPWyTnC!nOR&ieAzW3cr^>Uz%LkJ-4v?jfCzyI$r zfG(K!$>*@*B&rZ0v&~@AbK?IKG!hnq&cMZR+(^K@w!%S3^2usz&zb`dc(ahwrdEO7dnA+TmO`XfTW1Ww44mU;WePYYc zW}&5-BMg5coRcQ{>UZ9SXw36NlidU#-S)`czEW#NxcKZt(5=7H7BO9I@IIRbgbPXu zGA~z7VcGnVp?|N0%fX=JYf=V-$&>U+yuN=4PZ9?+-JeR;SnQe~u{QhN=n}XX;&>%) z;S^5-%8)s5j7Kd3U%+gzXi(?hmT{U*{%)nODMEj7D?J!;$ES<|C^9Yvd5|PFDEu}q zQXxbINJe0lP;?+J(x#ctK8GCFdDlM>-Gax*k=D@MGA^+bA*|g2{lEoa(0^O>i@qOq zlcw~DOKt&!E*JQ92sNM;k@$fw-9sRxioHX1G(h%dK%zCe9(!V5g)a=Io~C<_M;U=9 zL%Dzc43pzhihu{)?_L4f0}lJ~<` zg{?}~uCpbCe%k~XZcC)HRrE3T)h;}_7Y~1ogqB7q@M2cAiX&A^V%hoV3h5!(VtEKT zS>9&9l4w##n6(_l)^=e9!$1os)atd2tswq#ZUbaZkEWVq1!%Jj-pbg*S$d z#1cd6>&=Ays}cdH7-z>pn{t**C*(RG!VeenA0&F@?iO|vsn^V-brX4(-fS+76&EoG zuYpjEdPt~=O$7qRa-S1}S`u!^9~plp`v{WOF=p1jrkpEB`Zc6*5W;LB>o5u#Gz|E^Ri zC-6Jb1npb{lVg+4dgXLjRt$9QO9k;`Ti$Gt9Q=LH)@^u7JW-ut)7a5fcVRepkL%wYOTAA)GL zj&aY*SCkI0Snql-aMrpl;TlQbi^uvYC)MmElH?(cGE@ixBnZ)l{f>uM#nDV(d_NO} z$LNepj3b}&mesqy%EoS`x2=Dj&D-mf{-3S3FQ0(d{Mou=x6Mv!>+437^^-A?6^mvI zb-BgWVzqv938>FjK@S!o3cm!4=<~AbaxMdZ{}@+r) z)7x1>6@J-zjyyLr!oX2f1SY z0CmJf7MdUj-+-|V5Oyw3o2*d|@;xI-Y;~v#0Eu%{DUp9=brXUD&-!riz_099 zB6AA3W)ZsY4lFaJ=kw!F>!e(Jvo6R`QYLH_#h3Pc6*t?vNNvI49w6*G8_{d=aZ|aJ zv?r-Et&}j1xS9T2YACazT0RcRoMclrC!^)XI;5|3?-YWM;1+VvYoiEspabwLt%pk@ zM!sGIT{Bz785MtY!vYBz=65!SeFI<|ju|fSwmkfyUv~J33$_PT)&0xrO(M5jBrK7k zw%|Ob25YX{d1Zw|2nN|x$9jSNYpyGv%LR~ya^`W)P$HD*^3u71{ltuP+u|?lY>@mM zquZDnpTis0b%b}O%5gLrS|MmM9#OT}E1J1{J2WL;x~_k9g-vna*Ly};1#02SCW~-e z&L^?Hn=pG=(9JO1H-j5L99+WXT-4BYet_xi8nHLzW7rJ2AedQ`l3PY_56Ef?rE!Zz zf6lzP=eSVEq+W9gUUAH&s8GCT3i)GBL)3yQ6?@A6F8j%$V1(MgLHK4!65#?aerwtq za*$6%fZ~5#ZfZ1v_u{7QEgF*Cr^TiZ&@` zV#Ufgfhr#dUC_uagrV?=Hbk-KtEG(6zOtfbN>Wraux_;X^>NtD)kO|!2QyBQmGl= z94&u##fe0Bl`#TjfnPAg_R!nMAxLeQ!C4DF?gtW$y~Lt8K0G#2{1C>$6mifzsc zz-&G!Tw^56aXk&G1@GV=0WjE3lv^1)pRBUHs)V58l}Nlu(A@8IIzn~GwvH5EE`5G75Yd}eeLXfoy}Eg*D=Rg{0B zYCYT|3blP}E7xhAa3&p!ctcGD?k5y8wC@OIwXA}MZ75HJV+*}>8~Y+)0>6~<%Ih1C zfG~EXvJijA70x3; zbyImxFids?7-_Kmd=X2D3IaIk={wZC#Xj6IpcvpXUt??5RD!P*n4ska^ePYn2dtk< zHd6luF-A=47EW?xBUrO=X5hGWa83VSVN(l8!TGJN|1zb1&s!Vg)z`rlvX4#xdbF*i z1M2Y8>4xG+aYTdacNC@Q8JRB+N4H4goK?dgoKWP z|97k-l7&}eM@U)HC_e}Qp_ckL>(iW!y9)F?W&IQS;roawQOju-h2MXr`-*){^@5G% zJ3yOjNh-4GTtEzwNKqV=V8kZ70eG!LXU!{P15p1jjWCAyeT>K6+NP?c`sE9m0TAil zjt;mU2bnUL_}T*rSpE4*73p+4iW6zeRy!^io+`?lH>~I>$D9e=_(NQ5|M_N*+)#AW zs@NEnszjk^iQ$S?>{5Rt_wf^K80mS-xzw+&=C6#T;T6q<_LNp%r>DMMsNJ*v(gfdx zIM7nvg%KQ1j=}1VLrj?z|8vFUBjQ6g6^WQtzf|LJv~sgL(~Fn_E#?2;lisJmA`ai=WRQ}gi_+l+wCNv)`D1^6=abkPeD&T2vFL(z zBa#JAC9=d{y`*-Jk6%S9BAW84+H+NaIT=|68oU2cxJ8DbS8@gxgNY@bDMiyDcmqmx z%J|m)|i-Y>PmqMDr5XfBZffM6Ut2)I8^ zDspa3O_ee^o$bFNeV?H~9Ljio@;Z#KX~cJyJ7BofG1z@v6RejaIuI=2|Ff%5tvqP! zp_kavzSVzO#Xmf_Kp2A=ANdKkU$rO8x;tBfkr)Gv4EkLaolj3&j zG0p9FA)YzPSvk4kUG=DDSJ!>$%97Dgg-rTTN%Z&AzPt8n|urz*lid{=h29z8mrR zF%Lu?(4q%nKd)cJsdD-P7BnBAFn~ZbEtPzV#Xi5SaaVY@7$!IcyJ1xu%xCGk1$3#T zEbekGhV3d@^PTLkq}#wopZl4ix=F?n#~VUHGnlT^N63dIA{Ib591R0n!3BQWmvgDq z4f=nR4?fcH0zOgVujO)>=Qm3B2lrmSv-L)xjpcubS0W0LX^)um~M6n3+gj8V+3 z{>vhTEcyWllkjUK-zfxUD7+}!^BqCJK30EraKVr$4JzDXiZ>D}lg%PWhhn7R=OnS;u9ROa*+yOxrBHp=J8Rx`BnE8+^Jx zqrwtF*02#5>l}9rAHVth1Q$DesyQKxM{J;#kqu*pO|)?;HuR)&k2S@gEjW4;{QyfM zE`6Xfh;yYR#<3Y;C(=27Rm8j6*4lrY(P4qt>%fFtLFiZQNje$-dzevSR00-Y`V2C} z0#_Gix&OnwNFd`Uoskd)z+Cl!Bgli{KLaIMFoAUMb~9RFNQ-L?C*>lgNN_WpMAkgr znNBYBsHJcL6C%S-R-q=;7rQU5NHrlHp)*gz)Fl95s74eXTKD_~bO&*+nreS3*uPp$ zB)*X=y_mB?0^-{?-2d+RJxgK0goIeZK4GweNNu+?-}l?&Qj)x#DANxk008LFj;`Ps z6yAWhRWCYNDy81%1os0lk=0xdm%HR3JJ|+twytN0Q1fi@Fz(NCy0^t(ZflL6Hl@YU zw|~1z$%pNsvC+U5!KtSdfL4Fw2(`i5d)fbjEsVWOBhI4cgEIMy8Gye9=06Jre69U8 z+9=OBFXi^K=hmBOR`Gk~K_!fTfivf4s99>Ti&@$2DlI~DyExNdcg*+9+TNJL2rg{? z%$0kqrcH1~HsRf`L^R(&S)v@yCZ*jQ|(Csr5_7j zr54VwhbPrNB7$uM6yc&w@MRTo<`8icFnf$z8WaDX%a%24*}~Sao^ubrQr=G#C1T{ zn_wT(N_=1)cmfF1!9kT3CKLh<&#SqM6=U&rSte2SXr1wR8zFm2-XtEf`PV)^t}xFJ zH(WgAS(_%w*xs45FzV+y3V(MCx^&t+L+Qz903^qBh^ZV<&ozIkwF~PQ%w6Z?1%Fj) z*5*Lw^BE_Yl@*M=<4(8VsbDdM*xs95c!HFv< z_9R9s2urjt1+XI`)rmMPvo zN$IfaTZtS(X_|kpYoAPo^A*(o;+Y!2_v?Rk5EZsf`htNjoF_*;pLU|_=;@;$vc?eQ=)+ z*P31nFO7f1Qe%u6L&jR&T5p#;@xF~(K_-~FTExq|%9ITZh3C+&xR{lr^TO{&?@PSv z7D4X&b7;qJF&&iJ*99a}*K-xhMJ=a4AaoA4`5=5jmm09wo%Q%QtJOuQ1H41qbYC|* zMw0%Iu6+AysW;(*n)yksv#tpQSouR{H(+K+)pvg_NxifpW#1Ypl|G#fIpo(!|cRk;tVj z0)ZWF2I`g86`~GfzEb=CH$^<+k|LgazV@WWCTGiD%U2hH*xT z-&%iBpsuZ%3n8j$ZJ(iy4^pL%UMx_$dLIbD59`5ki)U@sms~#nC zaydXj^M2oA#MCs_i>!TiVbo(ld6QDp)a^Z7q5xWL> zqNXwX*!tAaU^1MdSp5DTAcN6?IyaB<{;A!R?*p+>k$kEEmML^1Adz!JBPrFHIGY%@ zTx$7c@UccM`KOf1Z3piIw+aCW3j}TeWB_geED&Ls2vEwAa zzrSsVja?h46)AH5R^sL*Zhj8$K%#$da#W7|y%o*(!r7-QK@fC(&p~&8(CtCThv%10 zpY-0znwy{8tRcutkci#1u~r z1mf=KFGBfx@Cp$;?<&Jkp{Kj57GyQ}*hSE4bJbw7p@?n1#8&}bA4k_09bR-3#K?gd z!>3bQBX^|s!pZ@hDR)!z@auo1DL!-E8t8z_(_(q`bcCzHZG7CJpN?o{mw2Ga1GDe{?dO_t^eW-_cY8H zIiUfk>3aDVvB-3gCtc4%R3EN7E6yfIm~UESb)b41#@bx0fA?JDM|YJyp8Nz_y%xrH ziNa+r#CtE2**^saPg~$+#g_396a_yTug^^&NvrUk?ajF%Xg3y!IZs&mLYV~((S#hq zj5Vyi&ChsxtcO!?K+AvQJc1;fhEV(fm(-7p4C1vDjpuD{ZSs=%g>QT7#IkB8@nbC4 zh?~UEdyN5WXri~1_}vN!$PCKaSvaLaf*1uT+?Ai#A$Aq(zN*H@m9g`S)a#rTo7iQi zkDJ}>%a&$nA}cN9*K3pU3xApg-=EWH)?{fe{rrajpn_Ap0;Yc@#!F-#(hX=HRc#C^ z_^(2NJt#e`UV}3D_^yyXg+<1yf?(fC0GK`(+^=N&0fc{arh^+S4;IGhK8XxhF&D7zYv1(6sce_3YKvYnRk&1>w!#r^f$EN2 z);ir1JH?fu?84P z7tJ_lD6W`_y6@+m)n(K03842bnckmM7n*R806?MLPUaj(TO%y-HsxA~VVz^Y%0h_; z5+>S*Ej)ViX^dN&z!;+B!rSYxCe-Gf=T3_mAU+H~p!|PDkyy!CE9Q|Q#YwyL>j};? zr|{a@3)x3iS|mjT5%eU^Rs`|=6}}2_r%O#;B>GKTDpr;(a{r{p`JK6B z&Qe}u@Jl|a<^4zbW7K7pTtPGd7Bfb6b`M{XklGhkNlrnf<)e%d^_b*M5T3!x<)ZE|_d0-q{K zIKIH>!fJ3Z-ObQ(SRhsyz_2eZ8#Yu`Ebd)21IyCZWAzv;b;Uc-gabmi60SDtOanS* z{oEQr|H!|RWSm2Vbff`VK(w>6hEsnr(maG&lT;A?NqC5@b3{!lHZ16VOwn-CQ<^Ch zxhVB41D`oWt)Tv_8T&$TyPq2ViI}miFJFp$%_Cn4@Wrf8pxk}EyC%~ROcuEAm-Z7W z13VaEC=teTH(~>x^pec1;@-zjeSya5iLJJ`j1a*&Z`VM)Q5LP}x0wJ-Or)si>H!6<&E{SF} zroxzwO!a48?k(#z#t)~YwhIz`!+b!mc=CI?(WScCsw(=jbPd#YkE5)4ix|f+?t*nF z9BhrilD~gkqQ#n8|_yV2Q9qQvMx7L8Ud77zs`5-Engzkt%OnD0Tj${YIwKE~%de z{t_)48V;UyRycSL-7|pQYqS3#>83e=QbBz0lR;a^M2Qb^QXkDZ4q$; zK4#l75C&dNXqOq*$hc~XV^n9vGnK+d16sSaS&{HfqL@X$q&oXE1ite|U=7bINl2B~ zMQ0M0qmsl)CC#9n?~z)tapIVQVs!aG^tatnnsek7Mbl2n`94EW_i{x6nlyipmbQC4 z;8w#O>Thx4TU38knM!h-5|uy;Ww^aMF|}W2LcRTN8*EY1iMsERpiVn5x+mSO02-y% z$iPh9dB@w9g;Lg&!W5pOhI-=jOPp-bE&P(nTd!vh@m^mdj@3xKEZjym7=|$^%{wBm zBv2r{w9$RB1ik?zHlMY(O?h8=&i6pcP+U4KwK8TvIoA;kmG zTfsj@N{_O->;q%+sgmuM`fv?B$@D|KwG3M8vkCYcCWdrc7q8|9}Y8g3oi+2O>qe(6V5gG)wS zQ}_QF&hSWo72o~1$Y7aFjzn_W3IaML)6|}PzQK>$8o5Tx926LG28yujF^ofDN0#I& zy7YX<8m~&E0!cMUwX864sW3PmMRidyE#z6MtpI<>a(2RqJ!+GX@j+=CnxnIUEv8Ap z9bwpJpTv7g3+)>hwuO8O^|zLS%51_mc(RO+2VqW;=m5AVmsC3@=(k=F|7DPYXeETadwlj~y!H2GwZ83XIMma67K%1~Z z%2$7qqGmBF+eEsP9ZIy{ft74vLtAbkZ#dxB+wS4$eTe&gi?F&Rj!s2Fje(dTV(twlhvdrU+Ci#vUAc;v0i!18XsFG@G~w-$`iH1E=zp3$kdDGH+`h-H&A=BY7irf0J$}xMY7v=!9Tw zFc9Sk`o<`v?pIUC3i+Y6YFy}K{^y_-v3#*_$;}rkDPFMV;fMig{vx%ozh&ZKjIvnU zCtKp=S}1au-*Rwc%0>x;##(SAKTA3w5kTWCk>HV@N@Hqk%eFZ2)vh8N+ZEw=`lHnr z!tY9MOR;A`_}!Og0bwT0WD9>IWKmbQ`XHUxm^)?zQgWdig`Uiu>Yv3}_Vgp$75HEm zo(J#<$WqY@IcjZK+Ll&s-Z$aWA4hG*f8T1%8R&lRK}5>(d*VL#!GZolAoto~Pv}!j&cyh$RDl zl?|d@-#Pkjtlz;vT~AT7=iSDOSll2*K_GyUDRBA;?!upZSmsR6 zZ{HG7yoG74gtstrPMrX6U6#;SBj_f;(|1?EBxuPQIdh&n@qSxe0=y*765u7EH8=Mf zcC6I4QUbih(35`v?+C9Xz#|fHA_1Px2@>EH0_G9mDR(zVfX5x6Y<+#^+adwo7X(1Q zEfV&RfvsU;b)9~158oD_TMp>@>Afw=UT<(geH{>}{+?T4C@g z#4sOnbDn?rDd0U9;OwV>*C{~G{8Ydjbbk2)S04;F zR0&p`{zHjc&q!r(2I^2Yuz+Rpzu;$6P&!d0>!p8}!pwa-wXhU;SY{y@`j+@#>*ERf zAixmpWiFjmlV1isv&EEDC>^TfF(D9)^$SuAz0ekoJdAW;l?+F`mF(2z}C^ zisSLLME5tQ4pW_hmzlnM+Y1&}&)@QpvVJ?q1o-TutNCRn#@STcW zt-F7K5Hi_waiAfrU;)#MMC#1^ousNFVE&b>r!voLkTfE&%9xxSuJy`;8V0AuEa9f` zNC;gBR|sBPb4a2flKG1KBzuts_BSKL4Cc5J538{-Z{&*c=u_NKQVtYAV`K`gikbVD zf4LvYE&D6*^|iDY6Vx&yAzudzf(Qn+uB(4azPdNDv5x7M6QiiK%_jV0i8A4ur@&_b zmW}hERlO951aXR)Z`P`G36|s1SNHWwnQ07_`6*6Cml}a{+yB|Jx!)Uz;Rl1S1tOxu z6js89D=BpXPLz_%aD{;L(A}9Qb?k8uzXXYrxexNZNrJBH7lyQe zqRR`wDMA zhght7WQK&G$imzBP?F9v`B1k%B!m%>eqG*5n&3M=_!{mUp@xQb1{>hC7@2`mb$2*f zQyI?+;ndpSV~Nq^aw8DzXMM>=u=GQ3_L_N3$lDm!0`B=bA6PGwZuXiSXKsI9tv}Q0 zWBy~EJ@9T!BHC;tMi&J_KXSkvAt5yp9!P*H!yARSrecFPT+I49jio=xcQ^F|u?Opn zcL3}q+mLJ8TBgQ=yX(j8xEB^0D_5^Q}zLTe-ShTf+)P=PNU`ax}e7hO6*dM zD#HUPQv#&sU~sFkPn_ZEUCwNJuq59AIg0=y)?&)Jom?*$mT!Uo~__I0X&s@hvHrLm+V=32_Gq`OP*DULetS63%|Je=Qypri;!4 zZKTi5q_KduPaYTQ`H+v@z{|#hWcSPc|M&=|e`+S0!NFc}R-W2{ee|9Rf{6Nv7*plX z&OZubu*{j{M2EK-@}9{AjUvxATimryr+0>nnT5Eb}mb0@!r$0@G#wQo1&c; zbg`|?K!$GQf>;mgwj4>D2LR!_*kN1)EScNRux+l+M86G&|SGN(htH*^(AX@g@x!wRXfo?21|B$qT znJHWRCy>@wA1q9VA&|Rub7!SZOaUBq&(SA`oM?XF@&Df3+_F<>yKHTmQ<^5D~C@wX3anw5!z%TWKY?n#w=HNR>JN9nZ6WYxc$3j)1-mD?3d{p@o1HO=K{ zTI`{o?PK1Pt3cR4{Z{V>JQ;SUAV<3h>{4!(cx4Z40$#NNUBaEQM7f0~aJ z)O`L1gtZ2>)6euSc6EKUjALF&j4%wYiXjM+aN8RUdE7S_qmO<%{k;6^7(SJFyj^4l zu|B=Ydbyd>HPjLkOMe^xtC;O6uSyzVAG<#;-oe9~!9q8+%FdFXZ^1RAk(BGT1JDQe_}XiR~A* z{-56C);$!=gB^cX`_w!JWueGsWdyNem!KgDmZf4*gj@vLQa zaae92TVj~>#h35Y1$Mm17GbLR4-mg@kDcRN+xmRb&qg_Z_)Rui)%oE!vlphoMgCe( zmP>C#zsFr)dc_8%r|GWX@-+w?WyUj6O(%Y);!$Q)raMPXOa#&T<(W(EqV&kYyV1d< zn4_!t;f&D<+AH4xeiwp`f9^p`Z}2;m_XZhF^S)PrTbSeE0uy}t)dhX*!6;zO6C*ds z{fIWY70;l9JLr@(JLBbKW)9SomSw5>Z;o}#w9weZph{@3bTixGR5Qx*53DoVuz~R& zhQZYaZpeyS_W)`NA?IYW3W?Ck6MPq8Ug?tnv`W?JYbQE*QvmCse@_QHpZO-L2So2D zir}1n4h@tqA!Lgg8b~<4$@>D#YcYj@s$T=oGo~MyZP;>O6T; zRwXI|O5fUEmb>yczXUZtAOi^gp$nsdzE$AG^`aYOVF z;yR*Vobokdzx>>iDl^OW#eI15i-srd$pg4YcXIDh^UF2Y09C)zzMC-q)(l+E)X0q~ z;cDxf5D<&ogI*k9!M+I*4hl!_DGx`R69c|Y_12I^tmdiWf44v)LQ}51aEoxUekX`! zv#BQrt1GZibfTn3R5$s62}4AiEiLR)ut&yeR^5SrOMo0|K@2YjaF8&DI?ww4MlGn( zz7xa|41$JF2CA?{gmHWbQ%0hFmS0G6TOI4ONc-k`bfC+)sVCzHR1O=h0CY&iJOsi* z%ZI2yWMCv}e^djR8S$*PYrWGPQmD%A9l_l(v2NTbvMxdcZ|%lR49gmvh+>Jf+o#yz z)u39teH?|zjQ^fuoRh)b_UT93ZQqtBw&E?|XpIPva~28jF$RuavGV;g&>nDYQeu)N zHKXm6#NPM9uko5d&Yaneu8@dx+kKCt!L4zQ>;6N~VKCl3zSZGIYG_k6hy=l;u0up< z13X_y#_Z9H4&8jAST0m?`UOCPebDnd+}Jkhe|>@W;)i<_NoPCMWzgD`@%v>-;}|xD d<})yURxdQ^VtNGQcj4>8n0rr$6SwXeO$P9}F2Vo+ delta 15094 zcmV~4q(5fK$asZsF)+P;}eGV+VRZPQn}P5Q6CR^H-m?jnAfPU!3MM2%17u-(Cc z*6jhWYr=iKFw6r=lqA)g0(@SGHd8vL{v@9pP@gd9Vaix~-okw2jXWsSlW{m2lXAQS z|Dd>gP91=xO5WGOZXADIv=VR$v_z-_@7W4WC1oi2bu#wv1zta=d@SZdm+M;qP+V*L zbHu%Bsq1otROL#_H5f+uZN4p&aNdf2)E0%)^2TaQdZWAbFng( z%5I_+9M87bTb@XLN)~Zj3{;88dz9eaI48T-tn|VJe^uKOE3ZS!J;dV{Jy)*ba6HOq z#8X=~EQi{--MRi_7>A5tsI#BioL0_py6wGU{gqat>+flaU#JxPnyjV)2}1w~01yxu z9448JBm#**&QO0e-v=aML5d8Em4tY%2m$~)0DxcscnAuc zLg$Em3;^X}Kz8pYd5ARO0@;kVY1mr8KxU{_7Elu`xR0p^B!k0}n$;tJX?90HGjG7m>rj z!uxE1TO6Bz$B(T;EFTEg2Atc0CLpC52Y4MsOWyu?OO(q$&$#)+RkiqUiyAj+%GQMC z{jK7I?2fJUzV~=_r7#OD>T?n|mg6NLTVR6j}V_s@e z+uIaKi<1SijFq4Y%`_CHX{&_&`E7Fl4C;SB!cl_*MdpVj9s*$N`{~Vet)YPaya&R! z>2a=_!8F|fTB4^vV#{vS=$AyPP6uXdnEA zGU&Oi6D6}RcN~41v{d6@A_d-q)6rzgt|Cf7L)rL*AcfgcK(JH`T=PCM#+JA;plEK& zF)^AyCIBOTYUEEf2+wWc6diJEjOUbSadLlhN$vdcIYZrIB3P`fv5>%Pr)=3>51Dzy z2-0zDl47E&1XzaV6%E$kRd_z(Buzy2AVj>R7d20I%gQD)Oize{qYUU674H{rEf-Lm zRu3oymQkQHo`$+0h0#)HgZuN}$9L~SgN28qe>r@Uuu4uJY8-F^q9%~r&0fh#!#IB- z36Pdb_Y5=66LR~SVJd(VGQW~Xa_dEL1PaW#pOF@dagO>XQ+Qjog|JXKZS~BOHFUB9 zGBu(rns@E4+w%Mk7IANP83d*4d=hQ`B|VG zdO;m2ocCW3(GvzQYvd;MM87}HMzn!z-U?q#;cb3*{Z!~4Yn_s&<$1)wN~nM5nze$> zL$rhKjyhksu5Yb(^}t#SmhHi<=n@xJC@vuEia*ho3@LT3o)PXw&501!c-b*-BH{W+ zJj(r`4?w_{p|*%sI<$xtg%;d;t+Dk`j1fj~7Me5U^iKQ%NW^tPp*v++f|9`agGgCa zEf=q<91LIfs(L$vcfbCDdnSKa2`HkxCTOtRY!XU*;%u^lJ6B0(2)=&C)^8{1T$IxpaMaQh%A2s0U8&JGYkC& z0#WsP^}fs=D_M`%T`q>5T-v{-VGw&KazrsKurFE+oC}JO`iY`hwJ3kf_y-LWP$l3z z*5iJNlx)@0;jf5)ClwYW2d-0vT*H;Qe>HGLcfeJg81l$tm63GsX^5`G6r1P=VyN7} zxN*)J(fp~!GLy=D+N*%>^+*8H9u55Z2tqQ zT>`o*2JI?>iyjMPz=;CLlS_w7Fc@_48hyR<;vpD5Vh4ll)Be zpd>=qDa8vCAVYGt2TpUHsL}M2a0JaTfrmA^gd{KqJdv^h%0qwrz!tX{q|WIYt_UM0 zWNIUljR6Fs&xf*JUP_AYhVp)xFm`S(ceLpn+Q!c0jK(CuI99bUvl8}jSUI?~Q0{me z3F>W*%~@;W=FSXMsr|P9>;>5Fn9v7;AD|Mz%J^(MP5THG1pR>F7td*3H9YMCNP*pg?&8*W9q+s^V6vypNN7wvcDYx`J1Pphg8F0H>)>Yojm{x~$LnuLQ>589)@ex2XD(5!2T{m-jBMskYf73T z_Ly$WcxMJcsq!pGrGowYq$y%%_XbPYe0ZIy4aP(2+cP;vW)NrUDkNs(v=iEVD>8G2 zL27f-L|A|Q&ZSU+F=}Yao8U*JLSfmrR=kh*6G*;Jf=oDhl0J*q;V)q$ap2DVDG-ds&dMW>D&39z-ij}dPvRCTpd_Fe zGU9c3(%ahwUk%0#>a^K1&idqUSo)d=75CGFC-;B0%0BKthQlC_#l!}5E8?OQLR2`& z2&{lYzr7gEW|(aKkb@}i}H-tHKb>$gEU zcuIfKA|74;RW18~UclyF83se?=}2(%j8@H5|3lE4%hx3sL$Lhv7;>_ujeI53 z;&0f=7UZH|#K}ro77(Nmetk6p(NpfBt7MPlFhZm!9qv2!1q`p4&DK0DJy@h z|8mM|sT{`4H*WW2`gY{a{ZyTkWM;bf^yb4oHr=>SP5;>Cu*o|AX|FPpKq@o-5`*#$mIZ&`Ul}+y7)N&aNh40_J-7X=~23gJXvpck;aO76NIyGOGG^+ z)a0iE`C`fE)~?n8cmGEQg!BOubqs&EwJ#&XKtU3Hd!QKe_bykIO8Y6&y}IoCrCebE zn|T%bwUNvryc})`KKJG15aJdb=|%)xRe`8UktjYx=lLpG@%f)~Iy~5MAoT*iBAURQ zBTRB!LNTxG3Xv6!uKlR!eQdv)@sfj=&xpG9#)c<^GbkD@x(Y9QsFrR;fuMh^p`8p+ zDfay_-tds<#7<@y&MbEH2a+{BSX++ywQv;`17fV0YpKdlHM9pN)~df z|E2yto&}&fX0^IT(sc3IKINpEoqUo!q)~=?Aul9DqFkS0$f`)2X^3xS;$`i{TwL}G1g4i@(DN63HvcWSY&(-JWu zwM-nvvZz;npAl1n#q1Ti%sTv_&~Rv{Rp?2X_lqJ0v8dG2P>Ir0KpS)|AhBn;e*xj= zh?ai*CLL~oOKTXBgB5uTiD$NBQC4hi!9~|ABK@i8S9OaO?NsG)P^J@IHqNA>h$z;$ zQn`f`k#lZv#A8ipwEck1^+&k?c^NpaL{p^i&7JSgnMhz+``YVEj!hn{&RLAFi2& z?rVHYQNHt$Pp2{IG)^1iq2&}4QN`Dn_ArZ^RW2q%uw@TBc1`T)75F%)T)Ntm)J!WS zA|r0#|CahUI#T_H!xJZol+AfcbxIn|oIN@WcJ!>TYWaUzeAyENtQK0I8^Y*{w;_cJsm#18QrC$JFZP zx>>GlV+g^jwDMgqFk;O`>bVpF8G6o8;{*aY2^}x}nUgNZNT><)vUv^ppJTs`nQ=Y5 z!4orhp{pEUqoLKE-p-Slz_Owl!P|sT;@tIDR}(3^&oY0zs6?JD+}LCVx8*>Tt#?e; z<_b(1hHgG^ABF@6zMR`1x??{SdK>)LX?zS5Lka}5X<~7k3ho(4NZ}QCxnIp0kIv)? zb#l~(F69}CKKH{Zin%nii#B#!KTC7+9JN( zcLyjr8agYRgZdq^(6B*F;dO2h6-sQ=FrcjLLtQcmUK_Oq6 zOmc7V*u6muF}RJ!J)#=Zq{g%y7?#^3ZP3!JK7t~d8tV_f8jBHWtg$AIkXtMix7yY< zX8RPD0U1It8g;qlHrV?A_QWEoCo1NUxGH}d6@#2{p+3uY(23|yK@s@`WP}1@bxqiL zkc^~kStTna08dZ1{Pfi%b6F*hlXV}Zy7qV>(rO1F+icCOe|DzH%Yry}R&ls?a?`SS zKk7??-2w(=G&DK!ro4lTUv?^t!PZphrhZ9M`Q)0>eSE)I{j1L$~To^VH&oItHX;>)#cCO@B1WevfK3-Kd&k@>39NBhF zZzkM}uFq?&yI(fjW+NNrI3^+3<-~Ul>4lU}oEFu!glR$4MCnT>G7xYnBY4WrvC|6h z)#PzRzChU(TnWn}KT<8Jc|BSh6$SMNW7h@jT?kKc;I0THDfTo27$ zy(=Av2wF^y5wt8Kh!jw$lAah`@=O$5dB+E#*nA>K#9&%sFyb&5<0gtEib{XlF);|) zeqdbgewpNZN3dquXJDjtaE+#3VRJOdmGcW)|K(c!u2dUy)YrEavX4CgJua-I0qQVO z=!W9Ba76Rb?bR;_G>nTK4Cw`ssB0j}xhet@c7KbX8P0Z&=q;jv*Jg!J)X=`}55ba%<2HQ*mfi zYB7bPC3YdSVnB_k$4{$Z-2hD-wyM`lT=6CCbf}$w+oU%(6PtMG$f%+cQ|&*9+Ie zYUhryrx&{yXzBBR;(9+X7IF9}Cxet9U6cnm_)Y%+%%AW(pTXAPE8sSW)fQMciz#?& zpC$gPgW5f(e-$|o;g?U<9;*V((8wyV+x^GNEm8r!q8V5)Oe}xtR4MBN!5dIkmFOsqpQ)vSi-=ynbNZSTQS`K23%h@U;;p2_KJv(%DD)M`0s|wmP6q; zb{w*~u`R|TTdpNk#d2VV&v0}Gw)rW}3>R>h<)qz3R<;@IfgWxXs?=}hjX1+Uwwug| zc3_l9g=(+U|dB!rmEj6$M2Zxoq>y=9dRC+1bT`{Z9=>qv<@*7i%{0Lp_SmzH>9^EnLf7RKtH(jryld0SIQ$LBQ2Hsr++m zg0ND?@ofJkr}qa1=1|7dHqoV~8$PSC-(0T*)L^m2a=V!YoyEZd{En94nq)=x-J-4=^rwfdRkd z!g??`TG3%1uQ_ol%Um zVyImut63oXD|Q>$7;`_fLpPau#PNn?&>Y9LX~ZMomxu)nhI7q;inzcB`f|of-2n0A zgPAnEEE6UE;;x)`_>G+X!IhiutfLWVWBK1r)pi_8Kg7(r>Lki1S6U8W|R{v#@ zA_e_`gGp#LQtcE1Gcdd;@BEHfz&_T+-(u&8nx9Z=hwKZ1G9XRuSPzC%YHugb8Vx7I zL@qY?BYqS?FNM!)oI#mO{9vkyy^2=jw?2RMdu%+2Jd|yXO4FT*PVy=^BWF1l1@~i* z!%i?(I0{w`fqfXI&4^bFN_#8gHWu$fTTqXKi3wFK@L{uZZOfCBSfk8P{Y=_|&Au}e zjv#%TwRVZc6vlg?l|W$sfP7DA3uf+|599JrrV_qk7H&4)&=P%F-N3@g2|it)QDJ}O zB5U-DnRJf1#f{%keuAnUEY)z31v&EKcJ8-fC39$^ZbEzJUw@$JI5z z-2Xc-5@7L@jweL1c&>WD3i4?AjFjWTtW@SX-arvHEKzD%Q3cp$k^dHs4eK&Hg z7gJP7Kzw1tz3iUfvmXqa5Ev_1C=6C0(srHab$xs6Mv}J^H~qT<2tWt#=n8+1N#G55 zTlJ!YN~P4(PH;b;6IsdSaJS11vXf;IXX}cF2sO_Z598h}r!!j&=C*3|*iu>?S^Ky5 zl$_HZ;yyZW(HeW80G=`qstpFcSNSh)VeGv&;w%baoXKbS0{jtR0)(ZoQpmg}hf7RKoNZ@N<8D#=A;QburVpUFAMBw;M72QOEqNtnDRL7%hcO;oPsc z(%67kWN_V$Dpe^7D`udrj4BRB5^{GGY6w(hH{arTwizvjad(|>sy$vn=|4hOnV0h; z@FcxAfj~0>MY!P$;;$kO=S2L1yzxe5%nsk`7iyyNSn0iII>kzc1Cr3u>X$?c$~)_U(N@FTX&@J&{CnE+KfGCO6wUcz1f`;BP;3ZUDdaCkRngk~EX~yD6-1nELG|++xY#p)nX4|(} zi4RlfH+Y~yTC$$O_mHvh=S#~bGL(S{U< zJZsY=8QVK^7DoC!58>}NXVvinu1Uc41YAx$Bs};ID$3*&e9u zU9n?;BBM#jkioNZX@jnUn^8FNR}8<$g1G!)zJr9x!Tc6tu5*7CcrH<+GdLp!#h%2I z3Pup^3k!BcWVaEAh3-uinR)v{_4iXf>Efg+P^d^H*6gnYh|$~=9X0SR>k2J6pZE@| zJ}8kxXif8V@sl-hzNXV(8B+s5e*HhPh|r!tphU>70ZLMoQZ@v&t;%r9S&(kVRAqma zwnaheh2c{*OWc1vuq(K7&uG5|04`M`j}2^?O1?c`4D5JV0r^G>WfnA*nGgs_G+~6( z)p|2(l6Z!@0j;D{fZyJANy7|0^*+N#vw9yM>M#lZ(v8?825mm-E&9Bd;Nq8Vu}vNX zDR_EQb8QAej@RUHXGA$sX{Y2(-+Ygv8iZ7P6cxG!ovMFeNZ>`BM^U{6x~1hNszO#z zii(>?&gnZ@C|Hpm`Z0s$bA{@mbNo>}tc7L6reI{oX?WSKp2?I5`T|K$W>CSY{I#Ih z!b>Z$)EHw%lCf4d*V`pezi*3H(1oe27V$ElK4k-g`W)KT7ZY=oRru}VeckW6McDfO zJm>f=Havf&_C$e1I`v$2zo^CI51h{3H6N=lKcoiKy0bHyDHo4)Ht2a}}# z?v>wP4VVp3(2jjl>!51_ZB{T)(x-zPIl-1rv$659!wO{J zW5+h+{vW-^&Vr8au>+oeBZ2t?!-JAR3>DBd)mMMQL6Onhi{h!OzcjHkWh8Q`n?PV^ zi@|)QRSi+62CUR@|C=HnaY+%6P{s=Zb<1DCT8rv#j{35lAhKS-j>OZ|=7w=eiQ8IH zptP;;g%H)WD4(I4RuMNyl;^ge>_Vu=fa)ftrfI1cnR}hov|CKJe52Ln2hmprjB%E^k(x%qI#w#N zUL9nWW6T9QA^PH&np*b9A2qEg;Q9sasA(Cq9ZcqOw|@AkB*buI08PsNktI0Gi<*|~ zW9w2wgUNG>VsVyxfDB6q>dZ-e{ik*fe;enCN5f5`vQ6*HtK?A|;vQh! zZ`)yGR|l$w6ght@aq|*4KZl1f(Kk6NNB&-m=3C+H)0H3yy1wV2yFcjmpyR{yOQ%nI z?_|yC&%>8LHK!b4HIHxcqEqW`skMKko0?yp1L&&4yVe7&Co=&xRUE~Zr)&b0uL*|) z35bey;Sga6)rz1esECMQU<6EUE#=F@(dWT_cepZN<;8(cCnee0-E*(Ch;6R6>GhG@ zbJsMh#RFYfeYn*JTR|h@CZ`ZFM=(2=Cr)NWK~Zq7&uQfW5(oLO8ERpR6IXwUDV`dJ zad-3=p?pPng$SN^b!o8B(_K}ISq?sS5wzM`gDtsoEZI=RHeceifUb|D>x&LAItgOr zK#bwjsjZPaQd?o=0M3)Usd@NS(iDHWZjSfOW?;baXsJRiCnbg}s52e48%dI6*bFlu z5P-md0mEbhc_11IMI(a|N%nsOBVZ5_X;=yci+~_F2m}IxKp@x%f*=TjAUuK~48kD{ z&j6K1Z(0Lh3q5=FTZa?(2#va85=s`5ycYZx46)C8l1kBo?I(aoB6=N~85T?kj28yrH<+ zvcKy#j-|c{gA0lo%n(iF5zJ7-+NfHLXUBT-_2y>Q(h)}~X9(d3xWj&A91yQ<(0E>P zE0YewFJ{}@Cbp|W2tR*rxgbgiKOff^76&N36vFTJ$NOZkoSi6BDiR=3K>l5MeQdDH zv}LJkd|VnkdVh7DPqB$z+B|Nq&a7vtLfa2gD7;=9h~I2VDtOW8jKoQnxYd?FcmNeV z#w#GCE?yw!A>ByNo@z@_LEI{2o8i%G?Uj?Ay?=vr=N1{S3vhpUrxJi&lZ(6>Fx!$< zQrcHEbN!Y3jxAg0gckmv3LsRk3{1BJ{Edy51@yVyyX{ibd)W3y(RhdoCqKXPl#9*Z>*P zI~hjj75V>#rx$;vVt_GnRK^CwgXpLTkxq9$E6)s{1iq?~OWH4WfB)VI03h1!FW@-( z6}gDFiO@oP);YedtTMBWaDZlSaYt1PVl=f0EQ0*D5jcI-mE~sAsk63Ta4_(LGcOAG zN(RO?kGxZyw51nX-`pmE*VSIgb5x}vDdI#hC~?jsi0^-|@KuO+er z>!^%9ONb5ZFNqGz`!zz?CK$keDeWF<{V)5e)ozu}bQaQN;3w%-srFZ-JID!nz9y`l z2`IjH}DWU4tCHazn3% z5KK-z)gym}kjfr1Bh=m0FV{V5kTZIkC)}j7dBVEh>GDdT37J)*lSfw!lA*7-!APN* zczMeniI<1H*$f?wvW9qhD+iU_N|fqU^_aP5B3xtibn!g8(6{K=zHN{!^ko+<;-N$! zOMDK*GVDYP9)H1ap)c_Ep=93e>o{=WM+sP@y5WBli1y$sbpyYt!iV-O@7N%O-v~R#ylSa^Z#? zCc*u}b%Bl7wdL4q=q2aft833hkcb=}PvKvwQ=pl-%_QWDFT*#l*x=eJa>iabA<=X^ ztE+!0N*D2z`&ZH{?|zcar4}Gq7(YDuN?*>?uvrFlfYbX`g=v%=h|>5OGe>hbUCNj? zgwSAkmb*&F7&%x$9??3W4ajF=>fX>STGk^NRUu_ki=a(Csj$edT57@<^TbSpKKR9w zvu9zmjS%19y@aFMWnx!-XJvvwKy);aT5x|DnD9(+AK&j$1IS$v-4yMRp}kiqVT`OZ5POyu1;#;qnIlp%G<4QYDSs0fEd<8#)SI{a%=Txde^ej2Yk#Dwi|`BM1pdVIrOB@Xz+ zRQI*+nrt5?nXq-g{7*gyc#wpl#Od?85eInHds!+>@BK5$aG@^cMVVIuaB$=2;x-dt zwMy!eZ0zwX4cbYf&LP6I^a6ToH%4Iu)FPoYYnUc@bsxfAA&3a-?n2C687_apgY$oM zNn$cjH?zh`D~`-A&njw6!85)Yq_Yg7 z1O;y=I&du(;TG7PM0k3ryOA^8Y@m-o1ad#b9y11m@4w%YBut>~^+z%DOjJL>lZ*40 zor2JE&q-gNITMjk@pT3APWyK@!eLbcY|$Qk`+w?C$Z}$*xOtH_6diwnupj?2+uWW= z1>F=%S?y!jmg$sE=4q?Hq+CNfzf;2I{LWHL8EDc_9{*5=NYG42u#UY_Y3oNt5(jD9 z5dGdYsbJxw-!|C9@73H70hzZS~NssKAzoh!LmZCR@s; z4c0_u18E!OYSt5YgsXq=RJ0?ysn{voY6$k7qJyDL#TL+;qVBgv`l8u%eb;1~ej*hE zzgt6B3$KvOYH#Kwc|bS@KCB&?=Z+7T+qHVZ;bOf@8&|0CZt3wVVWsdA!`gmME|&P6 zFMa#1$(%oBo^1=n4fu@Mj-~f%n%rJy%#v{pQyjRR5gaMc8%lqFZELeF;Ro__ED}gj z?C12^<{@tw&nmykP}VhP5@r}q!x>7N0krQO7!GlcoB~Z#=Q>nxmzlJ3g>uAES&=9q zL(le1I|0IKzDlHOM`d`KhKtbODa4mosCJd)CQ2lQ(#mi%ols@l$%GmMnKigU0u$0N zAA+c;zqn4igN}bJ@>z?4nL6l{Y~hVZX-Epg3cL!{iO(-F5J8vl3npj1p83XmeTg_$ zC9zq!9ob+Q#-uds2(FSq-S?6!QzuaeD=x#+Ss$XKo*KzL&EO!pfu~vI-oC{qW+*}` zX1Vqupd|E1azfW^KZn5M87X3h%$u$VK=f0SB zgbV;mtJh?m5o$3N2F3`JJ&Oj76Jb``*s(1nS*U-%=L)vSo^FGTWmJ1`;S{NG2YhVz zKE9EUORE=Gsu}^ML~_2j1u<};x94#JiTzg`g&R8Wyyc|;?6q4Ae<=9&`64`T?TccspQ!%CBUH~;Z^&5fEj+%JXPfM)raEKsRfMA-< zO~)Rk-IzWJ3QDwN2>Spp=Z$eH%ljPqMXKb>yM8AE-)xqL({pJ9s^z`3B5YCiqJT{+ zAwpc!%I3>07A3zlNHwiwZIW=Niu{HZ-}Qfk=7u7lW7x7(Y_)`-Od@)7&+w((C2OsV zgbPW9O=pG6sJg-mzCgYDPvll>q&)P~>_UGYKb(|tMVSqsvc%zj&w6@jN~36}E=Xgc zm5m2up(|7{%9qP980+=kZ81qjVY&6*89GJciF(10z8^a69 zVSKeiIMcOqA1{MR?KB<}`;kxLdS2dJR#w#TnQ<^-)(2yU@cB5k6Xe$45F42-^W(!r zI}wHA1B@L+VNXCuoU8_b>5U9r<-UKvA~%bi4|FT_F63-V;;Y%s7=1aLO%FTsZkFo%f|)CsQtCPt$8%BZHcht zjfr$ix|QIP6IuU(VS_;ihvfTFkosOb7%PE4!m9CjnOQu)n24pMZ6yc3_@sY$Uv+&W zDJHdww8Q=uXMM44v3O8eiIZy~gkXMyt&J%qMHmd$4jboG(!_~?i?gJ~?DNzhK~`HJ zor#gViagqGX1>!OO*YJYS1?$Ga-{L7$Lj5y445iyvEEi8<3If{bR!~AT~ zc1P2Eu>t2xIm z^JPCZGKyHZ&A8HmDf3PC?q9WF2R0%hfh7R1O6u*~HhnA-=#a1LPneoJYUtN>pt|JJ ziYkUchQs_UN|2b44$#yCJUGD@gt3r>o8?<{M1Ct_=IBx)2UG{35SV^uT?)0**&!pu2Q_`7vcLQ9Rj68=uxUAvK>C1>Q!d9K8} z*}}o!#c&P&E)}hr8QZV1@Thgc-|41S@ONEn6a1Z4!a?wN6qOGA-JCE6{tiUD75F>w zz-7|{*Oe_0(0zYSU|ZQDlz#|p?G;hi_3s!~w)nhdf%cxNY*FS>Ws7;5TZ&GfJhxNe zgzhsK62f?CnXP@Y1Q+mdcfyxvjo-{3J@Mi5X!6$1q1b^SD?AZ7lewfN;Rp1l!q@l_ zcu&}UccchAb5Gbe%h<4$$Tb~!KunWdwnlxT`a7)u)6B+*0GOU3sTM?CF{tn<$Ly7hyGM?vb z!1}viRKkCBwEhm0CA0ocIEQWC`a8RvP{6LgJ1(s2?*t7ICgJPvxUf~tDPe#2T!68S z{aqgcGV{v*ZvOe@mt1{tv!O~@@%!&cOL#^lgEOXw!oa?kMfpOWP5#tGQL|nk6acE# zRE(vFznBHG=v!iOtq;DZ2kD@%ua;^O0<{bhnXP|-nnGo$Zr+4eL@p_W->qSFCOU@J zwS|-qLe1eo!Gd=xclr4e^8t>m(-PCWj(YWV{x2k@qNm?q?+6+xleA1kYl&GhiEIzW zehSkW8u5DnIz+8VQx2=sH5HNZFvH}|#a(=WfCWr15~(xu=V_{nfcY<~p2|G00WgTb zQbvEE<8a*>3Thaf7PDZH!sFa>Ayy%5ZA~GGUQGrm@{L&@|`z><*-XuW2VNdD!1B+uxtsn*woSu7luDH?D%*a@PK(K>1=`RLw4JvhKM zL`=u8EL#I{)X5S{O+r8cvuqrKt@Mk98oYmpWo{Lh!k;hhf~79dL^3yGsBBW4k}jnO z=O$pMmI=PD+k}?_zA|*?jaQ+RIc@+vFl}w18i4Vi#I=t?C{O;OKKx`Bb0vd7UhQ+k zX+)up*1_^k1zJ&Pmz_|!ymcl?BE9K%FrX1de9QR%*JZdF@p>RX+5O+yZYWyOKXSKCtlVU{|Gpovp(B2eyLexU)$ronx z5*Q5;Fr@m2j)aI!M+Xc~1O@*l+?y|&7>mjiks77J3s_ogc4p8LSxe!~d5iq&w+WB< zi5T!O{jsd4zpL-HNn(6xRKuT zul@~Q*1Fl-!7g)zTK{wXo@JjpkRH2f{b-?xSTBkN{YU`jh>3I{ZlHD=^P31%ypG>z zSQgTD9(@;R>~;aH`G|@Oz%Cfx(gMRBf(leIznk@nouG*Z7z*t<0AdN3clH5%e^G)V zyy8xy~^vo>`c&$60(3w1O_mePFuuxH-J^ z$HvIeezJEyIQW(QahaC(*X^uJkZy1zXXh4eO%XNNP48MEn!#_Y6)T~4e_9xmWJ&ML zVC_7Xm!EG36HX1p54YP#71T%yf0hK{+=#^u2pBK1brO5k0cd!w06eJ&9I*$e(SgoP zFxX*3v@wU8k0b(`O|sjCfi{KTxJXR1iSZRPNz+6H4eR|aY04o`IFMAhgGPR{tqiXO z5}o8{zd3S?=l8&(d_bGHbCc7!LEGoYrFuTe$8P#%qe8MfasT|AfoYlPe?GWiZ#Y9P zoWMRaI|WBXPDG5U@@Hoo;U<{mO!6bg3i`we6+_2ix)cy&V6>ZkOFm%`pKaP6>1%uo+YwOW$M5X@J(16u1VZ) zXQ@H0HHfx%F1#rM+`9NrpaLG{5PmOkiCA)u8$1H+3W_!}92kiTTeH{ZTgVRfY`w>H{CL!I%5jVGp^GU{~H-m31xG z@RFskLm4xwe;I8R5kPWxJN~hU=|1s#} zXVA|JUOw*S0L7b5W}N8LhfT?6ElWN{l32jn_^gWAYE?^81N+$d-Q)0a;#I}vWuf3Q zYX$Pww!mf4%!<$A2WnyVd4={@LC8}OxIHvnADec<=yKeYf82k6Sg?5lC$L5~n1>VH zB?A!=bGonp-32;6p#fn3Gmllunz=5cGkzM_B=x`L*f4(bxcktw%0J#qI({L?w@O9E z9g>}N@q;hJ^pu!pL7M+f7`N{4!#qauXEnPXt-DmEf1E@B0XNii{%)H!GpPY0wf=W8 zr)?|zBI^5#e;ChN?MS|P>q4fBsY;Qz`X1_93bgyaP-Wf{YG&*145be zJND3s3S0$p5VcL_V`Ra>_c6oeWLR!DMZDS4|o_Nx@~Du7d|{PPP6KF(^~@6Pzy>1 zP}$ufi~*cy1AT)R!eied#PJ6~!xaNnI7I|FK7=Fx#y;y`=((*j_7RYMT{t=vVVsun z@VhNXMJtdT>R}!N=|FFXs6f~-awV#PfBc9rLOZ_RX%-YAuxF0o9-P25Zq#!Zp|NW1 zhItJ14NeSV32@rS@?fw~iPOHLLu3Zao?^7gP)7SA5i;5*=}A}d7U;CP1jxCLg!dQ; zjz(75Et%3DaBV3usRK0|cS@3MH{ch~HGwzI?8Zt+#JN;`kE7| {} EventMsg::ThreadSettingsApplied(thread_settings_event) => { let thread_settings = thread_settings_from_core_snapshot(thread_settings_event.thread_settings); diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index c394c8ad81..df0daaee38 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -26,7 +26,7 @@ use codex_protocol::error::CodexErr; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_rollout::state_db::StateDbHandle; -use codex_thread_store::ThreadStore; +use codex_thread_store::QueueStore; use crate::outgoing_message::OutgoingMessageSender; use crate::outgoing_message::ThreadScopedOutgoingMessageSender; @@ -44,8 +44,8 @@ pub(crate) struct ThreadExtensionDependencies { pub(crate) executor_skill_provider: Arc, pub(crate) git_attribution_base_url: String, pub(crate) http_client_factory: HttpClientFactory, - /// Process-scoped persistence backend for extensions that need stored thread history. - pub(crate) thread_store: Arc, + /// Process-scoped persistence backend for queued user messages. + pub(crate) queue_store: Option>, } pub(crate) fn thread_extensions( @@ -66,9 +66,17 @@ where executor_skill_provider, git_attribution_base_url, http_client_factory, - thread_store: _thread_store, + queue_store, } = dependencies; - let mut builder = ExtensionRegistryBuilder::::with_event_sink(event_sink); + let mut builder = ExtensionRegistryBuilder::::with_event_sink(Arc::clone(&event_sink)); + if let Some(queue) = queue_store { + let queue_service = Arc::new(codex_queue_extension::QueuedItemService::new( + queue, + thread_manager.clone(), + event_sink, + )); + codex_queue_extension::install(&mut builder, queue_service); + } if let Some(state_db) = state_db { codex_goal_extension::install_with_backend( &mut builder, diff --git a/codex-rs/app-server/src/mcp_refresh.rs b/codex-rs/app-server/src/mcp_refresh.rs index 405f575c16..e1bb838f95 100644 --- a/codex-rs/app-server/src/mcp_refresh.rs +++ b/codex-rs/app-server/src/mcp_refresh.rs @@ -339,7 +339,7 @@ enabled = false executor_skill_provider: Arc::clone(&executor_skill_provider), git_attribution_base_url: good_config.chatgpt_base_url.clone(), http_client_factory: good_config.http_client_factory(), - thread_store: Arc::clone(&thread_store), + queue_store: None, }, ), Arc::new(CodexHomeUserInstructionsProvider::new( diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index c11671bb5d..6637345472 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -66,6 +66,7 @@ use codex_chatgpt::workspace_settings; use codex_code_mode::CodeModeSessionProvider; use codex_core::ThreadManager; use codex_core::config::Config; +use codex_core::config::ThreadStoreConfig; use codex_exec_server::EnvironmentManager; use codex_feedback::CodexFeedback; use codex_goal_extension::GoalService; @@ -77,6 +78,8 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::W3cTraceContext; use codex_rollout::StateDbHandle; use codex_state::log_db::LogDbLayer; +use codex_thread_store::LocalQueueStore; +use codex_thread_store::QueueStore; use tokio::sync::Mutex; use tokio::sync::Semaphore; use tokio::sync::broadcast; @@ -247,6 +250,14 @@ impl MessageProcessor { // affect per-thread behavior, but they must not move newly started, // resumed, or forked threads to a different persistence backend/root. let thread_store = codex_core::thread_store_from_config(config.as_ref(), state_db.clone()); + // Queue persistence requires SQLite, so in-memory thread stores and + // app servers without a state database do not have a queue backend. + let queue_store: Option> = match &config.experimental_thread_store { + ThreadStoreConfig::Local => state_db.as_ref().map(|state_db| { + Arc::new(LocalQueueStore::new(Arc::clone(state_db))) as Arc + }), + ThreadStoreConfig::InMemory { .. } => None, + }; let environment_manager_for_requests = Arc::clone(&environment_manager); let environment_manager_for_extensions = Arc::clone(&environment_manager); let restriction_product = session_source.restriction_product(); @@ -281,7 +292,7 @@ impl MessageProcessor { executor_skill_provider: Arc::clone(&executor_skill_provider), git_attribution_base_url: config.chatgpt_base_url.clone(), http_client_factory: config.http_client_factory(), - thread_store: Arc::clone(&thread_store), + queue_store, }, ), Arc::new(CodexHomeUserInstructionsProvider::new( diff --git a/codex-rs/app-server/src/request_processors/thread_goal_processor.rs b/codex-rs/app-server/src/request_processors/thread_goal_processor.rs index 81cb9bf0be..835ae75563 100644 --- a/codex-rs/app-server/src/request_processors/thread_goal_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_goal_processor.rs @@ -65,18 +65,11 @@ impl ThreadGoalRequestProcessor { .map(|()| None) } - pub(crate) async fn emit_resume_goal_snapshot_and_continue( - &self, - thread_id: ThreadId, - thread: &CodexThread, - ) { + pub(crate) async fn emit_resume_goal_snapshot(&self, thread_id: ThreadId) { if !self.config.features.enabled(Feature::Goals) { return; } self.emit_thread_goal_snapshot(thread_id).await; - // App-server owns resume response and snapshot ordering, so wait until - // those are sent before letting extensions react to the idle thread. - thread.emit_thread_idle_lifecycle_if_idle().await; } pub(crate) async fn pending_resume_goal_state( diff --git a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs index 1c9f1668da..1119f1c56b 100644 --- a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs +++ b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs @@ -1,5 +1,6 @@ use super::*; use crate::extensions::send_thread_warning; +use codex_extension_api::ThreadIdleCause; use codex_protocol::config_types::MultiAgentMode; pub(super) const THREAD_UNLOADING_DELAY: Duration = Duration::from_secs(30 * 60); @@ -750,9 +751,9 @@ pub(super) async fn handle_pending_thread_resume_request( .await; // App-server owns resume response and snapshot ordering, so wait until // replay completes before letting extensions react to the idle thread. - if pending.emit_thread_goal_update { - conversation.emit_thread_idle_lifecycle_if_idle().await; - } + conversation + .emit_thread_idle_lifecycle_if_idle(ThreadIdleCause::Completed) + .await; } pub(super) async fn send_thread_goal_snapshot_notification( diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index 2164737c21..07c9d1755f 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -8,6 +8,7 @@ use codex_app_server_protocol::ThreadSection; use codex_app_server_protocol::ThreadSectionMoveParams; use codex_app_server_protocol::ThreadSectionMoveResponse; use codex_extension_api::ExtensionDataInit; +use codex_extension_api::ThreadIdleCause; use codex_protocol::config_types::MultiAgentMode; use codex_protocol::error::CodexErrorDetails; use codex_protocol::mcp::ClientMcpExtensions; @@ -3438,7 +3439,10 @@ impl ThreadRequestProcessor { .await; } self.thread_goal_processor - .emit_resume_goal_snapshot_and_continue(thread_id, codex_thread.as_ref()) + .emit_resume_goal_snapshot(thread_id) + .await; + codex_thread + .emit_thread_idle_lifecycle_if_idle(ThreadIdleCause::Completed) .await; } Err(err) => { diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index 3e5c3ad948..badcdca87d 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -6,8 +6,11 @@ use crate::session::SessionSettingsUpdate; use crate::session::SteerInputError; use crate::session::TurnInput; use crate::session::session::Session; +use crate::user_message_admission::PendingUserMessageAdmissionState; use crate::user_message_admission::UserMessageAdmission; +use crate::user_message_admission::UserMessageAdmissionError; use codex_exec_server::SelectedCapabilityRootsStatus; +use codex_extension_api::ThreadIdleCause; use codex_features::Feature; use codex_otel::SessionTelemetry; use codex_protocol::ThreadId; @@ -100,6 +103,12 @@ pub enum TryStartTurnIfIdleRejectionReason { /// Another turn or task is active, or the idle reservation was lost before /// the automatic turn could start. Busy, + /// A user-prompt hook consumed and rejected the submitted input. + RejectedByHook, + /// The automatic turn ended before its initial input was persisted. + TaskEndedBeforePersistence, + /// The initial input could not be durably written to the rollout. + PersistenceFailed, } /// Rejection returned when an extension asks to start automatic idle work but @@ -233,6 +242,11 @@ impl CodexThread { self.session.services.session_telemetry.clone() } + /// Returns extension-owned data attached to this thread runtime. + pub fn thread_extension_data(&self) -> &codex_extension_api::ExtensionData { + &self.session.services.thread_extension_data + } + pub async fn shutdown_and_wait(&self) -> CodexResult<()> { self.io.shutdown_and_wait().await } @@ -258,8 +272,8 @@ impl CodexThread { } } - pub async fn emit_thread_idle_lifecycle_if_idle(&self) { - self.session.emit_thread_idle_lifecycle_if_idle().await; + pub async fn emit_thread_idle_lifecycle_if_idle(&self, cause: ThreadIdleCause) { + self.session.emit_thread_idle_lifecycle_if_idle(cause).await; } #[doc(hidden)] @@ -298,28 +312,79 @@ impl CodexThread { .await } - /// Waits until Core has actually started a turn or steered the active turn. + /// Waits until Core has started a turn or steered the active turn. pub async fn submit_user_input_and_wait_for_admission( &self, op: Op, trace: Option, client_user_message_id: Option, ) -> CodexResult { - if !matches!(op, Op::UserInput { .. }) { - return Err(CodexErr::InvalidRequest( - "user message admission requires user input".to_string(), + self.submit_user_input_and_wait_for_admission_inner( + op, + trace, + client_user_message_id, + PendingUserMessageAdmissionState::Immediate, + ) + .await + .map_err(Into::into) + } + + /// Waits for durable admission and preserves its typed failure outcome. + /// + /// A client user-message id is required to identify the persisted message. + pub async fn submit_user_input_and_wait_for_persisted_admission( + &self, + op: Op, + trace: Option, + client_user_message_id: Option, + ) -> Result { + if client_user_message_id.is_none() { + return Err(UserMessageAdmissionError::Admission( + CodexErr::InvalidRequest( + "persisted user message admission requires a client user message id" + .to_string(), + ), + )); + } + self.submit_user_input_and_wait_for_admission_inner( + op, + trace, + client_user_message_id, + PendingUserMessageAdmissionState::WaitingForAdmission, + ) + .await + } + + async fn submit_user_input_and_wait_for_admission_inner( + &self, + op: Op, + trace: Option, + client_user_message_id: Option, + state: PendingUserMessageAdmissionState, + ) -> Result { + let Op::UserInput { items, .. } = &op else { + return Err(UserMessageAdmissionError::Admission( + CodexErr::InvalidRequest("user message admission requires user input".to_string()), + )); + }; + if items.is_empty() { + return Err(UserMessageAdmissionError::Admission( + CodexErr::InvalidRequest( + "user message admission requires nonempty user input".to_string(), + ), )); } self.session .services .agent_control .ensure_execution_capacity_for_op(self.session.thread_id(), &op) - .await?; + .await + .map_err(UserMessageAdmissionError::Admission)?; let submission_id = crate::session::new_submission_id(); let (_pending_admission, admission) = self .session .pending_user_message_admissions - .register(submission_id.clone()); + .register(submission_id.clone(), client_user_message_id.clone(), state); self.io .submit_with_id(Submission { id: submission_id.clone(), @@ -328,11 +393,15 @@ impl CodexThread { trace, parent_turn_id: None, }) - .await?; + .await + .map_err(UserMessageAdmissionError::Admission)?; tokio::select! { biased; - result = admission => result.map_err(|_| CodexErr::InternalAgentDied)?, - () = self.io.session_loop_termination.clone() => Err(CodexErr::InternalAgentDied), + result = admission => result + .unwrap_or(Err(UserMessageAdmissionError::TaskEndedBeforePersistence)), + () = self.io.session_loop_termination.clone() => { + Err(UserMessageAdmissionError::TaskEndedBeforePersistence) + }, } } diff --git a/codex-rs/core/src/guardian/review.rs b/codex-rs/core/src/guardian/review.rs index 41446bda71..c1f274c70a 100644 --- a/codex-rs/core/src/guardian/review.rs +++ b/codex-rs/core/src/guardian/review.rs @@ -6,6 +6,7 @@ use codex_analytics::GuardianReviewTerminalStatus; use codex_analytics::GuardianReviewTrackContext; use codex_analytics::GuardianReviewedAction; use codex_core_plugins::PluginCommandAttribution; +use codex_extension_api::ThreadIdleCause; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::openai_models::MODEL_SPECIALTY_CYBER; use codex_protocol::protocol::AskForApproval; @@ -281,7 +282,9 @@ async fn record_guardian_denial(session: &Arc, turn: &Arc, if aborted { // Guardian aborts bypass normal task completion, so emit its idle lifecycle here. // User interrupts deliberately do not take this path. - session.emit_thread_idle_lifecycle_if_idle().await; + session + .emit_thread_idle_lifecycle_if_idle(ThreadIdleCause::Interrupted) + .await; } }); } diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index 420ef3e991..ee4684a8c6 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -39,6 +39,7 @@ use codex_thread_store::ReadThreadParams; use serde_json::Value; use tracing::instrument; +use crate::codex_thread::TryStartTurnIfIdleRejectionReason; use crate::context::ContextualUserFragment; use crate::context::HookAdditionalContext; use crate::event_mapping::parse_turn_item; @@ -47,6 +48,7 @@ use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::tools::hook_names::HookToolName; use crate::tools::sandboxing::PermissionRequestPayload; +use crate::user_message_admission::UserMessageAdmissionError; pub(crate) struct HookRuntimeOutcome { pub should_stop: bool, @@ -568,14 +570,69 @@ pub(crate) async fn inspect_pending_input( } } +#[expect( + clippy::await_holding_invalid_type, + reason = "durable queue admission must settle before its active turn can be aborted" +)] pub(crate) async fn record_pending_input( sess: &Arc, turn_context: &Arc, pending_input: TurnInput, additional_contexts: Vec, -) { +) -> Result<(), TryStartTurnIfIdleRejectionReason> { match pending_input { TurnInput::UserInput { content, client_id } => { + let awaiting_admission = client_id.as_deref().is_some_and(|client_id| { + sess.pending_user_message_admissions + .contains_client_id(client_id) + }); + let mut active = sess.active_turn.lock().await; + let input_persisted = active + .as_mut() + .and_then(|turn| turn.task.as_mut()) + .filter(|task| Arc::ptr_eq(&task.turn_context, turn_context)) + .and_then(|task| task.input_persisted.take()); + if awaiting_admission || input_persisted.is_some() { + sess.record_user_prompt_and_emit_turn_item( + turn_context.as_ref(), + content.as_slice(), + client_id.clone(), + ) + .await; + record_additional_contexts(sess, turn_context, additional_contexts).await; + match sess.flush_rollout().await { + Ok(()) => { + if awaiting_admission && let Some(client_id) = client_id.as_deref() { + sess.pending_user_message_admissions + .complete_persistence(client_id, Ok(())); + } + if let Some(sender) = input_persisted { + let _ = sender.send(Ok(())); + } + } + Err(error) => { + let error = codex_protocol::error::CodexErr::from(error); + sess.send_event( + turn_context.as_ref(), + EventMsg::Error(error.to_error_event(/*message_prefix*/ None)), + ) + .await; + if awaiting_admission && let Some(client_id) = client_id.as_deref() { + sess.pending_user_message_admissions.complete_persistence( + client_id, + Err(UserMessageAdmissionError::PersistenceFailed(error)), + ); + } + if let Some(sender) = input_persisted { + let _ = sender + .send(Err(TryStartTurnIfIdleRejectionReason::PersistenceFailed)); + } + return Err(TryStartTurnIfIdleRejectionReason::PersistenceFailed); + } + } + return Ok(()); + } + drop(active); sess.record_user_prompt_and_emit_turn_item( turn_context.as_ref(), content.as_slice(), @@ -593,6 +650,32 @@ pub(crate) async fn record_pending_input( } } record_additional_contexts(sess, turn_context, additional_contexts).await; + Ok(()) +} + +pub(crate) async fn reject_pending_input( + sess: &Arc, + turn_context: &Arc, + pending_input: &TurnInput, +) { + let TurnInput::UserInput { client_id, .. } = pending_input else { + return; + }; + if let Some(client_id) = client_id.as_deref() { + sess.pending_user_message_admissions + .complete_persistence(client_id, Err(UserMessageAdmissionError::RejectedByHook)); + } + let input_persisted = { + let mut active = sess.active_turn.lock().await; + active + .as_mut() + .and_then(|turn| turn.task.as_mut()) + .filter(|task| Arc::ptr_eq(&task.turn_context, turn_context)) + .and_then(|task| task.input_persisted.take()) + }; + if let Some(sender) = input_persisted { + let _ = sender.send(Err(TryStartTurnIfIdleRejectionReason::RejectedByHook)); + } } async fn run_context_injecting_hook( diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 4cecf46aed..357785cf3d 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -35,6 +35,7 @@ pub use codex_thread::TryStartTurnIfIdleError; pub use codex_thread::TryStartTurnIfIdleRejectionReason; pub use session::turn_context::TurnContext; pub use user_message_admission::UserMessageAdmission; +pub use user_message_admission::UserMessageAdmissionError; mod agent; mod agent_communication; mod attestation; diff --git a/codex-rs/core/src/session/inject.rs b/codex-rs/core/src/session/inject.rs index 9b73b05370..c1855cf318 100644 --- a/codex-rs/core/src/session/inject.rs +++ b/codex-rs/core/src/session/inject.rs @@ -123,6 +123,9 @@ impl Session { )); } + let (input_persisted_sender, input_persisted_receiver) = + has_user_input.then(tokio::sync::oneshot::channel).unzip(); + let original_input = input.clone(); let task_input = if has_user_input { self.clear_connector_selection().await; for item in &input { @@ -141,9 +144,18 @@ impl Session { turn_context, task_input, RegularTask::new(), + input_persisted_sender, MailboxParentProvenance::Ignore, ) .await; + if let Some(receiver) = input_persisted_receiver { + return receiver + .await + .unwrap_or(Err( + TryStartTurnIfIdleRejectionReason::TaskEndedBeforePersistence, + )) + .map_err(|reason| TryStartTurnIfIdleError::new(reason, original_input)); + } Ok(()) } diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index d70dbe6306..a133ec106f 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -4,13 +4,15 @@ use crate::state::TurnState; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::user_input::UserInput; +use serde::Deserialize; +use serde::Serialize; use std::collections::VecDeque; use std::sync::Arc; use tokio::sync::Mutex; use tokio::sync::watch; /// Input consumed by a regular turn. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum TurnInput { UserInput { content: Vec, diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 0117b596ff..01c742be89 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1319,6 +1319,15 @@ impl Session { InitialHistory::Resumed(resumed_history) => { let turn_context = self.new_default_turn().await; let rollout_items = resumed_history.history; + if matches!( + rollout_items.iter().rev().find_map(|item| match item { + RolloutItem::EventMsg(event) => agent_status_from_event(event), + _ => None, + }), + Some(AgentStatus::Interrupted) + ) { + self.agent_status.send_replace(AgentStatus::Interrupted); + } let previous_turn_settings = self .apply_rollout_reconstruction(&turn_context, &rollout_items) .await; @@ -4025,7 +4034,7 @@ impl Session { .collect::>(); pending_input.push(TurnInput::UserInput { content: input, - client_id: client_user_message_id, + client_id: client_user_message_id.clone(), }); self.input_queue .extend_pending_input_and_accept_mailbox_delivery_for_turn_state( @@ -4033,6 +4042,10 @@ impl Session { pending_input, ) .await; + if let Some(client_id) = client_user_message_id.as_deref() { + self.pending_user_message_admissions + .associate_steered_by_client_id(client_id, active_turn_id); + } Ok(active_turn_id.clone()) } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index b2d7a14105..be87b859c7 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -92,6 +92,8 @@ use crate::tools::handlers::ShellCommandHandler; use crate::tools::registry::ToolExecutor; use crate::tools::router::ToolCallSource; use crate::turn_diff_tracker::TurnDiffTracker; +use crate::user_message_admission::PendingUserMessageAdmissionState; +use crate::user_message_admission::UserMessageAdmissionError; use codex_config::config_toml::ConfigToml; use codex_config::config_toml::ProjectConfig; use codex_config::permissions_toml::FilesystemPermissionToml; @@ -9865,6 +9867,83 @@ async fn attach_in_memory_thread_store( store } +#[tokio::test] +async fn failed_user_message_persistence_stops_turn_processing() { + let (mut session, turn_context, _events) = make_session_and_context_with_rx().await; + open_thread_persistence( + Arc::get_mut(&mut session).unwrap_or_else(|| panic!("session should be uniquely owned")), + ) + .await; + session + .services + .live_thread + .as_ref() + .unwrap_or_else(|| panic!("session should have a live thread")) + .shutdown() + .await + .unwrap_or_else(|error| panic!("live thread should shut down: {error}")); + + let client_id = "persistence-failure".to_string(); + let (_pending_admission, admission) = session.pending_user_message_admissions.register( + turn_context.sub_id.clone(), + Some(client_id.clone()), + PendingUserMessageAdmissionState::WaitingForAdmission, + ); + let failed_queued_prompt = vec![UserInput::Text { + text: "message with a failed rollout flush".to_string(), + text_elements: Vec::new(), + }]; + let later_steer = vec![UserInput::Text { + text: "later ordinary steer".to_string(), + text_elements: Vec::new(), + }]; + let later_response = user_message("later response item"); + let later_mailbox = InterAgentCommunication::new( + AgentPath::root().join("worker").expect("worker path"), + AgentPath::root(), + Vec::new(), + "later mailbox message".to_string(), + /*trigger_turn*/ false, + ); + let input = [ + TurnInput::UserInput { + content: failed_queued_prompt, + client_id: Some(client_id), + }, + TurnInput::UserInput { + content: later_steer.clone(), + client_id: None, + }, + TurnInput::ResponseItem(later_response.clone()), + TurnInput::InterAgentCommunication(later_mailbox.clone()), + ]; + + assert!( + super::turn::run_hooks_and_record_inputs(&session, &turn_context, &input).await, + "a rollout flush failure should stop turn processing" + ); + assert!(matches!( + admission.await, + Ok(Err(UserMessageAdmissionError::PersistenceFailed(_))) + )); + assert!(turn_context.terminal_error.lock().await.is_some()); + + let mut expected_suffix = vec![ + session.response_item_from_user_input(later_steer), + later_response, + later_mailbox.to_model_input_item(), + ]; + for item in &mut expected_suffix { + item.set_turn_id_if_missing(&turn_context.sub_id); + } + let history = session.clone_history().await; + let history_items = strip_response_item_ids(history.raw_items()); + assert!( + history_items.ends_with(&strip_response_item_ids(&expected_suffix)), + "later steer, response, and mailbox inputs should survive the failed queued message" + ); +} + #[tokio::test] async fn hook_transcript_path_does_not_persist_non_local_thread_store() { let (mut session, _) = make_session_and_context().await; @@ -10461,7 +10540,9 @@ async fn thread_idle_lifecycle_waits_for_trigger_turn_mailbox_work() { ) .await; - session.emit_thread_idle_lifecycle_if_idle().await; + session + .emit_thread_idle_lifecycle_if_idle(codex_extension_api::ThreadIdleCause::Completed) + .await; assert_eq!(0, calls.load(std::sync::atomic::Ordering::SeqCst)); } @@ -10960,7 +11041,7 @@ async fn steered_input_reopens_mailbox_delivery_for_current_turn() { text: "follow up".to_string(), text_elements: Vec::new(), }], - client_id: None + client_id: None, }, TurnInput::InterAgentCommunication(communication), ], @@ -11018,7 +11099,7 @@ async fn stale_defer_mailbox_delivery_does_not_override_steered_input() { text: "follow up".to_string(), text_elements: Vec::new(), }], - client_id: None + client_id: None, }, TurnInput::InterAgentCommunication(communication), ], diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 16ef4e7d36..eb18e3b24f 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -20,6 +20,7 @@ use crate::feedback_tags; use crate::hook_runtime::inspect_pending_input; use crate::hook_runtime::record_additional_contexts; use crate::hook_runtime::record_pending_input; +use crate::hook_runtime::reject_pending_input; use crate::hook_runtime::run_legacy_after_agent_hook; use crate::hook_runtime::run_pending_session_start_hooks; use crate::hook_runtime::run_turn_stop_hooks; @@ -583,25 +584,33 @@ pub(crate) async fn run_hooks_and_record_inputs( ) -> bool { let mut blocked_input = false; let mut accepted_user_input = false; + let mut persistence_failed = false; for input_item in input { let hook_outcome = inspect_pending_input(sess, turn_context, input_item).await; if hook_outcome.should_stop { blocked_input = true; + reject_pending_input(sess, turn_context, input_item).await; record_additional_contexts(sess, turn_context, hook_outcome.additional_contexts).await; } else { if matches!(input_item, TurnInput::UserInput { content, .. } if !content.is_empty()) { accepted_user_input = true; } - record_pending_input( + if record_pending_input( sess, turn_context, input_item.clone(), hook_outcome.additional_contexts, ) - .await; + .await + .is_err() + { + // Preserve later drained inputs, but stop before sampling so a + // later queue retry cannot execute this message twice. + persistence_failed = true; + } } } - blocked_input && !accepted_user_input + persistence_failed || (blocked_input && !accepted_user_input) } fn turn_user_input(input: &[TurnInput]) -> Vec { @@ -1750,6 +1759,7 @@ pub(super) fn realtime_text_for_event(msg: &EventMsg) -> Option<(String, Option< | EventMsg::EnvironmentConnected(_) | EventMsg::EnvironmentDisconnected(_) | EventMsg::ThreadGoalUpdated(_) + | EventMsg::ThreadQueueChanged(_) | EventMsg::McpStartupUpdate(_) | EventMsg::McpStartupComplete(_) | EventMsg::McpToolCallBegin(_) diff --git a/codex-rs/core/src/state/turn.rs b/codex-rs/core/src/state/turn.rs index 79c103745f..b284199d96 100644 --- a/codex-rs/core/src/state/turn.rs +++ b/codex-rs/core/src/state/turn.rs @@ -18,6 +18,7 @@ use rmcp::model::RequestId; use tokio::sync::oneshot; use crate::agent::control::AgentExecutionGuard; +use crate::codex_thread::TryStartTurnIfIdleRejectionReason; use crate::mcp_tool_call::McpToolApprovalMetadata; use crate::session::TurnInputQueue; use crate::session::turn_context::TurnContext; @@ -74,6 +75,8 @@ pub(crate) struct RunningTask { pub(crate) done: Arc, pub(crate) kind: TaskKind, pub(crate) task: Arc, + pub(crate) input_persisted: + Option>>, pub(crate) cancellation_token: CancellationToken, pub(crate) handle: AbortOnDropHandle<()>, pub(crate) turn_context: Arc, diff --git a/codex-rs/core/src/tasks/lifecycle.rs b/codex-rs/core/src/tasks/lifecycle.rs index 789955df13..0af3127d95 100644 --- a/codex-rs/core/src/tasks/lifecycle.rs +++ b/codex-rs/core/src/tasks/lifecycle.rs @@ -1,4 +1,5 @@ use codex_extension_api::ExtensionData; +use codex_extension_api::ThreadIdleCause; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TurnAbortReason; @@ -39,7 +40,7 @@ impl Session { } } - pub(crate) async fn emit_thread_idle_lifecycle_if_idle(&self) { + pub(crate) async fn emit_thread_idle_lifecycle_if_idle(&self, cause: ThreadIdleCause) { if self.active_turn.lock().await.is_some() || self.input_queue.has_trigger_turn_mailbox_items().await { @@ -49,6 +50,7 @@ impl Session { for contributor in self.services.extensions.thread_lifecycle_contributors() { contributor .on_thread_idle(codex_extension_api::ThreadIdleInput { + cause, session_store: &self.services.session_extension_data, thread_store: &self.services.thread_extension_data, }) diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index bfd18ef09d..75506c4e85 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use codex_extension_api::ThreadIdleCause; use futures::future::BoxFuture; use tokio::select; use tokio::sync::Notify; @@ -22,13 +23,12 @@ use tracing::trace_span; use tracing::warn; use crate::codex_thread::BackgroundTerminalInfo; +use crate::codex_thread::TryStartTurnIfIdleRejectionReason; use crate::config::Config; use crate::context::ContextualUserFragment; -use crate::hook_runtime::inspect_pending_input; -use crate::hook_runtime::record_additional_contexts; -use crate::hook_runtime::record_pending_input; use crate::session::TurnInput; use crate::session::session::Session; +use crate::session::turn::run_hooks_and_record_inputs; use crate::session::turn_context::TurnContext; use crate::state::ActiveTurn; use crate::state::RunningTask; @@ -281,8 +281,14 @@ impl Session { ) { self.abort_all_tasks(TurnAbortReason::Replaced).await; self.clear_connector_selection().await; - self.start_task(turn_context, input, task, MailboxParentProvenance::Ignore) - .await; + self.start_task( + turn_context, + input, + task, + /*input_persisted*/ None, + MailboxParentProvenance::Ignore, + ) + .await; } pub(crate) async fn start_task( @@ -290,6 +296,9 @@ impl Session { turn_context: Arc, input: Vec, task: T, + input_persisted: Option< + tokio::sync::oneshot::Sender>, + >, mailbox_parent_provenance: MailboxParentProvenance, ) { let task: Arc = Arc::new(task); @@ -408,6 +417,7 @@ impl Session { handle: AbortOnDropHandle::new(handle), kind: task_kind, task, + input_persisted, cancellation_token, turn_context: Arc::clone(&turn_context), _agent_execution_guard: agent_execution_guard, @@ -471,6 +481,7 @@ impl Session { turn_context, Vec::new(), RegularTask::new(), + /*input_persisted*/ None, MailboxParentProvenance::Attribute, ) .await; @@ -580,12 +591,24 @@ impl Session { let turn_state = { let mut active = self.active_turn.lock().await; active.as_mut().and_then(|active_turn| { - let task = active_turn.task.take()?; + let mut task = active_turn.task.take()?; + let task_ended_before_persistence = + if let Some(sender) = task.input_persisted.take() { + let _ = sender.send(Err( + TryStartTurnIfIdleRejectionReason::TaskEndedBeforePersistence, + )); + true + } else { + false + }; task.handle.detach(); - Some(Arc::clone(&active_turn.turn_state)) + Some(( + Arc::clone(&active_turn.turn_state), + task_ended_before_persistence, + )) }) }; - let Some(turn_state) = turn_state else { + let Some((turn_state, mut task_ended_before_persistence)) = turn_state else { return; }; let pending_input = self @@ -600,28 +623,10 @@ impl Session { ts.token_usage_at_turn_start.clone(), ) }; - if !pending_input.is_empty() { - for pending_input_item in pending_input { - let hook_outcome = - inspect_pending_input(self, &turn_context, &pending_input_item).await; - if hook_outcome.should_stop { - record_additional_contexts( - self, - &turn_context, - hook_outcome.additional_contexts, - ) - .await; - } else { - record_pending_input( - self, - &turn_context, - pending_input_item, - hook_outcome.additional_contexts, - ) - .await; - } - } - } + run_hooks_and_record_inputs(self, &turn_context, &pending_input).await; + task_ended_before_persistence |= self + .pending_user_message_admissions + .complete_task_end(&turn_context.sub_id); // Emit token usage metrics. { // TODO(jif): drop this @@ -764,6 +769,15 @@ impl Session { turn_id: turn_context.sub_id.clone(), profile, }); + let idle_cause = if matches!(abort_reason.as_ref(), Some(TurnAbortReason::Interrupted)) { + ThreadIdleCause::Interrupted + } else if task_ended_before_persistence + || (abort_reason.is_none() && turn_context.terminal_error.lock().await.is_some()) + { + ThreadIdleCause::Failed + } else { + ThreadIdleCause::Completed + }; let event = if let Some(reason) = abort_reason { self.emit_turn_abort_lifecycle(reason.clone(), turn_context.extension_data.as_ref()) .await; @@ -812,7 +826,7 @@ impl Session { } }; if cleared_active_turn { - self.emit_thread_idle_lifecycle_if_idle().await; + self.emit_thread_idle_lifecycle_if_idle(idle_cause).await; } // Regular items were flushed before this terminal event was appended; buffering // thread writers may not flush it without another explicit barrier. @@ -847,12 +861,19 @@ impl Session { .await } - async fn handle_task_abort(self: &Arc, task: RunningTask, reason: TurnAbortReason) { + async fn handle_task_abort(self: &Arc, mut task: RunningTask, reason: TurnAbortReason) { let sub_id = task.turn_context.sub_id.clone(); if task.cancellation_token.is_cancelled() { return; } + if let Some(sender) = task.input_persisted.take() { + let _ = sender.send(Err( + TryStartTurnIfIdleRejectionReason::TaskEndedBeforePersistence, + )); + } + self.pending_user_message_admissions + .complete_task_end(&sub_id); trace!(task_kind = ?task.kind, sub_id, "aborting running task"); task.cancellation_token.cancel(); task.turn_context diff --git a/codex-rs/core/src/user_message_admission.rs b/codex-rs/core/src/user_message_admission.rs index 1fded6b5ce..59196b8a6e 100644 --- a/codex-rs/core/src/user_message_admission.rs +++ b/codex-rs/core/src/user_message_admission.rs @@ -1,3 +1,4 @@ +use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; use std::collections::HashMap; use std::sync::Mutex; @@ -12,24 +13,72 @@ pub enum UserMessageAdmission { Steered { turn_id: String }, } +/// Explains why a submitted user message did not become durable. +#[derive(Debug)] +pub enum UserMessageAdmissionError { + /// Core rejected the submission before admitting it to a turn. + Admission(CodexErr), + /// A user-prompt hook rejected the submitted message. + RejectedByHook, + /// The owning task finished before the submitted message was persisted. + TaskEndedBeforePersistence, + /// The admitted message could not be durably written to the rollout. + PersistenceFailed(CodexErr), +} + +impl From for CodexErr { + fn from(error: UserMessageAdmissionError) -> Self { + match error { + UserMessageAdmissionError::Admission(error) => error, + UserMessageAdmissionError::RejectedByHook => { + Self::InvalidRequest("user message was rejected by a hook".to_string()) + } + UserMessageAdmissionError::TaskEndedBeforePersistence => Self::InternalAgentDied, + UserMessageAdmissionError::PersistenceFailed(error) => error, + } + } +} + #[derive(Default)] pub(crate) struct PendingUserMessageAdmissions { - pending: Mutex>>>, + pending: Mutex>, +} + +struct PendingUserMessageAdmission { + client_id: Option, + sender: oneshot::Sender>, + state: PendingUserMessageAdmissionState, +} + +pub(crate) enum PendingUserMessageAdmissionState { + Immediate, + WaitingForAdmission, + Admitted(UserMessageAdmission), + Persisted, } impl PendingUserMessageAdmissions { pub(crate) fn register( &self, submission_id: String, + client_id: Option, + state: PendingUserMessageAdmissionState, ) -> ( PendingUserMessageAdmissionGuard<'_>, - oneshot::Receiver>, + oneshot::Receiver>, ) { let (sender, receiver) = oneshot::channel(); self.pending .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert(submission_id.clone(), sender); + .insert( + submission_id.clone(), + PendingUserMessageAdmission { + client_id, + sender, + state, + }, + ); ( PendingUserMessageAdmissionGuard { admissions: self, @@ -44,16 +93,152 @@ impl PendingUserMessageAdmissions { submission_id: &str, admission: CodexResult, ) { - let admission_sender = self + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(entry) = pending.get_mut(submission_id) else { + return; + }; + + match admission { + Ok(admission) => match &entry.state { + PendingUserMessageAdmissionState::WaitingForAdmission => { + entry.state = PendingUserMessageAdmissionState::Admitted(admission); + } + PendingUserMessageAdmissionState::Immediate + | PendingUserMessageAdmissionState::Persisted => { + if let Some(entry) = pending.remove(submission_id) { + let _ = entry.sender.send(Ok(admission)); + } + } + PendingUserMessageAdmissionState::Admitted(_) => {} + }, + Err(error) => { + if let Some(entry) = pending.remove(submission_id) { + let _ = entry + .sender + .send(Err(UserMessageAdmissionError::Admission(error))); + } + } + } + } + + pub(crate) fn contains_client_id(&self, client_id: &str) -> bool { + self.pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .any(|entry| { + !matches!(&entry.state, PendingUserMessageAdmissionState::Immediate) + && entry.client_id.as_deref() == Some(client_id) + }) + } + + pub(crate) fn associate_steered_by_client_id(&self, client_id: &str, turn_id: &str) { + let submission_id = self .pending .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(submission_id); - if let Some(sender) = admission_sender { - let _ = sender.send(admission); + .iter() + .find_map(|(submission_id, entry)| { + (matches!( + &entry.state, + PendingUserMessageAdmissionState::WaitingForAdmission + | PendingUserMessageAdmissionState::Persisted + ) && entry.client_id.as_deref() == Some(client_id)) + .then(|| submission_id.clone()) + }); + if let Some(submission_id) = submission_id { + self.complete( + &submission_id, + Ok(UserMessageAdmission::Steered { + turn_id: turn_id.to_string(), + }), + ); } } + pub(crate) fn complete_persistence( + &self, + client_id: &str, + result: Result<(), UserMessageAdmissionError>, + ) { + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(submission_id) = pending.iter().find_map(|(submission_id, entry)| { + (!matches!(&entry.state, PendingUserMessageAdmissionState::Immediate) + && entry.client_id.as_deref() == Some(client_id)) + .then(|| submission_id.clone()) + }) else { + return; + }; + + if let Err(error) = result { + if let Some(entry) = pending.remove(&submission_id) { + let _ = entry.sender.send(Err(error)); + } + return; + } + + let Some(entry) = pending.get_mut(&submission_id) else { + return; + }; + match std::mem::replace( + &mut entry.state, + PendingUserMessageAdmissionState::Persisted, + ) { + PendingUserMessageAdmissionState::Admitted(admission) => { + if let Some(entry) = pending.remove(&submission_id) { + let _ = entry.sender.send(Ok(admission)); + } + } + PendingUserMessageAdmissionState::Immediate + | PendingUserMessageAdmissionState::WaitingForAdmission + | PendingUserMessageAdmissionState::Persisted => {} + } + } + + pub(crate) fn complete_task_end(&self, turn_id: &str) -> bool { + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let submission_ids = pending + .iter() + .filter_map(|(submission_id, entry)| { + let belongs_to_turn = match &entry.state { + PendingUserMessageAdmissionState::WaitingForAdmission => { + submission_id == turn_id + } + PendingUserMessageAdmissionState::Admitted( + UserMessageAdmission::Started { + turn_id: admission_turn_id, + } + | UserMessageAdmission::Steered { + turn_id: admission_turn_id, + }, + ) => admission_turn_id == turn_id, + PendingUserMessageAdmissionState::Immediate + | PendingUserMessageAdmissionState::Persisted => false, + }; + belongs_to_turn.then(|| submission_id.clone()) + }) + .collect::>(); + + let had_pending_admissions = !submission_ids.is_empty(); + for submission_id in submission_ids { + if let Some(entry) = pending.remove(&submission_id) { + let _ = entry + .sender + .send(Err(UserMessageAdmissionError::TaskEndedBeforePersistence)); + } + } + had_pending_admissions + } + fn remove(&self, submission_id: &str) { self.pending .lock() diff --git a/codex-rs/core/tests/suite/user_message_admission.rs b/codex-rs/core/tests/suite/user_message_admission.rs index c88c14a91c..60f84a0d11 100644 --- a/codex-rs/core/tests/suite/user_message_admission.rs +++ b/codex-rs/core/tests/suite/user_message_admission.rs @@ -73,7 +73,7 @@ async fn user_message_admission_reports_started_and_steered_for_concurrent_submi async move { barrier.wait().await; codex - .submit_user_input_and_wait_for_admission( + .submit_user_input_and_wait_for_persisted_admission( user_input("first message"), /*trace*/ None, Some("client-message-1".to_string()), @@ -87,7 +87,7 @@ async fn user_message_admission_reports_started_and_steered_for_concurrent_submi async move { barrier.wait().await; codex - .submit_user_input_and_wait_for_admission( + .submit_user_input_and_wait_for_persisted_admission( user_input("second message"), /*trace*/ None, Some("client-message-2".to_string()), @@ -97,11 +97,21 @@ async fn user_message_admission_reports_started_and_steered_for_concurrent_submi }); barrier.wait().await; + timeout( + Duration::from_secs(5), + server.wait_for_request_count(/*count*/ 1), + ) + .await + .expect("the started turn should reach its first model request"); + release_response + .send(()) + .expect("response gate should remain open"); + let (first_admission, second_admission) = timeout(Duration::from_secs(5), async { tokio::join!(first_submission, second_submission) }) .await - .expect("both concurrent admissions should resolve before sampling finishes"); + .expect("both concurrent admissions should resolve once their messages are persisted"); let first_admission = first_admission .expect("first submission task should finish") .expect("first user message should be admitted"); @@ -125,9 +135,33 @@ async fn user_message_admission_reports_started_and_steered_for_concurrent_submi }; assert_eq!(started_turn_id, steered_turn_id); - release_response - .send(()) - .expect("response gate should remain open"); + let rollout_path = codex.rollout_path().expect("user-message rollout path"); + let rollout = tokio::fs::read_to_string(rollout_path) + .await + .expect("read persisted user-message rollout"); + let mut persisted_client_message_ids: Vec<_> = rollout + .lines() + .map(serde_json::from_str::) + .collect::, _>>() + .expect("parse persisted user-message rollout") + .into_iter() + .filter_map(|line| match line.item { + RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => match event.item { + TurnItem::UserMessage(message) => message.client_id, + _ => None, + }, + _ => None, + }) + .collect(); + persisted_client_message_ids.sort(); + assert_eq!( + persisted_client_message_ids, + vec![ + "client-message-1".to_string(), + "client-message-2".to_string() + ] + ); + let mut observed_client_message_ids = Vec::new(); loop { let event = timeout(Duration::from_secs(10), codex.next_event()) @@ -165,36 +199,6 @@ async fn user_message_admission_reports_started_and_steered_for_concurrent_submi assert!(request_bodies[1].to_string().contains("first message")); assert!(request_bodies[1].to_string().contains("second message")); - codex - .flush_rollout() - .await - .expect("flush persisted user-message history"); - let rollout_path = codex.rollout_path().expect("user-message rollout path"); - let rollout = tokio::fs::read_to_string(rollout_path) - .await - .expect("read persisted user-message rollout"); - let mut persisted_client_message_ids: Vec<_> = rollout - .lines() - .map(serde_json::from_str::) - .collect::, _>>() - .expect("parse persisted user-message rollout") - .into_iter() - .filter_map(|line| match line.item { - RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => match event.item { - TurnItem::UserMessage(message) => message.client_id, - _ => None, - }, - _ => None, - }) - .collect(); - persisted_client_message_ids.sort(); - assert_eq!( - persisted_client_message_ids, - vec![ - "client-message-1".to_string(), - "client-message-2".to_string() - ] - ); server.shutdown().await; } diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 14e0aa32dc..048d614a50 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -29,6 +29,7 @@ pub use prompt::PromptFragment; pub use prompt::PromptSlot; pub use skill_invocation::SkillInvocationInput; pub use skill_invocation::SkillInvocationKind; +pub use thread_lifecycle::ThreadIdleCause; pub use thread_lifecycle::ThreadIdleInput; pub use thread_lifecycle::ThreadOriginator; pub use thread_lifecycle::ThreadResumeInput; diff --git a/codex-rs/ext/extension-api/src/contributors/thread_lifecycle.rs b/codex-rs/ext/extension-api/src/contributors/thread_lifecycle.rs index bef595ee19..d2b83bfe79 100644 --- a/codex-rs/ext/extension-api/src/contributors/thread_lifecycle.rs +++ b/codex-rs/ext/extension-api/src/contributors/thread_lifecycle.rs @@ -42,8 +42,21 @@ pub struct ThreadResumeInput<'a> { pub thread_store: &'a ExtensionData, } +/// Why a thread has no immediately pending work. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ThreadIdleCause { + /// The previous turn completed and automatic follow-up work can run. + Completed, + /// The user interrupted the previous turn. + Interrupted, + /// The previous turn ended with a terminal error. + Failed, +} + /// Input supplied when the host has no immediately pending thread work. pub struct ThreadIdleInput<'a> { + /// Why the thread became idle. + pub cause: ThreadIdleCause, /// Store scoped to the host session runtime. pub session_store: &'a ExtensionData, /// Store scoped to this thread runtime. diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index 8cae0e752b..2f21380b0f 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -47,6 +47,7 @@ pub use contributors::RenderedWorldStateFragment; pub use contributors::SkillInvocationContributor; pub use contributors::SkillInvocationInput; pub use contributors::SkillInvocationKind; +pub use contributors::ThreadIdleCause; pub use contributors::ThreadIdleInput; pub use contributors::ThreadLifecycleContributor; pub use contributors::ThreadOriginator; diff --git a/codex-rs/ext/queue/BUILD.bazel b/codex-rs/ext/queue/BUILD.bazel new file mode 100644 index 0000000000..21f5afd2c9 --- /dev/null +++ b/codex-rs/ext/queue/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "queue", + crate_name = "codex_queue_extension", +) diff --git a/codex-rs/ext/queue/Cargo.toml b/codex-rs/ext/queue/Cargo.toml new file mode 100644 index 0000000000..4b5e5d04f4 --- /dev/null +++ b/codex-rs/ext/queue/Cargo.toml @@ -0,0 +1,34 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-queue-extension" +version.workspace = true + +[lib] +name = "codex_queue_extension" +path = "src/lib.rs" +test = false +doctest = false + +[lints] +workspace = true + +[dependencies] +codex-core = { workspace = true } +codex-extension-api = { workspace = true } +codex-protocol = { workspace = true } +codex-thread-store = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt", "sync", "time"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["v7"] } + +[dev-dependencies] +anyhow = { workspace = true } +codex-state = { workspace = true } +codex-utils-absolute-path = { workspace = true } +core_test_support = { workspace = true } +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/ext/queue/src/lib.rs b/codex-rs/ext/queue/src/lib.rs new file mode 100644 index 0000000000..1f7be82988 --- /dev/null +++ b/codex-rs/ext/queue/src/lib.rs @@ -0,0 +1,19 @@ +//! Durable, storage-neutral user-message queue and idle dispatch. + +use std::sync::Arc; + +use codex_extension_api::ExtensionRegistryBuilder; + +mod service; + +pub use service::QueueServiceError; +pub use service::QueuedItem; +pub use service::QueuedItemService; + +/// Registers the caller-owned queue before lower-priority idle contributors. +pub fn install(registry: &mut ExtensionRegistryBuilder, service: Arc) +where + C: Send + Sync + 'static, +{ + registry.thread_lifecycle_contributor(service); +} diff --git a/codex-rs/ext/queue/src/service.rs b/codex-rs/ext/queue/src/service.rs new file mode 100644 index 0000000000..67c6b71958 --- /dev/null +++ b/codex-rs/ext/queue/src/service.rs @@ -0,0 +1,394 @@ +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex as StdMutex; +use std::sync::Weak; + +use codex_core::CodexThread; +use codex_core::ThreadManager; +use codex_core::TryStartTurnIfIdleRejectionReason; +use codex_core::TurnInput; +use codex_core::UserMessageAdmission; +use codex_core::UserMessageAdmissionError; +use codex_extension_api::ExtensionEventSink; +use codex_extension_api::ExtensionFuture; +use codex_extension_api::ThreadIdleCause; +use codex_extension_api::ThreadIdleInput; +use codex_extension_api::ThreadLifecycleContributor; +use codex_protocol::ThreadId; +use codex_protocol::error::CodexErr; +use codex_protocol::models::snapshot_local_user_input; +use codex_protocol::protocol::AgentStatus; +use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::ThreadQueueChangedEvent; +use codex_protocol::protocol::ThreadSettingsOverrides; +use codex_protocol::protocol::W3cTraceContext; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; +use codex_protocol::user_input::UserInput; +use codex_thread_store::MAX_QUEUE_ITEMS; +use codex_thread_store::QueueStore; +use codex_thread_store::QueuedUserSubmissionRecord; +use codex_thread_store::ThreadStoreError; +use thiserror::Error; +use tokio::sync::Mutex; +use tokio::sync::OwnedMutexGuard; +use uuid::Uuid; + +/// One user message waiting to start on its thread. +#[derive(Clone, Debug, PartialEq)] +pub struct QueuedItem { + pub id: String, + pub input: TurnInput, +} + +#[derive(Debug, Error)] +pub enum QueueServiceError { + #[error("queue storage failed: {0}")] + Storage(#[from] ThreadStoreError), + #[error("queued item payload is invalid: {0}")] + InvalidPayload(#[from] serde_json::Error), + #[error("local queued attachment is invalid: {0}")] + InvalidAttachment(#[from] std::io::Error), + #[error("queued user message could not be admitted: {0}")] + Admission(#[from] CodexErr), + #[error("only user input can be added to the user-message queue")] + InvalidInput, + #[error( + "queued user input exceeds the maximum length of {MAX_USER_INPUT_TEXT_CHARS} characters ({actual_chars} provided)" + )] + InputTooLarge { actual_chars: usize }, +} + +#[derive(Clone)] +pub struct QueuedItemService { + queue: Arc, + thread_manager: Weak, + event_sink: Arc, + dispatch_locks: Arc>>>>, +} + +impl QueuedItemService { + pub fn new( + queue: Arc, + thread_manager: Weak, + event_sink: Arc, + ) -> Self { + Self { + queue, + thread_manager, + event_sink, + dispatch_locks: Arc::new(StdMutex::new(HashMap::new())), + } + } + + fn dispatch_lock(&self, thread_id: ThreadId) -> Arc> { + let mut locks = self + .dispatch_locks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + locks.retain(|_, lock| lock.strong_count() != 0); + if let Some(lock) = locks.get(&thread_id).and_then(Weak::upgrade) { + return lock; + } + let lock = Arc::new(Mutex::new(())); + locks.insert(thread_id, Arc::downgrade(&lock)); + lock + } + + async fn dispatch_guard(&self, thread_id: ThreadId) -> OwnedMutexGuard<()> { + self.dispatch_lock(thread_id).lock_owned().await + } + + pub async fn enqueue( + &self, + thread_id: ThreadId, + input: TurnInput, + ) -> Result { + let input = prepare_queued_user_input(input).await?; + let payload = serde_json::to_string(&input)?; + let item = queued_item_from_record(self.queue.enqueue(thread_id, payload).await?)?; + self.emit_changed(thread_id); + self.wake_if_loaded(thread_id).await; + Ok(item) + } + + pub async fn list(&self, thread_id: ThreadId) -> Result, QueueServiceError> { + self.queue + .list_page(thread_id, /*offset*/ 0, MAX_QUEUE_ITEMS) + .await? + .into_iter() + .map(queued_item_from_record) + .collect() + } + + pub async fn update( + &self, + thread_id: ThreadId, + queued_item_id: String, + mut input: TurnInput, + ) -> Result, QueueServiceError> { + let _dispatch_guard = self.dispatch_guard(thread_id).await; + if let TurnInput::UserInput { client_id, .. } = &mut input + && client_id.is_none() + { + *client_id = self + .list(thread_id) + .await? + .into_iter() + .find_map(|item| match item { + QueuedItem { + id, + input: TurnInput::UserInput { client_id, .. }, + } if id == queued_item_id => client_id, + _ => None, + }); + } + let input = prepare_queued_user_input(input).await?; + let payload = serde_json::to_string(&input)?; + let item = self + .queue + .update(thread_id, queued_item_id, payload) + .await? + .map(queued_item_from_record) + .transpose()?; + if item.is_some() { + self.emit_changed(thread_id); + } + Ok(item) + } + + pub async fn delete( + &self, + thread_id: ThreadId, + queued_item_id: String, + ) -> Result { + let _dispatch_guard = self.dispatch_guard(thread_id).await; + self.delete_locked(thread_id, queued_item_id).await + } + + async fn delete_locked( + &self, + thread_id: ThreadId, + queued_item_id: String, + ) -> Result { + let deleted = self.queue.delete(thread_id, queued_item_id).await?; + if deleted { + self.emit_changed(thread_id); + } + Ok(deleted) + } + + pub async fn reorder( + &self, + thread_id: ThreadId, + ordered_ids: Vec, + ) -> Result<(), QueueServiceError> { + let _dispatch_guard = self.dispatch_guard(thread_id).await; + self.queue.reorder(thread_id, ordered_ids).await?; + self.emit_changed(thread_id); + Ok(()) + } + + /// Starts or steers the exact queued message and removes it after admission. + pub async fn start( + &self, + thread: &CodexThread, + queued_item_id: String, + trace: Option, + ) -> Result { + let thread_id = thread.session_configured().thread_id; + let _dispatch_guard = self.dispatch_guard(thread_id).await; + let item = self + .list(thread_id) + .await? + .into_iter() + .find(|item| item.id == queued_item_id) + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("queued item not found: {queued_item_id}"), + })?; + let TurnInput::UserInput { content, client_id } = item.input else { + return Err(QueueServiceError::InvalidInput); + }; + let admission = thread + .submit_user_input_and_wait_for_persisted_admission( + Op::UserInput { + items: content, + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::new(), + thread_settings: ThreadSettingsOverrides::default(), + }, + trace, + client_id, + ) + .await; + let admission = match admission { + Ok(admission) => admission, + Err(error) => { + if matches!(&error, UserMessageAdmissionError::RejectedByHook) { + self.delete_locked(thread_id, queued_item_id).await?; + } + return Err(QueueServiceError::Admission(error.into())); + } + }; + self.delete_locked(thread_id, queued_item_id).await?; + Ok(admission) + } + + async fn dispatch_if_idle(&self, thread_id: ThreadId) -> Result<(), QueueServiceError> { + let Some(manager) = self.thread_manager.upgrade() else { + return Ok(()); + }; + let Ok(thread) = manager.get_thread(thread_id).await else { + return Ok(()); + }; + + loop { + let Some(record) = self + .queue + .list_page(thread_id, /*offset*/ 0, /*limit*/ 1) + .await? + .into_iter() + .next() + else { + return Ok(()); + }; + let queued_item_id = record.id.clone(); + + let input = match serde_json::from_str::(&record.payload) { + Ok(input) => input, + Err(error) => { + tracing::warn!(%queued_item_id, %error, "discarding invalid queued item"); + self.delete_locked(thread_id, queued_item_id).await?; + continue; + } + }; + if !matches!(input, TurnInput::UserInput { .. }) { + tracing::warn!(%queued_item_id, "discarding non-user queued input"); + self.delete_locked(thread_id, queued_item_id).await?; + continue; + } + + if let Err(error) = thread.try_start_turn_if_idle(vec![input]).await { + if error.reason() == TryStartTurnIfIdleRejectionReason::RejectedByHook { + self.delete_locked(thread_id, queued_item_id).await?; + continue; + } + tracing::warn!( + %thread_id, + %queued_item_id, + reason = ?error.reason(), + "core could not start queued user input" + ); + return Ok(()); + } + + self.delete_locked(thread_id, queued_item_id).await?; + return Ok(()); + } + } + + async fn wake_if_loaded(&self, thread_id: ThreadId) { + let Some(manager) = self.thread_manager.upgrade() else { + return; + }; + if let Ok(thread) = manager.get_thread(thread_id).await + && !matches!(thread.agent_status().await, AgentStatus::Interrupted) + { + thread + .emit_thread_idle_lifecycle_if_idle(ThreadIdleCause::Completed) + .await; + } + } + + fn emit_changed(&self, thread_id: ThreadId) { + self.event_sink.emit(Event { + id: Uuid::now_v7().to_string(), + msg: EventMsg::ThreadQueueChanged(ThreadQueueChangedEvent { thread_id }), + }); + } +} + +async fn prepare_queued_user_input(mut input: TurnInput) -> Result { + validate_queued_user_input(&input)?; + let TurnInput::UserInput { content, client_id } = &mut input else { + return Err(QueueServiceError::InvalidInput); + }; + client_id.get_or_insert_with(|| Uuid::now_v7().to_string()); + if !content.iter().any(|item| { + matches!( + item, + UserInput::LocalImage { .. } | UserInput::LocalAudio { .. } + ) + }) { + return Ok(input); + } + + tokio::task::spawn_blocking(move || { + let mut input = input; + if let TurnInput::UserInput { content, .. } = &mut input { + for item in content { + snapshot_local_user_input(item)?; + } + } + Ok::(input) + }) + .await + .map_err(|error| QueueServiceError::InvalidAttachment(std::io::Error::other(error)))? + .map_err(QueueServiceError::InvalidAttachment) +} + +fn validate_queued_user_input(input: &TurnInput) -> Result<(), QueueServiceError> { + let TurnInput::UserInput { content, .. } = input else { + return Err(QueueServiceError::InvalidInput); + }; + if content.is_empty() { + return Err(QueueServiceError::InvalidInput); + } + let actual_chars: usize = content + .iter() + .filter_map(|item| match item { + UserInput::Text { text, .. } => Some(text.chars().count()), + _ => None, + }) + .sum(); + if actual_chars > MAX_USER_INPUT_TEXT_CHARS { + return Err(QueueServiceError::InputTooLarge { actual_chars }); + } + Ok(()) +} + +impl ThreadLifecycleContributor for QueuedItemService +where + C: Send + Sync + 'static, +{ + fn on_thread_idle<'a>(&'a self, input: ThreadIdleInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + if input.cause != ThreadIdleCause::Completed { + return; + } + let Ok(thread_id) = ThreadId::from_string(input.thread_store.level_id()) else { + tracing::warn!( + level_id = input.thread_store.level_id(), + "queue extension received an invalid thread id" + ); + return; + }; + let _guard = self.dispatch_guard(thread_id).await; + if let Err(error) = self.dispatch_if_idle(thread_id).await { + tracing::warn!(%thread_id, %error, "failed to dispatch queued user input"); + } + }) + } +} + +fn queued_item_from_record( + record: QueuedUserSubmissionRecord, +) -> Result { + Ok(QueuedItem { + id: record.id, + input: serde_json::from_str::(&record.payload)?, + }) +} diff --git a/codex-rs/ext/queue/tests/queue_service.rs b/codex-rs/ext/queue/tests/queue_service.rs new file mode 100644 index 0000000000..70167f643e --- /dev/null +++ b/codex-rs/ext/queue/tests/queue_service.rs @@ -0,0 +1,756 @@ +use std::path::Path; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::OnceLock; +use std::sync::Weak; +use std::time::Duration; + +use anyhow::Context; +use codex_core::TurnInput; +use codex_core::UserMessageAdmission; +use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionEventSink; +use codex_extension_api::ExtensionFuture; +use codex_extension_api::ExtensionRegistry; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::ExtensionWarning; +use codex_extension_api::NoopExtensionEventSink; +use codex_extension_api::ThreadIdleCause; +use codex_extension_api::ThreadIdleInput; +use codex_extension_api::ThreadLifecycleContributor; +use codex_protocol::ThreadId; +use codex_protocol::items::TurnItem; +use codex_protocol::models::ImageDetail; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; +use codex_protocol::user_input::UserInput; +use codex_queue_extension::QueueServiceError; +use codex_queue_extension::QueuedItemService; +use codex_state::SqliteConfig; +use codex_state::StateRuntime; +use codex_thread_store::LocalQueueStore; +use codex_thread_store::QueueStore; +use codex_thread_store::ThreadStoreError; +use codex_utils_absolute_path::test_support::PathExt; +use core_test_support::hooks::trust_discovered_hooks; +use core_test_support::responses; +use core_test_support::responses::start_mock_server; +use core_test_support::test_codex::TestCodex; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event_match; +use pretty_assertions::assert_eq; +use tempfile::TempDir; + +const TINY_PNG_BYTES: &[u8] = &[ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, + 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1, + 122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, +]; +const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII="; + +#[derive(Default)] +struct RecordingEventSink { + events: Mutex>, +} + +impl ExtensionEventSink for RecordingEventSink { + fn emit(&self, event: Event) { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(event); + } + + fn emit_warning(&self, _warning: ExtensionWarning) {} +} + +#[derive(Default)] +struct InstalledQueue(OnceLock>); + +impl ThreadLifecycleContributor for InstalledQueue { + fn on_thread_idle<'a>(&'a self, input: ThreadIdleInput<'a>) -> ExtensionFuture<'a, ()> { + match self.0.get() { + Some(service) => >::on_thread_idle(service.as_ref(), input), + None => Box::pin(async {}), + } + } +} + +fn registered_queue_extensions() -> ( + Arc, + Arc>, +) { + let installed = Arc::new(InstalledQueue::default()); + let mut extensions = ExtensionRegistryBuilder::new(); + extensions.thread_lifecycle_contributor(installed.clone()); + (installed, Arc::new(extensions.build())) +} + +fn install_registered_queue( + test: &TestCodex, + installed: &InstalledQueue, +) -> anyhow::Result> { + let service = Arc::new(QueuedItemService::new( + loaded_thread_queue(test)?, + Arc::downgrade(&test.thread_manager), + Arc::new(NoopExtensionEventSink), + )); + assert!(installed.0.set(Arc::clone(&service)).is_ok()); + Ok(service) +} + +fn write_rejecting_prompt_hook(home: &Path) { + let script_path = home.join("queue_prompt_hook.py"); + let log_path = home.join("queue_prompt_hook.log"); + let script = format!( + r#"import json +from pathlib import Path +import sys + +payload = json.load(sys.stdin) +with Path(r"{log_path}").open("a", encoding="utf-8") as log: + log.write(payload["prompt"] + "\n") +if payload["prompt"] == "blocked": + print(json.dumps({{"decision": "block", "reason": "blocked by queue hook"}})) +"#, + log_path = log_path.display(), + ); + std::fs::write(&script_path, script) + .unwrap_or_else(|error| panic!("write queue hook script: {error}")); + let hooks = serde_json::json!({ + "hooks": { + "UserPromptSubmit": [{ + "hooks": [{ + "type": "command", + "command": format!("python3 {}", script_path.display()), + }] + }] + } + }); + std::fs::write(home.join("hooks.json"), hooks.to_string()) + .unwrap_or_else(|error| panic!("write queue hooks: {error}")); +} + +async fn test_queue() -> anyhow::Result<(Arc, TempDir)> { + let home = tempfile::tempdir()?; + let sqlite = SqliteConfig::new_for_testing(home.path().abs()); + let runtime = StateRuntime::init(sqlite.clone(), "test-provider".to_string()).await?; + let queue: Arc = Arc::new(LocalQueueStore::new(runtime)); + Ok((queue, home)) +} + +fn loaded_thread_queue(test: &TestCodex) -> anyhow::Result> { + let runtime = test.codex.state_db().context("state runtime unavailable")?; + Ok(Arc::new(LocalQueueStore::new(runtime))) +} + +fn user_input(text: &str) -> TurnInput { + TurnInput::UserInput { + content: vec![UserInput::Text { + text: text.to_string(), + text_elements: Vec::new(), + }], + client_id: None, + } +} + +fn structured_user_input(text: &str) -> TurnInput { + TurnInput::UserInput { + content: vec![UserInput::Text { + text: text.to_string(), + text_elements: Vec::new(), + }], + client_id: Some("stable-client-message".to_string()), + } +} + +fn user_input_with_media(text: &str, image: UserInput, audio: UserInput) -> TurnInput { + let mut input = structured_user_input(text); + if let TurnInput::UserInput { content, .. } = &mut input { + content.extend([image, audio]); + } + input +} + +async fn emit_idle(service: &QueuedItemService, thread_id: ThreadId) { + emit_idle_with_cause(service, thread_id, ThreadIdleCause::Completed).await; +} + +async fn emit_idle_with_cause( + service: &QueuedItemService, + thread_id: ThreadId, + cause: ThreadIdleCause, +) { + let session_store = ExtensionData::new("session"); + let thread_store = ExtensionData::new(thread_id.to_string()); + >::on_thread_idle( + service, + ThreadIdleInput { + cause, + session_store: &session_store, + thread_store: &thread_store, + }, + ) + .await; +} + +#[tokio::test] +async fn queued_input_and_unique_event_ids_round_trip() -> anyhow::Result<()> { + let (queue, _home) = test_queue().await?; + let sink = Arc::new(RecordingEventSink::default()); + let event_sink: Arc = sink.clone(); + let service = QueuedItemService::new(queue, Weak::new(), event_sink); + let thread_id = ThreadId::new(); + let input = structured_user_input("structured message"); + + let first = service.enqueue(thread_id, input.clone()).await?; + let second = service.enqueue(thread_id, user_input("next")).await?; + let TurnInput::UserInput { + client_id: Some(generated_client_id), + .. + } = &second.input + else { + anyhow::bail!("queued message did not receive a client message id"); + }; + assert_eq!( + 7, + uuid::Uuid::parse_str(generated_client_id)?.get_version_num() + ); + assert_eq!(vec![first.clone(), second], service.list(thread_id).await?); + assert_eq!(input, first.input); + + let events = sink + .events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(2, events.len()); + for event in events.iter() { + let EventMsg::ThreadQueueChanged(change) = &event.msg else { + anyhow::bail!("event is not a queue change"); + }; + assert_eq!( + serde_json::json!({ "threadId": thread_id }), + serde_json::to_value(change)? + ); + } + assert_ne!(events[0].id, events[1].id); + Ok(()) +} + +#[tokio::test] +async fn editing_reordering_and_deleting_preserve_queue_identity() -> anyhow::Result<()> { + let (queue, _home) = test_queue().await?; + let service = QueuedItemService::new(queue, Weak::new(), Arc::new(NoopExtensionEventSink)); + let thread_id = ThreadId::new(); + let first = service + .enqueue(thread_id, structured_user_input("first")) + .await?; + let second = service.enqueue(thread_id, user_input("second")).await?; + let edited = service + .update(thread_id, first.id.clone(), user_input("edited")) + .await? + .context("queued item missing")?; + assert_eq!(first.id, edited.id); + let TurnInput::UserInput { client_id, .. } = &edited.input else { + anyhow::bail!("edited queue item does not contain user input"); + }; + assert_eq!(Some("stable-client-message"), client_id.as_deref()); + assert!(matches!( + service.reorder(thread_id, vec![first.id.clone()]).await, + Err(QueueServiceError::Storage( + ThreadStoreError::InvalidRequest { .. } + )) + )); + service + .reorder(thread_id, vec![second.id.clone(), first.id.clone()]) + .await?; + assert_eq!( + vec![second.clone(), edited.clone()], + service.list(thread_id).await? + ); + assert!(service.delete(thread_id, second.id).await?); + assert_eq!(vec![edited], service.list(thread_id).await?); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn starting_a_selected_item_preserves_the_remaining_queue() -> anyhow::Result<()> { + let server = start_mock_server().await; + let response = + responses::mount_sse_once(&server, responses::sse_completed("selected-turn")).await; + let test = test_codex().build_with_auto_env(&server).await?; + let thread_id = test.session_configured.thread_id; + let queue = loaded_thread_queue(&test)?; + let service = QueuedItemService::new(queue, Weak::new(), Arc::new(NoopExtensionEventSink)); + let first = service.enqueue(thread_id, user_input("first")).await?; + let second = service + .enqueue(thread_id, structured_user_input("second")) + .await?; + + let admission = service + .start(test.codex.as_ref(), second.id.clone(), /*trace*/ None) + .await?; + + assert!(matches!(admission, UserMessageAdmission::Started { .. })); + assert_eq!(vec![first], service.list(thread_id).await?); + wait_for_event_match(test.codex.as_ref(), |event| match event { + EventMsg::TurnComplete(_) => Some(()), + _ => None, + }) + .await; + assert_eq!( + Some("second"), + response + .single_request() + .message_input_texts("user") + .last() + .map(String::as_str) + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stopped_turns_leave_queued_messages_in_place() -> anyhow::Result<()> { + let server = start_mock_server().await; + let test = test_codex().build_with_auto_env(&server).await?; + let thread_id = test.session_configured.thread_id; + let queue = loaded_thread_queue(&test)?; + let service = QueuedItemService::new( + queue, + Arc::downgrade(&test.thread_manager), + Arc::new(NoopExtensionEventSink), + ); + let queued = service + .enqueue(thread_id, user_input("keep queued after stopping")) + .await?; + + for cause in [ThreadIdleCause::Interrupted, ThreadIdleCause::Failed] { + emit_idle_with_cause(&service, thread_id, cause).await; + + assert_eq!(vec![queued.clone()], service.list(thread_id).await?); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn registered_queue_lifecycle_starts_messages_in_fifo_order() -> anyhow::Result<()> { + let server = start_mock_server().await; + let responses = responses::mount_sse_sequence( + &server, + ["turn-a", "turn-b", "turn-c"] + .into_iter() + .map(responses::sse_completed) + .collect(), + ) + .await; + let (installed, extensions) = registered_queue_extensions(); + let test = test_codex() + .with_extensions(extensions) + .with_config(|config| config.include_environment_context = false) + .build_with_auto_env(&server) + .await?; + let thread_id = test.session_configured.thread_id; + let staging = QueuedItemService::new( + loaded_thread_queue(&test)?, + Weak::new(), + Arc::new(NoopExtensionEventSink), + ); + for prompt in ["B", "C"] { + staging.enqueue(thread_id, user_input(prompt)).await?; + } + let queue = install_registered_queue(&test, installed.as_ref())?; + + tokio::time::timeout(Duration::from_secs(10), async { + test.submit_text_turn("A").await?; + for _ in 0..2 { + wait_for_event_match(test.codex.as_ref(), |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + } + anyhow::Ok(()) + }) + .await??; + + let prompts = responses + .requests() + .into_iter() + .map(|request| { + request + .message_input_texts("user") + .pop() + .context("model request has no user input") + }) + .collect::>>()?; + assert_eq!(vec!["A", "B", "C"], prompts); + assert!(queue.list(thread_id).await?.is_empty()); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rejected_queue_messages_are_consumed_without_retrying_or_blocking_followups() +-> anyhow::Result<()> { + let server = start_mock_server().await; + let responses = responses::mount_sse_sequence( + &server, + ["initial-turn", "following-turn"] + .into_iter() + .map(responses::sse_completed) + .collect(), + ) + .await; + let (installed, extensions) = registered_queue_extensions(); + let test = test_codex() + .with_extensions(extensions) + .with_pre_build_hook(write_rejecting_prompt_hook) + .with_config(trust_discovered_hooks) + .with_config(|config| config.include_environment_context = false) + .build_with_auto_env(&server) + .await?; + let thread_id = test.session_configured.thread_id; + let staging = QueuedItemService::new( + loaded_thread_queue(&test)?, + Weak::new(), + Arc::new(NoopExtensionEventSink), + ); + for prompt in ["blocked", "C"] { + staging.enqueue(thread_id, user_input(prompt)).await?; + } + let queue = install_registered_queue(&test, installed.as_ref())?; + + tokio::time::timeout(Duration::from_secs(10), async { + test.submit_text_turn("A").await?; + for _ in 0..2 { + wait_for_event_match(test.codex.as_ref(), |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + } + anyhow::Ok(()) + }) + .await??; + + let prompts = responses + .requests() + .into_iter() + .map(|request| { + request + .message_input_texts("user") + .pop() + .context("model request has no user input") + }) + .collect::>>()?; + assert_eq!(vec!["A", "C"], prompts); + let hook_log = std::fs::read_to_string(test.codex_home_path().join("queue_prompt_hook.log"))?; + assert_eq!( + vec!["A", "blocked", "C"], + hook_log.lines().collect::>() + ); + assert!(queue.list(thread_id).await?.is_empty()); + + let rejected = staging.enqueue(thread_id, user_input("blocked")).await?; + let error = tokio::time::timeout( + Duration::from_secs(10), + queue.start(test.codex.as_ref(), rejected.id, /*trace*/ None), + ) + .await? + .expect_err("explicitly started blocked input should be rejected"); + assert!(matches!( + error, + QueueServiceError::Admission(error) + if matches!( + error.details(), + codex_protocol::error::CodexErrorDetails::InvalidRequest(message) + if message == "user message was rejected by a hook" + ) + )); + assert!(queue.list(thread_id).await?.is_empty()); + let hook_log = std::fs::read_to_string(test.codex_home_path().join("queue_prompt_hook.log"))?; + assert_eq!( + vec!["A", "blocked", "C", "blocked"], + hook_log.lines().collect::>() + ); + assert_eq!(2, responses.requests().len()); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow::Result<()> { + let (queue, home) = test_queue().await?; + let service = QueuedItemService::new( + Arc::clone(&queue), + Weak::new(), + Arc::new(NoopExtensionEventSink), + ); + let thread_id = ThreadId::new(); + let image_path = home.path().join("queued.png"); + let audio_path = home.path().join("queued.mp3"); + std::fs::write(&image_path, TINY_PNG_BYTES)?; + std::fs::write(&audio_path, b"audio")?; + let queued_input = user_input_with_media( + "queued attachments", + UserInput::LocalImage { + path: image_path.clone(), + detail: Some(ImageDetail::Original), + }, + UserInput::LocalAudio { + path: audio_path.clone(), + }, + ); + let expected_queued_input = user_input_with_media( + "queued attachments", + UserInput::Image { + image_url: TINY_PNG_DATA_URL.to_string(), + detail: Some(ImageDetail::Original), + }, + UserInput::Audio { + audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + }, + ); + + let queued = service.enqueue(thread_id, queued_input).await?; + std::fs::remove_file(&image_path)?; + std::fs::remove_file(&audio_path)?; + + assert_eq!(expected_queued_input, queued.input); + assert_eq!(vec![queued.clone()], service.list(thread_id).await?); + let queued_record = queue + .list_page(thread_id, /*offset*/ 0, /*limit*/ 1) + .await? + .into_iter() + .next() + .context("snapshotted queued attachments were not persisted")?; + let persisted_queued_input: TurnInput = serde_json::from_str(&queued_record.payload)?; + assert_eq!(expected_queued_input, persisted_queued_input); + + let edited_image_path = home.path().join("edited.png"); + let edited_audio_path = home.path().join("edited.m4a"); + std::fs::write(&edited_image_path, TINY_PNG_BYTES)?; + std::fs::write(&edited_audio_path, b"edited audio")?; + let edited_input = user_input_with_media( + "edited attachments", + UserInput::LocalImage { + path: edited_image_path.clone(), + detail: Some(ImageDetail::High), + }, + UserInput::LocalAudio { + path: edited_audio_path.clone(), + }, + ); + let expected_edited_input = user_input_with_media( + "edited attachments", + UserInput::Image { + image_url: TINY_PNG_DATA_URL.to_string(), + detail: Some(ImageDetail::High), + }, + UserInput::Audio { + audio_url: "data:audio/mp4;base64,ZWRpdGVkIGF1ZGlv".to_string(), + }, + ); + + let edited = service + .update(thread_id, queued.id.clone(), edited_input) + .await? + .context("snapshotted queued attachments were not updated")?; + std::fs::remove_file(&edited_image_path)?; + std::fs::remove_file(&edited_audio_path)?; + + assert_eq!(expected_edited_input, edited.input); + assert_eq!(vec![edited.clone()], service.list(thread_id).await?); + let edited_record = queue + .list_page(thread_id, /*offset*/ 0, /*limit*/ 1) + .await? + .into_iter() + .next() + .context("snapshotted edited attachments were not persisted")?; + let persisted_edited_input: TurnInput = serde_json::from_str(&edited_record.payload)?; + assert_eq!(expected_edited_input, persisted_edited_input); + + Ok(()) +} + +#[tokio::test] +async fn invalid_local_attachments_do_not_mutate_queue() -> anyhow::Result<()> { + let (queue, home) = test_queue().await?; + let service = QueuedItemService::new(queue, Weak::new(), Arc::new(NoopExtensionEventSink)); + let thread_id = ThreadId::new(); + let existing = service.enqueue(thread_id, user_input("existing")).await?; + let missing_path = home.path().join("missing.png"); + let invalid_input = user_input_with_media( + "invalid attachments", + UserInput::LocalImage { + path: missing_path, + detail: Some(ImageDetail::Original), + }, + UserInput::Audio { + audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + }, + ); + + assert!(matches!( + service + .enqueue(thread_id, invalid_input.clone()) + .await, + Err(QueueServiceError::InvalidAttachment(error)) + if error.kind() == std::io::ErrorKind::NotFound + )); + assert!(matches!( + service + .update( + thread_id, + existing.id.clone(), + invalid_input, + ) + .await, + Err(QueueServiceError::InvalidAttachment(error)) + if error.kind() == std::io::ErrorKind::NotFound + )); + assert_eq!(vec![existing], service.list(thread_id).await?); + + Ok(()) +} + +#[tokio::test] +async fn non_user_input_cannot_enter_the_user_message_queue() -> anyhow::Result<()> { + let (queue, _home) = test_queue().await?; + let service = QueuedItemService::new(queue, Weak::new(), Arc::new(NoopExtensionEventSink)); + let error = service + .enqueue( + ThreadId::new(), + TurnInput::ResponseItem(ResponseItem::Other), + ) + .await + .expect_err("response item should not enter the user queue"); + assert!(matches!(error, QueueServiceError::InvalidInput)); + Ok(()) +} + +#[tokio::test] +async fn queued_text_limit_is_enforced_across_all_input_items() -> anyhow::Result<()> { + let (queue, _home) = test_queue().await?; + let service = QueuedItemService::new(queue, Weak::new(), Arc::new(NoopExtensionEventSink)); + let thread_id = ThreadId::new(); + let existing = service.enqueue(thread_id, user_input("existing")).await?; + let mut oversized = user_input(&"x".repeat(MAX_USER_INPUT_TEXT_CHARS / 2)); + if let TurnInput::UserInput { content, .. } = &mut oversized { + content.push(UserInput::Text { + text: "y".repeat(MAX_USER_INPUT_TEXT_CHARS / 2 + 1), + text_elements: Vec::new(), + }); + } + + assert!(matches!( + service + .enqueue(thread_id, oversized.clone()) + .await, + Err(QueueServiceError::InputTooLarge { actual_chars }) + if actual_chars == MAX_USER_INPUT_TEXT_CHARS + 1 + )); + assert!(matches!( + service + .update( + thread_id, + existing.id.clone(), + oversized, + ) + .await, + Err(QueueServiceError::InputTooLarge { actual_chars }) + if actual_chars == MAX_USER_INPUT_TEXT_CHARS + 1 + )); + assert_eq!(vec![existing], service.list(thread_id).await?); + + Ok(()) +} + +#[tokio::test] +async fn queued_text_limit_counts_characters_and_ignores_non_text_items() -> anyhow::Result<()> { + let (queue, _home) = test_queue().await?; + let service = QueuedItemService::new(queue, Weak::new(), Arc::new(NoopExtensionEventSink)); + let mut input = structured_user_input(&"é".repeat(MAX_USER_INPUT_TEXT_CHARS)); + if let TurnInput::UserInput { content, .. } = &mut input { + content.push(UserInput::Mention { + name: "Demo App".to_string(), + path: "app://demo-app".to_string(), + }); + } + + let queued = service.enqueue(ThreadId::new(), input.clone()).await?; + assert_eq!(input, queued.input); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_head_is_skipped_and_a_live_user_turn_is_accepted() -> anyhow::Result<()> { + let server = start_mock_server().await; + let response = + responses::mount_sse_once(&server, responses::sse_completed("queued-turn")).await; + let test = test_codex() + .with_config(|config| config.include_environment_context = false) + .build_with_auto_env(&server) + .await?; + let thread_id = test.session_configured.thread_id; + let queue = loaded_thread_queue(&test)?; + queue + .enqueue(thread_id, r#"{"unsupported":true}"#.to_string()) + .await?; + let service = QueuedItemService::new( + queue, + Arc::downgrade(&test.thread_manager), + Arc::new(NoopExtensionEventSink), + ); + + service + .enqueue(thread_id, structured_user_input("durable follow-up")) + .await?; + emit_idle(&service, thread_id).await; + let client_id = wait_for_event_match(test.codex.as_ref(), |event| match event { + EventMsg::ItemCompleted(event) => match &event.item { + TurnItem::UserMessage(item) => Some(item.client_id.clone()), + _ => None, + }, + _ => None, + }) + .await; + assert_eq!(Some("stable-client-message".to_string()), client_id); + wait_for_event_match(test.codex.as_ref(), |event| match event { + EventMsg::TurnComplete(_) => Some(()), + _ => None, + }) + .await; + assert!(service.list(thread_id).await?.is_empty()); + let request = response.single_request(); + assert_eq!( + vec!["durable follow-up"], + request.message_input_texts("user") + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resumed_idle_dispatches_input_persisted_without_a_loaded_manager() -> anyhow::Result<()> { + let server = start_mock_server().await; + responses::mount_sse_once(&server, responses::sse_completed("resumed-turn")).await; + let test = test_codex().build_with_auto_env(&server).await?; + let thread_id = test.session_configured.thread_id; + let queue = loaded_thread_queue(&test)?; + QueuedItemService::new( + Arc::clone(&queue), + Weak::new(), + Arc::new(NoopExtensionEventSink), + ) + .enqueue(thread_id, user_input("queued while unloaded")) + .await?; + let service = QueuedItemService::new( + queue, + Arc::downgrade(&test.thread_manager), + Arc::new(NoopExtensionEventSink), + ); + emit_idle(&service, thread_id).await; + assert!(service.list(thread_id).await?.is_empty()); + Ok(()) +} diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index dd551a580a..e056e9b5c5 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -306,8 +306,8 @@ async fn run_codex_tool_session_inner( EventMsg::SessionConfigured(_) => { tracing::error!("unexpected SessionConfigured event"); } - EventMsg::ThreadGoalUpdated(_) => { - // Ignore thread goal metadata updates in MCP tool runner. + EventMsg::ThreadGoalUpdated(_) | EventMsg::ThreadQueueChanged(_) => { + // Ignore thread-scoped metadata updates in MCP tool runner. } EventMsg::McpStartupUpdate(_) | EventMsg::McpStartupComplete(_) => { // Ignored in MCP tool runner. diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index 5a969d4661..28fff2120a 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -20,6 +20,7 @@ pub mod error; pub mod exec_output; pub mod items; mod legacy_events; +pub mod local_media; pub mod mcp; pub mod mcp_approval_meta; pub mod memory_citation; diff --git a/codex-rs/protocol/src/local_media.rs b/codex-rs/protocol/src/local_media.rs new file mode 100644 index 0000000000..7e4cbd9dfd --- /dev/null +++ b/codex-rs/protocol/src/local_media.rs @@ -0,0 +1,93 @@ +use std::io; +use std::io::Read; +use std::path::Path; + +use codex_utils_image::MAX_PROMPT_IMAGE_INPUT_BYTES; +use codex_utils_image::PromptImageMode; +use codex_utils_image::data_url_from_bytes; +use codex_utils_image::load_for_prompt_bytes; + +use crate::models::ImageDetail; +use crate::user_input::UserInput; + +/// Maximum accepted decoded byte length for prompt audio inputs. +/// +/// This matches the Responses API audio input limit. +pub const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024; + +/// Snapshots local image and audio input into portable, validated data URLs. +pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> { + match input { + UserInput::LocalImage { path, detail } => { + let image_detail = *detail; + let mode = match image_detail { + Some(ImageDetail::Original) => PromptImageMode::Original, + Some(ImageDetail::Auto | ImageDetail::Low | ImageDetail::High) | None => { + PromptImageMode::ResizeToFit + } + }; + let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?; + let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?; + *input = UserInput::Image { + image_url: image.into_data_url(), + detail: image_detail, + }; + } + UserInput::LocalAudio { path } => { + let mime = audio_mime_for_path(path).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format") + })?; + let file_bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?; + *input = UserInput::Audio { + audio_url: data_url_from_bytes(mime, &file_bytes), + }; + } + UserInput::Text { .. } + | UserInput::Image { .. } + | UserInput::Audio { .. } + | UserInput::Skill { .. } + | UserInput::Mention { .. } => {} + } + Ok(()) +} + +fn read_bounded_local_media(path: &Path, max_bytes: usize, kind: &str) -> io::Result> { + let file = std::fs::File::open(path)?; + if file.metadata()?.len() > max_bytes as u64 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{kind} input exceeds {max_bytes} bytes"), + )); + } + + let mut bytes = Vec::new(); + file.take(max_bytes as u64 + 1).read_to_end(&mut bytes)?; + if bytes.len() > max_bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{kind} input exceeds {max_bytes} bytes"), + )); + } + Ok(bytes) +} + +pub(crate) fn audio_mime_for_path(path: &Path) -> Option<&'static str> { + let extension = path.extension()?.to_str()?; + if extension.eq_ignore_ascii_case("wav") { + Some("audio/wav") + } else if extension.eq_ignore_ascii_case("mp3") { + Some("audio/mpeg") + } else if extension.eq_ignore_ascii_case("m4a") { + Some("audio/mp4") + } else if extension.eq_ignore_ascii_case("webm") { + Some("audio/webm") + } else if extension.eq_ignore_ascii_case("ogg") { + Some("audio/ogg") + } else { + None + } +} + +#[cfg(test)] +#[path = "local_media_tests.rs"] +mod tests; diff --git a/codex-rs/protocol/src/local_media_tests.rs b/codex-rs/protocol/src/local_media_tests.rs new file mode 100644 index 0000000000..3d03c32620 --- /dev/null +++ b/codex-rs/protocol/src/local_media_tests.rs @@ -0,0 +1,128 @@ +use super::*; +use anyhow::Result; +use pretty_assertions::assert_eq; +use tempfile::tempdir; + +const TINY_PNG_BYTES: &[u8] = &[ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, + 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1, + 122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, +]; + +#[test] +fn snapshots_local_image_user_input_with_requested_detail() -> Result<()> { + let temp_dir = tempdir()?; + let image_path = temp_dir.path().join("sample.png"); + std::fs::write(&image_path, TINY_PNG_BYTES)?; + + for detail in [ + None, + Some(ImageDetail::Auto), + Some(ImageDetail::Low), + Some(ImageDetail::High), + Some(ImageDetail::Original), + ] { + let mut input = UserInput::LocalImage { + path: image_path.clone(), + detail, + }; + + snapshot_local_user_input(&mut input)?; + + assert_eq!( + UserInput::Image { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII=" + .to_string(), + detail, + }, + input + ); + } + + Ok(()) +} + +#[test] +fn snapshots_local_audio_user_input_with_supported_mime_types() -> Result<()> { + let temp_dir = tempdir()?; + + for (extension, mime) in [ + ("wav", "audio/wav"), + ("MP3", "audio/mpeg"), + ("m4a", "audio/mp4"), + ("webm", "audio/webm"), + ("ogg", "audio/ogg"), + ] { + let audio_path = temp_dir.path().join(format!("sample.{extension}")); + std::fs::write(&audio_path, b"audio")?; + let mut input = UserInput::LocalAudio { path: audio_path }; + + snapshot_local_user_input(&mut input)?; + + assert_eq!( + UserInput::Audio { + audio_url: format!("data:{mime};base64,YXVkaW8="), + }, + input + ); + } + + Ok(()) +} + +#[test] +fn rejects_invalid_local_media_without_changing_user_input() -> Result<()> { + let temp_dir = tempdir()?; + let invalid_image_path = temp_dir.path().join("invalid.png"); + let unsupported_audio_path = temp_dir.path().join("unsupported.flac"); + std::fs::write(&invalid_image_path, b"invalid image")?; + std::fs::write(&unsupported_audio_path, b"audio")?; + + let mut invalid_image = UserInput::LocalImage { + path: invalid_image_path, + detail: Some(ImageDetail::Original), + }; + let original_image = invalid_image.clone(); + let image_error = snapshot_local_user_input(&mut invalid_image) + .expect_err("an invalid local image must be rejected"); + assert_eq!(io::ErrorKind::Other, image_error.kind()); + assert_eq!(original_image, invalid_image); + + let mut unsupported_audio = UserInput::LocalAudio { + path: unsupported_audio_path, + }; + let original_audio = unsupported_audio.clone(); + let audio_error = snapshot_local_user_input(&mut unsupported_audio) + .expect_err("an unsupported local audio format must be rejected"); + assert_eq!(io::ErrorKind::InvalidData, audio_error.kind()); + assert_eq!(original_audio, unsupported_audio); + + Ok(()) +} + +#[test] +fn rejects_oversized_local_media_without_reading_it() -> Result<()> { + let temp_dir = tempdir()?; + + for (name, limit, is_image) in [ + ("oversized.png", MAX_PROMPT_IMAGE_INPUT_BYTES, true), + ("oversized.mp3", MAX_PROMPT_AUDIO_INPUT_BYTES, false), + ] { + let path = temp_dir.path().join(name); + std::fs::File::create(&path)?.set_len(limit as u64 + 1)?; + let mut input = if is_image { + UserInput::LocalImage { path, detail: None } + } else { + UserInput::LocalAudio { path } + }; + let original = input.clone(); + + let error = snapshot_local_user_input(&mut input) + .expect_err("oversized local media must be rejected before it is read"); + + assert_eq!(io::ErrorKind::InvalidData, error.kind()); + assert_eq!(original, input); + } + + Ok(()) +} diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 2c2a6d6b7a..99d6bf7f50 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -12,6 +12,7 @@ use serde::Serialize; use serde::ser::Serializer; use ts_rs::TS; +use crate::local_media::audio_mime_for_path; use crate::permissions::FileSystemAccessMode; use crate::permissions::FileSystemPath; use crate::permissions::FileSystemSandboxEntry; @@ -30,6 +31,8 @@ use crate::mcp::CallToolResult; mod executed_tool_calls; +pub use crate::local_media::MAX_PROMPT_AUDIO_INPUT_BYTES; +pub use crate::local_media::snapshot_local_user_input; pub use executed_tool_calls::ExecutedToolCall; pub use executed_tool_calls::ExecutedToolCallArguments; pub use executed_tool_calls::ExecutedToolCallTruncation; @@ -1529,23 +1532,6 @@ pub enum LocalImagePreparation { Defer, } -fn audio_mime_for_path(path: &std::path::Path) -> Option<&'static str> { - let extension = path.extension()?.to_str()?; - if extension.eq_ignore_ascii_case("wav") { - Some("audio/wav") - } else if extension.eq_ignore_ascii_case("mp3") { - Some("audio/mpeg") - } else if extension.eq_ignore_ascii_case("m4a") { - Some("audio/mp4") - } else if extension.eq_ignore_ascii_case("webm") { - Some("audio/webm") - } else if extension.eq_ignore_ascii_case("ogg") { - Some("audio/ogg") - } else { - None - } -} - fn unsupported_audio_error_placeholder(path: &std::path::Path) -> ContentItem { ContentItem::InputText { text: format!( diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 242a5657ae..48f0d36567 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1371,6 +1371,9 @@ pub enum EventMsg { /// Updated long-running goal metadata for the thread. ThreadGoalUpdated(ThreadGoalUpdatedEvent), + /// A durable thread-scoped user-message queue changed. + ThreadQueueChanged(ThreadQueueChangedEvent), + /// Incremental MCP startup progress updates. McpStartupUpdate(McpStartupUpdateEvent), @@ -2332,7 +2335,7 @@ pub struct AgentMessageEvent { pub memory_citation: Option, } -#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema, TS)] +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, JsonSchema, TS)] pub struct UserMessageEvent { #[serde(default, skip_serializing_if = "Option::is_none")] pub client_id: Option, @@ -4117,6 +4120,13 @@ pub struct ThreadGoalUpdatedEvent { pub goal: ThreadGoal, } +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "protocol/")] +pub struct ThreadQueueChangedEvent { + pub thread_id: ThreadId, +} + /// User's decision in response to an ExecApprovalRequest. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Display, JsonSchema, TS)] #[serde(rename_all = "snake_case")] diff --git a/codex-rs/rollout-trace/src/protocol_event.rs b/codex-rs/rollout-trace/src/protocol_event.rs index 22d2974514..e0a03a3737 100644 --- a/codex-rs/rollout-trace/src/protocol_event.rs +++ b/codex-rs/rollout-trace/src/protocol_event.rs @@ -371,6 +371,7 @@ pub(crate) fn tool_runtime_trace_event(event: &EventMsg) -> Option Option<&'static s | EventMsg::AgentReasoningRawContent(_) | EventMsg::AgentReasoningSectionBreak(_) | EventMsg::ThreadGoalUpdated(_) + | EventMsg::ThreadQueueChanged(_) | EventMsg::McpStartupUpdate(_) | EventMsg::McpStartupComplete(_) | EventMsg::McpToolCallBegin(_) diff --git a/codex-rs/rollout/src/policy.rs b/codex-rs/rollout/src/policy.rs index 65a8213f08..22718f6c95 100644 --- a/codex-rs/rollout/src/policy.rs +++ b/codex-rs/rollout/src/policy.rs @@ -120,6 +120,7 @@ pub fn should_persist_event_msg(ev: &EventMsg, history_mode: ThreadHistoryMode) // Transient, non-durable events. EventMsg::Error(_) + | EventMsg::ThreadQueueChanged(_) | EventMsg::GuardianAssessment(_) | EventMsg::ExecCommandEnd(_) | EventMsg::ViewImageToolCall(_) diff --git a/codex-rs/utils/audio/src/lib.rs b/codex-rs/utils/audio/src/lib.rs index ab836b5631..a2adf86f72 100644 --- a/codex-rs/utils/audio/src/lib.rs +++ b/codex-rs/utils/audio/src/lib.rs @@ -4,6 +4,7 @@ use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES; use codex_protocol::models::ResponseItem; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; @@ -25,10 +26,6 @@ const AUDIO_TOO_LARGE_PLACEHOLDER: &str = const UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER: &str = "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg"; -/// Maximum accepted decoded byte length for prompt audio inputs. -/// -/// This matches the Responses API audio input limit. -const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024; const MAX_PROMPT_AUDIO_BASE64_BYTES: usize = MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4; const AUDIO_TOKEN_ESTIMATE_CACHE_SIZE: usize = 32; const AUDIO_TOKENS_PER_SECOND: f64 = 10.0;