From ba69c2bec3a9ebae9278e424876b6c001ac48ac4 Mon Sep 17 00:00:00 2001 From: camysidron1 Date: Mon, 2 Nov 2026 09:32:40 -0800 Subject: [PATCH] fix: drain native Python callbacks before interpreter finalization --- changelog/fixed/sleepy-tokio-fireflies.md | 1 + temporalio/bridge/src/callbacks.rs | 224 ++++++++++++++++++++++ temporalio/bridge/src/client.rs | 2 +- temporalio/bridge/src/lib.rs | 2 + temporalio/bridge/src/runtime.rs | 92 ++++++++- temporalio/bridge/src/worker.rs | 80 ++++++-- temporalio/runtime.py | 6 + tests/test_runtime_shutdown.py | 201 +++++++++++++++++++ 8 files changed, 583 insertions(+), 25 deletions(-) create mode 100644 changelog/fixed/sleepy-tokio-fireflies.md create mode 100644 temporalio/bridge/src/callbacks.rs create mode 100644 tests/test_runtime_shutdown.py diff --git a/changelog/fixed/sleepy-tokio-fireflies.md b/changelog/fixed/sleepy-tokio-fireflies.md new file mode 100644 index 0000000000..2582c88edc --- /dev/null +++ b/changelog/fixed/sleepy-tokio-fireflies.md @@ -0,0 +1 @@ +Drain native-to-Python callbacks before interpreter finalization to prevent process-exit crashes after completed client operations. Normal Python exit handlers still run; workers and application tasks must still be shut down by the application. diff --git a/temporalio/bridge/src/callbacks.rs b/temporalio/bridge/src/callbacks.rs new file mode 100644 index 0000000000..dbbd27eb44 --- /dev/null +++ b/temporalio/bridge/src/callbacks.rs @@ -0,0 +1,224 @@ +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Condvar, Mutex, Weak}; + +#[derive(Default)] +pub(crate) struct CallbackRegistry { + closing_pid: AtomicU32, + callbacks: Mutex>>, +} + +impl CallbackRegistry { + pub(crate) fn is_closing(&self) -> bool { + self.closing_pid.load(Ordering::Acquire) == std::process::id() + } + + // Registration and closing are called with the GIL held. Callback threads + // never take this lock, so a fork cannot inherit it from one of those threads. + pub(crate) fn register(&self) -> Option> { + let mut callbacks = self.callbacks.lock().unwrap(); + if self.is_closing() { + return None; + } + callbacks.retain(|entry| entry.strong_count() != 0); + // Interpreter shutdown is shared: a callback on one runtime can be + // waiting for an operation already issued on another runtime. + if let Some(entry) = callbacks + .iter() + .filter_map(Weak::upgrade) + .find(|entry| entry.pid == std::process::id()) + { + return Some(entry); + } + let entry = Arc::new(Callbacks { + pid: std::process::id(), + state: Mutex::new(State::default()), + changed: Condvar::new(), + }); + callbacks.push(Arc::downgrade(&entry)); + Some(entry) + } + + pub(crate) fn begin_shutdown(&self) -> Vec> { + self.closing_pid + .store(std::process::id(), Ordering::Release); + let callbacks: Vec<_> = self + .callbacks + .lock() + .unwrap() + .iter() + .filter_map(Weak::upgrade) + // Inherited callback counts/locks belong to threads in the parent. + .filter(|entry| entry.pid == std::process::id()) + .collect(); + for entry in &callbacks { + entry.close(); + } + callbacks + } +} + +pub(crate) struct Callbacks { + pid: u32, + state: Mutex, + changed: Condvar, +} + +#[derive(Default)] +struct State { + closed: bool, + active: usize, +} + +impl Callbacks { + pub(crate) fn enter(self: &Arc) -> Option { + self.enter_inner(false) + } + + pub(crate) fn enter_completion(self: &Arc) -> Option { + self.enter_inner(true) + } + + fn enter_inner(self: &Arc, completion: bool) -> Option { + if self.pid != std::process::id() { + return None; + } + let mut state = self.state.lock().unwrap(); + // An active callback may be awaiting an already-issued RPC on another + // runtime. Drain its completions, but do not admit work after quiescence. + if state.closed && (!completion || state.active == 0) { + return None; + } + state.active += 1; + Some(CallbackGuard(self.clone())) + } + + fn close(&self) { + let mut state = self.state.lock().unwrap(); + state.closed = true; + self.changed.notify_all(); + } + + pub(crate) fn wait(&self) { + let mut state = self.state.lock().unwrap(); + while state.active != 0 { + state = self.changed.wait(state).unwrap(); + } + } +} + +pub(crate) struct CallbackGuard(Arc); + +impl Drop for CallbackGuard { + fn drop(&mut self) { + let mut state = self.0.state.lock().unwrap(); + state.active -= 1; + if state.closed && state.active == 0 { + self.0.changed.notify_all(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc::{channel, RecvTimeoutError}; + use std::thread; + use std::time::Duration; + + #[test] + fn shutdown_waits_for_callbacks_and_rejects_new_work() { + let registry = Arc::new(CallbackRegistry::default()); + let callbacks = registry.register().unwrap(); + let guard = callbacks.enter().unwrap(); + let (closed_tx, closed_rx) = channel(); + let (done_tx, done_rx) = channel(); + let closing_registry = registry.clone(); + let closer = thread::spawn(move || { + let callbacks = closing_registry.begin_shutdown(); + closed_tx.send(()).unwrap(); + for entry in callbacks { + entry.wait(); + } + done_tx.send(()).unwrap(); + }); + + closed_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(registry.register().is_none()); + assert!(callbacks.enter().is_none()); + assert_eq!( + done_rx.recv_timeout(Duration::from_millis(50)), + Err(RecvTimeoutError::Timeout) + ); + drop(guard); + done_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + closer.join().unwrap(); + for entry in registry.begin_shutdown() { + entry.wait(); + } + } + + #[test] + fn shutdown_ignores_inherited_callback_locks_and_counts() { + let registry = CallbackRegistry::default(); + let inherited = Arc::new(Callbacks { + pid: std::process::id().wrapping_add(1), + state: Mutex::new(State { + closed: false, + active: 1, + }), + changed: Condvar::new(), + }); + registry + .callbacks + .lock() + .unwrap() + .push(Arc::downgrade(&inherited)); + let _inherited_lock = inherited.state.lock().unwrap(); + assert!(inherited.enter().is_none()); + assert!(registry.begin_shutdown().is_empty()); + } + + #[test] + fn active_callbacks_can_drain_completions_from_other_runtimes() { + let registry = CallbackRegistry::default(); + let first_runtime = registry.register().unwrap(); + let second_runtime = registry.register().unwrap(); + let active = first_runtime.enter().unwrap(); + let closing = registry.begin_shutdown(); + assert!(second_runtime.enter().is_none()); + let completion = second_runtime.enter_completion().unwrap(); + drop(active); + assert!(first_runtime.enter_completion().is_some()); + drop(completion); + for entry in closing { + entry.wait(); + } + assert!(second_runtime.enter_completion().is_none()); + } + + #[test] + fn shutdown_races_safely_with_callback_entry() { + let registry = CallbackRegistry::default(); + let callbacks = registry.register().unwrap(); + let ready = Arc::new(std::sync::Barrier::new(9)); + let threads: Vec<_> = (0..8) + .map(|_| { + let callbacks = callbacks.clone(); + let ready = ready.clone(); + thread::spawn(move || { + ready.wait(); + drop(callbacks.enter()); + }) + }) + .collect(); + ready.wait(); + let closing = registry.begin_shutdown(); + for entry in closing { + entry.wait(); + } + for thread in threads { + thread.join().unwrap(); + } + assert!(callbacks.enter().is_none()); + } +} diff --git a/temporalio/bridge/src/client.rs b/temporalio/bridge/src/client.rs index 28b3bb37c8..6c0a2294db 100644 --- a/temporalio/bridge/src/client.rs +++ b/temporalio/bridge/src/client.rs @@ -210,7 +210,7 @@ where match res { Ok(resp) => Ok(resp.get_ref().encode_to_vec()), Err(err) => { - Python::attach(move |py| { + runtime::with_python(move |py| { // Create tuple of "status", "message", and optional "details" let code = err.code() as u32; let message = err.message().to_owned(); diff --git a/temporalio/bridge/src/lib.rs b/temporalio/bridge/src/lib.rs index a4fda26db6..c3f72bf423 100644 --- a/temporalio/bridge/src/lib.rs +++ b/temporalio/bridge/src/lib.rs @@ -1,6 +1,7 @@ use pyo3::prelude::*; use pyo3::types::PyTuple; +mod callbacks; mod client; mod client_rpc_generated; mod envconfig; @@ -77,6 +78,7 @@ fn temporal_sdk_bridge(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { envconfig_module.add_function(wrap_pyfunction!(envconfig::load_client_connect_config, m)?)?; m.add_submodule(&envconfig_module)?; + runtime::register_callback_shutdown(py)?; Ok(()) } diff --git a/temporalio/bridge/src/runtime.rs b/temporalio/bridge/src/runtime.rs index b5d70df08b..b71067af8f 100644 --- a/temporalio/bridge/src/runtime.rs +++ b/temporalio/bridge/src/runtime.rs @@ -2,12 +2,13 @@ use futures::channel::mpsc::Receiver; use pyo3::exceptions::{PyAssertionError, PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pythonize::pythonize; +use std::cell::RefCell; use std::collections::HashMap; use std::future::Future; use std::net::SocketAddr; use std::pin::Pin; use std::str::FromStr; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use std::time::{Duration, UNIX_EPOCH}; use temporalio_common::telemetry::metrics::core::MetricCallBufferer; use temporalio_common::telemetry::metrics::CoreMeter; @@ -23,8 +24,72 @@ use tokio_stream::StreamExt; use tracing::Level; use url::Url; +use crate::callbacks::{CallbackRegistry, Callbacks}; use crate::metric::{convert_metric_events, BufferedMetricRef, BufferedMetricUpdate}; +static CALLBACK_REGISTRY: LazyLock = LazyLock::new(CallbackRegistry::default); + +thread_local! { + static CALLBACK_CONTEXT: RefCell>> = const { RefCell::new(None) }; +} + +struct CallbackContext(Option>); + +impl CallbackContext { + fn enter(callbacks: Arc) -> Self { + Self(CALLBACK_CONTEXT.with(|current| current.replace(Some(callbacks)))) + } +} + +impl Drop for CallbackContext { + fn drop(&mut self) { + CALLBACK_CONTEXT.with(|current| current.replace(self.0.take())); + } +} + +fn current_callbacks() -> Arc { + CALLBACK_CONTEXT + .with(|current| current.borrow().clone()) + .or_else(|| CALLBACK_TASK.try_with(Clone::clone).ok()) + .expect("Python bridge task must have a callback context") +} + +fn shutdown_error() -> PyErr { + PyRuntimeError::new_err("Temporal runtime is shutting down") +} + +pub(crate) fn with_python(callback: F) -> PyResult +where + F: for<'py> FnOnce(Python<'py>) -> PyResult, +{ + let callbacks = current_callbacks(); + let _guard = callbacks.enter_completion().ok_or_else(shutdown_error)?; + Python::attach(callback) +} + +#[pyclass] +struct CallbackShutdown; + +#[pymethods] +impl CallbackShutdown { + fn __call__(&self, py: Python<'_>) { + let callbacks = CALLBACK_REGISTRY.begin_shutdown(); + // A callback may be waiting to reacquire the GIL after notifying its + // event loop. Let it return before CPython starts finalizing. + py.detach(move || { + for callback in callbacks { + callback.wait(); + } + }); + } +} + +pub(crate) fn register_callback_shutdown(py: Python<'_>) -> PyResult<()> { + py.import("atexit")? + .call_method1("register", (Py::new(py, CallbackShutdown)?,))?; + Ok(()) +} + #[pyclass] pub struct RuntimeRef { pub(crate) runtime: Runtime, @@ -34,6 +99,7 @@ pub struct RuntimeRef { pub(crate) struct Runtime { pub(crate) pid: u32, pub(crate) core: Arc, + pub(crate) callbacks: Arc, metrics_call_buffer: Option>>, log_forwarder_handle: Option>>, } @@ -118,6 +184,7 @@ const FORWARD_LOG_BUFFER_SIZE: usize = 2048; const FORWARD_LOG_MAX_FREQ_MS: u64 = 10; pub fn init_runtime(options: RuntimeOptions) -> PyResult { + let callbacks = CALLBACK_REGISTRY.register().ok_or_else(shutdown_error)?; let RuntimeOptions { telemetry: TelemetryConfig { logging, metrics }, worker_heartbeat_interval_millis, @@ -196,6 +263,7 @@ pub fn init_runtime(options: RuntimeOptions) -> PyResult { } // Start log forwarding if needed + let log_callbacks = callbacks.clone(); let log_forwarder_handle = log_forwarding.map(|(stream, callback)| { Arc::new(core.tokio_handle().spawn(async move { let mut stream = std::pin::pin!(stream.chunks_timeout( @@ -210,6 +278,9 @@ pub fn init_runtime(options: RuntimeOptions) -> PyResult { .collect::>(); // We silently swallow errors here because logging them could // cause a bad loop and we don't want to assume console presence + let Some(_guard) = log_callbacks.enter() else { + break; + }; let _ = Python::attach(|py| callback.call1(py, (entries,))); } })) @@ -219,6 +290,7 @@ pub fn init_runtime(options: RuntimeOptions) -> PyResult { runtime: Runtime { pid: std::process::id(), core: Arc::new(CoreRuntimeHandle(Some(core))), + callbacks, metrics_call_buffer, log_forwarder_handle, }, @@ -239,7 +311,12 @@ impl Runtime { F: Future> + Send + 'static, T: for<'py> IntoPyObject<'py> + Send + 'static, { + self.assert_same_process("use runtime")?; + if CALLBACK_REGISTRY.is_closing() { + return Err(shutdown_error()); + } let _guard = self.core.tokio_handle().enter(); + let _callbacks = CallbackContext::enter(self.callbacks.clone()); pyo3_async_runtimes::generic::future_into_py::(py, fut) } @@ -440,6 +517,7 @@ pub(crate) struct TokioRuntime; tokio::task_local! { static TASK_LOCALS: std::cell::OnceCell; + static CALLBACK_TASK: Arc; } impl pyo3_async_runtimes::generic::Runtime for TokioRuntime { @@ -450,14 +528,22 @@ impl pyo3_async_runtimes::generic::Runtime for TokioRuntime { where F: Future + Send + 'static, { - tokio::runtime::Handle::current().spawn(fut) + let callbacks = current_callbacks(); + tokio::runtime::Handle::current().spawn(CALLBACK_TASK.scope(callbacks, fut)) } fn spawn_blocking(f: F) -> Self::JoinHandle where F: FnOnce() + Send + 'static, { - tokio::runtime::Handle::current().spawn_blocking(f) + let callbacks = current_callbacks(); + tokio::runtime::Handle::current().spawn_blocking(move || { + let Some(_guard) = callbacks.enter_completion() else { + return; + }; + let _context = CallbackContext::enter(callbacks); + f(); + }) } } diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index e74698e944..aae3d2dcdb 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -25,6 +25,7 @@ use tokio::sync::mpsc::{channel, Sender}; use tokio_stream::wrappers::ReceiverStream; use tracing::error; +use crate::callbacks::Callbacks; use crate::client; use crate::runtime; @@ -311,6 +312,7 @@ pub struct CustomSlotSupplier { struct CustomSlotSupplierOfType { inner: Arc>, event_loop_task_locals: Arc>, + callbacks: Arc, _phantom: PhantomData, } @@ -344,16 +346,23 @@ impl CreatedTaskForSlotCallback { struct TaskCanceller { stored_task: Arc>>, + callbacks: Arc, } impl TaskCanceller { - fn new(stored_task: Arc>>) -> Self { - TaskCanceller { stored_task } + fn new(stored_task: Arc>>, callbacks: Arc) -> Self { + TaskCanceller { + stored_task, + callbacks, + } } } impl Drop for TaskCanceller { fn drop(&mut self) { + let Some(_guard) = self.callbacks.enter_completion() else { + return; + }; if let Some(task) = self.stored_task.get() { Python::attach(|py| { task.call_method0(py, "cancel") @@ -370,22 +379,28 @@ impl SlotSupplierTrait for CustomSlotSupplierOfType< async fn reserve_slot(&self, ctx: &dyn SlotReservationContext) -> SlotSupplierPermit { loop { let stored_task = Arc::new(OnceLock::new()); - let _task_canceller = TaskCanceller::new(stored_task.clone()); - let pypermit = match Python::attach(|py| { - let py_obj = self.inner.bind(py); - let called = py_obj.call_method1( - "reserve_slot", - ( - SlotReserveCtx::from_ctx(SK::kind(), ctx), - CreatedTaskForSlotCallback { stored_task }, - ), - )?; - let tl = self - .event_loop_task_locals - .get() - .expect("task locals must be set"); - pyo3_async_runtimes::into_future_with_locals(tl, called) - }) { + let _task_canceller = TaskCanceller::new(stored_task.clone(), self.callbacks.clone()); + let pypermit = { + let Some(_guard) = self.callbacks.enter_completion() else { + return std::future::pending().await; + }; + Python::attach(|py| { + let py_obj = self.inner.bind(py); + let called = py_obj.call_method1( + "reserve_slot", + ( + SlotReserveCtx::from_ctx(SK::kind(), ctx), + CreatedTaskForSlotCallback { stored_task }, + ), + )?; + let tl = self + .event_loop_task_locals + .get() + .expect("task locals must be set"); + pyo3_async_runtimes::into_future_with_locals(tl, called) + }) + }; + let pypermit = match pypermit { Ok(f) => f, Err(e) => { error!("Unexpected error in custom slot supplier `reserve_slot`: {e}"); @@ -406,6 +421,7 @@ impl SlotSupplierTrait for CustomSlotSupplierOfType< } fn try_reserve_slot(&self, ctx: &dyn SlotReservationContext) -> Option { + let _guard = self.callbacks.enter_completion()?; Python::attach(|py| { let py_obj = self.inner.bind(py); let pa = py_obj.call_method1( @@ -427,6 +443,9 @@ impl SlotSupplierTrait for CustomSlotSupplierOfType< } fn mark_slot_used(&self, ctx: &dyn SlotMarkUsedContext) { + let Some(_guard) = self.callbacks.enter_completion() else { + return; + }; if let Err(e) = Python::attach(|py| { let permit = ctx .permit() @@ -448,6 +467,9 @@ impl SlotSupplierTrait for CustomSlotSupplierOfType< } fn release_slot(&self, ctx: &dyn SlotReleaseContext) { + let Some(_guard) = self.callbacks.enter_completion() else { + return; + }; if let Err(e) = Python::attach(|py| { let permit = ctx .permit() @@ -495,7 +517,11 @@ pub fn new_worker( enter_sync!(runtime_ref.runtime); runtime_ref.runtime.assert_same_process("create worker")?; let event_loop_task_locals = Arc::new(OnceLock::new()); - let config = convert_worker_config(config, event_loop_task_locals.clone())?; + let config = convert_worker_config( + config, + event_loop_task_locals.clone(), + runtime_ref.runtime.callbacks.clone(), + )?; let worker = temporalio_sdk_core::init_worker( &runtime_ref.runtime.core, config, @@ -519,7 +545,11 @@ pub fn new_replay_worker<'a>( .runtime .assert_same_process("create replay worker")?; let event_loop_task_locals = Arc::new(OnceLock::new()); - let config = convert_worker_config(config, event_loop_task_locals.clone())?; + let config = convert_worker_config( + config, + event_loop_task_locals.clone(), + runtime_ref.runtime.callbacks.clone(), + )?; let (history_pusher, stream) = HistoryPusher::new(runtime_ref.runtime.clone()); let worker = WorkerRef { worker: Some(Arc::new( @@ -715,8 +745,9 @@ impl WorkerRef { fn convert_worker_config( conf: WorkerConfig, task_locals: Arc>, + callbacks: Arc, ) -> PyResult { - let converted_tuner = convert_tuner_holder(conf.tuner, task_locals)?; + let converted_tuner = convert_tuner_holder(conf.tuner, task_locals, callbacks)?; let converted_versioning_strategy = convert_versioning_strategy(conf.versioning_strategy); temporalio_sdk_core::WorkerConfig::builder() .namespace(conf.namespace) @@ -787,6 +818,7 @@ fn convert_worker_config( fn convert_tuner_holder( holder: TunerHolder, task_locals: Arc>, + callbacks: Arc, ) -> PyResult { // Verify all resource-based options are the same if any are set let maybe_wf_resource_opts = @@ -844,18 +876,22 @@ fn convert_tuner_holder( .workflow_slot_options(convert_slot_supplier( holder.workflow_slot_supplier, task_locals.clone(), + callbacks.clone(), )?) .activity_slot_options(convert_slot_supplier( holder.activity_slot_supplier, task_locals.clone(), + callbacks.clone(), )?) .local_activity_slot_options(convert_slot_supplier( holder.local_activity_slot_supplier, task_locals.clone(), + callbacks.clone(), )?) .nexus_slot_options(convert_slot_supplier( holder.nexus_slot_supplier, task_locals, + callbacks, )?) .build() .map_err(|e| PyValueError::new_err(format!("Invalid tuner holder options: {e}")))? @@ -866,6 +902,7 @@ fn convert_tuner_holder( fn convert_slot_supplier( supplier: SlotSupplier, task_locals: Arc>, + callbacks: Arc, ) -> PyResult> { Ok(match supplier { SlotSupplier::FixedSize(fs) => temporalio_sdk_core::SlotSupplierOptions::FixedSize { @@ -884,6 +921,7 @@ fn convert_slot_supplier( > { inner: cs.inner, event_loop_task_locals: task_locals, + callbacks, _phantom: PhantomData, })) } diff --git a/temporalio/runtime.py b/temporalio/runtime.py index d20b44ede6..d110cd429f 100644 --- a/temporalio/runtime.py +++ b/temporalio/runtime.py @@ -68,6 +68,12 @@ class Runtime: :py:meth:`prevent_default` and :py:meth:`set_default` to ensure each fork creates it's own runtime. + During normal interpreter exit, the SDK stops admitting native callbacks + into Python and waits for callbacks already executing to return before + Python finalizes. This does not replace shutting down workers or awaiting + application tasks. Creating runtimes or starting asynchronous SDK operations + after the SDK's interpreter-exit handler has run raises ``RuntimeError``. + """ @classmethod diff --git a/tests/test_runtime_shutdown.py b/tests/test_runtime_shutdown.py new file mode 100644 index 0000000000..60bd7ddeca --- /dev/null +++ b/tests/test_runtime_shutdown.py @@ -0,0 +1,201 @@ +import socket +import subprocess +import sys +import uuid + +import pytest + +import temporalio.testing + +# The registration wrapper only exposes the boundary of the real SDK exit hook. +# It lets the parent release a real native callback without timing-based sleeps. +_CHILD = r""" +import asyncio +import atexit +import contextlib +import logging +import socket +import sys +import threading + +connection = socket.create_connection(("127.0.0.1", int(sys.argv[1]))) +send_lock = threading.Lock() + +def send(message): + with send_lock: + connection.sendall((message + "\n").encode()) + +def user_finalizer(): + try: + temporalio.runtime.Runtime(telemetry=temporalio.runtime.TelemetryConfig()) + except RuntimeError: + send("late-runtime-rejected") + else: + raise AssertionError("SDK accepted a runtime after shutdown") + send("python-finalizer") + +atexit.register(user_finalizer) +register = atexit.register + +def instrument_registration(callback, *args, **kwargs): + def wrapped(): + send("shutdown-entered") + callback(*args, **kwargs) + send("shutdown-finished") + register(wrapped) + return callback + +atexit.register = instrument_registration +try: + import temporalio.runtime + import temporalio.client + import temporalio.api.enums.v1 + import temporalio.api.taskqueue.v1 + import temporalio.api.workflowservice.v1 +finally: + atexit.register = register + +started = threading.Event() + +def hold_callback(): + send("callback-started") + started.set() + if connection.recv(1) != b"r": + raise AssertionError("Callback was not released") + send("callback-finished") + +class BlockingHandler(logging.Handler): + def emit(self, record): + if "shutdown-regression-log" in record.getMessage(): + if sys.argv[2] == "nested-rpc": + asyncio.run(nested_poll()) + else: + hold_callback() + +class ControlledLoop(asyncio.SelectorEventLoop): + held = False + + def call_soon_threadsafe(self, callback, *args, context=None): + handle = super().call_soon_threadsafe(callback, *args, context=context) + if threading.current_thread() is not threading.main_thread() and not self.held: + self.held = True + hold_callback() + return handle + +async def connect(): + await temporalio.client.Client.connect(sys.argv[3]) + if not started.wait(10): + raise AssertionError("Native callback did not start") + +async def start_poll(): + client = await temporalio.client.Client.connect(sys.argv[3]) + request = temporalio.api.workflowservice.v1.PollWorkflowTaskQueueRequest( + namespace="default", + task_queue=temporalio.api.taskqueue.v1.TaskQueue( + name="shutdown-poll-" + sys.argv[1], + kind=temporalio.api.enums.v1.TaskQueueKind.TASK_QUEUE_KIND_NORMAL, + ), + identity="shutdown-regression", + ) + task = asyncio.create_task(client.workflow_service.poll_workflow_task_queue(request)) + ready = asyncio.get_running_loop().create_future() + asyncio.get_running_loop().call_soon(ready.set_result, None) + await ready + return task + +async def nested_poll(): + # Connect before announcing the outstanding poll. This is a different + # runtime from the log forwarder whose callback is awaiting this operation. + task = await start_poll() + send("callback-started") + started.set() + await task + send("callback-finished") + +async def cancel_poll(): + task = await start_poll() + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + +mode = sys.argv[2] +if mode in ("logging", "nested-rpc"): + logger = logging.getLogger("runtime-shutdown") + logger.setLevel(logging.INFO) + logger.addHandler(BlockingHandler()) + runtime = temporalio.runtime.Runtime( + telemetry=temporalio.runtime.TelemetryConfig( + logging=temporalio.runtime.LoggingConfig( + filter="INFO", + forwarding=temporalio.runtime.LogForwardingConfig(logger=logger), + ) + ) + ) + runtime._core_runtime.write_test_info_log("shutdown-regression-log", "") + if not started.wait(10): + raise AssertionError("Native log callback did not start") +elif mode == "rpc": + loop = ControlledLoop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(connect()) + loop.run_until_complete(loop.shutdown_asyncgens()) + finally: + loop.close() +else: + asyncio.run(cancel_poll()) +send("main-returned") +""" + + +@pytest.mark.requires_local_server +@pytest.mark.parametrize("mode", ["logging", "rpc", "cancelled-poll", "nested-rpc"]) +async def test_runtime_shutdown( + env: temporalio.testing.WorkflowEnvironment, mode: str +) -> None: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + listener.settimeout(20) + process = subprocess.Popen( + [ + sys.executable, + "-c", + _CHILD, + str(listener.getsockname()[1]), + mode, + env.client.service_client.config.target_host, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + workflow_handle = None + try: + with listener.accept()[0] as connection: + connection.settimeout(20) + with connection.makefile("rb") as messages: + if mode != "cancelled-poll": + assert messages.readline() == b"callback-started\n" + assert messages.readline() == b"main-returned\n" + assert messages.readline() == b"shutdown-entered\n" + if mode == "nested-rpc": + workflow_handle = await env.client.start_workflow( + "ShutdownPollWorkflow", + id=f"shutdown-{uuid.uuid4()}", + task_queue=f"shutdown-poll-{listener.getsockname()[1]}", + ) + assert messages.readline() == b"callback-finished\n" + elif mode != "cancelled-poll": + connection.sendall(b"r") + assert messages.readline() == b"callback-finished\n" + assert messages.readline() == b"shutdown-finished\n" + assert messages.readline() == b"late-runtime-rejected\n" + assert messages.readline() == b"python-finalizer\n" + stdout, stderr = process.communicate(timeout=20) + assert process.returncode == 0, (stdout, stderr) + finally: + if process.poll() is None: + process.kill() + process.communicate(timeout=10) + if workflow_handle is not None: + await workflow_handle.terminate()