From dc63e2dcdba438a8e220c31e7e4f4573e791a135 Mon Sep 17 00:00:00 2001 From: Erik Salter Date: Wed, 4 Nov 2026 15:04:28 -0500 Subject: [PATCH] fix: Kill a Python backend stub that outlives a failed instance launch When an instance failed to launch after its stub had started, for example after missed health updates, the server waited indefinitely for a stub that considered itself initialized, so the model load and server startup hung and the original error was never logged. Wait up to the stub timeout for the stub to exit, then kill it and log a warning. --- src/stub_launcher.cc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/stub_launcher.cc b/src/stub_launcher.cc index 4bc8979c..4f8c1557 100644 --- a/src/stub_launcher.cc +++ b/src/stub_launcher.cc @@ -539,12 +539,25 @@ StubLauncher::Launch() stub_message_queue_->Push(DUMMY_MESSAGE); } - // If the model is not initialized, wait for the stub process to exit. + // If the model is not initialized, make sure the stub process exits. A + // stub that reported an initialization error normally exits on its own. A + // stub the server gave up on, for example after missed health updates, + // may still consider itself initialized and would wait for commands + // forever, so a stub that has not exited when the stub timeout expires is + // killed. if (!is_initialized_) { stub_message_queue_.reset(); parent_message_queue_.reset(); memory_manager_.reset(); - WaitForStubProcess(); + if (!WaitForStubProcessWithTimeout(stub_timeout_seconds_)) { + LOG_MESSAGE( + TRITONSERVER_LOG_WARN, + (std::string("Python backend stub '") + model_instance_name_ + + "' did not exit within " + std::to_string(stub_timeout_seconds_) + + " seconds after its initialization failed; killing it.") + .c_str()); + KillStubProcess(); + } } });