From c87eee8bf626934ae9ad337ec9fa561604d231ca Mon Sep 17 00:00:00 2001 From: Marco Giacalone Date: Mon, 26 Oct 2026 12:59:18 +0100 Subject: [PATCH 1/3] Fix correct TBB pool closure in dpl-eventgen with TTL --- .../include/Generators/GeneratorHybrid.h | 1 + Generators/src/GeneratorHybrid.cxx | 24 ++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Generators/include/Generators/GeneratorHybrid.h b/Generators/include/Generators/GeneratorHybrid.h index a47ef0b7cce98..ee53780259b5e 100644 --- a/Generators/include/Generators/GeneratorHybrid.h +++ b/Generators/include/Generators/GeneratorHybrid.h @@ -64,6 +64,7 @@ class GeneratorHybrid : public Generator Bool_t generateEvent() override; Bool_t importParticles() override; void updateHeader(o2::dataformats::MCEventHeader* eventHeader) override; + void stop() override; // end-of-stream hook that closes the TBB task arena Bool_t parseJSON(const std::string& path); Bool_t confSetter(const auto& gen); diff --git a/Generators/src/GeneratorHybrid.cxx b/Generators/src/GeneratorHybrid.cxx index 29ea6e6737a8c..994b70822f39a 100644 --- a/Generators/src/GeneratorHybrid.cxx +++ b/Generators/src/GeneratorHybrid.cxx @@ -10,6 +10,7 @@ // or submit itself to any jurisdiction. #include "Generators/GeneratorHybrid.h" +#include #include #include #include @@ -191,7 +192,24 @@ GeneratorHybrid::GeneratorHybrid(const std::string& inputgens) GeneratorHybrid::~GeneratorHybrid() { LOG(info) << "Destructor of generator hybrid called"; + // stop() is the designated shutdown hook (called via GeneratorService::stopGenerators), + // normally running end-of-stream, in which case this is a no-op. It is not reached on every exit path, e.g. when the + // driver quits before the event budget or the TTL is up, so it is repeated here. + stop(); +} + +void GeneratorHybrid::stop() +{ mStopFlag = true; + // The worker lambdas reference mInputTaskQueue, mResultQueue and mTaskArena, which are + // destroyed together with this object, so they must not be left running. + // We wait for the thread to finish, which will happen once the worker lambdas notice mStopFlag. + if (mTBBTaskPoolRunner.joinable()) { + auto t0 = std::chrono::steady_clock::now(); + mTBBTaskPoolRunner.join(); + auto ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + LOG(info) << "HybridGen: stop() joined TBB task pool after " << ms << " ms"; + } } Bool_t GeneratorHybrid::Init() @@ -320,7 +338,7 @@ Bool_t GeneratorHybrid::Init() // } } bool isTriggered = false; - while (!isTriggered) { + while (!isTriggered && !mStopFlag) { generator->clearParticles(); generator->generateEvent(); generator->importParticles(); @@ -355,8 +373,8 @@ Bool_t GeneratorHybrid::Init() // let the TBB task system run in it's own thread mTBBTaskPoolRunner = std::thread([this, worker_function]() { mTaskArena.execute([&]() { tbb::parallel_for(0, mTaskArena.max_concurrency(), [&](int) { worker_function(); }); }); }); - mTBBTaskPoolRunner.detach(); // detaching because we don't like to wait on the thread to finish - // some of the generators might still be generating when we are done + // NOTE: thread deliberately NOT detached. We never wait for it during the run, but the + // pool must be joined in the destructor before the queues and the arena are destroyed. // let's also push initial generation tasks for each event generator for (size_t genindex = 0; genindex < gens.size(); ++genindex) { From e4a587631a99ed900e7c9f69418320d28136fc65 Mon Sep 17 00:00:00 2001 From: ALICE Action Bot Date: Mon, 26 Oct 2026 12:11:20 +0000 Subject: [PATCH 2/3] Please consider the following formatting changes --- Generators/src/GeneratorHybrid.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Generators/src/GeneratorHybrid.cxx b/Generators/src/GeneratorHybrid.cxx index 994b70822f39a..9f859501457c3 100644 --- a/Generators/src/GeneratorHybrid.cxx +++ b/Generators/src/GeneratorHybrid.cxx @@ -192,7 +192,7 @@ GeneratorHybrid::GeneratorHybrid(const std::string& inputgens) GeneratorHybrid::~GeneratorHybrid() { LOG(info) << "Destructor of generator hybrid called"; - // stop() is the designated shutdown hook (called via GeneratorService::stopGenerators), + // stop() is the designated shutdown hook (called via GeneratorService::stopGenerators), // normally running end-of-stream, in which case this is a no-op. It is not reached on every exit path, e.g. when the // driver quits before the event budget or the TTL is up, so it is repeated here. stop(); From 360bc73b3e1fa0bf85782c52d430bf5ec83ce4a0 Mon Sep 17 00:00:00 2001 From: Marco Giacalone Date: Tue, 27 Oct 2026 09:58:56 +0100 Subject: [PATCH 3/3] Add Init protection + stop() subgens + protection against nextEvent --- .../include/Generators/GeneratorHybrid.h | 2 +- Generators/src/GeneratorHybrid.cxx | 25 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Generators/include/Generators/GeneratorHybrid.h b/Generators/include/Generators/GeneratorHybrid.h index ee53780259b5e..061eed088e5d3 100644 --- a/Generators/include/Generators/GeneratorHybrid.h +++ b/Generators/include/Generators/GeneratorHybrid.h @@ -64,7 +64,7 @@ class GeneratorHybrid : public Generator Bool_t generateEvent() override; Bool_t importParticles() override; void updateHeader(o2::dataformats::MCEventHeader* eventHeader) override; - void stop() override; // end-of-stream hook that closes the TBB task arena + void stop() override; // end-of-stream hook that joins the TBB task pool Bool_t parseJSON(const std::string& path); Bool_t confSetter(const auto& gen); diff --git a/Generators/src/GeneratorHybrid.cxx b/Generators/src/GeneratorHybrid.cxx index 9f859501457c3..1afb9446cbe05 100644 --- a/Generators/src/GeneratorHybrid.cxx +++ b/Generators/src/GeneratorHybrid.cxx @@ -210,10 +210,20 @@ void GeneratorHybrid::stop() auto ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); LOG(info) << "HybridGen: stop() joined TBB task pool after " << ms << " ms"; } + // Forward the hook to the sub-generators + for (auto& gen : gens) { + if (gen) { + gen->stop(); + } + } } Bool_t GeneratorHybrid::Init() { + // protection against additional Init() which would assign over a joinable mTBBTaskPoolRunner, which calls std::terminate + if (mIsInitialized) { + return true; + } // init all sub-gens int count = 0; for (auto& gen : mGens) { @@ -474,6 +484,9 @@ bool GeneratorHybrid::importParticles() // this is important when using event pools where the full header information is forwarded from the generator // otherwise some events might have mixed header information from different generators mMCEventHeader.Reset(); + // Once the event budget is exhausted there is no further event to prepare, and re-queueing a + // sub-generator would start an event nobody consumes and that stop() then has to wait for + const bool queueNextEvent = getTotalNEvents() == 0 || mEventCounter + 1 < getTotalNEvents(); if (mCocktailMode) { // in cocktail mode we need to merge the particles from the different generators for (auto subIndex : subGenIndex) { @@ -504,8 +517,10 @@ bool GeneratorHybrid::importParticles() gens[subIndex]->updateHeader(&mMCEventHeader); mHeaderGeneratorIndex = subIndex; // store index of generator updating the header } - mInputTaskQueue.push(subIndex); - mTasksStarted++; + if (queueNextEvent) { + mInputTaskQueue.push(subIndex); + mTasksStarted++; + } } } else { if (!mIsHyperloop) { @@ -528,8 +543,10 @@ bool GeneratorHybrid::importParticles() // fetch the event Header information from the underlying generator gens[genIndex]->updateHeader(&mMCEventHeader); mHeaderGeneratorIndex = genIndex; // store index of generator updating the header - mInputTaskQueue.push(genIndex); - mTasksStarted++; + if (queueNextEvent) { + mInputTaskQueue.push(genIndex); + mTasksStarted++; + } } mseqCounter++;