From b8a825788e9399084b93c488457ea92205b5f2d6 Mon Sep 17 00:00:00 2001 From: flipflip Date: Mon, 5 Oct 2026 16:58:31 +0200 Subject: [PATCH 1/6] fpltool extract: parallel video decoding, optional hw (VA-API) decoding fpltool: - Decode video in parallel when extracting camera data to a ROS bag. Each group of pictures (GOP) is decoded by its own decoder in its own thread. All messages go through a queue and are written in the order they are in the log. - Add -j, --jobs option for the number of parallel video decoders (default: number of CPUs - 2, at least 1) - Add -a, --accel option to select the hw acceleration of the video decoders: SW, HW or AUTO (default: SW) - Write the last frame of each camera, which was decoded but not written - Abort the extraction if a GOP has more than 100 frames fpsdk_common: - video: make hardware (VA-API) decoding work. The filter graph is set up from the format of the first decoded frame. Hardware frames are scaled on the GPU, downloaded as yuv420p and converted to the output pixel format in software. HwAccel::HW fails if the decoder cannot use the hardware, HwAccel::AUTO falls back to software decoding. - video: document hardware vs. software decoding performance in video.hpp - fpl: fix rec_time in the CamData info string Docker, devcontainer: - Build FFmpeg with the h264 and hevc VA-API hwaccels - Pass /dev/dri to the trixie-dev container - Install the time utility - Keep persistent devcontainer data in .devcontainer/.persist, created by the new .devcontainer/helper.sh Co-Authored-By: Claude Opus 5.5 --- docker/docker-compose.yaml | 2 + fpsdk_apps/fpltool/fpltool_extract.cpp | 349 ++++++++++++++++---- fpsdk_apps/fpltool/fpltool_extract.hpp | 66 +++- fpsdk_apps/fpltool/fpltool_opts.hpp | 40 ++- fpsdk_common/include/fpsdk_common/video.hpp | 24 ++ fpsdk_common/src/fpl.cpp | 2 +- fpsdk_common/src/video.cpp | 66 ++-- 7 files changed, 445 insertions(+), 104 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 7eeedd6..2c2fb48 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -50,6 +50,8 @@ services: - ../.devcontainer/.persist/.local:/home/fpsdk/.local # Workspace - ../.:/home/fpsdk/fixposition-sdk + devices: + - /dev/dri cap_add: - SYS_PTRACE # Set kernel.yama.ptrace_scope = 0 in /etc/sysctl.d/10-ptrace.conf hostname: fpsdk-trixie-dev-${USER} diff --git a/fpsdk_apps/fpltool/fpltool_extract.cpp b/fpsdk_apps/fpltool/fpltool_extract.cpp index ff6ec85..526cb5f 100644 --- a/fpsdk_apps/fpltool/fpltool_extract.cpp +++ b/fpsdk_apps/fpltool/fpltool_extract.cpp @@ -63,6 +63,9 @@ using namespace fpsdk::common::video; using namespace fpsdk::common::ros2; #endif +// Maximum number of frames in a group of pictures (GOP), abort extraction if larger +static constexpr std::size_t MAX_GOP_SIZE = 100; + // --------------------------------------------------------------------------------------------------------------------- FplToolExtract::FplToolExtract(FplToolOptions& opts) /* clang-format off */ : @@ -143,6 +146,11 @@ bool FplToolExtract::Run() NOTICE("Extracting to %s", output_bag.c_str()); } +#if FPSDK_USE_FFMPEG + max_gops_pending_ = opts_.jobs_; + DEBUG("max_gops_pending=%" PRIuMAX, max_gops_pending_); +#endif + // Handle SIGINT (C-c) to abort nicely SigIntHelper sig_int; @@ -153,7 +161,6 @@ bool FplToolExtract::Run() FplMessage fpl_msg; bool do_extract = true; uint32_t time_into_log = 0; - std::size_t errors = 0; while (!sig_int.ShouldAbort() && fpl_reader.Next(fpl_msg) && ok) { // Report progress if (opts_.progress_ > 0) { @@ -168,37 +175,43 @@ bool FplToolExtract::Run() break; } - // Process message - const auto log_type = fpl_msg.PayloadType(); - ProcRes res = ProcRes::OK; - switch (log_type) { // clang-format off - case FplType::LOGSTATUS: res = ProcessLogStatus(fpl_msg, do_extract, time_into_log); break; - case FplType::LOGMETA: res = ProcessLogMeta(fpl_msg, do_extract); break; - case FplType::STREAMMSG: res = ProcessStreamMsg(fpl_msg, do_extract); break; - case FplType::ROSMSGDEF: res = ProcessRosMsgDef(fpl_msg, do_extract); break; - case FplType::ROSMSGBIN: res = ProcessRosMsgBin(fpl_msg, do_extract); break; - case FplType::FILEDUMP: res = ProcessFileDump(fpl_msg, do_extract); break; - case FplType::CAMDATA: res = ProcessCamData(fpl_msg, do_extract); break; + // Queue message for processing + QueueItem item; + item.do_extract_ = do_extract; + switch (fpl_msg.PayloadType()) { + case FplType::LOGSTATUS: { + const LogStatus logstatus(fpl_msg); + if (logstatus.valid_) { + time_into_log = logstatus.log_duration_; + } + break; + } + case FplType::CAMDATA: +#if FPSDK_USE_FFMPEG + if (do_extract && do_cam_ && do_ros_) { + ok = QueueCamData(fpl_msg, item.job_, item.job_ix_); + } +#endif + break; + case FplType::LOGMETA: + case FplType::STREAMMSG: + case FplType::ROSMSGDEF: + case FplType::ROSMSGBIN: + case FplType::FILEDUMP: + break; case FplType::BLOB: case FplType::INT_D: case FplType::INT_F: case FplType::INT_X: - case FplType::UNSPECIFIED: break; - } // clang-format on + case FplType::UNSPECIFIED: + continue; + } + item.fpl_msg_ = std::move(fpl_msg); + queue_.push_back(std::move(item)); - switch (res) { - case ProcRes::OK: - break; - case ProcRes::ERROR: - if (errors >= 100) { - WARNING("Too many errors, giving up"); - ok = false; - } - break; - case ProcRes::FATAL: - WARNING("Giving up"); - ok = false; - break; + // Process what we can + if (ok) { + ok = ProcessQueue(QueueWait::NONE); } } @@ -207,6 +220,20 @@ bool FplToolExtract::Run() ok = false; } + // Process what is left +#if FPSDK_USE_FFMPEG + for (auto& entry : gop_jobs_) { + entry.second->Close(); + } + gop_jobs_.clear(); +#endif + if (ok) { + ok = ProcessQueue(QueueWait::ALL); + } + if (!ok) { + AbortQueue(); + } + // Close output files CloseAll(ok); if (do_ros_) { @@ -230,8 +257,77 @@ bool FplToolExtract::Run() // --------------------------------------------------------------------------------------------------------------------- -FplToolExtract::ProcRes FplToolExtract::ProcessLogStatus( - const common::fpl::FplMessage& fpl_msg, const bool do_extract, uint32_t& time_into_log) +bool FplToolExtract::ProcessQueue(const QueueWait wait) +{ + bool may_wait = (wait != QueueWait::NONE); + while (!queue_.empty()) { + auto& item = queue_.front(); + +#if FPSDK_USE_FFMPEG + // Camera frames that are not decoded yet hold back everything + if (item.job_ && !item.job_->IsDecoded(item.job_ix_)) { + if (!may_wait) { + break; + } + may_wait = (wait == QueueWait::ALL); + } +#endif + + const auto& fpl_msg = item.fpl_msg_; + const bool do_extract = item.do_extract_; + ProcRes res = ProcRes::OK; + switch (fpl_msg.PayloadType()) { // clang-format off + case FplType::LOGSTATUS: res = ProcessLogStatus(fpl_msg, do_extract); break; + case FplType::LOGMETA: res = ProcessLogMeta(fpl_msg, do_extract); break; + case FplType::STREAMMSG: res = ProcessStreamMsg(fpl_msg, do_extract); break; + case FplType::ROSMSGDEF: res = ProcessRosMsgDef(fpl_msg, do_extract); break; + case FplType::ROSMSGBIN: res = ProcessRosMsgBin(fpl_msg, do_extract); break; + case FplType::FILEDUMP: res = ProcessFileDump(fpl_msg, do_extract); break; + case FplType::CAMDATA: res = ProcessCamData(item); break; + case FplType::BLOB: + case FplType::INT_D: + case FplType::INT_F: + case FplType::INT_X: + case FplType::UNSPECIFIED: break; + } // clang-format on + + queue_.pop_front(); + + switch (res) { + case ProcRes::OK: + break; + case ProcRes::ERROR: + if (errors_ >= 100) { + WARNING("Too many errors, giving up"); + return false; + } + break; + case ProcRes::FATAL: + WARNING("Giving up"); + return false; + } + } + return true; +} + +// --------------------------------------------------------------------------------------------------------------------- + +void FplToolExtract::AbortQueue() +{ +#if FPSDK_USE_FFMPEG + for (auto& item : queue_) { + if (item.job_) { + item.job_->Abort(); + } + } + gops_pending_ = 0; +#endif + queue_.clear(); +} + +// --------------------------------------------------------------------------------------------------------------------- + +FplToolExtract::ProcRes FplToolExtract::ProcessLogStatus(const common::fpl::FplMessage& fpl_msg, const bool do_extract) { const LogStatus logstatus(fpl_msg); if (!logstatus.valid_) { @@ -240,8 +336,6 @@ FplToolExtract::ProcRes FplToolExtract::ProcessLogStatus( } TRACE("LOGSTATUS %s", logstatus.info_.c_str()); - time_into_log = logstatus.log_duration_; - if (do_extract && do_jsonl_ && !WriteJson(jsonl_name_, logstatus)) { return ProcRes::FATAL; } @@ -451,13 +545,13 @@ FplToolExtract::ProcRes FplToolExtract::ProcessFileDump(const FplMessage& fpl_ms // --------------------------------------------------------------------------------------------------------------------- -FplToolExtract::ProcRes FplToolExtract::ProcessCamData(const FplMessage& fpl_msg, const bool do_extract) +FplToolExtract::ProcRes FplToolExtract::ProcessCamData(const QueueItem& item) { - if (!do_extract) { + if (!item.do_extract_) { return ProcRes::OK; } - fpsdk::common::fpl::CamData camdata(fpl_msg); + fpsdk::common::fpl::CamData camdata(item.fpl_msg_); if (!camdata.valid_) { WARNING("Invalid CAMDATA"); return ProcRes::ERROR; @@ -471,13 +565,39 @@ FplToolExtract::ProcRes FplToolExtract::ProcessCamData(const FplMessage& fpl_msg return ProcRes::FATAL; } +#if FPSDK_USE_FFMPEG + // Frames that were decoded (and stringified to JSON) in a GopJob + if (item.job_) { + auto& job = *item.job_; + auto decdata = job.GetDecoded(item.job_ix_); + job.num_done_++; + if (job.closed_ && (job.num_done_ == job.num_frames_)) { + gops_pending_--; + } + if (do_jsonl_ && do_cam_ && !WriteJson(jsonl_name_, decdata.json_)) { + return ProcRes::FATAL; + } + return ProcessAsyncDecData(std::move(decdata)); + } +#endif + if (do_jsonl_ && do_cam_ && !WriteJson(jsonl_name_, camdata)) { return ProcRes::FATAL; } - ProcRes res = ProcRes::OK; + return ProcRes::OK; +} + +// --------------------------------------------------------------------------------------------------------------------- #if FPSDK_USE_FFMPEG +bool FplToolExtract::QueueCamData(const FplMessage& fpl_msg, std::shared_ptr& job, std::size_t& job_ix) +{ + fpsdk::common::fpl::CamData camdata(fpl_msg); + if (!camdata.valid_) { + return true; // ProcessCamData() will complain + } + VideoCodec codec = VideoCodec::UNSPECIFIED; switch (camdata.fmt_) { // clang-format off case CamDataFmt::H264_NAL: codec = VideoCodec::H264; break; @@ -489,46 +609,138 @@ FplToolExtract::ProcRes FplToolExtract::ProcessCamData(const FplMessage& fpl_msg case CamDataFmt::NV12: case CamDataFmt::RGB24: break; } // clang-format on - while (do_cam_ && do_ros_ && (codec != VideoCodec::UNSPECIFIED)) { - auto entry = decoder_states_.find(name); - if (entry == decoder_states_.end()) { - // We can only start decoding from the first I-frame onwards - if (camdata.frm_ != CamDataFrm::I_FRAME) { - break; + if (codec == VideoCodec::UNSPECIFIED) { + return true; + } + + const std::string name = Sprintf( + "%s-%s-%s", CamIdToStr(camdata.cam_id_), CamDataTypeToStr(camdata.type_), CamDataFmtToStr(camdata.fmt_)); + auto entry = gop_jobs_.find(name); + + // An I-frame starts a new GOP, which is decoded independently of the previous one + if (camdata.frm_ == CamDataFrm::I_FRAME) { + if (entry != gop_jobs_.end()) { + auto& prev = *entry->second; + prev.Close(); + if (prev.num_done_ < prev.num_frames_) { + gops_pending_++; } - AsyncDecState state; - state.dec_ = CreateVideoFrameDecoder({ name, codec, opts_.pixelfmt_, opts_.scale_ }); - entry = decoder_states_.emplace(name, std::move(state)).first; + gop_jobs_.erase(entry); } - if (!entry->second.dec_) { - WARNING("No decoder for %s", name.c_str()); - return ProcRes::FATAL; + // Don't run too many decoders at the same time, and don't queue too much decoded data + while (gops_pending_ >= max_gops_pending_) { + if (!ProcessQueue(QueueWait::HEAD)) { + return false; + } } - auto& dec = entry->second.dec_; - auto& fut = entry->second.fut_; - // Wait for decoding results from previous frame - if (fut.valid()) { - res = ProcessAsyncDecData(fut.get()); - } + // Each decoder makes its own hw device, if hw decoding is used + const VideoDecoderParams params = { name, codec, opts_.pixelfmt_, opts_.scale_, opts_.accel_ }; + entry = gop_jobs_.emplace(name, std::make_shared(params, do_jsonl_)).first; + } + // We can only start decoding from the first I-frame onwards + else if (entry == gop_jobs_.end()) { + return true; + } - // Decode this frame - fut = std::async(std::launch::async, [&dec, /*copy!*/ camdata]() -> AsyncDecData { - return { camdata, dec->DecodeFrame(camdata.data_), dec->IsOkay() }; - }); + if (entry->second->num_frames_ >= MAX_GOP_SIZE) { + WARNING("Too many frames (> %" PRIuMAX ") without an I-frame in %s", MAX_GOP_SIZE, name.c_str()); + return false; + } + job = entry->second; + job_ix = job->AddFrame(std::move(camdata)); + + return true; +} + +// --------------------------------------------------------------------------------------------------------------------- - break; +FplToolExtract::GopJob::GopJob(const VideoDecoderParams& params, const bool do_json) /* clang-format off */ : + do_json_ { do_json } // clang-format on +{ + fut_ = std::async(std::launch::async, [this, params]() { Run(params); }); +} + +FplToolExtract::GopJob::~GopJob() +{ + Abort(); +} + +std::size_t FplToolExtract::GopJob::AddFrame(fpsdk::common::fpl::CamData&& camdata) +{ + { + std::unique_lock lock(mutex_); + todo_.push_back(std::move(camdata)); } -#endif + cond_.notify_all(); + return num_frames_++; +} - return res; +void FplToolExtract::GopJob::Close() +{ + { + std::unique_lock lock(mutex_); + finish_ = true; + } + cond_.notify_all(); + closed_ = true; +} + +void FplToolExtract::GopJob::Abort() +{ + { + std::unique_lock lock(mutex_); + finish_ = true; + abort_ = true; + } + cond_.notify_all(); +} + +bool FplToolExtract::GopJob::IsDecoded(const std::size_t ix) +{ + std::unique_lock lock(mutex_); + return ix < done_.size(); +} + +FplToolExtract::AsyncDecData FplToolExtract::GopJob::GetDecoded(const std::size_t ix) +{ + std::unique_lock lock(mutex_); + cond_.wait(lock, [this, ix]() { return ix < done_.size(); }); + return std::move(done_[ix]); +} + +void FplToolExtract::GopJob::Run(const VideoDecoderParams& params) +{ + auto dec = CreateVideoFrameDecoder(params); + while (true) { + std::unique_lock lock(mutex_); + cond_.wait(lock, [this]() { return !todo_.empty() || finish_; }); + if (todo_.empty() || abort_) { + break; + } + AsyncDecData decdata = { std::move(todo_.front()), std::nullopt, false, "" }; + todo_.pop_front(); + lock.unlock(); + + if (dec) { + decdata.img_ = dec->DecodeFrame(decdata.camdata_.data_); + decdata.dec_is_okay_ = dec->IsOkay(); + } + if (do_json_) { + decdata.json_ = nlohmann::json(decdata.camdata_).dump(); + } + + lock.lock(); + done_.push_back(std::move(decdata)); + lock.unlock(); + cond_.notify_all(); + } } // --------------------------------------------------------------------------------------------------------------------- -#if FPSDK_USE_FFMPEG -FplToolExtract::ProcRes FplToolExtract::ProcessAsyncDecData(const FplToolExtract::AsyncDecData& decdata) +FplToolExtract::ProcRes FplToolExtract::ProcessAsyncDecData(FplToolExtract::AsyncDecData&& decdata) { auto& camdata = decdata.camdata_; if (!decdata.img_) { @@ -560,7 +772,7 @@ FplToolExtract::ProcRes FplToolExtract::ProcessAsyncDecData(const FplToolExtract case PixelFmt::GBRP: rosmsg.encoding = "8UC3"; rosmsg.step = img.width_; break; case PixelFmt::UNSPECIFIED: break; } // clang-format on - rosmsg.data = { img.data_.data(), img.data_.data() + img.data_.size() }; + rosmsg.data = std::move(img.data_); // Abuse pixel (0, 0) to store the exposure duration in [0.1ms] if (!rosmsg.data.empty()) { const uint32_t dt01ms = camdata.dt_ / 100000; // [ns] -> [0.1ms] @@ -596,6 +808,17 @@ bool FplToolExtract::WriteJson(const std::string& name, const nlohmann::json& js // --------------------------------------------------------------------------------------------------------------------- +bool FplToolExtract::WriteJson(const std::string& name, const std::string& json) +{ + auto f = GetOutputFile(name); + if (!f) { + return false; + } + return f->Write(json) && f->Write("\n"); +} + +// --------------------------------------------------------------------------------------------------------------------- + bool FplToolExtract::WriteStreamMsg(const std::string& name, const StreamMsg& streammsg, const ParserMsg& parsermsg) { parsermsg.MakeInfo(); diff --git a/fpsdk_apps/fpltool/fpltool_extract.hpp b/fpsdk_apps/fpltool/fpltool_extract.hpp index e85906f..514b4e8 100644 --- a/fpsdk_apps/fpltool/fpltool_extract.hpp +++ b/fpsdk_apps/fpltool/fpltool_extract.hpp @@ -14,8 +14,13 @@ #define __FPSDK_APPS_FPLTOOL_FPLTOOL_EXTRACT_HPP__ /* LIBC/STL */ +#include +#include #include #include +#include +#include +#include #include #include #include @@ -29,6 +34,7 @@ #include #include #include +#include #include /* PACKAGE */ @@ -60,13 +66,12 @@ class FplToolExtract // FPL // clang-format off enum class ProcRes { OK, ERROR, FATAL }; // clang-format off - ProcRes ProcessLogStatus(const common::fpl::FplMessage& fpl_msg, const bool do_extract, uint32_t& time_into_log); + ProcRes ProcessLogStatus(const common::fpl::FplMessage& fpl_msg, const bool do_extract); ProcRes ProcessLogMeta(const common::fpl::FplMessage& fpl_msg, const bool do_extract); ProcRes ProcessRosMsgDef(const common::fpl::FplMessage& fpl_msg, const bool do_extract); ProcRes ProcessRosMsgBin(const common::fpl::FplMessage& fpl_msg, const bool do_extract); ProcRes ProcessStreamMsg(const common::fpl::FplMessage& fpl_msg, const bool do_extract); ProcRes ProcessFileDump(const common::fpl::FplMessage& fpl_msg, const bool do_extract); - ProcRes ProcessCamData(const common::fpl::FplMessage& fpl_msg, const bool do_extract); // LOGMETA bool have_logmeta_ = false; @@ -90,16 +95,62 @@ class FplToolExtract common::fpl::CamData camdata_; std::optional img_; bool dec_is_okay_ = true; + std::string json_; }; - struct AsyncDecState + + // Decoding of one group of pictures (GOP, one I-frame and the P-frames that follow it), using its own decoder + // in its own thread. The main thread adds the frames as it reads them from the log and collects the decoded + // frames in the same order. + class GopJob : private common::types::NoCopyNoMove { - common::video::VideoFrameDecoderPtr dec_; - std::future fut_; + public: + GopJob(const common::video::VideoDecoderParams& params, const bool do_json); + ~GopJob(); + std::size_t AddFrame(common::fpl::CamData&& camdata); // Add frame to decode, returns the frame index + void Close(); // No more frames will be added (end of GOP) + void Abort(); // Stop decoding as soon as possible + bool IsDecoded(const std::size_t ix); // Check if frame is decoded + AsyncDecData GetDecoded(const std::size_t ix); // Get decoded frame, waits for it if necessary + std::size_t num_frames_ = 0; // Number of frames added (main thread only) + std::size_t num_done_ = 0; // Number of frames collected (main thread only) + bool closed_ = false; // No more frames will be added (main thread only) + + private: + void Run(const common::video::VideoDecoderParams& params); + std::mutex mutex_; + std::condition_variable cond_; + std::deque todo_; + std::vector done_; + bool finish_ = false; + bool abort_ = false; + const bool do_json_; + std::future fut_; // Must be last, so that the thread is gone before the other members are }; - std::map decoder_states_; + std::map> gop_jobs_; // The GOP we're currently reading, per stream + std::size_t gops_pending_ = 0; // Number of complete GOPs not fully written yet + std::size_t max_gops_pending_ = 1; + + bool QueueCamData(const common::fpl::FplMessage& fpl_msg, std::shared_ptr& job, std::size_t& job_ix); + ProcRes ProcessAsyncDecData(AsyncDecData&& decdata); +#endif - ProcRes ProcessAsyncDecData(const AsyncDecData& decdata); + // Messages are queued and processed in the order they are in the log. Camera frames that need decoding are held + // back, along with everything that follows them, until they are decoded. + struct QueueItem + { + common::fpl::FplMessage fpl_msg_; + bool do_extract_ = true; +#if FPSDK_USE_FFMPEG + std::shared_ptr job_; + std::size_t job_ix_ = 0; #endif + }; + std::deque queue_; + enum class QueueWait { NONE, HEAD, ALL }; + bool ProcessQueue(const QueueWait wait); + ProcRes ProcessCamData(const QueueItem& item); + void AbortQueue(); + std::size_t errors_ = 0; // Output files #if FPSDK_USE_ROS2 @@ -111,6 +162,7 @@ class FplToolExtract common::path::OutputFile* GetOutputFile(const std::string& name); bool WriteData(const std::string& name, const std::vector& data); bool WriteJson(const std::string& name, const nlohmann::json& json); + bool WriteJson(const std::string& name, const std::string& json); bool WriteStreamMsg( const std::string& name, const common::fpl::StreamMsg& streammsg, const common::parser::ParserMsg& parsermsg); void CloseAll(const bool ok = false); diff --git a/fpsdk_apps/fpltool/fpltool_opts.hpp b/fpsdk_apps/fpltool/fpltool_opts.hpp index 8f21c5c..9a05b4f 100644 --- a/fpsdk_apps/fpltool/fpltool_opts.hpp +++ b/fpsdk_apps/fpltool/fpltool_opts.hpp @@ -16,6 +16,7 @@ /* LIBC/STL */ #include #include +#include #include /* EXTERNAL */ @@ -54,6 +55,8 @@ class FplToolOptions : public common::app::ProgramOptions { 'e', true, "formats" }, { 's', true, "scale" }, { 't', true, "pixelfmt" }, + { 'j', true, "jobs" }, + { 'a', true, "accel" }, }) {}; // clang-format on /** @@ -88,6 +91,8 @@ class FplToolOptions : public common::app::ProgramOptions #if FPSDK_USE_FFMPEG double scale_ = 1.0; //!< Scale factor for decoded video frames common::video::PixelFmt pixelfmt_ = common::video::PixelFmt::RGB24; //!< Pixel format for decoded video frames + uint32_t jobs_ = 0; //!< Number of parallel video decoders (0 = auto) + common::video::HwAccel accel_ = common::video::HwAccel::SW; //!< Hw acceleration for video decoders #endif // clang-format on @@ -125,6 +130,8 @@ class FplToolOptions : public common::app::ProgramOptions " -e, --formats -- Comma-separated list of output formats for the extract (default: jsonl,raw,file)\n" " -s, --scale -- Scale factor for decoded video frames: 0.1 - 1.0 (default: 1.0)\n" " -t, --pixelfmt -- Pixel format for decoded video frames: Y8, RGB24, GBRP (default: RGB24)\n" + " -j, --jobs -- Number of video decoders to run in parallel (default: number of CPUs - 2, at least 1)\n" + " -a, --accel -- Hardware acceleration for video decoders: SW (default, recommended), HW or AUTO\n" " -- The command, see below\n" " -- The .fpl (or .fpl.gz) file to process\n" " \n" @@ -160,7 +167,7 @@ class FplToolOptions : public common::app::ProgramOptions "\n" " extract -- Extract the data in a .fpl file\n" "\n" - " fpltool [-vqpPfocSDst] [-e ] extract \n" + " fpltool [-vqpPfocSDstja] [-e ] extract \n" "\n" " The data is extracted to different files in the current directory. The files are named like the\n" " with added suffixes and different file extension, depending on the kind of data that\n" @@ -373,6 +380,27 @@ class FplToolOptions : public common::app::ProgramOptions #else WARNING("Cannot use --pixelfmt, this fpltool is not compiled with FFmpeg"); ok = false; +#endif + break; + case 'j': +#if FPSDK_USE_FFMPEG + if (!common::string::StrToValue(argument, jobs_) || (jobs_ < 1)) { + ok = false; + } +#else + WARNING("Cannot use --jobs, this fpltool is not compiled with FFmpeg"); + ok = false; +#endif + break; + case 'a': +#if FPSDK_USE_FFMPEG + accel_ = HwAccelFromStrOr(argument.c_str(), common::video::HwAccel::UNSPECIFIED); + if (accel_ == common::video::HwAccel::UNSPECIFIED) { + ok = false; + } +#else + WARNING("Cannot use --accel, this fpltool is not compiled with FFmpeg"); + ok = false; #endif break; default: @@ -416,6 +444,14 @@ class FplToolOptions : public common::app::ProgramOptions progress_ = 1; } +#if FPSDK_USE_FFMPEG + // Default number of jobs: leave two CPUs for the rest (reading, writing) + if (jobs_ == 0) { + const uint32_t num_cpus = std::thread::hardware_concurrency(); + jobs_ = (num_cpus > 2 ? num_cpus - 2 : 1); + } +#endif + // Debug DEBUG("command_ = '%s'", command_str_.c_str()); for (std::size_t ix = 0; ix < inputs_.size(); ix++) { @@ -435,6 +471,8 @@ class FplToolOptions : public common::app::ProgramOptions #if FPSDK_USE_FFMPEG DEBUG("scale = %.1f", scale_); DEBUG("pixelfmt = %s", common::video::PixelFmtToStr(pixelfmt_)); + DEBUG("jobs = %d", jobs_); + DEBUG("accel = %s", common::video::HwAccelToStr(accel_)); #endif return ok; } diff --git a/fpsdk_common/include/fpsdk_common/video.hpp b/fpsdk_common/include/fpsdk_common/video.hpp index 18d4eca..599b7cd 100644 --- a/fpsdk_common/include/fpsdk_common/video.hpp +++ b/fpsdk_common/include/fpsdk_common/video.hpp @@ -15,6 +15,30 @@ * **API**: fpsdk_common/video.hpp and fpsdk::common::video * * @note This is only available if compiled with FFmpeg, see @ref FPSDK_BUILD_DEPS. + * + * @section FPSDK_COMMON_VIDEO_HWACCEL Hardware vs. software decoding + * + * Hardware decoding (fpsdk::common::video::HwAccel::HW, currently VA-API) is not necessarily faster than software + * decoding (fpsdk::common::video::HwAccel::SW). The following are wall-clock times for "fpltool extract" of a 32 s + * recording with two cameras (H.265, 1024x768, 25 fps, 10 frames per GOP, 1547 frames in total) decoded to RGB24 + * (a 3.5 GB ROS1 bag), using different numbers of parallel decoders (-j). Measured on an AMD Ryzen 7 PRO 8840HS + * (8 cores, 16 threads) with Radeon 780M graphics, FFmpeg 7.1, Release build, median of three runs: + * + * | Decoders | -e jsonl,raw,file,ros,cam -a SW | -e jsonl,raw,file,ros,cam -a HW | -e ros,cam -a SW | -e ros,cam -a HW | + * |----------|---------------------------------|---------------------------------|------------------|------------------| + * | -j 1 | 5.3 s | 5.9 s | 4.4 s | 5.7 s | + * | -j 2 | 4.7 s | 5.8 s | 4.7 s | 6.2 s | + * | -j 4 | 4.2 s | 6.1 s | 4.2 s | 5.6 s | + * | -j 8 | 4.2 s | 6.5 s | 4.0 s | 5.5 s | + * + * Findings: + * + * - Software decoding is faster than hardware decoding in all cases + * - Decoding this kind of video is cheap on the CPU (about 1.2 ms per P-frame and 8 ms per I-frame on one core), + * and it scales with the number of decoders. + * - The hardware decoder is limited by getting the decoded frames back from the GPU, and the GPU does not scale with + * the number of decoders. The conversion to the output pixel format is done in software in both cases. + * - The results may be different for other video (larger frames, H.264, longer GOPs), GPUs and drivers. */ #ifndef __FPSDK_COMMON_VIDEO_HPP__ #define __FPSDK_COMMON_VIDEO_HPP__ diff --git a/fpsdk_common/src/fpl.cpp b/fpsdk_common/src/fpl.cpp index 155fa7e..3deb55b 100644 --- a/fpsdk_common/src/fpl.cpp +++ b/fpsdk_common/src/fpl.cpp @@ -755,7 +755,7 @@ CamData::CamData(const FplMessage& log_msg) info_ = string::Sprintf("rec_time=%.3f cam_id=%s type=%s fmt=%s frm=%s %" PRIu32 "x%" PRIu32 " seq=%" PRIuMAX " ts=%.3f size=%" PRIuMAX, - rec_time_.ToSec() * 1e-9, CamIdToStr(cam_id_), CamDataTypeToStr(type_), CamDataFmtToStr(fmt_), + rec_time_.ToSec(), CamIdToStr(cam_id_), CamDataTypeToStr(type_), CamDataFmtToStr(fmt_), CamDataFrmToStr(frm_), width_, height_, seq_, (double)ts_ * 1e-9, data_.size()); } else { WARNING("CamData: invalid message"); diff --git a/fpsdk_common/src/video.cpp b/fpsdk_common/src/video.cpp index abddd67..770cdbf 100644 --- a/fpsdk_common/src/video.cpp +++ b/fpsdk_common/src/video.cpp @@ -14,6 +14,7 @@ /* LIBC/STL */ #include #include +#include #include #include @@ -210,14 +211,11 @@ class VideoFrameDecoderImpl : public VideoFrameDecoder, private NoCopyNoMove std::optional DecodeFrame(const uint8_t* data, const std::size_t size) override final; bool IsOkay() const override final; - enum AVPixelFormat GetHwFormat(const enum AVPixelFormat* fmts) const; - private: // clang-format off enum class State { DEINIT, INIT_AV, INIT_GRAPH, ERROR }; State state_ = State::DEINIT; AVHWDeviceType hw_type_ = AV_HWDEVICE_TYPE_NONE; // AV_HWDEVICE_TYPE_VAAPI - enum AVPixelFormat src_fmt_ = AV_PIX_FMT_YUVJ420P; // AV_PIX_FMT_VAAPI const AVCodec* codec_ = nullptr; AVCodecContext* ctx_ = nullptr; AVBufferRef* hw_ctx_ = nullptr; // hwdevice_ctx @@ -265,21 +263,6 @@ bool VideoFrameDecoderImpl::IsOkay() const // --------------------------------------------------------------------------------------------------------------------- -static enum AVPixelFormat GetHwFormatCb(AVCodecContext* ctx, const enum AVPixelFormat* fmts) -{ - return static_cast(ctx->opaque)->GetHwFormat(fmts); -} - -enum AVPixelFormat VideoFrameDecoderImpl::GetHwFormat(const enum AVPixelFormat* fmts) const -{ - for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; ++p) { - if (*p == src_fmt_) { - return *p; - } - } - return AV_PIX_FMT_NONE; -} - bool VideoFrameDecoderImpl::InitAv() { DEBUG("VideoFrameDecoderImpl(%s) init", params_.name_.c_str()); @@ -354,14 +337,10 @@ bool VideoFrameDecoderImpl::InitAv() } hw = nullptr; } else { - // hw_ctx_ = reinterpret_cast(hw); - hw_ctx_ = av_buffer_ref(hw); + // FFmpeg's default get_format() selects the hw pixel format if the decoder can use this device. + // Otherwise it decodes in sw and we get sw frames, see InitGraph(). + hw_ctx_ = hw; hw_type_ = AV_HWDEVICE_TYPE_VAAPI; - src_fmt_ = AV_PIX_FMT_VAAPI; - // @todo this crashes.. - // ctx_->opaque = this; - // ctx_->get_format = GetHwFormatCb; - UNUSED(GetHwFormatCb); } } @@ -449,13 +428,33 @@ bool VideoFrameDecoderImpl::InitGraph() int src_width = 0; int src_height = 0; - AVPixelFormat src_format = AV_PIX_FMT_NONE; // TODO check <-> src_fmt_ + AVPixelFormat src_format = AV_PIX_FMT_NONE; AVRational src_time_base = { 1, 25 }; AVRational src_sar = { 1, 1 }; src_width = frame_->width; src_height = frame_->height; - src_format = static_cast(frame_->format); // yuvj420p, check that + src_format = static_cast(frame_->format); // sw: yuvj420p, hw: vaapi + + // We may have a hw device, but the decoder may not be able to use it (codec, profile, FFmpeg build, ...) + const bool hw_frame = (src_format == AV_PIX_FMT_VAAPI); + if (!hw_frame && (params_.accel_ == HwAccel::HW)) { + WARNING("VideoFrameDecoder(%s) no hw decoding for this video", params_.name_.c_str()); + return false; + } + if (!hw_frame && hw_ctx_) { + DEBUG("VideoFrameDecoder(%s) no hw decoding for this video, using sw decoding", params_.name_.c_str()); + } + + // Tell the user once, not for each decoder + static std::atomic_flag info_done = ATOMIC_FLAG_INIT; + if (!info_done.test_and_set()) { + if (hw_frame) { + INFO("Using hardware (%s) video decoder", av_hwdevice_get_type_name(hw_type_)); + } else { + INFO("Using software video decoder"); + } + } if ((ctx_->time_base.num > 0) && (ctx_->time_base.den > 0)) { src_time_base = ctx_->time_base; } else { @@ -506,12 +505,12 @@ bool VideoFrameDecoderImpl::InitGraph() ok = false; break; } - par->format = src_fmt_; + par->format = src_format; par->width = src_width; par->height = src_height; par->time_base = src_time_base; par->sample_aspect_ratio = src_sar; - if (hw_ctx_) { + if (hw_frame) { if (frame_->hw_frames_ctx) { // Should be present now that we have decoded the first frame par->hw_frames_ctx = av_buffer_ref(frame_->hw_frames_ctx); } else { @@ -581,9 +580,12 @@ bool VideoFrameDecoderImpl::InitGraph() # endif char filter_desc[256]; - if (hw_ctx_) { - std::snprintf(filter_desc, sizeof(filter_desc), "scale_vaapi=w=%d:h=%d:format=nv12,hwdownload,format=nv12", - dst_width, dst_height); + if (hw_frame) { + // Scale on the GPU, download the frame, convert the pixel format in sw. Download yuv420p, not nv12, as + // swscale is much slower for nv12 to rgb24. + std::snprintf(filter_desc, sizeof(filter_desc), + "scale_vaapi=w=%d:h=%d:format=yuv420p,hwdownload,format=yuv420p,scale,format=pix_fmts=%s", dst_width, + dst_height, PixelFmtToFlag(params_.fmt_)); } else { std::snprintf(filter_desc, sizeof(filter_desc), "scale=w=%d:h=%d:flags=%s,format=pix_fmts=%s", dst_width, dst_height, ScalingQualToFlag(params_.qual_), PixelFmtToFlag(params_.fmt_)); From b50f57c1cb93f999081e06e29f454ce1120fa86b Mon Sep 17 00:00:00 2001 From: flipflip Date: Mon, 5 Oct 2026 17:05:07 +0200 Subject: [PATCH 2/6] fpltool extract: count errors, so that the limit of 100 errors has an effect Co-Authored-By: Claude Opus 5.5 --- fpsdk_apps/fpltool/fpltool_extract.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fpsdk_apps/fpltool/fpltool_extract.cpp b/fpsdk_apps/fpltool/fpltool_extract.cpp index 526cb5f..cde2df6 100644 --- a/fpsdk_apps/fpltool/fpltool_extract.cpp +++ b/fpsdk_apps/fpltool/fpltool_extract.cpp @@ -297,6 +297,7 @@ bool FplToolExtract::ProcessQueue(const QueueWait wait) case ProcRes::OK: break; case ProcRes::ERROR: + errors_++; if (errors_ >= 100) { WARNING("Too many errors, giving up"); return false; From 283e0836549ec01a038853b247e3e5188aa74087 Mon Sep 17 00:00:00 2001 From: flipflip Date: Mon, 5 Oct 2026 17:27:44 +0200 Subject: [PATCH 3/6] fpltool extract: fix build with USE_FFMPEG=OFF (unused variable) Co-Authored-By: Claude Opus 5.5 --- fpsdk_apps/fpltool/fpltool_extract.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fpsdk_apps/fpltool/fpltool_extract.cpp b/fpsdk_apps/fpltool/fpltool_extract.cpp index cde2df6..bdb2402 100644 --- a/fpsdk_apps/fpltool/fpltool_extract.cpp +++ b/fpsdk_apps/fpltool/fpltool_extract.cpp @@ -259,7 +259,11 @@ bool FplToolExtract::Run() bool FplToolExtract::ProcessQueue(const QueueWait wait) { +#if FPSDK_USE_FFMPEG bool may_wait = (wait != QueueWait::NONE); +#else + UNUSED(wait); +#endif while (!queue_.empty()) { auto& item = queue_.front(); From 5b9fe4e13941c29c07769af9c036f22f6c9d67ca Mon Sep 17 00:00:00 2001 From: flipflip Date: Tue, 6 Oct 2026 09:15:27 +0200 Subject: [PATCH 4/6] codenazi --- fpsdk_common/include/fpsdk_common/video.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fpsdk_common/include/fpsdk_common/video.hpp b/fpsdk_common/include/fpsdk_common/video.hpp index 599b7cd..dfec3ba 100644 --- a/fpsdk_common/include/fpsdk_common/video.hpp +++ b/fpsdk_common/include/fpsdk_common/video.hpp @@ -9,7 +9,9 @@ * * @file * @brief Fixposition SDK: Video frame decoding - * + */ +// clang-format off +/** * @page FPSDK_COMMON_VIDEO Video frame decoding * * **API**: fpsdk_common/video.hpp and fpsdk::common::video @@ -40,6 +42,7 @@ * the number of decoders. The conversion to the output pixel format is done in software in both cases. * - The results may be different for other video (larger frames, H.264, longer GOPs), GPUs and drivers. */ +// clang-format on #ifndef __FPSDK_COMMON_VIDEO_HPP__ #define __FPSDK_COMMON_VIDEO_HPP__ From 5ab91aaaf44c1d58892194174f2ed751c8107d74 Mon Sep 17 00:00:00 2001 From: flipflip Date: Wed, 28 Oct 2026 18:24:58 +0100 Subject: [PATCH 5/6] examples: fix CMakeLists.txt typo in Makefile deps The Makefiles listed "CMakeList.txt" in their dependencies, so changes to CMakeLists.txt didn't trigger a rebuild. Also fix the same typo in the build docs. Co-Authored-By: Claude Opus 5.5 --- examples/camera_streaming/Makefile | 2 +- examples/fpb_measurements/Makefile | 2 +- examples/fusion_epoch/Makefile | 2 +- examples/parser_intro/Makefile | 2 +- fpsdk_doc/fpsdk_build.hpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/camera_streaming/Makefile b/examples/camera_streaming/Makefile index 74e6dad..6d96910 100644 --- a/examples/camera_streaming/Makefile +++ b/examples/camera_streaming/Makefile @@ -1,6 +1,6 @@ .SUFFIXES: -deps = $(sort $(wildcard CMakeList.txt Makefile ../../fpsdk_doc/* *.cpp \ +deps = $(sort $(wildcard CMakeLists.txt Makefile ../../fpsdk_doc/* *.cpp \ ../../fpsdk_common/* ../../fpsdk_common/*/* ../../fpsdk_common/*/*/* ../../fpsdk_common/*/*/*/* \ ../../fpsdk_apps/* ../../fpsdk_apps/*/* ../../fpsdk_apps/*/*/* ../../fpsdk_apps/*/*/*/*)) diff --git a/examples/fpb_measurements/Makefile b/examples/fpb_measurements/Makefile index 74e6dad..6d96910 100644 --- a/examples/fpb_measurements/Makefile +++ b/examples/fpb_measurements/Makefile @@ -1,6 +1,6 @@ .SUFFIXES: -deps = $(sort $(wildcard CMakeList.txt Makefile ../../fpsdk_doc/* *.cpp \ +deps = $(sort $(wildcard CMakeLists.txt Makefile ../../fpsdk_doc/* *.cpp \ ../../fpsdk_common/* ../../fpsdk_common/*/* ../../fpsdk_common/*/*/* ../../fpsdk_common/*/*/*/* \ ../../fpsdk_apps/* ../../fpsdk_apps/*/* ../../fpsdk_apps/*/*/* ../../fpsdk_apps/*/*/*/*)) diff --git a/examples/fusion_epoch/Makefile b/examples/fusion_epoch/Makefile index 74e6dad..6d96910 100644 --- a/examples/fusion_epoch/Makefile +++ b/examples/fusion_epoch/Makefile @@ -1,6 +1,6 @@ .SUFFIXES: -deps = $(sort $(wildcard CMakeList.txt Makefile ../../fpsdk_doc/* *.cpp \ +deps = $(sort $(wildcard CMakeLists.txt Makefile ../../fpsdk_doc/* *.cpp \ ../../fpsdk_common/* ../../fpsdk_common/*/* ../../fpsdk_common/*/*/* ../../fpsdk_common/*/*/*/* \ ../../fpsdk_apps/* ../../fpsdk_apps/*/* ../../fpsdk_apps/*/*/* ../../fpsdk_apps/*/*/*/*)) diff --git a/examples/parser_intro/Makefile b/examples/parser_intro/Makefile index 74e6dad..6d96910 100644 --- a/examples/parser_intro/Makefile +++ b/examples/parser_intro/Makefile @@ -1,6 +1,6 @@ .SUFFIXES: -deps = $(sort $(wildcard CMakeList.txt Makefile ../../fpsdk_doc/* *.cpp \ +deps = $(sort $(wildcard CMakeLists.txt Makefile ../../fpsdk_doc/* *.cpp \ ../../fpsdk_common/* ../../fpsdk_common/*/* ../../fpsdk_common/*/*/* ../../fpsdk_common/*/*/*/* \ ../../fpsdk_apps/* ../../fpsdk_apps/*/* ../../fpsdk_apps/*/*/* ../../fpsdk_apps/*/*/*/*)) diff --git a/fpsdk_doc/fpsdk_build.hpp b/fpsdk_doc/fpsdk_build.hpp index 1dd647f..4a6316e 100644 --- a/fpsdk_doc/fpsdk_build.hpp +++ b/fpsdk_doc/fpsdk_build.hpp @@ -244,7 +244,7 @@ namespace fpsdk { @subsection FPSDK_BUILD_DETAILS Details - Refer to the various CMakeList.txt files, the CI workflow configuration (`.github/workflows/ci.yml`) + Refer to the various CMakeLists.txt files, the CI workflow configuration (`.github/workflows/ci.yml`) and the CI script (`docker/ci.sh`) for details on how things are done. From b8be5a89055fa3c8c73a8d118050c8cb35220822 Mon Sep 17 00:00:00 2001 From: flipflip Date: Wed, 28 Oct 2026 18:24:58 +0100 Subject: [PATCH 6/6] video: document decoding-only SW vs. HW performance measurements - Add a table with decoding rates (fps) for SW and HW decoding with 1, 2, 4 and 8 parallel decoders, each decoder decoding all frames vs. a new decoder per GOP (as fpltool extract does) - Update the findings: SW decoding scales about 3x with 8 decoders, HW is only faster with 2-3 decoders, HW stops scaling at about 800 fps, and the HW device setup on the first frame makes a new decoder per GOP expensive Co-Authored-By: Claude Opus 5.5 --- fpsdk_common/include/fpsdk_common/video.hpp | 31 +++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/fpsdk_common/include/fpsdk_common/video.hpp b/fpsdk_common/include/fpsdk_common/video.hpp index dfec3ba..fae34e4 100644 --- a/fpsdk_common/include/fpsdk_common/video.hpp +++ b/fpsdk_common/include/fpsdk_common/video.hpp @@ -33,13 +33,34 @@ * | -j 4 | 4.2 s | 6.1 s | 4.2 s | 5.6 s | * | -j 8 | 4.2 s | 6.5 s | 4.0 s | 5.5 s | * + * The following are decoding rates (average frames per second, fps) measured with a test program that only decodes + * (VideoFrameDecoder::DecodeFrame()) on the same machine, for the first 100 frames (10 I frames, 90 P frames) of a + * camera recording (H.265, 1024x768) decoded to RGB24, 10 repetitions. In "indep" mode each of the -j decoders decodes all + * frames (that is, -j times the work). In "gop" mode each GOP is decoded by a new decoder and the GOPs are distributed + * to the -j decoder threads, which is what "fpltool extract" does (with 11 GOPs this mode stops scaling at about -j 5): + * + * | Decoders | indep -a SW | indep -a HW | gop -a SW | gop -a HW | + * |----------|-------------|-------------|-----------|-----------| + * | -j 1 | 324 fps | 304 fps | 286 fps | 225 fps | + * | -j 2 | 521 fps | 643 fps | 476 fps | 372 fps | + * | -j 4 | 789 fps | 751 fps | 679 fps | 480 fps | + * | -j 8 | 1030 fps | 820 fps | 800 fps | 534 fps | + * * Findings: * - * - Software decoding is faster than hardware decoding in all cases - * - Decoding this kind of video is cheap on the CPU (about 1.2 ms per P-frame and 8 ms per I-frame on one core), - * and it scales with the number of decoders. - * - The hardware decoder is limited by getting the decoded frames back from the GPU, and the GPU does not scale with - * the number of decoders. The conversion to the output pixel format is done in software in both cases. + * - Software decoding is faster than hardware decoding in almost all cases. Only with 2 or 3 decoders running at the + * same time, each with its own continuous stream of frames, hardware decoding is slightly faster. + * - Decoding this kind of video is cheap on the CPU (about 1.2-1.6 ms per P frame and 8-15 ms per I frame on one core, + * depending on the recording). It scales with the number of decoders, but not linearly: with 8 decoders the + * throughput is about 3 times that of one decoder, as each decoder gets slower (for example, 1.6 ms per P frame with + * one decoder, 4.8 ms with 8 decoders). + * - The hardware decoder decodes I frames faster than software (about 3.6 ms), but P frames slower (about 3.0 ms). It + * is limited by getting the decoded frames back from the GPU, and the GPU does not scale with the number of decoders + * (the throughput stops increasing at about 800 fps with 3 decoders). The conversion to the output pixel format is + * done in software in both cases. + * - The hardware device is set up when the first frame is decoded, which costs about 10 ms (up to about 40 ms when + * many decoders do this at the same time). This makes hardware decoding even slower when a new decoder is used for + * each GOP. * - The results may be different for other video (larger frames, H.264, longer GOPs), GPUs and drivers. */ // clang-format on