From f7dc39fb8730ccf854d60b819a985050fee0d4b1 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 20 May 2022 01:11:35 +0200 Subject: [PATCH] Extract J2K frame encoder classes for CPU and remote. --- src/lib/cpu_j2k_frame_encoder.cc | 59 ++++++++++++++++++++++++ src/lib/cpu_j2k_frame_encoder.h | 38 +++++++++++++++ src/lib/j2k_encoder.cc | 62 ++++++------------------- src/lib/j2k_encoder.h | 10 ++-- src/lib/j2k_frame_encoder.h | 45 ++++++++++++++++++ src/lib/remote_j2k_frame_encoder.cc | 71 +++++++++++++++++++++++++++++ src/lib/remote_j2k_frame_encoder.h | 46 +++++++++++++++++++ src/lib/wscript | 2 + 8 files changed, 282 insertions(+), 51 deletions(-) create mode 100644 src/lib/cpu_j2k_frame_encoder.cc create mode 100644 src/lib/cpu_j2k_frame_encoder.h create mode 100644 src/lib/j2k_frame_encoder.h create mode 100644 src/lib/remote_j2k_frame_encoder.cc create mode 100644 src/lib/remote_j2k_frame_encoder.h diff --git a/src/lib/cpu_j2k_frame_encoder.cc b/src/lib/cpu_j2k_frame_encoder.cc new file mode 100644 index 000000000..dcfbe9dcf --- /dev/null +++ b/src/lib/cpu_j2k_frame_encoder.cc @@ -0,0 +1,59 @@ +/* + Copyright (C) 2022 Carl Hetherington + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + +#include "cpu_j2k_frame_encoder.h" +#include "cross.h" +#include "dcp_video.h" +#include "dcpomatic_log.h" + +#include "i18n.h" + + +using std::shared_ptr; +using boost::optional; +using dcp::ArrayData; + + +optional +CPUJ2KFrameEncoder::encode (DCPVideo const& vf) +{ + optional encoded; + + try { + LOG_TIMING("start-local-encode thread=%1 frame=%2", thread_id(), vf.index()); + encoded = vf.encode_locally(); + LOG_TIMING("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index()); + } catch (std::exception& e) { + /* This is very bad, so don't cope with it, just pass it on */ + LOG_ERROR(N_("Local encode failed (%1)"), e.what()); + throw; + } + + return encoded; +} + + +void +CPUJ2KFrameEncoder::log_thread_start () +{ + LOG_TIMING("start-encoder-thread thread=%1 server=localhost", thread_id()); +} + diff --git a/src/lib/cpu_j2k_frame_encoder.h b/src/lib/cpu_j2k_frame_encoder.h new file mode 100644 index 000000000..5e866b086 --- /dev/null +++ b/src/lib/cpu_j2k_frame_encoder.h @@ -0,0 +1,38 @@ +/* + Copyright (C) 2022 Carl Hetherington + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + +#ifndef DCPOMATIC_CPU_J2K_FRAME_ENCODER +#define DCPOMATIC_CPU_J2K_FRAME_ENCODER + + +#include "j2k_frame_encoder.h" + + +class CPUJ2KFrameEncoder : public J2KFrameEncoder +{ +public: + boost::optional encode (DCPVideo const &) override; + void log_thread_start () override; +}; + + +#endif + diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 985708f05..47d8b1295 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -26,6 +26,7 @@ #include "compose.hpp" #include "config.h" +#include "cpu_j2k_frame_encoder.h" #include "cross.h" #include "dcp_video.h" #include "dcpomatic_log.h" @@ -36,6 +37,7 @@ #include "log.h" #include "player.h" #include "player_video.h" +#include "remote_j2k_frame_encoder.h" #include "util.h" #include "writer.h" #include @@ -280,15 +282,12 @@ J2KEncoder::terminate_threads () void -J2KEncoder::encoder_thread (optional server) +J2KEncoder::encoder_thread (weak_ptr weak_worker) try { - start_of_thread ("J2KEncoder"); - - if (server) { - LOG_TIMING ("start-encoder-thread thread=%1 server=%2", thread_id (), server->host_name ()); - } else { - LOG_TIMING ("start-encoder-thread thread=%1 server=localhost", thread_id ()); + auto worker = weak_worker.lock(); + if (!worker) { + return; } /* Number of seconds that we currently wait between attempts @@ -320,45 +319,10 @@ try lock.unlock (); - shared_ptr encoded; - - /* We need to encode this input */ - if (server) { - try { - encoded = make_shared(vf.encode_remotely(server.get())); - - if (remote_backoff > 0) { - LOG_GENERAL ("%1 was lost, but now she is found; removing backoff", server->host_name ()); - } - - /* This job succeeded, so remove any backoff */ - remote_backoff = 0; - - } catch (std::exception& e) { - if (remote_backoff < 60) { - /* back off more */ - remote_backoff += 10; - } - LOG_ERROR ( - N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"), - vf.index(), server->host_name(), e.what(), remote_backoff - ); - } - - } else { - try { - LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index()); - encoded = make_shared(vf.encode_locally()); - LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index()); - } catch (std::exception& e) { - /* This is very bad, so don't cope with it, just pass it on */ - LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); - throw; - } - } + auto encoded = worker->encode(vf); if (encoded) { - _writer->write (encoded, vf.index(), vf.eyes()); + _writer->write (make_shared(*encoded), vf.index(), vf.eyes()); frame_done (); } else { lock.lock (); @@ -401,11 +365,13 @@ J2KEncoder::servers_list_changed () if (!Config::instance()->only_servers_encode ()) { for (int i = 0; i < Config::instance()->master_encoding_threads (); ++i) { + auto worker = make_shared(); + _workers.push_back(worker); #ifdef DCPOMATIC_LINUX - auto t = _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional())); + auto t = _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, worker)); pthread_setname_np (t->native_handle(), "encode-worker"); #else - _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional())); + _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, worker)); #endif } } @@ -417,7 +383,9 @@ J2KEncoder::servers_list_changed () LOG_GENERAL (N_("Adding %1 worker threads for remote %2"), i.threads(), i.host_name ()); for (int j = 0; j < i.threads(); ++j) { - _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, i)); + auto worker = make_shared(i); + _workers.push_back(worker); + _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, worker)); } } diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index ea0a2bef8..5a7db6441 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -41,12 +41,13 @@ #include -class Film; -class EncodeServerDescription; class DCPVideo; -class Writer; +class EncodeServerDescription; +class Film; +class J2KFrameEncoder; class Job; class PlayerVideo; +class Writer; /** @class J2KEncoder @@ -84,7 +85,7 @@ private: void frame_done (); - void encoder_thread (boost::optional); + void encoder_thread (std::weak_ptr worker); void terminate_threads (); /** Film that we are encoding */ @@ -97,6 +98,7 @@ private: mutable boost::mutex _queue_mutex; std::list _queue; + std::list> _workers; /** condition to manage thread wakeups when we have nothing to do */ boost::condition _empty_condition; /** condition to manage thread wakeups when we have too much to do */ diff --git a/src/lib/j2k_frame_encoder.h b/src/lib/j2k_frame_encoder.h new file mode 100644 index 000000000..33f987653 --- /dev/null +++ b/src/lib/j2k_frame_encoder.h @@ -0,0 +1,45 @@ +/* + Copyright (C) 2022 Carl Hetherington + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + +#ifndef DCPOMATIC_J2K_FRAME_ENCODER +#define DCPOMATIC_J2K_FRAME_ENCODER + + +#include +#include +#include + + +class DCPVideo; + + +class J2KFrameEncoder +{ +public: + virtual ~J2KFrameEncoder() {} + + virtual boost::optional encode (DCPVideo const &) = 0; + virtual void log_thread_start () = 0; +}; + + +#endif + diff --git a/src/lib/remote_j2k_frame_encoder.cc b/src/lib/remote_j2k_frame_encoder.cc new file mode 100644 index 000000000..3917cc07b --- /dev/null +++ b/src/lib/remote_j2k_frame_encoder.cc @@ -0,0 +1,71 @@ +/* + Copyright (C) 2022 Carl Hetherington + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + +#include "cross.h" +#include "dcp_video.h" +#include "dcpomatic_log.h" +#include "remote_j2k_frame_encoder.h" + +#include "i18n.h" + + +using std::make_shared; +using std::shared_ptr; +using boost::optional; +using dcp::Data; + + +optional +RemoteJ2KFrameEncoder::encode(DCPVideo const& vf) +{ + optional encoded; + + try { + encoded = vf.encode_remotely(_server); + + if (_remote_backoff > 0) { + LOG_GENERAL ("%1 was lost, but now she is found; removing backoff", _server.host_name()); + } + + /* This job succeeded, so remove any backoff */ + _remote_backoff = 0; + + } catch (std::exception& e) { + if (_remote_backoff < 60) { + /* back off more */ + _remote_backoff += 10; + } + LOG_ERROR ( + N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"), + vf.index(), _server.host_name(), e.what(), _remote_backoff + ); + } + + return encoded; +} + + +void +RemoteJ2KFrameEncoder::log_thread_start () +{ + LOG_TIMING("start-encoder-thread thread=%1 server=%2", thread_id(), _server.host_name()); +} + diff --git a/src/lib/remote_j2k_frame_encoder.h b/src/lib/remote_j2k_frame_encoder.h new file mode 100644 index 000000000..e532782b2 --- /dev/null +++ b/src/lib/remote_j2k_frame_encoder.h @@ -0,0 +1,46 @@ +/* + Copyright (C) 2022 Carl Hetherington + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + +#ifndef DCPOMATIC_REMOTE_J2K_FRAME_ENCODER +#define DCPOMATIC_REMOTE_J2K_FRAME_ENCODER + + +#include "j2k_frame_encoder.h" + + +class RemoteJ2KFrameEncoder : public J2KFrameEncoder +{ +public: + RemoteJ2KFrameEncoder(EncodeServerDescription s) + : _server(s) + {} + + boost::optional encode (DCPVideo const &) override; + void log_thread_start () override; + +private: + EncodeServerDescription _server; + int _remote_backoff = 0; +}; + + +#endif + diff --git a/src/lib/wscript b/src/lib/wscript index 7400c5bf1..1e8fd6f25 100644 --- a/src/lib/wscript +++ b/src/lib/wscript @@ -57,6 +57,7 @@ sources = """ content_factory.cc combine_dcp_job.cc copy_dcp_details_to_film.cc + cpu_j2k_frame_encoder.cc create_cli.cc cross_common.cc crypto.cc @@ -151,6 +152,7 @@ sources = """ ratio.cc raw_image_proxy.cc reel_writer.cc + remote_j2k_frame_encoder.cc render_text.cc resampler.cc rgba.cc -- 2.30.2