2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/j2k_encoder.cc
23 * @brief J2K encoder class.
27 #include "compose.hpp"
30 #include "dcp_video.h"
31 #include "dcpomatic_log.h"
32 #include "encode_server_description.h"
33 #include "encode_server_finder.h"
35 #include "j2k_encoder.h"
37 #include "player_video.h"
40 #include <libcxml/cxml.h>
49 using std::make_shared;
50 using std::shared_ptr;
52 using boost::optional;
54 using namespace dcpomatic;
56 static grk_plugin::GrokInitializer grokInitializer;
58 /** @param film Film that we are encoding.
59 * @param writer Writer that we are using.
61 J2KEncoder::J2KEncoder(shared_ptr<const Film> film, Writer& writer)
65 , _dcpomatic_context(film, writer, _history, Config::instance()->gpu_binary_location())
66 , _context(Config::instance()->enable_gpu() ? new grk_plugin::GrokContext(_dcpomatic_context) : nullptr)
71 J2KEncoder::~J2KEncoder ()
73 _server_found_connection.disconnect();
76 boost::mutex::scoped_lock lm(_threads_mutex);
86 _server_found_connection = EncodeServerFinder::instance()->ServersListChanged.connect(
87 boost::bind(&J2KEncoder::servers_list_changed, this)
89 servers_list_changed ();
96 if (!Config::instance()->enable_gpu()) {
101 boost::mutex::scoped_lock lm (_threads_mutex);
102 terminate_threads ();
105 /* Something might have been thrown during terminate_threads */
113 void J2KEncoder::resume()
115 if (!Config::instance()->enable_gpu()) {
119 _context = new grk_plugin::GrokContext(_dcpomatic_context);
120 servers_list_changed();
127 boost::mutex::scoped_lock lock (_queue_mutex);
129 LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ());
131 /* Keep waking workers until the queue is empty */
132 while (!_queue.empty ()) {
134 _empty_condition.notify_all ();
135 _full_condition.wait (lock);
139 LOG_GENERAL_NC (N_("Terminating encoder threads"));
142 boost::mutex::scoped_lock lm (_threads_mutex);
143 terminate_threads ();
146 /* Something might have been thrown during terminate_threads */
149 LOG_GENERAL (N_("Mopping up %1"), _queue.size());
151 /* The following sequence of events can occur in the above code:
152 1. a remote worker takes the last image off the queue
153 2. the loop above terminates
154 3. the remote worker fails to encode the image and puts it back on the queue
155 4. the remote worker is then terminated by terminate_threads
157 So just mop up anything left in the queue here.
159 for (auto & i: _queue) {
160 if (Config::instance()->enable_gpu ()) {
161 if (!_context->scheduleCompress(i)){
162 LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index());
167 LOG_GENERAL(N_("Encode left-over frame %1"), i.index());
170 make_shared<dcp::ArrayData>(i.encode_locally()),
175 } catch (std::exception& e) {
176 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
186 /** @return an estimate of the current number of frames we are encoding per second,
190 J2KEncoder::current_encoding_rate () const
192 return _history.rate ();
196 /** @return Number of video frames that have been queued for encoding */
198 J2KEncoder::video_frames_enqueued () const
200 if (!_last_player_video_time) {
204 return _last_player_video_time->frames_floor (_film->video_frame_rate ());
208 /** Should be called when a frame has been encoded successfully */
210 J2KEncoder::frame_done ()
216 /** Called to request encoding of the next video frame in the DCP. This is called in order,
217 * so each time the supplied frame is the one after the previous one.
218 * pv represents one video frame, and could be empty if there is nothing to encode
219 * for this DCP frame.
221 * @param pv PlayerVideo to encode.
222 * @param time Time of \p pv within the DCP.
225 J2KEncoder::encode (shared_ptr<PlayerVideo> pv, DCPTime time)
231 boost::mutex::scoped_lock lm (_threads_mutex);
233 threads = _threads->size();
235 threads = std::thread::hardware_concurrency();
238 boost::mutex::scoped_lock queue_lock (_queue_mutex);
240 /* Wait until the queue has gone down a bit. Allow one thing in the queue even
241 when there are no threads.
243 while (_queue.size() >= (threads * 2) + 1) {
244 LOG_TIMING ("decoder-sleep queue=%1 threads=%2", _queue.size(), threads);
245 _full_condition.wait (queue_lock);
246 LOG_TIMING ("decoder-wake queue=%1 threads=%2", _queue.size(), threads);
250 /* Re-throw any exception raised by one of our threads. If more
251 than one has thrown an exception, only one will be rethrown, I think;
252 but then, if that happens something has gone badly wrong.
256 auto const position = time.frames_floor(_film->video_frame_rate());
258 if (_writer.can_fake_write(position)) {
259 /* We can fake-write this frame */
260 LOG_DEBUG_ENCODE("Frame @ %1 FAKE", to_string(time));
261 _writer.fake_write(position, pv->eyes ());
263 } else if (pv->has_j2k() && !_film->reencode_j2k()) {
264 LOG_DEBUG_ENCODE("Frame @ %1 J2K", to_string(time));
265 /* This frame already has J2K data, so just write it */
266 _writer.write(pv->j2k(), position, pv->eyes ());
268 } else if (_last_player_video[pv->eyes()] && _writer.can_repeat(position) && pv->same(_last_player_video[pv->eyes()])) {
269 LOG_DEBUG_ENCODE("Frame @ %1 REPEAT", to_string(time));
270 _writer.repeat(position, pv->eyes());
272 LOG_DEBUG_ENCODE("Frame @ %1 ENCODE", to_string(time));
273 /* Queue this new frame for encoding */
274 LOG_TIMING ("add-frame-to-queue queue=%1", _queue.size ());
275 auto dcpv = DCPVideo(
278 _film->video_frame_rate(),
279 _film->j2k_bandwidth(),
282 _queue.push_back (dcpv);
284 /* The queue might not be empty any more, so notify anything which is
287 _empty_condition.notify_all ();
290 _last_player_video[pv->eyes()] = pv;
291 _last_player_video_time = time;
295 /** Caller must hold a lock on _threads_mutex */
297 J2KEncoder::terminate_threads ()
299 boost::this_thread::disable_interruption dis;
305 _threads->interrupt_all ();
307 _threads->join_all ();
308 } catch (exception& e) {
309 LOG_ERROR ("join() threw an exception: %1", e.what());
311 LOG_ERROR_NC ("join() threw an exception");
319 J2KEncoder::encoder_thread (optional<EncodeServerDescription> server)
322 auto config = Config::instance ();
324 start_of_thread ("J2KEncoder");
327 LOG_TIMING ("start-encoder-thread thread=%1 server=%2", thread_id (), server->host_name ());
329 LOG_TIMING ("start-encoder-thread thread=%1 server=localhost", thread_id ());
332 /* Number of seconds that we currently wait between attempts
333 to connect to the server; not relevant for localhost
336 int remote_backoff = 0;
340 LOG_TIMING ("encoder-sleep thread=%1", thread_id ());
341 boost::mutex::scoped_lock lock (_queue_mutex);
342 while (_queue.empty ()) {
343 _empty_condition.wait (lock);
346 LOG_TIMING ("encoder-wake thread=%1 queue=%2", thread_id(), _queue.size());
347 auto vf = _queue.front ();
349 /* We're about to commit to either encoding this frame or putting it back onto the queue,
350 so we must not be interrupted until one or other of these things have happened. This
351 block has thread interruption disabled.
354 boost::this_thread::disable_interruption dis;
356 LOG_TIMING ("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), vf.index(), static_cast<int>(vf.eyes()));
361 shared_ptr<Data> encoded;
363 /* We need to encode this input */
366 encoded = make_shared<dcp::ArrayData>(vf.encode_remotely(server.get()));
368 if (remote_backoff > 0) {
369 LOG_GENERAL ("%1 was lost, but now she is found; removing backoff", server->host_name ());
372 /* This job succeeded, so remove any backoff */
375 } catch (std::exception& e) {
376 if (remote_backoff < 60) {
378 remote_backoff += 10;
381 N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"),
382 vf.index(), server->host_name(), e.what(), remote_backoff
388 if (!_context->launch(vf, config->selected_gpu()) || !_context->scheduleCompress(vf)) {
389 LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index());
390 _queue.push_front (vf);
394 LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index());
395 encoded = make_shared<dcp::ArrayData>(vf.encode_locally());
396 LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index());
397 } catch (std::exception& e) {
398 /* This is very bad, so don't cope with it, just pass it on */
399 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
406 _writer.write(encoded, vf.index(), vf.eyes());
409 if (!Config::instance()->enable_gpu ()) {
411 LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index());
412 _queue.push_front (vf);
418 if (remote_backoff > 0) {
419 boost::this_thread::sleep (boost::posix_time::seconds (remote_backoff));
422 /* The queue might not be full any more, so notify anything that is waiting on that */
424 _full_condition.notify_all ();
427 catch (boost::thread_interrupted& e) {
428 /* Ignore these and just stop the thread */
429 _full_condition.notify_all ();
434 /* Wake anything waiting on _full_condition so it can see the exception */
435 _full_condition.notify_all ();
440 J2KEncoder::servers_list_changed ()
442 boost::mutex::scoped_lock lm (_threads_mutex);
444 terminate_threads ();
445 _threads = make_shared<boost::thread_group>();
447 /* XXX: could re-use threads */
449 if (!Config::instance()->only_servers_encode ()) {
450 for (int i = 0; i < Config::instance()->master_encoding_threads (); ++i) {
451 #ifdef DCPOMATIC_LINUX
452 auto t = _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional<EncodeServerDescription>()));
453 pthread_setname_np (t->native_handle(), "encode-worker");
455 _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional<EncodeServerDescription>()));
460 for (auto i: EncodeServerFinder::instance()->servers()) {
461 if (!i.current_link_version()) {
465 LOG_GENERAL (N_("Adding %1 worker threads for remote %2"), i.threads(), i.host_name ());
466 for (int j = 0; j < i.threads(); ++j) {
467 _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, i));
471 _writer.set_encoder_threads(_threads->size());