2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/j2k_encoder.cc
23 * @brief J2K encoder class.
27 #include "compose.hpp"
30 #include "dcp_video.h"
31 #include "dcpomatic_log.h"
32 #include "encode_server_description.h"
33 #include "encode_server_finder.h"
35 #include "j2k_encoder.h"
37 #include "player_video.h"
40 #include <libcxml/cxml.h>
49 using std::make_shared;
50 using std::shared_ptr;
52 using boost::optional;
54 using namespace dcpomatic;
57 /** @param film Film that we are encoding.
58 * @param writer Writer that we are using.
60 J2KEncoder::J2KEncoder(shared_ptr<const Film> film, Writer& writer)
68 J2KEncoder::~J2KEncoder ()
70 _server_found_connection.disconnect();
72 boost::mutex::scoped_lock lm (_threads_mutex);
80 _server_found_connection = EncodeServerFinder::instance()->ServersListChanged.connect(
81 boost::bind(&J2KEncoder::servers_list_changed, this)
83 servers_list_changed ();
90 boost::mutex::scoped_lock lock (_queue_mutex);
92 LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ());
94 /* Keep waking workers until the queue is empty */
95 while (!_queue.empty ()) {
97 _empty_condition.notify_all ();
98 _full_condition.wait (lock);
103 LOG_GENERAL_NC (N_("Terminating encoder threads"));
106 boost::mutex::scoped_lock lm (_threads_mutex);
107 terminate_threads ();
110 /* Something might have been thrown during terminate_threads */
113 LOG_GENERAL (N_("Mopping up %1"), _queue.size());
115 /* The following sequence of events can occur in the above code:
116 1. a remote worker takes the last image off the queue
117 2. the loop above terminates
118 3. the remote worker fails to encode the image and puts it back on the queue
119 4. the remote worker is then terminated by terminate_threads
121 So just mop up anything left in the queue here.
124 for (auto const& i: _queue) {
125 LOG_GENERAL(N_("Encode left-over frame %1"), i.index());
128 make_shared<dcp::ArrayData>(i.encode_locally()),
133 } catch (std::exception& e) {
134 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
140 /** @return an estimate of the current number of frames we are encoding per second,
144 J2KEncoder::current_encoding_rate () const
146 return _history.rate ();
150 /** @return Number of video frames that have been queued for encoding */
152 J2KEncoder::video_frames_enqueued () const
154 if (!_last_player_video_time) {
158 return _last_player_video_time->frames_floor (_film->video_frame_rate ());
162 /** Should be called when a frame has been encoded successfully */
164 J2KEncoder::frame_done ()
170 /** Called to request encoding of the next video frame in the DCP. This is called in order,
171 * so each time the supplied frame is the one after the previous one.
172 * pv represents one video frame, and could be empty if there is nothing to encode
173 * for this DCP frame.
175 * @param pv PlayerVideo to encode.
176 * @param time Time of \p pv within the DCP.
179 J2KEncoder::encode (shared_ptr<PlayerVideo> pv, DCPTime time)
185 boost::mutex::scoped_lock lm (_threads_mutex);
186 threads = _threads->size();
189 boost::mutex::scoped_lock queue_lock (_queue_mutex);
191 /* Wait until the queue has gone down a bit. Allow one thing in the queue even
192 when there are no threads.
194 while (_queue.size() >= (threads * 2) + 1) {
195 LOG_TIMING ("decoder-sleep queue=%1 threads=%2", _queue.size(), threads);
196 _full_condition.wait (queue_lock);
197 LOG_TIMING ("decoder-wake queue=%1 threads=%2", _queue.size(), threads);
201 /* Re-throw any exception raised by one of our threads. If more
202 than one has thrown an exception, only one will be rethrown, I think;
203 but then, if that happens something has gone badly wrong.
207 auto const position = time.frames_floor(_film->video_frame_rate());
209 if (_writer.can_fake_write(position)) {
210 /* We can fake-write this frame */
211 LOG_DEBUG_ENCODE("Frame @ %1 FAKE", to_string(time));
212 _writer.fake_write(position, pv->eyes ());
214 } else if (pv->has_j2k() && !_film->reencode_j2k()) {
215 LOG_DEBUG_ENCODE("Frame @ %1 J2K", to_string(time));
216 /* This frame already has J2K data, so just write it */
217 _writer.write(pv->j2k(), position, pv->eyes ());
219 } else if (_last_player_video[pv->eyes()] && _writer.can_repeat(position) && pv->same(_last_player_video[pv->eyes()])) {
220 LOG_DEBUG_ENCODE("Frame @ %1 REPEAT", to_string(time));
221 _writer.repeat(position, pv->eyes());
223 LOG_DEBUG_ENCODE("Frame @ %1 ENCODE", to_string(time));
224 /* Queue this new frame for encoding */
225 LOG_TIMING ("add-frame-to-queue queue=%1", _queue.size ());
226 _queue.push_back (DCPVideo(
229 _film->video_frame_rate(),
230 _film->j2k_bandwidth(),
234 /* The queue might not be empty any more, so notify anything which is
237 _empty_condition.notify_all ();
240 _last_player_video[pv->eyes()] = pv;
241 _last_player_video_time = time;
245 /** Caller must hold a lock on _threads_mutex */
247 J2KEncoder::terminate_threads ()
249 boost::this_thread::disable_interruption dis;
255 _threads->interrupt_all ();
257 _threads->join_all ();
258 } catch (exception& e) {
259 LOG_ERROR ("join() threw an exception: %1", e.what());
261 LOG_ERROR_NC ("join() threw an exception");
269 J2KEncoder::encoder_thread (optional<EncodeServerDescription> server)
272 start_of_thread ("J2KEncoder");
275 LOG_TIMING ("start-encoder-thread thread=%1 server=%2", thread_id (), server->host_name ());
277 LOG_TIMING ("start-encoder-thread thread=%1 server=localhost", thread_id ());
280 /* Number of seconds that we currently wait between attempts
281 to connect to the server; not relevant for localhost
284 int remote_backoff = 0;
288 LOG_TIMING ("encoder-sleep thread=%1", thread_id ());
289 boost::mutex::scoped_lock lock (_queue_mutex);
290 while (_queue.empty ()) {
291 _empty_condition.wait (lock);
294 LOG_TIMING ("encoder-wake thread=%1 queue=%2", thread_id(), _queue.size());
295 auto vf = _queue.front ();
297 /* We're about to commit to either encoding this frame or putting it back onto the queue,
298 so we must not be interrupted until one or other of these things have happened. This
299 block has thread interruption disabled.
302 boost::this_thread::disable_interruption dis;
304 LOG_TIMING ("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), vf.index(), static_cast<int>(vf.eyes()));
309 shared_ptr<Data> encoded;
311 /* We need to encode this input */
314 encoded = make_shared<dcp::ArrayData>(vf.encode_remotely(server.get()));
316 if (remote_backoff > 0) {
317 LOG_GENERAL ("%1 was lost, but now she is found; removing backoff", server->host_name ());
320 /* This job succeeded, so remove any backoff */
323 } catch (std::exception& e) {
324 if (remote_backoff < 60) {
326 remote_backoff += 10;
329 N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"),
330 vf.index(), server->host_name(), e.what(), remote_backoff
336 LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index());
337 encoded = make_shared<dcp::ArrayData>(vf.encode_locally());
338 LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index());
339 } catch (std::exception& e) {
340 /* This is very bad, so don't cope with it, just pass it on */
341 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
347 _writer.write(encoded, vf.index(), vf.eyes());
351 LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index());
352 _queue.push_front (vf);
357 if (remote_backoff > 0) {
358 boost::this_thread::sleep (boost::posix_time::seconds (remote_backoff));
361 /* The queue might not be full any more, so notify anything that is waiting on that */
363 _full_condition.notify_all ();
366 catch (boost::thread_interrupted& e) {
367 /* Ignore these and just stop the thread */
368 _full_condition.notify_all ();
373 /* Wake anything waiting on _full_condition so it can see the exception */
374 _full_condition.notify_all ();
379 J2KEncoder::servers_list_changed ()
381 boost::mutex::scoped_lock lm (_threads_mutex);
383 terminate_threads ();
384 _threads = make_shared<boost::thread_group>();
386 /* XXX: could re-use threads */
388 if (!Config::instance()->only_servers_encode ()) {
389 for (int i = 0; i < Config::instance()->master_encoding_threads (); ++i) {
390 #ifdef DCPOMATIC_LINUX
391 auto t = _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional<EncodeServerDescription>()));
392 pthread_setname_np (t->native_handle(), "encode-worker");
394 _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional<EncodeServerDescription>()));
399 for (auto i: EncodeServerFinder::instance()->servers()) {
400 if (!i.current_link_version()) {
404 LOG_GENERAL (N_("Adding %1 worker threads for remote %2"), i.threads(), i.host_name ());
405 for (int j = 0; j < i.threads(); ++j) {
406 _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, i));
410 _writer.set_encoder_threads(_threads->size());