2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <boost/foreach.hpp>
49 /* OS X strikes again */
61 using boost::shared_ptr;
62 using boost::weak_ptr;
63 using boost::dynamic_pointer_cast;
64 using boost::optional;
66 using namespace dcpomatic;
68 Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
73 , _queued_full_in_memory (0)
74 /* These will be reset to sensible values when J2KEncoder is created */
75 , _maximum_frames_in_memory (8)
76 , _maximum_queue_size (8)
82 shared_ptr<Job> job = _job.lock ();
83 DCPOMATIC_ASSERT (job);
86 list<DCPTimePeriod> const reels = _film->reels ();
87 BOOST_FOREACH (DCPTimePeriod p, reels) {
88 _reels.push_back (ReelWriter (film, p, job, reel_index++, reels.size(), _film->content_summary(p)));
91 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
92 and captions arrive to the Writer in sequence. This is not so for video.
94 _audio_reel = _reels.begin ();
95 _subtitle_reel = _reels.begin ();
96 BOOST_FOREACH (DCPTextTrack i, _film->closed_caption_tracks()) {
97 _caption_reels[i] = _reels.begin ();
100 /* Check that the signer is OK if we need one */
102 if (_film->is_signed() && !Config::instance()->signer_chain()->valid(&reason)) {
103 throw InvalidSignerError (reason);
110 _thread = new boost::thread (boost::bind (&Writer::thread, this));
111 #ifdef DCPOMATIC_LINUX
112 pthread_setname_np (_thread->native_handle(), "writer");
118 terminate_thread (false);
121 /** Pass a video frame to the writer for writing to disk at some point.
122 * This method can be called with frames out of order.
123 * @param encoded JPEG2000-encoded data.
124 * @param frame Frame index within the DCP.
125 * @param eyes Eyes that this frame image is for.
128 Writer::write (Data encoded, Frame frame, Eyes eyes)
130 boost::mutex::scoped_lock lock (_state_mutex);
132 while (_queued_full_in_memory > _maximum_frames_in_memory) {
133 /* There are too many full frames in memory; wake the main writer thread and
134 wait until it sorts everything out */
135 _empty_condition.notify_all ();
136 _full_condition.wait (lock);
140 qi.type = QueueItem::FULL;
141 qi.encoded = encoded;
142 qi.reel = video_reel (frame);
143 qi.frame = frame - _reels[qi.reel].start ();
145 if (_film->three_d() && eyes == EYES_BOTH) {
146 /* 2D material in a 3D DCP; fake the 3D */
148 _queue.push_back (qi);
149 ++_queued_full_in_memory;
150 qi.eyes = EYES_RIGHT;
151 _queue.push_back (qi);
152 ++_queued_full_in_memory;
155 _queue.push_back (qi);
156 ++_queued_full_in_memory;
159 /* Now there's something to do: wake anything wait()ing on _empty_condition */
160 _empty_condition.notify_all ();
164 Writer::can_repeat (Frame frame) const
166 return frame > _reels[video_reel(frame)].start();
169 /** Repeat the last frame that was written to a reel as a new frame.
170 * @param frame Frame index within the DCP of the new (repeated) frame.
171 * @param eyes Eyes that this repeated frame image is for.
174 Writer::repeat (Frame frame, Eyes eyes)
176 boost::mutex::scoped_lock lock (_state_mutex);
178 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
179 /* The queue is too big, and the main writer thread can run and fix it, so
180 wake it and wait until it has done.
182 _empty_condition.notify_all ();
183 _full_condition.wait (lock);
187 qi.type = QueueItem::REPEAT;
188 qi.reel = video_reel (frame);
189 qi.frame = frame - _reels[qi.reel].start ();
190 if (_film->three_d() && eyes == EYES_BOTH) {
192 _queue.push_back (qi);
193 qi.eyes = EYES_RIGHT;
194 _queue.push_back (qi);
197 _queue.push_back (qi);
200 /* Now there's something to do: wake anything wait()ing on _empty_condition */
201 _empty_condition.notify_all ();
205 Writer::fake_write (Frame frame, Eyes eyes)
207 boost::mutex::scoped_lock lock (_state_mutex);
209 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
210 /* The queue is too big, and the main writer thread can run and fix it, so
211 wake it and wait until it has done.
213 _empty_condition.notify_all ();
214 _full_condition.wait (lock);
217 size_t const reel = video_reel (frame);
218 Frame const reel_frame = frame - _reels[reel].start ();
221 qi.type = QueueItem::FAKE;
224 shared_ptr<InfoFileHandle> info_file = _film->info_file_handle(_reels[reel].period(), true);
225 qi.size = _reels[reel].read_frame_info(info_file, reel_frame, eyes).size;
229 qi.frame = reel_frame;
230 if (_film->three_d() && eyes == EYES_BOTH) {
232 _queue.push_back (qi);
233 qi.eyes = EYES_RIGHT;
234 _queue.push_back (qi);
237 _queue.push_back (qi);
240 /* Now there's something to do: wake anything wait()ing on _empty_condition */
241 _empty_condition.notify_all ();
244 /** Write some audio frames to the DCP.
245 * @param audio Audio data.
246 * @param time Time of this data within the DCP.
247 * This method is not thread safe.
250 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
252 DCPOMATIC_ASSERT (audio);
254 int const afr = _film->audio_frame_rate();
256 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
258 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
263 if (_audio_reel == _reels.end ()) {
264 /* This audio is off the end of the last reel; ignore it */
268 if (end <= _audio_reel->period().to) {
269 /* Easy case: we can write all the audio to this reel */
270 _audio_reel->write (audio);
272 } else if (_audio_reel->period().to <= t) {
273 /* This reel is entirely before the start of our audio; just skip the reel */
276 /* This audio is over a reel boundary; split the audio into two and write the first part */
277 DCPTime part_lengths[2] = {
278 _audio_reel->period().to - t,
279 end - _audio_reel->period().to
282 Frame part_frames[2] = {
283 part_lengths[0].frames_ceil(afr),
284 part_lengths[1].frames_ceil(afr)
287 if (part_frames[0]) {
288 shared_ptr<AudioBuffers> part (new AudioBuffers (audio->channels(), part_frames[0]));
289 part->copy_from (audio.get(), part_frames[0], 0, 0);
290 _audio_reel->write (part);
293 if (part_frames[1]) {
294 shared_ptr<AudioBuffers> part (new AudioBuffers (audio->channels(), part_frames[1]));
295 part->copy_from (audio.get(), part_frames[1], part_frames[0], 0);
302 t += part_lengths[0];
307 /** This must be called from Writer::thread() with an appropriate lock held */
309 Writer::have_sequenced_image_at_queue_head ()
311 if (_queue.empty ()) {
317 QueueItem const & f = _queue.front();
318 ReelWriter const & reel = _reels[f.reel];
320 /* The queue should contain only EYES_LEFT/EYES_RIGHT pairs or EYES_BOTH */
322 if (f.eyes == EYES_BOTH) {
324 return f.frame == (reel.last_written_video_frame() + 1);
329 if (reel.last_written_eyes() == EYES_LEFT && f.frame == reel.last_written_video_frame() && f.eyes == EYES_RIGHT) {
333 if (reel.last_written_eyes() == EYES_RIGHT && f.frame == (reel.last_written_video_frame() + 1) && f.eyes == EYES_LEFT) {
346 boost::mutex::scoped_lock lock (_state_mutex);
350 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
351 /* We've got something to do: go and do it */
355 /* Nothing to do: wait until something happens which may indicate that we do */
356 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
357 _empty_condition.wait (lock);
358 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
361 if (_finish && _queue.empty()) {
365 /* We stop here if we have been asked to finish, and if either the queue
366 is empty or we do not have a sequenced image at its head (if this is the
367 case we will never terminate as no new frames will be sent once
370 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
371 /* (Hopefully temporarily) log anything that was not written */
372 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
373 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
374 for (list<QueueItem>::const_iterator i = _queue.begin(); i != _queue.end(); ++i) {
375 if (i->type == QueueItem::FULL) {
376 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i->frame, (int) i->eyes);
378 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i->size, i->frame, (int) i->eyes);
385 /* Write any frames that we can write; i.e. those that are in sequence. */
386 while (have_sequenced_image_at_queue_head ()) {
387 QueueItem qi = _queue.front ();
389 if (qi.type == QueueItem::FULL && qi.encoded) {
390 --_queued_full_in_memory;
395 ReelWriter& reel = _reels[qi.reel];
398 case QueueItem::FULL:
399 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
401 qi.encoded = Data (_film->j2c_path (qi.reel, qi.frame, qi.eyes, false));
403 reel.write (qi.encoded, qi.frame, qi.eyes);
406 case QueueItem::FAKE:
407 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
408 reel.fake_write (qi.frame, qi.eyes, qi.size);
411 case QueueItem::REPEAT:
412 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
413 reel.repeat_write (qi.frame, qi.eyes);
419 _full_condition.notify_all ();
422 while (_queued_full_in_memory > _maximum_frames_in_memory) {
423 /* Too many frames in memory which can't yet be written to the stream.
424 Write some FULL frames to disk.
427 /* Find one from the back of the queue */
429 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
430 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
434 DCPOMATIC_ASSERT (i != _queue.rend());
436 /* For the log message below */
437 int const awaiting = _reels[_queue.front().reel].last_written_video_frame() + 1;
440 /* i is valid here, even though we don't hold a lock on the mutex,
441 since list iterators are unaffected by insertion and only this
442 thread could erase the last item in the list.
445 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
447 i->encoded->write_via_temp (
448 _film->j2c_path (i->reel, i->frame, i->eyes, true),
449 _film->j2c_path (i->reel, i->frame, i->eyes, false)
454 --_queued_full_in_memory;
455 _full_condition.notify_all ();
465 Writer::terminate_thread (bool can_throw)
467 boost::mutex::scoped_lock lock (_state_mutex);
473 _empty_condition.notify_all ();
474 _full_condition.notify_all ();
477 if (_thread->joinable ()) {
496 LOG_GENERAL_NC ("Terminating writer thread");
498 terminate_thread (true);
500 LOG_GENERAL_NC ("Finishing ReelWriters");
502 BOOST_FOREACH (ReelWriter& i, _reels) {
506 LOG_GENERAL_NC ("Writing XML");
508 dcp::DCP dcp (_film->dir (_film->dcp_name()));
510 shared_ptr<dcp::CPL> cpl (
513 _film->dcp_content_type()->libdcp_kind ()
519 /* Calculate digests for each reel in parallel */
521 shared_ptr<Job> job = _job.lock ();
522 job->sub (_("Computing digests"));
524 boost::asio::io_service service;
525 boost::thread_group pool;
527 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
529 int const threads = max (1, Config::instance()->master_encoding_threads ());
531 for (int i = 0; i < threads; ++i) {
532 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
535 BOOST_FOREACH (ReelWriter& i, _reels) {
536 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
537 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
544 /* Add reels to CPL */
546 BOOST_FOREACH (ReelWriter& i, _reels) {
547 cpl->add (i.create_reel (_reel_assets, _fonts));
550 dcp::XMLMetadata meta;
551 meta.annotation_text = cpl->annotation_text ();
552 meta.creator = Config::instance()->dcp_creator ();
553 if (meta.creator.empty ()) {
554 meta.creator = String::compose ("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
556 meta.issuer = Config::instance()->dcp_issuer ();
557 if (meta.issuer.empty ()) {
558 meta.issuer = String::compose ("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
560 meta.set_issue_date_now ();
562 cpl->set_metadata (meta);
563 cpl->set_ratings (vector_to_list(_film->ratings()));
564 cpl->set_content_version_label_text (_film->content_version());
566 shared_ptr<const dcp::CertificateChain> signer;
567 if (_film->is_signed ()) {
568 signer = Config::instance()->signer_chain ();
569 /* We did check earlier, but check again here to be on the safe side */
571 if (!signer->valid (&reason)) {
572 throw InvalidSignerError (reason);
576 dcp.write_xml (_film->interop () ? dcp::INTEROP : dcp::SMPTE, meta, signer, Config::instance()->dcp_metadata_filename_format());
579 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
582 write_cover_sheet ();
586 Writer::write_cover_sheet ()
588 boost::filesystem::path const cover = _film->file ("COVER_SHEET.txt");
589 FILE* f = fopen_boost (cover, "w");
591 throw OpenFileError (cover, errno, OpenFileError::WRITE);
594 string text = Config::instance()->cover_sheet ();
595 boost::algorithm::replace_all (text, "$CPL_NAME", _film->name());
596 boost::algorithm::replace_all (text, "$TYPE", _film->dcp_content_type()->pretty_name());
597 boost::algorithm::replace_all (text, "$CONTAINER", _film->container()->container_nickname());
598 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _film->isdcf_metadata().audio_language);
600 optional<string> subtitle_language;
601 BOOST_FOREACH (shared_ptr<Content> i, _film->content()) {
602 BOOST_FOREACH (shared_ptr<TextContent> j, i->text) {
603 if (j->type() == TEXT_OPEN_SUBTITLE && j->use()) {
604 subtitle_language = j->language ();
608 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_language.get_value_or("None"));
610 boost::uintmax_t size = 0;
612 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(_film->dir(_film->dcp_name()));
613 i != boost::filesystem::recursive_directory_iterator();
615 if (boost::filesystem::is_regular_file (i->path ())) {
616 size += boost::filesystem::file_size (i->path ());
620 if (size > (1000000000L)) {
621 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
623 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
626 pair<int, int> ch = audio_channel_types (_film->mapped_audio_channels(), _film->audio_channels());
627 string description = String::compose("%1.%2", ch.first, ch.second);
629 if (description == "0.0") {
630 description = _("None");
631 } else if (description == "1.0") {
632 description = _("Mono");
633 } else if (description == "2.0") {
634 description = _("Stereo");
636 boost::algorithm::replace_all (text, "$AUDIO", description);
639 _film->length().split (_film->video_frame_rate(), h, m, s, fr);
641 if (h == 0 && m == 0) {
642 length = String::compose("%1s", s);
643 } else if (h == 0 && m > 0) {
644 length = String::compose("%1m%2s", m, s);
645 } else if (h > 0 && m > 0) {
646 length = String::compose("%1h%2m%3s", h, m, s);
649 boost::algorithm::replace_all (text, "$LENGTH", length);
651 checked_fwrite (text.c_str(), text.length(), f, cover);
655 /** @param frame Frame index within the whole DCP.
656 * @return true if we can fake-write this frame.
659 Writer::can_fake_write (Frame frame) const
661 if (_film->encrypted()) {
662 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
666 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
667 parameters in the asset writer.
670 ReelWriter const & reel = _reels[video_reel(frame)];
672 /* Make frame relative to the start of the reel */
673 frame -= reel.start ();
674 return (frame != 0 && frame < reel.first_nonexistant_frame());
677 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
679 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
681 vector<ReelWriter>::iterator* reel = 0;
684 case TEXT_OPEN_SUBTITLE:
685 reel = &_subtitle_reel;
687 case TEXT_CLOSED_CAPTION:
688 DCPOMATIC_ASSERT (track);
689 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
690 reel = &_caption_reels[*track];
693 DCPOMATIC_ASSERT (false);
696 DCPOMATIC_ASSERT (*reel != _reels.end());
697 while ((*reel)->period().to <= period.from) {
699 DCPOMATIC_ASSERT (*reel != _reels.end());
702 (*reel)->write (text, type, track, period);
706 Writer::write (list<shared_ptr<Font> > fonts)
708 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
710 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
712 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
719 _fonts.push_back (i);
725 operator< (QueueItem const & a, QueueItem const & b)
727 if (a.reel != b.reel) {
728 return a.reel < b.reel;
731 if (a.frame != b.frame) {
732 return a.frame < b.frame;
735 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
739 operator== (QueueItem const & a, QueueItem const & b)
741 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
745 Writer::set_encoder_threads (int threads)
747 boost::mutex::scoped_lock lm (_state_mutex);
748 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
749 _maximum_queue_size = threads * 16;
753 Writer::write (ReferencedReelAsset asset)
755 _reel_assets.push_back (asset);
759 Writer::video_reel (int frame) const
761 DCPTime t = DCPTime::from_frames (frame, _film->video_frame_rate ());
763 while (i < _reels.size() && !_reels[i].period().contains (t)) {
767 DCPOMATIC_ASSERT (i < _reels.size ());
772 Writer::set_digest_progress (Job* job, float progress)
774 /* I believe this is thread-safe */
775 _digest_progresses[boost::this_thread::get_id()] = progress;
777 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
778 float min_progress = FLT_MAX;
779 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
780 min_progress = min (min_progress, i->second);
783 job->set_progress (min_progress);