2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <dcp/reel_mxf.h>
42 #include <boost/foreach.hpp>
50 /* OS X strikes again */
62 using boost::shared_ptr;
63 using boost::weak_ptr;
64 using boost::dynamic_pointer_cast;
65 using boost::optional;
66 #if BOOST_VERSION >= 106100
67 using namespace boost::placeholders;
71 using namespace dcpomatic;
76 ignore_progress (float)
82 /** @param j Job to report progress to, or 0.
83 * @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
85 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
86 : WeakConstFilm (weak_film)
89 , _queued_full_in_memory (0)
90 /* These will be reset to sensible values when J2KEncoder is created */
91 , _maximum_frames_in_memory (8)
92 , _maximum_queue_size (8)
97 , _text_only (text_only)
98 , _have_subtitles (false)
100 shared_ptr<Job> job = _job.lock ();
103 list<DCPTimePeriod> const reels = film()->reels();
104 BOOST_FOREACH (DCPTimePeriod p, reels) {
105 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
108 _last_written.resize (reels.size());
110 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
111 and captions arrive to the Writer in sequence. This is not so for video.
113 _audio_reel = _reels.begin ();
114 _subtitle_reel = _reels.begin ();
115 BOOST_FOREACH (DCPTextTrack i, film()->closed_caption_tracks()) {
116 _caption_reels[i] = _reels.begin ();
118 _atmos_reel = _reels.begin ();
120 /* Check that the signer is OK */
122 if (!Config::instance()->signer_chain()->valid(&reason)) {
123 throw InvalidSignerError (reason);
131 _thread = boost::thread (boost::bind(&Writer::thread, this));
132 #ifdef DCPOMATIC_LINUX
133 pthread_setname_np (_thread.native_handle(), "writer");
141 terminate_thread (false);
145 /** Pass a video frame to the writer for writing to disk at some point.
146 * This method can be called with frames out of order.
147 * @param encoded JPEG2000-encoded data.
148 * @param frame Frame index within the DCP.
149 * @param eyes Eyes that this frame image is for.
152 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
154 boost::mutex::scoped_lock lock (_state_mutex);
156 while (_queued_full_in_memory > _maximum_frames_in_memory) {
157 /* There are too many full frames in memory; wake the main writer thread and
158 wait until it sorts everything out */
159 _empty_condition.notify_all ();
160 _full_condition.wait (lock);
164 qi.type = QueueItem::FULL;
165 qi.encoded = encoded;
166 qi.reel = video_reel (frame);
167 qi.frame = frame - _reels[qi.reel].start ();
169 if (film()->three_d() && eyes == EYES_BOTH) {
170 /* 2D material in a 3D DCP; fake the 3D */
172 _queue.push_back (qi);
173 ++_queued_full_in_memory;
174 qi.eyes = EYES_RIGHT;
175 _queue.push_back (qi);
176 ++_queued_full_in_memory;
179 _queue.push_back (qi);
180 ++_queued_full_in_memory;
183 /* Now there's something to do: wake anything wait()ing on _empty_condition */
184 _empty_condition.notify_all ();
188 Writer::can_repeat (Frame frame) const
190 return frame > _reels[video_reel(frame)].start();
193 /** Repeat the last frame that was written to a reel as a new frame.
194 * @param frame Frame index within the DCP of the new (repeated) frame.
195 * @param eyes Eyes that this repeated frame image is for.
198 Writer::repeat (Frame frame, Eyes eyes)
200 boost::mutex::scoped_lock lock (_state_mutex);
202 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
203 /* The queue is too big, and the main writer thread can run and fix it, so
204 wake it and wait until it has done.
206 _empty_condition.notify_all ();
207 _full_condition.wait (lock);
211 qi.type = QueueItem::REPEAT;
212 qi.reel = video_reel (frame);
213 qi.frame = frame - _reels[qi.reel].start ();
214 if (film()->three_d() && eyes == EYES_BOTH) {
216 _queue.push_back (qi);
217 qi.eyes = EYES_RIGHT;
218 _queue.push_back (qi);
221 _queue.push_back (qi);
224 /* Now there's something to do: wake anything wait()ing on _empty_condition */
225 _empty_condition.notify_all ();
229 Writer::fake_write (Frame frame, Eyes eyes)
231 boost::mutex::scoped_lock lock (_state_mutex);
233 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
234 /* The queue is too big, and the main writer thread can run and fix it, so
235 wake it and wait until it has done.
237 _empty_condition.notify_all ();
238 _full_condition.wait (lock);
241 size_t const reel = video_reel (frame);
242 Frame const frame_in_reel = frame - _reels[reel].start ();
245 qi.type = QueueItem::FAKE;
248 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
249 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
253 qi.frame = frame_in_reel;
254 if (film()->three_d() && eyes == EYES_BOTH) {
256 _queue.push_back (qi);
257 qi.eyes = EYES_RIGHT;
258 _queue.push_back (qi);
261 _queue.push_back (qi);
264 /* Now there's something to do: wake anything wait()ing on _empty_condition */
265 _empty_condition.notify_all ();
268 /** Write some audio frames to the DCP.
269 * @param audio Audio data.
270 * @param time Time of this data within the DCP.
271 * This method is not thread safe.
274 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
276 DCPOMATIC_ASSERT (audio);
278 int const afr = film()->audio_frame_rate();
280 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
282 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
287 if (_audio_reel == _reels.end ()) {
288 /* This audio is off the end of the last reel; ignore it */
292 if (end <= _audio_reel->period().to) {
293 /* Easy case: we can write all the audio to this reel */
294 _audio_reel->write (audio);
296 } else if (_audio_reel->period().to <= t) {
297 /* This reel is entirely before the start of our audio; just skip the reel */
300 /* This audio is over a reel boundary; split the audio into two and write the first part */
301 DCPTime part_lengths[2] = {
302 _audio_reel->period().to - t,
303 end - _audio_reel->period().to
306 Frame part_frames[2] = {
307 part_lengths[0].frames_ceil(afr),
308 part_lengths[1].frames_ceil(afr)
311 if (part_frames[0]) {
312 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
313 _audio_reel->write (part);
316 if (part_frames[1]) {
317 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
323 t += part_lengths[0];
330 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
332 if (_atmos_reel->period().to == time) {
334 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
337 /* We assume that we get a video frame's worth of data here */
338 _atmos_reel->write (atmos, metadata);
342 /** Caller must hold a lock on _state_mutex */
344 Writer::have_sequenced_image_at_queue_head ()
346 if (_queue.empty ()) {
351 QueueItem const & f = _queue.front();
352 return _last_written[f.reel].next(f);
357 Writer::LastWritten::next (QueueItem qi) const
359 if (qi.eyes == EYES_BOTH) {
361 return qi.frame == (_frame + 1);
366 if (_eyes == EYES_LEFT && qi.frame == _frame && qi.eyes == EYES_RIGHT) {
370 if (_eyes == EYES_RIGHT && qi.frame == (_frame + 1) && qi.eyes == EYES_LEFT) {
379 Writer::LastWritten::update (QueueItem qi)
392 boost::mutex::scoped_lock lock (_state_mutex);
396 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
397 /* We've got something to do: go and do it */
401 /* Nothing to do: wait until something happens which may indicate that we do */
402 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
403 _empty_condition.wait (lock);
404 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
407 /* We stop here if we have been asked to finish, and if either the queue
408 is empty or we do not have a sequenced image at its head (if this is the
409 case we will never terminate as no new frames will be sent once
412 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
413 /* (Hopefully temporarily) log anything that was not written */
414 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
415 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
416 BOOST_FOREACH (QueueItem const& i, _queue) {
417 if (i.type == QueueItem::FULL) {
418 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
420 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
427 /* Write any frames that we can write; i.e. those that are in sequence. */
428 while (have_sequenced_image_at_queue_head ()) {
429 QueueItem qi = _queue.front ();
430 _last_written[qi.reel].update (qi);
432 if (qi.type == QueueItem::FULL && qi.encoded) {
433 --_queued_full_in_memory;
438 ReelWriter& reel = _reels[qi.reel];
441 case QueueItem::FULL:
442 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
444 qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
446 reel.write (qi.encoded, qi.frame, qi.eyes);
449 case QueueItem::FAKE:
450 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
451 reel.fake_write (qi.size);
454 case QueueItem::REPEAT:
455 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
456 reel.repeat_write (qi.frame, qi.eyes);
462 _full_condition.notify_all ();
465 while (_queued_full_in_memory > _maximum_frames_in_memory) {
466 /* Too many frames in memory which can't yet be written to the stream.
467 Write some FULL frames to disk.
470 /* Find one from the back of the queue */
472 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
473 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
477 DCPOMATIC_ASSERT (i != _queue.rend());
479 /* For the log message below */
480 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
483 /* i is valid here, even though we don't hold a lock on the mutex,
484 since list iterators are unaffected by insertion and only this
485 thread could erase the last item in the list.
488 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
490 i->encoded->write_via_temp (
491 film()->j2c_path(i->reel, i->frame, i->eyes, true),
492 film()->j2c_path(i->reel, i->frame, i->eyes, false)
497 --_queued_full_in_memory;
498 _full_condition.notify_all ();
508 Writer::terminate_thread (bool can_throw)
510 boost::this_thread::disable_interruption dis;
512 boost::mutex::scoped_lock lock (_state_mutex);
515 _empty_condition.notify_all ();
516 _full_condition.notify_all ();
529 /** @param output_dcp Path to DCP folder to write */
531 Writer::finish (boost::filesystem::path output_dcp)
533 if (_thread.joinable()) {
534 LOG_GENERAL_NC ("Terminating writer thread");
535 terminate_thread (true);
538 LOG_GENERAL_NC ("Finishing ReelWriters");
540 BOOST_FOREACH (ReelWriter& i, _reels) {
541 i.finish (output_dcp);
544 LOG_GENERAL_NC ("Writing XML");
546 dcp::DCP dcp (output_dcp);
548 shared_ptr<dcp::CPL> cpl (
551 film()->dcp_content_type()->libdcp_kind ()
557 /* Calculate digests for each reel in parallel */
559 shared_ptr<Job> job = _job.lock ();
561 job->sub (_("Computing digests"));
564 boost::asio::io_service service;
565 boost::thread_group pool;
567 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
569 int const threads = max (1, Config::instance()->master_encoding_threads ());
571 for (int i = 0; i < threads; ++i) {
572 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
575 boost::function<void (float)> set_progress;
577 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
579 set_progress = &ignore_progress;
582 BOOST_FOREACH (ReelWriter& i, _reels) {
583 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
585 service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
593 BOOST_FOREACH (ReelWriter& i, _reels) {
594 cpl->add (i.create_reel(_reel_assets, _fonts, output_dcp, _have_subtitles, _have_closed_captions));
599 string creator = Config::instance()->dcp_creator();
600 if (creator.empty()) {
601 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
604 string issuer = Config::instance()->dcp_issuer();
605 if (issuer.empty()) {
606 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
609 cpl->set_ratings (film()->ratings());
611 vector<dcp::ContentVersion> cv;
612 BOOST_FOREACH (string i, film()->content_versions()) {
613 cv.push_back (dcp::ContentVersion(i));
615 cpl->set_content_versions (cv);
617 cpl->set_full_content_title_text (film()->name());
618 cpl->set_full_content_title_text_language (film()->name_language());
619 cpl->set_release_territory (film()->release_territory());
620 cpl->set_version_number (film()->version_number());
621 cpl->set_status (film()->status());
622 cpl->set_chain (film()->chain());
623 cpl->set_distributor (film()->distributor());
624 cpl->set_facility (film()->facility());
625 cpl->set_luminance (film()->luminance());
627 list<int> ac = film()->mapped_audio_channels();
628 dcp::MCASoundField field = (
629 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSL)) != ac.end() ||
630 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSR)) != ac.end()
631 ) ? dcp::SEVEN_POINT_ONE : dcp::FIVE_POINT_ONE;
633 dcp::MainSoundConfiguration msc (field, film()->audio_channels());
634 BOOST_FOREACH (int i, ac) {
635 if (i < film()->audio_channels()) {
636 msc.set_mapping (i, static_cast<dcp::Channel>(i));
640 cpl->set_main_sound_configuration (msc.to_string());
641 cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
642 cpl->set_main_picture_stored_area (film()->frame_size());
643 cpl->set_main_picture_active_area (film()->active_area());
645 vector<dcp::LanguageTag> sl = film()->subtitle_languages();
647 cpl->set_additional_subtitle_languages(std::vector<dcp::LanguageTag>(sl.begin() + 1, sl.end()));
650 shared_ptr<const dcp::CertificateChain> signer;
651 signer = Config::instance()->signer_chain ();
652 /* We did check earlier, but check again here to be on the safe side */
654 if (!signer->valid (&reason)) {
655 throw InvalidSignerError (reason);
659 film()->interop() ? dcp::INTEROP : dcp::SMPTE,
662 dcp::LocalTime().as_string(),
663 String::compose("Created by libdcp %1", dcp::version),
665 Config::instance()->dcp_metadata_filename_format()
669 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
672 write_cover_sheet (output_dcp);
676 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
678 boost::filesystem::path const cover = film()->file("COVER_SHEET.txt");
679 FILE* f = fopen_boost (cover, "w");
681 throw OpenFileError (cover, errno, OpenFileError::WRITE);
684 string text = Config::instance()->cover_sheet ();
685 boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
686 boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
687 boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
688 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", film()->isdcf_metadata().audio_language);
690 vector<dcp::LanguageTag> subtitle_languages = film()->subtitle_languages();
691 if (subtitle_languages.empty()) {
692 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", "None");
694 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.front().description());
697 boost::uintmax_t size = 0;
699 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(output_dcp);
700 i != boost::filesystem::recursive_directory_iterator();
702 if (boost::filesystem::is_regular_file (i->path ())) {
703 size += boost::filesystem::file_size (i->path ());
707 if (size > (1000000000L)) {
708 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
710 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
713 pair<int, int> ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
714 string description = String::compose("%1.%2", ch.first, ch.second);
716 if (description == "0.0") {
717 description = _("None");
718 } else if (description == "1.0") {
719 description = _("Mono");
720 } else if (description == "2.0") {
721 description = _("Stereo");
723 boost::algorithm::replace_all (text, "$AUDIO", description);
726 film()->length().split(film()->video_frame_rate(), h, m, s, fr);
728 if (h == 0 && m == 0) {
729 length = String::compose("%1s", s);
730 } else if (h == 0 && m > 0) {
731 length = String::compose("%1m%2s", m, s);
732 } else if (h > 0 && m > 0) {
733 length = String::compose("%1h%2m%3s", h, m, s);
736 boost::algorithm::replace_all (text, "$LENGTH", length);
738 checked_fwrite (text.c_str(), text.length(), f, cover);
742 /** @param frame Frame index within the whole DCP.
743 * @return true if we can fake-write this frame.
746 Writer::can_fake_write (Frame frame) const
748 if (film()->encrypted()) {
749 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
753 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
754 parameters in the asset writer.
757 ReelWriter const & reel = _reels[video_reel(frame)];
759 /* Make frame relative to the start of the reel */
760 frame -= reel.start ();
761 return (frame != 0 && frame < reel.first_nonexistant_frame());
764 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
766 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
768 vector<ReelWriter>::iterator* reel = 0;
771 case TEXT_OPEN_SUBTITLE:
772 reel = &_subtitle_reel;
773 _have_subtitles = true;
775 case TEXT_CLOSED_CAPTION:
776 DCPOMATIC_ASSERT (track);
777 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
778 reel = &_caption_reels[*track];
779 _have_closed_captions.insert (*track);
782 DCPOMATIC_ASSERT (false);
785 DCPOMATIC_ASSERT (*reel != _reels.end());
786 while ((*reel)->period().to <= period.from) {
788 DCPOMATIC_ASSERT (*reel != _reels.end());
791 (*reel)->write (text, type, track, period);
795 Writer::write (list<shared_ptr<Font> > fonts)
797 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
799 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
801 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
808 _fonts.push_back (i);
814 operator< (QueueItem const & a, QueueItem const & b)
816 if (a.reel != b.reel) {
817 return a.reel < b.reel;
820 if (a.frame != b.frame) {
821 return a.frame < b.frame;
824 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
828 operator== (QueueItem const & a, QueueItem const & b)
830 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
834 Writer::set_encoder_threads (int threads)
836 boost::mutex::scoped_lock lm (_state_mutex);
837 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
838 _maximum_queue_size = threads * 16;
842 Writer::write (ReferencedReelAsset asset)
844 _reel_assets.push_back (asset);
848 Writer::video_reel (int frame) const
850 DCPTime t = DCPTime::from_frames (frame, film()->video_frame_rate());
852 while (i < _reels.size() && !_reels[i].period().contains (t)) {
856 DCPOMATIC_ASSERT (i < _reels.size ());
861 Writer::set_digest_progress (Job* job, float progress)
863 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
865 _digest_progresses[boost::this_thread::get_id()] = progress;
866 float min_progress = FLT_MAX;
867 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
868 min_progress = min (min_progress, i->second);
871 job->set_progress (min_progress);
878 /** Calculate hashes for any referenced MXF assets which do not already have one */
880 Writer::calculate_referenced_digests (boost::function<void (float)> set_progress)
882 BOOST_FOREACH (ReferencedReelAsset const& i, _reel_assets) {
883 shared_ptr<dcp::ReelMXF> mxf = dynamic_pointer_cast<dcp::ReelMXF>(i.asset);
884 if (mxf && !mxf->hash()) {
885 mxf->asset_ref().asset()->hash (set_progress);
886 mxf->set_hash (mxf->asset_ref().asset()->hash());