2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
35 #include "font_data.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <dcp/reel_mxf.h>
49 /* OS X strikes again */
61 using std::shared_ptr;
63 using std::dynamic_pointer_cast;
64 using boost::optional;
65 #if BOOST_VERSION >= 106100
66 using namespace boost::placeholders;
70 using namespace dcpomatic;
75 ignore_progress (float)
81 /** @param j Job to report progress to, or 0.
82 * @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
84 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
85 : WeakConstFilm (weak_film)
88 , _queued_full_in_memory (0)
89 /* These will be reset to sensible values when J2KEncoder is created */
90 , _maximum_frames_in_memory (8)
91 , _maximum_queue_size (8)
96 , _text_only (text_only)
97 , _have_subtitles (false)
99 shared_ptr<Job> job = _job.lock ();
102 list<DCPTimePeriod> const reels = film()->reels();
103 for (auto p: reels) {
104 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
107 _last_written.resize (reels.size());
109 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
110 and captions arrive to the Writer in sequence. This is not so for video.
112 _audio_reel = _reels.begin ();
113 _subtitle_reel = _reels.begin ();
114 for (auto i: film()->closed_caption_tracks()) {
115 _caption_reels[i] = _reels.begin ();
117 _atmos_reel = _reels.begin ();
119 /* Check that the signer is OK */
121 if (!Config::instance()->signer_chain()->valid(&reason)) {
122 throw InvalidSignerError (reason);
130 _thread = boost::thread (boost::bind(&Writer::thread, this));
131 #ifdef DCPOMATIC_LINUX
132 pthread_setname_np (_thread.native_handle(), "writer");
140 terminate_thread (false);
144 /** Pass a video frame to the writer for writing to disk at some point.
145 * This method can be called with frames out of order.
146 * @param encoded JPEG2000-encoded data.
147 * @param frame Frame index within the DCP.
148 * @param eyes Eyes that this frame image is for.
151 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
153 boost::mutex::scoped_lock lock (_state_mutex);
155 while (_queued_full_in_memory > _maximum_frames_in_memory) {
156 /* There are too many full frames in memory; wake the main writer thread and
157 wait until it sorts everything out */
158 _empty_condition.notify_all ();
159 _full_condition.wait (lock);
163 qi.type = QueueItem::FULL;
164 qi.encoded = encoded;
165 qi.reel = video_reel (frame);
166 qi.frame = frame - _reels[qi.reel].start ();
168 if (film()->three_d() && eyes == EYES_BOTH) {
169 /* 2D material in a 3D DCP; fake the 3D */
171 _queue.push_back (qi);
172 ++_queued_full_in_memory;
173 qi.eyes = EYES_RIGHT;
174 _queue.push_back (qi);
175 ++_queued_full_in_memory;
178 _queue.push_back (qi);
179 ++_queued_full_in_memory;
182 /* Now there's something to do: wake anything wait()ing on _empty_condition */
183 _empty_condition.notify_all ();
187 Writer::can_repeat (Frame frame) const
189 return frame > _reels[video_reel(frame)].start();
192 /** Repeat the last frame that was written to a reel as a new frame.
193 * @param frame Frame index within the DCP of the new (repeated) frame.
194 * @param eyes Eyes that this repeated frame image is for.
197 Writer::repeat (Frame frame, Eyes eyes)
199 boost::mutex::scoped_lock lock (_state_mutex);
201 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
202 /* The queue is too big, and the main writer thread can run and fix it, so
203 wake it and wait until it has done.
205 _empty_condition.notify_all ();
206 _full_condition.wait (lock);
210 qi.type = QueueItem::REPEAT;
211 qi.reel = video_reel (frame);
212 qi.frame = frame - _reels[qi.reel].start ();
213 if (film()->three_d() && eyes == EYES_BOTH) {
215 _queue.push_back (qi);
216 qi.eyes = EYES_RIGHT;
217 _queue.push_back (qi);
220 _queue.push_back (qi);
223 /* Now there's something to do: wake anything wait()ing on _empty_condition */
224 _empty_condition.notify_all ();
228 Writer::fake_write (Frame frame, Eyes eyes)
230 boost::mutex::scoped_lock lock (_state_mutex);
232 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
233 /* The queue is too big, and the main writer thread can run and fix it, so
234 wake it and wait until it has done.
236 _empty_condition.notify_all ();
237 _full_condition.wait (lock);
240 size_t const reel = video_reel (frame);
241 Frame const frame_in_reel = frame - _reels[reel].start ();
244 qi.type = QueueItem::FAKE;
247 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
248 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
252 qi.frame = frame_in_reel;
253 if (film()->three_d() && eyes == EYES_BOTH) {
255 _queue.push_back (qi);
256 qi.eyes = EYES_RIGHT;
257 _queue.push_back (qi);
260 _queue.push_back (qi);
263 /* Now there's something to do: wake anything wait()ing on _empty_condition */
264 _empty_condition.notify_all ();
267 /** Write some audio frames to the DCP.
268 * @param audio Audio data.
269 * @param time Time of this data within the DCP.
270 * This method is not thread safe.
273 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
275 DCPOMATIC_ASSERT (audio);
277 int const afr = film()->audio_frame_rate();
279 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
281 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
286 if (_audio_reel == _reels.end ()) {
287 /* This audio is off the end of the last reel; ignore it */
291 if (end <= _audio_reel->period().to) {
292 /* Easy case: we can write all the audio to this reel */
293 _audio_reel->write (audio);
295 } else if (_audio_reel->period().to <= t) {
296 /* This reel is entirely before the start of our audio; just skip the reel */
299 /* This audio is over a reel boundary; split the audio into two and write the first part */
300 DCPTime part_lengths[2] = {
301 _audio_reel->period().to - t,
302 end - _audio_reel->period().to
305 Frame part_frames[2] = {
306 part_lengths[0].frames_ceil(afr),
307 part_lengths[1].frames_ceil(afr)
310 if (part_frames[0]) {
311 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
312 _audio_reel->write (part);
315 if (part_frames[1]) {
316 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
322 t += part_lengths[0];
329 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
331 if (_atmos_reel->period().to == time) {
333 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
336 /* We assume that we get a video frame's worth of data here */
337 _atmos_reel->write (atmos, metadata);
341 /** Caller must hold a lock on _state_mutex */
343 Writer::have_sequenced_image_at_queue_head ()
345 if (_queue.empty ()) {
350 QueueItem const & f = _queue.front();
351 return _last_written[f.reel].next(f);
356 Writer::LastWritten::next (QueueItem qi) const
358 if (qi.eyes == EYES_BOTH) {
360 return qi.frame == (_frame + 1);
365 if (_eyes == EYES_LEFT && qi.frame == _frame && qi.eyes == EYES_RIGHT) {
369 if (_eyes == EYES_RIGHT && qi.frame == (_frame + 1) && qi.eyes == EYES_LEFT) {
378 Writer::LastWritten::update (QueueItem qi)
391 boost::mutex::scoped_lock lock (_state_mutex);
395 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
396 /* We've got something to do: go and do it */
400 /* Nothing to do: wait until something happens which may indicate that we do */
401 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
402 _empty_condition.wait (lock);
403 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
406 /* We stop here if we have been asked to finish, and if either the queue
407 is empty or we do not have a sequenced image at its head (if this is the
408 case we will never terminate as no new frames will be sent once
411 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
412 /* (Hopefully temporarily) log anything that was not written */
413 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
414 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
415 for (auto const& i: _queue) {
416 if (i.type == QueueItem::FULL) {
417 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
419 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
426 /* Write any frames that we can write; i.e. those that are in sequence. */
427 while (have_sequenced_image_at_queue_head ()) {
428 QueueItem qi = _queue.front ();
429 _last_written[qi.reel].update (qi);
431 if (qi.type == QueueItem::FULL && qi.encoded) {
432 --_queued_full_in_memory;
437 ReelWriter& reel = _reels[qi.reel];
440 case QueueItem::FULL:
441 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
443 qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
445 reel.write (qi.encoded, qi.frame, qi.eyes);
448 case QueueItem::FAKE:
449 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
450 reel.fake_write (qi.size);
453 case QueueItem::REPEAT:
454 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
455 reel.repeat_write (qi.frame, qi.eyes);
461 _full_condition.notify_all ();
464 while (_queued_full_in_memory > _maximum_frames_in_memory) {
465 /* Too many frames in memory which can't yet be written to the stream.
466 Write some FULL frames to disk.
469 /* Find one from the back of the queue */
471 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
472 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
476 DCPOMATIC_ASSERT (i != _queue.rend());
478 /* For the log message below */
479 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
482 /* i is valid here, even though we don't hold a lock on the mutex,
483 since list iterators are unaffected by insertion and only this
484 thread could erase the last item in the list.
487 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
489 i->encoded->write_via_temp (
490 film()->j2c_path(i->reel, i->frame, i->eyes, true),
491 film()->j2c_path(i->reel, i->frame, i->eyes, false)
496 --_queued_full_in_memory;
497 _full_condition.notify_all ();
507 Writer::terminate_thread (bool can_throw)
509 boost::this_thread::disable_interruption dis;
511 boost::mutex::scoped_lock lock (_state_mutex);
514 _empty_condition.notify_all ();
515 _full_condition.notify_all ();
528 /** @param output_dcp Path to DCP folder to write */
530 Writer::finish (boost::filesystem::path output_dcp)
532 if (_thread.joinable()) {
533 LOG_GENERAL_NC ("Terminating writer thread");
534 terminate_thread (true);
537 LOG_GENERAL_NC ("Finishing ReelWriters");
539 for (auto& i: _reels) {
540 i.finish (output_dcp);
543 LOG_GENERAL_NC ("Writing XML");
545 dcp::DCP dcp (output_dcp);
547 shared_ptr<dcp::CPL> cpl (
550 film()->dcp_content_type()->libdcp_kind ()
556 /* Calculate digests for each reel in parallel */
558 shared_ptr<Job> job = _job.lock ();
560 job->sub (_("Computing digests"));
563 boost::asio::io_service service;
564 boost::thread_group pool;
566 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
568 int const threads = max (1, Config::instance()->master_encoding_threads ());
570 for (int i = 0; i < threads; ++i) {
571 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
574 boost::function<void (float)> set_progress;
576 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
578 set_progress = &ignore_progress;
581 for (auto& i: _reels) {
582 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
584 service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
592 for (auto& i: _reels) {
593 cpl->add (i.create_reel(_reel_assets, _fonts, output_dcp, _have_subtitles, _have_closed_captions));
598 string creator = Config::instance()->dcp_creator();
599 if (creator.empty()) {
600 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
603 string issuer = Config::instance()->dcp_issuer();
604 if (issuer.empty()) {
605 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
608 cpl->set_ratings (film()->ratings());
610 vector<dcp::ContentVersion> cv;
611 for (auto i: film()->content_versions()) {
612 cv.push_back (dcp::ContentVersion(i));
614 cpl->set_content_versions (cv);
616 cpl->set_full_content_title_text (film()->name());
617 cpl->set_full_content_title_text_language (film()->name_language());
618 cpl->set_release_territory (film()->release_territory());
619 cpl->set_version_number (film()->version_number());
620 cpl->set_status (film()->status());
621 cpl->set_chain (film()->chain());
622 cpl->set_distributor (film()->distributor());
623 cpl->set_facility (film()->facility());
624 cpl->set_luminance (film()->luminance());
626 auto ac = film()->mapped_audio_channels();
627 dcp::MCASoundField field = (
628 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSL)) != ac.end() ||
629 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSR)) != ac.end()
630 ) ? dcp::MCASoundField::SEVEN_POINT_ONE : dcp::MCASoundField::FIVE_POINT_ONE;
632 dcp::MainSoundConfiguration msc (field, film()->audio_channels());
634 if (static_cast<int>(i) < film()->audio_channels()) {
635 msc.set_mapping (i, static_cast<dcp::Channel>(i));
639 cpl->set_main_sound_configuration (msc.to_string());
640 cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
641 cpl->set_main_picture_stored_area (film()->frame_size());
642 cpl->set_main_picture_active_area (film()->active_area());
644 vector<dcp::LanguageTag> sl = film()->subtitle_languages();
646 cpl->set_additional_subtitle_languages(std::vector<dcp::LanguageTag>(sl.begin() + 1, sl.end()));
649 shared_ptr<const dcp::CertificateChain> signer;
650 signer = Config::instance()->signer_chain ();
651 /* We did check earlier, but check again here to be on the safe side */
653 if (!signer->valid (&reason)) {
654 throw InvalidSignerError (reason);
658 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE,
661 dcp::LocalTime().as_string(),
662 String::compose("Created by libdcp %1", dcp::version),
664 Config::instance()->dcp_metadata_filename_format()
668 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
671 write_cover_sheet (output_dcp);
675 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
677 boost::filesystem::path const cover = film()->file("COVER_SHEET.txt");
678 FILE* f = fopen_boost (cover, "w");
680 throw OpenFileError (cover, errno, OpenFileError::WRITE);
683 string text = Config::instance()->cover_sheet ();
684 boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
685 boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
686 boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
687 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", film()->isdcf_metadata().audio_language);
689 vector<dcp::LanguageTag> subtitle_languages = film()->subtitle_languages();
690 if (subtitle_languages.empty()) {
691 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", "None");
693 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.front().description());
696 boost::uintmax_t size = 0;
698 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(output_dcp);
699 i != boost::filesystem::recursive_directory_iterator();
701 if (boost::filesystem::is_regular_file (i->path ())) {
702 size += boost::filesystem::file_size (i->path ());
706 if (size > (1000000000L)) {
707 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
709 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
712 pair<int, int> ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
713 string description = String::compose("%1.%2", ch.first, ch.second);
715 if (description == "0.0") {
716 description = _("None");
717 } else if (description == "1.0") {
718 description = _("Mono");
719 } else if (description == "2.0") {
720 description = _("Stereo");
722 boost::algorithm::replace_all (text, "$AUDIO", description);
725 film()->length().split(film()->video_frame_rate(), h, m, s, fr);
727 if (h == 0 && m == 0) {
728 length = String::compose("%1s", s);
729 } else if (h == 0 && m > 0) {
730 length = String::compose("%1m%2s", m, s);
731 } else if (h > 0 && m > 0) {
732 length = String::compose("%1h%2m%3s", h, m, s);
735 boost::algorithm::replace_all (text, "$LENGTH", length);
737 checked_fwrite (text.c_str(), text.length(), f, cover);
741 /** @param frame Frame index within the whole DCP.
742 * @return true if we can fake-write this frame.
745 Writer::can_fake_write (Frame frame) const
747 if (film()->encrypted()) {
748 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
752 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
753 parameters in the asset writer.
756 ReelWriter const & reel = _reels[video_reel(frame)];
758 /* Make frame relative to the start of the reel */
759 frame -= reel.start ();
760 return (frame != 0 && frame < reel.first_nonexistant_frame());
763 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
765 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
767 vector<ReelWriter>::iterator* reel = 0;
770 case TEXT_OPEN_SUBTITLE:
771 reel = &_subtitle_reel;
772 _have_subtitles = true;
774 case TEXT_CLOSED_CAPTION:
775 DCPOMATIC_ASSERT (track);
776 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
777 reel = &_caption_reels[*track];
778 _have_closed_captions.insert (*track);
781 DCPOMATIC_ASSERT (false);
784 DCPOMATIC_ASSERT (*reel != _reels.end());
785 while ((*reel)->period().to <= period.from) {
787 DCPOMATIC_ASSERT (*reel != _reels.end());
790 (*reel)->write (text, type, track, period);
794 Writer::write (vector<FontData> fonts)
796 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
798 for (auto const& i: fonts) {
800 for (auto& j: _fonts) {
807 _fonts.push_back (i);
813 operator< (QueueItem const & a, QueueItem const & b)
815 if (a.reel != b.reel) {
816 return a.reel < b.reel;
819 if (a.frame != b.frame) {
820 return a.frame < b.frame;
823 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
827 operator== (QueueItem const & a, QueueItem const & b)
829 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
833 Writer::set_encoder_threads (int threads)
835 boost::mutex::scoped_lock lm (_state_mutex);
836 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
837 _maximum_queue_size = threads * 16;
841 Writer::write (ReferencedReelAsset asset)
843 _reel_assets.push_back (asset);
847 Writer::video_reel (int frame) const
849 DCPTime t = DCPTime::from_frames (frame, film()->video_frame_rate());
851 while (i < _reels.size() && !_reels[i].period().contains (t)) {
855 DCPOMATIC_ASSERT (i < _reels.size ());
860 Writer::set_digest_progress (Job* job, float progress)
862 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
864 _digest_progresses[boost::this_thread::get_id()] = progress;
865 float min_progress = FLT_MAX;
866 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
867 min_progress = min (min_progress, i->second);
870 job->set_progress (min_progress);
877 /** Calculate hashes for any referenced MXF assets which do not already have one */
879 Writer::calculate_referenced_digests (boost::function<void (float)> set_progress)
881 for (auto const& i: _reel_assets) {
882 shared_ptr<dcp::ReelMXF> mxf = dynamic_pointer_cast<dcp::ReelMXF>(i.asset);
883 if (mxf && !mxf->hash()) {
884 mxf->asset_ref().asset()->hash (set_progress);
885 mxf->set_hash (mxf->asset_ref().asset()->hash());