2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <dcp/reel_mxf.h>
42 #include <boost/foreach.hpp>
50 /* OS X strikes again */
62 using boost::shared_ptr;
63 using boost::weak_ptr;
64 using boost::dynamic_pointer_cast;
65 using boost::optional;
66 #if BOOST_VERSION >= 106100
67 using namespace boost::placeholders;
71 using namespace dcpomatic;
73 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j)
74 : WeakConstFilm (weak_film)
77 , _queued_full_in_memory (0)
78 /* These will be reset to sensible values when J2KEncoder is created */
79 , _maximum_frames_in_memory (8)
80 , _maximum_queue_size (8)
86 shared_ptr<Job> job = _job.lock ();
87 DCPOMATIC_ASSERT (job);
90 list<DCPTimePeriod> const reels = film()->reels();
91 BOOST_FOREACH (DCPTimePeriod p, reels) {
92 _reels.push_back (ReelWriter(film(), p, job, reel_index++, reels.size()));
95 _last_written.resize (reels.size());
97 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
98 and captions arrive to the Writer in sequence. This is not so for video.
100 _audio_reel = _reels.begin ();
101 _subtitle_reel = _reels.begin ();
102 BOOST_FOREACH (DCPTextTrack i, film()->closed_caption_tracks()) {
103 _caption_reels[i] = _reels.begin ();
105 _atmos_reel = _reels.begin ();
107 /* Check that the signer is OK */
109 if (!Config::instance()->signer_chain()->valid(&reason)) {
110 throw InvalidSignerError (reason);
117 _thread = boost::thread (boost::bind(&Writer::thread, this));
118 #ifdef DCPOMATIC_LINUX
119 pthread_setname_np (_thread.native_handle(), "writer");
125 terminate_thread (false);
128 /** Pass a video frame to the writer for writing to disk at some point.
129 * This method can be called with frames out of order.
130 * @param encoded JPEG2000-encoded data.
131 * @param frame Frame index within the DCP.
132 * @param eyes Eyes that this frame image is for.
135 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
137 boost::mutex::scoped_lock lock (_state_mutex);
139 while (_queued_full_in_memory > _maximum_frames_in_memory) {
140 /* There are too many full frames in memory; wake the main writer thread and
141 wait until it sorts everything out */
142 _empty_condition.notify_all ();
143 _full_condition.wait (lock);
147 qi.type = QueueItem::FULL;
148 qi.encoded = encoded;
149 qi.reel = video_reel (frame);
150 qi.frame = frame - _reels[qi.reel].start ();
152 if (film()->three_d() && eyes == EYES_BOTH) {
153 /* 2D material in a 3D DCP; fake the 3D */
155 _queue.push_back (qi);
156 ++_queued_full_in_memory;
157 qi.eyes = EYES_RIGHT;
158 _queue.push_back (qi);
159 ++_queued_full_in_memory;
162 _queue.push_back (qi);
163 ++_queued_full_in_memory;
166 /* Now there's something to do: wake anything wait()ing on _empty_condition */
167 _empty_condition.notify_all ();
171 Writer::can_repeat (Frame frame) const
173 return frame > _reels[video_reel(frame)].start();
176 /** Repeat the last frame that was written to a reel as a new frame.
177 * @param frame Frame index within the DCP of the new (repeated) frame.
178 * @param eyes Eyes that this repeated frame image is for.
181 Writer::repeat (Frame frame, Eyes eyes)
183 boost::mutex::scoped_lock lock (_state_mutex);
185 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
186 /* The queue is too big, and the main writer thread can run and fix it, so
187 wake it and wait until it has done.
189 _empty_condition.notify_all ();
190 _full_condition.wait (lock);
194 qi.type = QueueItem::REPEAT;
195 qi.reel = video_reel (frame);
196 qi.frame = frame - _reels[qi.reel].start ();
197 if (film()->three_d() && eyes == EYES_BOTH) {
199 _queue.push_back (qi);
200 qi.eyes = EYES_RIGHT;
201 _queue.push_back (qi);
204 _queue.push_back (qi);
207 /* Now there's something to do: wake anything wait()ing on _empty_condition */
208 _empty_condition.notify_all ();
212 Writer::fake_write (Frame frame, Eyes eyes)
214 boost::mutex::scoped_lock lock (_state_mutex);
216 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
217 /* The queue is too big, and the main writer thread can run and fix it, so
218 wake it and wait until it has done.
220 _empty_condition.notify_all ();
221 _full_condition.wait (lock);
224 size_t const reel = video_reel (frame);
225 Frame const frame_in_reel = frame - _reels[reel].start ();
228 qi.type = QueueItem::FAKE;
231 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
232 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
236 qi.frame = frame_in_reel;
237 if (film()->three_d() && eyes == EYES_BOTH) {
239 _queue.push_back (qi);
240 qi.eyes = EYES_RIGHT;
241 _queue.push_back (qi);
244 _queue.push_back (qi);
247 /* Now there's something to do: wake anything wait()ing on _empty_condition */
248 _empty_condition.notify_all ();
251 /** Write some audio frames to the DCP.
252 * @param audio Audio data.
253 * @param time Time of this data within the DCP.
254 * This method is not thread safe.
257 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
259 DCPOMATIC_ASSERT (audio);
261 int const afr = film()->audio_frame_rate();
263 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
265 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
270 if (_audio_reel == _reels.end ()) {
271 /* This audio is off the end of the last reel; ignore it */
275 if (end <= _audio_reel->period().to) {
276 /* Easy case: we can write all the audio to this reel */
277 _audio_reel->write (audio);
279 } else if (_audio_reel->period().to <= t) {
280 /* This reel is entirely before the start of our audio; just skip the reel */
283 /* This audio is over a reel boundary; split the audio into two and write the first part */
284 DCPTime part_lengths[2] = {
285 _audio_reel->period().to - t,
286 end - _audio_reel->period().to
289 Frame part_frames[2] = {
290 part_lengths[0].frames_ceil(afr),
291 part_lengths[1].frames_ceil(afr)
294 if (part_frames[0]) {
295 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
296 _audio_reel->write (part);
299 if (part_frames[1]) {
300 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
306 t += part_lengths[0];
313 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
315 if (_atmos_reel->period().to == time) {
317 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
320 /* We assume that we get a video frame's worth of data here */
321 _atmos_reel->write (atmos, metadata);
325 /** Caller must hold a lock on _state_mutex */
327 Writer::have_sequenced_image_at_queue_head ()
329 if (_queue.empty ()) {
334 QueueItem const & f = _queue.front();
335 return _last_written[f.reel].next(f);
340 Writer::LastWritten::next (QueueItem qi) const
342 if (qi.eyes == EYES_BOTH) {
344 return qi.frame == (_frame + 1);
349 if (_eyes == EYES_LEFT && qi.frame == _frame && qi.eyes == EYES_RIGHT) {
353 if (_eyes == EYES_RIGHT && qi.frame == (_frame + 1) && qi.eyes == EYES_LEFT) {
362 Writer::LastWritten::update (QueueItem qi)
375 boost::mutex::scoped_lock lock (_state_mutex);
379 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
380 /* We've got something to do: go and do it */
384 /* Nothing to do: wait until something happens which may indicate that we do */
385 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
386 _empty_condition.wait (lock);
387 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
390 /* We stop here if we have been asked to finish, and if either the queue
391 is empty or we do not have a sequenced image at its head (if this is the
392 case we will never terminate as no new frames will be sent once
395 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
396 /* (Hopefully temporarily) log anything that was not written */
397 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
398 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
399 BOOST_FOREACH (QueueItem const& i, _queue) {
400 if (i.type == QueueItem::FULL) {
401 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
403 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
410 /* Write any frames that we can write; i.e. those that are in sequence. */
411 while (have_sequenced_image_at_queue_head ()) {
412 QueueItem qi = _queue.front ();
413 _last_written[qi.reel].update (qi);
415 if (qi.type == QueueItem::FULL && qi.encoded) {
416 --_queued_full_in_memory;
421 ReelWriter& reel = _reels[qi.reel];
424 case QueueItem::FULL:
425 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
427 qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
429 reel.write (qi.encoded, qi.frame, qi.eyes);
432 case QueueItem::FAKE:
433 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
434 reel.fake_write (qi.size);
437 case QueueItem::REPEAT:
438 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
439 reel.repeat_write (qi.frame, qi.eyes);
445 _full_condition.notify_all ();
448 while (_queued_full_in_memory > _maximum_frames_in_memory) {
449 /* Too many frames in memory which can't yet be written to the stream.
450 Write some FULL frames to disk.
453 /* Find one from the back of the queue */
455 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
456 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
460 DCPOMATIC_ASSERT (i != _queue.rend());
462 /* For the log message below */
463 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
466 /* i is valid here, even though we don't hold a lock on the mutex,
467 since list iterators are unaffected by insertion and only this
468 thread could erase the last item in the list.
471 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
473 i->encoded->write_via_temp (
474 film()->j2c_path(i->reel, i->frame, i->eyes, true),
475 film()->j2c_path(i->reel, i->frame, i->eyes, false)
480 --_queued_full_in_memory;
481 _full_condition.notify_all ();
491 Writer::terminate_thread (bool can_throw)
493 boost::this_thread::disable_interruption dis;
495 boost::mutex::scoped_lock lock (_state_mutex);
498 _empty_condition.notify_all ();
499 _full_condition.notify_all ();
514 if (!_thread.joinable()) {
518 LOG_GENERAL_NC ("Terminating writer thread");
520 terminate_thread (true);
522 LOG_GENERAL_NC ("Finishing ReelWriters");
524 BOOST_FOREACH (ReelWriter& i, _reels) {
528 LOG_GENERAL_NC ("Writing XML");
530 dcp::DCP dcp (film()->dir(film()->dcp_name()));
532 shared_ptr<dcp::CPL> cpl (
535 film()->dcp_content_type()->libdcp_kind ()
541 /* Calculate digests for each reel in parallel */
543 shared_ptr<Job> job = _job.lock ();
544 job->sub (_("Computing digests"));
546 boost::asio::io_service service;
547 boost::thread_group pool;
549 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
551 int const threads = max (1, Config::instance()->master_encoding_threads ());
553 for (int i = 0; i < threads; ++i) {
554 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
557 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
558 BOOST_FOREACH (ReelWriter& i, _reels) {
559 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
561 service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
569 BOOST_FOREACH (ReelWriter& i, _reels) {
570 cpl->add (i.create_reel (_reel_assets, _fonts));
575 string creator = Config::instance()->dcp_creator();
576 if (creator.empty()) {
577 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
580 string issuer = Config::instance()->dcp_issuer();
581 if (issuer.empty()) {
582 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
585 cpl->set_ratings (film()->ratings());
587 vector<dcp::ContentVersion> cv;
588 BOOST_FOREACH (string i, film()->content_versions()) {
589 cv.push_back (dcp::ContentVersion(i));
591 cpl->set_content_versions (cv);
593 cpl->set_full_content_title_text (film()->name());
594 cpl->set_full_content_title_text_language (film()->name_language());
595 cpl->set_release_territory (film()->release_territory());
596 cpl->set_version_number (film()->version_number());
597 cpl->set_status (film()->status());
598 cpl->set_chain (film()->chain());
599 cpl->set_distributor (film()->distributor());
600 cpl->set_facility (film()->facility());
601 cpl->set_luminance (film()->luminance());
603 list<int> ac = film()->mapped_audio_channels();
604 dcp::MCASoundField field = (
605 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSL)) != ac.end() ||
606 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSR)) != ac.end()
607 ) ? dcp::SEVEN_POINT_ONE : dcp::FIVE_POINT_ONE;
609 dcp::MainSoundConfiguration msc (field, film()->audio_channels());
610 BOOST_FOREACH (int i, ac) {
611 if (i < film()->audio_channels()) {
612 msc.set_mapping (i, static_cast<dcp::Channel>(i));
616 cpl->set_main_sound_configuration (msc.to_string());
617 cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
618 cpl->set_main_picture_stored_area (film()->frame_size());
619 cpl->set_main_picture_active_area (film()->active_area());
621 vector<dcp::LanguageTag> sl = film()->subtitle_languages();
623 cpl->set_additional_subtitle_languages(std::vector<dcp::LanguageTag>(sl.begin() + 1, sl.end()));
626 shared_ptr<const dcp::CertificateChain> signer;
627 signer = Config::instance()->signer_chain ();
628 /* We did check earlier, but check again here to be on the safe side */
630 if (!signer->valid (&reason)) {
631 throw InvalidSignerError (reason);
635 film()->interop() ? dcp::INTEROP : dcp::SMPTE,
638 dcp::LocalTime().as_string(),
639 String::compose("Created by libdcp %1", dcp::version),
641 Config::instance()->dcp_metadata_filename_format()
645 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
648 write_cover_sheet ();
652 Writer::write_cover_sheet ()
654 boost::filesystem::path const cover = film()->file("COVER_SHEET.txt");
655 FILE* f = fopen_boost (cover, "w");
657 throw OpenFileError (cover, errno, OpenFileError::WRITE);
660 string text = Config::instance()->cover_sheet ();
661 boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
662 boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
663 boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
664 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", film()->isdcf_metadata().audio_language);
666 vector<dcp::LanguageTag> subtitle_languages = film()->subtitle_languages();
667 if (subtitle_languages.empty()) {
668 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", "None");
670 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.front().description());
673 boost::uintmax_t size = 0;
675 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(film()->dir(film()->dcp_name()));
676 i != boost::filesystem::recursive_directory_iterator();
678 if (boost::filesystem::is_regular_file (i->path ())) {
679 size += boost::filesystem::file_size (i->path ());
683 if (size > (1000000000L)) {
684 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
686 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
689 pair<int, int> ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
690 string description = String::compose("%1.%2", ch.first, ch.second);
692 if (description == "0.0") {
693 description = _("None");
694 } else if (description == "1.0") {
695 description = _("Mono");
696 } else if (description == "2.0") {
697 description = _("Stereo");
699 boost::algorithm::replace_all (text, "$AUDIO", description);
702 film()->length().split(film()->video_frame_rate(), h, m, s, fr);
704 if (h == 0 && m == 0) {
705 length = String::compose("%1s", s);
706 } else if (h == 0 && m > 0) {
707 length = String::compose("%1m%2s", m, s);
708 } else if (h > 0 && m > 0) {
709 length = String::compose("%1h%2m%3s", h, m, s);
712 boost::algorithm::replace_all (text, "$LENGTH", length);
714 checked_fwrite (text.c_str(), text.length(), f, cover);
718 /** @param frame Frame index within the whole DCP.
719 * @return true if we can fake-write this frame.
722 Writer::can_fake_write (Frame frame) const
724 if (film()->encrypted()) {
725 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
729 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
730 parameters in the asset writer.
733 ReelWriter const & reel = _reels[video_reel(frame)];
735 /* Make frame relative to the start of the reel */
736 frame -= reel.start ();
737 return (frame != 0 && frame < reel.first_nonexistant_frame());
740 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
742 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
744 vector<ReelWriter>::iterator* reel = 0;
747 case TEXT_OPEN_SUBTITLE:
748 reel = &_subtitle_reel;
750 case TEXT_CLOSED_CAPTION:
751 DCPOMATIC_ASSERT (track);
752 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
753 reel = &_caption_reels[*track];
756 DCPOMATIC_ASSERT (false);
759 DCPOMATIC_ASSERT (*reel != _reels.end());
760 while ((*reel)->period().to <= period.from) {
762 DCPOMATIC_ASSERT (*reel != _reels.end());
765 (*reel)->write (text, type, track, period);
769 Writer::write (list<shared_ptr<Font> > fonts)
771 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
773 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
775 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
782 _fonts.push_back (i);
788 operator< (QueueItem const & a, QueueItem const & b)
790 if (a.reel != b.reel) {
791 return a.reel < b.reel;
794 if (a.frame != b.frame) {
795 return a.frame < b.frame;
798 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
802 operator== (QueueItem const & a, QueueItem const & b)
804 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
808 Writer::set_encoder_threads (int threads)
810 boost::mutex::scoped_lock lm (_state_mutex);
811 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
812 _maximum_queue_size = threads * 16;
816 Writer::write (ReferencedReelAsset asset)
818 _reel_assets.push_back (asset);
822 Writer::video_reel (int frame) const
824 DCPTime t = DCPTime::from_frames (frame, film()->video_frame_rate());
826 while (i < _reels.size() && !_reels[i].period().contains (t)) {
830 DCPOMATIC_ASSERT (i < _reels.size ());
835 Writer::set_digest_progress (Job* job, float progress)
837 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
839 _digest_progresses[boost::this_thread::get_id()] = progress;
840 float min_progress = FLT_MAX;
841 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
842 min_progress = min (min_progress, i->second);
845 job->set_progress (min_progress);
852 /** Calculate hashes for any referenced MXF assets which do not already have one */
854 Writer::calculate_referenced_digests (boost::function<void (float)> set_progress)
856 BOOST_FOREACH (ReferencedReelAsset const& i, _reel_assets) {
857 shared_ptr<dcp::ReelMXF> mxf = dynamic_pointer_cast<dcp::ReelMXF>(i.asset);
858 if (mxf && !mxf->hash()) {
859 mxf->asset_ref().asset()->hash (set_progress);
860 mxf->set_hash (mxf->asset_ref().asset()->hash());