2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <dcp/reel_mxf.h>
42 #include <boost/foreach.hpp>
50 /* OS X strikes again */
62 using boost::shared_ptr;
63 using boost::weak_ptr;
64 using boost::dynamic_pointer_cast;
65 using boost::optional;
66 #if BOOST_VERSION >= 106100
67 using namespace boost::placeholders;
71 using namespace dcpomatic;
73 Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
77 , _queued_full_in_memory (0)
78 /* These will be reset to sensible values when J2KEncoder is created */
79 , _maximum_frames_in_memory (8)
80 , _maximum_queue_size (8)
86 shared_ptr<Job> job = _job.lock ();
87 DCPOMATIC_ASSERT (job);
90 list<DCPTimePeriod> const reels = _film->reels ();
91 BOOST_FOREACH (DCPTimePeriod p, reels) {
92 _reels.push_back (ReelWriter (film, p, job, reel_index++, reels.size(), _film->content_summary(p)));
95 _last_written.resize (reels.size());
97 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
98 and captions arrive to the Writer in sequence. This is not so for video.
100 _audio_reel = _reels.begin ();
101 _subtitle_reel = _reels.begin ();
102 BOOST_FOREACH (DCPTextTrack i, _film->closed_caption_tracks()) {
103 _caption_reels[i] = _reels.begin ();
105 _atmos_reel = _reels.begin ();
107 /* Check that the signer is OK */
109 if (!Config::instance()->signer_chain()->valid(&reason)) {
110 throw InvalidSignerError (reason);
117 _thread = boost::thread (boost::bind(&Writer::thread, this));
118 #ifdef DCPOMATIC_LINUX
119 pthread_setname_np (_thread.native_handle(), "writer");
125 terminate_thread (false);
128 /** Pass a video frame to the writer for writing to disk at some point.
129 * This method can be called with frames out of order.
130 * @param encoded JPEG2000-encoded data.
131 * @param frame Frame index within the DCP.
132 * @param eyes Eyes that this frame image is for.
135 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
137 boost::mutex::scoped_lock lock (_state_mutex);
139 while (_queued_full_in_memory > _maximum_frames_in_memory) {
140 /* There are too many full frames in memory; wake the main writer thread and
141 wait until it sorts everything out */
142 _empty_condition.notify_all ();
143 _full_condition.wait (lock);
147 qi.type = QueueItem::FULL;
148 qi.encoded = encoded;
149 qi.reel = video_reel (frame);
150 qi.frame = frame - _reels[qi.reel].start ();
152 if (_film->three_d() && eyes == EYES_BOTH) {
153 /* 2D material in a 3D DCP; fake the 3D */
155 _queue.push_back (qi);
156 ++_queued_full_in_memory;
157 qi.eyes = EYES_RIGHT;
158 _queue.push_back (qi);
159 ++_queued_full_in_memory;
162 _queue.push_back (qi);
163 ++_queued_full_in_memory;
166 /* Now there's something to do: wake anything wait()ing on _empty_condition */
167 _empty_condition.notify_all ();
171 Writer::can_repeat (Frame frame) const
173 return frame > _reels[video_reel(frame)].start();
176 /** Repeat the last frame that was written to a reel as a new frame.
177 * @param frame Frame index within the DCP of the new (repeated) frame.
178 * @param eyes Eyes that this repeated frame image is for.
181 Writer::repeat (Frame frame, Eyes eyes)
183 boost::mutex::scoped_lock lock (_state_mutex);
185 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
186 /* The queue is too big, and the main writer thread can run and fix it, so
187 wake it and wait until it has done.
189 _empty_condition.notify_all ();
190 _full_condition.wait (lock);
194 qi.type = QueueItem::REPEAT;
195 qi.reel = video_reel (frame);
196 qi.frame = frame - _reels[qi.reel].start ();
197 if (_film->three_d() && eyes == EYES_BOTH) {
199 _queue.push_back (qi);
200 qi.eyes = EYES_RIGHT;
201 _queue.push_back (qi);
204 _queue.push_back (qi);
207 /* Now there's something to do: wake anything wait()ing on _empty_condition */
208 _empty_condition.notify_all ();
212 Writer::fake_write (Frame frame, Eyes eyes)
214 boost::mutex::scoped_lock lock (_state_mutex);
216 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
217 /* The queue is too big, and the main writer thread can run and fix it, so
218 wake it and wait until it has done.
220 _empty_condition.notify_all ();
221 _full_condition.wait (lock);
224 size_t const reel = video_reel (frame);
225 Frame const frame_in_reel = frame - _reels[reel].start ();
228 qi.type = QueueItem::FAKE;
231 shared_ptr<InfoFileHandle> info_file = _film->info_file_handle(_reels[reel].period(), true);
232 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
236 qi.frame = frame_in_reel;
237 if (_film->three_d() && eyes == EYES_BOTH) {
239 _queue.push_back (qi);
240 qi.eyes = EYES_RIGHT;
241 _queue.push_back (qi);
244 _queue.push_back (qi);
247 /* Now there's something to do: wake anything wait()ing on _empty_condition */
248 _empty_condition.notify_all ();
251 /** Write some audio frames to the DCP.
252 * @param audio Audio data.
253 * @param time Time of this data within the DCP.
254 * This method is not thread safe.
257 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
259 DCPOMATIC_ASSERT (audio);
261 int const afr = _film->audio_frame_rate();
263 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
265 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
270 if (_audio_reel == _reels.end ()) {
271 /* This audio is off the end of the last reel; ignore it */
275 if (end <= _audio_reel->period().to) {
276 /* Easy case: we can write all the audio to this reel */
277 _audio_reel->write (audio);
279 } else if (_audio_reel->period().to <= t) {
280 /* This reel is entirely before the start of our audio; just skip the reel */
283 /* This audio is over a reel boundary; split the audio into two and write the first part */
284 DCPTime part_lengths[2] = {
285 _audio_reel->period().to - t,
286 end - _audio_reel->period().to
289 Frame part_frames[2] = {
290 part_lengths[0].frames_ceil(afr),
291 part_lengths[1].frames_ceil(afr)
294 if (part_frames[0]) {
295 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
296 _audio_reel->write (part);
299 if (part_frames[1]) {
300 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
306 t += part_lengths[0];
313 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
315 if (_atmos_reel->period().to == time) {
317 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
320 /* We assume that we get a video frame's worth of data here */
321 _atmos_reel->write (atmos, metadata);
325 /** Caller must hold a lock on _state_mutex */
327 Writer::have_sequenced_image_at_queue_head ()
329 if (_queue.empty ()) {
334 QueueItem const & f = _queue.front();
335 return _last_written[f.reel].next(f);
340 Writer::LastWritten::next (QueueItem qi) const
342 if (qi.eyes == EYES_BOTH) {
344 return qi.frame == (_frame + 1);
349 if (_eyes == EYES_LEFT && qi.frame == _frame && qi.eyes == EYES_RIGHT) {
353 if (_eyes == EYES_RIGHT && qi.frame == (_frame + 1) && qi.eyes == EYES_LEFT) {
362 Writer::LastWritten::update (QueueItem qi)
375 boost::mutex::scoped_lock lock (_state_mutex);
379 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
380 /* We've got something to do: go and do it */
384 /* Nothing to do: wait until something happens which may indicate that we do */
385 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
386 _empty_condition.wait (lock);
387 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
390 if (_finish && _queue.empty()) {
394 /* We stop here if we have been asked to finish, and if either the queue
395 is empty or we do not have a sequenced image at its head (if this is the
396 case we will never terminate as no new frames will be sent once
399 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
400 /* (Hopefully temporarily) log anything that was not written */
401 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
402 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
403 BOOST_FOREACH (QueueItem const& i, _queue) {
404 if (i.type == QueueItem::FULL) {
405 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
407 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
414 /* Write any frames that we can write; i.e. those that are in sequence. */
415 while (have_sequenced_image_at_queue_head ()) {
416 QueueItem qi = _queue.front ();
417 _last_written[qi.reel].update (qi);
419 if (qi.type == QueueItem::FULL && qi.encoded) {
420 --_queued_full_in_memory;
425 ReelWriter& reel = _reels[qi.reel];
428 case QueueItem::FULL:
429 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
431 qi.encoded.reset (new ArrayData(_film->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
433 reel.write (qi.encoded, qi.frame, qi.eyes);
436 case QueueItem::FAKE:
437 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
438 reel.fake_write (qi.size);
441 case QueueItem::REPEAT:
442 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
443 reel.repeat_write (qi.frame, qi.eyes);
449 _full_condition.notify_all ();
452 while (_queued_full_in_memory > _maximum_frames_in_memory) {
453 /* Too many frames in memory which can't yet be written to the stream.
454 Write some FULL frames to disk.
457 /* Find one from the back of the queue */
459 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
460 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
464 DCPOMATIC_ASSERT (i != _queue.rend());
466 /* For the log message below */
467 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
470 /* i is valid here, even though we don't hold a lock on the mutex,
471 since list iterators are unaffected by insertion and only this
472 thread could erase the last item in the list.
475 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
477 i->encoded->write_via_temp (
478 _film->j2c_path (i->reel, i->frame, i->eyes, true),
479 _film->j2c_path (i->reel, i->frame, i->eyes, false)
484 --_queued_full_in_memory;
485 _full_condition.notify_all ();
495 Writer::terminate_thread (bool can_throw)
497 boost::this_thread::disable_interruption dis;
499 boost::mutex::scoped_lock lock (_state_mutex);
502 _empty_condition.notify_all ();
503 _full_condition.notify_all ();
518 if (!_thread.joinable()) {
522 LOG_GENERAL_NC ("Terminating writer thread");
524 terminate_thread (true);
526 LOG_GENERAL_NC ("Finishing ReelWriters");
528 BOOST_FOREACH (ReelWriter& i, _reels) {
532 LOG_GENERAL_NC ("Writing XML");
534 dcp::DCP dcp (_film->dir (_film->dcp_name()));
536 shared_ptr<dcp::CPL> cpl (
539 _film->dcp_content_type()->libdcp_kind ()
545 /* Calculate digests for each reel in parallel */
547 shared_ptr<Job> job = _job.lock ();
548 job->sub (_("Computing digests"));
550 boost::asio::io_service service;
551 boost::thread_group pool;
553 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
555 int const threads = max (1, Config::instance()->master_encoding_threads ());
557 for (int i = 0; i < threads; ++i) {
558 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
561 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
562 BOOST_FOREACH (ReelWriter& i, _reels) {
563 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
565 service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
573 BOOST_FOREACH (ReelWriter& i, _reels) {
574 cpl->add (i.create_reel (_reel_assets, _fonts));
579 string creator = Config::instance()->dcp_creator();
580 if (creator.empty()) {
581 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
584 string issuer = Config::instance()->dcp_issuer();
585 if (issuer.empty()) {
586 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
589 cpl->set_ratings (_film->ratings());
591 vector<dcp::ContentVersion> cv;
592 BOOST_FOREACH (string i, _film->content_versions()) {
593 cv.push_back (dcp::ContentVersion(i));
595 cpl->set_content_versions (cv);
597 cpl->set_full_content_title_text (_film->name());
598 cpl->set_full_content_title_text_language (_film->name_language());
599 cpl->set_release_territory (_film->release_territory());
600 cpl->set_version_number (_film->version_number());
601 cpl->set_status (_film->status());
602 cpl->set_chain (_film->chain());
603 cpl->set_distributor (_film->distributor());
604 cpl->set_facility (_film->facility());
605 cpl->set_luminance (_film->luminance());
607 list<int> ac = _film->mapped_audio_channels ();
608 dcp::MCASoundField field = (
609 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSL)) != ac.end() ||
610 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSR)) != ac.end()
611 ) ? dcp::SEVEN_POINT_ONE : dcp::FIVE_POINT_ONE;
613 dcp::MainSoundConfiguration msc (field, _film->audio_channels());
614 BOOST_FOREACH (int i, ac) {
615 if (i < _film->audio_channels()) {
616 msc.set_mapping (i, static_cast<dcp::Channel>(i));
620 cpl->set_main_sound_configuration (msc.to_string());
621 cpl->set_main_sound_sample_rate (_film->audio_frame_rate());
622 cpl->set_main_picture_stored_area (_film->frame_size());
623 cpl->set_main_picture_active_area (_film->active_area());
625 vector<dcp::LanguageTag> sl = _film->subtitle_languages();
627 cpl->set_additional_subtitle_languages(std::vector<dcp::LanguageTag>(sl.begin() + 1, sl.end()));
630 shared_ptr<const dcp::CertificateChain> signer;
631 signer = Config::instance()->signer_chain ();
632 /* We did check earlier, but check again here to be on the safe side */
634 if (!signer->valid (&reason)) {
635 throw InvalidSignerError (reason);
639 _film->interop() ? dcp::INTEROP : dcp::SMPTE,
642 dcp::LocalTime().as_string(),
643 String::compose("Created by libdcp %1", dcp::version),
645 Config::instance()->dcp_metadata_filename_format()
649 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
652 write_cover_sheet ();
656 Writer::write_cover_sheet ()
658 boost::filesystem::path const cover = _film->file ("COVER_SHEET.txt");
659 FILE* f = fopen_boost (cover, "w");
661 throw OpenFileError (cover, errno, OpenFileError::WRITE);
664 string text = Config::instance()->cover_sheet ();
665 boost::algorithm::replace_all (text, "$CPL_NAME", _film->name());
666 boost::algorithm::replace_all (text, "$TYPE", _film->dcp_content_type()->pretty_name());
667 boost::algorithm::replace_all (text, "$CONTAINER", _film->container()->container_nickname());
668 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _film->isdcf_metadata().audio_language);
670 vector<dcp::LanguageTag> subtitle_languages = _film->subtitle_languages();
671 if (subtitle_languages.empty()) {
672 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", "None");
674 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.front().description());
677 boost::uintmax_t size = 0;
679 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(_film->dir(_film->dcp_name()));
680 i != boost::filesystem::recursive_directory_iterator();
682 if (boost::filesystem::is_regular_file (i->path ())) {
683 size += boost::filesystem::file_size (i->path ());
687 if (size > (1000000000L)) {
688 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
690 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
693 pair<int, int> ch = audio_channel_types (_film->mapped_audio_channels(), _film->audio_channels());
694 string description = String::compose("%1.%2", ch.first, ch.second);
696 if (description == "0.0") {
697 description = _("None");
698 } else if (description == "1.0") {
699 description = _("Mono");
700 } else if (description == "2.0") {
701 description = _("Stereo");
703 boost::algorithm::replace_all (text, "$AUDIO", description);
706 _film->length().split (_film->video_frame_rate(), h, m, s, fr);
708 if (h == 0 && m == 0) {
709 length = String::compose("%1s", s);
710 } else if (h == 0 && m > 0) {
711 length = String::compose("%1m%2s", m, s);
712 } else if (h > 0 && m > 0) {
713 length = String::compose("%1h%2m%3s", h, m, s);
716 boost::algorithm::replace_all (text, "$LENGTH", length);
718 checked_fwrite (text.c_str(), text.length(), f, cover);
722 /** @param frame Frame index within the whole DCP.
723 * @return true if we can fake-write this frame.
726 Writer::can_fake_write (Frame frame) const
728 if (_film->encrypted()) {
729 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
733 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
734 parameters in the asset writer.
737 ReelWriter const & reel = _reels[video_reel(frame)];
739 /* Make frame relative to the start of the reel */
740 frame -= reel.start ();
741 return (frame != 0 && frame < reel.first_nonexistant_frame());
744 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
746 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
748 vector<ReelWriter>::iterator* reel = 0;
751 case TEXT_OPEN_SUBTITLE:
752 reel = &_subtitle_reel;
754 case TEXT_CLOSED_CAPTION:
755 DCPOMATIC_ASSERT (track);
756 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
757 reel = &_caption_reels[*track];
760 DCPOMATIC_ASSERT (false);
763 DCPOMATIC_ASSERT (*reel != _reels.end());
764 while ((*reel)->period().to <= period.from) {
766 DCPOMATIC_ASSERT (*reel != _reels.end());
769 (*reel)->write (text, type, track, period);
773 Writer::write (list<shared_ptr<Font> > fonts)
775 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
777 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
779 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
786 _fonts.push_back (i);
792 operator< (QueueItem const & a, QueueItem const & b)
794 if (a.reel != b.reel) {
795 return a.reel < b.reel;
798 if (a.frame != b.frame) {
799 return a.frame < b.frame;
802 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
806 operator== (QueueItem const & a, QueueItem const & b)
808 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
812 Writer::set_encoder_threads (int threads)
814 boost::mutex::scoped_lock lm (_state_mutex);
815 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
816 _maximum_queue_size = threads * 16;
820 Writer::write (ReferencedReelAsset asset)
822 _reel_assets.push_back (asset);
826 Writer::video_reel (int frame) const
828 DCPTime t = DCPTime::from_frames (frame, _film->video_frame_rate ());
830 while (i < _reels.size() && !_reels[i].period().contains (t)) {
834 DCPOMATIC_ASSERT (i < _reels.size ());
839 Writer::set_digest_progress (Job* job, float progress)
841 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
843 _digest_progresses[boost::this_thread::get_id()] = progress;
844 float min_progress = FLT_MAX;
845 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
846 min_progress = min (min_progress, i->second);
849 job->set_progress (min_progress);
856 /** Calculate hashes for any referenced MXF assets which do not already have one */
858 Writer::calculate_referenced_digests (boost::function<void (float)> set_progress)
860 BOOST_FOREACH (ReferencedReelAsset const& i, _reel_assets) {
861 shared_ptr<dcp::ReelMXF> mxf = dynamic_pointer_cast<dcp::ReelMXF>(i.asset);
862 if (mxf && !mxf->hash()) {
863 mxf->asset_ref().asset()->hash (set_progress);
864 mxf->set_hash (mxf->asset_ref().asset()->hash());