2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <boost/foreach.hpp>
49 /* OS X strikes again */
61 using boost::shared_ptr;
62 using boost::weak_ptr;
63 using boost::dynamic_pointer_cast;
64 using boost::optional;
65 #if BOOST_VERSION >= 106100
66 using namespace boost::placeholders;
70 using namespace dcpomatic;
72 Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
76 , _queued_full_in_memory (0)
77 /* These will be reset to sensible values when J2KEncoder is created */
78 , _maximum_frames_in_memory (8)
79 , _maximum_queue_size (8)
85 shared_ptr<Job> job = _job.lock ();
86 DCPOMATIC_ASSERT (job);
89 list<DCPTimePeriod> const reels = _film->reels ();
90 BOOST_FOREACH (DCPTimePeriod p, reels) {
91 _reels.push_back (ReelWriter (film, p, job, reel_index++, reels.size(), _film->content_summary(p)));
94 _last_written.resize (reels.size());
96 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
97 and captions arrive to the Writer in sequence. This is not so for video.
99 _audio_reel = _reels.begin ();
100 _subtitle_reel = _reels.begin ();
101 BOOST_FOREACH (DCPTextTrack i, _film->closed_caption_tracks()) {
102 _caption_reels[i] = _reels.begin ();
104 _atmos_reel = _reels.begin ();
106 /* Check that the signer is OK */
108 if (!Config::instance()->signer_chain()->valid(&reason)) {
109 throw InvalidSignerError (reason);
116 _thread = boost::thread (boost::bind(&Writer::thread, this));
117 #ifdef DCPOMATIC_LINUX
118 pthread_setname_np (_thread.native_handle(), "writer");
124 terminate_thread (false);
127 /** Pass a video frame to the writer for writing to disk at some point.
128 * This method can be called with frames out of order.
129 * @param encoded JPEG2000-encoded data.
130 * @param frame Frame index within the DCP.
131 * @param eyes Eyes that this frame image is for.
134 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
136 boost::mutex::scoped_lock lock (_state_mutex);
138 while (_queued_full_in_memory > _maximum_frames_in_memory) {
139 /* There are too many full frames in memory; wake the main writer thread and
140 wait until it sorts everything out */
141 _empty_condition.notify_all ();
142 _full_condition.wait (lock);
146 qi.type = QueueItem::FULL;
147 qi.encoded = encoded;
148 qi.reel = video_reel (frame);
149 qi.frame = frame - _reels[qi.reel].start ();
151 if (_film->three_d() && eyes == EYES_BOTH) {
152 /* 2D material in a 3D DCP; fake the 3D */
154 _queue.push_back (qi);
155 ++_queued_full_in_memory;
156 qi.eyes = EYES_RIGHT;
157 _queue.push_back (qi);
158 ++_queued_full_in_memory;
161 _queue.push_back (qi);
162 ++_queued_full_in_memory;
165 /* Now there's something to do: wake anything wait()ing on _empty_condition */
166 _empty_condition.notify_all ();
170 Writer::can_repeat (Frame frame) const
172 return frame > _reels[video_reel(frame)].start();
175 /** Repeat the last frame that was written to a reel as a new frame.
176 * @param frame Frame index within the DCP of the new (repeated) frame.
177 * @param eyes Eyes that this repeated frame image is for.
180 Writer::repeat (Frame frame, Eyes eyes)
182 boost::mutex::scoped_lock lock (_state_mutex);
184 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
185 /* The queue is too big, and the main writer thread can run and fix it, so
186 wake it and wait until it has done.
188 _empty_condition.notify_all ();
189 _full_condition.wait (lock);
193 qi.type = QueueItem::REPEAT;
194 qi.reel = video_reel (frame);
195 qi.frame = frame - _reels[qi.reel].start ();
196 if (_film->three_d() && eyes == EYES_BOTH) {
198 _queue.push_back (qi);
199 qi.eyes = EYES_RIGHT;
200 _queue.push_back (qi);
203 _queue.push_back (qi);
206 /* Now there's something to do: wake anything wait()ing on _empty_condition */
207 _empty_condition.notify_all ();
211 Writer::fake_write (Frame frame, Eyes eyes)
213 boost::mutex::scoped_lock lock (_state_mutex);
215 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
216 /* The queue is too big, and the main writer thread can run and fix it, so
217 wake it and wait until it has done.
219 _empty_condition.notify_all ();
220 _full_condition.wait (lock);
223 size_t const reel = video_reel (frame);
224 Frame const frame_in_reel = frame - _reels[reel].start ();
227 qi.type = QueueItem::FAKE;
230 shared_ptr<InfoFileHandle> info_file = _film->info_file_handle(_reels[reel].period(), true);
231 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
235 qi.frame = frame_in_reel;
236 if (_film->three_d() && eyes == EYES_BOTH) {
238 _queue.push_back (qi);
239 qi.eyes = EYES_RIGHT;
240 _queue.push_back (qi);
243 _queue.push_back (qi);
246 /* Now there's something to do: wake anything wait()ing on _empty_condition */
247 _empty_condition.notify_all ();
250 /** Write some audio frames to the DCP.
251 * @param audio Audio data.
252 * @param time Time of this data within the DCP.
253 * This method is not thread safe.
256 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
258 DCPOMATIC_ASSERT (audio);
260 int const afr = _film->audio_frame_rate();
262 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
264 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
269 if (_audio_reel == _reels.end ()) {
270 /* This audio is off the end of the last reel; ignore it */
274 if (end <= _audio_reel->period().to) {
275 /* Easy case: we can write all the audio to this reel */
276 _audio_reel->write (audio);
278 } else if (_audio_reel->period().to <= t) {
279 /* This reel is entirely before the start of our audio; just skip the reel */
282 /* This audio is over a reel boundary; split the audio into two and write the first part */
283 DCPTime part_lengths[2] = {
284 _audio_reel->period().to - t,
285 end - _audio_reel->period().to
288 Frame part_frames[2] = {
289 part_lengths[0].frames_ceil(afr),
290 part_lengths[1].frames_ceil(afr)
293 if (part_frames[0]) {
294 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
295 _audio_reel->write (part);
298 if (part_frames[1]) {
299 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
305 t += part_lengths[0];
312 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
314 if (_atmos_reel->period().to == time) {
316 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
319 /* We assume that we get a video frame's worth of data here */
320 _atmos_reel->write (atmos, metadata);
324 /** Caller must hold a lock on _state_mutex */
326 Writer::have_sequenced_image_at_queue_head ()
328 if (_queue.empty ()) {
333 QueueItem const & f = _queue.front();
334 return _last_written[f.reel].next(f);
339 Writer::LastWritten::next (QueueItem qi) const
341 if (qi.eyes == EYES_BOTH) {
343 return qi.frame == (_frame + 1);
348 if (_eyes == EYES_LEFT && qi.frame == _frame && qi.eyes == EYES_RIGHT) {
352 if (_eyes == EYES_RIGHT && qi.frame == (_frame + 1) && qi.eyes == EYES_LEFT) {
361 Writer::LastWritten::update (QueueItem qi)
374 boost::mutex::scoped_lock lock (_state_mutex);
378 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
379 /* We've got something to do: go and do it */
383 /* Nothing to do: wait until something happens which may indicate that we do */
384 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
385 _empty_condition.wait (lock);
386 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
389 if (_finish && _queue.empty()) {
393 /* We stop here if we have been asked to finish, and if either the queue
394 is empty or we do not have a sequenced image at its head (if this is the
395 case we will never terminate as no new frames will be sent once
398 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
399 /* (Hopefully temporarily) log anything that was not written */
400 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
401 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
402 BOOST_FOREACH (QueueItem const& i, _queue) {
403 if (i.type == QueueItem::FULL) {
404 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
406 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
413 /* Write any frames that we can write; i.e. those that are in sequence. */
414 while (have_sequenced_image_at_queue_head ()) {
415 QueueItem qi = _queue.front ();
416 _last_written[qi.reel].update (qi);
418 if (qi.type == QueueItem::FULL && qi.encoded) {
419 --_queued_full_in_memory;
424 ReelWriter& reel = _reels[qi.reel];
427 case QueueItem::FULL:
428 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
430 qi.encoded.reset (new ArrayData(_film->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
432 reel.write (qi.encoded, qi.frame, qi.eyes);
435 case QueueItem::FAKE:
436 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
437 reel.fake_write (qi.size);
440 case QueueItem::REPEAT:
441 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
442 reel.repeat_write (qi.frame, qi.eyes);
448 _full_condition.notify_all ();
451 while (_queued_full_in_memory > _maximum_frames_in_memory) {
452 /* Too many frames in memory which can't yet be written to the stream.
453 Write some FULL frames to disk.
456 /* Find one from the back of the queue */
458 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
459 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
463 DCPOMATIC_ASSERT (i != _queue.rend());
465 /* For the log message below */
466 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
469 /* i is valid here, even though we don't hold a lock on the mutex,
470 since list iterators are unaffected by insertion and only this
471 thread could erase the last item in the list.
474 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
476 i->encoded->write_via_temp (
477 _film->j2c_path (i->reel, i->frame, i->eyes, true),
478 _film->j2c_path (i->reel, i->frame, i->eyes, false)
483 --_queued_full_in_memory;
484 _full_condition.notify_all ();
494 Writer::terminate_thread (bool can_throw)
496 boost::this_thread::disable_interruption dis;
498 boost::mutex::scoped_lock lock (_state_mutex);
501 _empty_condition.notify_all ();
502 _full_condition.notify_all ();
517 if (!_thread.joinable()) {
521 LOG_GENERAL_NC ("Terminating writer thread");
523 terminate_thread (true);
525 LOG_GENERAL_NC ("Finishing ReelWriters");
527 BOOST_FOREACH (ReelWriter& i, _reels) {
531 LOG_GENERAL_NC ("Writing XML");
533 dcp::DCP dcp (_film->dir (_film->dcp_name()));
535 shared_ptr<dcp::CPL> cpl (
538 _film->dcp_content_type()->libdcp_kind ()
544 /* Calculate digests for each reel in parallel */
546 shared_ptr<Job> job = _job.lock ();
547 job->sub (_("Computing digests"));
549 boost::asio::io_service service;
550 boost::thread_group pool;
552 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
554 int const threads = max (1, Config::instance()->master_encoding_threads ());
556 for (int i = 0; i < threads; ++i) {
557 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
560 BOOST_FOREACH (ReelWriter& i, _reels) {
561 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
562 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
571 BOOST_FOREACH (ReelWriter& i, _reels) {
572 cpl->add (i.create_reel (_reel_assets, _fonts));
577 string creator = Config::instance()->dcp_creator();
578 if (creator.empty()) {
579 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
582 string issuer = Config::instance()->dcp_issuer();
583 if (issuer.empty()) {
584 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
587 cpl->set_ratings (_film->ratings());
589 vector<dcp::ContentVersion> cv;
590 BOOST_FOREACH (string i, _film->content_versions()) {
591 cv.push_back (dcp::ContentVersion(i));
593 cpl->set_content_versions (cv);
595 cpl->set_full_content_title_text (_film->name());
596 cpl->set_full_content_title_text_language (_film->name_language());
597 cpl->set_release_territory (_film->release_territory());
598 cpl->set_version_number (_film->version_number());
599 cpl->set_status (_film->status());
600 cpl->set_chain (_film->chain());
601 cpl->set_distributor (_film->distributor());
602 cpl->set_facility (_film->facility());
603 cpl->set_luminance (_film->luminance());
605 list<int> ac = _film->mapped_audio_channels ();
606 dcp::MCASoundField field = (
607 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSL)) != ac.end() ||
608 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSR)) != ac.end()
609 ) ? dcp::SEVEN_POINT_ONE : dcp::FIVE_POINT_ONE;
611 dcp::MainSoundConfiguration msc (field, _film->audio_channels());
612 BOOST_FOREACH (int i, ac) {
613 if (i < _film->audio_channels()) {
614 msc.set_mapping (i, static_cast<dcp::Channel>(i));
618 cpl->set_main_sound_configuration (msc.to_string());
619 cpl->set_main_sound_sample_rate (_film->audio_frame_rate());
620 cpl->set_main_picture_stored_area (_film->frame_size());
621 cpl->set_main_picture_active_area (_film->active_area());
623 shared_ptr<const dcp::CertificateChain> signer;
624 signer = Config::instance()->signer_chain ();
625 /* We did check earlier, but check again here to be on the safe side */
627 if (!signer->valid (&reason)) {
628 throw InvalidSignerError (reason);
632 _film->interop() ? dcp::INTEROP : dcp::SMPTE,
635 dcp::LocalTime().as_string(),
636 String::compose("Created by libdcp %1", dcp::version),
638 Config::instance()->dcp_metadata_filename_format()
642 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
645 write_cover_sheet ();
649 Writer::write_cover_sheet ()
651 boost::filesystem::path const cover = _film->file ("COVER_SHEET.txt");
652 FILE* f = fopen_boost (cover, "w");
654 throw OpenFileError (cover, errno, OpenFileError::WRITE);
657 string text = Config::instance()->cover_sheet ();
658 boost::algorithm::replace_all (text, "$CPL_NAME", _film->name());
659 boost::algorithm::replace_all (text, "$TYPE", _film->dcp_content_type()->pretty_name());
660 boost::algorithm::replace_all (text, "$CONTAINER", _film->container()->container_nickname());
661 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _film->isdcf_metadata().audio_language);
663 optional<string> subtitle_language;
664 BOOST_FOREACH (shared_ptr<Content> i, _film->content()) {
665 BOOST_FOREACH (shared_ptr<TextContent> j, i->text) {
666 if (j->type() == TEXT_OPEN_SUBTITLE && j->use()) {
667 subtitle_language = j->language ();
671 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_language.get_value_or("None"));
673 boost::uintmax_t size = 0;
675 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(_film->dir(_film->dcp_name()));
676 i != boost::filesystem::recursive_directory_iterator();
678 if (boost::filesystem::is_regular_file (i->path ())) {
679 size += boost::filesystem::file_size (i->path ());
683 if (size > (1000000000L)) {
684 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
686 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
689 pair<int, int> ch = audio_channel_types (_film->mapped_audio_channels(), _film->audio_channels());
690 string description = String::compose("%1.%2", ch.first, ch.second);
692 if (description == "0.0") {
693 description = _("None");
694 } else if (description == "1.0") {
695 description = _("Mono");
696 } else if (description == "2.0") {
697 description = _("Stereo");
699 boost::algorithm::replace_all (text, "$AUDIO", description);
702 _film->length().split (_film->video_frame_rate(), h, m, s, fr);
704 if (h == 0 && m == 0) {
705 length = String::compose("%1s", s);
706 } else if (h == 0 && m > 0) {
707 length = String::compose("%1m%2s", m, s);
708 } else if (h > 0 && m > 0) {
709 length = String::compose("%1h%2m%3s", h, m, s);
712 boost::algorithm::replace_all (text, "$LENGTH", length);
714 checked_fwrite (text.c_str(), text.length(), f, cover);
718 /** @param frame Frame index within the whole DCP.
719 * @return true if we can fake-write this frame.
722 Writer::can_fake_write (Frame frame) const
724 if (_film->encrypted()) {
725 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
729 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
730 parameters in the asset writer.
733 ReelWriter const & reel = _reels[video_reel(frame)];
735 /* Make frame relative to the start of the reel */
736 frame -= reel.start ();
737 return (frame != 0 && frame < reel.first_nonexistant_frame());
740 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
742 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
744 vector<ReelWriter>::iterator* reel = 0;
747 case TEXT_OPEN_SUBTITLE:
748 reel = &_subtitle_reel;
750 case TEXT_CLOSED_CAPTION:
751 DCPOMATIC_ASSERT (track);
752 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
753 reel = &_caption_reels[*track];
756 DCPOMATIC_ASSERT (false);
759 DCPOMATIC_ASSERT (*reel != _reels.end());
760 while ((*reel)->period().to <= period.from) {
762 DCPOMATIC_ASSERT (*reel != _reels.end());
765 (*reel)->write (text, type, track, period);
769 Writer::write (list<shared_ptr<Font> > fonts)
771 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
773 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
775 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
782 _fonts.push_back (i);
788 operator< (QueueItem const & a, QueueItem const & b)
790 if (a.reel != b.reel) {
791 return a.reel < b.reel;
794 if (a.frame != b.frame) {
795 return a.frame < b.frame;
798 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
802 operator== (QueueItem const & a, QueueItem const & b)
804 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
808 Writer::set_encoder_threads (int threads)
810 boost::mutex::scoped_lock lm (_state_mutex);
811 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
812 _maximum_queue_size = threads * 16;
816 Writer::write (ReferencedReelAsset asset)
818 _reel_assets.push_back (asset);
822 Writer::video_reel (int frame) const
824 DCPTime t = DCPTime::from_frames (frame, _film->video_frame_rate ());
826 while (i < _reels.size() && !_reels[i].period().contains (t)) {
830 DCPOMATIC_ASSERT (i < _reels.size ());
835 Writer::set_digest_progress (Job* job, float progress)
837 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
839 _digest_progresses[boost::this_thread::get_id()] = progress;
840 float min_progress = FLT_MAX;
841 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
842 min_progress = min (min_progress, i->second);
845 job->set_progress (min_progress);