2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
23 #include "compose.hpp"
27 #include "dcpomatic_log.h"
28 #include "dcp_video.h"
29 #include "dcp_content_type.h"
30 #include "audio_mapping.h"
34 #include "audio_buffers.h"
36 #include "font_data.h"
38 #include "reel_writer.h"
39 #include "text_content.h"
41 #include <dcp/locale_convert.h>
42 #include <dcp/reel_file_asset.h>
51 /* OS X strikes again */
56 using std::dynamic_pointer_cast;
59 using std::make_shared;
64 using std::shared_ptr;
68 using boost::optional;
69 #if BOOST_VERSION >= 106100
70 using namespace boost::placeholders;
74 using namespace dcpomatic;
79 ignore_progress (float)
85 /** @param j Job to report progress to, or 0.
86 * @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
88 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
89 : WeakConstFilm (weak_film)
91 /* These will be reset to sensible values when J2KEncoder is created */
92 , _maximum_frames_in_memory (8)
93 , _maximum_queue_size (8)
94 , _text_only (text_only)
96 auto job = _job.lock ();
99 auto const reels = film()->reels();
100 for (auto p: reels) {
101 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
104 _last_written.resize (reels.size());
106 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
107 and captions arrive to the Writer in sequence. This is not so for video.
109 _audio_reel = _reels.begin ();
110 _subtitle_reel = _reels.begin ();
111 for (auto i: film()->closed_caption_tracks()) {
112 _caption_reels[i] = _reels.begin ();
114 _atmos_reel = _reels.begin ();
116 /* Check that the signer is OK */
118 if (!Config::instance()->signer_chain()->valid(&reason)) {
119 throw InvalidSignerError (reason);
128 _thread = boost::thread (boost::bind(&Writer::thread, this));
129 #ifdef DCPOMATIC_LINUX
130 pthread_setname_np (_thread.native_handle(), "writer");
139 terminate_thread (false);
144 /** Pass a video frame to the writer for writing to disk at some point.
145 * This method can be called with frames out of order.
146 * @param encoded JPEG2000-encoded data.
147 * @param frame Frame index within the DCP.
148 * @param eyes Eyes that this frame image is for.
151 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
153 boost::mutex::scoped_lock lock (_state_mutex);
155 while (_queued_full_in_memory > _maximum_frames_in_memory) {
156 /* There are too many full frames in memory; wake the main writer thread and
157 wait until it sorts everything out */
158 _empty_condition.notify_all ();
159 _full_condition.wait (lock);
163 qi.type = QueueItem::Type::FULL;
164 qi.encoded = encoded;
165 qi.reel = video_reel (frame);
166 qi.frame = frame - _reels[qi.reel].start ();
168 if (film()->three_d() && eyes == Eyes::BOTH) {
169 /* 2D material in a 3D DCP; fake the 3D */
170 qi.eyes = Eyes::LEFT;
171 _queue.push_back (qi);
172 ++_queued_full_in_memory;
173 qi.eyes = Eyes::RIGHT;
174 _queue.push_back (qi);
175 ++_queued_full_in_memory;
178 _queue.push_back (qi);
179 ++_queued_full_in_memory;
182 /* Now there's something to do: wake anything wait()ing on _empty_condition */
183 _empty_condition.notify_all ();
188 Writer::can_repeat (Frame frame) const
190 return frame > _reels[video_reel(frame)].start();
194 /** Repeat the last frame that was written to a reel as a new frame.
195 * @param frame Frame index within the DCP of the new (repeated) frame.
196 * @param eyes Eyes that this repeated frame image is for.
199 Writer::repeat (Frame frame, Eyes eyes)
201 boost::mutex::scoped_lock lock (_state_mutex);
203 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
204 /* The queue is too big, and the main writer thread can run and fix it, so
205 wake it and wait until it has done.
207 _empty_condition.notify_all ();
208 _full_condition.wait (lock);
212 qi.type = QueueItem::Type::REPEAT;
213 qi.reel = video_reel (frame);
214 qi.frame = frame - _reels[qi.reel].start ();
215 if (film()->three_d() && eyes == Eyes::BOTH) {
216 qi.eyes = Eyes::LEFT;
217 _queue.push_back (qi);
218 qi.eyes = Eyes::RIGHT;
219 _queue.push_back (qi);
222 _queue.push_back (qi);
225 /* Now there's something to do: wake anything wait()ing on _empty_condition */
226 _empty_condition.notify_all ();
231 Writer::fake_write (Frame frame, Eyes eyes)
233 boost::mutex::scoped_lock lock (_state_mutex);
235 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
236 /* The queue is too big, and the main writer thread can run and fix it, so
237 wake it and wait until it has done.
239 _empty_condition.notify_all ();
240 _full_condition.wait (lock);
243 size_t const reel = video_reel (frame);
244 Frame const frame_in_reel = frame - _reels[reel].start ();
247 qi.type = QueueItem::Type::FAKE;
250 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
251 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
255 qi.frame = frame_in_reel;
256 if (film()->three_d() && eyes == Eyes::BOTH) {
257 qi.eyes = Eyes::LEFT;
258 _queue.push_back (qi);
259 qi.eyes = Eyes::RIGHT;
260 _queue.push_back (qi);
263 _queue.push_back (qi);
266 /* Now there's something to do: wake anything wait()ing on _empty_condition */
267 _empty_condition.notify_all ();
271 /** Write some audio frames to the DCP.
272 * @param audio Audio data.
273 * @param time Time of this data within the DCP.
274 * This method is not thread safe.
277 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
279 DCPOMATIC_ASSERT (audio);
281 int const afr = film()->audio_frame_rate();
283 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
285 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
290 if (_audio_reel == _reels.end ()) {
291 /* This audio is off the end of the last reel; ignore it */
295 if (end <= _audio_reel->period().to) {
296 /* Easy case: we can write all the audio to this reel */
297 _audio_reel->write (audio);
299 } else if (_audio_reel->period().to <= t) {
300 /* This reel is entirely before the start of our audio; just skip the reel */
303 /* This audio is over a reel boundary; split the audio into two and write the first part */
304 DCPTime part_lengths[2] = {
305 _audio_reel->period().to - t,
306 end - _audio_reel->period().to
309 /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
310 Frame part_frames[2] = {
311 part_lengths[0].frames_ceil(afr),
312 part_lengths[1].frames_floor(afr)
315 DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
317 if (part_frames[0]) {
318 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
319 _audio_reel->write (part);
322 if (part_frames[1]) {
323 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
329 t += part_lengths[0];
336 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
338 if (_atmos_reel->period().to == time) {
340 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
343 /* We assume that we get a video frame's worth of data here */
344 _atmos_reel->write (atmos, metadata);
348 /** Caller must hold a lock on _state_mutex */
350 Writer::have_sequenced_image_at_queue_head ()
352 if (_queue.empty ()) {
357 auto const & f = _queue.front();
358 return _last_written[f.reel].next(f);
363 Writer::LastWritten::next (QueueItem qi) const
365 if (qi.eyes == Eyes::BOTH) {
367 return qi.frame == (_frame + 1);
372 if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
376 if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
385 Writer::LastWritten::update (QueueItem qi)
398 boost::mutex::scoped_lock lock (_state_mutex);
402 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
403 /* We've got something to do: go and do it */
407 /* Nothing to do: wait until something happens which may indicate that we do */
408 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
409 _empty_condition.wait (lock);
410 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
413 /* We stop here if we have been asked to finish, and if either the queue
414 is empty or we do not have a sequenced image at its head (if this is the
415 case we will never terminate as no new frames will be sent once
418 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
419 /* (Hopefully temporarily) log anything that was not written */
420 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
421 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
422 for (auto const& i: _queue) {
423 if (i.type == QueueItem::Type::FULL) {
424 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
426 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
433 /* Write any frames that we can write; i.e. those that are in sequence. */
434 while (have_sequenced_image_at_queue_head ()) {
435 auto qi = _queue.front ();
436 _last_written[qi.reel].update (qi);
438 if (qi.type == QueueItem::Type::FULL && qi.encoded) {
439 --_queued_full_in_memory;
444 auto& reel = _reels[qi.reel];
447 case QueueItem::Type::FULL:
448 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
450 qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
452 reel.write (qi.encoded, qi.frame, qi.eyes);
455 case QueueItem::Type::FAKE:
456 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
457 reel.fake_write (qi.size);
460 case QueueItem::Type::REPEAT:
461 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
462 reel.repeat_write (qi.frame, qi.eyes);
468 _full_condition.notify_all ();
471 while (_queued_full_in_memory > _maximum_frames_in_memory) {
472 /* Too many frames in memory which can't yet be written to the stream.
473 Write some FULL frames to disk.
476 /* Find one from the back of the queue */
478 auto i = _queue.rbegin ();
479 while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
483 DCPOMATIC_ASSERT (i != _queue.rend());
485 /* For the log message below */
486 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
489 /* i is valid here, even though we don't hold a lock on the mutex,
490 since list iterators are unaffected by insertion and only this
491 thread could erase the last item in the list.
494 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
496 i->encoded->write_via_temp (
497 film()->j2c_path(i->reel, i->frame, i->eyes, true),
498 film()->j2c_path(i->reel, i->frame, i->eyes, false)
503 --_queued_full_in_memory;
504 _full_condition.notify_all ();
515 Writer::terminate_thread (bool can_throw)
517 boost::this_thread::disable_interruption dis;
519 boost::mutex::scoped_lock lock (_state_mutex);
522 _empty_condition.notify_all ();
523 _full_condition.notify_all ();
536 /** @param output_dcp Path to DCP folder to write */
538 Writer::finish (boost::filesystem::path output_dcp)
540 if (_thread.joinable()) {
541 LOG_GENERAL_NC ("Terminating writer thread");
542 terminate_thread (true);
545 LOG_GENERAL_NC ("Finishing ReelWriters");
547 for (auto& i: _reels) {
548 i.finish (output_dcp);
551 LOG_GENERAL_NC ("Writing XML");
553 dcp::DCP dcp (output_dcp);
555 auto cpl = make_shared<dcp::CPL>(
557 film()->dcp_content_type()->libdcp_kind()
562 /* Calculate digests for each reel in parallel */
564 auto job = _job.lock ();
566 job->sub (_("Computing digests"));
569 boost::asio::io_service service;
570 boost::thread_group pool;
572 auto work = make_shared<boost::asio::io_service::work>(service);
574 int const threads = max (1, Config::instance()->master_encoding_threads ());
576 for (int i = 0; i < threads; ++i) {
577 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
580 boost::function<void (float)> set_progress;
582 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
584 set_progress = &ignore_progress;
587 for (auto& i: _reels) {
588 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
590 service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
598 for (auto& i: _reels) {
599 cpl->add (i.create_reel(_reel_assets, _fonts, output_dcp, _have_subtitles, _have_closed_captions));
604 auto creator = Config::instance()->dcp_creator();
605 if (creator.empty()) {
606 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
609 auto issuer = Config::instance()->dcp_issuer();
610 if (issuer.empty()) {
611 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
614 cpl->set_ratings (film()->ratings());
616 vector<dcp::ContentVersion> cv;
617 for (auto i: film()->content_versions()) {
618 cv.push_back (dcp::ContentVersion(i));
620 cpl->set_content_versions (cv);
622 cpl->set_full_content_title_text (film()->name());
623 cpl->set_full_content_title_text_language (film()->name_language());
624 cpl->set_release_territory (film()->release_territory());
625 cpl->set_version_number (film()->version_number());
626 cpl->set_status (film()->status());
627 cpl->set_chain (film()->chain());
628 cpl->set_distributor (film()->distributor());
629 cpl->set_facility (film()->facility());
630 cpl->set_luminance (film()->luminance());
632 auto ac = film()->mapped_audio_channels();
633 dcp::MCASoundField field = (
634 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSL)) != ac.end() ||
635 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSR)) != ac.end()
636 ) ? dcp::MCASoundField::SEVEN_POINT_ONE : dcp::MCASoundField::FIVE_POINT_ONE;
638 dcp::MainSoundConfiguration msc (field, film()->audio_channels());
640 if (static_cast<int>(i) < film()->audio_channels()) {
641 msc.set_mapping (i, static_cast<dcp::Channel>(i));
645 cpl->set_main_sound_configuration (msc.to_string());
646 cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
647 cpl->set_main_picture_stored_area (film()->frame_size());
649 auto active_area = film()->active_area();
650 if (active_area.width > 0 && active_area.height > 0) {
651 /* It's not allowed to have a zero active area width or height */
652 cpl->set_main_picture_active_area (active_area);
655 vector<dcp::LanguageTag> sl = film()->subtitle_languages();
657 cpl->set_additional_subtitle_languages(std::vector<dcp::LanguageTag>(sl.begin() + 1, sl.end()));
660 auto signer = Config::instance()->signer_chain();
661 /* We did check earlier, but check again here to be on the safe side */
663 if (!signer->valid (&reason)) {
664 throw InvalidSignerError (reason);
668 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE,
671 dcp::LocalTime().as_string(),
674 Config::instance()->dcp_metadata_filename_format()
678 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
681 write_cover_sheet (output_dcp);
686 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
688 auto const cover = film()->file("COVER_SHEET.txt");
689 auto f = fopen_boost (cover, "w");
691 throw OpenFileError (cover, errno, OpenFileError::WRITE);
694 auto text = Config::instance()->cover_sheet ();
695 boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
696 boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
697 boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
698 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", film()->isdcf_metadata().audio_language);
700 auto subtitle_languages = film()->subtitle_languages();
701 if (subtitle_languages.empty()) {
702 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", "None");
704 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.front().description());
707 boost::uintmax_t size = 0;
709 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
710 i != boost::filesystem::recursive_directory_iterator();
712 if (boost::filesystem::is_regular_file (i->path())) {
713 size += boost::filesystem::file_size (i->path());
717 if (size > (1000000000L)) {
718 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
720 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
723 auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
724 auto description = String::compose("%1.%2", ch.first, ch.second);
726 if (description == "0.0") {
727 description = _("None");
728 } else if (description == "1.0") {
729 description = _("Mono");
730 } else if (description == "2.0") {
731 description = _("Stereo");
733 boost::algorithm::replace_all (text, "$AUDIO", description);
736 film()->length().split(film()->video_frame_rate(), h, m, s, fr);
738 if (h == 0 && m == 0) {
739 length = String::compose("%1s", s);
740 } else if (h == 0 && m > 0) {
741 length = String::compose("%1m%2s", m, s);
742 } else if (h > 0 && m > 0) {
743 length = String::compose("%1h%2m%3s", h, m, s);
746 boost::algorithm::replace_all (text, "$LENGTH", length);
748 checked_fwrite (text.c_str(), text.length(), f, cover);
753 /** @param frame Frame index within the whole DCP.
754 * @return true if we can fake-write this frame.
757 Writer::can_fake_write (Frame frame) const
759 if (film()->encrypted()) {
760 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
764 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
765 parameters in the asset writer.
768 auto const & reel = _reels[video_reel(frame)];
770 /* Make frame relative to the start of the reel */
771 frame -= reel.start ();
772 return (frame != 0 && frame < reel.first_nonexistant_frame());
776 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
778 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
780 vector<ReelWriter>::iterator* reel = nullptr;
783 case TextType::OPEN_SUBTITLE:
784 reel = &_subtitle_reel;
785 _have_subtitles = true;
787 case TextType::CLOSED_CAPTION:
788 DCPOMATIC_ASSERT (track);
789 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
790 reel = &_caption_reels[*track];
791 _have_closed_captions.insert (*track);
794 DCPOMATIC_ASSERT (false);
797 DCPOMATIC_ASSERT (*reel != _reels.end());
798 while ((*reel)->period().to <= period.from) {
800 DCPOMATIC_ASSERT (*reel != _reels.end());
803 (*reel)->write (text, type, track, period);
808 Writer::write (vector<FontData> fonts)
810 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
812 for (auto const& i: fonts) {
814 for (auto& j: _fonts) {
821 _fonts.push_back (i);
828 operator< (QueueItem const & a, QueueItem const & b)
830 if (a.reel != b.reel) {
831 return a.reel < b.reel;
834 if (a.frame != b.frame) {
835 return a.frame < b.frame;
838 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
843 operator== (QueueItem const & a, QueueItem const & b)
845 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
850 Writer::set_encoder_threads (int threads)
852 boost::mutex::scoped_lock lm (_state_mutex);
853 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
854 _maximum_queue_size = threads * 16;
859 Writer::write (ReferencedReelAsset asset)
861 _reel_assets.push_back (asset);
866 Writer::video_reel (int frame) const
868 auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
870 while (i < _reels.size() && !_reels[i].period().contains (t)) {
874 DCPOMATIC_ASSERT (i < _reels.size ());
880 Writer::set_digest_progress (Job* job, float progress)
882 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
884 _digest_progresses[boost::this_thread::get_id()] = progress;
885 float min_progress = FLT_MAX;
886 for (auto const& i: _digest_progresses) {
887 min_progress = min (min_progress, i.second);
890 job->set_progress (min_progress);
897 /** Calculate hashes for any referenced MXF assets which do not already have one */
899 Writer::calculate_referenced_digests (boost::function<void (float)> set_progress)
901 for (auto const& i: _reel_assets) {
902 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
903 if (file && !file->hash()) {
904 file->asset_ref().asset()->hash (set_progress);
905 file->set_hash (file->asset_ref().asset()->hash());