2 Copyright (C) 2012-2018 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcp_video.h"
27 #include "dcp_content_type.h"
28 #include "audio_mapping.h"
32 #include "audio_buffers.h"
36 #include "reel_writer.h"
38 #include <dcp/locale_convert.h>
39 #include <boost/foreach.hpp>
47 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
48 #define LOG_GENERAL_NC(...) _film->log()->log (__VA_ARGS__, LogEntry::TYPE_GENERAL);
49 #define LOG_DEBUG_ENCODE(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_DEBUG_ENCODE);
50 #define LOG_TIMING(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_TIMING);
51 #define LOG_WARNING_NC(...) _film->log()->log (__VA_ARGS__, LogEntry::TYPE_WARNING);
52 #define LOG_WARNING(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_WARNING);
53 #define LOG_ERROR(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_ERROR);
55 /* OS X strikes again */
67 using boost::shared_ptr;
68 using boost::weak_ptr;
69 using boost::dynamic_pointer_cast;
70 using boost::optional;
73 Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
78 , _queued_full_in_memory (0)
79 /* These will be reset to sensible values when J2KEncoder is created */
80 , _maximum_frames_in_memory (8)
81 , _maximum_queue_size (8)
87 shared_ptr<Job> job = _job.lock ();
88 DCPOMATIC_ASSERT (job);
91 list<DCPTimePeriod> const reels = _film->reels ();
92 BOOST_FOREACH (DCPTimePeriod p, reels) {
93 _reels.push_back (ReelWriter (film, p, job, reel_index++, reels.size(), _film->content_summary(p)));
96 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
97 and captions arrive to the Writer in sequence. This is not so for video.
99 _audio_reel = _reels.begin ();
100 _subtitle_reel = _reels.begin ();
101 BOOST_FOREACH (DCPTextTrack i, _film->closed_caption_tracks()) {
102 _caption_reels[i] = _reels.begin ();
105 /* Check that the signer is OK if we need one */
107 if (_film->is_signed() && !Config::instance()->signer_chain()->valid(&reason)) {
108 throw InvalidSignerError (reason);
115 _thread = new boost::thread (boost::bind (&Writer::thread, this));
116 #ifdef DCPOMATIC_LINUX
117 pthread_setname_np (_thread->native_handle(), "writer");
123 terminate_thread (false);
126 /** Pass a video frame to the writer for writing to disk at some point.
127 * This method can be called with frames out of order.
128 * @param encoded JPEG2000-encoded data.
129 * @param frame Frame index within the DCP.
130 * @param eyes Eyes that this frame image is for.
133 Writer::write (Data encoded, Frame frame, Eyes eyes)
135 boost::mutex::scoped_lock lock (_state_mutex);
137 while (_queued_full_in_memory > _maximum_frames_in_memory) {
138 /* There are too many full frames in memory; wake the main writer thread and
139 wait until it sorts everything out */
140 _empty_condition.notify_all ();
141 _full_condition.wait (lock);
145 qi.type = QueueItem::FULL;
146 qi.encoded = encoded;
147 qi.reel = video_reel (frame);
148 qi.frame = frame - _reels[qi.reel].start ();
150 if (_film->three_d() && eyes == EYES_BOTH) {
151 /* 2D material in a 3D DCP; fake the 3D */
153 _queue.push_back (qi);
154 ++_queued_full_in_memory;
155 qi.eyes = EYES_RIGHT;
156 _queue.push_back (qi);
157 ++_queued_full_in_memory;
160 _queue.push_back (qi);
161 ++_queued_full_in_memory;
164 /* Now there's something to do: wake anything wait()ing on _empty_condition */
165 _empty_condition.notify_all ();
169 Writer::can_repeat (Frame frame) const
171 return frame > _reels[video_reel(frame)].start();
174 /** Repeat the last frame that was written to a reel as a new frame.
175 * @param frame Frame index within the DCP of the new (repeated) frame.
176 * @param eyes Eyes that this repeated frame image is for.
179 Writer::repeat (Frame frame, Eyes eyes)
181 boost::mutex::scoped_lock lock (_state_mutex);
183 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
184 /* The queue is too big, and the main writer thread can run and fix it, so
185 wake it and wait until it has done.
187 _empty_condition.notify_all ();
188 _full_condition.wait (lock);
192 qi.type = QueueItem::REPEAT;
193 qi.reel = video_reel (frame);
194 qi.frame = frame - _reels[qi.reel].start ();
195 if (_film->three_d() && eyes == EYES_BOTH) {
197 _queue.push_back (qi);
198 qi.eyes = EYES_RIGHT;
199 _queue.push_back (qi);
202 _queue.push_back (qi);
205 /* Now there's something to do: wake anything wait()ing on _empty_condition */
206 _empty_condition.notify_all ();
210 Writer::fake_write (Frame frame, Eyes eyes)
212 boost::mutex::scoped_lock lock (_state_mutex);
214 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
215 /* The queue is too big, and the main writer thread can run and fix it, so
216 wake it and wait until it has done.
218 _empty_condition.notify_all ();
219 _full_condition.wait (lock);
222 size_t const reel = video_reel (frame);
223 Frame const reel_frame = frame - _reels[reel].start ();
225 FILE* file = fopen_boost (_film->info_file(_reels[reel].period()), "rb");
227 throw ReadFileError (_film->info_file(_reels[reel].period()));
229 dcp::FrameInfo info = _reels[reel].read_frame_info (file, reel_frame, eyes);
233 qi.type = QueueItem::FAKE;
236 qi.frame = reel_frame;
237 if (_film->three_d() && eyes == EYES_BOTH) {
239 _queue.push_back (qi);
240 qi.eyes = EYES_RIGHT;
241 _queue.push_back (qi);
244 _queue.push_back (qi);
247 /* Now there's something to do: wake anything wait()ing on _empty_condition */
248 _empty_condition.notify_all ();
251 /** Write some audio frames to the DCP.
252 * @param audio Audio data.
253 * @param time Time of this data within the DCP.
254 * This method is not thread safe.
257 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
259 DCPOMATIC_ASSERT (audio);
261 int const afr = _film->audio_frame_rate();
263 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
265 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
270 if (_audio_reel == _reels.end ()) {
271 /* This audio is off the end of the last reel; ignore it */
275 if (end <= _audio_reel->period().to) {
276 /* Easy case: we can write all the audio to this reel */
277 _audio_reel->write (audio);
280 /* Split the audio into two and write the first part */
281 DCPTime part_lengths[2] = {
282 _audio_reel->period().to - t,
283 end - _audio_reel->period().to
286 Frame part_frames[2] = {
287 part_lengths[0].frames_ceil(afr),
288 part_lengths[1].frames_ceil(afr)
291 if (part_frames[0]) {
292 shared_ptr<AudioBuffers> part (new AudioBuffers (audio->channels(), part_frames[0]));
293 part->copy_from (audio.get(), part_frames[0], 0, 0);
294 _audio_reel->write (part);
297 if (part_frames[1]) {
298 shared_ptr<AudioBuffers> part (new AudioBuffers (audio->channels(), part_frames[1]));
299 part->copy_from (audio.get(), part_frames[1], part_frames[0], 0);
306 t += part_lengths[0];
311 /** This must be called from Writer::thread() with an appropriate lock held */
313 Writer::have_sequenced_image_at_queue_head ()
315 if (_queue.empty ()) {
321 QueueItem const & f = _queue.front();
322 ReelWriter const & reel = _reels[f.reel];
324 /* The queue should contain only EYES_LEFT/EYES_RIGHT pairs or EYES_BOTH */
326 if (f.eyes == EYES_BOTH) {
328 return f.frame == (reel.last_written_video_frame() + 1);
333 if (reel.last_written_eyes() == EYES_LEFT && f.frame == reel.last_written_video_frame() && f.eyes == EYES_RIGHT) {
337 if (reel.last_written_eyes() == EYES_RIGHT && f.frame == (reel.last_written_video_frame() + 1) && f.eyes == EYES_LEFT) {
350 boost::mutex::scoped_lock lock (_state_mutex);
354 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
355 /* We've got something to do: go and do it */
359 /* Nothing to do: wait until something happens which may indicate that we do */
360 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
361 _empty_condition.wait (lock);
362 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
365 if (_finish && _queue.empty()) {
369 /* We stop here if we have been asked to finish, and if either the queue
370 is empty or we do not have a sequenced image at its head (if this is the
371 case we will never terminate as no new frames will be sent once
374 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
375 /* (Hopefully temporarily) log anything that was not written */
376 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
377 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
378 for (list<QueueItem>::const_iterator i = _queue.begin(); i != _queue.end(); ++i) {
379 if (i->type == QueueItem::FULL) {
380 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i->frame, (int) i->eyes);
382 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i->size, i->frame, (int) i->eyes);
389 /* Write any frames that we can write; i.e. those that are in sequence. */
390 while (have_sequenced_image_at_queue_head ()) {
391 QueueItem qi = _queue.front ();
393 if (qi.type == QueueItem::FULL && qi.encoded) {
394 --_queued_full_in_memory;
399 ReelWriter& reel = _reels[qi.reel];
402 case QueueItem::FULL:
403 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
405 qi.encoded = Data (_film->j2c_path (qi.reel, qi.frame, qi.eyes, false));
407 reel.write (qi.encoded, qi.frame, qi.eyes);
410 case QueueItem::FAKE:
411 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
412 reel.fake_write (qi.frame, qi.eyes, qi.size);
415 case QueueItem::REPEAT:
416 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
417 reel.repeat_write (qi.frame, qi.eyes);
423 _full_condition.notify_all ();
426 while (_queued_full_in_memory > _maximum_frames_in_memory) {
427 /* Too many frames in memory which can't yet be written to the stream.
428 Write some FULL frames to disk.
431 /* Find one from the back of the queue */
433 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
434 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
438 DCPOMATIC_ASSERT (i != _queue.rend());
440 /* For the log message below */
441 int const awaiting = _reels[_queue.front().reel].last_written_video_frame() + 1;
444 /* i is valid here, even though we don't hold a lock on the mutex,
445 since list iterators are unaffected by insertion and only this
446 thread could erase the last item in the list.
449 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
451 i->encoded->write_via_temp (
452 _film->j2c_path (i->reel, i->frame, i->eyes, true),
453 _film->j2c_path (i->reel, i->frame, i->eyes, false)
458 --_queued_full_in_memory;
459 _full_condition.notify_all ();
469 Writer::terminate_thread (bool can_throw)
471 boost::mutex::scoped_lock lock (_state_mutex);
477 _empty_condition.notify_all ();
478 _full_condition.notify_all ();
481 if (_thread->joinable ()) {
500 LOG_GENERAL_NC ("Terminating writer thread");
502 terminate_thread (true);
504 LOG_GENERAL_NC ("Finishing ReelWriters");
506 BOOST_FOREACH (ReelWriter& i, _reels) {
510 LOG_GENERAL_NC ("Writing XML");
512 dcp::DCP dcp (_film->dir (_film->dcp_name()));
514 shared_ptr<dcp::CPL> cpl (
517 _film->dcp_content_type()->libdcp_kind ()
523 /* Calculate digests for each reel in parallel */
525 shared_ptr<Job> job = _job.lock ();
526 job->sub (_("Computing digests"));
528 boost::asio::io_service service;
529 boost::thread_group pool;
531 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
533 int const threads = max (1, Config::instance()->master_encoding_threads ());
535 for (int i = 0; i < threads; ++i) {
536 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
539 BOOST_FOREACH (ReelWriter& i, _reels) {
540 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
541 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
548 /* Add reels to CPL */
550 BOOST_FOREACH (ReelWriter& i, _reels) {
551 cpl->add (i.create_reel (_reel_assets, _fonts));
554 dcp::XMLMetadata meta;
555 meta.annotation_text = cpl->annotation_text ();
556 meta.creator = Config::instance()->dcp_creator ();
557 if (meta.creator.empty ()) {
558 meta.creator = String::compose ("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
560 meta.issuer = Config::instance()->dcp_issuer ();
561 if (meta.issuer.empty ()) {
562 meta.issuer = String::compose ("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
564 meta.set_issue_date_now ();
566 cpl->set_metadata (meta);
568 shared_ptr<const dcp::CertificateChain> signer;
569 if (_film->is_signed ()) {
570 signer = Config::instance()->signer_chain ();
571 /* We did check earlier, but check again here to be on the safe side */
573 if (!signer->valid (&reason)) {
574 throw InvalidSignerError (reason);
578 dcp.write_xml (_film->interop () ? dcp::INTEROP : dcp::SMPTE, meta, signer, Config::instance()->dcp_metadata_filename_format());
581 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
584 write_cover_sheet ();
588 Writer::write_cover_sheet ()
590 boost::filesystem::path const cover = _film->file ("COVER_SHEET.txt");
591 FILE* f = fopen_boost (cover, "w");
593 throw OpenFileError (cover, errno, false);
596 string text = Config::instance()->cover_sheet ();
597 boost::algorithm::replace_all (text, "$CPL_NAME", _film->name());
598 boost::algorithm::replace_all (text, "$TYPE", _film->dcp_content_type()->pretty_name());
599 boost::algorithm::replace_all (text, "$CONTAINER", _film->container()->container_nickname());
600 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _film->isdcf_metadata().audio_language);
601 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _film->isdcf_metadata().subtitle_language);
603 boost::uintmax_t size = 0;
605 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(_film->dir(_film->dcp_name()));
606 i != boost::filesystem::recursive_directory_iterator();
608 if (boost::filesystem::is_regular_file (i->path ())) {
609 size += boost::filesystem::file_size (i->path ());
613 if (size > (1000000000L)) {
614 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
616 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
619 pair<int, int> ch = audio_channel_types (_film->mapped_audio_channels(), _film->audio_channels());
620 string description = String::compose("%1.%2", ch.first, ch.second);
622 if (description == "0.0") {
623 description = _("None");
624 } else if (description == "1.0") {
625 description = _("Mono");
626 } else if (description == "2.0") {
627 description = _("Stereo");
629 boost::algorithm::replace_all (text, "$AUDIO", description);
632 _film->length().split (_film->video_frame_rate(), h, m, s, fr);
634 if (h == 0 && m == 0) {
635 length = String::compose("%1s", s);
636 } else if (h == 0 && m > 0) {
637 length = String::compose("%1m%2s", m, s);
638 } else if (h > 0 && m > 0) {
639 length = String::compose("%1h%2m%3s", h, m, s);
642 boost::algorithm::replace_all (text, "$LENGTH", length);
644 fwrite (text.c_str(), 1, text.length(), f);
648 /** @param frame Frame index within the whole DCP.
649 * @return true if we can fake-write this frame.
652 Writer::can_fake_write (Frame frame) const
654 if (_film->encrypted()) {
655 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
659 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
660 parameters in the asset writer.
663 ReelWriter const & reel = _reels[video_reel(frame)];
665 /* Make frame relative to the start of the reel */
666 frame -= reel.start ();
667 return (frame != 0 && frame < reel.first_nonexistant_frame());
670 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
672 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
674 vector<ReelWriter>::iterator* reel = 0;
677 case TEXT_OPEN_SUBTITLE:
678 reel = &_subtitle_reel;
680 case TEXT_CLOSED_CAPTION:
681 DCPOMATIC_ASSERT (track);
682 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
683 reel = &_caption_reels[*track];
686 DCPOMATIC_ASSERT (false);
689 DCPOMATIC_ASSERT (*reel != _reels.end());
690 while ((*reel)->period().to <= period.from) {
692 DCPOMATIC_ASSERT (*reel != _reels.end());
695 (*reel)->write (text, type, track, period);
699 Writer::write (list<shared_ptr<Font> > fonts)
701 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
703 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
705 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
712 _fonts.push_back (i);
718 operator< (QueueItem const & a, QueueItem const & b)
720 if (a.reel != b.reel) {
721 return a.reel < b.reel;
724 if (a.frame != b.frame) {
725 return a.frame < b.frame;
728 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
732 operator== (QueueItem const & a, QueueItem const & b)
734 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
738 Writer::set_encoder_threads (int threads)
740 boost::mutex::scoped_lock lm (_state_mutex);
741 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
742 _maximum_queue_size = threads * 16;
746 Writer::write (ReferencedReelAsset asset)
748 _reel_assets.push_back (asset);
752 Writer::video_reel (int frame) const
754 DCPTime t = DCPTime::from_frames (frame, _film->video_frame_rate ());
756 while (i < _reels.size() && !_reels[i].period().contains (t)) {
760 DCPOMATIC_ASSERT (i < _reels.size ());
765 Writer::set_digest_progress (Job* job, float progress)
767 /* I believe this is thread-safe */
768 _digest_progresses[boost::this_thread::get_id()] = progress;
770 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
771 float min_progress = FLT_MAX;
772 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
773 min_progress = min (min_progress, i->second);
776 job->set_progress (min_progress);