2 Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
23 #include "audio_buffers.h"
24 #include "content_audio.h"
25 #include "dcp_content.h"
28 #include "raw_image_proxy.h"
31 #include "render_subtitles.h"
33 #include "content_video.h"
34 #include "player_video.h"
35 #include "frame_rate_change.h"
36 #include "audio_processor.h"
38 #include "referenced_reel_asset.h"
39 #include "decoder_factory.h"
41 #include "video_decoder.h"
42 #include "audio_decoder.h"
43 #include "subtitle_content.h"
44 #include "subtitle_decoder.h"
45 #include "ffmpeg_content.h"
46 #include "audio_content.h"
47 #include "content_subtitle.h"
48 #include "dcp_decoder.h"
49 #include "image_decoder.h"
51 #include <dcp/reel_sound_asset.h>
52 #include <dcp/reel_subtitle_asset.h>
53 #include <dcp/reel_picture_asset.h>
54 #include <boost/foreach.hpp>
61 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
73 using boost::shared_ptr;
74 using boost::weak_ptr;
75 using boost::dynamic_pointer_cast;
76 using boost::optional;
77 using boost::scoped_ptr;
80 has_video (Content* c)
82 return static_cast<bool>(c->video);
86 has_audio (Content* c)
88 return static_cast<bool>(c->audio);
92 has_subtitle (Content* c)
94 return static_cast<bool>(c->subtitle);
97 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
99 , _playlist (playlist)
100 , _have_valid_pieces (false)
101 , _ignore_video (false)
102 , _ignore_audio (false)
103 , _always_burn_subtitles (false)
105 , _play_referenced (false)
107 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
108 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
109 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
110 set_video_container_size (_film->frame_size ());
112 film_changed (Film::AUDIO_PROCESSOR);
116 Player::setup_pieces ()
120 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
122 if (!i->paths_valid ()) {
126 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
127 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
130 /* Not something that we can decode; e.g. Atmos content */
134 if (decoder->video && _ignore_video) {
135 decoder->video->set_ignore ();
138 if (decoder->audio && _ignore_audio) {
139 decoder->audio->set_ignore ();
142 if (decoder->audio && _fast) {
143 decoder->audio->set_fast ();
146 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
147 if (dcp && _play_referenced) {
148 dcp->set_decode_referenced ();
151 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc)));
154 _have_valid_pieces = true;
158 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
160 shared_ptr<Content> c = w.lock ();
166 property == ContentProperty::POSITION ||
167 property == ContentProperty::LENGTH ||
168 property == ContentProperty::TRIM_START ||
169 property == ContentProperty::TRIM_END ||
170 property == ContentProperty::PATH ||
171 property == VideoContentProperty::FRAME_TYPE ||
172 property == DCPContentProperty::NEEDS_ASSETS ||
173 property == DCPContentProperty::NEEDS_KDM ||
174 property == SubtitleContentProperty::COLOUR ||
175 property == SubtitleContentProperty::OUTLINE ||
176 property == SubtitleContentProperty::SHADOW ||
177 property == SubtitleContentProperty::EFFECT_COLOUR ||
178 property == FFmpegContentProperty::SUBTITLE_STREAM ||
179 property == VideoContentProperty::COLOUR_CONVERSION
182 _have_valid_pieces = false;
186 property == SubtitleContentProperty::LINE_SPACING ||
187 property == SubtitleContentProperty::OUTLINE_WIDTH ||
188 property == SubtitleContentProperty::Y_SCALE ||
189 property == SubtitleContentProperty::FADE_IN ||
190 property == SubtitleContentProperty::FADE_OUT
193 /* These changes just need the pieces' decoders to be reset.
194 It's quite possible that other changes could be handled by
195 this branch rather than the _have_valid_pieces = false branch
196 above. This would make things a lot faster.
203 property == ContentProperty::VIDEO_FRAME_RATE ||
204 property == SubtitleContentProperty::USE ||
205 property == SubtitleContentProperty::X_OFFSET ||
206 property == SubtitleContentProperty::Y_OFFSET ||
207 property == SubtitleContentProperty::X_SCALE ||
208 property == SubtitleContentProperty::FONTS ||
209 property == VideoContentProperty::CROP ||
210 property == VideoContentProperty::SCALE ||
211 property == VideoContentProperty::FADE_IN ||
212 property == VideoContentProperty::FADE_OUT
220 Player::set_video_container_size (dcp::Size s)
222 _video_container_size = s;
224 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
225 _black_image->make_black ();
229 Player::playlist_changed ()
231 _have_valid_pieces = false;
236 Player::film_changed (Film::Property p)
238 /* Here we should notice Film properties that affect our output, and
239 alert listeners that our output now would be different to how it was
240 last time we were run.
243 if (p == Film::CONTAINER) {
245 } else if (p == Film::VIDEO_FRAME_RATE) {
246 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
247 so we need new pieces here.
249 _have_valid_pieces = false;
251 } else if (p == Film::AUDIO_PROCESSOR) {
252 if (_film->audio_processor ()) {
253 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
259 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
261 list<PositionImage> all;
263 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
268 /* We will scale the subtitle up to fit _video_container_size */
269 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
271 /* Then we need a corrective translation, consisting of two parts:
273 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
274 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
276 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
277 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
278 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
280 * Combining these two translations gives these expressions.
287 dcp::YUV_TO_RGB_REC601,
288 i->image->pixel_format (),
293 lrint (_video_container_size.width * i->rectangle.x),
294 lrint (_video_container_size.height * i->rectangle.y)
303 shared_ptr<PlayerVideo>
304 Player::black_player_video_frame (DCPTime time) const
306 return shared_ptr<PlayerVideo> (
308 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
312 _video_container_size,
313 _video_container_size,
316 PresetColourConversion::all().front().conversion
321 /** @return All PlayerVideos at the given time. There may be none if the content
322 * at `time' is a DCP which we are passing through (i.e. referring to by reference)
323 * or 2 if we have 3D.
325 list<shared_ptr<PlayerVideo> >
326 Player::get_video (DCPTime time, bool accurate)
328 if (!_have_valid_pieces) {
332 /* Find subtitles for possible burn-in */
334 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true, accurate);
336 list<PositionImage> sub_images;
338 /* Image subtitles */
339 list<PositionImage> c = transform_image_subtitles (ps.image);
340 copy (c.begin(), c.end(), back_inserter (sub_images));
342 /* Text subtitles (rendered to an image) */
343 if (!ps.text.empty ()) {
344 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size, time);
345 copy (s.begin (), s.end (), back_inserter (sub_images));
348 optional<PositionImage> subtitles;
349 if (!sub_images.empty ()) {
350 subtitles = merge (sub_images);
353 /* Find pieces containing video which is happening now */
355 list<shared_ptr<Piece> > ov = overlaps (
357 time + DCPTime::from_frames (1, _film->video_frame_rate ()),
361 list<shared_ptr<PlayerVideo> > pvf;
364 /* No video content at this time */
365 pvf.push_back (black_player_video_frame (time));
367 /* Some video content at this time */
368 shared_ptr<Piece> last = *(ov.rbegin ());
369 VideoFrameType const last_type = last->content->video->frame_type ();
371 /* Get video from appropriate piece(s) */
372 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
374 shared_ptr<VideoDecoder> decoder = piece->decoder->video;
375 DCPOMATIC_ASSERT (decoder);
377 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (piece->content);
378 if (dcp_content && dcp_content->reference_video () && !_play_referenced) {
383 /* always use the last video */
385 /* with a corresponding L/R eye if appropriate */
386 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
387 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
390 /* We want to use this piece */
391 list<ContentVideo> content_video = decoder->get (dcp_to_content_video (piece, time), accurate);
392 if (content_video.empty ()) {
393 pvf.push_back (black_player_video_frame (time));
395 dcp::Size image_size = piece->content->video->scale().size (
396 piece->content->video, _video_container_size, _film->frame_size ()
399 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
401 shared_ptr<PlayerVideo> (
405 piece->content->video->crop (),
406 piece->content->video->fade (i->frame.index()),
408 _video_container_size,
411 piece->content->video->colour_conversion ()
418 /* Discard unused video */
419 decoder->get (dcp_to_content_video (piece, time), accurate);
425 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
426 p->set_subtitle (subtitles.get ());
433 /** @return Audio data or 0 if the only audio data here is referenced DCP data */
434 shared_ptr<AudioBuffers>
435 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
437 if (!_have_valid_pieces) {
441 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
443 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
444 audio->make_silent ();
446 list<shared_ptr<Piece> > ov = overlaps (time, time + length, has_audio);
451 bool all_referenced = true;
452 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
453 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (i->content);
454 if (i->content->audio && (!dcp_content || !dcp_content->reference_audio ())) {
455 /* There is audio content which is not from a DCP or not set to be referenced */
456 all_referenced = false;
460 if (all_referenced && !_play_referenced) {
461 return shared_ptr<AudioBuffers> ();
464 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
466 DCPOMATIC_ASSERT (i->content->audio);
467 shared_ptr<AudioDecoder> decoder = i->decoder->audio;
468 DCPOMATIC_ASSERT (decoder);
470 /* The time that we should request from the content */
471 DCPTime request = time - DCPTime::from_seconds (i->content->audio->delay() / 1000.0);
472 Frame request_frames = length_frames;
474 if (request < DCPTime ()) {
475 /* We went off the start of the content, so we will need to offset
476 the stuff we get back.
479 request_frames += request.frames_round (_film->audio_frame_rate ());
480 if (request_frames < 0) {
483 request = DCPTime ();
486 Frame const content_frame = dcp_to_resampled_audio (i, request);
488 BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams ()) {
490 if (j->channels() == 0) {
491 /* Some content (e.g. DCPs) can have streams with no channels */
495 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
496 ContentAudio all = decoder->get (j, content_frame, request_frames, accurate);
499 if (i->content->audio->gain() != 0) {
500 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
501 gain->apply_gain (i->content->audio->gain ());
506 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
507 dcp_mapped->make_silent ();
508 AudioMapping map = j->mapping ();
509 for (int i = 0; i < map.input_channels(); ++i) {
510 for (int j = 0; j < _film->audio_channels(); ++j) {
511 if (map.get (i, j) > 0) {
512 dcp_mapped->accumulate_channel (
522 if (_audio_processor) {
523 dcp_mapped = _audio_processor->run (dcp_mapped, _film->audio_channels ());
526 all.audio = dcp_mapped;
528 audio->accumulate_frames (
530 content_frame - all.frame,
531 offset.frames_round (_film->audio_frame_rate()),
532 min (Frame (all.audio->frames()), request_frames)
541 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
543 DCPTime s = t - piece->content->position ();
544 s = min (piece->content->length_after_trim(), s);
545 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
547 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
548 then convert that ContentTime to frames at the content's rate. However this fails for
549 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
550 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
552 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
554 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
558 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
560 /* See comment in dcp_to_content_video */
561 DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
562 return max (DCPTime (), d + piece->content->position ());
566 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
568 DCPTime s = t - piece->content->position ();
569 s = min (piece->content->length_after_trim(), s);
570 /* See notes in dcp_to_content_video */
571 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
575 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
577 DCPTime s = t - piece->content->position ();
578 s = min (piece->content->length_after_trim(), s);
579 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
583 Player::content_subtitle_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
585 return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
588 /** @param burnt true to return only subtitles to be burnt, false to return only
589 * subtitles that should not be burnt. This parameter will be ignored if
590 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
593 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt, bool accurate)
595 list<shared_ptr<Piece> > subs = overlaps (time, time + length, has_subtitle);
597 PlayerSubtitles ps (time);
599 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
600 if (!(*j)->content->subtitle->use () || (!_always_burn_subtitles && (burnt != (*j)->content->subtitle->burn ()))) {
604 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> ((*j)->content);
605 if (dcp_content && dcp_content->reference_subtitle () && !_play_referenced) {
609 shared_ptr<SubtitleDecoder> subtitle_decoder = (*j)->decoder->subtitle;
610 ContentTime const from = dcp_to_content_subtitle (*j, time);
611 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
612 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
614 list<ContentImageSubtitle> image = subtitle_decoder->get_image (ContentTimePeriod (from, to), starting, accurate);
615 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
617 /* Apply content's subtitle offsets */
618 i->sub.rectangle.x += (*j)->content->subtitle->x_offset ();
619 i->sub.rectangle.y += (*j)->content->subtitle->y_offset ();
621 /* Apply content's subtitle scale */
622 i->sub.rectangle.width *= (*j)->content->subtitle->x_scale ();
623 i->sub.rectangle.height *= (*j)->content->subtitle->y_scale ();
625 /* Apply a corrective translation to keep the subtitle centred after that scale */
626 i->sub.rectangle.x -= i->sub.rectangle.width * ((*j)->content->subtitle->x_scale() - 1);
627 i->sub.rectangle.y -= i->sub.rectangle.height * ((*j)->content->subtitle->y_scale() - 1);
629 ps.image.push_back (i->sub);
632 list<ContentTextSubtitle> text = subtitle_decoder->get_text (ContentTimePeriod (from, to), starting, accurate);
633 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
634 BOOST_FOREACH (dcp::SubtitleString s, ts.subs) {
635 s.set_h_position (s.h_position() + (*j)->content->subtitle->x_offset ());
636 s.set_v_position (s.v_position() + (*j)->content->subtitle->y_offset ());
637 float const xs = (*j)->content->subtitle->x_scale();
638 float const ys = (*j)->content->subtitle->y_scale();
639 float size = s.size();
641 /* Adjust size to express the common part of the scaling;
642 e.g. if xs = ys = 0.5 we scale size by 2.
644 if (xs > 1e-5 && ys > 1e-5) {
645 size *= 1 / min (1 / xs, 1 / ys);
649 /* Then express aspect ratio changes */
650 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
651 s.set_aspect_adjust (xs / ys);
653 s.set_in (dcp::Time(content_subtitle_to_dcp (*j, ts.period().from).seconds(), 1000));
654 s.set_out (dcp::Time(content_subtitle_to_dcp (*j, ts.period().to).seconds(), 1000));
655 ps.text.push_back (SubtitleString (s, (*j)->content->subtitle->outline_width()));
656 ps.add_fonts ((*j)->content->subtitle->fonts ());
664 list<shared_ptr<Font> >
665 Player::get_subtitle_fonts ()
667 if (!_have_valid_pieces) {
671 list<shared_ptr<Font> > fonts;
672 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
673 if (p->content->subtitle) {
674 /* XXX: things may go wrong if there are duplicate font IDs
675 with different font files.
677 list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
678 copy (f.begin(), f.end(), back_inserter (fonts));
685 /** Set this player never to produce any video data */
687 Player::set_ignore_video ()
689 _ignore_video = true;
692 /** Set this player never to produce any audio data */
694 Player::set_ignore_audio ()
696 _ignore_audio = true;
699 /** Set whether or not this player should always burn text subtitles into the image,
700 * regardless of the content settings.
701 * @param burn true to always burn subtitles, false to obey content settings.
704 Player::set_always_burn_subtitles (bool burn)
706 _always_burn_subtitles = burn;
713 _have_valid_pieces = false;
717 Player::set_play_referenced ()
719 _play_referenced = true;
720 _have_valid_pieces = false;
723 list<ReferencedReelAsset>
724 Player::get_reel_assets ()
726 list<ReferencedReelAsset> a;
728 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
729 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
734 scoped_ptr<DCPDecoder> decoder;
736 decoder.reset (new DCPDecoder (j, _film->log()));
742 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
744 DCPOMATIC_ASSERT (j->video_frame_rate ());
745 double const cfr = j->video_frame_rate().get();
746 Frame const trim_start = j->trim_start().frames_round (cfr);
747 Frame const trim_end = j->trim_end().frames_round (cfr);
749 DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
750 if (j->reference_video ()) {
751 DCPOMATIC_ASSERT (k->main_picture ());
752 k->main_picture()->set_entry_point (trim_start);
753 k->main_picture()->set_duration (k->main_picture()->intrinsic_duration() - trim_start - trim_end);
755 ReferencedReelAsset (
757 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_picture()->duration(), _film->video_frame_rate()))
762 if (j->reference_audio ()) {
763 DCPOMATIC_ASSERT (k->main_sound ());
764 k->main_sound()->set_entry_point (trim_start);
765 k->main_sound()->set_duration (k->main_sound()->intrinsic_duration() - trim_start - trim_end);
767 ReferencedReelAsset (
769 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_sound()->duration(), _film->video_frame_rate()))
774 if (j->reference_subtitle ()) {
775 DCPOMATIC_ASSERT (k->main_subtitle ());
776 k->main_subtitle()->set_entry_point (trim_start);
777 k->main_subtitle()->set_duration (k->main_subtitle()->intrinsic_duration() - trim_start - trim_end);
779 ReferencedReelAsset (
781 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_subtitle()->duration(), _film->video_frame_rate()))
786 /* Assume that main picture duration is the length of the reel */
787 offset += k->main_picture()->duration ();
794 list<shared_ptr<Piece> >
795 Player::overlaps (DCPTime from, DCPTime to, boost::function<bool (Content *)> valid)
797 if (!_have_valid_pieces) {
801 list<shared_ptr<Piece> > overlaps;
802 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
803 if (valid (i->content.get ()) && i->content->position() < to && i->content->end() > from) {
804 overlaps.push_back (i);
812 Player::reset_pieces ()
814 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
815 i->decoder->reset ();