2 Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
23 #include "audio_buffers.h"
24 #include "content_audio.h"
25 #include "dcp_content.h"
28 #include "raw_image_proxy.h"
31 #include "render_subtitles.h"
33 #include "content_video.h"
34 #include "player_video.h"
35 #include "frame_rate_change.h"
36 #include "audio_processor.h"
38 #include "referenced_reel_asset.h"
39 #include "decoder_factory.h"
41 #include "video_decoder.h"
42 #include "audio_decoder.h"
43 #include "subtitle_content.h"
44 #include "subtitle_decoder.h"
45 #include "ffmpeg_content.h"
46 #include "audio_content.h"
47 #include "content_subtitle.h"
48 #include "dcp_decoder.h"
49 #include "image_decoder.h"
51 #include <dcp/reel_sound_asset.h>
52 #include <dcp/reel_subtitle_asset.h>
53 #include <dcp/reel_picture_asset.h>
54 #include <boost/foreach.hpp>
61 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
73 using boost::shared_ptr;
74 using boost::weak_ptr;
75 using boost::dynamic_pointer_cast;
76 using boost::optional;
77 using boost::scoped_ptr;
80 has_video (Content* c)
82 return static_cast<bool>(c->video);
86 has_audio (Content* c)
88 return static_cast<bool>(c->audio);
92 has_subtitle (Content* c)
94 return static_cast<bool>(c->subtitle);
97 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
99 , _playlist (playlist)
100 , _have_valid_pieces (false)
101 , _ignore_video (false)
102 , _ignore_audio (false)
103 , _always_burn_subtitles (false)
105 , _play_referenced (false)
107 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
108 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
109 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
110 set_video_container_size (_film->frame_size ());
112 film_changed (Film::AUDIO_PROCESSOR);
116 Player::setup_pieces ()
120 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
122 if (!i->paths_valid ()) {
126 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
127 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
130 /* Not something that we can decode; e.g. Atmos content */
134 if (decoder->video && _ignore_video) {
135 decoder->video->set_ignore ();
138 if (decoder->audio && _ignore_audio) {
139 decoder->audio->set_ignore ();
142 if (decoder->audio && _fast) {
143 decoder->audio->set_fast ();
146 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
147 if (dcp && _play_referenced) {
148 dcp->set_decode_referenced ();
151 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc)));
154 _have_valid_pieces = true;
158 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
160 shared_ptr<Content> c = w.lock ();
166 property == ContentProperty::POSITION ||
167 property == ContentProperty::LENGTH ||
168 property == ContentProperty::TRIM_START ||
169 property == ContentProperty::TRIM_END ||
170 property == ContentProperty::PATH ||
171 property == VideoContentProperty::FRAME_TYPE ||
172 property == DCPContentProperty::CAN_BE_PLAYED ||
173 property == SubtitleContentProperty::COLOUR ||
174 property == SubtitleContentProperty::OUTLINE ||
175 property == SubtitleContentProperty::SHADOW ||
176 property == SubtitleContentProperty::EFFECT_COLOUR ||
177 property == FFmpegContentProperty::SUBTITLE_STREAM ||
178 property == VideoContentProperty::COLOUR_CONVERSION
181 _have_valid_pieces = false;
185 property == SubtitleContentProperty::LINE_SPACING ||
186 property == SubtitleContentProperty::OUTLINE_WIDTH ||
187 property == SubtitleContentProperty::Y_SCALE ||
188 property == SubtitleContentProperty::FADE_IN ||
189 property == SubtitleContentProperty::FADE_OUT
192 /* These changes just need the pieces' decoders to be reset.
193 It's quite possible that other changes could be handled by
194 this branch rather than the _have_valid_pieces = false branch
195 above. This would make things a lot faster.
202 property == ContentProperty::VIDEO_FRAME_RATE ||
203 property == SubtitleContentProperty::USE ||
204 property == SubtitleContentProperty::X_OFFSET ||
205 property == SubtitleContentProperty::Y_OFFSET ||
206 property == SubtitleContentProperty::X_SCALE ||
207 property == SubtitleContentProperty::FONTS ||
208 property == VideoContentProperty::CROP ||
209 property == VideoContentProperty::SCALE ||
210 property == VideoContentProperty::FADE_IN ||
211 property == VideoContentProperty::FADE_OUT
219 Player::set_video_container_size (dcp::Size s)
221 _video_container_size = s;
223 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
224 _black_image->make_black ();
228 Player::playlist_changed ()
230 _have_valid_pieces = false;
235 Player::film_changed (Film::Property p)
237 /* Here we should notice Film properties that affect our output, and
238 alert listeners that our output now would be different to how it was
239 last time we were run.
242 if (p == Film::CONTAINER) {
244 } else if (p == Film::VIDEO_FRAME_RATE) {
245 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
246 so we need new pieces here.
248 _have_valid_pieces = false;
250 } else if (p == Film::AUDIO_PROCESSOR) {
251 if (_film->audio_processor ()) {
252 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
258 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
260 list<PositionImage> all;
262 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
267 /* We will scale the subtitle up to fit _video_container_size */
268 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
270 /* Then we need a corrective translation, consisting of two parts:
272 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
273 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
275 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
276 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
277 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
279 * Combining these two translations gives these expressions.
286 dcp::YUV_TO_RGB_REC601,
287 i->image->pixel_format (),
292 lrint (_video_container_size.width * i->rectangle.x),
293 lrint (_video_container_size.height * i->rectangle.y)
302 shared_ptr<PlayerVideo>
303 Player::black_player_video_frame (DCPTime time) const
305 return shared_ptr<PlayerVideo> (
307 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
311 _video_container_size,
312 _video_container_size,
315 PresetColourConversion::all().front().conversion
320 /** @return All PlayerVideos at the given time. There may be none if the content
321 * at `time' is a DCP which we are passing through (i.e. referring to by reference)
322 * or 2 if we have 3D.
324 list<shared_ptr<PlayerVideo> >
325 Player::get_video (DCPTime time, bool accurate)
327 if (!_have_valid_pieces) {
331 /* Find subtitles for possible burn-in */
333 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true, accurate);
335 list<PositionImage> sub_images;
337 /* Image subtitles */
338 list<PositionImage> c = transform_image_subtitles (ps.image);
339 copy (c.begin(), c.end(), back_inserter (sub_images));
341 /* Text subtitles (rendered to an image) */
342 if (!ps.text.empty ()) {
343 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size, time);
344 copy (s.begin (), s.end (), back_inserter (sub_images));
347 optional<PositionImage> subtitles;
348 if (!sub_images.empty ()) {
349 subtitles = merge (sub_images);
352 /* Find pieces containing video which is happening now */
354 list<shared_ptr<Piece> > ov = overlaps (
356 time + DCPTime::from_frames (1, _film->video_frame_rate ()),
360 list<shared_ptr<PlayerVideo> > pvf;
363 /* No video content at this time */
364 pvf.push_back (black_player_video_frame (time));
366 /* Some video content at this time */
367 shared_ptr<Piece> last = *(ov.rbegin ());
368 VideoFrameType const last_type = last->content->video->frame_type ();
370 /* Get video from appropriate piece(s) */
371 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
373 shared_ptr<VideoDecoder> decoder = piece->decoder->video;
374 DCPOMATIC_ASSERT (decoder);
376 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (piece->content);
377 if (dcp_content && dcp_content->reference_video () && !_play_referenced) {
382 /* always use the last video */
384 /* with a corresponding L/R eye if appropriate */
385 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
386 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
389 /* We want to use this piece */
390 list<ContentVideo> content_video = decoder->get (dcp_to_content_video (piece, time), accurate);
391 if (content_video.empty ()) {
392 pvf.push_back (black_player_video_frame (time));
394 dcp::Size image_size = piece->content->video->scale().size (
395 piece->content->video, _video_container_size, _film->frame_size ()
398 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
400 shared_ptr<PlayerVideo> (
404 piece->content->video->crop (),
405 piece->content->video->fade (i->frame.index()),
407 _video_container_size,
410 piece->content->video->colour_conversion ()
417 /* Discard unused video */
418 decoder->get (dcp_to_content_video (piece, time), accurate);
424 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
425 p->set_subtitle (subtitles.get ());
432 /** @return Audio data or 0 if the only audio data here is referenced DCP data */
433 shared_ptr<AudioBuffers>
434 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
436 if (!_have_valid_pieces) {
440 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
442 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
443 audio->make_silent ();
445 list<shared_ptr<Piece> > ov = overlaps (time, time + length, has_audio);
450 bool all_referenced = true;
451 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
452 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (i->content);
453 if (i->content->audio && (!dcp_content || !dcp_content->reference_audio ())) {
454 /* There is audio content which is not from a DCP or not set to be referenced */
455 all_referenced = false;
459 if (all_referenced && !_play_referenced) {
460 return shared_ptr<AudioBuffers> ();
463 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
465 DCPOMATIC_ASSERT (i->content->audio);
466 shared_ptr<AudioDecoder> decoder = i->decoder->audio;
467 DCPOMATIC_ASSERT (decoder);
469 /* The time that we should request from the content */
470 DCPTime request = time - DCPTime::from_seconds (i->content->audio->delay() / 1000.0);
471 Frame request_frames = length_frames;
473 if (request < DCPTime ()) {
474 /* We went off the start of the content, so we will need to offset
475 the stuff we get back.
478 request_frames += request.frames_round (_film->audio_frame_rate ());
479 if (request_frames < 0) {
482 request = DCPTime ();
485 Frame const content_frame = dcp_to_resampled_audio (i, request);
487 BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams ()) {
489 if (j->channels() == 0) {
490 /* Some content (e.g. DCPs) can have streams with no channels */
494 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
495 ContentAudio all = decoder->get (j, content_frame, request_frames, accurate);
498 if (i->content->audio->gain() != 0) {
499 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
500 gain->apply_gain (i->content->audio->gain ());
505 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
506 dcp_mapped->make_silent ();
507 AudioMapping map = j->mapping ();
508 for (int i = 0; i < map.input_channels(); ++i) {
509 for (int j = 0; j < _film->audio_channels(); ++j) {
510 if (map.get (i, j) > 0) {
511 dcp_mapped->accumulate_channel (
521 if (_audio_processor) {
522 dcp_mapped = _audio_processor->run (dcp_mapped, _film->audio_channels ());
525 all.audio = dcp_mapped;
527 audio->accumulate_frames (
529 content_frame - all.frame,
530 offset.frames_round (_film->audio_frame_rate()),
531 min (Frame (all.audio->frames()), request_frames)
540 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
542 DCPTime s = t - piece->content->position ();
543 s = min (piece->content->length_after_trim(), s);
544 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
546 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
547 then convert that ContentTime to frames at the content's rate. However this fails for
548 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
549 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
551 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
553 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
557 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
559 /* See comment in dcp_to_content_video */
560 DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
561 return max (DCPTime (), d + piece->content->position ());
565 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
567 DCPTime s = t - piece->content->position ();
568 s = min (piece->content->length_after_trim(), s);
569 /* See notes in dcp_to_content_video */
570 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
574 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
576 DCPTime s = t - piece->content->position ();
577 s = min (piece->content->length_after_trim(), s);
578 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
582 Player::content_subtitle_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
584 return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
587 /** @param burnt true to return only subtitles to be burnt, false to return only
588 * subtitles that should not be burnt. This parameter will be ignored if
589 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
592 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt, bool accurate)
594 list<shared_ptr<Piece> > subs = overlaps (time, time + length, has_subtitle);
596 PlayerSubtitles ps (time);
598 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
599 if (!(*j)->content->subtitle->use () || (!_always_burn_subtitles && (burnt != (*j)->content->subtitle->burn ()))) {
603 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> ((*j)->content);
604 if (dcp_content && dcp_content->reference_subtitle () && !_play_referenced) {
608 shared_ptr<SubtitleDecoder> subtitle_decoder = (*j)->decoder->subtitle;
609 ContentTime const from = dcp_to_content_subtitle (*j, time);
610 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
611 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
613 list<ContentImageSubtitle> image = subtitle_decoder->get_image (ContentTimePeriod (from, to), starting, accurate);
614 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
616 /* Apply content's subtitle offsets */
617 i->sub.rectangle.x += (*j)->content->subtitle->x_offset ();
618 i->sub.rectangle.y += (*j)->content->subtitle->y_offset ();
620 /* Apply content's subtitle scale */
621 i->sub.rectangle.width *= (*j)->content->subtitle->x_scale ();
622 i->sub.rectangle.height *= (*j)->content->subtitle->y_scale ();
624 /* Apply a corrective translation to keep the subtitle centred after that scale */
625 i->sub.rectangle.x -= i->sub.rectangle.width * ((*j)->content->subtitle->x_scale() - 1);
626 i->sub.rectangle.y -= i->sub.rectangle.height * ((*j)->content->subtitle->y_scale() - 1);
628 ps.image.push_back (i->sub);
631 list<ContentTextSubtitle> text = subtitle_decoder->get_text (ContentTimePeriod (from, to), starting, accurate);
632 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
633 BOOST_FOREACH (dcp::SubtitleString s, ts.subs) {
634 s.set_h_position (s.h_position() + (*j)->content->subtitle->x_offset ());
635 s.set_v_position (s.v_position() + (*j)->content->subtitle->y_offset ());
636 float const xs = (*j)->content->subtitle->x_scale();
637 float const ys = (*j)->content->subtitle->y_scale();
638 float size = s.size();
640 /* Adjust size to express the common part of the scaling;
641 e.g. if xs = ys = 0.5 we scale size by 2.
643 if (xs > 1e-5 && ys > 1e-5) {
644 size *= 1 / min (1 / xs, 1 / ys);
648 /* Then express aspect ratio changes */
649 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
650 s.set_aspect_adjust (xs / ys);
652 s.set_in (dcp::Time(content_subtitle_to_dcp (*j, ts.period().from).seconds(), 1000));
653 s.set_out (dcp::Time(content_subtitle_to_dcp (*j, ts.period().to).seconds(), 1000));
654 ps.text.push_back (SubtitleString (s, (*j)->content->subtitle->outline_width()));
655 ps.add_fonts ((*j)->content->subtitle->fonts ());
663 list<shared_ptr<Font> >
664 Player::get_subtitle_fonts ()
666 if (!_have_valid_pieces) {
670 list<shared_ptr<Font> > fonts;
671 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
672 if (p->content->subtitle) {
673 /* XXX: things may go wrong if there are duplicate font IDs
674 with different font files.
676 list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
677 copy (f.begin(), f.end(), back_inserter (fonts));
684 /** Set this player never to produce any video data */
686 Player::set_ignore_video ()
688 _ignore_video = true;
691 /** Set this player never to produce any audio data */
693 Player::set_ignore_audio ()
695 _ignore_audio = true;
698 /** Set whether or not this player should always burn text subtitles into the image,
699 * regardless of the content settings.
700 * @param burn true to always burn subtitles, false to obey content settings.
703 Player::set_always_burn_subtitles (bool burn)
705 _always_burn_subtitles = burn;
712 _have_valid_pieces = false;
716 Player::set_play_referenced ()
718 _play_referenced = true;
719 _have_valid_pieces = false;
722 list<ReferencedReelAsset>
723 Player::get_reel_assets ()
725 list<ReferencedReelAsset> a;
727 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
728 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
733 scoped_ptr<DCPDecoder> decoder;
735 decoder.reset (new DCPDecoder (j, _film->log()));
741 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
742 DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
743 if (j->reference_video ()) {
745 ReferencedReelAsset (
747 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_picture()->duration(), _film->video_frame_rate()))
752 if (j->reference_audio ()) {
754 ReferencedReelAsset (
756 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_sound()->duration(), _film->video_frame_rate()))
761 if (j->reference_subtitle ()) {
762 DCPOMATIC_ASSERT (k->main_subtitle ());
764 ReferencedReelAsset (
766 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_subtitle()->duration(), _film->video_frame_rate()))
771 /* Assume that main picture duration is the length of the reel */
772 offset += k->main_picture()->duration ();
779 list<shared_ptr<Piece> >
780 Player::overlaps (DCPTime from, DCPTime to, boost::function<bool (Content *)> valid)
782 if (!_have_valid_pieces) {
786 list<shared_ptr<Piece> > overlaps;
787 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
788 if (valid (i->content.get ()) && i->content->position() < to && i->content->end() > from) {
789 overlaps.push_back (i);
797 Player::reset_pieces ()
799 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
800 i->decoder->reset ();