2 Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
23 #include "audio_buffers.h"
24 #include "content_audio.h"
25 #include "dcp_content.h"
28 #include "raw_image_proxy.h"
31 #include "render_subtitles.h"
33 #include "content_video.h"
34 #include "player_video.h"
35 #include "frame_rate_change.h"
36 #include "audio_processor.h"
38 #include "referenced_reel_asset.h"
39 #include "decoder_factory.h"
41 #include "video_decoder.h"
42 #include "audio_decoder.h"
43 #include "subtitle_content.h"
44 #include "subtitle_decoder.h"
45 #include "ffmpeg_content.h"
46 #include "audio_content.h"
47 #include "content_subtitle.h"
48 #include "dcp_decoder.h"
49 #include "image_decoder.h"
51 #include <dcp/reel_sound_asset.h>
52 #include <dcp/reel_subtitle_asset.h>
53 #include <dcp/reel_picture_asset.h>
54 #include <boost/foreach.hpp>
61 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
73 using boost::shared_ptr;
74 using boost::weak_ptr;
75 using boost::dynamic_pointer_cast;
76 using boost::optional;
77 using boost::scoped_ptr;
80 has_video (Content* c)
82 return static_cast<bool>(c->video);
86 has_audio (Content* c)
88 return static_cast<bool>(c->audio);
92 has_subtitle (Content* c)
94 return static_cast<bool>(c->subtitle);
97 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
99 , _playlist (playlist)
100 , _have_valid_pieces (false)
101 , _ignore_video (false)
102 , _ignore_audio (false)
103 , _always_burn_subtitles (false)
105 , _play_referenced (false)
107 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
108 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
109 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
110 set_video_container_size (_film->frame_size ());
112 film_changed (Film::AUDIO_PROCESSOR);
116 Player::setup_pieces ()
120 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
122 if (!i->paths_valid ()) {
126 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
127 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
130 /* Not something that we can decode; e.g. Atmos content */
134 if (decoder->video && _ignore_video) {
135 decoder->video->set_ignore ();
138 if (decoder->audio && _ignore_audio) {
139 decoder->audio->set_ignore ();
142 if (decoder->audio && _fast) {
143 decoder->audio->set_fast ();
146 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
147 if (dcp && _play_referenced) {
148 dcp->set_decode_referenced ();
151 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc)));
154 _have_valid_pieces = true;
158 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
160 shared_ptr<Content> c = w.lock ();
166 property == ContentProperty::POSITION ||
167 property == ContentProperty::LENGTH ||
168 property == ContentProperty::TRIM_START ||
169 property == ContentProperty::TRIM_END ||
170 property == ContentProperty::PATH ||
171 property == VideoContentProperty::FRAME_TYPE ||
172 property == DCPContentProperty::CAN_BE_PLAYED ||
173 property == SubtitleContentProperty::COLOUR ||
174 property == SubtitleContentProperty::OUTLINE ||
175 property == SubtitleContentProperty::SHADOW ||
176 property == SubtitleContentProperty::EFFECT_COLOUR ||
177 property == FFmpegContentProperty::SUBTITLE_STREAM ||
178 property == VideoContentProperty::COLOUR_CONVERSION
181 _have_valid_pieces = false;
185 property == SubtitleContentProperty::LINE_SPACING ||
186 property == SubtitleContentProperty::Y_SCALE
189 /* These changes just need the pieces' decoders to be reset.
190 It's quite possible that other changes could be handled by
191 this branch rather than the _have_valid_pieces = false branch
192 above. This would make things a lot faster.
199 property == ContentProperty::VIDEO_FRAME_RATE ||
200 property == SubtitleContentProperty::USE ||
201 property == SubtitleContentProperty::X_OFFSET ||
202 property == SubtitleContentProperty::Y_OFFSET ||
203 property == SubtitleContentProperty::X_SCALE ||
204 property == SubtitleContentProperty::FONTS ||
205 property == VideoContentProperty::CROP ||
206 property == VideoContentProperty::SCALE ||
207 property == VideoContentProperty::FADE_IN ||
208 property == VideoContentProperty::FADE_OUT
216 Player::set_video_container_size (dcp::Size s)
218 _video_container_size = s;
220 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
221 _black_image->make_black ();
225 Player::playlist_changed ()
227 _have_valid_pieces = false;
232 Player::film_changed (Film::Property p)
234 /* Here we should notice Film properties that affect our output, and
235 alert listeners that our output now would be different to how it was
236 last time we were run.
239 if (p == Film::CONTAINER) {
241 } else if (p == Film::VIDEO_FRAME_RATE) {
242 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
243 so we need new pieces here.
245 _have_valid_pieces = false;
247 } else if (p == Film::AUDIO_PROCESSOR) {
248 if (_film->audio_processor ()) {
249 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
255 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
257 list<PositionImage> all;
259 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
264 /* We will scale the subtitle up to fit _video_container_size */
265 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
267 /* Then we need a corrective translation, consisting of two parts:
269 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
270 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
272 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
273 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
274 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
276 * Combining these two translations gives these expressions.
283 dcp::YUV_TO_RGB_REC601,
284 i->image->pixel_format (),
289 lrint (_video_container_size.width * i->rectangle.x),
290 lrint (_video_container_size.height * i->rectangle.y)
299 shared_ptr<PlayerVideo>
300 Player::black_player_video_frame (DCPTime time) const
302 return shared_ptr<PlayerVideo> (
304 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
308 _video_container_size,
309 _video_container_size,
312 PresetColourConversion::all().front().conversion
317 /** @return All PlayerVideos at the given time. There may be none if the content
318 * at `time' is a DCP which we are passing through (i.e. referring to by reference)
319 * or 2 if we have 3D.
321 list<shared_ptr<PlayerVideo> >
322 Player::get_video (DCPTime time, bool accurate)
324 if (!_have_valid_pieces) {
328 /* Find subtitles for possible burn-in */
330 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true, accurate);
332 list<PositionImage> sub_images;
334 /* Image subtitles */
335 list<PositionImage> c = transform_image_subtitles (ps.image);
336 copy (c.begin(), c.end(), back_inserter (sub_images));
338 /* Text subtitles (rendered to an image) */
339 if (!ps.text.empty ()) {
340 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size);
341 copy (s.begin (), s.end (), back_inserter (sub_images));
344 optional<PositionImage> subtitles;
345 if (!sub_images.empty ()) {
346 subtitles = merge (sub_images);
349 /* Find pieces containing video which is happening now */
351 list<shared_ptr<Piece> > ov = overlaps (
353 time + DCPTime::from_frames (1, _film->video_frame_rate ()),
357 list<shared_ptr<PlayerVideo> > pvf;
360 /* No video content at this time */
361 pvf.push_back (black_player_video_frame (time));
363 /* Some video content at this time */
364 shared_ptr<Piece> last = *(ov.rbegin ());
365 VideoFrameType const last_type = last->content->video->frame_type ();
367 /* Get video from appropriate piece(s) */
368 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
370 shared_ptr<VideoDecoder> decoder = piece->decoder->video;
371 DCPOMATIC_ASSERT (decoder);
373 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (piece->content);
374 if (dcp_content && dcp_content->reference_video () && !_play_referenced) {
379 /* always use the last video */
381 /* with a corresponding L/R eye if appropriate */
382 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
383 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
386 /* We want to use this piece */
387 list<ContentVideo> content_video = decoder->get (dcp_to_content_video (piece, time), accurate);
388 if (content_video.empty ()) {
389 pvf.push_back (black_player_video_frame (time));
391 dcp::Size image_size = piece->content->video->scale().size (
392 piece->content->video, _video_container_size, _film->frame_size ()
395 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
397 shared_ptr<PlayerVideo> (
401 piece->content->video->crop (),
402 piece->content->video->fade (i->frame.index()),
404 _video_container_size,
407 piece->content->video->colour_conversion ()
414 /* Discard unused video */
415 decoder->get (dcp_to_content_video (piece, time), accurate);
421 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
422 p->set_subtitle (subtitles.get ());
429 /** @return Audio data or 0 if the only audio data here is referenced DCP data */
430 shared_ptr<AudioBuffers>
431 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
433 if (!_have_valid_pieces) {
437 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
439 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
440 audio->make_silent ();
442 list<shared_ptr<Piece> > ov = overlaps (time, time + length, has_audio);
447 bool all_referenced = true;
448 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
449 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (i->content);
450 if (i->content->audio && (!dcp_content || !dcp_content->reference_audio ())) {
451 /* There is audio content which is not from a DCP or not set to be referenced */
452 all_referenced = false;
456 if (all_referenced && !_play_referenced) {
457 return shared_ptr<AudioBuffers> ();
460 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
462 DCPOMATIC_ASSERT (i->content->audio);
463 shared_ptr<AudioDecoder> decoder = i->decoder->audio;
464 DCPOMATIC_ASSERT (decoder);
466 /* The time that we should request from the content */
467 DCPTime request = time - DCPTime::from_seconds (i->content->audio->delay() / 1000.0);
468 Frame request_frames = length_frames;
470 if (request < DCPTime ()) {
471 /* We went off the start of the content, so we will need to offset
472 the stuff we get back.
475 request_frames += request.frames_round (_film->audio_frame_rate ());
476 if (request_frames < 0) {
479 request = DCPTime ();
482 Frame const content_frame = dcp_to_resampled_audio (i, request);
484 BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams ()) {
486 if (j->channels() == 0) {
487 /* Some content (e.g. DCPs) can have streams with no channels */
491 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
492 ContentAudio all = decoder->get (j, content_frame, request_frames, accurate);
495 if (i->content->audio->gain() != 0) {
496 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
497 gain->apply_gain (i->content->audio->gain ());
502 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
503 dcp_mapped->make_silent ();
504 AudioMapping map = j->mapping ();
505 for (int i = 0; i < map.input_channels(); ++i) {
506 for (int j = 0; j < _film->audio_channels(); ++j) {
507 if (map.get (i, j) > 0) {
508 dcp_mapped->accumulate_channel (
518 if (_audio_processor) {
519 dcp_mapped = _audio_processor->run (dcp_mapped, _film->audio_channels ());
522 all.audio = dcp_mapped;
524 audio->accumulate_frames (
526 content_frame - all.frame,
527 offset.frames_round (_film->audio_frame_rate()),
528 min (Frame (all.audio->frames()), request_frames)
537 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
539 DCPTime s = t - piece->content->position ();
540 s = min (piece->content->length_after_trim(), s);
541 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
543 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
544 then convert that ContentTime to frames at the content's rate. However this fails for
545 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
546 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
548 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
550 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
554 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
556 /* See comment in dcp_to_content_video */
557 DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
558 return max (DCPTime (), d + piece->content->position ());
562 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
564 DCPTime s = t - piece->content->position ();
565 s = min (piece->content->length_after_trim(), s);
566 /* See notes in dcp_to_content_video */
567 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
571 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
573 DCPTime s = t - piece->content->position ();
574 s = min (piece->content->length_after_trim(), s);
575 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
579 Player::content_subtitle_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
581 return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
584 /** @param burnt true to return only subtitles to be burnt, false to return only
585 * subtitles that should not be burnt. This parameter will be ignored if
586 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
589 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt, bool accurate)
591 list<shared_ptr<Piece> > subs = overlaps (time, time + length, has_subtitle);
593 PlayerSubtitles ps (time);
595 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
596 if (!(*j)->content->subtitle->use () || (!_always_burn_subtitles && (burnt != (*j)->content->subtitle->burn ()))) {
600 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> ((*j)->content);
601 if (dcp_content && dcp_content->reference_subtitle () && !_play_referenced) {
605 shared_ptr<SubtitleDecoder> subtitle_decoder = (*j)->decoder->subtitle;
606 ContentTime const from = dcp_to_content_subtitle (*j, time);
607 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
608 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
610 list<ContentImageSubtitle> image = subtitle_decoder->get_image (ContentTimePeriod (from, to), starting, accurate);
611 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
613 /* Apply content's subtitle offsets */
614 i->sub.rectangle.x += (*j)->content->subtitle->x_offset ();
615 i->sub.rectangle.y += (*j)->content->subtitle->y_offset ();
617 /* Apply content's subtitle scale */
618 i->sub.rectangle.width *= (*j)->content->subtitle->x_scale ();
619 i->sub.rectangle.height *= (*j)->content->subtitle->y_scale ();
621 /* Apply a corrective translation to keep the subtitle centred after that scale */
622 i->sub.rectangle.x -= i->sub.rectangle.width * ((*j)->content->subtitle->x_scale() - 1);
623 i->sub.rectangle.y -= i->sub.rectangle.height * ((*j)->content->subtitle->y_scale() - 1);
625 ps.image.push_back (i->sub);
628 list<ContentTextSubtitle> text = subtitle_decoder->get_text (ContentTimePeriod (from, to), starting, accurate);
629 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
630 BOOST_FOREACH (dcp::SubtitleString s, ts.subs) {
631 s.set_h_position (s.h_position() + (*j)->content->subtitle->x_offset ());
632 s.set_v_position (s.v_position() + (*j)->content->subtitle->y_offset ());
633 float const xs = (*j)->content->subtitle->x_scale();
634 float const ys = (*j)->content->subtitle->y_scale();
635 float size = s.size();
637 /* Adjust size to express the common part of the scaling;
638 e.g. if xs = ys = 0.5 we scale size by 2.
640 if (xs > 1e-5 && ys > 1e-5) {
641 size *= 1 / min (1 / xs, 1 / ys);
645 /* Then express aspect ratio changes */
646 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
647 s.set_aspect_adjust (xs / ys);
649 s.set_in (dcp::Time(content_subtitle_to_dcp (*j, ts.period().from).seconds(), 1000));
650 s.set_out (dcp::Time(content_subtitle_to_dcp (*j, ts.period().to).seconds(), 1000));
651 ps.text.push_back (s);
652 ps.add_fonts ((*j)->content->subtitle->fonts ());
660 list<shared_ptr<Font> >
661 Player::get_subtitle_fonts ()
663 if (!_have_valid_pieces) {
667 list<shared_ptr<Font> > fonts;
668 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
669 if (p->content->subtitle) {
670 /* XXX: things may go wrong if there are duplicate font IDs
671 with different font files.
673 list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
674 copy (f.begin(), f.end(), back_inserter (fonts));
681 /** Set this player never to produce any video data */
683 Player::set_ignore_video ()
685 _ignore_video = true;
688 /** Set this player never to produce any audio data */
690 Player::set_ignore_audio ()
692 _ignore_audio = true;
695 /** Set whether or not this player should always burn text subtitles into the image,
696 * regardless of the content settings.
697 * @param burn true to always burn subtitles, false to obey content settings.
700 Player::set_always_burn_subtitles (bool burn)
702 _always_burn_subtitles = burn;
709 _have_valid_pieces = false;
713 Player::set_play_referenced ()
715 _play_referenced = true;
716 _have_valid_pieces = false;
719 list<ReferencedReelAsset>
720 Player::get_reel_assets ()
722 list<ReferencedReelAsset> a;
724 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
725 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
730 scoped_ptr<DCPDecoder> decoder;
732 decoder.reset (new DCPDecoder (j, _film->log()));
738 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
739 DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
740 if (j->reference_video ()) {
742 ReferencedReelAsset (
744 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_picture()->duration(), _film->video_frame_rate()))
749 if (j->reference_audio ()) {
751 ReferencedReelAsset (
753 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_sound()->duration(), _film->video_frame_rate()))
758 if (j->reference_subtitle ()) {
759 DCPOMATIC_ASSERT (k->main_subtitle ());
761 ReferencedReelAsset (
763 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_subtitle()->duration(), _film->video_frame_rate()))
768 /* Assume that main picture duration is the length of the reel */
769 offset += k->main_picture()->duration ();
776 list<shared_ptr<Piece> >
777 Player::overlaps (DCPTime from, DCPTime to, boost::function<bool (Content *)> valid)
779 if (!_have_valid_pieces) {
783 list<shared_ptr<Piece> > overlaps;
784 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
785 if (valid (i->content.get ()) && i->content->position() < to && i->content->end() > from) {
786 overlaps.push_back (i);
794 Player::reset_pieces ()
796 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
797 i->decoder->reset ();