2 Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
35 #include "raw_image_proxy.h"
38 #include "render_subtitles.h"
40 #include "content_video.h"
41 #include "player_video.h"
42 #include "frame_rate_change.h"
43 #include "dcp_content.h"
44 #include "dcp_decoder.h"
45 #include "dcp_subtitle_content.h"
46 #include "dcp_subtitle_decoder.h"
47 #include "audio_processor.h"
49 #include <boost/foreach.hpp>
56 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
68 using boost::shared_ptr;
69 using boost::weak_ptr;
70 using boost::dynamic_pointer_cast;
71 using boost::optional;
73 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
75 , _playlist (playlist)
76 , _have_valid_pieces (false)
77 , _ignore_video (false)
78 , _ignore_audio (false)
79 , _always_burn_subtitles (false)
82 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
83 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
84 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
85 set_video_container_size (_film->frame_size ());
87 film_changed (Film::AUDIO_PROCESSOR);
91 Player::setup_pieces ()
93 list<shared_ptr<Piece> > old_pieces = _pieces;
96 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
98 if (!i->paths_valid ()) {
102 shared_ptr<Decoder> decoder;
103 optional<FrameRateChange> frc;
105 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
106 DCPTime best_overlap_t;
107 shared_ptr<VideoContent> best_overlap;
108 BOOST_FOREACH (shared_ptr<Content> j, _playlist->content ()) {
109 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (j);
114 DCPTime const overlap = max (vc->position(), i->position()) - min (vc->end(), i->end());
115 if (overlap > best_overlap_t) {
117 best_overlap_t = overlap;
121 optional<FrameRateChange> best_overlap_frc;
123 best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
125 /* No video overlap; e.g. if the DCP is just audio */
126 best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
130 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (i);
132 decoder.reset (new FFmpegDecoder (fc, _film->log(), _fast));
133 frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
136 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (i);
138 decoder.reset (new DCPDecoder (dc, _fast));
139 frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
143 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (i);
145 /* See if we can re-use an old ImageDecoder */
146 for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
147 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
148 if (imd && imd->content() == ic) {
154 decoder.reset (new ImageDecoder (ic));
157 frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
161 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (i);
163 decoder.reset (new SndfileDecoder (sc, _fast));
164 frc = best_overlap_frc;
168 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (i);
170 decoder.reset (new SubRipDecoder (rc));
171 frc = best_overlap_frc;
174 /* DCPSubtitleContent */
175 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (i);
177 decoder.reset (new DCPSubtitleDecoder (dsc));
178 frc = best_overlap_frc;
181 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
182 if (vd && _ignore_video) {
183 vd->set_ignore_video ();
186 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> (decoder);
187 if (ad && _ignore_audio) {
188 ad->set_ignore_audio ();
191 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc.get ())));
194 _have_valid_pieces = true;
198 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
200 shared_ptr<Content> c = w.lock ();
206 property == ContentProperty::POSITION ||
207 property == ContentProperty::LENGTH ||
208 property == ContentProperty::TRIM_START ||
209 property == ContentProperty::TRIM_END ||
210 property == ContentProperty::PATH ||
211 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
212 property == DCPContentProperty::CAN_BE_PLAYED
215 _have_valid_pieces = false;
219 property == SubtitleContentProperty::USE_SUBTITLES ||
220 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
221 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
222 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
223 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
224 property == SubtitleContentProperty::FONTS ||
225 property == VideoContentProperty::VIDEO_CROP ||
226 property == VideoContentProperty::VIDEO_SCALE ||
227 property == VideoContentProperty::VIDEO_FRAME_RATE ||
228 property == VideoContentProperty::VIDEO_FADE_IN ||
229 property == VideoContentProperty::VIDEO_FADE_OUT ||
230 property == VideoContentProperty::COLOUR_CONVERSION
238 Player::set_video_container_size (dcp::Size s)
240 _video_container_size = s;
242 _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
243 _black_image->make_black ();
247 Player::playlist_changed ()
249 _have_valid_pieces = false;
254 Player::film_changed (Film::Property p)
256 /* Here we should notice Film properties that affect our output, and
257 alert listeners that our output now would be different to how it was
258 last time we were run.
261 if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
263 } else if (p == Film::AUDIO_PROCESSOR) {
264 if (_film->audio_processor ()) {
265 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
271 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
273 list<PositionImage> all;
275 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
280 /* We will scale the subtitle up to fit _video_container_size */
281 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
283 /* Then we need a corrective translation, consisting of two parts:
285 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
286 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
288 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
289 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
290 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
292 * Combining these two translations gives these expressions.
299 dcp::YUV_TO_RGB_REC601,
300 i->image->pixel_format (),
304 lrint (_video_container_size.width * i->rectangle.x),
305 lrint (_video_container_size.height * i->rectangle.y)
314 shared_ptr<PlayerVideo>
315 Player::black_player_video_frame (DCPTime time) const
317 return shared_ptr<PlayerVideo> (
319 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
323 _video_container_size,
324 _video_container_size,
327 PresetColourConversion::all().front().conversion
332 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
333 list<shared_ptr<PlayerVideo> >
334 Player::get_video (DCPTime time, bool accurate)
336 if (!_have_valid_pieces) {
340 /* Find subtitles for possible burn-in */
342 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true);
344 list<PositionImage> sub_images;
346 /* Image subtitles */
347 list<PositionImage> c = transform_image_subtitles (ps.image);
348 copy (c.begin(), c.end(), back_inserter (sub_images));
350 /* Text subtitles (rendered to an image) */
351 if (!ps.text.empty ()) {
352 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size);
353 copy (s.begin (), s.end (), back_inserter (sub_images));
356 optional<PositionImage> subtitles;
357 if (!sub_images.empty ()) {
358 subtitles = merge (sub_images);
361 /* Find pieces containing video which is happening now */
363 list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
365 time + DCPTime::from_frames (1, _film->video_frame_rate ()) - DCPTime::delta()
368 list<shared_ptr<PlayerVideo> > pvf;
371 /* No video content at this time */
372 pvf.push_back (black_player_video_frame (time));
374 /* Some video content at this time */
375 shared_ptr<Piece> last = *(ov.rbegin ());
376 VideoFrameType const last_type = dynamic_pointer_cast<VideoContent> (last->content)->video_frame_type ();
378 /* Get video from appropriate piece(s) */
379 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
381 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
382 DCPOMATIC_ASSERT (decoder);
383 shared_ptr<VideoContent> video_content = dynamic_pointer_cast<VideoContent> (piece->content);
384 DCPOMATIC_ASSERT (video_content);
387 /* always use the last video */
389 /* with a corresponding L/R eye if appropriate */
390 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
391 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
394 /* We want to use this piece */
395 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
396 if (content_video.empty ()) {
397 pvf.push_back (black_player_video_frame (time));
399 dcp::Size image_size = video_content->scale().size (video_content, _video_container_size, _film->frame_size ());
401 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
403 shared_ptr<PlayerVideo> (
406 content_video_to_dcp (piece, i->frame),
407 video_content->crop (),
408 video_content->fade (i->frame),
410 _video_container_size,
413 video_content->colour_conversion ()
420 /* Discard unused video */
421 decoder->get_video (dcp_to_content_video (piece, time), accurate);
427 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
428 p->set_subtitle (subtitles.get ());
435 shared_ptr<AudioBuffers>
436 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
438 if (!_have_valid_pieces) {
442 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
444 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
445 audio->make_silent ();
447 list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
452 for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
454 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
455 DCPOMATIC_ASSERT (content);
456 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
457 DCPOMATIC_ASSERT (decoder);
459 /* The time that we should request from the content */
460 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
461 Frame request_frames = length_frames;
463 if (request < DCPTime ()) {
464 /* We went off the start of the content, so we will need to offset
465 the stuff we get back.
468 request_frames += request.frames_round (_film->audio_frame_rate ());
469 if (request_frames < 0) {
472 request = DCPTime ();
475 Frame const content_frame = dcp_to_resampled_audio (*i, request);
477 BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
479 if (j->channels() == 0) {
480 /* Some content (e.g. DCPs) can have streams with no channels */
484 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
485 ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
488 if (content->audio_gain() != 0) {
489 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
490 gain->apply_gain (content->audio_gain ());
495 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
496 dcp_mapped->make_silent ();
497 AudioMapping map = j->mapping ();
498 for (int i = 0; i < map.input_channels(); ++i) {
499 for (int j = 0; j < _film->audio_channels(); ++j) {
500 if (map.get (i, j) > 0) {
501 dcp_mapped->accumulate_channel (
511 if (_audio_processor) {
512 dcp_mapped = _audio_processor->run (dcp_mapped, _film->audio_channels ());
515 all.audio = dcp_mapped;
517 audio->accumulate_frames (
519 content_frame - all.frame,
520 offset.frames_round (_film->audio_frame_rate()),
521 min (Frame (all.audio->frames()), request_frames)
530 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
532 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
533 DCPTime s = t - piece->content->position ();
534 s = min (piece->content->length_after_trim(), s);
535 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
537 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
538 then convert that ContentTime to frames at the content's rate. However this fails for
539 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
540 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
542 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
544 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
548 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
550 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
551 /* See comment in dcp_to_content_video */
552 DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
553 return max (DCPTime (), d + piece->content->position ());
557 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
559 DCPTime s = t - piece->content->position ();
560 s = min (piece->content->length_after_trim(), s);
561 /* See notes in dcp_to_content_video */
562 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
566 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
568 DCPTime s = t - piece->content->position ();
569 s = min (piece->content->length_after_trim(), s);
570 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
573 /** @param burnt true to return only subtitles to be burnt, false to return only
574 * subtitles that should not be burnt. This parameter will be ignored if
575 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
578 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt)
580 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
582 PlayerSubtitles ps (time, length);
584 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
585 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
586 if (!subtitle_content->use_subtitles () || (!_always_burn_subtitles && (burnt != subtitle_content->burn_subtitles ()))) {
590 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
591 ContentTime const from = dcp_to_content_subtitle (*j, time);
592 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
593 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
595 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
596 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
598 /* Apply content's subtitle offsets */
599 i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
600 i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
602 /* Apply content's subtitle scale */
603 i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
604 i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
606 /* Apply a corrective translation to keep the subtitle centred after that scale */
607 i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
608 i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
610 ps.image.push_back (i->sub);
613 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
614 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
615 BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
616 s.set_h_position (s.h_position() + subtitle_content->subtitle_x_offset ());
617 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
618 float const xs = subtitle_content->subtitle_x_scale();
619 float const ys = subtitle_content->subtitle_y_scale();
620 float const average = s.size() * (xs + ys) / 2;
621 s.set_size (average);
622 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
623 s.set_aspect_adjust (xs / ys);
625 ps.text.push_back (s);
626 ps.add_fonts (subtitle_content->fonts ());
634 list<shared_ptr<Font> >
635 Player::get_subtitle_fonts ()
637 if (!_have_valid_pieces) {
641 list<shared_ptr<Font> > fonts;
642 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
643 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
645 /* XXX: things may go wrong if there are duplicate font IDs
646 with different font files.
648 list<shared_ptr<Font> > f = sc->fonts ();
649 copy (f.begin(), f.end(), back_inserter (fonts));
656 /** Set this player never to produce any video data */
658 Player::set_ignore_video ()
660 _ignore_video = true;
663 /** Set this player never to produce any audio data */
665 Player::set_ignore_audio ()
667 _ignore_audio = true;
670 /** Set whether or not this player should always burn text subtitles into the image,
671 * regardless of the content settings.
672 * @param burn true to always burn subtitles, false to obey content settings.
675 Player::set_always_burn_subtitles (bool burn)
677 _always_burn_subtitles = burn;
684 _have_valid_pieces = false;