e3d88a54c212dbe36e3225a1f337090d76050d13
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "log.h"
35 #include "scaler.h"
36
37 using std::list;
38 using std::cout;
39 using std::min;
40 using std::max;
41 using std::vector;
42 using std::pair;
43 using std::map;
44 using boost::shared_ptr;
45 using boost::weak_ptr;
46 using boost::dynamic_pointer_cast;
47 using boost::optional;
48
49 class Piece
50 {
51 public:
52         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
53                 : content (c)
54                 , decoder (d)
55                 , frc (f)
56         {}
57
58         shared_ptr<Content> content;
59         shared_ptr<Decoder> decoder;
60         FrameRateChange frc;
61 };
62
63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
64         : _film (f)
65         , _playlist (p)
66         , _video (true)
67         , _audio (true)
68         , _have_valid_pieces (false)
69         , _video_position (0)
70         , _audio_position (0)
71         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
72         , _last_emit_was_black (false)
73         , _just_did_inaccurate_seek (false)
74         , _approximate_size (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
80 }
81
82 void
83 Player::disable_video ()
84 {
85         _video = false;
86 }
87
88 void
89 Player::disable_audio ()
90 {
91         _audio = false;
92 }
93
94 bool
95 Player::pass ()
96 {
97         if (!_have_valid_pieces) {
98                 setup_pieces ();
99         }
100
101         /* Interrogate all our pieces to find the one with the earliest decoded data */
102
103         shared_ptr<Piece> earliest_piece;
104         shared_ptr<Decoded> earliest_decoded;
105         DCPTime earliest_time = TIME_MAX;
106         DCPTime earliest_audio = TIME_MAX;
107
108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
109
110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
111
112                 if (dec) {
113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
114                 }
115
116                 if (dec && dec->dcp_time < earliest_time) {
117                         earliest_piece = *i;
118                         earliest_decoded = dec;
119                         earliest_time = dec->dcp_time;
120                 }
121
122                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
123                         earliest_audio = dec->dcp_time;
124                 }
125         }
126                 
127         if (!earliest_piece) {
128                 flush ();
129                 return true;
130         }
131
132         if (earliest_audio != TIME_MAX) {
133                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
134                 Audio (tb.audio, tb.time);
135                 /* This assumes that the audio_frames_to_time conversion is exact
136                    so that there are no accumulated errors caused by rounding.
137                 */
138                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
139         }
140
141         /* Emit the earliest thing */
142
143         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
144         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
145         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
146
147         /* Will be set to false if we shouldn't consume the peeked DecodedThing */
148         bool consume = true;
149
150         /* This is the margin either side of _{video,audio}_position that we will accept
151            as a starting point for a frame consecutive to the previous.
152         */
153         DCPTime const margin = TIME_HZ / (2 * _film->video_frame_rate ());
154         
155         if (dv && _video) {
156
157                 if (_just_did_inaccurate_seek) {
158
159                         /* Just emit; no subtlety */
160                         emit_video (earliest_piece, dv);
161                         step_video_position (dv);
162                         
163                 } else if (dv->dcp_time - _video_position > margin) {
164
165                         /* Too far ahead */
166
167                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
168                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
169                                 ++i;
170                         }
171
172                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
173                                 /* We're outside all video content */
174                                 emit_black ();
175                         } else {
176                                 /* We're inside some video; repeat the frame */
177                                 _last_incoming_video.video->dcp_time = _video_position;
178                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
179                                 step_video_position (_last_incoming_video.video);
180                         }
181
182                         consume = false;
183
184                 } else if (abs (dv->dcp_time - _video_position) < margin) {
185                         /* We're ok */
186                         emit_video (earliest_piece, dv);
187                         step_video_position (dv);
188                 } else {
189                         /* Too far behind: skip */
190                 }
191
192                 _just_did_inaccurate_seek = false;
193
194         } else if (da && _audio) {
195
196                 if (da->dcp_time - _audio_position > margin) {
197                         /* Too far ahead */
198                         emit_silence (da->dcp_time - _audio_position);
199                         consume = false;
200                 } else if (abs (da->dcp_time - _audio_position) < margin) {
201                         /* We're ok */
202                         emit_audio (earliest_piece, da);
203                 } else {
204                         /* Too far behind: skip */
205                 }
206                 
207         } else if (ds && _video) {
208                 _in_subtitle.piece = earliest_piece;
209                 _in_subtitle.subtitle = ds;
210                 update_subtitle ();
211         }
212
213         if (consume) {
214                 earliest_piece->decoder->consume ();
215         }                       
216         
217         return false;
218 }
219
220 void
221 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
222 {
223         /* Keep a note of what came in so that we can repeat it if required */
224         _last_incoming_video.weak_piece = weak_piece;
225         _last_incoming_video.video = video;
226         
227         shared_ptr<Piece> piece = weak_piece.lock ();
228         if (!piece) {
229                 return;
230         }
231
232         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
233         assert (content);
234
235         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
236
237         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
238         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
239         if (_approximate_size) {
240                 image_size.width &= ~3;
241                 image_size.height &= ~3;
242         }
243
244         shared_ptr<PlayerImage> pi (
245                 new PlayerImage (
246                         video->image,
247                         content->crop(),
248                         image_size,
249                         _video_container_size,
250                         _film->scaler()
251                         )
252                 );
253         
254         if (
255                 _film->with_subtitles () &&
256                 _out_subtitle.image &&
257                 video->dcp_time >= _out_subtitle.from && video->dcp_time <= _out_subtitle.to
258                 ) {
259
260                 Position<int> const container_offset (
261                         (_video_container_size.width - image_size.width) / 2,
262                         (_video_container_size.height - image_size.height) / 2
263                         );
264
265                 pi->set_subtitle (_out_subtitle.image, _out_subtitle.position + container_offset);
266         }
267                                             
268 #ifdef DCPOMATIC_DEBUG
269         _last_video = piece->content;
270 #endif
271
272         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
273         
274         _last_emit_was_black = false;
275 }
276
277 void
278 Player::step_video_position (shared_ptr<DecodedVideo> video)
279 {
280         /* This is a bit of a hack; don't update _video_position if EYES_RIGHT is on its way */
281         if (video->eyes != EYES_LEFT) {
282                 /* This assumes that the video_frames_to_time conversion is exact
283                    so that there are no accumulated errors caused by rounding.
284                 */
285                 _video_position += _film->video_frames_to_time (1);
286         }
287 }
288
289 void
290 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
291 {
292         shared_ptr<Piece> piece = weak_piece.lock ();
293         if (!piece) {
294                 return;
295         }
296
297         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
298         assert (content);
299
300         /* Gain */
301         if (content->audio_gain() != 0) {
302                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
303                 gain->apply_gain (content->audio_gain ());
304                 audio->data = gain;
305         }
306
307         if (content->trimmed (audio->dcp_time - content->position ())) {
308                 return;
309         }
310
311         /* Remap channels */
312         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
313         dcp_mapped->make_silent ();
314         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
315         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
316                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
317                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
318                 }
319         }
320
321         audio->data = dcp_mapped;
322
323         /* Delay */
324         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
325         if (audio->dcp_time < 0) {
326                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
327                 if (frames >= audio->data->frames ()) {
328                         return;
329                 }
330
331                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
332                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
333
334                 audio->data = trimmed;
335                 audio->dcp_time = 0;
336         }
337
338         _audio_merger.push (audio->data, audio->dcp_time);
339 }
340
341 void
342 Player::flush ()
343 {
344         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
345         if (tb.audio) {
346                 Audio (tb.audio, tb.time);
347                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
348         }
349
350         while (_video_position < _audio_position) {
351                 emit_black ();
352         }
353
354         while (_audio_position < _video_position) {
355                 emit_silence (_video_position - _audio_position);
356         }
357         
358 }
359
360 /** Seek so that the next pass() will yield (approximately) the requested frame.
361  *  Pass accurate = true to try harder to get close to the request.
362  *  @return true on error
363  */
364 void
365 Player::seek (DCPTime t, bool accurate)
366 {
367         if (!_have_valid_pieces) {
368                 setup_pieces ();
369         }
370
371         if (_pieces.empty ()) {
372                 return;
373         }
374
375         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
376                 /* s is the offset of t from the start position of this content */
377                 DCPTime s = t - (*i)->content->position ();
378                 s = max (static_cast<DCPTime> (0), s);
379                 s = min ((*i)->content->length_after_trim(), s);
380
381                 /* Convert this to the content time */
382                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
383
384                 /* And seek the decoder */
385                 (*i)->decoder->seek (ct, accurate);
386         }
387
388         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
389         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
390
391         _audio_merger.clear (_audio_position);
392
393         if (!accurate) {
394                 /* We just did an inaccurate seek, so it's likely that the next thing seen
395                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
396                    this flag stops pass() from trying to fix that: we assume that if it
397                    was an inaccurate seek then the caller does not care too much about
398                    inserting black/silence to keep the time tidy.
399                 */
400                 _just_did_inaccurate_seek = true;
401         }
402 }
403
404 void
405 Player::setup_pieces ()
406 {
407         list<shared_ptr<Piece> > old_pieces = _pieces;
408         _pieces.clear ();
409
410         ContentList content = _playlist->content ();
411
412         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
413
414                 shared_ptr<Decoder> decoder;
415                 optional<FrameRateChange> frc;
416
417                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
418                 if (fc) {
419                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
420                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
421                 }
422                 
423                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
424                 if (ic) {
425                         /* See if we can re-use an old ImageDecoder */
426                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
427                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
428                                 if (imd && imd->content() == ic) {
429                                         decoder = imd;
430                                 }
431                         }
432
433                         if (!decoder) {
434                                 decoder.reset (new ImageDecoder (_film, ic));
435                         }
436
437                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
438                 }
439
440                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
441                 if (sc) {
442                         decoder.reset (new SndfileDecoder (_film, sc));
443
444                         /* Working out the frc for this content is a bit tricky: what if it overlaps
445                            two pieces of video content with different frame rates?  For now, use
446                            the one with the best overlap.
447                         */
448
449                         DCPTime best_overlap_t = 0;
450                         shared_ptr<VideoContent> best_overlap;
451                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
452                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
453                                 if (!vc) {
454                                         continue;
455                                 }
456
457                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
458                                 if (overlap > best_overlap_t) {
459                                         best_overlap = vc;
460                                         best_overlap_t = overlap;
461                                 }
462                         }
463
464                         if (best_overlap) {
465                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
466                         } else {
467                                 /* No video overlap; e.g. if the DCP is just audio */
468                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
469                         }
470                 }
471
472                 decoder->seek ((*i)->trim_start (), true);
473                 
474                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
475         }
476
477         _have_valid_pieces = true;
478
479         /* The Piece for the _last_incoming_video will no longer be valid */
480         _last_incoming_video.video.reset ();
481
482         _video_position = _audio_position = 0;
483 }
484
485 void
486 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
487 {
488         shared_ptr<Content> c = w.lock ();
489         if (!c) {
490                 return;
491         }
492
493         if (
494                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
495                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
496                 property == VideoContentProperty::VIDEO_FRAME_TYPE 
497                 ) {
498                 
499                 _have_valid_pieces = false;
500                 Changed (frequent);
501
502         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
503
504                 update_subtitle ();
505                 Changed (frequent);
506
507         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
508                 
509                 Changed (frequent);
510
511         } else if (property == ContentProperty::PATH) {
512
513                 Changed (frequent);
514         }
515 }
516
517 void
518 Player::playlist_changed ()
519 {
520         _have_valid_pieces = false;
521         Changed (false);
522 }
523
524 void
525 Player::set_video_container_size (libdcp::Size s)
526 {
527         _video_container_size = s;
528
529         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
530         im->make_black ();
531         
532         _black_frame.reset (
533                 new PlayerImage (
534                         im,
535                         Crop(),
536                         _video_container_size,
537                         _video_container_size,
538                         Scaler::from_id ("bicubic")
539                         )
540                 );
541 }
542
543 void
544 Player::emit_black ()
545 {
546 #ifdef DCPOMATIC_DEBUG
547         _last_video.reset ();
548 #endif
549
550         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
551         _video_position += _film->video_frames_to_time (1);
552         _last_emit_was_black = true;
553 }
554
555 void
556 Player::emit_silence (DCPTime most)
557 {
558         if (most == 0) {
559                 return;
560         }
561         
562         DCPTime t = min (most, TIME_HZ / 2);
563         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
564         silence->make_silent ();
565         Audio (silence, _audio_position);
566         
567         _audio_position += t;
568 }
569
570 void
571 Player::film_changed (Film::Property p)
572 {
573         /* Here we should notice Film properties that affect our output, and
574            alert listeners that our output now would be different to how it was
575            last time we were run.
576         */
577
578         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
579                 Changed (false);
580         }
581 }
582
583 void
584 Player::update_subtitle ()
585 {
586         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
587         if (!piece) {
588                 return;
589         }
590
591         if (!_in_subtitle.subtitle->image) {
592                 _out_subtitle.image.reset ();
593                 return;
594         }
595
596         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
597         assert (sc);
598
599         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
600         libdcp::Size scaled_size;
601
602         in_rect.y += sc->subtitle_offset ();
603
604         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
605         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
606         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
607
608         /* Then we need a corrective translation, consisting of two parts:
609          *
610          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
611          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
612          *
613          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
614          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
615          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
616          *
617          * Combining these two translations gives these expressions.
618          */
619         
620         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
621         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
622
623         _out_subtitle.image = _in_subtitle.subtitle->image->scale (
624                 scaled_size,
625                 Scaler::from_id ("bicubic"),
626                 PIX_FMT_RGBA,
627                 true
628                 );
629
630         _out_subtitle.from = _in_subtitle.subtitle->dcp_time;
631         _out_subtitle.to = _in_subtitle.subtitle->dcp_time_to;
632 }
633
634 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
635  *  @return false if this could not be done.
636  */
637 bool
638 Player::repeat_last_video ()
639 {
640         if (!_last_incoming_video.video || !_have_valid_pieces) {
641                 return false;
642         }
643
644         emit_video (
645                 _last_incoming_video.weak_piece,
646                 _last_incoming_video.video
647                 );
648
649         return true;
650 }
651
652 void
653 Player::set_approximate_size ()
654 {
655         _approximate_size = true;
656 }
657                               
658
659 PlayerImage::PlayerImage (
660         shared_ptr<const Image> in,
661         Crop crop,
662         libdcp::Size inter_size,
663         libdcp::Size out_size,
664         Scaler const * scaler
665         )
666         : _in (in)
667         , _crop (crop)
668         , _inter_size (inter_size)
669         , _out_size (out_size)
670         , _scaler (scaler)
671 {
672
673 }
674
675 void
676 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
677 {
678         _subtitle_image = image;
679         _subtitle_position = pos;
680 }
681
682 shared_ptr<Image>
683 PlayerImage::image (AVPixelFormat format, bool aligned)
684 {
685         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, format, aligned);
686         
687         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
688
689         if (_subtitle_image) {
690                 out->alpha_blend (_subtitle_image, _subtitle_position);
691         }
692
693         return out;
694 }
695