Repeat frames rather than using black for missing frames when we're inside some content.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "resampler.h"
35 #include "log.h"
36 #include "scaler.h"
37
38 using std::list;
39 using std::cout;
40 using std::min;
41 using std::max;
42 using std::vector;
43 using std::pair;
44 using std::map;
45 using boost::shared_ptr;
46 using boost::weak_ptr;
47 using boost::dynamic_pointer_cast;
48 using boost::optional;
49
50 class Piece
51 {
52 public:
53         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
54                 : content (c)
55                 , decoder (d)
56                 , frc (f)
57         {}
58
59         shared_ptr<Content> content;
60         shared_ptr<Decoder> decoder;
61         FrameRateChange frc;
62 };
63
64 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
65         : _film (f)
66         , _playlist (p)
67         , _video (true)
68         , _audio (true)
69         , _have_valid_pieces (false)
70         , _video_position (0)
71         , _audio_position (0)
72         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
73         , _last_emit_was_black (false)
74         , _just_did_inaccurate_seek (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
80 }
81
82 void
83 Player::disable_video ()
84 {
85         _video = false;
86 }
87
88 void
89 Player::disable_audio ()
90 {
91         _audio = false;
92 }
93
94 bool
95 Player::pass ()
96 {
97         if (!_have_valid_pieces) {
98                 setup_pieces ();
99         }
100
101         /* Interrogate all our pieces to find the one with the earliest decoded data */
102
103         shared_ptr<Piece> earliest_piece;
104         shared_ptr<Decoded> earliest_decoded;
105         DCPTime earliest_time = TIME_MAX;
106         DCPTime earliest_audio = TIME_MAX;
107
108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
109
110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
111
112                 if (dec) {
113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
114                 }
115
116                 /* XXX: don't know what to do with this */
117 #if 0           
118                 if (ad->done()) {
119                         shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> ((*i)->content);
120                         assert (ac);
121                         shared_ptr<Resampler> re = resampler (ac, false);
122                         if (re) {
123                                 shared_ptr<const AudioBuffers> b = re->flush ();
124                                 if (b->frames ()) {
125                                         process_audio (earliest, b, ac->audio_length ());
126                                 }
127                         }
128                 }
129 #endif          
130
131                 if (dec && dec->dcp_time < earliest_time) {
132                         earliest_piece = *i;
133                         earliest_decoded = dec;
134                         earliest_time = dec->dcp_time;
135                 }
136
137                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
138                         earliest_audio = dec->dcp_time;
139                 }
140         }
141                 
142         if (!earliest_piece) {
143                 flush ();
144                 return true;
145         }
146
147         if (earliest_audio != TIME_MAX) {
148                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
149                 Audio (tb.audio, tb.time);
150                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
151         }
152
153         /* Emit the earliest thing */
154
155         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
156         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
157         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
158
159         if (dv) {
160                 if (!_just_did_inaccurate_seek && earliest_time > _video_position) {
161
162                         /* See if we're inside some video content */
163                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
164                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
165                                 ++i;
166                         }
167
168                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
169                                 /* We're outside all video content */
170                                 emit_black ();
171                         } else {
172                                 _last_incoming_video.video->dcp_time = _video_position;
173                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
174                         }
175                 } else {
176                         emit_video (earliest_piece, dv);
177                         earliest_piece->decoder->get ();
178                 }
179         } else if (da) {
180                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
181                         emit_silence (earliest_time - _audio_position);
182                 } else {
183                         emit_audio (earliest_piece, da);
184                         earliest_piece->decoder->get ();
185                 }
186         } else if (ds) {
187                 _in_subtitle.piece = earliest_piece;
188                 _in_subtitle.subtitle = ds;
189                 update_subtitle ();
190                 earliest_piece->decoder->get ();
191         }
192
193         _just_did_inaccurate_seek = false;
194
195         return false;
196 }
197
198 void
199 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
200 {
201         /* Keep a note of what came in so that we can repeat it if required */
202         _last_incoming_video.weak_piece = weak_piece;
203         _last_incoming_video.video = video;
204         
205         shared_ptr<Piece> piece = weak_piece.lock ();
206         if (!piece) {
207                 return;
208         }
209
210         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
211         assert (content);
212
213         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
214 #if 0
215         XXX
216         if (frc.skip && (frame % 2) == 1) {
217                 return;
218         }
219 #endif  
220
221         if (content->trimmed (video->dcp_time - content->position ())) {
222                 return;
223         }
224
225         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
226         libdcp::Size const image_size = fit_ratio_within (ratio, _video_container_size);
227
228         shared_ptr<PlayerImage> pi (
229                 new PlayerImage (
230                         video->image,
231                         content->crop(),
232                         image_size,
233                         _video_container_size,
234                         _film->scaler()
235                         )
236                 );
237         
238         if (
239                 _film->with_subtitles () &&
240                 _out_subtitle.subtitle->image &&
241                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
242                 ) {
243
244                 Position<int> const container_offset (
245                         (_video_container_size.width - image_size.width) / 2,
246                         (_video_container_size.height - image_size.width) / 2
247                         );
248
249                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
250         }
251                                             
252 #ifdef DCPOMATIC_DEBUG
253         _last_video = piece->content;
254 #endif
255
256         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
257
258         _last_emit_was_black = false;
259         _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
260 }
261
262 void
263 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
264 {
265         shared_ptr<Piece> piece = weak_piece.lock ();
266         if (!piece) {
267                 return;
268         }
269
270         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
271         assert (content);
272
273         /* Gain */
274         if (content->audio_gain() != 0) {
275                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
276                 gain->apply_gain (content->audio_gain ());
277                 audio->data = gain;
278         }
279
280         /* Resample */
281         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
282                 audio->data = resampler(content, true)->run (audio->data);
283         }
284         
285         if (content->trimmed (audio->dcp_time - content->position ())) {
286                 return;
287         }
288
289         /* Remap channels */
290         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
291         dcp_mapped->make_silent ();
292         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
293         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
294                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
295                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
296                 }
297         }
298
299         audio->data = dcp_mapped;
300
301         /* Delay */
302         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
303         if (audio->dcp_time < 0) {
304                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
305                 if (frames >= audio->data->frames ()) {
306                         return;
307                 }
308
309                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
310                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
311
312                 audio->data = trimmed;
313                 audio->dcp_time = 0;
314         }
315
316         _audio_merger.push (audio->data, audio->dcp_time);
317 }
318
319 void
320 Player::flush ()
321 {
322         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
323         if (tb.audio) {
324                 Audio (tb.audio, tb.time);
325                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
326         }
327
328         while (_video_position < _audio_position) {
329                 emit_black ();
330         }
331
332         while (_audio_position < _video_position) {
333                 emit_silence (_video_position - _audio_position);
334         }
335         
336 }
337
338 /** Seek so that the next pass() will yield (approximately) the requested frame.
339  *  Pass accurate = true to try harder to get close to the request.
340  *  @return true on error
341  */
342 void
343 Player::seek (DCPTime t, bool accurate)
344 {
345         if (!_have_valid_pieces) {
346                 setup_pieces ();
347         }
348
349         if (_pieces.empty ()) {
350                 return;
351         }
352
353         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
354                 /* s is the offset of t from the start position of this content */
355                 DCPTime s = t - (*i)->content->position ();
356                 s = max (static_cast<DCPTime> (0), s);
357                 s = min ((*i)->content->length_after_trim(), s);
358
359                 /* Convert this to the content time */
360                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
361
362                 /* And seek the decoder */
363                 (*i)->decoder->seek (ct, accurate);
364         }
365
366         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
367         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
368
369         _audio_merger.clear (_audio_position);
370
371         if (!accurate) {
372                 /* We just did an inaccurate seek, so it's likely that the next thing seen
373                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
374                    this flag stops pass() from trying to fix that: we assume that if it
375                    was an inaccurate seek then the caller does not care too much about
376                    inserting black/silence to keep the time tidy.
377                 */
378                 _just_did_inaccurate_seek = true;
379         }
380 }
381
382 void
383 Player::setup_pieces ()
384 {
385         list<shared_ptr<Piece> > old_pieces = _pieces;
386         _pieces.clear ();
387
388         ContentList content = _playlist->content ();
389
390         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
391
392                 shared_ptr<Decoder> decoder;
393                 optional<FrameRateChange> frc;
394
395                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
396                 if (fc) {
397                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
398                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
399                 }
400                 
401                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
402                 if (ic) {
403                         /* See if we can re-use an old ImageDecoder */
404                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
405                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
406                                 if (imd && imd->content() == ic) {
407                                         decoder = imd;
408                                 }
409                         }
410
411                         if (!decoder) {
412                                 decoder.reset (new ImageDecoder (_film, ic));
413                         }
414
415                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
416                 }
417
418                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
419                 if (sc) {
420                         decoder.reset (new SndfileDecoder (_film, sc));
421
422                         /* Working out the frc for this content is a bit tricky: what if it overlaps
423                            two pieces of video content with different frame rates?  For now, use
424                            the one with the best overlap.
425                         */
426
427                         DCPTime best_overlap_t = 0;
428                         shared_ptr<VideoContent> best_overlap;
429                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
430                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
431                                 if (!vc) {
432                                         continue;
433                                 }
434
435                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
436                                 if (overlap > best_overlap_t) {
437                                         best_overlap = vc;
438                                         best_overlap_t = overlap;
439                                 }
440                         }
441
442                         if (best_overlap) {
443                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
444                         } else {
445                                 /* No video overlap; e.g. if the DCP is just audio */
446                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
447                         }
448                 }
449
450                 decoder->seek ((*i)->trim_start (), true);
451                 
452                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
453         }
454
455         _have_valid_pieces = true;
456 }
457
458 void
459 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
460 {
461         shared_ptr<Content> c = w.lock ();
462         if (!c) {
463                 return;
464         }
465
466         if (
467                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
468                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
469                 ) {
470                 
471                 _have_valid_pieces = false;
472                 Changed (frequent);
473
474         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
475
476                 update_subtitle ();
477                 Changed (frequent);
478
479         } else if (
480                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
481                 property == VideoContentProperty::VIDEO_RATIO
482                 ) {
483                 
484                 Changed (frequent);
485
486         } else if (property == ContentProperty::PATH) {
487
488                 Changed (frequent);
489         }
490 }
491
492 void
493 Player::playlist_changed ()
494 {
495         _have_valid_pieces = false;
496         Changed (false);
497 }
498
499 void
500 Player::set_video_container_size (libdcp::Size s)
501 {
502         _video_container_size = s;
503
504         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
505         im->make_black ();
506         
507         _black_frame.reset (
508                 new PlayerImage (
509                         im,
510                         Crop(),
511                         _video_container_size,
512                         _video_container_size,
513                         Scaler::from_id ("bicubic")
514                         )
515                 );
516 }
517
518 shared_ptr<Resampler>
519 Player::resampler (shared_ptr<AudioContent> c, bool create)
520 {
521         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
522         if (i != _resamplers.end ()) {
523                 return i->second;
524         }
525
526         if (!create) {
527                 return shared_ptr<Resampler> ();
528         }
529
530         _film->log()->log (
531                 String::compose (
532                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
533                         )
534                 );
535         
536         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
537         _resamplers[c] = r;
538         return r;
539 }
540
541 void
542 Player::emit_black ()
543 {
544 #ifdef DCPOMATIC_DEBUG
545         _last_video.reset ();
546 #endif
547
548         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
549         _video_position += _film->video_frames_to_time (1);
550         _last_emit_was_black = true;
551 }
552
553 void
554 Player::emit_silence (DCPTime most)
555 {
556         if (most == 0) {
557                 return;
558         }
559         
560         DCPTime t = min (most, TIME_HZ / 2);
561         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
562         silence->make_silent ();
563         Audio (silence, _audio_position);
564         _audio_position += t;
565 }
566
567 void
568 Player::film_changed (Film::Property p)
569 {
570         /* Here we should notice Film properties that affect our output, and
571            alert listeners that our output now would be different to how it was
572            last time we were run.
573         */
574
575         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
576                 Changed (false);
577         }
578 }
579
580 void
581 Player::update_subtitle ()
582 {
583         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
584         if (!piece) {
585                 return;
586         }
587
588         if (!_in_subtitle.subtitle->image) {
589                 _out_subtitle.subtitle->image.reset ();
590                 return;
591         }
592
593         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
594         assert (sc);
595
596         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
597         libdcp::Size scaled_size;
598
599         in_rect.y += sc->subtitle_offset ();
600
601         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
602         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
603         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
604
605         /* Then we need a corrective translation, consisting of two parts:
606          *
607          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
608          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
609          *
610          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
611          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
612          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
613          *
614          * Combining these two translations gives these expressions.
615          */
616         
617         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
618         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
619         
620         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
621                 scaled_size,
622                 Scaler::from_id ("bicubic"),
623                 _in_subtitle.subtitle->image->pixel_format (),
624                 true
625                 );
626         
627         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
628         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
629 }
630
631 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
632  *  @return false if this could not be done.
633  */
634 bool
635 Player::repeat_last_video ()
636 {
637         if (!_last_incoming_video.video || !_have_valid_pieces) {
638                 return false;
639         }
640
641         emit_video (
642                 _last_incoming_video.weak_piece,
643                 _last_incoming_video.video
644                 );
645
646         return true;
647 }
648
649 PlayerImage::PlayerImage (
650         shared_ptr<const Image> in,
651         Crop crop,
652         libdcp::Size inter_size,
653         libdcp::Size out_size,
654         Scaler const * scaler
655         )
656         : _in (in)
657         , _crop (crop)
658         , _inter_size (inter_size)
659         , _out_size (out_size)
660         , _scaler (scaler)
661 {
662
663 }
664
665 void
666 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
667 {
668         _subtitle_image = image;
669         _subtitle_position = pos;
670 }
671
672 shared_ptr<Image>
673 PlayerImage::image ()
674 {
675         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, PIX_FMT_RGB24, false);
676
677         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
678
679         if (_subtitle_image) {
680                 out->alpha_blend (_subtitle_image, _subtitle_position);
681         }
682
683         return out;
684 }