Fix seek with respect to video/audio frame boundary alignment.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "resampler.h"
35 #include "log.h"
36 #include "scaler.h"
37
38 using std::list;
39 using std::cout;
40 using std::min;
41 using std::max;
42 using std::vector;
43 using std::pair;
44 using std::map;
45 using boost::shared_ptr;
46 using boost::weak_ptr;
47 using boost::dynamic_pointer_cast;
48 using boost::optional;
49
50 class Piece
51 {
52 public:
53         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
54                 : content (c)
55                 , decoder (d)
56                 , frc (f)
57         {}
58
59         shared_ptr<Content> content;
60         shared_ptr<Decoder> decoder;
61         FrameRateChange frc;
62 };
63
64 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
65         : _film (f)
66         , _playlist (p)
67         , _video (true)
68         , _audio (true)
69         , _have_valid_pieces (false)
70         , _video_position (0)
71         , _audio_position (0)
72         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
73         , _last_emit_was_black (false)
74         , _just_did_inaccurate_seek (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
80 }
81
82 void
83 Player::disable_video ()
84 {
85         _video = false;
86 }
87
88 void
89 Player::disable_audio ()
90 {
91         _audio = false;
92 }
93
94 bool
95 Player::pass ()
96 {
97         if (!_have_valid_pieces) {
98                 setup_pieces ();
99         }
100
101         /* Interrogate all our pieces to find the one with the earliest decoded data */
102
103         shared_ptr<Piece> earliest_piece;
104         shared_ptr<Decoded> earliest_decoded;
105         DCPTime earliest_time = TIME_MAX;
106         DCPTime earliest_audio = TIME_MAX;
107
108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
109
110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
111
112                 if (dec) {
113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
114                 }
115
116                 /* XXX: don't know what to do with this */
117 #if 0           
118                 if (ad->done()) {
119                         shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> ((*i)->content);
120                         assert (ac);
121                         shared_ptr<Resampler> re = resampler (ac, false);
122                         if (re) {
123                                 shared_ptr<const AudioBuffers> b = re->flush ();
124                                 if (b->frames ()) {
125                                         process_audio (earliest, b, ac->audio_length ());
126                                 }
127                         }
128                 }
129 #endif          
130
131                 if (dec && dec->dcp_time < earliest_time) {
132                         earliest_piece = *i;
133                         earliest_decoded = dec;
134                         earliest_time = dec->dcp_time;
135                 }
136
137                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
138                         earliest_audio = dec->dcp_time;
139                 }
140         }
141                 
142         if (!earliest_piece) {
143                 flush ();
144                 return true;
145         }
146
147         if (earliest_audio != TIME_MAX) {
148                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
149                 Audio (tb.audio, tb.time);
150                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
151         }
152
153         /* Emit the earliest thing */
154
155         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
156         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
157         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
158
159         if (dv) {
160                 if (!_just_did_inaccurate_seek && earliest_time > _video_position) {
161                         /* XXX: if we're inside some content, repeat the last frame... otherwise emit black */
162                         emit_black ();
163                 } else {
164                         emit_video (earliest_piece, dv);
165                         earliest_piece->decoder->get ();
166                 }
167         } else if (da) {
168                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
169                         emit_silence (earliest_time - _audio_position);
170                 } else {
171                         emit_audio (earliest_piece, da);
172                         earliest_piece->decoder->get ();
173                 }
174         } else if (ds) {
175                 _in_subtitle.piece = earliest_piece;
176                 _in_subtitle.subtitle = ds;
177                 update_subtitle ();
178                 earliest_piece->decoder->get ();
179         }
180
181         _just_did_inaccurate_seek = false;
182
183         return false;
184 }
185
186 void
187 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
188 {
189         /* Keep a note of what came in so that we can repeat it if required */
190         _last_incoming_video.weak_piece = weak_piece;
191         _last_incoming_video.video = video;
192         
193         shared_ptr<Piece> piece = weak_piece.lock ();
194         if (!piece) {
195                 return;
196         }
197
198         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
199         assert (content);
200
201         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
202 #if 0
203         XXX
204         if (frc.skip && (frame % 2) == 1) {
205                 return;
206         }
207 #endif  
208
209         if (content->trimmed (video->dcp_time - content->position ())) {
210                 return;
211         }
212
213         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
214         libdcp::Size const image_size = fit_ratio_within (ratio, _video_container_size);
215
216         shared_ptr<PlayerImage> pi (
217                 new PlayerImage (
218                         video->image,
219                         content->crop(),
220                         image_size,
221                         _video_container_size,
222                         _film->scaler()
223                         )
224                 );
225         
226         if (
227                 _film->with_subtitles () &&
228                 _out_subtitle.subtitle->image &&
229                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
230                 ) {
231
232                 Position<int> const container_offset (
233                         (_video_container_size.width - image_size.width) / 2,
234                         (_video_container_size.height - image_size.width) / 2
235                         );
236
237                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
238         }
239                                             
240 #ifdef DCPOMATIC_DEBUG
241         _last_video = piece->content;
242 #endif
243
244         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
245
246         _last_emit_was_black = false;
247         _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
248 }
249
250 void
251 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
252 {
253         shared_ptr<Piece> piece = weak_piece.lock ();
254         if (!piece) {
255                 return;
256         }
257
258         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
259         assert (content);
260
261         /* Gain */
262         if (content->audio_gain() != 0) {
263                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
264                 gain->apply_gain (content->audio_gain ());
265                 audio->data = gain;
266         }
267
268         /* Resample */
269         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
270                 audio->data = resampler(content, true)->run (audio->data);
271         }
272         
273         if (content->trimmed (audio->dcp_time - content->position ())) {
274                 return;
275         }
276
277         /* Remap channels */
278         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
279         dcp_mapped->make_silent ();
280         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
281         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
282                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
283                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
284                 }
285         }
286
287         audio->data = dcp_mapped;
288
289         /* Delay */
290         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
291         if (audio->dcp_time < 0) {
292                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
293                 if (frames >= audio->data->frames ()) {
294                         return;
295                 }
296
297                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
298                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
299
300                 audio->data = trimmed;
301                 audio->dcp_time = 0;
302         }
303
304         _audio_merger.push (audio->data, audio->dcp_time);
305 }
306
307 void
308 Player::flush ()
309 {
310         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
311         if (tb.audio) {
312                 Audio (tb.audio, tb.time);
313                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
314         }
315
316         while (_video_position < _audio_position) {
317                 emit_black ();
318         }
319
320         while (_audio_position < _video_position) {
321                 emit_silence (_video_position - _audio_position);
322         }
323         
324 }
325
326 /** Seek so that the next pass() will yield (approximately) the requested frame.
327  *  Pass accurate = true to try harder to get close to the request.
328  *  @return true on error
329  */
330 void
331 Player::seek (DCPTime t, bool accurate)
332 {
333         if (!_have_valid_pieces) {
334                 setup_pieces ();
335         }
336
337         if (_pieces.empty ()) {
338                 return;
339         }
340
341         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
342                 /* s is the offset of t from the start position of this content */
343                 DCPTime s = t - (*i)->content->position ();
344                 s = max (static_cast<DCPTime> (0), s);
345                 s = min ((*i)->content->length_after_trim(), s);
346
347                 /* Convert this to the content time */
348                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
349
350                 /* And seek the decoder */
351                 (*i)->decoder->seek (ct, accurate);
352         }
353
354         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
355         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
356
357         _audio_merger.clear (_audio_position);
358
359         if (!accurate) {
360                 /* We just did an inaccurate seek, so it's likely that the next thing seen
361                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
362                    this flag stops pass() from trying to fix that: we assume that if it
363                    was an inaccurate seek then the caller does not care too much about
364                    inserting black/silence to keep the time tidy.
365                 */
366                 _just_did_inaccurate_seek = true;
367         }
368 }
369
370 void
371 Player::setup_pieces ()
372 {
373         list<shared_ptr<Piece> > old_pieces = _pieces;
374         _pieces.clear ();
375
376         ContentList content = _playlist->content ();
377
378         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
379
380                 shared_ptr<Decoder> decoder;
381                 optional<FrameRateChange> frc;
382
383                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
384                 if (fc) {
385                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
386                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
387                 }
388                 
389                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
390                 if (ic) {
391                         /* See if we can re-use an old ImageDecoder */
392                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
393                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
394                                 if (imd && imd->content() == ic) {
395                                         decoder = imd;
396                                 }
397                         }
398
399                         if (!decoder) {
400                                 decoder.reset (new ImageDecoder (_film, ic));
401                         }
402
403                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
404                 }
405
406                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
407                 if (sc) {
408                         decoder.reset (new SndfileDecoder (_film, sc));
409
410                         /* Working out the frc for this content is a bit tricky: what if it overlaps
411                            two pieces of video content with different frame rates?  For now, use
412                            the one with the best overlap.
413                         */
414
415                         DCPTime best_overlap_t = 0;
416                         shared_ptr<VideoContent> best_overlap;
417                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
418                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
419                                 if (!vc) {
420                                         continue;
421                                 }
422
423                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
424                                 if (overlap > best_overlap_t) {
425                                         best_overlap = vc;
426                                         best_overlap_t = overlap;
427                                 }
428                         }
429
430                         if (best_overlap) {
431                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
432                         } else {
433                                 /* No video overlap; e.g. if the DCP is just audio */
434                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
435                         }
436                 }
437
438                 decoder->seek ((*i)->trim_start (), true);
439                 
440                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
441         }
442
443         _have_valid_pieces = true;
444 }
445
446 void
447 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
448 {
449         shared_ptr<Content> c = w.lock ();
450         if (!c) {
451                 return;
452         }
453
454         if (
455                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
456                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
457                 ) {
458                 
459                 _have_valid_pieces = false;
460                 Changed (frequent);
461
462         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
463
464                 update_subtitle ();
465                 Changed (frequent);
466
467         } else if (
468                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
469                 property == VideoContentProperty::VIDEO_RATIO
470                 ) {
471                 
472                 Changed (frequent);
473
474         } else if (property == ContentProperty::PATH) {
475
476                 Changed (frequent);
477         }
478 }
479
480 void
481 Player::playlist_changed ()
482 {
483         _have_valid_pieces = false;
484         Changed (false);
485 }
486
487 void
488 Player::set_video_container_size (libdcp::Size s)
489 {
490         _video_container_size = s;
491
492         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
493         im->make_black ();
494         
495         _black_frame.reset (
496                 new PlayerImage (
497                         im,
498                         Crop(),
499                         _video_container_size,
500                         _video_container_size,
501                         Scaler::from_id ("bicubic")
502                         )
503                 );
504 }
505
506 shared_ptr<Resampler>
507 Player::resampler (shared_ptr<AudioContent> c, bool create)
508 {
509         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
510         if (i != _resamplers.end ()) {
511                 return i->second;
512         }
513
514         if (!create) {
515                 return shared_ptr<Resampler> ();
516         }
517
518         _film->log()->log (
519                 String::compose (
520                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
521                         )
522                 );
523         
524         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
525         _resamplers[c] = r;
526         return r;
527 }
528
529 void
530 Player::emit_black ()
531 {
532 #ifdef DCPOMATIC_DEBUG
533         _last_video.reset ();
534 #endif
535
536         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
537         _video_position += _film->video_frames_to_time (1);
538         _last_emit_was_black = true;
539 }
540
541 void
542 Player::emit_silence (DCPTime most)
543 {
544         if (most == 0) {
545                 return;
546         }
547         
548         DCPTime t = min (most, TIME_HZ / 2);
549         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
550         silence->make_silent ();
551         Audio (silence, _audio_position);
552         _audio_position += t;
553 }
554
555 void
556 Player::film_changed (Film::Property p)
557 {
558         /* Here we should notice Film properties that affect our output, and
559            alert listeners that our output now would be different to how it was
560            last time we were run.
561         */
562
563         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
564                 Changed (false);
565         }
566 }
567
568 void
569 Player::update_subtitle ()
570 {
571         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
572         if (!piece) {
573                 return;
574         }
575
576         if (!_in_subtitle.subtitle->image) {
577                 _out_subtitle.subtitle->image.reset ();
578                 return;
579         }
580
581         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
582         assert (sc);
583
584         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
585         libdcp::Size scaled_size;
586
587         in_rect.y += sc->subtitle_offset ();
588
589         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
590         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
591         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
592
593         /* Then we need a corrective translation, consisting of two parts:
594          *
595          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
596          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
597          *
598          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
599          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
600          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
601          *
602          * Combining these two translations gives these expressions.
603          */
604         
605         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
606         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
607         
608         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
609                 scaled_size,
610                 Scaler::from_id ("bicubic"),
611                 _in_subtitle.subtitle->image->pixel_format (),
612                 true
613                 );
614         
615         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
616         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
617 }
618
619 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
620  *  @return false if this could not be done.
621  */
622 bool
623 Player::repeat_last_video ()
624 {
625         if (!_last_incoming_video.video || !_have_valid_pieces) {
626                 return false;
627         }
628
629         emit_video (
630                 _last_incoming_video.weak_piece,
631                 _last_incoming_video.video
632                 );
633
634         return true;
635 }
636
637 PlayerImage::PlayerImage (
638         shared_ptr<const Image> in,
639         Crop crop,
640         libdcp::Size inter_size,
641         libdcp::Size out_size,
642         Scaler const * scaler
643         )
644         : _in (in)
645         , _crop (crop)
646         , _inter_size (inter_size)
647         , _out_size (out_size)
648         , _scaler (scaler)
649 {
650
651 }
652
653 void
654 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
655 {
656         _subtitle_image = image;
657         _subtitle_position = pos;
658 }
659
660 shared_ptr<Image>
661 PlayerImage::image ()
662 {
663         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, PIX_FMT_RGB24, false);
664
665         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
666
667         if (_subtitle_image) {
668                 out->alpha_blend (_subtitle_image, _subtitle_position);
669         }
670
671         return out;
672 }