Fix subtitle position.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "log.h"
35 #include "scaler.h"
36
37 using std::list;
38 using std::cout;
39 using std::min;
40 using std::max;
41 using std::vector;
42 using std::pair;
43 using std::map;
44 using boost::shared_ptr;
45 using boost::weak_ptr;
46 using boost::dynamic_pointer_cast;
47 using boost::optional;
48
49 class Piece
50 {
51 public:
52         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
53                 : content (c)
54                 , decoder (d)
55                 , frc (f)
56         {}
57
58         shared_ptr<Content> content;
59         shared_ptr<Decoder> decoder;
60         FrameRateChange frc;
61 };
62
63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
64         : _film (f)
65         , _playlist (p)
66         , _video (true)
67         , _audio (true)
68         , _have_valid_pieces (false)
69         , _video_position (0)
70         , _audio_position (0)
71         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
72         , _last_emit_was_black (false)
73         , _just_did_inaccurate_seek (false)
74         , _approximate_size (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
80 }
81
82 void
83 Player::disable_video ()
84 {
85         _video = false;
86 }
87
88 void
89 Player::disable_audio ()
90 {
91         _audio = false;
92 }
93
94 bool
95 Player::pass ()
96 {
97         if (!_have_valid_pieces) {
98                 setup_pieces ();
99         }
100
101         /* Interrogate all our pieces to find the one with the earliest decoded data */
102
103         shared_ptr<Piece> earliest_piece;
104         shared_ptr<Decoded> earliest_decoded;
105         DCPTime earliest_time = TIME_MAX;
106         DCPTime earliest_audio = TIME_MAX;
107
108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
109
110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
111
112                 if (dec) {
113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
114                 }
115
116                 if (dec && dec->dcp_time < earliest_time) {
117                         earliest_piece = *i;
118                         earliest_decoded = dec;
119                         earliest_time = dec->dcp_time;
120                 }
121
122                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
123                         earliest_audio = dec->dcp_time;
124                 }
125         }
126                 
127         if (!earliest_piece) {
128                 flush ();
129                 return true;
130         }
131
132         if (earliest_audio != TIME_MAX) {
133                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
134                 Audio (tb.audio, tb.time);
135                 /* This assumes that the audio_frames_to_time conversion is exact
136                    so that there are no accumulated errors caused by rounding.
137                 */
138                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
139         }
140
141         /* Emit the earliest thing */
142
143         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
144         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
145         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
146
147         /* Will be set to false if we shouldn't consume the peeked DecodedThing */
148         bool consume = true;
149
150         /* This is the margin either side of _{video,audio}_position that we will accept
151            as a starting point for a frame consecutive to the previous.
152         */
153         DCPTime const margin = TIME_HZ / (2 * _film->video_frame_rate ());
154         
155         if (dv && _video) {
156
157                 if (_just_did_inaccurate_seek) {
158
159                         /* Just emit; no subtlety */
160                         emit_video (earliest_piece, dv);
161                         step_video_position (dv);
162                         
163                 } else if (dv->dcp_time - _video_position > margin) {
164
165                         /* Too far ahead */
166
167                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
168                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
169                                 ++i;
170                         }
171
172                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
173                                 /* We're outside all video content */
174                                 emit_black ();
175                         } else {
176                                 /* We're inside some video; repeat the frame */
177                                 _last_incoming_video.video->dcp_time = _video_position;
178                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
179                                 step_video_position (_last_incoming_video.video);
180                         }
181
182                         consume = false;
183
184                 } else if (abs (dv->dcp_time - _video_position) < margin) {
185                         /* We're ok */
186                         emit_video (earliest_piece, dv);
187                         step_video_position (dv);
188                 } else {
189                         /* Too far behind: skip */
190                 }
191
192         } else if (da && _audio) {
193
194                 if (_just_did_inaccurate_seek) {
195                         /* Just emit; no subtlety */
196                         emit_audio (earliest_piece, da);
197                 } else if (da->dcp_time - _audio_position > margin) {
198                         /* Too far ahead */
199                         emit_silence (da->dcp_time - _audio_position);
200                         consume = false;
201                 } else if (abs (da->dcp_time - _audio_position) < margin) {
202                         /* We're ok */
203                         emit_audio (earliest_piece, da);
204                 } else {
205                         /* Too far behind: skip */
206                 }
207                 
208         } else if (ds && _video) {
209                 _in_subtitle.piece = earliest_piece;
210                 _in_subtitle.subtitle = ds;
211                 update_subtitle ();
212         }
213
214         if (consume) {
215                 earliest_piece->decoder->consume ();
216         }                       
217         
218         _just_did_inaccurate_seek = false;
219
220         return false;
221 }
222
223 void
224 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
225 {
226         /* Keep a note of what came in so that we can repeat it if required */
227         _last_incoming_video.weak_piece = weak_piece;
228         _last_incoming_video.video = video;
229         
230         shared_ptr<Piece> piece = weak_piece.lock ();
231         if (!piece) {
232                 return;
233         }
234
235         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
236         assert (content);
237
238         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
239
240         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
241         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
242         if (_approximate_size) {
243                 image_size.width &= ~3;
244                 image_size.height &= ~3;
245         }
246
247         shared_ptr<PlayerImage> pi (
248                 new PlayerImage (
249                         video->image,
250                         content->crop(),
251                         image_size,
252                         _video_container_size,
253                         _film->scaler()
254                         )
255                 );
256         
257         if (
258                 _film->with_subtitles () &&
259                 _out_subtitle.image &&
260                 video->dcp_time >= _out_subtitle.from && video->dcp_time <= _out_subtitle.to
261                 ) {
262
263                 Position<int> const container_offset (
264                         (_video_container_size.width - image_size.width) / 2,
265                         (_video_container_size.height - image_size.height) / 2
266                         );
267
268                 pi->set_subtitle (_out_subtitle.image, _out_subtitle.position + container_offset);
269         }
270                                             
271 #ifdef DCPOMATIC_DEBUG
272         _last_video = piece->content;
273 #endif
274
275         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
276         
277         _last_emit_was_black = false;
278 }
279
280 void
281 Player::step_video_position (shared_ptr<DecodedVideo> video)
282 {
283         /* This is a bit of a hack; don't update _video_position if EYES_RIGHT is on its way */
284         if (video->eyes != EYES_LEFT) {
285                 /* This assumes that the video_frames_to_time conversion is exact
286                    so that there are no accumulated errors caused by rounding.
287                 */
288                 _video_position += _film->video_frames_to_time (1);
289         }
290 }
291
292 void
293 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
294 {
295         shared_ptr<Piece> piece = weak_piece.lock ();
296         if (!piece) {
297                 return;
298         }
299
300         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
301         assert (content);
302
303         /* Gain */
304         if (content->audio_gain() != 0) {
305                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
306                 gain->apply_gain (content->audio_gain ());
307                 audio->data = gain;
308         }
309
310         if (content->trimmed (audio->dcp_time - content->position ())) {
311                 return;
312         }
313
314         /* Remap channels */
315         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
316         dcp_mapped->make_silent ();
317         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
318         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
319                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
320                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
321                 }
322         }
323
324         audio->data = dcp_mapped;
325
326         /* Delay */
327         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
328         if (audio->dcp_time < 0) {
329                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
330                 if (frames >= audio->data->frames ()) {
331                         return;
332                 }
333
334                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
335                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
336
337                 audio->data = trimmed;
338                 audio->dcp_time = 0;
339         }
340
341         _audio_merger.push (audio->data, audio->dcp_time);
342 }
343
344 void
345 Player::flush ()
346 {
347         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
348         if (tb.audio) {
349                 Audio (tb.audio, tb.time);
350                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
351         }
352
353         while (_video_position < _audio_position) {
354                 emit_black ();
355         }
356
357         while (_audio_position < _video_position) {
358                 emit_silence (_video_position - _audio_position);
359         }
360         
361 }
362
363 /** Seek so that the next pass() will yield (approximately) the requested frame.
364  *  Pass accurate = true to try harder to get close to the request.
365  *  @return true on error
366  */
367 void
368 Player::seek (DCPTime t, bool accurate)
369 {
370         if (!_have_valid_pieces) {
371                 setup_pieces ();
372         }
373
374         if (_pieces.empty ()) {
375                 return;
376         }
377
378         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
379                 /* s is the offset of t from the start position of this content */
380                 DCPTime s = t - (*i)->content->position ();
381                 s = max (static_cast<DCPTime> (0), s);
382                 s = min ((*i)->content->length_after_trim(), s);
383
384                 /* Convert this to the content time */
385                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
386
387                 /* And seek the decoder */
388                 (*i)->decoder->seek (ct, accurate);
389         }
390
391         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
392         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
393
394         _audio_merger.clear (_audio_position);
395
396         if (!accurate) {
397                 /* We just did an inaccurate seek, so it's likely that the next thing seen
398                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
399                    this flag stops pass() from trying to fix that: we assume that if it
400                    was an inaccurate seek then the caller does not care too much about
401                    inserting black/silence to keep the time tidy.
402                 */
403                 _just_did_inaccurate_seek = true;
404         }
405 }
406
407 void
408 Player::setup_pieces ()
409 {
410         list<shared_ptr<Piece> > old_pieces = _pieces;
411         _pieces.clear ();
412
413         ContentList content = _playlist->content ();
414
415         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
416
417                 shared_ptr<Decoder> decoder;
418                 optional<FrameRateChange> frc;
419
420                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
421                 if (fc) {
422                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
423                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
424                 }
425                 
426                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
427                 if (ic) {
428                         /* See if we can re-use an old ImageDecoder */
429                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
430                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
431                                 if (imd && imd->content() == ic) {
432                                         decoder = imd;
433                                 }
434                         }
435
436                         if (!decoder) {
437                                 decoder.reset (new ImageDecoder (_film, ic));
438                         }
439
440                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
441                 }
442
443                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
444                 if (sc) {
445                         decoder.reset (new SndfileDecoder (_film, sc));
446
447                         /* Working out the frc for this content is a bit tricky: what if it overlaps
448                            two pieces of video content with different frame rates?  For now, use
449                            the one with the best overlap.
450                         */
451
452                         DCPTime best_overlap_t = 0;
453                         shared_ptr<VideoContent> best_overlap;
454                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
455                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
456                                 if (!vc) {
457                                         continue;
458                                 }
459
460                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
461                                 if (overlap > best_overlap_t) {
462                                         best_overlap = vc;
463                                         best_overlap_t = overlap;
464                                 }
465                         }
466
467                         if (best_overlap) {
468                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
469                         } else {
470                                 /* No video overlap; e.g. if the DCP is just audio */
471                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
472                         }
473                 }
474
475                 decoder->seek ((*i)->trim_start (), true);
476                 
477                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
478         }
479
480         _have_valid_pieces = true;
481
482         /* The Piece for the _last_incoming_video will no longer be valid */
483         _last_incoming_video.video.reset ();
484
485         _video_position = _audio_position = 0;
486 }
487
488 void
489 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
490 {
491         shared_ptr<Content> c = w.lock ();
492         if (!c) {
493                 return;
494         }
495
496         if (
497                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
498                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
499                 property == VideoContentProperty::VIDEO_FRAME_TYPE 
500                 ) {
501                 
502                 _have_valid_pieces = false;
503                 Changed (frequent);
504
505         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
506
507                 update_subtitle ();
508                 Changed (frequent);
509
510         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
511                 
512                 Changed (frequent);
513
514         } else if (property == ContentProperty::PATH) {
515
516                 Changed (frequent);
517         }
518 }
519
520 void
521 Player::playlist_changed ()
522 {
523         _have_valid_pieces = false;
524         Changed (false);
525 }
526
527 void
528 Player::set_video_container_size (libdcp::Size s)
529 {
530         _video_container_size = s;
531
532         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
533         im->make_black ();
534         
535         _black_frame.reset (
536                 new PlayerImage (
537                         im,
538                         Crop(),
539                         _video_container_size,
540                         _video_container_size,
541                         Scaler::from_id ("bicubic")
542                         )
543                 );
544 }
545
546 void
547 Player::emit_black ()
548 {
549 #ifdef DCPOMATIC_DEBUG
550         _last_video.reset ();
551 #endif
552
553         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
554         _video_position += _film->video_frames_to_time (1);
555         _last_emit_was_black = true;
556 }
557
558 void
559 Player::emit_silence (DCPTime most)
560 {
561         if (most == 0) {
562                 return;
563         }
564         
565         DCPTime t = min (most, TIME_HZ / 2);
566         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
567         silence->make_silent ();
568         Audio (silence, _audio_position);
569         
570         _audio_position += t;
571 }
572
573 void
574 Player::film_changed (Film::Property p)
575 {
576         /* Here we should notice Film properties that affect our output, and
577            alert listeners that our output now would be different to how it was
578            last time we were run.
579         */
580
581         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
582                 Changed (false);
583         }
584 }
585
586 void
587 Player::update_subtitle ()
588 {
589         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
590         if (!piece) {
591                 return;
592         }
593
594         if (!_in_subtitle.subtitle->image) {
595                 _out_subtitle.image.reset ();
596                 return;
597         }
598
599         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
600         assert (sc);
601
602         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
603         libdcp::Size scaled_size;
604
605         in_rect.y += sc->subtitle_offset ();
606
607         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
608         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
609         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
610
611         /* Then we need a corrective translation, consisting of two parts:
612          *
613          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
614          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
615          *
616          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
617          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
618          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
619          *
620          * Combining these two translations gives these expressions.
621          */
622         
623         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
624         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
625
626         _out_subtitle.image = _in_subtitle.subtitle->image->scale (
627                 scaled_size,
628                 Scaler::from_id ("bicubic"),
629                 PIX_FMT_RGBA,
630                 true
631                 );
632
633         _out_subtitle.from = _in_subtitle.subtitle->dcp_time;
634         _out_subtitle.to = _in_subtitle.subtitle->dcp_time_to;
635 }
636
637 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
638  *  @return false if this could not be done.
639  */
640 bool
641 Player::repeat_last_video ()
642 {
643         if (!_last_incoming_video.video || !_have_valid_pieces) {
644                 return false;
645         }
646
647         emit_video (
648                 _last_incoming_video.weak_piece,
649                 _last_incoming_video.video
650                 );
651
652         return true;
653 }
654
655 void
656 Player::set_approximate_size ()
657 {
658         _approximate_size = true;
659 }
660                               
661
662 PlayerImage::PlayerImage (
663         shared_ptr<const Image> in,
664         Crop crop,
665         libdcp::Size inter_size,
666         libdcp::Size out_size,
667         Scaler const * scaler
668         )
669         : _in (in)
670         , _crop (crop)
671         , _inter_size (inter_size)
672         , _out_size (out_size)
673         , _scaler (scaler)
674 {
675
676 }
677
678 void
679 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
680 {
681         _subtitle_image = image;
682         _subtitle_position = pos;
683 }
684
685 shared_ptr<Image>
686 PlayerImage::image (AVPixelFormat format, bool aligned)
687 {
688         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, format, aligned);
689         
690         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
691
692         if (_subtitle_image) {
693                 out->alpha_blend (_subtitle_image, _subtitle_position);
694         }
695
696         return out;
697 }
698