Merge still/moving image classes.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "resampler.h"
35 #include "log.h"
36 #include "scaler.h"
37
38 using std::list;
39 using std::cout;
40 using std::min;
41 using std::max;
42 using std::vector;
43 using std::pair;
44 using std::map;
45 using boost::shared_ptr;
46 using boost::weak_ptr;
47 using boost::dynamic_pointer_cast;
48
49 class Piece
50 {
51 public:
52         Piece (shared_ptr<Content> c)
53                 : content (c)
54                 , video_position (c->position ())
55                 , audio_position (c->position ())
56                 , repeat_to_do (0)
57                 , repeat_done (0)
58         {}
59         
60         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
61                 : content (c)
62                 , decoder (d)
63                 , video_position (c->position ())
64                 , audio_position (c->position ())
65         {}
66
67         /** Set this piece to repeat a video frame a given number of times */
68         void set_repeat (IncomingVideo video, int num)
69         {
70                 repeat_video = video;
71                 repeat_to_do = num;
72                 repeat_done = 0;
73         }
74
75         void reset_repeat ()
76         {
77                 repeat_video.image.reset ();
78                 repeat_to_do = 0;
79                 repeat_done = 0;
80         }
81
82         bool repeating () const
83         {
84                 return repeat_done != repeat_to_do;
85         }
86
87         void repeat (Player* player)
88         {
89                 player->process_video (
90                         repeat_video.weak_piece,
91                         repeat_video.image,
92                         repeat_video.eyes,
93                         repeat_done > 0,
94                         repeat_video.frame,
95                         (repeat_done + 1) * (TIME_HZ / player->_film->video_frame_rate ())
96                         );
97
98                 ++repeat_done;
99         }
100         
101         shared_ptr<Content> content;
102         shared_ptr<Decoder> decoder;
103         /** Time of the last video we emitted relative to the start of the DCP */
104         Time video_position;
105         /** Time of the last audio we emitted relative to the start of the DCP */
106         Time audio_position;
107
108         IncomingVideo repeat_video;
109         int repeat_to_do;
110         int repeat_done;
111 };
112
113 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
114         : _film (f)
115         , _playlist (p)
116         , _video (true)
117         , _audio (true)
118         , _have_valid_pieces (false)
119         , _video_position (0)
120         , _audio_position (0)
121         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
122         , _last_emit_was_black (false)
123 {
124         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
125         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
126         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
127         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
128 }
129
130 void
131 Player::disable_video ()
132 {
133         _video = false;
134 }
135
136 void
137 Player::disable_audio ()
138 {
139         _audio = false;
140 }
141
142 bool
143 Player::pass ()
144 {
145         if (!_have_valid_pieces) {
146                 setup_pieces ();
147                 _have_valid_pieces = true;
148         }
149
150         Time earliest_t = TIME_MAX;
151         shared_ptr<Piece> earliest;
152         enum {
153                 VIDEO,
154                 AUDIO
155         } type = VIDEO;
156
157         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
158                 if ((*i)->decoder->done ()) {
159                         continue;
160                 }
161
162                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
163                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
164
165                 if (_video && vd) {
166                         if ((*i)->video_position < earliest_t) {
167                                 earliest_t = (*i)->video_position;
168                                 earliest = *i;
169                                 type = VIDEO;
170                         }
171                 }
172
173                 if (_audio && ad && ad->has_audio ()) {
174                         if ((*i)->audio_position < earliest_t) {
175                                 earliest_t = (*i)->audio_position;
176                                 earliest = *i;
177                                 type = AUDIO;
178                         }
179                 }
180         }
181
182         if (!earliest) {
183                 flush ();
184                 return true;
185         }
186
187         switch (type) {
188         case VIDEO:
189                 if (earliest_t > _video_position) {
190                         emit_black ();
191                 } else {
192                         if (earliest->repeating ()) {
193                                 earliest->repeat (this);
194                         } else {
195                                 earliest->decoder->pass ();
196                         }
197                 }
198                 break;
199
200         case AUDIO:
201                 if (earliest_t > _audio_position) {
202                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
203                 } else {
204                         earliest->decoder->pass ();
205
206                         if (earliest->decoder->done()) {
207                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
208                                 assert (ac);
209                                 shared_ptr<Resampler> re = resampler (ac, false);
210                                 if (re) {
211                                         shared_ptr<const AudioBuffers> b = re->flush ();
212                                         if (b->frames ()) {
213                                                 process_audio (earliest, b, ac->audio_length ());
214                                         }
215                                 }
216                         }
217                 }
218                 break;
219         }
220
221         if (_audio) {
222                 boost::optional<Time> audio_done_up_to;
223                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
224                         if ((*i)->decoder->done ()) {
225                                 continue;
226                         }
227
228                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
229                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
230                         }
231                 }
232
233                 if (audio_done_up_to) {
234                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
235                         Audio (tb.audio, tb.time);
236                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
237                 }
238         }
239                 
240         return false;
241 }
242
243 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
244 void
245 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
246 {
247         /* Keep a note of what came in so that we can repeat it if required */
248         _last_incoming_video.weak_piece = weak_piece;
249         _last_incoming_video.image = image;
250         _last_incoming_video.eyes = eyes;
251         _last_incoming_video.same = same;
252         _last_incoming_video.frame = frame;
253         _last_incoming_video.extra = extra;
254         
255         shared_ptr<Piece> piece = weak_piece.lock ();
256         if (!piece) {
257                 return;
258         }
259
260         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
261         assert (content);
262
263         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
264         if (frc.skip && (frame % 2) == 1) {
265                 return;
266         }
267
268         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
269         if (content->trimmed (relative_time)) {
270                 return;
271         }
272
273         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
274         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
275
276         work_image = work_image->crop (content->crop(), true);
277
278         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
279         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
280         
281         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
282
283         Time time = content->position() + relative_time + extra - content->trim_start ();
284             
285         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
286                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
287         }
288
289         if (image_size != _video_container_size) {
290                 assert (image_size.width <= _video_container_size.width);
291                 assert (image_size.height <= _video_container_size.height);
292                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
293                 im->make_black ();
294                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
295                 work_image = im;
296         }
297
298 #ifdef DCPOMATIC_DEBUG
299         _last_video = piece->content;
300 #endif
301
302         Video (work_image, eyes, content->colour_conversion(), same, time);
303
304         time += TIME_HZ / _film->video_frame_rate();
305         _last_emit_was_black = false;
306         _video_position = piece->video_position = time;
307
308         if (frc.repeat > 1 && !piece->repeating ()) {
309                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
310         }
311 }
312
313 void
314 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
315 {
316         shared_ptr<Piece> piece = weak_piece.lock ();
317         if (!piece) {
318                 return;
319         }
320
321         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
322         assert (content);
323
324         /* Gain */
325         if (content->audio_gain() != 0) {
326                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
327                 gain->apply_gain (content->audio_gain ());
328                 audio = gain;
329         }
330
331         /* Resample */
332         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
333                 shared_ptr<Resampler> r = resampler (content, true);
334                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
335                 audio = ro.first;
336                 frame = ro.second;
337         }
338         
339         Time const relative_time = _film->audio_frames_to_time (frame);
340
341         if (content->trimmed (relative_time)) {
342                 return;
343         }
344
345         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
346         
347         /* Remap channels */
348         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
349         dcp_mapped->make_silent ();
350         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
351         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
352                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
353                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
354                 }
355         }
356
357         audio = dcp_mapped;
358
359         /* We must cut off anything that comes before the start of all time */
360         if (time < 0) {
361                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
362                 if (frames >= audio->frames ()) {
363                         return;
364                 }
365
366                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
367                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
368
369                 audio = trimmed;
370                 time = 0;
371         }
372
373         _audio_merger.push (audio, time);
374         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
375 }
376
377 void
378 Player::flush ()
379 {
380         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
381         if (tb.audio) {
382                 Audio (tb.audio, tb.time);
383                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
384         }
385
386         while (_video_position < _audio_position) {
387                 emit_black ();
388         }
389
390         while (_audio_position < _video_position) {
391                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
392         }
393         
394 }
395
396 /** Seek so that the next pass() will yield (approximately) the requested frame.
397  *  Pass accurate = true to try harder to get close to the request.
398  *  @return true on error
399  */
400 void
401 Player::seek (Time t, bool accurate)
402 {
403         if (!_have_valid_pieces) {
404                 setup_pieces ();
405                 _have_valid_pieces = true;
406         }
407
408         if (_pieces.empty ()) {
409                 return;
410         }
411
412         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
413                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
414                 if (!vc) {
415                         continue;
416                 }
417
418                 /* s is the offset of t from the start position of this content */
419                 Time s = t - vc->position ();
420                 s = max (static_cast<Time> (0), s);
421                 s = min (vc->length_after_trim(), s);
422
423                 /* Hence set the piece positions to the `global' time */
424                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
425
426                 /* And seek the decoder */
427                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
428                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
429                         );
430
431                 (*i)->reset_repeat ();
432         }
433
434         _video_position = _audio_position = t;
435
436         /* XXX: don't seek audio because we don't need to... */
437 }
438
439 void
440 Player::setup_pieces ()
441 {
442         list<shared_ptr<Piece> > old_pieces = _pieces;
443
444         _pieces.clear ();
445
446         ContentList content = _playlist->content ();
447         sort (content.begin(), content.end(), ContentSorter ());
448
449         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
450
451                 shared_ptr<Piece> piece (new Piece (*i));
452
453                 /* XXX: into content? */
454
455                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
456                 if (fc) {
457                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
458                         
459                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
460                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
461                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
462
463                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
464                         piece->decoder = fd;
465                 }
466                 
467                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
468                 if (ic) {
469                         bool reusing = false;
470                         
471                         /* See if we can re-use an old ImageDecoder */
472                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
473                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
474                                 if (imd && imd->content() == ic) {
475                                         piece = *j;
476                                         reusing = true;
477                                 }
478                         }
479
480                         if (!reusing) {
481                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
482                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
483                                 piece->decoder = id;
484                         }
485                 }
486
487                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
488                 if (sc) {
489                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
490                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
491
492                         piece->decoder = sd;
493                 }
494
495                 _pieces.push_back (piece);
496         }
497 }
498
499 void
500 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
501 {
502         shared_ptr<Content> c = w.lock ();
503         if (!c) {
504                 return;
505         }
506
507         if (
508                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
509                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
510                 ) {
511                 
512                 _have_valid_pieces = false;
513                 Changed (frequent);
514
515         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
516
517                 update_subtitle ();
518                 Changed (frequent);
519
520         } else if (
521                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
522                 property == VideoContentProperty::VIDEO_RATIO
523                 ) {
524                 
525                 Changed (frequent);
526
527         } else if (property == ContentProperty::PATH) {
528
529                 Changed (frequent);
530         }
531 }
532
533 void
534 Player::playlist_changed ()
535 {
536         _have_valid_pieces = false;
537         Changed (false);
538 }
539
540 void
541 Player::set_video_container_size (libdcp::Size s)
542 {
543         _video_container_size = s;
544         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
545         _black_frame->make_black ();
546 }
547
548 shared_ptr<Resampler>
549 Player::resampler (shared_ptr<AudioContent> c, bool create)
550 {
551         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
552         if (i != _resamplers.end ()) {
553                 return i->second;
554         }
555
556         if (!create) {
557                 return shared_ptr<Resampler> ();
558         }
559
560         _film->log()->log (
561                 String::compose (
562                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
563                         )
564                 );
565         
566         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
567         _resamplers[c] = r;
568         return r;
569 }
570
571 void
572 Player::emit_black ()
573 {
574 #ifdef DCPOMATIC_DEBUG
575         _last_video.reset ();
576 #endif
577
578         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
579         _video_position += _film->video_frames_to_time (1);
580         _last_emit_was_black = true;
581 }
582
583 void
584 Player::emit_silence (OutputAudioFrame most)
585 {
586         if (most == 0) {
587                 return;
588         }
589         
590         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
591         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
592         silence->make_silent ();
593         Audio (silence, _audio_position);
594         _audio_position += _film->audio_frames_to_time (N);
595 }
596
597 void
598 Player::film_changed (Film::Property p)
599 {
600         /* Here we should notice Film properties that affect our output, and
601            alert listeners that our output now would be different to how it was
602            last time we were run.
603         */
604
605         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
606                 Changed (false);
607         }
608 }
609
610 void
611 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
612 {
613         _in_subtitle.piece = weak_piece;
614         _in_subtitle.image = image;
615         _in_subtitle.rect = rect;
616         _in_subtitle.from = from;
617         _in_subtitle.to = to;
618
619         update_subtitle ();
620 }
621
622 void
623 Player::update_subtitle ()
624 {
625         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
626         if (!piece) {
627                 return;
628         }
629
630         if (!_in_subtitle.image) {
631                 _out_subtitle.image.reset ();
632                 return;
633         }
634
635         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
636         assert (sc);
637
638         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
639         libdcp::Size scaled_size;
640
641         in_rect.y += sc->subtitle_offset ();
642
643         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
644         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
645         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
646
647         /* Then we need a corrective translation, consisting of two parts:
648          *
649          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
650          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
651          *
652          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
653          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
654          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
655          *
656          * Combining these two translations gives these expressions.
657          */
658         
659         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
660         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
661         
662         _out_subtitle.image = _in_subtitle.image->scale (
663                 scaled_size,
664                 Scaler::from_id ("bicubic"),
665                 _in_subtitle.image->pixel_format (),
666                 true
667                 );
668         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
669         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
670 }
671
672 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
673  *  @return false if this could not be done.
674  */
675 bool
676 Player::repeat_last_video ()
677 {
678         if (!_last_incoming_video.image) {
679                 return false;
680         }
681
682         process_video (
683                 _last_incoming_video.weak_piece,
684                 _last_incoming_video.image,
685                 _last_incoming_video.eyes,
686                 _last_incoming_video.same,
687                 _last_incoming_video.frame,
688                 _last_incoming_video.extra
689                 );
690
691         return true;
692 }