647095793b9ac7e2874c8e3169d28f2de376c4c6
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 //#define DEBUG_PLAYER 1
52
53 class Piece
54 {
55 public:
56         Piece (shared_ptr<Content> c)
57                 : content (c)
58                 , video_position (c->start ())
59                 , audio_position (c->start ())
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->start ())
66                 , audio_position (c->start ())
67         {}
68         
69         shared_ptr<Content> content;
70         shared_ptr<Decoder> decoder;
71         Time video_position;
72         Time audio_position;
73 };
74
75 #ifdef DEBUG_PLAYER
76 std::ostream& operator<<(std::ostream& s, Piece const & p)
77 {
78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
79                 s << "\tffmpeg     ";
80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
81                 s << "\tstill image";
82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
83                 s << "\tsndfile    ";
84         }
85         
86         s << " at " << p.content->start() << " until " << p.content->end();
87         
88         return s;
89 }
90 #endif  
91
92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
93         : _film (f)
94         , _playlist (p)
95         , _video (true)
96         , _audio (true)
97         , _have_valid_pieces (false)
98         , _video_position (0)
99         , _audio_position (0)
100         , _audio_buffers (f->audio_channels(), 0)
101 {
102         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
103         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
104         _film->Changed.connect (bind (&Player::film_changed, this, _1));
105         set_video_container_size (_film->container()->size (_film->full_frame ()));
106 }
107
108 void
109 Player::disable_video ()
110 {
111         _video = false;
112 }
113
114 void
115 Player::disable_audio ()
116 {
117         _audio = false;
118 }
119
120 bool
121 Player::pass ()
122 {
123         if (!_have_valid_pieces) {
124                 setup_pieces ();
125                 _have_valid_pieces = true;
126         }
127
128 #ifdef DEBUG_PLAYER
129         cout << "= PASS\n";
130 #endif  
131
132         Time earliest_t = TIME_MAX;
133         shared_ptr<Piece> earliest;
134         enum {
135                 VIDEO,
136                 AUDIO
137         } type = VIDEO;
138
139         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
140                 if ((*i)->decoder->done ()) {
141                         continue;
142                 }
143
144                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
145                         if ((*i)->video_position < earliest_t) {
146                                 earliest_t = (*i)->video_position;
147                                 earliest = *i;
148                                 type = VIDEO;
149                         }
150                 }
151
152                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
153                         if ((*i)->audio_position < earliest_t) {
154                                 earliest_t = (*i)->audio_position;
155                                 earliest = *i;
156                                 type = AUDIO;
157                         }
158                 }
159         }
160
161         if (!earliest) {
162 #ifdef DEBUG_PLAYER
163                 cout << "no earliest piece.\n";
164 #endif          
165                 
166                 flush ();
167                 return true;
168         }
169
170         switch (type) {
171         case VIDEO:
172                 if (earliest_t > _video_position) {
173 #ifdef DEBUG_PLAYER
174                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
175 #endif
176                         emit_black ();
177                 } else {
178 #ifdef DEBUG_PLAYER
179                         cout << "Pass " << *earliest << "\n";
180 #endif                  
181                         earliest->decoder->pass ();
182                 }
183                 break;
184
185         case AUDIO:
186                 if (earliest_t > _audio_position) {
187 #ifdef DEBUG_PLAYER
188                         cout << "no audio here; emitting silence.\n";
189 #endif
190                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
191                 } else {
192 #ifdef DEBUG_PLAYER
193                         cout << "Pass " << *earliest << "\n";
194 #endif                  
195                         earliest->decoder->pass ();
196
197                         if (earliest->decoder->done()) {
198                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
199                                 assert (ac);
200                                 shared_ptr<Resampler> re = resampler (ac, false);
201                                 if (re) {
202                                         shared_ptr<const AudioBuffers> b = re->flush ();
203                                         if (b->frames ()) {
204                                                 process_audio (earliest, b, ac->audio_length ());
205                                         }
206                                 }
207                         }
208         
209                 }
210                 break;
211         }
212
213 #ifdef DEBUG_PLAYER
214         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
215 #endif  
216
217         return false;
218 }
219
220 void
221 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
222 {
223         shared_ptr<Piece> piece = weak_piece.lock ();
224         if (!piece) {
225                 return;
226         }
227
228         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
229         assert (content);
230
231         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
232         if (frc.skip && (frame % 2) == 1) {
233                 return;
234         }
235
236         shared_ptr<Image> work_image = image->crop (content->crop(), true);
237
238         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
239         
240         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
241
242         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
243         
244         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
245                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
246         }
247
248         if (image_size != _video_container_size) {
249                 assert (image_size.width <= _video_container_size.width);
250                 assert (image_size.height <= _video_container_size.height);
251                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
252                 im->make_black ();
253                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
254                 work_image = im;
255         }
256
257 #ifdef DCPOMATIC_DEBUG
258         _last_video = piece->content;
259 #endif
260
261         Video (work_image, eyes, same, time);
262         time += TIME_HZ / _film->video_frame_rate();
263
264         if (frc.repeat) {
265                 Video (work_image, eyes, true, time);
266                 time += TIME_HZ / _film->video_frame_rate();
267         }
268
269         _video_position = piece->video_position = time;
270 }
271
272 void
273 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
274 {
275         shared_ptr<Piece> piece = weak_piece.lock ();
276         if (!piece) {
277                 return;
278         }
279
280         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
281         assert (content);
282
283         /* Resample */
284         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
285                 shared_ptr<Resampler> r = resampler (content, true);
286                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
287                 audio = ro.first;
288                 frame = ro.second;
289         }
290         
291         /* Remap channels */
292         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
293         dcp_mapped->make_silent ();
294         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
295         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
296                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
297                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
298                 }
299         }
300
301         audio = dcp_mapped;
302
303         Time time = content->start()
304                 + _film->audio_frames_to_time (frame)
305                 + (content->audio_delay() * TIME_HZ / 1000);
306
307         /* We must cut off anything that comes before the start of all time */
308         if (time < 0) {
309                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
310                 if (frames >= audio->frames ()) {
311                         return;
312                 }
313
314                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
315                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
316
317                 audio = trimmed;
318                 time = 0;
319         }
320
321         /* The time of this audio may indicate that some of our buffered audio is not going to
322            be added to any more, so it can be emitted.
323         */
324
325         if (time > _audio_position) {
326                 /* We can emit some audio from our buffers; this is how many frames */
327                 OutputAudioFrame const N = _film->time_to_audio_frames (time - _audio_position);
328                 if (N > _audio_buffers.frames()) {
329                         /* We need some extra silence before whatever is in the buffers */
330                         _audio_buffers.ensure_size (N);
331                         _audio_buffers.move (0, N - _audio_buffers.frames(), _audio_buffers.frames ());
332                         _audio_buffers.make_silent (0, _audio_buffers.frames());
333                         _audio_buffers.set_frames (N);
334                 }
335                 assert (N <= _audio_buffers.frames());
336
337                 /* XXX: not convinced that a copy is necessary here */
338                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), N));
339                 emit->copy_from (&_audio_buffers, N, 0, 0);
340                 Audio (emit, _audio_position);
341
342                 _audio_position = piece->audio_position = _audio_position + _film->audio_frames_to_time (N);
343
344                 /* And remove it from our buffers */
345                 if (_audio_buffers.frames() > N) {
346                         _audio_buffers.move (N, 0, _audio_buffers.frames() - N);
347                 }
348                 _audio_buffers.set_frames (_audio_buffers.frames() - N);
349         }
350
351         /* Now accumulate the new audio into our buffers */
352         _audio_buffers.ensure_size (_audio_buffers.frames() + audio->frames());
353         _audio_buffers.accumulate_frames (audio.get(), 0, 0, audio->frames ());
354         _audio_buffers.set_frames (_audio_buffers.frames() + audio->frames());
355 }
356
357 void
358 Player::flush ()
359 {
360         if (_audio_buffers.frames() > 0) {
361                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), _audio_buffers.frames()));
362                 emit->copy_from (&_audio_buffers, _audio_buffers.frames(), 0, 0);
363                 Audio (emit, _audio_position);
364                 _audio_position += _film->audio_frames_to_time (_audio_buffers.frames ());
365                 _audio_buffers.set_frames (0);
366         }
367
368         while (_video_position < _audio_position) {
369                 emit_black ();
370         }
371
372         while (_audio_position < _video_position) {
373                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
374         }
375         
376 }
377
378 /** Seek so that the next pass() will yield (approximately) the requested frame.
379  *  Pass accurate = true to try harder to get close to the request.
380  *  @return true on error
381  */
382 void
383 Player::seek (Time t, bool accurate)
384 {
385         if (!_have_valid_pieces) {
386                 setup_pieces ();
387                 _have_valid_pieces = true;
388         }
389
390         if (_pieces.empty ()) {
391                 return;
392         }
393
394         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
395                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
396                 if (!vc) {
397                         continue;
398                 }
399                 
400                 Time s = t - vc->start ();
401                 s = max (static_cast<Time> (0), s);
402                 s = min (vc->length(), s);
403
404                 (*i)->video_position = (*i)->audio_position = vc->start() + s;
405
406                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
407                 /* Here we are converting from time (in the DCP) to a frame number in the content.
408                    Hence we need to use the DCP's frame rate and the double/skip correction, not
409                    the source's rate.
410                 */
411                 VideoContent::Frame f = s * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
412                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
413         }
414
415         _video_position = _audio_position = t;
416         
417         /* XXX: don't seek audio because we don't need to... */
418 }
419
420 void
421 Player::setup_pieces ()
422 {
423         list<shared_ptr<Piece> > old_pieces = _pieces;
424
425         _pieces.clear ();
426
427         ContentList content = _playlist->content ();
428         sort (content.begin(), content.end(), ContentSorter ());
429
430         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
431
432                 shared_ptr<Piece> piece (new Piece (*i));
433
434                 /* XXX: into content? */
435
436                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
437                 if (fc) {
438                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
439                         
440                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
441                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
442                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
443
444                         piece->decoder = fd;
445                 }
446                 
447                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
448                 if (ic) {
449                         shared_ptr<StillImageDecoder> id;
450                         
451                         /* See if we can re-use an old StillImageDecoder */
452                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
453                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
454                                 if (imd && imd->content() == ic) {
455                                         id = imd;
456                                 }
457                         }
458
459                         if (!id) {
460                                 id.reset (new StillImageDecoder (_film, ic));
461                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
462                         }
463
464                         piece->decoder = id;
465                 }
466
467                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
468                 if (mc) {
469                         shared_ptr<MovingImageDecoder> md;
470
471                         if (!md) {
472                                 md.reset (new MovingImageDecoder (_film, mc));
473                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
474                         }
475
476                         piece->decoder = md;
477                 }
478
479                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
480                 if (sc) {
481                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
482                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
483
484                         piece->decoder = sd;
485                 }
486
487                 _pieces.push_back (piece);
488         }
489
490 #ifdef DEBUG_PLAYER
491         cout << "=== Player setup:\n";
492         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
493                 cout << *(i->get()) << "\n";
494         }
495 #endif  
496 }
497
498 void
499 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
500 {
501         shared_ptr<Content> c = w.lock ();
502         if (!c) {
503                 return;
504         }
505
506         if (
507                 property == ContentProperty::START || property == ContentProperty::LENGTH ||
508                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
509                 ) {
510                 
511                 _have_valid_pieces = false;
512                 Changed (frequent);
513
514         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
515                 update_subtitle ();
516                 Changed (frequent);
517         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
518                 Changed (frequent);
519         }
520 }
521
522 void
523 Player::playlist_changed ()
524 {
525         _have_valid_pieces = false;
526         Changed (false);
527 }
528
529 void
530 Player::set_video_container_size (libdcp::Size s)
531 {
532         _video_container_size = s;
533         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
534         _black_frame->make_black ();
535 }
536
537 shared_ptr<Resampler>
538 Player::resampler (shared_ptr<AudioContent> c, bool create)
539 {
540         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
541         if (i != _resamplers.end ()) {
542                 return i->second;
543         }
544
545         if (!create) {
546                 return shared_ptr<Resampler> ();
547         }
548         
549         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
550         _resamplers[c] = r;
551         return r;
552 }
553
554 void
555 Player::emit_black ()
556 {
557 #ifdef DCPOMATIC_DEBUG
558         _last_video.reset ();
559 #endif
560         
561         /* XXX: use same here */
562         Video (_black_frame, EYES_BOTH, false, _video_position);
563         _video_position += _film->video_frames_to_time (1);
564 }
565
566 void
567 Player::emit_silence (OutputAudioFrame most)
568 {
569         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
570         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
571         silence->make_silent ();
572         Audio (silence, _audio_position);
573         _audio_position += _film->audio_frames_to_time (N);
574 }
575
576 void
577 Player::film_changed (Film::Property p)
578 {
579         /* Here we should notice Film properties that affect our output, and
580            alert listeners that our output now would be different to how it was
581            last time we were run.
582         */
583
584         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
585                 Changed (false);
586         }
587 }
588
589 void
590 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
591 {
592         _in_subtitle.piece = weak_piece;
593         _in_subtitle.image = image;
594         _in_subtitle.rect = rect;
595         _in_subtitle.from = from;
596         _in_subtitle.to = to;
597
598         update_subtitle ();
599 }
600
601 void
602 Player::update_subtitle ()
603 {
604         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
605         if (!piece) {
606                 return;
607         }
608
609         if (!_in_subtitle.image) {
610                 _out_subtitle.image.reset ();
611                 return;
612         }
613
614         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
615         assert (sc);
616
617         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
618         libdcp::Size scaled_size;
619
620         in_rect.y += sc->subtitle_offset ();
621
622         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
623         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
624         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
625
626         /* Then we need a corrective translation, consisting of two parts:
627          *
628          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
629          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
630          *
631          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
632          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
633          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
634          *
635          * Combining these two translations gives these expressions.
636          */
637         
638         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
639         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
640         
641         _out_subtitle.image = _in_subtitle.image->scale (libdcp::Size (scaled_size.width, scaled_size.height), Scaler::from_id ("bicubic"), true);
642         _out_subtitle.from = _in_subtitle.from + piece->content->start ();
643         _out_subtitle.to = _in_subtitle.to + piece->content->start ();
644 }