Fix hang on attempt to emit zero frames of silence.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 //#define DEBUG_PLAYER 1
52
53 class Piece
54 {
55 public:
56         Piece (shared_ptr<Content> c)
57                 : content (c)
58                 , video_position (c->position ())
59                 , audio_position (c->position ())
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->position ())
66                 , audio_position (c->position ())
67         {}
68         
69         shared_ptr<Content> content;
70         shared_ptr<Decoder> decoder;
71         Time video_position;
72         Time audio_position;
73 };
74
75 #ifdef DEBUG_PLAYER
76 std::ostream& operator<<(std::ostream& s, Piece const & p)
77 {
78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
79                 s << "\tffmpeg     ";
80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
81                 s << "\tstill image";
82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
83                 s << "\tsndfile    ";
84         }
85         
86         s << " at " << p.content->position() << " until " << p.content->end();
87         
88         return s;
89 }
90 #endif  
91
92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
93         : _film (f)
94         , _playlist (p)
95         , _video (true)
96         , _audio (true)
97         , _have_valid_pieces (false)
98         , _video_position (0)
99         , _audio_position (0)
100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
101         , _last_emit_was_black (false)
102 {
103         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
104         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
105         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
106         set_video_container_size (_film->container()->size (_film->full_frame ()));
107 }
108
109 void
110 Player::disable_video ()
111 {
112         _video = false;
113 }
114
115 void
116 Player::disable_audio ()
117 {
118         _audio = false;
119 }
120
121 bool
122 Player::pass ()
123 {
124         if (!_have_valid_pieces) {
125                 setup_pieces ();
126                 _have_valid_pieces = true;
127         }
128
129 #ifdef DEBUG_PLAYER
130         cout << "= PASS\n";
131 #endif  
132
133         Time earliest_t = TIME_MAX;
134         shared_ptr<Piece> earliest;
135         enum {
136                 VIDEO,
137                 AUDIO
138         } type = VIDEO;
139
140         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
141                 if ((*i)->decoder->done ()) {
142                         continue;
143                 }
144
145                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
146                         if ((*i)->video_position < earliest_t) {
147                                 earliest_t = (*i)->video_position;
148                                 earliest = *i;
149                                 type = VIDEO;
150                         }
151                 }
152
153                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
154                         if ((*i)->audio_position < earliest_t) {
155                                 earliest_t = (*i)->audio_position;
156                                 earliest = *i;
157                                 type = AUDIO;
158                         }
159                 }
160         }
161
162         if (!earliest) {
163 #ifdef DEBUG_PLAYER
164                 cout << "no earliest piece.\n";
165 #endif          
166                 
167                 flush ();
168                 return true;
169         }
170
171         switch (type) {
172         case VIDEO:
173                 if (earliest_t > _video_position) {
174 #ifdef DEBUG_PLAYER
175                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
176 #endif
177                         emit_black ();
178                 } else {
179 #ifdef DEBUG_PLAYER
180                         cout << "Pass video " << *earliest << "\n";
181 #endif                  
182                         earliest->decoder->pass ();
183                 }
184                 break;
185
186         case AUDIO:
187                 if (earliest_t > _audio_position) {
188 #ifdef DEBUG_PLAYER
189                         cout << "no audio here (none until " << earliest_t << "); emitting silence.\n";
190 #endif
191                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
192                 } else {
193 #ifdef DEBUG_PLAYER
194                         cout << "Pass audio " << *earliest << "\n";
195 #endif
196                         earliest->decoder->pass ();
197
198                         if (earliest->decoder->done()) {
199                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
200                                 assert (ac);
201                                 shared_ptr<Resampler> re = resampler (ac, false);
202                                 if (re) {
203                                         shared_ptr<const AudioBuffers> b = re->flush ();
204                                         if (b->frames ()) {
205                                                 process_audio (earliest, b, ac->audio_length ());
206                                         }
207                                 }
208                         }
209                 }
210                 break;
211         }
212
213         if (_audio) {
214                 Time audio_done_up_to = TIME_MAX;
215                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
216                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
217                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
218                         }
219                 }
220
221                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
222                 Audio (tb.audio, tb.time);
223                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
224         }
225                 
226 #ifdef DEBUG_PLAYER
227         cout << "\tpost pass _video_position=" << _video_position << " _audio_position=" << _audio_position << "\n";
228 #endif  
229
230         return false;
231 }
232
233 void
234 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
235 {
236         shared_ptr<Piece> piece = weak_piece.lock ();
237         if (!piece) {
238                 return;
239         }
240
241         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
242         assert (content);
243
244         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
245         if (frc.skip && (frame % 2) == 1) {
246                 return;
247         }
248
249         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
250         if (content->trimmed (relative_time)) {
251                 return;
252         }
253         
254         shared_ptr<Image> work_image = image->crop (content->crop(), true);
255
256         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
257         
258         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
259
260         Time time = content->position() + relative_time - content->trim_start ();
261             
262         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
263                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
264         }
265
266         if (image_size != _video_container_size) {
267                 assert (image_size.width <= _video_container_size.width);
268                 assert (image_size.height <= _video_container_size.height);
269                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
270                 im->make_black ();
271                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
272                 work_image = im;
273         }
274
275 #ifdef DCPOMATIC_DEBUG
276         _last_video = piece->content;
277 #endif
278
279         Video (work_image, eyes, content->colour_conversion(), same, time);
280         time += TIME_HZ / _film->video_frame_rate();
281
282         if (frc.repeat) {
283                 Video (work_image, eyes, content->colour_conversion(), true, time);
284                 time += TIME_HZ / _film->video_frame_rate();
285         }
286
287         _last_emit_was_black = false;
288
289         _video_position = piece->video_position = time;
290 }
291
292 void
293 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
294 {
295         shared_ptr<Piece> piece = weak_piece.lock ();
296         if (!piece) {
297                 return;
298         }
299
300         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
301         assert (content);
302
303         /* Gain */
304         if (content->audio_gain() != 0) {
305                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
306                 gain->apply_gain (content->audio_gain ());
307                 audio = gain;
308         }
309
310         /* Resample */
311         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
312                 shared_ptr<Resampler> r = resampler (content, true);
313                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
314                 audio = ro.first;
315                 frame = ro.second;
316         }
317         
318         Time const relative_time = _film->audio_frames_to_time (frame);
319
320         if (content->trimmed (relative_time)) {
321                 return;
322         }
323
324         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
325         
326         /* Remap channels */
327         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
328         dcp_mapped->make_silent ();
329         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
330         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
331                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
332                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
333                 }
334         }
335
336         audio = dcp_mapped;
337
338         /* We must cut off anything that comes before the start of all time */
339         if (time < 0) {
340                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
341                 if (frames >= audio->frames ()) {
342                         return;
343                 }
344
345                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
346                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
347
348                 audio = trimmed;
349                 time = 0;
350         }
351
352         _audio_merger.push (audio, time);
353         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
354 }
355
356 void
357 Player::flush ()
358 {
359         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
360         if (tb.audio) {
361                 Audio (tb.audio, tb.time);
362                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
363         }
364
365         while (_video_position < _audio_position) {
366                 emit_black ();
367         }
368
369         while (_audio_position < _video_position) {
370                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
371         }
372         
373 }
374
375 /** Seek so that the next pass() will yield (approximately) the requested frame.
376  *  Pass accurate = true to try harder to get close to the request.
377  *  @return true on error
378  */
379 void
380 Player::seek (Time t, bool accurate)
381 {
382         if (!_have_valid_pieces) {
383                 setup_pieces ();
384                 _have_valid_pieces = true;
385         }
386
387         if (_pieces.empty ()) {
388                 return;
389         }
390
391         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
392                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
393                 if (!vc) {
394                         continue;
395                 }
396                 
397                 Time s = t - vc->position ();
398                 s = max (static_cast<Time> (0), s);
399                 s = min (vc->length_after_trim(), s);
400
401                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
402
403                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
404                 /* Here we are converting from time (in the DCP) to a frame number in the content.
405                    Hence we need to use the DCP's frame rate and the double/skip correction, not
406                    the source's rate.
407                 */
408                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
409                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
410         }
411
412         _video_position = _audio_position = t;
413         
414         /* XXX: don't seek audio because we don't need to... */
415 }
416
417 void
418 Player::setup_pieces ()
419 {
420         list<shared_ptr<Piece> > old_pieces = _pieces;
421
422         _pieces.clear ();
423
424         ContentList content = _playlist->content ();
425         sort (content.begin(), content.end(), ContentSorter ());
426
427         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
428
429                 shared_ptr<Piece> piece (new Piece (*i));
430
431                 /* XXX: into content? */
432
433                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
434                 if (fc) {
435                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
436                         
437                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
438                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
439                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
440
441                         piece->decoder = fd;
442                 }
443                 
444                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
445                 if (ic) {
446                         shared_ptr<StillImageDecoder> id;
447                         
448                         /* See if we can re-use an old StillImageDecoder */
449                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
450                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
451                                 if (imd && imd->content() == ic) {
452                                         id = imd;
453                                 }
454                         }
455
456                         if (!id) {
457                                 id.reset (new StillImageDecoder (_film, ic));
458                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
459                         }
460
461                         piece->decoder = id;
462                 }
463
464                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
465                 if (mc) {
466                         shared_ptr<MovingImageDecoder> md;
467
468                         if (!md) {
469                                 md.reset (new MovingImageDecoder (_film, mc));
470                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
471                         }
472
473                         piece->decoder = md;
474                 }
475
476                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
477                 if (sc) {
478                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
479                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
480
481                         piece->decoder = sd;
482                 }
483
484                 _pieces.push_back (piece);
485         }
486
487 #ifdef DEBUG_PLAYER
488         cout << "=== Player setup:\n";
489         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
490                 cout << *(i->get()) << "\n";
491         }
492 #endif  
493 }
494
495 void
496 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
497 {
498         shared_ptr<Content> c = w.lock ();
499         if (!c) {
500                 return;
501         }
502
503         if (
504                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
505                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
506                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
507                 ) {
508                 
509                 _have_valid_pieces = false;
510                 Changed (frequent);
511
512         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
513                 update_subtitle ();
514                 Changed (frequent);
515         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
516                 Changed (frequent);
517         }
518 }
519
520 void
521 Player::playlist_changed ()
522 {
523         _have_valid_pieces = false;
524         Changed (false);
525 }
526
527 void
528 Player::set_video_container_size (libdcp::Size s)
529 {
530         _video_container_size = s;
531         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
532         _black_frame->make_black ();
533 }
534
535 shared_ptr<Resampler>
536 Player::resampler (shared_ptr<AudioContent> c, bool create)
537 {
538         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
539         if (i != _resamplers.end ()) {
540                 return i->second;
541         }
542
543         if (!create) {
544                 return shared_ptr<Resampler> ();
545         }
546         
547         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
548         _resamplers[c] = r;
549         return r;
550 }
551
552 void
553 Player::emit_black ()
554 {
555 #ifdef DCPOMATIC_DEBUG
556         _last_video.reset ();
557 #endif
558         
559         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
560         _video_position += _film->video_frames_to_time (1);
561         _last_emit_was_black = true;
562 }
563
564 void
565 Player::emit_silence (OutputAudioFrame most)
566 {
567         if (most == 0) {
568                 return;
569         }
570         
571         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
572         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
573         silence->make_silent ();
574         Audio (silence, _audio_position);
575         _audio_position += _film->audio_frames_to_time (N);
576 }
577
578 void
579 Player::film_changed (Film::Property p)
580 {
581         /* Here we should notice Film properties that affect our output, and
582            alert listeners that our output now would be different to how it was
583            last time we were run.
584         */
585
586         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
587                 Changed (false);
588         }
589 }
590
591 void
592 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
593 {
594         _in_subtitle.piece = weak_piece;
595         _in_subtitle.image = image;
596         _in_subtitle.rect = rect;
597         _in_subtitle.from = from;
598         _in_subtitle.to = to;
599
600         update_subtitle ();
601 }
602
603 void
604 Player::update_subtitle ()
605 {
606         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
607         if (!piece) {
608                 return;
609         }
610
611         if (!_in_subtitle.image) {
612                 _out_subtitle.image.reset ();
613                 return;
614         }
615
616         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
617         assert (sc);
618
619         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
620         libdcp::Size scaled_size;
621
622         in_rect.y += sc->subtitle_offset ();
623
624         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
625         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
626         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
627
628         /* Then we need a corrective translation, consisting of two parts:
629          *
630          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
631          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
632          *
633          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
634          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
635          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
636          *
637          * Combining these two translations gives these expressions.
638          */
639         
640         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
641         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
642         
643         _out_subtitle.image = _in_subtitle.image->scale (libdcp::Size (scaled_size.width, scaled_size.height), Scaler::from_id ("bicubic"), true);
644         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
645         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
646 }