Fix a few memory leaks.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 class Piece
52 {
53 public:
54         Piece (shared_ptr<Content> c)
55                 : content (c)
56                 , video_position (c->position ())
57                 , audio_position (c->position ())
58                 , repeat_to_do (0)
59                 , repeat_done (0)
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->position ())
66                 , audio_position (c->position ())
67         {}
68
69         /** Set this piece to repeat a video frame a given number of times */
70         void set_repeat (IncomingVideo video, int num)
71         {
72                 repeat_video = video;
73                 repeat_to_do = num;
74                 repeat_done = 0;
75         }
76
77         void reset_repeat ()
78         {
79                 repeat_video.image.reset ();
80                 repeat_to_do = 0;
81                 repeat_done = 0;
82         }
83
84         bool repeating () const
85         {
86                 return repeat_done != repeat_to_do;
87         }
88
89         void repeat (Player* player)
90         {
91                 player->process_video (
92                         repeat_video.weak_piece,
93                         repeat_video.image,
94                         repeat_video.eyes,
95                         repeat_done > 0,
96                         repeat_video.frame,
97                         (repeat_done + 1) * (TIME_HZ / player->_film->video_frame_rate ())
98                         );
99
100                 ++repeat_done;
101         }
102         
103         shared_ptr<Content> content;
104         shared_ptr<Decoder> decoder;
105         /** Time of the last video we emitted relative to the start of the DCP */
106         Time video_position;
107         /** Time of the last audio we emitted relative to the start of the DCP */
108         Time audio_position;
109
110         IncomingVideo repeat_video;
111         int repeat_to_do;
112         int repeat_done;
113 };
114
115 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
116         : _film (f)
117         , _playlist (p)
118         , _video (true)
119         , _audio (true)
120         , _have_valid_pieces (false)
121         , _video_position (0)
122         , _audio_position (0)
123         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
124         , _last_emit_was_black (false)
125 {
126         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
127         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
128         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
129         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
130 }
131
132 void
133 Player::disable_video ()
134 {
135         _video = false;
136 }
137
138 void
139 Player::disable_audio ()
140 {
141         _audio = false;
142 }
143
144 bool
145 Player::pass ()
146 {
147         if (!_have_valid_pieces) {
148                 setup_pieces ();
149                 _have_valid_pieces = true;
150         }
151
152         Time earliest_t = TIME_MAX;
153         shared_ptr<Piece> earliest;
154         enum {
155                 VIDEO,
156                 AUDIO
157         } type = VIDEO;
158
159         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
160                 if ((*i)->decoder->done ()) {
161                         continue;
162                 }
163
164                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
165                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
166
167                 if (_video && vd) {
168                         if ((*i)->video_position < earliest_t) {
169                                 earliest_t = (*i)->video_position;
170                                 earliest = *i;
171                                 type = VIDEO;
172                         }
173                 }
174
175                 if (_audio && ad && ad->has_audio ()) {
176                         if ((*i)->audio_position < earliest_t) {
177                                 earliest_t = (*i)->audio_position;
178                                 earliest = *i;
179                                 type = AUDIO;
180                         }
181                 }
182         }
183
184         if (!earliest) {
185                 flush ();
186                 return true;
187         }
188
189         switch (type) {
190         case VIDEO:
191                 if (earliest_t > _video_position) {
192                         emit_black ();
193                 } else {
194                         if (earliest->repeating ()) {
195                                 earliest->repeat (this);
196                         } else {
197                                 earliest->decoder->pass ();
198                         }
199                 }
200                 break;
201
202         case AUDIO:
203                 if (earliest_t > _audio_position) {
204                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
205                 } else {
206                         earliest->decoder->pass ();
207
208                         if (earliest->decoder->done()) {
209                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
210                                 assert (ac);
211                                 shared_ptr<Resampler> re = resampler (ac, false);
212                                 if (re) {
213                                         shared_ptr<const AudioBuffers> b = re->flush ();
214                                         if (b->frames ()) {
215                                                 process_audio (earliest, b, ac->audio_length ());
216                                         }
217                                 }
218                         }
219                 }
220                 break;
221         }
222
223         if (_audio) {
224                 Time audio_done_up_to = TIME_MAX;
225                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
226                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
227                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
228                         }
229                 }
230
231                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
232                 Audio (tb.audio, tb.time);
233                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
234         }
235                 
236         return false;
237 }
238
239 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
240 void
241 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
242 {
243         /* Keep a note of what came in so that we can repeat it if required */
244         _last_incoming_video.weak_piece = weak_piece;
245         _last_incoming_video.image = image;
246         _last_incoming_video.eyes = eyes;
247         _last_incoming_video.same = same;
248         _last_incoming_video.frame = frame;
249         _last_incoming_video.extra = extra;
250         
251         shared_ptr<Piece> piece = weak_piece.lock ();
252         if (!piece) {
253                 return;
254         }
255
256         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
257         assert (content);
258
259         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
260         if (frc.skip && (frame % 2) == 1) {
261                 return;
262         }
263
264         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
265         if (content->trimmed (relative_time)) {
266                 return;
267         }
268
269         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
270         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
271
272         work_image = work_image->crop (content->crop(), true);
273
274         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
275         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
276         
277         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
278
279         Time time = content->position() + relative_time + extra - content->trim_start ();
280             
281         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
282                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
283         }
284
285         if (image_size != _video_container_size) {
286                 assert (image_size.width <= _video_container_size.width);
287                 assert (image_size.height <= _video_container_size.height);
288                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
289                 im->make_black ();
290                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
291                 work_image = im;
292         }
293
294 #ifdef DCPOMATIC_DEBUG
295         _last_video = piece->content;
296 #endif
297
298         Video (work_image, eyes, content->colour_conversion(), same, time);
299
300         time += TIME_HZ / _film->video_frame_rate();
301         _last_emit_was_black = false;
302         _video_position = piece->video_position = time;
303
304         if (frc.repeat > 1 && !piece->repeating ()) {
305                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
306         }
307 }
308
309 void
310 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
311 {
312         shared_ptr<Piece> piece = weak_piece.lock ();
313         if (!piece) {
314                 return;
315         }
316
317         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
318         assert (content);
319
320         /* Gain */
321         if (content->audio_gain() != 0) {
322                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
323                 gain->apply_gain (content->audio_gain ());
324                 audio = gain;
325         }
326
327         /* Resample */
328         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
329                 shared_ptr<Resampler> r = resampler (content, true);
330                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
331                 audio = ro.first;
332                 frame = ro.second;
333         }
334         
335         Time const relative_time = _film->audio_frames_to_time (frame);
336
337         if (content->trimmed (relative_time)) {
338                 return;
339         }
340
341         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
342         
343         /* Remap channels */
344         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
345         dcp_mapped->make_silent ();
346         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
347         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
348                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
349                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
350                 }
351         }
352
353         audio = dcp_mapped;
354
355         /* We must cut off anything that comes before the start of all time */
356         if (time < 0) {
357                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
358                 if (frames >= audio->frames ()) {
359                         return;
360                 }
361
362                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
363                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
364
365                 audio = trimmed;
366                 time = 0;
367         }
368
369         _audio_merger.push (audio, time);
370         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
371 }
372
373 void
374 Player::flush ()
375 {
376         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
377         if (tb.audio) {
378                 Audio (tb.audio, tb.time);
379                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
380         }
381
382         while (_video_position < _audio_position) {
383                 emit_black ();
384         }
385
386         while (_audio_position < _video_position) {
387                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
388         }
389         
390 }
391
392 /** Seek so that the next pass() will yield (approximately) the requested frame.
393  *  Pass accurate = true to try harder to get close to the request.
394  *  @return true on error
395  */
396 void
397 Player::seek (Time t, bool accurate)
398 {
399         if (!_have_valid_pieces) {
400                 setup_pieces ();
401                 _have_valid_pieces = true;
402         }
403
404         if (_pieces.empty ()) {
405                 return;
406         }
407
408         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
409                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
410                 if (!vc) {
411                         continue;
412                 }
413
414                 /* s is the offset of t from the start position of this content */
415                 Time s = t - vc->position ();
416                 s = max (static_cast<Time> (0), s);
417                 s = min (vc->length_after_trim(), s);
418
419                 /* Hence set the piece positions to the `global' time */
420                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
421
422                 /* And seek the decoder */
423                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
424                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
425                         );
426
427                 (*i)->reset_repeat ();
428         }
429
430         _video_position = _audio_position = t;
431
432         /* XXX: don't seek audio because we don't need to... */
433 }
434
435 void
436 Player::setup_pieces ()
437 {
438         list<shared_ptr<Piece> > old_pieces = _pieces;
439
440         _pieces.clear ();
441
442         ContentList content = _playlist->content ();
443         sort (content.begin(), content.end(), ContentSorter ());
444
445         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
446
447                 shared_ptr<Piece> piece (new Piece (*i));
448
449                 /* XXX: into content? */
450
451                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
452                 if (fc) {
453                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
454                         
455                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
456                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
457                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
458
459                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
460                         piece->decoder = fd;
461                 }
462                 
463                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
464                 if (ic) {
465                         shared_ptr<StillImageDecoder> id;
466                         
467                         /* See if we can re-use an old StillImageDecoder */
468                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
469                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
470                                 if (imd && imd->content() == ic) {
471                                         id = imd;
472                                 }
473                         }
474
475                         if (!id) {
476                                 id.reset (new StillImageDecoder (_film, ic));
477                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
478                         }
479
480                         piece->decoder = id;
481                 }
482
483                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
484                 if (mc) {
485                         shared_ptr<MovingImageDecoder> md;
486
487                         if (!md) {
488                                 md.reset (new MovingImageDecoder (_film, mc));
489                                 md->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
490                         }
491
492                         piece->decoder = md;
493                 }
494
495                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
496                 if (sc) {
497                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
498                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
499
500                         piece->decoder = sd;
501                 }
502
503                 _pieces.push_back (piece);
504         }
505 }
506
507 void
508 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
509 {
510         shared_ptr<Content> c = w.lock ();
511         if (!c) {
512                 return;
513         }
514
515         if (
516                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
517                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
518                 ) {
519                 
520                 _have_valid_pieces = false;
521                 Changed (frequent);
522
523         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
524
525                 update_subtitle ();
526                 Changed (frequent);
527
528         } else if (
529                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
530                 property == VideoContentProperty::VIDEO_RATIO
531                 ) {
532                 
533                 Changed (frequent);
534
535         } else if (property == ContentProperty::PATH) {
536
537                 Changed (frequent);
538         }
539 }
540
541 void
542 Player::playlist_changed ()
543 {
544         _have_valid_pieces = false;
545         Changed (false);
546 }
547
548 void
549 Player::set_video_container_size (libdcp::Size s)
550 {
551         _video_container_size = s;
552         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
553         _black_frame->make_black ();
554 }
555
556 shared_ptr<Resampler>
557 Player::resampler (shared_ptr<AudioContent> c, bool create)
558 {
559         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
560         if (i != _resamplers.end ()) {
561                 return i->second;
562         }
563
564         if (!create) {
565                 return shared_ptr<Resampler> ();
566         }
567         
568         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
569         _resamplers[c] = r;
570         return r;
571 }
572
573 void
574 Player::emit_black ()
575 {
576 #ifdef DCPOMATIC_DEBUG
577         _last_video.reset ();
578 #endif
579
580         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
581         _video_position += _film->video_frames_to_time (1);
582         _last_emit_was_black = true;
583 }
584
585 void
586 Player::emit_silence (OutputAudioFrame most)
587 {
588         if (most == 0) {
589                 return;
590         }
591         
592         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
593         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
594         silence->make_silent ();
595         Audio (silence, _audio_position);
596         _audio_position += _film->audio_frames_to_time (N);
597 }
598
599 void
600 Player::film_changed (Film::Property p)
601 {
602         /* Here we should notice Film properties that affect our output, and
603            alert listeners that our output now would be different to how it was
604            last time we were run.
605         */
606
607         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
608                 Changed (false);
609         }
610 }
611
612 void
613 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
614 {
615         _in_subtitle.piece = weak_piece;
616         _in_subtitle.image = image;
617         _in_subtitle.rect = rect;
618         _in_subtitle.from = from;
619         _in_subtitle.to = to;
620
621         update_subtitle ();
622 }
623
624 void
625 Player::update_subtitle ()
626 {
627         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
628         if (!piece) {
629                 return;
630         }
631
632         if (!_in_subtitle.image) {
633                 _out_subtitle.image.reset ();
634                 return;
635         }
636
637         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
638         assert (sc);
639
640         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
641         libdcp::Size scaled_size;
642
643         in_rect.y += sc->subtitle_offset ();
644
645         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
646         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
647         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
648
649         /* Then we need a corrective translation, consisting of two parts:
650          *
651          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
652          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
653          *
654          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
655          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
656          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
657          *
658          * Combining these two translations gives these expressions.
659          */
660         
661         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
662         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
663         
664         _out_subtitle.image = _in_subtitle.image->scale (
665                 scaled_size,
666                 Scaler::from_id ("bicubic"),
667                 _in_subtitle.image->pixel_format (),
668                 true
669                 );
670         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
671         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
672 }
673
674 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
675  *  @return false if this could not be done.
676  */
677 bool
678 Player::repeat_last_video ()
679 {
680         if (!_last_incoming_video.image) {
681                 return false;
682         }
683
684         process_video (
685                 _last_incoming_video.weak_piece,
686                 _last_incoming_video.image,
687                 _last_incoming_video.eyes,
688                 _last_incoming_video.same,
689                 _last_incoming_video.frame,
690                 _last_incoming_video.extra
691                 );
692
693         return true;
694 }