Hacks.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 class Piece
52 {
53 public:
54         Piece (shared_ptr<Content> c)
55                 : content (c)
56                 , video_position (c->position ())
57                 , audio_position (c->position ())
58                 , repeat_to_do (0)
59                 , repeat_done (0)
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->position ())
66                 , audio_position (c->position ())
67         {}
68
69         void set_repeat (IncomingVideo video, int num)
70         {
71                 cout << "Set repeat " << num << "\n";
72                 repeat_video = video;
73                 repeat_to_do = num;
74                 repeat_done = 0;
75         }
76
77         void reset_repeat ()
78         {
79                 repeat_video.image.reset ();
80                 repeat_to_do = 0;
81                 repeat_done = 0;
82         }
83
84         bool repeating () const
85         {
86                 return repeat_done != repeat_to_do;
87         }
88
89         void repeat (Player* player)
90         {
91                 cout << "repeating; " << repeat_done << "\n";
92                 player->process_video (
93                         repeat_video.weak_piece,
94                         repeat_video.image,
95                         repeat_video.eyes,
96                         repeat_video.same,
97                         repeat_video.frame,
98                         (repeat_done + 1) * (TIME_HZ / player->_film->video_frame_rate ())
99                         );
100
101                 ++repeat_done;
102         }
103         
104         shared_ptr<Content> content;
105         shared_ptr<Decoder> decoder;
106         Time video_position;
107         Time audio_position;
108
109         IncomingVideo repeat_video;
110         int repeat_to_do;
111         int repeat_done;
112 };
113
114 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
115         : _film (f)
116         , _playlist (p)
117         , _video (true)
118         , _audio (true)
119         , _have_valid_pieces (false)
120         , _video_position (0)
121         , _audio_position (0)
122         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
123         , _last_emit_was_black (false)
124 {
125         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
126         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
127         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
128         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
129 }
130
131 void
132 Player::disable_video ()
133 {
134         _video = false;
135 }
136
137 void
138 Player::disable_audio ()
139 {
140         _audio = false;
141 }
142
143 bool
144 Player::pass ()
145 {
146         if (!_have_valid_pieces) {
147                 setup_pieces ();
148                 _have_valid_pieces = true;
149         }
150
151         Time earliest_t = TIME_MAX;
152         shared_ptr<Piece> earliest;
153         enum {
154                 VIDEO,
155                 AUDIO
156         } type = VIDEO;
157
158         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
159                 if ((*i)->decoder->done ()) {
160                         cout << "Scan: done.\n";
161                         continue;
162                 }
163
164                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
165                         if ((*i)->video_position < earliest_t) {
166                                 earliest_t = (*i)->video_position;
167                                 earliest = *i;
168                                 type = VIDEO;
169                         }
170                 }
171
172                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
173                         if ((*i)->audio_position < earliest_t) {
174                                 earliest_t = (*i)->audio_position;
175                                 earliest = *i;
176                                 type = AUDIO;
177                         }
178                 }
179         }
180
181         if (!earliest) {
182                 cout << "No earliest: out.\n";
183                 flush ();
184                 return true;
185         }
186
187         cout << "Earliest: " << earliest_t << "\n";
188
189         switch (type) {
190         case VIDEO:
191                 cout << "VIDEO.\n";
192                 if (earliest_t > _video_position) {
193                         emit_black ();
194                 } else {
195                         if (earliest->repeating ()) {
196                                 cout << "-repeating.\n";
197                                 earliest->repeat (this);
198                         } else {
199                                 cout << "-passing.\n";
200                                 earliest->decoder->pass ();
201                         }
202                 }
203                 break;
204
205         case AUDIO:
206                 cout << "SOUND.\n";
207                 if (earliest_t > _audio_position) {
208                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
209                 } else {
210                         earliest->decoder->pass ();
211
212                         if (earliest->decoder->done()) {
213                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
214                                 assert (ac);
215                                 shared_ptr<Resampler> re = resampler (ac, false);
216                                 if (re) {
217                                         shared_ptr<const AudioBuffers> b = re->flush ();
218                                         if (b->frames ()) {
219                                                 process_audio (earliest, b, ac->audio_length ());
220                                         }
221                                 }
222                         }
223                 }
224                 break;
225         }
226
227         if (_audio) {
228                 Time audio_done_up_to = TIME_MAX;
229                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
230                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
231                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
232                         }
233                 }
234
235                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
236                 Audio (tb.audio, tb.time);
237                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
238         }
239                 
240         return false;
241 }
242
243 void
244 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
245 {
246         cout << "PLAYER RECEIVES A VIDEO FRAME, extra " << extra << "\n";
247         
248         /* Keep a note of what came in so that we can repeat it if required */
249         _last_incoming_video.weak_piece = weak_piece;
250         _last_incoming_video.image = image;
251         _last_incoming_video.eyes = eyes;
252         _last_incoming_video.same = same;
253         _last_incoming_video.frame = frame;
254         
255         shared_ptr<Piece> piece = weak_piece.lock ();
256         if (!piece) {
257                 return;
258         }
259
260         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
261         assert (content);
262
263         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
264         if (frc.skip && (frame % 2) == 1) {
265                 return;
266         }
267
268         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
269         if (content->trimmed (relative_time)) {
270                 return;
271         }
272
273         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
274         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
275
276         work_image = work_image->crop (content->crop(), true);
277
278         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
279         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
280         
281         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
282
283         Time time = content->position() + relative_time + extra - content->trim_start ();
284             
285         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
286                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
287         }
288
289         if (image_size != _video_container_size) {
290                 assert (image_size.width <= _video_container_size.width);
291                 assert (image_size.height <= _video_container_size.height);
292                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
293                 im->make_black ();
294                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
295                 work_image = im;
296         }
297
298 #ifdef DCPOMATIC_DEBUG
299         _last_video = piece->content;
300 #endif
301
302         Video (work_image, eyes, content->colour_conversion(), same, time);
303
304         time += TIME_HZ / _film->video_frame_rate();
305         _last_emit_was_black = false;
306         _video_position = piece->video_position = time;
307
308         cout << "frc.repeat=" << frc.repeat << "; vp now " << _video_position << "\n";
309
310         if (frc.repeat > 1 && !piece->repeating ()) {
311                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
312         }
313 }
314
315 void
316 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
317 {
318         shared_ptr<Piece> piece = weak_piece.lock ();
319         if (!piece) {
320                 return;
321         }
322
323         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
324         assert (content);
325
326         /* Gain */
327         if (content->audio_gain() != 0) {
328                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
329                 gain->apply_gain (content->audio_gain ());
330                 audio = gain;
331         }
332
333         /* Resample */
334         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
335                 shared_ptr<Resampler> r = resampler (content, true);
336                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
337                 audio = ro.first;
338                 frame = ro.second;
339         }
340         
341         Time const relative_time = _film->audio_frames_to_time (frame);
342
343         if (content->trimmed (relative_time)) {
344                 return;
345         }
346
347         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
348         
349         /* Remap channels */
350         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
351         dcp_mapped->make_silent ();
352         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
353         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
354                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
355                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
356                 }
357         }
358
359         audio = dcp_mapped;
360
361         /* We must cut off anything that comes before the start of all time */
362         if (time < 0) {
363                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
364                 if (frames >= audio->frames ()) {
365                         return;
366                 }
367
368                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
369                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
370
371                 audio = trimmed;
372                 time = 0;
373         }
374
375         _audio_merger.push (audio, time);
376         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
377 }
378
379 void
380 Player::flush ()
381 {
382         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
383         if (tb.audio) {
384                 Audio (tb.audio, tb.time);
385                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
386         }
387
388         while (_video_position < _audio_position) {
389                 emit_black ();
390         }
391
392         while (_audio_position < _video_position) {
393                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
394         }
395         
396 }
397
398 /** Seek so that the next pass() will yield (approximately) the requested frame.
399  *  Pass accurate = true to try harder to get close to the request.
400  *  @return true on error
401  */
402 void
403 Player::seek (Time t, bool accurate)
404 {
405         if (!_have_valid_pieces) {
406                 setup_pieces ();
407                 _have_valid_pieces = true;
408         }
409
410         if (_pieces.empty ()) {
411                 return;
412         }
413
414         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
415                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
416                 if (!vc) {
417                         continue;
418                 }
419                 
420                 Time s = t - vc->position ();
421                 s = max (static_cast<Time> (0), s);
422                 s = min (vc->length_after_trim(), s);
423
424                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
425
426                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
427                 /* Here we are converting from time (in the DCP) to a frame number in the content.
428                    Hence we need to use the DCP's frame rate and the double/skip correction, not
429                    the source's rate.
430                 */
431                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
432                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
433
434                 (*i)->reset_repeat ();
435         }
436
437         _video_position = _audio_position = t;
438
439         /* XXX: don't seek audio because we don't need to... */
440 }
441
442 void
443 Player::setup_pieces ()
444 {
445         list<shared_ptr<Piece> > old_pieces = _pieces;
446
447         _pieces.clear ();
448
449         ContentList content = _playlist->content ();
450         sort (content.begin(), content.end(), ContentSorter ());
451
452         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
453
454                 shared_ptr<Piece> piece (new Piece (*i));
455
456                 /* XXX: into content? */
457
458                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
459                 if (fc) {
460                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
461                         
462                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
463                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
464                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
465
466                         piece->decoder = fd;
467                 }
468                 
469                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
470                 if (ic) {
471                         shared_ptr<StillImageDecoder> id;
472                         
473                         /* See if we can re-use an old StillImageDecoder */
474                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
475                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
476                                 if (imd && imd->content() == ic) {
477                                         id = imd;
478                                 }
479                         }
480
481                         if (!id) {
482                                 id.reset (new StillImageDecoder (_film, ic));
483                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
484                         }
485
486                         piece->decoder = id;
487                 }
488
489                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
490                 if (mc) {
491                         shared_ptr<MovingImageDecoder> md;
492
493                         if (!md) {
494                                 md.reset (new MovingImageDecoder (_film, mc));
495                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
496                         }
497
498                         piece->decoder = md;
499                 }
500
501                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
502                 if (sc) {
503                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
504                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
505
506                         piece->decoder = sd;
507                 }
508
509                 _pieces.push_back (piece);
510         }
511 }
512
513 void
514 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
515 {
516         shared_ptr<Content> c = w.lock ();
517         if (!c) {
518                 return;
519         }
520
521         if (
522                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
523                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
524                 ) {
525                 
526                 _have_valid_pieces = false;
527                 Changed (frequent);
528
529         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
530
531                 update_subtitle ();
532                 Changed (frequent);
533
534         } else if (
535                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
536                 property == VideoContentProperty::VIDEO_RATIO
537                 ) {
538                 
539                 Changed (frequent);
540         }
541 }
542
543 void
544 Player::playlist_changed ()
545 {
546         _have_valid_pieces = false;
547         Changed (false);
548 }
549
550 void
551 Player::set_video_container_size (libdcp::Size s)
552 {
553         _video_container_size = s;
554         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
555         _black_frame->make_black ();
556 }
557
558 shared_ptr<Resampler>
559 Player::resampler (shared_ptr<AudioContent> c, bool create)
560 {
561         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
562         if (i != _resamplers.end ()) {
563                 return i->second;
564         }
565
566         if (!create) {
567                 return shared_ptr<Resampler> ();
568         }
569         
570         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
571         _resamplers[c] = r;
572         return r;
573 }
574
575 void
576 Player::emit_black ()
577 {
578 #ifdef DCPOMATIC_DEBUG
579         _last_video.reset ();
580 #endif
581         
582         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
583         _video_position += _film->video_frames_to_time (1);
584         _last_emit_was_black = true;
585 }
586
587 void
588 Player::emit_silence (OutputAudioFrame most)
589 {
590         if (most == 0) {
591                 return;
592         }
593         
594         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
595         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
596         silence->make_silent ();
597         Audio (silence, _audio_position);
598         _audio_position += _film->audio_frames_to_time (N);
599 }
600
601 void
602 Player::film_changed (Film::Property p)
603 {
604         /* Here we should notice Film properties that affect our output, and
605            alert listeners that our output now would be different to how it was
606            last time we were run.
607         */
608
609         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
610                 Changed (false);
611         }
612 }
613
614 void
615 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
616 {
617         _in_subtitle.piece = weak_piece;
618         _in_subtitle.image = image;
619         _in_subtitle.rect = rect;
620         _in_subtitle.from = from;
621         _in_subtitle.to = to;
622
623         update_subtitle ();
624 }
625
626 void
627 Player::update_subtitle ()
628 {
629         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
630         if (!piece) {
631                 return;
632         }
633
634         if (!_in_subtitle.image) {
635                 _out_subtitle.image.reset ();
636                 return;
637         }
638
639         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
640         assert (sc);
641
642         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
643         libdcp::Size scaled_size;
644
645         in_rect.y += sc->subtitle_offset ();
646
647         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
648         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
649         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
650
651         /* Then we need a corrective translation, consisting of two parts:
652          *
653          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
654          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
655          *
656          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
657          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
658          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
659          *
660          * Combining these two translations gives these expressions.
661          */
662         
663         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
664         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
665         
666         _out_subtitle.image = _in_subtitle.image->scale (
667                 scaled_size,
668                 Scaler::from_id ("bicubic"),
669                 _in_subtitle.image->pixel_format (),
670                 true
671                 );
672         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
673         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
674 }
675
676 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
677  *  @return false if this could not be done.
678  */
679 bool
680 Player::repeat_last_video ()
681 {
682         if (!_last_incoming_video.image) {
683                 return false;
684         }
685
686         process_video (
687                 _last_incoming_video.weak_piece,
688                 _last_incoming_video.image,
689                 _last_incoming_video.eyes,
690                 _last_incoming_video.same,
691                 _last_incoming_video.frame,
692                 0
693                 );
694
695         return true;
696 }