src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 class Piece
  52 {
  53 public:
  54         Piece (shared_ptr<Content> c)
  55                 : content (c)
  56                 , video_position (c->position ())
  57                 , audio_position (c->position ())
  58         {}
  59
  60         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  61                 : content (c)
  62                 , decoder (d)
  63                 , video_position (c->position ())
  64                 , audio_position (c->position ())
  65         {}
  66
  67         shared_ptr<Content> content;
  68         shared_ptr<Decoder> decoder;
  69         Time video_position;
  70         Time audio_position;
  71 };
  72
  73 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  74         : _film (f)
  75         , _playlist (p)
  76         , _video (true)
  77         , _audio (true)
  78         , _have_valid_pieces (false)
  79         , _video_position (0)
  80         , _audio_position (0)
  81         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  82         , _last_emit_was_black (false)
  83 {
  84         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  85         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  86         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  87         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
  88 }
  89
  90 void
  91 Player::disable_video ()
  92 {
  93         _video = false;
  94 }
  95
  96 void
  97 Player::disable_audio ()
  98 {
  99         _audio = false;
 100 }
 101
 102 bool
 103 Player::pass ()
 104 {
 105         if (!_have_valid_pieces) {
 106                 setup_pieces ();
 107                 _have_valid_pieces = true;
 108         }
 109
 110         Time earliest_t = TIME_MAX;
 111         shared_ptr<Piece> earliest;
 112         enum {
 113                 VIDEO,
 114                 AUDIO
 115         } type = VIDEO;
 116
 117         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 118                 if ((*i)->decoder->done ()) {
 119                         continue;
 120                 }
 121
 122                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 123                         if ((*i)->video_position < earliest_t) {
 124                                 earliest_t = (*i)->video_position;
 125                                 earliest = *i;
 126                                 type = VIDEO;
 127                         }
 128                 }
 129
 130                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 131                         if ((*i)->audio_position < earliest_t) {
 132                                 earliest_t = (*i)->audio_position;
 133                                 earliest = *i;
 134                                 type = AUDIO;
 135                         }
 136                 }
 137         }
 138
 139         if (!earliest) {
 140                 flush ();
 141                 return true;
 142         }
 143
 144         switch (type) {
 145         case VIDEO:
 146                 if (earliest_t > _video_position) {
 147                         emit_black ();
 148                 } else {
 149                         earliest->decoder->pass ();
 150                 }
 151                 break;
 152
 153         case AUDIO:
 154                 if (earliest_t > _audio_position) {
 155                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 156                 } else {
 157                         earliest->decoder->pass ();
 158
 159                         if (earliest->decoder->done()) {
 160                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 161                                 assert (ac);
 162                                 shared_ptr<Resampler> re = resampler (ac, false);
 163                                 if (re) {
 164                                         shared_ptr<const AudioBuffers> b = re->flush ();
 165                                         if (b->frames ()) {
 166                                                 process_audio (earliest, b, ac->audio_length ());
 167                                         }
 168                                 }
 169                         }
 170                 }
 171                 break;
 172         }
 173
 174         if (_audio) {
 175                 Time audio_done_up_to = TIME_MAX;
 176                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 177                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 178                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
 179                         }
 180                 }
 181
 182                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
 183                 Audio (tb.audio, tb.time);
 184                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 185         }
 186
 187         return false;
 188 }
 189
 190 void
 191 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 192 {
 193         shared_ptr<Piece> piece = weak_piece.lock ();
 194         if (!piece) {
 195                 return;
 196         }
 197
 198         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 199         assert (content);
 200
 201         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 202         if (frc.skip && (frame % 2) == 1) {
 203                 return;
 204         }
 205
 206         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 207         if (content->trimmed (relative_time)) {
 208                 return;
 209         }
 210
 211         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
 212         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
 213
 214         work_image = work_image->crop (content->crop(), true);
 215
 216         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
 217         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
 218
 219         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
 220
 221         Time time = content->position() + relative_time - content->trim_start ();
 222
 223         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 224                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 225         }
 226
 227         if (image_size != _video_container_size) {
 228                 assert (image_size.width <= _video_container_size.width);
 229                 assert (image_size.height <= _video_container_size.height);
 230                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 231                 im->make_black ();
 232                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 233                 work_image = im;
 234         }
 235
 236 #ifdef DCPOMATIC_DEBUG
 237         _last_video = piece->content;
 238 #endif
 239
 240         Video (work_image, eyes, content->colour_conversion(), same, time);
 241         time += TIME_HZ / _film->video_frame_rate();
 242
 243         if (frc.repeat) {
 244                 Video (work_image, eyes, content->colour_conversion(), true, time);
 245                 time += TIME_HZ / _film->video_frame_rate();
 246         }
 247
 248         _last_emit_was_black = false;
 249
 250         _video_position = piece->video_position = time;
 251 }
 252
 253 void
 254 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 255 {
 256         shared_ptr<Piece> piece = weak_piece.lock ();
 257         if (!piece) {
 258                 return;
 259         }
 260
 261         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 262         assert (content);
 263
 264         /* Gain */
 265         if (content->audio_gain() != 0) {
 266                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 267                 gain->apply_gain (content->audio_gain ());
 268                 audio = gain;
 269         }
 270
 271         /* Resample */
 272         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 273                 shared_ptr<Resampler> r = resampler (content, true);
 274                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 275                 audio = ro.first;
 276                 frame = ro.second;
 277         }
 278
 279         Time const relative_time = _film->audio_frames_to_time (frame);
 280
 281         if (content->trimmed (relative_time)) {
 282                 return;
 283         }
 284
 285         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
 286
 287         /* Remap channels */
 288         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 289         dcp_mapped->make_silent ();
 290         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 291         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 292                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 293                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 294                 }
 295         }
 296
 297         audio = dcp_mapped;
 298
 299         /* We must cut off anything that comes before the start of all time */
 300         if (time < 0) {
 301                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 302                 if (frames >= audio->frames ()) {
 303                         return;
 304                 }
 305
 306                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 307                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 308
 309                 audio = trimmed;
 310                 time = 0;
 311         }
 312
 313         _audio_merger.push (audio, time);
 314         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 315 }
 316
 317 void
 318 Player::flush ()
 319 {
 320         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 321         if (tb.audio) {
 322                 Audio (tb.audio, tb.time);
 323                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 324         }
 325
 326         while (_video_position < _audio_position) {
 327                 emit_black ();
 328         }
 329
 330         while (_audio_position < _video_position) {
 331                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 332         }
 333
 334 }
 335
 336 /** Seek so that the next pass() will yield (approximately) the requested frame.
 337  *  Pass accurate = true to try harder to get close to the request.
 338  *  @return true on error
 339  */
 340 void
 341 Player::seek (Time t, bool accurate)
 342 {
 343         if (!_have_valid_pieces) {
 344                 setup_pieces ();
 345                 _have_valid_pieces = true;
 346         }
 347
 348         if (_pieces.empty ()) {
 349                 return;
 350         }
 351
 352         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 353                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 354                 if (!vc) {
 355                         continue;
 356                 }
 357
 358                 Time s = t - vc->position ();
 359                 s = max (static_cast<Time> (0), s);
 360                 s = min (vc->length_after_trim(), s);
 361
 362                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 363
 364                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 365                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 366                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 367                    the source's rate.
 368                 */
 369                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 370                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 371         }
 372
 373         _video_position = _audio_position = t;
 374
 375         /* XXX: don't seek audio because we don't need to... */
 376 }
 377
 378 void
 379 Player::setup_pieces ()
 380 {
 381         list<shared_ptr<Piece> > old_pieces = _pieces;
 382
 383         _pieces.clear ();
 384
 385         ContentList content = _playlist->content ();
 386         sort (content.begin(), content.end(), ContentSorter ());
 387
 388         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 389
 390                 shared_ptr<Piece> piece (new Piece (*i));
 391
 392                 /* XXX: into content? */
 393
 394                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 395                 if (fc) {
 396                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 397
 398                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 399                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 400                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 401
 402                         piece->decoder = fd;
 403                 }
 404
 405                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 406                 if (ic) {
 407                         shared_ptr<StillImageDecoder> id;
 408
 409                         /* See if we can re-use an old StillImageDecoder */
 410                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 411                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 412                                 if (imd && imd->content() == ic) {
 413                                         id = imd;
 414                                 }
 415                         }
 416
 417                         if (!id) {
 418                                 id.reset (new StillImageDecoder (_film, ic));
 419                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 420                         }
 421
 422                         piece->decoder = id;
 423                 }
 424
 425                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 426                 if (mc) {
 427                         shared_ptr<MovingImageDecoder> md;
 428
 429                         if (!md) {
 430                                 md.reset (new MovingImageDecoder (_film, mc));
 431                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 432                         }
 433
 434                         piece->decoder = md;
 435                 }
 436
 437                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 438                 if (sc) {
 439                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 440                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 441
 442                         piece->decoder = sd;
 443                 }
 444
 445                 _pieces.push_back (piece);
 446         }
 447 }
 448
 449 void
 450 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 451 {
 452         shared_ptr<Content> c = w.lock ();
 453         if (!c) {
 454                 return;
 455         }
 456
 457         if (
 458                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 459                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
 460                 ) {
 461
 462                 _have_valid_pieces = false;
 463                 Changed (frequent);
 464
 465         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 466
 467                 update_subtitle ();
 468                 Changed (frequent);
 469
 470         } else if (
 471                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
 472                 property == VideoContentProperty::VIDEO_RATIO
 473                 ) {
 474
 475                 Changed (frequent);
 476         }
 477 }
 478
 479 void
 480 Player::playlist_changed ()
 481 {
 482         _have_valid_pieces = false;
 483         Changed (false);
 484 }
 485
 486 void
 487 Player::set_video_container_size (libdcp::Size s)
 488 {
 489         _video_container_size = s;
 490         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 491         _black_frame->make_black ();
 492 }
 493
 494 shared_ptr<Resampler>
 495 Player::resampler (shared_ptr<AudioContent> c, bool create)
 496 {
 497         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 498         if (i != _resamplers.end ()) {
 499                 return i->second;
 500         }
 501
 502         if (!create) {
 503                 return shared_ptr<Resampler> ();
 504         }
 505
 506         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 507         _resamplers[c] = r;
 508         return r;
 509 }
 510
 511 void
 512 Player::emit_black ()
 513 {
 514 #ifdef DCPOMATIC_DEBUG
 515         _last_video.reset ();
 516 #endif
 517
 518         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 519         _video_position += _film->video_frames_to_time (1);
 520         _last_emit_was_black = true;
 521 }
 522
 523 void
 524 Player::emit_silence (OutputAudioFrame most)
 525 {
 526         if (most == 0) {
 527                 return;
 528         }
 529
 530         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 531         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 532         silence->make_silent ();
 533         Audio (silence, _audio_position);
 534         _audio_position += _film->audio_frames_to_time (N);
 535 }
 536
 537 void
 538 Player::film_changed (Film::Property p)
 539 {
 540         /* Here we should notice Film properties that affect our output, and
 541            alert listeners that our output now would be different to how it was
 542            last time we were run.
 543         */
 544
 545         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 546                 Changed (false);
 547         }
 548 }
 549
 550 void
 551 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 552 {
 553         _in_subtitle.piece = weak_piece;
 554         _in_subtitle.image = image;
 555         _in_subtitle.rect = rect;
 556         _in_subtitle.from = from;
 557         _in_subtitle.to = to;
 558
 559         update_subtitle ();
 560 }
 561
 562 void
 563 Player::update_subtitle ()
 564 {
 565         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 566         if (!piece) {
 567                 return;
 568         }
 569
 570         if (!_in_subtitle.image) {
 571                 _out_subtitle.image.reset ();
 572                 return;
 573         }
 574
 575         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 576         assert (sc);
 577
 578         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 579         libdcp::Size scaled_size;
 580
 581         in_rect.y += sc->subtitle_offset ();
 582
 583         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 584         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 585         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 586
 587         /* Then we need a corrective translation, consisting of two parts:
 588          *
 589          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 590          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 591          *
 592          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 593          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 594          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 595          *
 596          * Combining these two translations gives these expressions.
 597          */
 598
 599         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 600         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 601
 602         _out_subtitle.image = _in_subtitle.image->scale (
 603                 scaled_size,
 604                 Scaler::from_id ("bicubic"),
 605                 _in_subtitle.image->pixel_format (),
 606                 true
 607                 );
 608         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
 609         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
 610 }