src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "resampler.h"
  35 #include "log.h"
  36 #include "scaler.h"
  37
  38 using std::list;
  39 using std::cout;
  40 using std::min;
  41 using std::max;
  42 using std::vector;
  43 using std::pair;
  44 using std::map;
  45 using boost::shared_ptr;
  46 using boost::weak_ptr;
  47 using boost::dynamic_pointer_cast;
  48
  49 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  50         : _film (f)
  51         , _playlist (p)
  52         , _video (true)
  53         , _audio (true)
  54         , _have_valid_pieces (false)
  55         , _video_position (0)
  56         , _audio_position (0)
  57         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  58         , _last_emit_was_black (false)
  59 {
  60         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  61         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  62         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  63         set_video_container_size (_film->frame_size ());
  64 }
  65
  66 void
  67 Player::disable_video ()
  68 {
  69         _video = false;
  70 }
  71
  72 void
  73 Player::disable_audio ()
  74 {
  75         _audio = false;
  76 }
  77
  78 bool
  79 Player::pass ()
  80 {
  81         if (!_have_valid_pieces) {
  82                 setup_pieces ();
  83         }
  84
  85         Time earliest_t = TIME_MAX;
  86         shared_ptr<Piece> earliest;
  87         enum {
  88                 VIDEO,
  89                 AUDIO
  90         } type = VIDEO;
  91
  92         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  93                 if ((*i)->decoder->done ()) {
  94                         continue;
  95                 }
  96
  97                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
  98                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
  99
 100                 if (_video && vd) {
 101                         if ((*i)->video_position < earliest_t) {
 102                                 earliest_t = (*i)->video_position;
 103                                 earliest = *i;
 104                                 type = VIDEO;
 105                         }
 106                 }
 107
 108                 if (_audio && ad && ad->has_audio ()) {
 109                         if ((*i)->audio_position < earliest_t) {
 110                                 earliest_t = (*i)->audio_position;
 111                                 earliest = *i;
 112                                 type = AUDIO;
 113                         }
 114                 }
 115         }
 116
 117         if (!earliest) {
 118                 flush ();
 119                 return true;
 120         }
 121
 122         switch (type) {
 123         case VIDEO:
 124                 if (earliest_t > _video_position) {
 125                         emit_black ();
 126                 } else {
 127                         if (earliest->repeating ()) {
 128                                 earliest->repeat (this);
 129                         } else {
 130                                 earliest->decoder->pass ();
 131                         }
 132                 }
 133                 break;
 134
 135         case AUDIO:
 136                 if (earliest_t > _audio_position) {
 137                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 138                 } else {
 139                         earliest->decoder->pass ();
 140
 141                         if (earliest->decoder->done()) {
 142                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 143                                 assert (ac);
 144                                 shared_ptr<Resampler> re = resampler (ac, false);
 145                                 if (re) {
 146                                         shared_ptr<const AudioBuffers> b = re->flush ();
 147                                         if (b->frames ()) {
 148                                                 process_audio (earliest, b, ac->audio_length ());
 149                                         }
 150                                 }
 151                         }
 152                 }
 153                 break;
 154         }
 155
 156         if (_audio) {
 157                 boost::optional<Time> audio_done_up_to;
 158                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 159                         if ((*i)->decoder->done ()) {
 160                                 continue;
 161                         }
 162
 163                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 164                         if (ad && ad->has_audio ()) {
 165                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 166                         }
 167                 }
 168
 169                 if (audio_done_up_to) {
 170                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 171                         Audio (tb.audio, tb.time);
 172                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 173                 }
 174         }
 175
 176         return false;
 177 }
 178
 179 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 180 void
 181 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
 182 {
 183         /* Keep a note of what came in so that we can repeat it if required */
 184         _last_incoming_video.weak_piece = weak_piece;
 185         _last_incoming_video.image = image;
 186         _last_incoming_video.eyes = eyes;
 187         _last_incoming_video.same = same;
 188         _last_incoming_video.frame = frame;
 189         _last_incoming_video.extra = extra;
 190
 191         shared_ptr<Piece> piece = weak_piece.lock ();
 192         if (!piece) {
 193                 return;
 194         }
 195
 196         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 197         assert (content);
 198
 199         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 200         if (frc.skip && (frame % 2) == 1) {
 201                 return;
 202         }
 203
 204         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 205         if (content->trimmed (relative_time)) {
 206                 return;
 207         }
 208
 209         Time const time = content->position() + relative_time + extra - content->trim_start ();
 210         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 211
 212         shared_ptr<PlayerImage> pi (
 213                 new PlayerImage (
 214                         image,
 215                         content->crop(),
 216                         image_size,
 217                         _video_container_size,
 218                         _film->scaler()
 219                         )
 220                 );
 221
 222         if (_film->with_subtitles ()) {
 223                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 224                         if (i->covers (time)) {
 225                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 226                                    one is the one we want to use, so that's ok.
 227                                 */
 228                                 Position<int> const container_offset (
 229                                         (_video_container_size.width - image_size.width) / 2,
 230                                         (_video_container_size.height - image_size.width) / 2
 231                                         );
 232
 233                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 234                         }
 235                 }
 236         }
 237
 238         /* Clear out old subtitles */
 239         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 240                 list<Subtitle>::iterator j = i;
 241                 ++j;
 242
 243                 if (i->ends_before (time)) {
 244                         _subtitles.erase (i);
 245                 }
 246
 247                 i = j;
 248         }
 249
 250 #ifdef DCPOMATIC_DEBUG
 251         _last_video = piece->content;
 252 #endif
 253
 254         Video (pi, eyes, content->colour_conversion(), same, time);
 255
 256         _last_emit_was_black = false;
 257         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 258
 259         if (frc.repeat > 1 && !piece->repeating ()) {
 260                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 261         }
 262 }
 263
 264 void
 265 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 266 {
 267         shared_ptr<Piece> piece = weak_piece.lock ();
 268         if (!piece) {
 269                 return;
 270         }
 271
 272         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 273         assert (content);
 274
 275         /* Gain */
 276         if (content->audio_gain() != 0) {
 277                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 278                 gain->apply_gain (content->audio_gain ());
 279                 audio = gain;
 280         }
 281
 282         /* Resample */
 283         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 284                 shared_ptr<Resampler> r = resampler (content, true);
 285                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 286                 audio = ro.first;
 287                 frame = ro.second;
 288         }
 289
 290         Time const relative_time = _film->audio_frames_to_time (frame);
 291
 292         if (content->trimmed (relative_time)) {
 293                 return;
 294         }
 295
 296         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 297
 298         /* Remap channels */
 299         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 300         dcp_mapped->make_silent ();
 301
 302         AudioMapping map = content->audio_mapping ();
 303         for (int i = 0; i < map.content_channels(); ++i) {
 304                 for (int j = 0; j < _film->audio_channels(); ++j) {
 305                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 306                                 dcp_mapped->accumulate_channel (
 307                                         audio.get(),
 308                                         i,
 309                                         static_cast<libdcp::Channel> (j),
 310                                         map.get (i, static_cast<libdcp::Channel> (j))
 311                                         );
 312                         }
 313                 }
 314         }
 315
 316         audio = dcp_mapped;
 317
 318         /* We must cut off anything that comes before the start of all time */
 319         if (time < 0) {
 320                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 321                 if (frames >= audio->frames ()) {
 322                         return;
 323                 }
 324
 325                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 326                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 327
 328                 audio = trimmed;
 329                 time = 0;
 330         }
 331
 332         _audio_merger.push (audio, time);
 333         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 334 }
 335
 336 void
 337 Player::flush ()
 338 {
 339         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 340         if (_audio && tb.audio) {
 341                 Audio (tb.audio, tb.time);
 342                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 343         }
 344
 345         while (_video && _video_position < _audio_position) {
 346                 emit_black ();
 347         }
 348
 349         while (_audio && _audio_position < _video_position) {
 350                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 351         }
 352
 353 }
 354
 355 /** Seek so that the next pass() will yield (approximately) the requested frame.
 356  *  Pass accurate = true to try harder to get close to the request.
 357  *  @return true on error
 358  */
 359 void
 360 Player::seek (Time t, bool accurate)
 361 {
 362         if (!_have_valid_pieces) {
 363                 setup_pieces ();
 364         }
 365
 366         if (_pieces.empty ()) {
 367                 return;
 368         }
 369
 370         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 371                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 372                 if (!vc) {
 373                         continue;
 374                 }
 375
 376                 /* s is the offset of t from the start position of this content */
 377                 Time s = t - vc->position ();
 378                 s = max (static_cast<Time> (0), s);
 379                 s = min (vc->length_after_trim(), s);
 380
 381                 /* Hence set the piece positions to the `global' time */
 382                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 383
 384                 /* And seek the decoder */
 385                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 386                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 387                         );
 388
 389                 (*i)->reset_repeat ();
 390         }
 391
 392         _video_position = _audio_position = t;
 393
 394         /* XXX: don't seek audio because we don't need to... */
 395 }
 396
 397 void
 398 Player::setup_pieces ()
 399 {
 400         list<shared_ptr<Piece> > old_pieces = _pieces;
 401
 402         _pieces.clear ();
 403
 404         ContentList content = _playlist->content ();
 405         sort (content.begin(), content.end(), ContentSorter ());
 406
 407         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 408
 409                 if (!(*i)->paths_valid ()) {
 410                         continue;
 411                 }
 412
 413                 shared_ptr<Piece> piece (new Piece (*i));
 414
 415                 /* XXX: into content? */
 416
 417                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 418                 if (fc) {
 419                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 420
 421                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 422                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 423                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 424
 425                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 426                         piece->decoder = fd;
 427                 }
 428
 429                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 430                 if (ic) {
 431                         bool reusing = false;
 432
 433                         /* See if we can re-use an old ImageDecoder */
 434                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 435                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 436                                 if (imd && imd->content() == ic) {
 437                                         piece = *j;
 438                                         reusing = true;
 439                                 }
 440                         }
 441
 442                         if (!reusing) {
 443                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 444                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 445                                 piece->decoder = id;
 446                         }
 447                 }
 448
 449                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 450                 if (sc) {
 451                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 452                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 453
 454                         piece->decoder = sd;
 455                 }
 456
 457                 _pieces.push_back (piece);
 458         }
 459
 460         _have_valid_pieces = true;
 461 }
 462
 463 void
 464 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 465 {
 466         shared_ptr<Content> c = w.lock ();
 467         if (!c) {
 468                 return;
 469         }
 470
 471         if (
 472                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 473                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 474                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 475                 ) {
 476
 477                 _have_valid_pieces = false;
 478                 Changed (frequent);
 479
 480         } else if (
 481                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 482                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 483                 property == SubtitleContentProperty::SUBTITLE_SCALE
 484                 ) {
 485
 486                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 487                         i->update (_film, _video_container_size);
 488                 }
 489
 490                 Changed (frequent);
 491
 492         } else if (
 493                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 494                 property == VideoContentProperty::VIDEO_FRAME_RATE
 495                 ) {
 496
 497                 Changed (frequent);
 498
 499         } else if (property == ContentProperty::PATH) {
 500
 501                 _have_valid_pieces = false;
 502                 Changed (frequent);
 503         }
 504 }
 505
 506 void
 507 Player::playlist_changed ()
 508 {
 509         _have_valid_pieces = false;
 510         Changed (false);
 511 }
 512
 513 void
 514 Player::set_video_container_size (libdcp::Size s)
 515 {
 516         _video_container_size = s;
 517
 518         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 519         im->make_black ();
 520
 521         _black_frame.reset (
 522                 new PlayerImage (
 523                         im,
 524                         Crop(),
 525                         _video_container_size,
 526                         _video_container_size,
 527                         Scaler::from_id ("bicubic")
 528                         )
 529                 );
 530 }
 531
 532 shared_ptr<Resampler>
 533 Player::resampler (shared_ptr<AudioContent> c, bool create)
 534 {
 535         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 536         if (i != _resamplers.end ()) {
 537                 return i->second;
 538         }
 539
 540         if (!create) {
 541                 return shared_ptr<Resampler> ();
 542         }
 543
 544         _film->log()->log (
 545                 String::compose (
 546                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 547                         )
 548                 );
 549
 550         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 551         _resamplers[c] = r;
 552         return r;
 553 }
 554
 555 void
 556 Player::emit_black ()
 557 {
 558 #ifdef DCPOMATIC_DEBUG
 559         _last_video.reset ();
 560 #endif
 561
 562         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 563         _video_position += _film->video_frames_to_time (1);
 564         _last_emit_was_black = true;
 565 }
 566
 567 void
 568 Player::emit_silence (OutputAudioFrame most)
 569 {
 570         if (most == 0) {
 571                 return;
 572         }
 573
 574         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 575         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 576         silence->make_silent ();
 577         Audio (silence, _audio_position);
 578         _audio_position += _film->audio_frames_to_time (N);
 579 }
 580
 581 void
 582 Player::film_changed (Film::Property p)
 583 {
 584         /* Here we should notice Film properties that affect our output, and
 585            alert listeners that our output now would be different to how it was
 586            last time we were run.
 587         */
 588
 589         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 590                 Changed (false);
 591         }
 592 }
 593
 594 void
 595 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 596 {
 597         if (!image) {
 598                 /* A null image means that we should stop any current subtitles at `from' */
 599                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 600                         i->set_stop (from);
 601                 }
 602         } else {
 603                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 604         }
 605 }
 606
 607 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 608  *  @return false if this could not be done.
 609  */
 610 bool
 611 Player::repeat_last_video ()
 612 {
 613         if (!_last_incoming_video.image || !_have_valid_pieces) {
 614                 return false;
 615         }
 616
 617         process_video (
 618                 _last_incoming_video.weak_piece,
 619                 _last_incoming_video.image,
 620                 _last_incoming_video.eyes,
 621                 _last_incoming_video.same,
 622                 _last_incoming_video.frame,
 623                 _last_incoming_video.extra
 624                 );
 625
 626         return true;
 627 }
 628
 629 PlayerImage::PlayerImage (
 630         shared_ptr<const Image> in,
 631         Crop crop,
 632         libdcp::Size inter_size,
 633         libdcp::Size out_size,
 634         Scaler const * scaler
 635         )
 636         : _in (in)
 637         , _crop (crop)
 638         , _inter_size (inter_size)
 639         , _out_size (out_size)
 640         , _scaler (scaler)
 641 {
 642
 643 }
 644
 645 void
 646 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
 647 {
 648         _subtitle_image = image;
 649         _subtitle_position = pos;
 650 }
 651
 652 shared_ptr<Image>
 653 PlayerImage::image ()
 654 {
 655         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, PIX_FMT_RGB24, false);
 656
 657         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
 658
 659         if (_subtitle_image) {
 660                 out->alpha_blend (_subtitle_image, _subtitle_position);
 661         }
 662
 663         return out;
 664 }