src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "resampler.h"
  35 #include "log.h"
  36 #include "scaler.h"
  37
  38 using std::list;
  39 using std::cout;
  40 using std::min;
  41 using std::max;
  42 using std::vector;
  43 using std::pair;
  44 using std::map;
  45 using boost::shared_ptr;
  46 using boost::weak_ptr;
  47 using boost::dynamic_pointer_cast;
  48
  49 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  50         : _film (f)
  51         , _playlist (p)
  52         , _video (true)
  53         , _audio (true)
  54         , _have_valid_pieces (false)
  55         , _video_position (0)
  56         , _audio_position (0)
  57         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  58         , _last_emit_was_black (false)
  59 {
  60         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  61         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  62         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  63         set_video_container_size (_film->frame_size ());
  64 }
  65
  66 void
  67 Player::disable_video ()
  68 {
  69         _video = false;
  70 }
  71
  72 void
  73 Player::disable_audio ()
  74 {
  75         _audio = false;
  76 }
  77
  78 bool
  79 Player::pass ()
  80 {
  81         if (!_have_valid_pieces) {
  82                 setup_pieces ();
  83         }
  84
  85         Time earliest_t = TIME_MAX;
  86         shared_ptr<Piece> earliest;
  87         enum {
  88                 VIDEO,
  89                 AUDIO
  90         } type = VIDEO;
  91
  92         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  93                 if ((*i)->decoder->done ()) {
  94                         continue;
  95                 }
  96
  97                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
  98                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
  99
 100                 if (_video && vd) {
 101                         if ((*i)->video_position < earliest_t) {
 102                                 earliest_t = (*i)->video_position;
 103                                 earliest = *i;
 104                                 type = VIDEO;
 105                         }
 106                 }
 107
 108                 if (_audio && ad && ad->has_audio ()) {
 109                         if ((*i)->audio_position < earliest_t) {
 110                                 earliest_t = (*i)->audio_position;
 111                                 earliest = *i;
 112                                 type = AUDIO;
 113                         }
 114                 }
 115         }
 116
 117         if (!earliest) {
 118                 flush ();
 119                 return true;
 120         }
 121
 122         switch (type) {
 123         case VIDEO:
 124                 if (earliest_t > _video_position) {
 125                         emit_black ();
 126                 } else {
 127                         if (earliest->repeating ()) {
 128                                 earliest->repeat (this);
 129                         } else {
 130                                 earliest->decoder->pass ();
 131                         }
 132                 }
 133                 break;
 134
 135         case AUDIO:
 136                 if (earliest_t > _audio_position) {
 137                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 138                 } else {
 139                         earliest->decoder->pass ();
 140
 141                         if (earliest->decoder->done()) {
 142                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 143                                 assert (ac);
 144                                 shared_ptr<Resampler> re = resampler (ac, false);
 145                                 if (re) {
 146                                         shared_ptr<const AudioBuffers> b = re->flush ();
 147                                         if (b->frames ()) {
 148                                                 process_audio (earliest, b, ac->audio_length ());
 149                                         }
 150                                 }
 151                         }
 152                 }
 153                 break;
 154         }
 155
 156         if (_audio) {
 157                 boost::optional<Time> audio_done_up_to;
 158                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 159                         if ((*i)->decoder->done ()) {
 160                                 continue;
 161                         }
 162
 163                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 164                         if (ad && ad->has_audio ()) {
 165                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 166                         }
 167                 }
 168
 169                 if (audio_done_up_to) {
 170                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 171                         Audio (tb.audio, tb.time);
 172                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 173                 }
 174         }
 175
 176         return false;
 177 }
 178
 179 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 180 void
 181 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
 182 {
 183         /* Keep a note of what came in so that we can repeat it if required */
 184         _last_incoming_video.weak_piece = weak_piece;
 185         _last_incoming_video.image = image;
 186         _last_incoming_video.eyes = eyes;
 187         _last_incoming_video.same = same;
 188         _last_incoming_video.frame = frame;
 189         _last_incoming_video.extra = extra;
 190
 191         shared_ptr<Piece> piece = weak_piece.lock ();
 192         if (!piece) {
 193                 return;
 194         }
 195
 196         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 197         assert (content);
 198
 199         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 200         if (frc.skip && (frame % 2) == 1) {
 201                 return;
 202         }
 203
 204         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 205         if (content->trimmed (relative_time)) {
 206                 return;
 207         }
 208
 209         Time const time = content->position() + relative_time + extra - content->trim_start ();
 210         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 211
 212         shared_ptr<PlayerImage> pi (
 213                 new PlayerImage (
 214                         image,
 215                         content->crop(),
 216                         image_size,
 217                         _video_container_size,
 218                         _film->scaler()
 219                         )
 220                 );
 221
 222         if (_film->with_subtitles () && _subtitle && _subtitle->out_image() && _subtitle->covers (time)) {
 223
 224                 Position<int> const container_offset (
 225                         (_video_container_size.width - image_size.width) / 2,
 226                         (_video_container_size.height - image_size.width) / 2
 227                         );
 228
 229                 pi->set_subtitle (_subtitle->out_image(), _subtitle->out_position() + container_offset);
 230         }
 231
 232
 233 #ifdef DCPOMATIC_DEBUG
 234         _last_video = piece->content;
 235 #endif
 236
 237         Video (pi, eyes, content->colour_conversion(), same, time);
 238
 239         _last_emit_was_black = false;
 240         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 241
 242         if (frc.repeat > 1 && !piece->repeating ()) {
 243                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 244         }
 245 }
 246
 247 void
 248 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 249 {
 250         shared_ptr<Piece> piece = weak_piece.lock ();
 251         if (!piece) {
 252                 return;
 253         }
 254
 255         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 256         assert (content);
 257
 258         /* Gain */
 259         if (content->audio_gain() != 0) {
 260                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 261                 gain->apply_gain (content->audio_gain ());
 262                 audio = gain;
 263         }
 264
 265         /* Resample */
 266         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 267                 shared_ptr<Resampler> r = resampler (content, true);
 268                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 269                 audio = ro.first;
 270                 frame = ro.second;
 271         }
 272
 273         Time const relative_time = _film->audio_frames_to_time (frame);
 274
 275         if (content->trimmed (relative_time)) {
 276                 return;
 277         }
 278
 279         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 280
 281         /* Remap channels */
 282         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 283         dcp_mapped->make_silent ();
 284
 285         AudioMapping map = content->audio_mapping ();
 286         for (int i = 0; i < map.content_channels(); ++i) {
 287                 for (int j = 0; j < _film->audio_channels(); ++j) {
 288                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 289                                 dcp_mapped->accumulate_channel (
 290                                         audio.get(),
 291                                         i,
 292                                         static_cast<libdcp::Channel> (j),
 293                                         map.get (i, static_cast<libdcp::Channel> (j))
 294                                         );
 295                         }
 296                 }
 297         }
 298
 299         audio = dcp_mapped;
 300
 301         /* We must cut off anything that comes before the start of all time */
 302         if (time < 0) {
 303                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 304                 if (frames >= audio->frames ()) {
 305                         return;
 306                 }
 307
 308                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 309                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 310
 311                 audio = trimmed;
 312                 time = 0;
 313         }
 314
 315         _audio_merger.push (audio, time);
 316         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 317 }
 318
 319 void
 320 Player::flush ()
 321 {
 322         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 323         if (_audio && tb.audio) {
 324                 Audio (tb.audio, tb.time);
 325                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 326         }
 327
 328         while (_video && _video_position < _audio_position) {
 329                 emit_black ();
 330         }
 331
 332         while (_audio && _audio_position < _video_position) {
 333                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 334         }
 335
 336 }
 337
 338 /** Seek so that the next pass() will yield (approximately) the requested frame.
 339  *  Pass accurate = true to try harder to get close to the request.
 340  *  @return true on error
 341  */
 342 void
 343 Player::seek (Time t, bool accurate)
 344 {
 345         if (!_have_valid_pieces) {
 346                 setup_pieces ();
 347         }
 348
 349         if (_pieces.empty ()) {
 350                 return;
 351         }
 352
 353         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 354                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 355                 if (!vc) {
 356                         continue;
 357                 }
 358
 359                 /* s is the offset of t from the start position of this content */
 360                 Time s = t - vc->position ();
 361                 s = max (static_cast<Time> (0), s);
 362                 s = min (vc->length_after_trim(), s);
 363
 364                 /* Hence set the piece positions to the `global' time */
 365                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 366
 367                 /* And seek the decoder */
 368                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 369                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 370                         );
 371
 372                 (*i)->reset_repeat ();
 373         }
 374
 375         _video_position = _audio_position = t;
 376
 377         /* XXX: don't seek audio because we don't need to... */
 378 }
 379
 380 void
 381 Player::setup_pieces ()
 382 {
 383         list<shared_ptr<Piece> > old_pieces = _pieces;
 384
 385         _pieces.clear ();
 386
 387         ContentList content = _playlist->content ();
 388         sort (content.begin(), content.end(), ContentSorter ());
 389
 390         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 391
 392                 if (!(*i)->paths_valid ()) {
 393                         continue;
 394                 }
 395
 396                 shared_ptr<Piece> piece (new Piece (*i));
 397
 398                 /* XXX: into content? */
 399
 400                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 401                 if (fc) {
 402                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 403
 404                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 405                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 406                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 407
 408                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 409                         piece->decoder = fd;
 410                 }
 411
 412                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 413                 if (ic) {
 414                         bool reusing = false;
 415
 416                         /* See if we can re-use an old ImageDecoder */
 417                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 418                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 419                                 if (imd && imd->content() == ic) {
 420                                         piece = *j;
 421                                         reusing = true;
 422                                 }
 423                         }
 424
 425                         if (!reusing) {
 426                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 427                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 428                                 piece->decoder = id;
 429                         }
 430                 }
 431
 432                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 433                 if (sc) {
 434                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 435                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 436
 437                         piece->decoder = sd;
 438                 }
 439
 440                 _pieces.push_back (piece);
 441         }
 442
 443         _have_valid_pieces = true;
 444 }
 445
 446 void
 447 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 448 {
 449         shared_ptr<Content> c = w.lock ();
 450         if (!c) {
 451                 return;
 452         }
 453
 454         if (
 455                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 456                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 457                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 458                 ) {
 459
 460                 _have_valid_pieces = false;
 461                 Changed (frequent);
 462
 463         } else if (
 464                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 465                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 466                 property == SubtitleContentProperty::SUBTITLE_SCALE
 467                 ) {
 468
 469                 if (_subtitle) {
 470                         _subtitle->update (_film, _video_container_size);
 471                 }
 472                 Changed (frequent);
 473
 474         } else if (
 475                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 476                 property == VideoContentProperty::VIDEO_FRAME_RATE
 477                 ) {
 478
 479                 Changed (frequent);
 480
 481         } else if (property == ContentProperty::PATH) {
 482
 483                 _have_valid_pieces = false;
 484                 Changed (frequent);
 485         }
 486 }
 487
 488 void
 489 Player::playlist_changed ()
 490 {
 491         _have_valid_pieces = false;
 492         Changed (false);
 493 }
 494
 495 void
 496 Player::set_video_container_size (libdcp::Size s)
 497 {
 498         _video_container_size = s;
 499
 500         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 501         im->make_black ();
 502
 503         _black_frame.reset (
 504                 new PlayerImage (
 505                         im,
 506                         Crop(),
 507                         _video_container_size,
 508                         _video_container_size,
 509                         Scaler::from_id ("bicubic")
 510                         )
 511                 );
 512 }
 513
 514 shared_ptr<Resampler>
 515 Player::resampler (shared_ptr<AudioContent> c, bool create)
 516 {
 517         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 518         if (i != _resamplers.end ()) {
 519                 return i->second;
 520         }
 521
 522         if (!create) {
 523                 return shared_ptr<Resampler> ();
 524         }
 525
 526         _film->log()->log (
 527                 String::compose (
 528                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 529                         )
 530                 );
 531
 532         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 533         _resamplers[c] = r;
 534         return r;
 535 }
 536
 537 void
 538 Player::emit_black ()
 539 {
 540 #ifdef DCPOMATIC_DEBUG
 541         _last_video.reset ();
 542 #endif
 543
 544         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 545         _video_position += _film->video_frames_to_time (1);
 546         _last_emit_was_black = true;
 547 }
 548
 549 void
 550 Player::emit_silence (OutputAudioFrame most)
 551 {
 552         if (most == 0) {
 553                 return;
 554         }
 555
 556         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 557         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 558         silence->make_silent ();
 559         Audio (silence, _audio_position);
 560         _audio_position += _film->audio_frames_to_time (N);
 561 }
 562
 563 void
 564 Player::film_changed (Film::Property p)
 565 {
 566         /* Here we should notice Film properties that affect our output, and
 567            alert listeners that our output now would be different to how it was
 568            last time we were run.
 569         */
 570
 571         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 572                 Changed (false);
 573         }
 574 }
 575
 576 void
 577 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 578 {
 579         _subtitle = Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to);
 580 }
 581
 582 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 583  *  @return false if this could not be done.
 584  */
 585 bool
 586 Player::repeat_last_video ()
 587 {
 588         if (!_last_incoming_video.image || !_have_valid_pieces) {
 589                 return false;
 590         }
 591
 592         process_video (
 593                 _last_incoming_video.weak_piece,
 594                 _last_incoming_video.image,
 595                 _last_incoming_video.eyes,
 596                 _last_incoming_video.same,
 597                 _last_incoming_video.frame,
 598                 _last_incoming_video.extra
 599                 );
 600
 601         return true;
 602 }
 603
 604 PlayerImage::PlayerImage (
 605         shared_ptr<const Image> in,
 606         Crop crop,
 607         libdcp::Size inter_size,
 608         libdcp::Size out_size,
 609         Scaler const * scaler
 610         )
 611         : _in (in)
 612         , _crop (crop)
 613         , _inter_size (inter_size)
 614         , _out_size (out_size)
 615         , _scaler (scaler)
 616 {
 617
 618 }
 619
 620 void
 621 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
 622 {
 623         _subtitle_image = image;
 624         _subtitle_position = pos;
 625 }
 626
 627 shared_ptr<Image>
 628 PlayerImage::image ()
 629 {
 630         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, PIX_FMT_RGB24, false);
 631
 632         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
 633
 634         if (_subtitle_image) {
 635                 out->alpha_blend (_subtitle_image, _subtitle_position);
 636         }
 637
 638         return out;
 639 }