src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "resampler.h"
  35 #include "log.h"
  36 #include "scaler.h"
  37 #include "player_video_frame.h"
  38
  39 using std::list;
  40 using std::cout;
  41 using std::min;
  42 using std::max;
  43 using std::vector;
  44 using std::pair;
  45 using std::map;
  46 using boost::shared_ptr;
  47 using boost::weak_ptr;
  48 using boost::dynamic_pointer_cast;
  49
  50 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  51         : _film (f)
  52         , _playlist (p)
  53         , _video (true)
  54         , _audio (true)
  55         , _have_valid_pieces (false)
  56         , _video_position (0)
  57         , _audio_position (0)
  58         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  59         , _last_emit_was_black (false)
  60 {
  61         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  62         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  63         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  64         set_video_container_size (_film->frame_size ());
  65 }
  66
  67 void
  68 Player::disable_video ()
  69 {
  70         _video = false;
  71 }
  72
  73 void
  74 Player::disable_audio ()
  75 {
  76         _audio = false;
  77 }
  78
  79 bool
  80 Player::pass ()
  81 {
  82         if (!_have_valid_pieces) {
  83                 setup_pieces ();
  84         }
  85
  86         Time earliest_t = TIME_MAX;
  87         shared_ptr<Piece> earliest;
  88         enum {
  89                 VIDEO,
  90                 AUDIO
  91         } type = VIDEO;
  92
  93         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  94                 if ((*i)->decoder->done ()) {
  95                         continue;
  96                 }
  97
  98                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
  99                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 100
 101                 if (_video && vd) {
 102                         if ((*i)->video_position < earliest_t) {
 103                                 earliest_t = (*i)->video_position;
 104                                 earliest = *i;
 105                                 type = VIDEO;
 106                         }
 107                 }
 108
 109                 if (_audio && ad && ad->has_audio ()) {
 110                         if ((*i)->audio_position < earliest_t) {
 111                                 earliest_t = (*i)->audio_position;
 112                                 earliest = *i;
 113                                 type = AUDIO;
 114                         }
 115                 }
 116         }
 117
 118         if (!earliest) {
 119                 flush ();
 120                 return true;
 121         }
 122
 123         switch (type) {
 124         case VIDEO:
 125                 if (earliest_t > _video_position) {
 126                         emit_black ();
 127                 } else {
 128                         if (earliest->repeating ()) {
 129                                 earliest->repeat (this);
 130                         } else {
 131                                 earliest->decoder->pass ();
 132                         }
 133                 }
 134                 break;
 135
 136         case AUDIO:
 137                 if (earliest_t > _audio_position) {
 138                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 139                 } else {
 140                         earliest->decoder->pass ();
 141
 142                         if (earliest->decoder->done()) {
 143                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 144                                 assert (ac);
 145                                 shared_ptr<Resampler> re = resampler (ac, false);
 146                                 if (re) {
 147                                         shared_ptr<const AudioBuffers> b = re->flush ();
 148                                         if (b->frames ()) {
 149                                                 process_audio (earliest, b, ac->audio_length ());
 150                                         }
 151                                 }
 152                         }
 153                 }
 154                 break;
 155         }
 156
 157         if (_audio) {
 158                 boost::optional<Time> audio_done_up_to;
 159                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 160                         if ((*i)->decoder->done ()) {
 161                                 continue;
 162                         }
 163
 164                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 165                         if (ad && ad->has_audio ()) {
 166                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 167                         }
 168                 }
 169
 170                 if (audio_done_up_to) {
 171                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 172                         Audio (tb.audio, tb.time);
 173                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 174                 }
 175         }
 176
 177         return false;
 178 }
 179
 180 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 181 void
 182 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
 183 {
 184         /* Keep a note of what came in so that we can repeat it if required */
 185         _last_incoming_video.weak_piece = weak_piece;
 186         _last_incoming_video.image = image;
 187         _last_incoming_video.eyes = eyes;
 188         _last_incoming_video.same = same;
 189         _last_incoming_video.frame = frame;
 190         _last_incoming_video.extra = extra;
 191
 192         shared_ptr<Piece> piece = weak_piece.lock ();
 193         if (!piece) {
 194                 return;
 195         }
 196
 197         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 198         assert (content);
 199
 200         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 201         if (frc.skip && (frame % 2) == 1) {
 202                 return;
 203         }
 204
 205         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 206         if (content->trimmed (relative_time)) {
 207                 return;
 208         }
 209
 210         Time const time = content->position() + relative_time + extra - content->trim_start ();
 211         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 212
 213         shared_ptr<PlayerVideoFrame> pi (
 214                 new PlayerVideoFrame (
 215                         image,
 216                         content->crop(),
 217                         image_size,
 218                         _video_container_size,
 219                         _film->scaler(),
 220                         eyes,
 221                         content->colour_conversion()
 222                         )
 223                 );
 224
 225         if (_film->with_subtitles ()) {
 226                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 227                         if (i->covers (time)) {
 228                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 229                                    one is the one we want to use, so that's ok.
 230                                 */
 231                                 Position<int> const container_offset (
 232                                         (_video_container_size.width - image_size.width) / 2,
 233                                         (_video_container_size.height - image_size.width) / 2
 234                                         );
 235
 236                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 237                         }
 238                 }
 239         }
 240
 241         /* Clear out old subtitles */
 242         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 243                 list<Subtitle>::iterator j = i;
 244                 ++j;
 245
 246                 if (i->ends_before (time)) {
 247                         _subtitles.erase (i);
 248                 }
 249
 250                 i = j;
 251         }
 252
 253 #ifdef DCPOMATIC_DEBUG
 254         _last_video = piece->content;
 255 #endif
 256
 257         Video (pi, same, time);
 258
 259         _last_emit_was_black = false;
 260         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 261
 262         if (frc.repeat > 1 && !piece->repeating ()) {
 263                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 264         }
 265 }
 266
 267 void
 268 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 269 {
 270         shared_ptr<Piece> piece = weak_piece.lock ();
 271         if (!piece) {
 272                 return;
 273         }
 274
 275         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 276         assert (content);
 277
 278         /* Gain */
 279         if (content->audio_gain() != 0) {
 280                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 281                 gain->apply_gain (content->audio_gain ());
 282                 audio = gain;
 283         }
 284
 285         /* Resample */
 286         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 287                 shared_ptr<Resampler> r = resampler (content, true);
 288                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 289                 audio = ro.first;
 290                 frame = ro.second;
 291         }
 292
 293         Time const relative_time = _film->audio_frames_to_time (frame);
 294
 295         if (content->trimmed (relative_time)) {
 296                 return;
 297         }
 298
 299         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 300
 301         /* Remap channels */
 302         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 303         dcp_mapped->make_silent ();
 304
 305         AudioMapping map = content->audio_mapping ();
 306         for (int i = 0; i < map.content_channels(); ++i) {
 307                 for (int j = 0; j < _film->audio_channels(); ++j) {
 308                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 309                                 dcp_mapped->accumulate_channel (
 310                                         audio.get(),
 311                                         i,
 312                                         static_cast<libdcp::Channel> (j),
 313                                         map.get (i, static_cast<libdcp::Channel> (j))
 314                                         );
 315                         }
 316                 }
 317         }
 318
 319         audio = dcp_mapped;
 320
 321         /* We must cut off anything that comes before the start of all time */
 322         if (time < 0) {
 323                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 324                 if (frames >= audio->frames ()) {
 325                         return;
 326                 }
 327
 328                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 329                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 330
 331                 audio = trimmed;
 332                 time = 0;
 333         }
 334
 335         _audio_merger.push (audio, time);
 336         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 337 }
 338
 339 void
 340 Player::flush ()
 341 {
 342         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 343         if (_audio && tb.audio) {
 344                 Audio (tb.audio, tb.time);
 345                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 346         }
 347
 348         while (_video && _video_position < _audio_position) {
 349                 emit_black ();
 350         }
 351
 352         while (_audio && _audio_position < _video_position) {
 353                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 354         }
 355
 356 }
 357
 358 /** Seek so that the next pass() will yield (approximately) the requested frame.
 359  *  Pass accurate = true to try harder to get close to the request.
 360  *  @return true on error
 361  */
 362 void
 363 Player::seek (Time t, bool accurate)
 364 {
 365         if (!_have_valid_pieces) {
 366                 setup_pieces ();
 367         }
 368
 369         if (_pieces.empty ()) {
 370                 return;
 371         }
 372
 373         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 374                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 375                 if (!vc) {
 376                         continue;
 377                 }
 378
 379                 /* s is the offset of t from the start position of this content */
 380                 Time s = t - vc->position ();
 381                 s = max (static_cast<Time> (0), s);
 382                 s = min (vc->length_after_trim(), s);
 383
 384                 /* Hence set the piece positions to the `global' time */
 385                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 386
 387                 /* And seek the decoder */
 388                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 389                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 390                         );
 391
 392                 (*i)->reset_repeat ();
 393         }
 394
 395         _video_position = _audio_position = t;
 396
 397         /* XXX: don't seek audio because we don't need to... */
 398 }
 399
 400 void
 401 Player::setup_pieces ()
 402 {
 403         list<shared_ptr<Piece> > old_pieces = _pieces;
 404
 405         _pieces.clear ();
 406
 407         ContentList content = _playlist->content ();
 408         sort (content.begin(), content.end(), ContentSorter ());
 409
 410         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 411
 412                 if (!(*i)->paths_valid ()) {
 413                         continue;
 414                 }
 415
 416                 shared_ptr<Piece> piece (new Piece (*i));
 417
 418                 /* XXX: into content? */
 419
 420                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 421                 if (fc) {
 422                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 423
 424                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 425                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 426                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 427
 428                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 429                         piece->decoder = fd;
 430                 }
 431
 432                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 433                 if (ic) {
 434                         bool reusing = false;
 435
 436                         /* See if we can re-use an old ImageDecoder */
 437                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 438                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 439                                 if (imd && imd->content() == ic) {
 440                                         piece = *j;
 441                                         reusing = true;
 442                                 }
 443                         }
 444
 445                         if (!reusing) {
 446                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 447                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 448                                 piece->decoder = id;
 449                         }
 450                 }
 451
 452                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 453                 if (sc) {
 454                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 455                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 456
 457                         piece->decoder = sd;
 458                 }
 459
 460                 _pieces.push_back (piece);
 461         }
 462
 463         _have_valid_pieces = true;
 464 }
 465
 466 void
 467 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 468 {
 469         shared_ptr<Content> c = w.lock ();
 470         if (!c) {
 471                 return;
 472         }
 473
 474         if (
 475                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 476                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 477                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 478                 ) {
 479
 480                 _have_valid_pieces = false;
 481                 Changed (frequent);
 482
 483         } else if (
 484                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 485                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 486                 property == SubtitleContentProperty::SUBTITLE_SCALE
 487                 ) {
 488
 489                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 490                         i->update (_film, _video_container_size);
 491                 }
 492
 493                 Changed (frequent);
 494
 495         } else if (
 496                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 497                 property == VideoContentProperty::VIDEO_FRAME_RATE
 498                 ) {
 499
 500                 Changed (frequent);
 501
 502         } else if (property == ContentProperty::PATH) {
 503
 504                 _have_valid_pieces = false;
 505                 Changed (frequent);
 506         }
 507 }
 508
 509 void
 510 Player::playlist_changed ()
 511 {
 512         _have_valid_pieces = false;
 513         Changed (false);
 514 }
 515
 516 void
 517 Player::set_video_container_size (libdcp::Size s)
 518 {
 519         _video_container_size = s;
 520
 521         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 522         im->make_black ();
 523
 524         _black_frame.reset (
 525                 new PlayerVideoFrame (
 526                         im,
 527                         Crop(),
 528                         _video_container_size,
 529                         _video_container_size,
 530                         Scaler::from_id ("bicubic"),
 531                         EYES_BOTH,
 532                         ColourConversion ()
 533                         )
 534                 );
 535 }
 536
 537 shared_ptr<Resampler>
 538 Player::resampler (shared_ptr<AudioContent> c, bool create)
 539 {
 540         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 541         if (i != _resamplers.end ()) {
 542                 return i->second;
 543         }
 544
 545         if (!create) {
 546                 return shared_ptr<Resampler> ();
 547         }
 548
 549         _film->log()->log (
 550                 String::compose (
 551                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 552                         )
 553                 );
 554
 555         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 556         _resamplers[c] = r;
 557         return r;
 558 }
 559
 560 void
 561 Player::emit_black ()
 562 {
 563 #ifdef DCPOMATIC_DEBUG
 564         _last_video.reset ();
 565 #endif
 566
 567         Video (_black_frame, _last_emit_was_black, _video_position);
 568         _video_position += _film->video_frames_to_time (1);
 569         _last_emit_was_black = true;
 570 }
 571
 572 void
 573 Player::emit_silence (OutputAudioFrame most)
 574 {
 575         if (most == 0) {
 576                 return;
 577         }
 578
 579         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 580         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 581         silence->make_silent ();
 582         Audio (silence, _audio_position);
 583         _audio_position += _film->audio_frames_to_time (N);
 584 }
 585
 586 void
 587 Player::film_changed (Film::Property p)
 588 {
 589         /* Here we should notice Film properties that affect our output, and
 590            alert listeners that our output now would be different to how it was
 591            last time we were run.
 592         */
 593
 594         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 595                 Changed (false);
 596         }
 597 }
 598
 599 void
 600 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 601 {
 602         if (!image) {
 603                 /* A null image means that we should stop any current subtitles at `from' */
 604                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 605                         i->set_stop (from);
 606                 }
 607         } else {
 608                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 609         }
 610 }
 611
 612 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 613  *  @return false if this could not be done.
 614  */
 615 bool
 616 Player::repeat_last_video ()
 617 {
 618         if (!_last_incoming_video.image || !_have_valid_pieces) {
 619                 return false;
 620         }
 621
 622         process_video (
 623                 _last_incoming_video.weak_piece,
 624                 _last_incoming_video.image,
 625                 _last_incoming_video.eyes,
 626                 _last_incoming_video.same,
 627                 _last_incoming_video.frame,
 628                 _last_incoming_video.extra
 629                 );
 630
 631         return true;
 632 }