src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "log.h"
  35 #include "scaler.h"
  36
  37 using std::list;
  38 using std::cout;
  39 using std::min;
  40 using std::max;
  41 using std::vector;
  42 using std::pair;
  43 using std::map;
  44 using boost::shared_ptr;
  45 using boost::weak_ptr;
  46 using boost::dynamic_pointer_cast;
  47 using boost::optional;
  48
  49 class Piece
  50 {
  51 public:
  52         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
  53                 : content (c)
  54                 , decoder (d)
  55                 , frc (f)
  56         {}
  57
  58         shared_ptr<Content> content;
  59         shared_ptr<Decoder> decoder;
  60         FrameRateChange frc;
  61 };
  62
  63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  64         : _film (f)
  65         , _playlist (p)
  66         , _video (true)
  67         , _audio (true)
  68         , _have_valid_pieces (false)
  69         , _video_position (0)
  70         , _audio_position (0)
  71         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  72         , _last_emit_was_black (false)
  73         , _just_did_inaccurate_seek (false)
  74         , _approximate_size (false)
  75 {
  76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
  80 }
  81
  82 void
  83 Player::disable_video ()
  84 {
  85         _video = false;
  86 }
  87
  88 void
  89 Player::disable_audio ()
  90 {
  91         _audio = false;
  92 }
  93
  94 bool
  95 Player::pass ()
  96 {
  97         if (!_have_valid_pieces) {
  98                 setup_pieces ();
  99         }
 100
 101         /* Interrogate all our pieces to find the one with the earliest decoded data */
 102
 103         shared_ptr<Piece> earliest_piece;
 104         shared_ptr<Decoded> earliest_decoded;
 105         DCPTime earliest_time = TIME_MAX;
 106         DCPTime earliest_audio = TIME_MAX;
 107
 108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 109
 110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
 111
 112                 if (dec) {
 113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
 114                 }
 115
 116                 if (dec && dec->dcp_time < earliest_time) {
 117                         earliest_piece = *i;
 118                         earliest_decoded = dec;
 119                         earliest_time = dec->dcp_time;
 120                 }
 121
 122                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
 123                         earliest_audio = dec->dcp_time;
 124                 }
 125         }
 126
 127         if (!earliest_piece) {
 128                 flush ();
 129                 return true;
 130         }
 131
 132         if (earliest_audio != TIME_MAX) {
 133                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
 134                 Audio (tb.audio, tb.time);
 135                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 136         }
 137
 138         /* Emit the earliest thing */
 139
 140         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
 141         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
 142         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
 143
 144         if (dv && _video) {
 145                 if (!_just_did_inaccurate_seek && earliest_time > _video_position) {
 146
 147                         /* See if we're inside some video content */
 148                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
 149                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
 150                                 ++i;
 151                         }
 152
 153                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
 154                                 /* We're outside all video content */
 155                                 emit_black ();
 156                         } else {
 157                                 _last_incoming_video.video->dcp_time = _video_position;
 158                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
 159                         }
 160                 } else {
 161                         emit_video (earliest_piece, dv);
 162                         earliest_piece->decoder->get ();
 163                 }
 164         } else if (da && _audio) {
 165                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
 166                         emit_silence (earliest_time - _audio_position);
 167                 } else {
 168                         emit_audio (earliest_piece, da);
 169                         earliest_piece->decoder->get ();
 170                 }
 171         } else if (ds && _video) {
 172                 _in_subtitle.piece = earliest_piece;
 173                 _in_subtitle.subtitle = ds;
 174                 update_subtitle ();
 175                 earliest_piece->decoder->get ();
 176         }
 177
 178         _just_did_inaccurate_seek = false;
 179
 180         return false;
 181 }
 182
 183 void
 184 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
 185 {
 186         /* Keep a note of what came in so that we can repeat it if required */
 187         _last_incoming_video.weak_piece = weak_piece;
 188         _last_incoming_video.video = video;
 189
 190         shared_ptr<Piece> piece = weak_piece.lock ();
 191         if (!piece) {
 192                 return;
 193         }
 194
 195         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 196         assert (content);
 197
 198         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
 199 #if 0
 200         XXX
 201         if (frc.skip && (frame % 2) == 1) {
 202                 return;
 203         }
 204 #endif
 205
 206         if (content->trimmed (video->dcp_time - content->position ())) {
 207                 return;
 208         }
 209
 210         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
 211         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
 212         if (_approximate_size) {
 213                 image_size.width &= ~3;
 214                 image_size.height &= ~3;
 215         }
 216
 217         shared_ptr<PlayerImage> pi (
 218                 new PlayerImage (
 219                         video->image,
 220                         content->crop(),
 221                         image_size,
 222                         _video_container_size,
 223                         _film->scaler()
 224                         )
 225                 );
 226
 227         if (
 228                 _film->with_subtitles () &&
 229                 _out_subtitle.subtitle->image &&
 230                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
 231                 ) {
 232
 233                 Position<int> const container_offset (
 234                         (_video_container_size.width - image_size.width) / 2,
 235                         (_video_container_size.height - image_size.width) / 2
 236                         );
 237
 238                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
 239         }
 240
 241 #ifdef DCPOMATIC_DEBUG
 242         _last_video = piece->content;
 243 #endif
 244
 245         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
 246
 247         _last_emit_was_black = false;
 248         _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
 249 }
 250
 251 void
 252 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
 253 {
 254         shared_ptr<Piece> piece = weak_piece.lock ();
 255         if (!piece) {
 256                 return;
 257         }
 258
 259         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 260         assert (content);
 261
 262         /* Gain */
 263         if (content->audio_gain() != 0) {
 264                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
 265                 gain->apply_gain (content->audio_gain ());
 266                 audio->data = gain;
 267         }
 268
 269         if (content->trimmed (audio->dcp_time - content->position ())) {
 270                 return;
 271         }
 272
 273         /* Remap channels */
 274         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
 275         dcp_mapped->make_silent ();
 276         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 277         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 278                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
 279                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
 280                 }
 281         }
 282
 283         audio->data = dcp_mapped;
 284
 285         /* Delay */
 286         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
 287         if (audio->dcp_time < 0) {
 288                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
 289                 if (frames >= audio->data->frames ()) {
 290                         return;
 291                 }
 292
 293                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
 294                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
 295
 296                 audio->data = trimmed;
 297                 audio->dcp_time = 0;
 298         }
 299
 300         _audio_merger.push (audio->data, audio->dcp_time);
 301 }
 302
 303 void
 304 Player::flush ()
 305 {
 306         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
 307         if (tb.audio) {
 308                 Audio (tb.audio, tb.time);
 309                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 310         }
 311
 312         while (_video_position < _audio_position) {
 313                 emit_black ();
 314         }
 315
 316         while (_audio_position < _video_position) {
 317                 emit_silence (_video_position - _audio_position);
 318         }
 319
 320 }
 321
 322 /** Seek so that the next pass() will yield (approximately) the requested frame.
 323  *  Pass accurate = true to try harder to get close to the request.
 324  *  @return true on error
 325  */
 326 void
 327 Player::seek (DCPTime t, bool accurate)
 328 {
 329         if (!_have_valid_pieces) {
 330                 setup_pieces ();
 331         }
 332
 333         if (_pieces.empty ()) {
 334                 return;
 335         }
 336
 337         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 338                 /* s is the offset of t from the start position of this content */
 339                 DCPTime s = t - (*i)->content->position ();
 340                 s = max (static_cast<DCPTime> (0), s);
 341                 s = min ((*i)->content->length_after_trim(), s);
 342
 343                 /* Convert this to the content time */
 344                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
 345
 346                 /* And seek the decoder */
 347                 (*i)->decoder->seek (ct, accurate);
 348         }
 349
 350         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
 351         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
 352
 353         _audio_merger.clear (_audio_position);
 354
 355         if (!accurate) {
 356                 /* We just did an inaccurate seek, so it's likely that the next thing seen
 357                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
 358                    this flag stops pass() from trying to fix that: we assume that if it
 359                    was an inaccurate seek then the caller does not care too much about
 360                    inserting black/silence to keep the time tidy.
 361                 */
 362                 _just_did_inaccurate_seek = true;
 363         }
 364 }
 365
 366 void
 367 Player::setup_pieces ()
 368 {
 369         list<shared_ptr<Piece> > old_pieces = _pieces;
 370         _pieces.clear ();
 371
 372         ContentList content = _playlist->content ();
 373
 374         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 375
 376                 shared_ptr<Decoder> decoder;
 377                 optional<FrameRateChange> frc;
 378
 379                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 380                 if (fc) {
 381                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
 382                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 383                 }
 384
 385                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 386                 if (ic) {
 387                         /* See if we can re-use an old ImageDecoder */
 388                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 389                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 390                                 if (imd && imd->content() == ic) {
 391                                         decoder = imd;
 392                                 }
 393                         }
 394
 395                         if (!decoder) {
 396                                 decoder.reset (new ImageDecoder (_film, ic));
 397                         }
 398
 399                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 400                 }
 401
 402                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 403                 if (sc) {
 404                         decoder.reset (new SndfileDecoder (_film, sc));
 405
 406                         /* Working out the frc for this content is a bit tricky: what if it overlaps
 407                            two pieces of video content with different frame rates?  For now, use
 408                            the one with the best overlap.
 409                         */
 410
 411                         DCPTime best_overlap_t = 0;
 412                         shared_ptr<VideoContent> best_overlap;
 413                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
 414                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
 415                                 if (!vc) {
 416                                         continue;
 417                                 }
 418
 419                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
 420                                 if (overlap > best_overlap_t) {
 421                                         best_overlap = vc;
 422                                         best_overlap_t = overlap;
 423                                 }
 424                         }
 425
 426                         if (best_overlap) {
 427                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 428                         } else {
 429                                 /* No video overlap; e.g. if the DCP is just audio */
 430                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 431                         }
 432                 }
 433
 434                 decoder->seek ((*i)->trim_start (), true);
 435
 436                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 437         }
 438
 439         _have_valid_pieces = true;
 440 }
 441
 442 void
 443 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 444 {
 445         shared_ptr<Content> c = w.lock ();
 446         if (!c) {
 447                 return;
 448         }
 449
 450         if (
 451                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 452                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 453                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 454                 ) {
 455
 456                 _have_valid_pieces = false;
 457                 Changed (frequent);
 458
 459         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 460
 461                 update_subtitle ();
 462                 Changed (frequent);
 463
 464         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
 465
 466                 Changed (frequent);
 467
 468         } else if (property == ContentProperty::PATH) {
 469
 470                 Changed (frequent);
 471         }
 472 }
 473
 474 void
 475 Player::playlist_changed ()
 476 {
 477         _have_valid_pieces = false;
 478         Changed (false);
 479 }
 480
 481 void
 482 Player::set_video_container_size (libdcp::Size s)
 483 {
 484         _video_container_size = s;
 485
 486         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 487         im->make_black ();
 488
 489         _black_frame.reset (
 490                 new PlayerImage (
 491                         im,
 492                         Crop(),
 493                         _video_container_size,
 494                         _video_container_size,
 495                         Scaler::from_id ("bicubic")
 496                         )
 497                 );
 498 }
 499
 500 void
 501 Player::emit_black ()
 502 {
 503 #ifdef DCPOMATIC_DEBUG
 504         _last_video.reset ();
 505 #endif
 506
 507         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 508         _video_position += _film->video_frames_to_time (1);
 509         _last_emit_was_black = true;
 510 }
 511
 512 void
 513 Player::emit_silence (DCPTime most)
 514 {
 515         if (most == 0) {
 516                 return;
 517         }
 518
 519         DCPTime t = min (most, TIME_HZ / 2);
 520         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
 521         silence->make_silent ();
 522         Audio (silence, _audio_position);
 523         _audio_position += t;
 524 }
 525
 526 void
 527 Player::film_changed (Film::Property p)
 528 {
 529         /* Here we should notice Film properties that affect our output, and
 530            alert listeners that our output now would be different to how it was
 531            last time we were run.
 532         */
 533
 534         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 535                 Changed (false);
 536         }
 537 }
 538
 539 void
 540 Player::update_subtitle ()
 541 {
 542         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 543         if (!piece) {
 544                 return;
 545         }
 546
 547         if (!_in_subtitle.subtitle->image) {
 548                 _out_subtitle.subtitle->image.reset ();
 549                 return;
 550         }
 551
 552         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 553         assert (sc);
 554
 555         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
 556         libdcp::Size scaled_size;
 557
 558         in_rect.y += sc->subtitle_offset ();
 559
 560         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 561         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 562         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 563
 564         /* Then we need a corrective translation, consisting of two parts:
 565          *
 566          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 567          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 568          *
 569          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 570          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 571          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 572          *
 573          * Combining these two translations gives these expressions.
 574          */
 575
 576         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 577         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 578
 579         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
 580                 scaled_size,
 581                 Scaler::from_id ("bicubic"),
 582                 _in_subtitle.subtitle->image->pixel_format (),
 583                 true
 584                 );
 585
 586         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
 587         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
 588 }
 589
 590 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 591  *  @return false if this could not be done.
 592  */
 593 bool
 594 Player::repeat_last_video ()
 595 {
 596         if (!_last_incoming_video.video || !_have_valid_pieces) {
 597                 return false;
 598         }
 599
 600         emit_video (
 601                 _last_incoming_video.weak_piece,
 602                 _last_incoming_video.video
 603                 );
 604
 605         return true;
 606 }
 607
 608 void
 609 Player::set_approximate_size ()
 610 {
 611         _approximate_size = true;
 612 }
 613
 614
 615 PlayerImage::PlayerImage (
 616         shared_ptr<const Image> in,
 617         Crop crop,
 618         libdcp::Size inter_size,
 619         libdcp::Size out_size,
 620         Scaler const * scaler
 621         )
 622         : _in (in)
 623         , _crop (crop)
 624         , _inter_size (inter_size)
 625         , _out_size (out_size)
 626         , _scaler (scaler)
 627 {
 628
 629 }
 630
 631 void
 632 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
 633 {
 634         _subtitle_image = image;
 635         _subtitle_position = pos;
 636 }
 637
 638 shared_ptr<Image>
 639 PlayerImage::image (AVPixelFormat format, bool aligned)
 640 {
 641         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, format, aligned);
 642
 643         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
 644
 645         if (_subtitle_image) {
 646                 out->alpha_blend (_subtitle_image, _subtitle_position);
 647         }
 648
 649         return out;
 650 }
 651