src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "log.h"
  35 #include "scaler.h"
  36
  37 using std::list;
  38 using std::cout;
  39 using std::min;
  40 using std::max;
  41 using std::vector;
  42 using std::pair;
  43 using std::map;
  44 using boost::shared_ptr;
  45 using boost::weak_ptr;
  46 using boost::dynamic_pointer_cast;
  47 using boost::optional;
  48
  49 class Piece
  50 {
  51 public:
  52         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
  53                 : content (c)
  54                 , decoder (d)
  55                 , frc (f)
  56         {}
  57
  58         shared_ptr<Content> content;
  59         shared_ptr<Decoder> decoder;
  60         FrameRateChange frc;
  61 };
  62
  63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  64         : _film (f)
  65         , _playlist (p)
  66         , _video (true)
  67         , _audio (true)
  68         , _have_valid_pieces (false)
  69         , _video_position (0)
  70         , _audio_position (0)
  71         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  72         , _last_emit_was_black (false)
  73         , _just_did_inaccurate_seek (false)
  74 {
  75         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  76         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  77         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  78         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
  79 }
  80
  81 void
  82 Player::disable_video ()
  83 {
  84         _video = false;
  85 }
  86
  87 void
  88 Player::disable_audio ()
  89 {
  90         _audio = false;
  91 }
  92
  93 bool
  94 Player::pass ()
  95 {
  96         if (!_have_valid_pieces) {
  97                 setup_pieces ();
  98         }
  99
 100         /* Interrogate all our pieces to find the one with the earliest decoded data */
 101
 102         shared_ptr<Piece> earliest_piece;
 103         shared_ptr<Decoded> earliest_decoded;
 104         DCPTime earliest_time = TIME_MAX;
 105         DCPTime earliest_audio = TIME_MAX;
 106
 107         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 108
 109                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
 110
 111                 if (dec) {
 112                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
 113                 }
 114
 115                 if (dec && dec->dcp_time < earliest_time) {
 116                         earliest_piece = *i;
 117                         earliest_decoded = dec;
 118                         earliest_time = dec->dcp_time;
 119                 }
 120
 121                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
 122                         earliest_audio = dec->dcp_time;
 123                 }
 124         }
 125
 126         if (!earliest_piece) {
 127                 flush ();
 128                 return true;
 129         }
 130
 131         if (earliest_audio != TIME_MAX) {
 132                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
 133                 Audio (tb.audio, tb.time);
 134                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 135         }
 136
 137         /* Emit the earliest thing */
 138
 139         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
 140         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
 141         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
 142
 143         if (dv && _video) {
 144                 if (!_just_did_inaccurate_seek && earliest_time > _video_position) {
 145
 146                         /* See if we're inside some video content */
 147                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
 148                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
 149                                 ++i;
 150                         }
 151
 152                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
 153                                 /* We're outside all video content */
 154                                 emit_black ();
 155                         } else {
 156                                 _last_incoming_video.video->dcp_time = _video_position;
 157                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
 158                         }
 159                 } else {
 160                         emit_video (earliest_piece, dv);
 161                         earliest_piece->decoder->get ();
 162                 }
 163         } else if (da && _audio) {
 164                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
 165                         emit_silence (earliest_time - _audio_position);
 166                 } else {
 167                         emit_audio (earliest_piece, da);
 168                         earliest_piece->decoder->get ();
 169                 }
 170         } else if (ds && _video) {
 171                 _in_subtitle.piece = earliest_piece;
 172                 _in_subtitle.subtitle = ds;
 173                 update_subtitle ();
 174                 earliest_piece->decoder->get ();
 175         }
 176
 177         _just_did_inaccurate_seek = false;
 178
 179         return false;
 180 }
 181
 182 void
 183 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
 184 {
 185         /* Keep a note of what came in so that we can repeat it if required */
 186         _last_incoming_video.weak_piece = weak_piece;
 187         _last_incoming_video.video = video;
 188
 189         shared_ptr<Piece> piece = weak_piece.lock ();
 190         if (!piece) {
 191                 return;
 192         }
 193
 194         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 195         assert (content);
 196
 197         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
 198 #if 0
 199         XXX
 200         if (frc.skip && (frame % 2) == 1) {
 201                 return;
 202         }
 203 #endif
 204
 205         if (content->trimmed (video->dcp_time - content->position ())) {
 206                 return;
 207         }
 208
 209         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
 210         libdcp::Size const image_size = fit_ratio_within (ratio, _video_container_size);
 211
 212         shared_ptr<PlayerImage> pi (
 213                 new PlayerImage (
 214                         video->image,
 215                         content->crop(),
 216                         image_size,
 217                         _video_container_size,
 218                         _film->scaler()
 219                         )
 220                 );
 221
 222         if (
 223                 _film->with_subtitles () &&
 224                 _out_subtitle.subtitle->image &&
 225                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
 226                 ) {
 227
 228                 Position<int> const container_offset (
 229                         (_video_container_size.width - image_size.width) / 2,
 230                         (_video_container_size.height - image_size.width) / 2
 231                         );
 232
 233                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
 234         }
 235
 236 #ifdef DCPOMATIC_DEBUG
 237         _last_video = piece->content;
 238 #endif
 239
 240         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
 241
 242         _last_emit_was_black = false;
 243         _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
 244 }
 245
 246 void
 247 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
 248 {
 249         shared_ptr<Piece> piece = weak_piece.lock ();
 250         if (!piece) {
 251                 return;
 252         }
 253
 254         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 255         assert (content);
 256
 257         /* Gain */
 258         if (content->audio_gain() != 0) {
 259                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
 260                 gain->apply_gain (content->audio_gain ());
 261                 audio->data = gain;
 262         }
 263
 264         if (content->trimmed (audio->dcp_time - content->position ())) {
 265                 return;
 266         }
 267
 268         /* Remap channels */
 269         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
 270         dcp_mapped->make_silent ();
 271         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 272         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 273                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
 274                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
 275                 }
 276         }
 277
 278         audio->data = dcp_mapped;
 279
 280         /* Delay */
 281         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
 282         if (audio->dcp_time < 0) {
 283                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
 284                 if (frames >= audio->data->frames ()) {
 285                         return;
 286                 }
 287
 288                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
 289                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
 290
 291                 audio->data = trimmed;
 292                 audio->dcp_time = 0;
 293         }
 294
 295         _audio_merger.push (audio->data, audio->dcp_time);
 296 }
 297
 298 void
 299 Player::flush ()
 300 {
 301         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
 302         if (tb.audio) {
 303                 Audio (tb.audio, tb.time);
 304                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 305         }
 306
 307         while (_video_position < _audio_position) {
 308                 emit_black ();
 309         }
 310
 311         while (_audio_position < _video_position) {
 312                 emit_silence (_video_position - _audio_position);
 313         }
 314
 315 }
 316
 317 /** Seek so that the next pass() will yield (approximately) the requested frame.
 318  *  Pass accurate = true to try harder to get close to the request.
 319  *  @return true on error
 320  */
 321 void
 322 Player::seek (DCPTime t, bool accurate)
 323 {
 324         if (!_have_valid_pieces) {
 325                 setup_pieces ();
 326         }
 327
 328         if (_pieces.empty ()) {
 329                 return;
 330         }
 331
 332         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 333                 /* s is the offset of t from the start position of this content */
 334                 DCPTime s = t - (*i)->content->position ();
 335                 s = max (static_cast<DCPTime> (0), s);
 336                 s = min ((*i)->content->length_after_trim(), s);
 337
 338                 /* Convert this to the content time */
 339                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
 340
 341                 /* And seek the decoder */
 342                 (*i)->decoder->seek (ct, accurate);
 343         }
 344
 345         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
 346         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
 347
 348         _audio_merger.clear (_audio_position);
 349
 350         if (!accurate) {
 351                 /* We just did an inaccurate seek, so it's likely that the next thing seen
 352                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
 353                    this flag stops pass() from trying to fix that: we assume that if it
 354                    was an inaccurate seek then the caller does not care too much about
 355                    inserting black/silence to keep the time tidy.
 356                 */
 357                 _just_did_inaccurate_seek = true;
 358         }
 359 }
 360
 361 void
 362 Player::setup_pieces ()
 363 {
 364         list<shared_ptr<Piece> > old_pieces = _pieces;
 365         _pieces.clear ();
 366
 367         ContentList content = _playlist->content ();
 368
 369         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 370
 371                 shared_ptr<Decoder> decoder;
 372                 optional<FrameRateChange> frc;
 373
 374                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 375                 if (fc) {
 376                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
 377                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 378                 }
 379
 380                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 381                 if (ic) {
 382                         /* See if we can re-use an old ImageDecoder */
 383                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 384                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 385                                 if (imd && imd->content() == ic) {
 386                                         decoder = imd;
 387                                 }
 388                         }
 389
 390                         if (!decoder) {
 391                                 decoder.reset (new ImageDecoder (_film, ic));
 392                         }
 393
 394                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 395                 }
 396
 397                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 398                 if (sc) {
 399                         decoder.reset (new SndfileDecoder (_film, sc));
 400
 401                         /* Working out the frc for this content is a bit tricky: what if it overlaps
 402                            two pieces of video content with different frame rates?  For now, use
 403                            the one with the best overlap.
 404                         */
 405
 406                         DCPTime best_overlap_t = 0;
 407                         shared_ptr<VideoContent> best_overlap;
 408                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
 409                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
 410                                 if (!vc) {
 411                                         continue;
 412                                 }
 413
 414                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
 415                                 if (overlap > best_overlap_t) {
 416                                         best_overlap = vc;
 417                                         best_overlap_t = overlap;
 418                                 }
 419                         }
 420
 421                         if (best_overlap) {
 422                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 423                         } else {
 424                                 /* No video overlap; e.g. if the DCP is just audio */
 425                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 426                         }
 427                 }
 428
 429                 decoder->seek ((*i)->trim_start (), true);
 430
 431                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 432         }
 433
 434         _have_valid_pieces = true;
 435 }
 436
 437 void
 438 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 439 {
 440         shared_ptr<Content> c = w.lock ();
 441         if (!c) {
 442                 return;
 443         }
 444
 445         if (
 446                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 447                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 448                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 449                 ) {
 450
 451                 _have_valid_pieces = false;
 452                 Changed (frequent);
 453
 454         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 455
 456                 update_subtitle ();
 457                 Changed (frequent);
 458
 459         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
 460
 461                 Changed (frequent);
 462
 463         } else if (property == ContentProperty::PATH) {
 464
 465                 Changed (frequent);
 466         }
 467 }
 468
 469 void
 470 Player::playlist_changed ()
 471 {
 472         _have_valid_pieces = false;
 473         Changed (false);
 474 }
 475
 476 void
 477 Player::set_video_container_size (libdcp::Size s)
 478 {
 479         _video_container_size = s;
 480
 481         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 482         im->make_black ();
 483
 484         _black_frame.reset (
 485                 new PlayerImage (
 486                         im,
 487                         Crop(),
 488                         _video_container_size,
 489                         _video_container_size,
 490                         Scaler::from_id ("bicubic")
 491                         )
 492                 );
 493 }
 494
 495 void
 496 Player::emit_black ()
 497 {
 498 #ifdef DCPOMATIC_DEBUG
 499         _last_video.reset ();
 500 #endif
 501
 502         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 503         _video_position += _film->video_frames_to_time (1);
 504         _last_emit_was_black = true;
 505 }
 506
 507 void
 508 Player::emit_silence (DCPTime most)
 509 {
 510         if (most == 0) {
 511                 return;
 512         }
 513
 514         DCPTime t = min (most, TIME_HZ / 2);
 515         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
 516         silence->make_silent ();
 517         Audio (silence, _audio_position);
 518         _audio_position += t;
 519 }
 520
 521 void
 522 Player::film_changed (Film::Property p)
 523 {
 524         /* Here we should notice Film properties that affect our output, and
 525            alert listeners that our output now would be different to how it was
 526            last time we were run.
 527         */
 528
 529         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 530                 Changed (false);
 531         }
 532 }
 533
 534 void
 535 Player::update_subtitle ()
 536 {
 537         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 538         if (!piece) {
 539                 return;
 540         }
 541
 542         if (!_in_subtitle.subtitle->image) {
 543                 _out_subtitle.subtitle->image.reset ();
 544                 return;
 545         }
 546
 547         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 548         assert (sc);
 549
 550         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
 551         libdcp::Size scaled_size;
 552
 553         in_rect.y += sc->subtitle_offset ();
 554
 555         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 556         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 557         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 558
 559         /* Then we need a corrective translation, consisting of two parts:
 560          *
 561          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 562          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 563          *
 564          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 565          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 566          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 567          *
 568          * Combining these two translations gives these expressions.
 569          */
 570
 571         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 572         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 573
 574         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
 575                 scaled_size,
 576                 Scaler::from_id ("bicubic"),
 577                 _in_subtitle.subtitle->image->pixel_format (),
 578                 true
 579                 );
 580
 581         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
 582         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
 583 }
 584
 585 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 586  *  @return false if this could not be done.
 587  */
 588 bool
 589 Player::repeat_last_video ()
 590 {
 591         if (!_last_incoming_video.video || !_have_valid_pieces) {
 592                 return false;
 593         }
 594
 595         emit_video (
 596                 _last_incoming_video.weak_piece,
 597                 _last_incoming_video.video
 598                 );
 599
 600         return true;
 601 }
 602
 603 PlayerImage::PlayerImage (
 604         shared_ptr<const Image> in,
 605         Crop crop,
 606         libdcp::Size inter_size,
 607         libdcp::Size out_size,
 608         Scaler const * scaler
 609         )
 610         : _in (in)
 611         , _crop (crop)
 612         , _inter_size (inter_size)
 613         , _out_size (out_size)
 614         , _scaler (scaler)
 615 {
 616
 617 }
 618
 619 void
 620 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
 621 {
 622         _subtitle_image = image;
 623         _subtitle_position = pos;
 624 }
 625
 626 shared_ptr<Image>
 627 PlayerImage::image ()
 628 {
 629         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, PIX_FMT_RGB24, false);
 630
 631         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
 632
 633         if (_subtitle_image) {
 634                 out->alpha_blend (_subtitle_image, _subtitle_position);
 635         }
 636
 637         return out;
 638 }