src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 class Piece
  52 {
  53 public:
  54         Piece (shared_ptr<Content> c)
  55                 : content (c)
  56                 , video_position (c->position ())
  57                 , audio_position (c->position ())
  58         {}
  59
  60         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  61                 : content (c)
  62                 , decoder (d)
  63                 , video_position (c->position ())
  64                 , audio_position (c->position ())
  65         {}
  66
  67         shared_ptr<Content> content;
  68         shared_ptr<Decoder> decoder;
  69         Time video_position;
  70         Time audio_position;
  71 };
  72
  73 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  74         : _film (f)
  75         , _playlist (p)
  76         , _video (true)
  77         , _audio (true)
  78         , _have_valid_pieces (false)
  79         , _video_position (0)
  80         , _audio_position (0)
  81         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  82         , _last_emit_was_black (false)
  83 {
  84         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  85         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  86         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  87         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
  88 }
  89
  90 void
  91 Player::disable_video ()
  92 {
  93         _video = false;
  94 }
  95
  96 void
  97 Player::disable_audio ()
  98 {
  99         _audio = false;
 100 }
 101
 102 bool
 103 Player::pass ()
 104 {
 105         if (!_have_valid_pieces) {
 106                 setup_pieces ();
 107                 _have_valid_pieces = true;
 108         }
 109
 110         Time earliest_t = TIME_MAX;
 111         shared_ptr<Piece> earliest;
 112         enum {
 113                 VIDEO,
 114                 AUDIO
 115         } type = VIDEO;
 116
 117         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 118                 if ((*i)->decoder->done ()) {
 119                         continue;
 120                 }
 121
 122                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 123                         if ((*i)->video_position < earliest_t) {
 124                                 earliest_t = (*i)->video_position;
 125                                 earliest = *i;
 126                                 type = VIDEO;
 127                         }
 128                 }
 129
 130                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 131                         if ((*i)->audio_position < earliest_t) {
 132                                 earliest_t = (*i)->audio_position;
 133                                 earliest = *i;
 134                                 type = AUDIO;
 135                         }
 136                 }
 137         }
 138
 139         if (!earliest) {
 140                 flush ();
 141                 return true;
 142         }
 143
 144         switch (type) {
 145         case VIDEO:
 146                 if (earliest_t > _video_position) {
 147                         emit_black ();
 148                 } else {
 149                         earliest->decoder->pass ();
 150                 }
 151                 break;
 152
 153         case AUDIO:
 154                 if (earliest_t > _audio_position) {
 155                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 156                 } else {
 157                         earliest->decoder->pass ();
 158
 159                         if (earliest->decoder->done()) {
 160                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 161                                 assert (ac);
 162                                 shared_ptr<Resampler> re = resampler (ac, false);
 163                                 if (re) {
 164                                         shared_ptr<const AudioBuffers> b = re->flush ();
 165                                         if (b->frames ()) {
 166                                                 process_audio (earliest, b, ac->audio_length ());
 167                                         }
 168                                 }
 169                         }
 170                 }
 171                 break;
 172         }
 173
 174         if (_audio) {
 175                 Time audio_done_up_to = TIME_MAX;
 176                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 177                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 178                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
 179                         }
 180                 }
 181
 182                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
 183                 Audio (tb.audio, tb.time);
 184                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 185         }
 186
 187         return false;
 188 }
 189
 190 void
 191 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 192 {
 193         /* Keep a note of what came in so that we can repeat it if required */
 194         _last_process_video.weak_piece = weak_piece;
 195         _last_process_video.image = image;
 196         _last_process_video.eyes = eyes;
 197         _last_process_video.same = same;
 198         _last_process_video.frame = frame;
 199
 200         shared_ptr<Piece> piece = weak_piece.lock ();
 201         if (!piece) {
 202                 return;
 203         }
 204
 205         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 206         assert (content);
 207
 208         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 209         if (frc.skip && (frame % 2) == 1) {
 210                 return;
 211         }
 212
 213         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 214         if (content->trimmed (relative_time)) {
 215                 return;
 216         }
 217
 218         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
 219         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
 220
 221         work_image = work_image->crop (content->crop(), true);
 222
 223         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
 224         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
 225
 226         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
 227
 228         Time time = content->position() + relative_time - content->trim_start ();
 229
 230         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 231                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 232         }
 233
 234         if (image_size != _video_container_size) {
 235                 assert (image_size.width <= _video_container_size.width);
 236                 assert (image_size.height <= _video_container_size.height);
 237                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 238                 im->make_black ();
 239                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 240                 work_image = im;
 241         }
 242
 243 #ifdef DCPOMATIC_DEBUG
 244         _last_video = piece->content;
 245 #endif
 246
 247         Video (work_image, eyes, content->colour_conversion(), same, time);
 248         time += TIME_HZ / _film->video_frame_rate();
 249
 250         if (frc.repeat) {
 251                 Video (work_image, eyes, content->colour_conversion(), true, time);
 252                 time += TIME_HZ / _film->video_frame_rate();
 253         }
 254
 255         _last_emit_was_black = false;
 256
 257         _video_position = piece->video_position = time;
 258 }
 259
 260 void
 261 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 262 {
 263         shared_ptr<Piece> piece = weak_piece.lock ();
 264         if (!piece) {
 265                 return;
 266         }
 267
 268         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 269         assert (content);
 270
 271         /* Gain */
 272         if (content->audio_gain() != 0) {
 273                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 274                 gain->apply_gain (content->audio_gain ());
 275                 audio = gain;
 276         }
 277
 278         /* Resample */
 279         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 280                 shared_ptr<Resampler> r = resampler (content, true);
 281                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 282                 audio = ro.first;
 283                 frame = ro.second;
 284         }
 285
 286         Time const relative_time = _film->audio_frames_to_time (frame);
 287
 288         if (content->trimmed (relative_time)) {
 289                 return;
 290         }
 291
 292         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
 293
 294         /* Remap channels */
 295         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 296         dcp_mapped->make_silent ();
 297         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 298         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 299                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 300                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 301                 }
 302         }
 303
 304         audio = dcp_mapped;
 305
 306         /* We must cut off anything that comes before the start of all time */
 307         if (time < 0) {
 308                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 309                 if (frames >= audio->frames ()) {
 310                         return;
 311                 }
 312
 313                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 314                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 315
 316                 audio = trimmed;
 317                 time = 0;
 318         }
 319
 320         _audio_merger.push (audio, time);
 321         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 322 }
 323
 324 void
 325 Player::flush ()
 326 {
 327         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 328         if (tb.audio) {
 329                 Audio (tb.audio, tb.time);
 330                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 331         }
 332
 333         while (_video_position < _audio_position) {
 334                 emit_black ();
 335         }
 336
 337         while (_audio_position < _video_position) {
 338                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 339         }
 340
 341 }
 342
 343 /** Seek so that the next pass() will yield (approximately) the requested frame.
 344  *  Pass accurate = true to try harder to get close to the request.
 345  *  @return true on error
 346  */
 347 void
 348 Player::seek (Time t, bool accurate)
 349 {
 350         if (!_have_valid_pieces) {
 351                 setup_pieces ();
 352                 _have_valid_pieces = true;
 353         }
 354
 355         if (_pieces.empty ()) {
 356                 return;
 357         }
 358
 359         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 360                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 361                 if (!vc) {
 362                         continue;
 363                 }
 364
 365                 Time s = t - vc->position ();
 366                 s = max (static_cast<Time> (0), s);
 367                 s = min (vc->length_after_trim(), s);
 368
 369                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 370
 371                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 372                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 373                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 374                    the source's rate.
 375                 */
 376                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 377                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 378         }
 379
 380         _video_position = _audio_position = t;
 381
 382         /* XXX: don't seek audio because we don't need to... */
 383 }
 384
 385 void
 386 Player::setup_pieces ()
 387 {
 388         list<shared_ptr<Piece> > old_pieces = _pieces;
 389
 390         _pieces.clear ();
 391
 392         ContentList content = _playlist->content ();
 393         sort (content.begin(), content.end(), ContentSorter ());
 394
 395         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 396
 397                 shared_ptr<Piece> piece (new Piece (*i));
 398
 399                 /* XXX: into content? */
 400
 401                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 402                 if (fc) {
 403                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 404
 405                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 406                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 407                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 408
 409                         piece->decoder = fd;
 410                 }
 411
 412                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 413                 if (ic) {
 414                         shared_ptr<StillImageDecoder> id;
 415
 416                         /* See if we can re-use an old StillImageDecoder */
 417                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 418                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 419                                 if (imd && imd->content() == ic) {
 420                                         id = imd;
 421                                 }
 422                         }
 423
 424                         if (!id) {
 425                                 id.reset (new StillImageDecoder (_film, ic));
 426                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 427                         }
 428
 429                         piece->decoder = id;
 430                 }
 431
 432                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 433                 if (mc) {
 434                         shared_ptr<MovingImageDecoder> md;
 435
 436                         if (!md) {
 437                                 md.reset (new MovingImageDecoder (_film, mc));
 438                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 439                         }
 440
 441                         piece->decoder = md;
 442                 }
 443
 444                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 445                 if (sc) {
 446                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 447                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 448
 449                         piece->decoder = sd;
 450                 }
 451
 452                 _pieces.push_back (piece);
 453         }
 454 }
 455
 456 void
 457 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 458 {
 459         shared_ptr<Content> c = w.lock ();
 460         if (!c) {
 461                 return;
 462         }
 463
 464         if (
 465                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 466                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
 467                 ) {
 468
 469                 _have_valid_pieces = false;
 470                 Changed (frequent);
 471
 472         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 473
 474                 update_subtitle ();
 475                 Changed (frequent);
 476
 477         } else if (
 478                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
 479                 property == VideoContentProperty::VIDEO_RATIO
 480                 ) {
 481
 482                 Changed (frequent);
 483
 484         } else if (property == ContentProperty::PATH) {
 485
 486                 Changed (frequent);
 487         }
 488 }
 489
 490 void
 491 Player::playlist_changed ()
 492 {
 493         _have_valid_pieces = false;
 494         Changed (false);
 495 }
 496
 497 void
 498 Player::set_video_container_size (libdcp::Size s)
 499 {
 500         _video_container_size = s;
 501         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 502         _black_frame->make_black ();
 503 }
 504
 505 shared_ptr<Resampler>
 506 Player::resampler (shared_ptr<AudioContent> c, bool create)
 507 {
 508         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 509         if (i != _resamplers.end ()) {
 510                 return i->second;
 511         }
 512
 513         if (!create) {
 514                 return shared_ptr<Resampler> ();
 515         }
 516
 517         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 518         _resamplers[c] = r;
 519         return r;
 520 }
 521
 522 void
 523 Player::emit_black ()
 524 {
 525 #ifdef DCPOMATIC_DEBUG
 526         _last_video.reset ();
 527 #endif
 528
 529         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 530         _video_position += _film->video_frames_to_time (1);
 531         _last_emit_was_black = true;
 532 }
 533
 534 void
 535 Player::emit_silence (OutputAudioFrame most)
 536 {
 537         if (most == 0) {
 538                 return;
 539         }
 540
 541         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 542         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 543         silence->make_silent ();
 544         Audio (silence, _audio_position);
 545         _audio_position += _film->audio_frames_to_time (N);
 546 }
 547
 548 void
 549 Player::film_changed (Film::Property p)
 550 {
 551         /* Here we should notice Film properties that affect our output, and
 552            alert listeners that our output now would be different to how it was
 553            last time we were run.
 554         */
 555
 556         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 557                 Changed (false);
 558         }
 559 }
 560
 561 void
 562 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 563 {
 564         _in_subtitle.piece = weak_piece;
 565         _in_subtitle.image = image;
 566         _in_subtitle.rect = rect;
 567         _in_subtitle.from = from;
 568         _in_subtitle.to = to;
 569
 570         update_subtitle ();
 571 }
 572
 573 void
 574 Player::update_subtitle ()
 575 {
 576         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 577         if (!piece) {
 578                 return;
 579         }
 580
 581         if (!_in_subtitle.image) {
 582                 _out_subtitle.image.reset ();
 583                 return;
 584         }
 585
 586         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 587         assert (sc);
 588
 589         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 590         libdcp::Size scaled_size;
 591
 592         in_rect.y += sc->subtitle_offset ();
 593
 594         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 595         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 596         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 597
 598         /* Then we need a corrective translation, consisting of two parts:
 599          *
 600          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 601          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 602          *
 603          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 604          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 605          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 606          *
 607          * Combining these two translations gives these expressions.
 608          */
 609
 610         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 611         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 612
 613         _out_subtitle.image = _in_subtitle.image->scale (
 614                 scaled_size,
 615                 Scaler::from_id ("bicubic"),
 616                 _in_subtitle.image->pixel_format (),
 617                 true
 618                 );
 619         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
 620         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
 621 }
 622
 623 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 624  *  @return false if this could not be done.
 625  */
 626 bool
 627 Player::repeat_last_video ()
 628 {
 629         if (!_last_process_video.image) {
 630                 return false;
 631         }
 632
 633         process_video (
 634                 _last_process_video.weak_piece,
 635                 _last_process_video.image,
 636                 _last_process_video.eyes,
 637                 _last_process_video.same,
 638                 _last_process_video.frame
 639                 );
 640
 641         return true;
 642 }