src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "image_proxy.h"
  38 #include "ratio.h"
  39 #include "log.h"
  40 #include "scaler.h"
  41 #include "render_subtitles.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44 #include "player_video_frame.h"
  45 #include "frame_rate_change.h"
  46
  47 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  48
  49 using std::list;
  50 using std::cout;
  51 using std::min;
  52 using std::max;
  53 using std::min;
  54 using std::vector;
  55 using std::pair;
  56 using std::map;
  57 using std::make_pair;
  58 using boost::shared_ptr;
  59 using boost::weak_ptr;
  60 using boost::dynamic_pointer_cast;
  61 using boost::optional;
  62
  63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  64         : _film (f)
  65         , _playlist (p)
  66         , _have_valid_pieces (false)
  67         , _approximate_size (false)
  68         , _burn_subtitles (false)
  69 {
  70         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  71         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  72         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  73         set_video_container_size (_film->frame_size ());
  74 }
  75
  76 void
  77 Player::setup_pieces ()
  78 {
  79         list<shared_ptr<Piece> > old_pieces = _pieces;
  80         _pieces.clear ();
  81
  82         ContentList content = _playlist->content ();
  83
  84         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  85
  86                 if (!(*i)->paths_valid ()) {
  87                         continue;
  88                 }
  89
  90                 shared_ptr<Decoder> decoder;
  91                 optional<FrameRateChange> frc;
  92
  93                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  94                 DCPTime best_overlap_t;
  95                 shared_ptr<VideoContent> best_overlap;
  96                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  97                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  98                         if (!vc) {
  99                                 continue;
 100                         }
 101
 102                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
 103                         if (overlap > best_overlap_t) {
 104                                 best_overlap = vc;
 105                                 best_overlap_t = overlap;
 106                         }
 107                 }
 108
 109                 optional<FrameRateChange> best_overlap_frc;
 110                 if (best_overlap) {
 111                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 112                 } else {
 113                         /* No video overlap; e.g. if the DCP is just audio */
 114                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 115                 }
 116
 117                 /* FFmpeg */
 118                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 119                 if (fc) {
 120                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 121                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 122                 }
 123
 124                 /* ImageContent */
 125                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 126                 if (ic) {
 127                         /* See if we can re-use an old ImageDecoder */
 128                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 129                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 130                                 if (imd && imd->content() == ic) {
 131                                         decoder = imd;
 132                                 }
 133                         }
 134
 135                         if (!decoder) {
 136                                 decoder.reset (new ImageDecoder (ic));
 137                         }
 138
 139                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 140                 }
 141
 142                 /* SndfileContent */
 143                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 144                 if (sc) {
 145                         decoder.reset (new SndfileDecoder (sc));
 146                         frc = best_overlap_frc;
 147                 }
 148
 149                 /* SubRipContent */
 150                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 151                 if (rc) {
 152                         decoder.reset (new SubRipDecoder (rc));
 153                         frc = best_overlap_frc;
 154                 }
 155
 156                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 157         }
 158
 159         _have_valid_pieces = true;
 160 }
 161
 162 void
 163 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 164 {
 165         shared_ptr<Content> c = w.lock ();
 166         if (!c) {
 167                 return;
 168         }
 169
 170         if (
 171                 property == ContentProperty::POSITION ||
 172                 property == ContentProperty::LENGTH ||
 173                 property == ContentProperty::TRIM_START ||
 174                 property == ContentProperty::TRIM_END ||
 175                 property == ContentProperty::PATH ||
 176                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 177                 ) {
 178
 179                 _have_valid_pieces = false;
 180                 Changed (frequent);
 181
 182         } else if (
 183                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 184                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 185                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 186                 property == VideoContentProperty::VIDEO_CROP ||
 187                 property == VideoContentProperty::VIDEO_SCALE ||
 188                 property == VideoContentProperty::VIDEO_FRAME_RATE
 189                 ) {
 190
 191                 Changed (frequent);
 192         }
 193 }
 194
 195 /** @param already_resampled true if this data has already been through the chain up to the resampler */
 196 void
 197 Player::playlist_changed ()
 198 {
 199         _have_valid_pieces = false;
 200         Changed (false);
 201 }
 202
 203 void
 204 Player::set_video_container_size (dcp::Size s)
 205 {
 206         _video_container_size = s;
 207
 208         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 209         _black_image->make_black ();
 210 }
 211
 212 void
 213 Player::film_changed (Film::Property p)
 214 {
 215         /* Here we should notice Film properties that affect our output, and
 216            alert listeners that our output now would be different to how it was
 217            last time we were run.
 218         */
 219
 220         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 221                 Changed (false);
 222         }
 223 }
 224
 225 list<PositionImage>
 226 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs) const
 227 {
 228         list<PositionImage> all;
 229
 230         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 231                 if (!(*i)->image) {
 232                         continue;
 233                 }
 234
 235                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 236                 dcp::Size scaled_size;
 237
 238                 in_rect.x += content->subtitle_x_offset ();
 239                 in_rect.y += content->subtitle_y_offset ();
 240
 241                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 242                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 243                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 244
 245                 /* Then we need a corrective translation, consisting of two parts:
 246                  *
 247                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 248                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 249                  *
 250                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 251                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 252                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 253                  *
 254                  * Combining these two translations gives these expressions.
 255                  */
 256
 257                 all.push_back (
 258                         PositionImage (
 259                                 (*i)->image->scale (
 260                                         scaled_size,
 261                                         Scaler::from_id ("bicubic"),
 262                                         (*i)->image->pixel_format (),
 263                                         true
 264                                         ),
 265                                 Position<int> (
 266                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 267                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 268                                         )
 269                                 )
 270                         );
 271         }
 272
 273         return all;
 274 }
 275
 276 list<PositionImage>
 277 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub) const
 278 {
 279         list<PositionImage> all;
 280         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 281                 if (!(*i)->subs.empty ()) {
 282                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 283                 }
 284         }
 285
 286         return all;
 287 }
 288
 289 void
 290 Player::set_approximate_size ()
 291 {
 292         _approximate_size = true;
 293 }
 294
 295 shared_ptr<PlayerVideoFrame>
 296 Player::black_player_video_frame () const
 297 {
 298         return shared_ptr<PlayerVideoFrame> (
 299                 new PlayerVideoFrame (
 300                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image, _film->log ())),
 301                         Crop (),
 302                         _video_container_size,
 303                         _video_container_size,
 304                         Scaler::from_id ("bicubic"),
 305                         EYES_BOTH,
 306                         PART_WHOLE,
 307                         Config::instance()->colour_conversions().front().conversion
 308                 )
 309         );
 310 }
 311
 312 shared_ptr<PlayerVideoFrame>
 313 Player::content_to_player_video_frame (
 314         shared_ptr<VideoContent> content,
 315         ContentVideo content_video,
 316         list<shared_ptr<Piece> > subs,
 317         DCPTime time,
 318         dcp::Size image_size) const
 319 {
 320         shared_ptr<PlayerVideoFrame> pvf (
 321                 new PlayerVideoFrame (
 322                         content_video.image,
 323                         content->crop (),
 324                         image_size,
 325                         _video_container_size,
 326                         _film->scaler(),
 327                         content_video.eyes,
 328                         content_video.part,
 329                         content->colour_conversion ()
 330                         )
 331                 );
 332
 333
 334         /* Add subtitles */
 335
 336         list<PositionImage> sub_images;
 337
 338         for (list<shared_ptr<Piece> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 339                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 340                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 341                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 342                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 343
 344                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to));
 345                 if (!image_subtitles.empty ()) {
 346                         list<PositionImage> im = process_content_image_subtitles (
 347                                 subtitle_content,
 348                                 image_subtitles
 349                                 );
 350
 351                         copy (im.begin(), im.end(), back_inserter (sub_images));
 352                 }
 353
 354                 if (_burn_subtitles) {
 355                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to));
 356                         if (!text_subtitles.empty ()) {
 357                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 358                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 359                         }
 360                 }
 361         }
 362
 363         if (!sub_images.empty ()) {
 364                 pvf->set_subtitle (merge (sub_images));
 365         }
 366
 367         return pvf;
 368 }
 369
 370 /** @return All PlayerVideoFrames at the given time (there may be two frames for 3D) */
 371 list<shared_ptr<PlayerVideoFrame> >
 372 Player::get_video (DCPTime time, bool accurate)
 373 {
 374         if (!_have_valid_pieces) {
 375                 setup_pieces ();
 376         }
 377
 378         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
 379                 time,
 380                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
 381                 );
 382
 383         list<shared_ptr<PlayerVideoFrame> > pvf;
 384
 385         if (ov.empty ()) {
 386                 /* No video content at this time */
 387                 pvf.push_back (black_player_video_frame ());
 388                 return pvf;
 389         }
 390
 391         /* Create a PlayerVideoFrame from the content's video at this time */
 392
 393         shared_ptr<Piece> piece = ov.back ();
 394         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 395         assert (decoder);
 396         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 397         assert (content);
 398
 399         list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 400         if (content_video.empty ()) {
 401                 pvf.push_back (black_player_video_frame ());
 402                 return pvf;
 403         }
 404
 405         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 406         if (_approximate_size) {
 407                 image_size.width &= ~3;
 408                 image_size.height &= ~3;
 409         }
 410
 411         for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
 412                 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (
 413                         time,
 414                         time + DCPTime::from_frames (1, _film->video_frame_rate ())
 415                         );
 416
 417                 pvf.push_back (content_to_player_video_frame (content, *i, subs, time, image_size));
 418         }
 419
 420         return pvf;
 421 }
 422
 423 shared_ptr<AudioBuffers>
 424 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 425 {
 426         if (!_have_valid_pieces) {
 427                 setup_pieces ();
 428         }
 429
 430         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 431
 432         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 433         audio->make_silent ();
 434
 435         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
 436         if (ov.empty ()) {
 437                 return audio;
 438         }
 439
 440         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 441
 442                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 443                 assert (content);
 444                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 445                 assert (decoder);
 446
 447                 if (content->audio_frame_rate() == 0) {
 448                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
 449                          * audio stream).
 450                          */
 451                         continue;
 452                 }
 453
 454                 /* The time that we should request from the content */
 455                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
 456                 DCPTime offset;
 457                 if (request < DCPTime ()) {
 458                         /* We went off the start of the content, so we will need to offset
 459                            the stuff we get back.
 460                         */
 461                         offset = -request;
 462                         request = DCPTime ();
 463                 }
 464
 465                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
 466
 467                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
 468                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
 469
 470                 /* Gain */
 471                 if (content->audio_gain() != 0) {
 472                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 473                         gain->apply_gain (content->audio_gain ());
 474                         all->audio = gain;
 475                 }
 476
 477                 /* Remap channels */
 478                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 479                 dcp_mapped->make_silent ();
 480                 AudioMapping map = content->audio_mapping ();
 481                 for (int i = 0; i < map.content_channels(); ++i) {
 482                         for (int j = 0; j < _film->audio_channels(); ++j) {
 483                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 484                                         dcp_mapped->accumulate_channel (
 485                                                 all->audio.get(),
 486                                                 i,
 487                                                 j,
 488                                                 map.get (i, static_cast<dcp::Channel> (j))
 489                                                 );
 490                                 }
 491                         }
 492                 }
 493
 494                 all->audio = dcp_mapped;
 495
 496                 audio->accumulate_frames (
 497                         all->audio.get(),
 498                         content_frame - all->frame,
 499                         offset.frames (_film->audio_frame_rate()),
 500                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
 501                         );
 502         }
 503 }
 504
 505 VideoFrame
 506 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 507 {
 508         /* s is the offset of t from the start position of this content */
 509         DCPTime s = t - piece->content->position ();
 510         s = DCPTime (max (int64_t (0), s.get ()));
 511         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 512
 513         /* Convert this to the content frame */
 514         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 515 }
 516
 517 AudioFrame
 518 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 519 {
 520         /* s is the offset of t from the start position of this content */
 521         DCPTime s = t - piece->content->position ();
 522         s = DCPTime (max (int64_t (0), s.get ()));
 523         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 524
 525         /* Convert this to the content frame */
 526         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 527 }
 528
 529 ContentTime
 530 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 531 {
 532         /* s is the offset of t from the start position of this content */
 533         DCPTime s = t - piece->content->position ();
 534         s = DCPTime (max (int64_t (0), s.get ()));
 535         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 536
 537         return ContentTime (s + piece->content->trim_start(), piece->frc);
 538 }
 539
 540 void
 541 PlayerStatistics::dump (shared_ptr<Log> log) const
 542 {
 543         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
 544         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
 545 }
 546
 547 PlayerStatistics const &
 548 Player::statistics () const
 549 {
 550         return _statistics;
 551 }