src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "image_proxy.h"
  38 #include "ratio.h"
  39 #include "log.h"
  40 #include "scaler.h"
  41 #include "render_subtitles.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44 #include "player_video.h"
  45 #include "frame_rate_change.h"
  46
  47 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  48
  49 using std::list;
  50 using std::cout;
  51 using std::min;
  52 using std::max;
  53 using std::min;
  54 using std::vector;
  55 using std::pair;
  56 using std::map;
  57 using std::make_pair;
  58 using boost::shared_ptr;
  59 using boost::weak_ptr;
  60 using boost::dynamic_pointer_cast;
  61 using boost::optional;
  62
  63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  64         : _film (f)
  65         , _playlist (p)
  66         , _have_valid_pieces (false)
  67         , _approximate_size (false)
  68 {
  69         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  70         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  71         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  72         set_video_container_size (_film->frame_size ());
  73 }
  74
  75 void
  76 Player::setup_pieces ()
  77 {
  78         list<shared_ptr<Piece> > old_pieces = _pieces;
  79         _pieces.clear ();
  80
  81         ContentList content = _playlist->content ();
  82
  83         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  84
  85                 if (!(*i)->paths_valid ()) {
  86                         continue;
  87                 }
  88
  89                 shared_ptr<Decoder> decoder;
  90                 optional<FrameRateChange> frc;
  91
  92                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  93                 DCPTime best_overlap_t;
  94                 shared_ptr<VideoContent> best_overlap;
  95                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  96                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  97                         if (!vc) {
  98                                 continue;
  99                         }
 100
 101                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
 102                         if (overlap > best_overlap_t) {
 103                                 best_overlap = vc;
 104                                 best_overlap_t = overlap;
 105                         }
 106                 }
 107
 108                 optional<FrameRateChange> best_overlap_frc;
 109                 if (best_overlap) {
 110                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 111                 } else {
 112                         /* No video overlap; e.g. if the DCP is just audio */
 113                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 114                 }
 115
 116                 /* FFmpeg */
 117                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 118                 if (fc) {
 119                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 120                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 121                 }
 122
 123                 /* ImageContent */
 124                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 125                 if (ic) {
 126                         /* See if we can re-use an old ImageDecoder */
 127                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 128                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 129                                 if (imd && imd->content() == ic) {
 130                                         decoder = imd;
 131                                 }
 132                         }
 133
 134                         if (!decoder) {
 135                                 decoder.reset (new ImageDecoder (ic));
 136                         }
 137
 138                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 139                 }
 140
 141                 /* SndfileContent */
 142                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 143                 if (sc) {
 144                         decoder.reset (new SndfileDecoder (sc));
 145                         frc = best_overlap_frc;
 146                 }
 147
 148                 /* SubRipContent */
 149                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 150                 if (rc) {
 151                         decoder.reset (new SubRipDecoder (rc));
 152                         frc = best_overlap_frc;
 153                 }
 154
 155                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 156         }
 157
 158         _have_valid_pieces = true;
 159 }
 160
 161 void
 162 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 163 {
 164         shared_ptr<Content> c = w.lock ();
 165         if (!c) {
 166                 return;
 167         }
 168
 169         if (
 170                 property == ContentProperty::POSITION ||
 171                 property == ContentProperty::LENGTH ||
 172                 property == ContentProperty::TRIM_START ||
 173                 property == ContentProperty::TRIM_END ||
 174                 property == ContentProperty::PATH ||
 175                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 176                 ) {
 177
 178                 _have_valid_pieces = false;
 179                 Changed (frequent);
 180
 181         } else if (
 182                 property == SubtitleContentProperty::SUBTITLE_USE ||
 183                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 184                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 185                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 186                 property == VideoContentProperty::VIDEO_CROP ||
 187                 property == VideoContentProperty::VIDEO_SCALE ||
 188                 property == VideoContentProperty::VIDEO_FRAME_RATE
 189                 ) {
 190
 191                 Changed (frequent);
 192         }
 193 }
 194
 195 /** @param already_resampled true if this data has already been through the chain up to the resampler */
 196 void
 197 Player::playlist_changed ()
 198 {
 199         _have_valid_pieces = false;
 200         Changed (false);
 201 }
 202
 203 void
 204 Player::set_video_container_size (dcp::Size s)
 205 {
 206         _video_container_size = s;
 207
 208         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 209         _black_image->make_black ();
 210 }
 211
 212 void
 213 Player::film_changed (Film::Property p)
 214 {
 215         /* Here we should notice Film properties that affect our output, and
 216            alert listeners that our output now would be different to how it was
 217            last time we were run.
 218         */
 219
 220         if (p == Film::SCALER || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 221                 Changed (false);
 222         }
 223 }
 224
 225 list<PositionImage>
 226 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 227 {
 228         list<PositionImage> all;
 229
 230         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 231                 if (!i->image) {
 232                         continue;
 233                 }
 234
 235                 /* We will scale the subtitle up to fit _video_container_size */
 236                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 237
 238                 /* Then we need a corrective translation, consisting of two parts:
 239                  *
 240                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 241                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 242                  *
 243                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 244                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 245                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 246                  *
 247                  * Combining these two translations gives these expressions.
 248                  */
 249
 250                 all.push_back (
 251                         PositionImage (
 252                                 i->image->scale (
 253                                         scaled_size,
 254                                         Scaler::from_id ("bicubic"),
 255                                         i->image->pixel_format (),
 256                                         true
 257                                         ),
 258                                 Position<int> (
 259                                         rint (_video_container_size.width * i->rectangle.x),
 260                                         rint (_video_container_size.height * i->rectangle.y)
 261                                         )
 262                                 )
 263                         );
 264         }
 265
 266         return all;
 267 }
 268
 269 void
 270 Player::set_approximate_size ()
 271 {
 272         _approximate_size = true;
 273 }
 274
 275 shared_ptr<PlayerVideo>
 276 Player::black_player_video_frame (DCPTime time) const
 277 {
 278         return shared_ptr<PlayerVideo> (
 279                 new PlayerVideo (
 280                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image, _film->log ())),
 281                         time,
 282                         Crop (),
 283                         _video_container_size,
 284                         _video_container_size,
 285                         Scaler::from_id ("bicubic"),
 286                         EYES_BOTH,
 287                         PART_WHOLE,
 288                         Config::instance()->colour_conversions().front().conversion
 289                 )
 290         );
 291 }
 292
 293 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
 294 list<shared_ptr<PlayerVideo> >
 295 Player::get_video (DCPTime time, bool accurate)
 296 {
 297         if (!_have_valid_pieces) {
 298                 setup_pieces ();
 299         }
 300
 301         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
 302                 time,
 303                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
 304                 );
 305
 306         list<shared_ptr<PlayerVideo> > pvf;
 307
 308         if (ov.empty ()) {
 309                 /* No video content at this time */
 310                 pvf.push_back (black_player_video_frame (time));
 311         } else {
 312                 /* Create a PlayerVideo from the content's video at this time */
 313
 314                 shared_ptr<Piece> piece = ov.back ();
 315                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 316                 assert (decoder);
 317                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 318                 assert (content);
 319
 320                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 321                 if (content_video.empty ()) {
 322                         pvf.push_back (black_player_video_frame (time));
 323                         return pvf;
 324                 }
 325
 326                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 327                 if (_approximate_size) {
 328                         image_size.width &= ~3;
 329                         image_size.height &= ~3;
 330                 }
 331
 332                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
 333                         pvf.push_back (
 334                                 shared_ptr<PlayerVideo> (
 335                                         new PlayerVideo (
 336                                                 i->image,
 337                                                 content_video_to_dcp (piece, i->frame),
 338                                                 content->crop (),
 339                                                 image_size,
 340                                                 _video_container_size,
 341                                                 _film->scaler(),
 342                                                 i->eyes,
 343                                                 i->part,
 344                                                 content->colour_conversion ()
 345                                                 )
 346                                         )
 347                                 );
 348                 }
 349         }
 350
 351         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
 352
 353         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
 354
 355         list<PositionImage> sub_images;
 356
 357         /* Image subtitles */
 358         list<PositionImage> c = transform_image_subtitles (ps.image);
 359         copy (c.begin(), c.end(), back_inserter (sub_images));
 360
 361         /* Text subtitles (rendered to images) */
 362         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
 363
 364         if (!sub_images.empty ()) {
 365                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
 366                         (*i)->set_subtitle (merge (sub_images));
 367                 }
 368         }
 369
 370         return pvf;
 371 }
 372
 373 shared_ptr<AudioBuffers>
 374 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 375 {
 376         if (!_have_valid_pieces) {
 377                 setup_pieces ();
 378         }
 379
 380         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 381
 382         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 383         audio->make_silent ();
 384
 385         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
 386         if (ov.empty ()) {
 387                 return audio;
 388         }
 389
 390         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 391
 392                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 393                 assert (content);
 394                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 395                 assert (decoder);
 396
 397                 if (content->audio_frame_rate() == 0) {
 398                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
 399                          * audio stream).
 400                          */
 401                         continue;
 402                 }
 403
 404                 /* The time that we should request from the content */
 405                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
 406                 DCPTime offset;
 407                 if (request < DCPTime ()) {
 408                         /* We went off the start of the content, so we will need to offset
 409                            the stuff we get back.
 410                         */
 411                         offset = -request;
 412                         request = DCPTime ();
 413                 }
 414
 415                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
 416
 417                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
 418                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
 419
 420                 /* Gain */
 421                 if (content->audio_gain() != 0) {
 422                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 423                         gain->apply_gain (content->audio_gain ());
 424                         all->audio = gain;
 425                 }
 426
 427                 /* Remap channels */
 428                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 429                 dcp_mapped->make_silent ();
 430                 AudioMapping map = content->audio_mapping ();
 431                 for (int i = 0; i < map.content_channels(); ++i) {
 432                         for (int j = 0; j < _film->audio_channels(); ++j) {
 433                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 434                                         dcp_mapped->accumulate_channel (
 435                                                 all->audio.get(),
 436                                                 i,
 437                                                 j,
 438                                                 map.get (i, static_cast<dcp::Channel> (j))
 439                                                 );
 440                                 }
 441                         }
 442                 }
 443
 444                 all->audio = dcp_mapped;
 445
 446                 audio->accumulate_frames (
 447                         all->audio.get(),
 448                         content_frame - all->frame,
 449                         offset.frames (_film->audio_frame_rate()),
 450                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
 451                         );
 452         }
 453
 454         return audio;
 455 }
 456
 457 VideoFrame
 458 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 459 {
 460         /* s is the offset of t from the start position of this content */
 461         DCPTime s = t - piece->content->position ();
 462         s = DCPTime (max (DCPTime::Type (0), s.get ()));
 463         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 464
 465         /* Convert this to the content frame */
 466         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 467 }
 468
 469 DCPTime
 470 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
 471 {
 472         DCPTime t = DCPTime::from_frames (f / piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
 473         if (t < DCPTime ()) {
 474                 t = DCPTime ();
 475         }
 476
 477         return t;
 478 }
 479
 480 AudioFrame
 481 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 482 {
 483         /* s is the offset of t from the start position of this content */
 484         DCPTime s = t - piece->content->position ();
 485         s = DCPTime (max (DCPTime::Type (0), s.get ()));
 486         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 487
 488         /* Convert this to the content frame */
 489         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 490 }
 491
 492 ContentTime
 493 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 494 {
 495         /* s is the offset of t from the start position of this content */
 496         DCPTime s = t - piece->content->position ();
 497         s = DCPTime (max (DCPTime::Type (0), s.get ()));
 498         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 499
 500         return ContentTime (s + piece->content->trim_start(), piece->frc);
 501 }
 502
 503 void
 504 PlayerStatistics::dump (shared_ptr<Log> log) const
 505 {
 506         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
 507         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
 508 }
 509
 510 PlayerStatistics const &
 511 Player::statistics () const
 512 {
 513         return _statistics;
 514 }
 515
 516 PlayerSubtitles
 517 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
 518 {
 519         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
 520
 521         PlayerSubtitles ps (time, length);
 522
 523         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
 524                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
 525                 if (!subtitle_content->subtitle_use ()) {
 526                         continue;
 527                 }
 528
 529                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
 530                 ContentTime const from = dcp_to_content_subtitle (*j, time);
 531                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
 532                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
 533
 534                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
 535                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
 536
 537                         /* Apply content's subtitle offsets */
 538                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
 539                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
 540
 541                         /* Apply content's subtitle scale */
 542                         i->sub.rectangle.width *= subtitle_content->subtitle_scale ();
 543                         i->sub.rectangle.height *= subtitle_content->subtitle_scale ();
 544
 545                         /* Apply a corrective translation to keep the subtitle centred after that scale */
 546                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_scale() - 1);
 547                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_scale() - 1);
 548
 549                         ps.image.push_back (i->sub);
 550                 }
 551
 552                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
 553                 for (list<ContentTextSubtitle>::const_iterator i = text.begin(); i != text.end(); ++i) {
 554                         copy (i->subs.begin(), i->subs.end(), back_inserter (ps.text));
 555                 }
 556         }
 557
 558         return ps;
 559 }