src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "ratio.h"
  38 #include "log.h"
  39 #include "scaler.h"
  40 #include "render_subtitles.h"
  41 #include "dcp_video.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44
  45 using std::list;
  46 using std::cout;
  47 using std::min;
  48 using std::max;
  49 using std::min;
  50 using std::vector;
  51 using std::pair;
  52 using std::map;
  53 using std::make_pair;
  54 using boost::shared_ptr;
  55 using boost::weak_ptr;
  56 using boost::dynamic_pointer_cast;
  57 using boost::optional;
  58
  59 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  60         : _film (f)
  61         , _playlist (p)
  62         , _have_valid_pieces (false)
  63         , _approximate_size (false)
  64         , _burn_subtitles (false)
  65 {
  66         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  67         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  68         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  69         set_video_container_size (_film->frame_size ());
  70 }
  71
  72 void
  73 Player::setup_pieces ()
  74 {
  75         list<shared_ptr<Piece> > old_pieces = _pieces;
  76         _pieces.clear ();
  77
  78         ContentList content = _playlist->content ();
  79
  80         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  81
  82                 if (!(*i)->paths_valid ()) {
  83                         continue;
  84                 }
  85
  86                 shared_ptr<Decoder> decoder;
  87                 optional<FrameRateChange> frc;
  88
  89                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  90                 DCPTime best_overlap_t;
  91                 shared_ptr<VideoContent> best_overlap;
  92                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  93                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  94                         if (!vc) {
  95                                 continue;
  96                         }
  97
  98                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
  99                         if (overlap > best_overlap_t) {
 100                                 best_overlap = vc;
 101                                 best_overlap_t = overlap;
 102                         }
 103                 }
 104
 105                 optional<FrameRateChange> best_overlap_frc;
 106                 if (best_overlap) {
 107                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 108                 } else {
 109                         /* No video overlap; e.g. if the DCP is just audio */
 110                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 111                 }
 112
 113                 /* FFmpeg */
 114                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 115                 if (fc) {
 116                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 117                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 118                 }
 119
 120                 /* ImageContent */
 121                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 122                 if (ic) {
 123                         /* See if we can re-use an old ImageDecoder */
 124                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 125                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 126                                 if (imd && imd->content() == ic) {
 127                                         decoder = imd;
 128                                 }
 129                         }
 130
 131                         if (!decoder) {
 132                                 decoder.reset (new ImageDecoder (ic));
 133                         }
 134
 135                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 136                 }
 137
 138                 /* SndfileContent */
 139                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 140                 if (sc) {
 141                         decoder.reset (new SndfileDecoder (sc));
 142                         frc = best_overlap_frc;
 143                 }
 144
 145                 /* SubRipContent */
 146                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 147                 if (rc) {
 148                         decoder.reset (new SubRipDecoder (rc));
 149                         frc = best_overlap_frc;
 150                 }
 151
 152                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 153         }
 154
 155         _have_valid_pieces = true;
 156 }
 157
 158 void
 159 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 160 {
 161         shared_ptr<Content> c = w.lock ();
 162         if (!c) {
 163                 return;
 164         }
 165
 166         if (
 167                 property == ContentProperty::POSITION ||
 168                 property == ContentProperty::LENGTH ||
 169                 property == ContentProperty::TRIM_START ||
 170                 property == ContentProperty::TRIM_END ||
 171                 property == ContentProperty::PATH ||
 172                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 173                 ) {
 174
 175                 _have_valid_pieces = false;
 176                 Changed (frequent);
 177
 178         } else if (
 179                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 180                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 181                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 182                 property == VideoContentProperty::VIDEO_CROP ||
 183                 property == VideoContentProperty::VIDEO_SCALE ||
 184                 property == VideoContentProperty::VIDEO_FRAME_RATE
 185                 ) {
 186
 187                 Changed (frequent);
 188         }
 189 }
 190
 191 void
 192 Player::playlist_changed ()
 193 {
 194         _have_valid_pieces = false;
 195         Changed (false);
 196 }
 197
 198 void
 199 Player::set_video_container_size (dcp::Size s)
 200 {
 201         _video_container_size = s;
 202
 203         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 204         _black_image->make_black ();
 205 }
 206
 207 void
 208 Player::film_changed (Film::Property p)
 209 {
 210         /* Here we should notice Film properties that affect our output, and
 211            alert listeners that our output now would be different to how it was
 212            last time we were run.
 213         */
 214
 215         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 216                 Changed (false);
 217         }
 218 }
 219
 220 list<PositionImage>
 221 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs)
 222 {
 223         list<PositionImage> all;
 224
 225         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 226                 if (!(*i)->image) {
 227                         continue;
 228                 }
 229
 230                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 231                 dcp::Size scaled_size;
 232
 233                 in_rect.x += content->subtitle_x_offset ();
 234                 in_rect.y += content->subtitle_y_offset ();
 235
 236                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 237                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 238                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 239
 240                 /* Then we need a corrective translation, consisting of two parts:
 241                  *
 242                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 243                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 244                  *
 245                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 246                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 247                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 248                  *
 249                  * Combining these two translations gives these expressions.
 250                  */
 251
 252                 all.push_back (
 253                         PositionImage (
 254                                 (*i)->image->scale (
 255                                         scaled_size,
 256                                         Scaler::from_id ("bicubic"),
 257                                         (*i)->image->pixel_format (),
 258                                         true
 259                                         ),
 260                                 Position<int> (
 261                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 262                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 263                                         )
 264                                 )
 265                         );
 266         }
 267
 268         return all;
 269 }
 270
 271 list<PositionImage>
 272 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub)
 273 {
 274         list<PositionImage> all;
 275         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 276                 if (!(*i)->subs.empty ()) {
 277                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 278                 }
 279         }
 280
 281         return all;
 282 }
 283
 284 void
 285 Player::set_approximate_size ()
 286 {
 287         _approximate_size = true;
 288 }
 289
 290 shared_ptr<DCPVideo>
 291 Player::black_dcp_video (DCPTime time) const
 292 {
 293         return shared_ptr<DCPVideo> (
 294                 new DCPVideo (
 295                         _black_image,
 296                         EYES_BOTH,
 297                         Crop (),
 298                         _video_container_size,
 299                         _video_container_size,
 300                         Scaler::from_id ("bicubic"),
 301                         Config::instance()->colour_conversions().front().conversion,
 302                         time
 303                 )
 304         );
 305 }
 306
 307 shared_ptr<DCPVideo>
 308 Player::get_video (DCPTime time, bool accurate)
 309 {
 310         if (!_have_valid_pieces) {
 311                 setup_pieces ();
 312         }
 313
 314         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (time);
 315         if (ov.empty ()) {
 316                 /* No video content at this time */
 317                 return black_dcp_video (time);
 318         }
 319
 320         /* Create a DCPVideo from the content's video at this time */
 321
 322         shared_ptr<Piece> piece = ov.back ();
 323         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 324         assert (decoder);
 325         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 326         assert (content);
 327
 328         optional<ContentVideo> dec = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 329         if (!dec) {
 330                 return black_dcp_video (time);
 331         }
 332
 333         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 334         if (_approximate_size) {
 335                 image_size.width &= ~3;
 336                 image_size.height &= ~3;
 337         }
 338
 339         shared_ptr<DCPVideo> dcp_video (
 340                 new DCPVideo (
 341                         dec->image,
 342                         dec->eyes,
 343                         content->crop (),
 344                         image_size,
 345                         _video_container_size,
 346                         _film->scaler(),
 347                         content->colour_conversion (),
 348                         time
 349                         )
 350                 );
 351
 352         /* Add subtitles */
 353
 354         ov = overlaps<SubtitleContent> (time);
 355         list<PositionImage> sub_images;
 356
 357         for (list<shared_ptr<Piece> >::const_iterator i = ov.begin(); i != ov.end(); ++i) {
 358                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 359                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 360                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 361                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 362
 363                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (from, to);
 364                 if (!image_subtitles.empty ()) {
 365                         list<PositionImage> im = process_content_image_subtitles (
 366                                 subtitle_content,
 367                                 image_subtitles
 368                                 );
 369
 370                         copy (im.begin(), im.end(), back_inserter (sub_images));
 371                 }
 372
 373                 if (_burn_subtitles) {
 374                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (from, to);
 375                         if (!text_subtitles.empty ()) {
 376                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 377                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 378                         }
 379                 }
 380         }
 381
 382         if (!sub_images.empty ()) {
 383                 dcp_video->set_subtitle (merge (sub_images));
 384         }
 385
 386         return dcp_video;
 387 }
 388
 389 shared_ptr<AudioBuffers>
 390 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 391 {
 392         if (!_have_valid_pieces) {
 393                 setup_pieces ();
 394         }
 395
 396         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 397
 398         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 399         audio->make_silent ();
 400
 401         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time);
 402         if (ov.empty ()) {
 403                 return audio;
 404         }
 405
 406         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 407
 408                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 409                 assert (content);
 410                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 411                 assert (decoder);
 412
 413                 if (content->content_audio_frame_rate() == 0) {
 414                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
 415                          * audio stream).
 416                          */
 417                         continue;
 418                 }
 419
 420                 AudioFrame const content_time = dcp_to_content_audio (*i, time);
 421
 422                 /* Audio from this piece's decoder (which might be more than what we asked for) */
 423                 shared_ptr<ContentAudio> all = decoder->get_audio (content_time, length_frames, accurate);
 424
 425                 /* Gain */
 426                 if (content->audio_gain() != 0) {
 427                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 428                         gain->apply_gain (content->audio_gain ());
 429                         all->audio = gain;
 430                 }
 431
 432                 /* Remap channels */
 433                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 434                 dcp_mapped->make_silent ();
 435                 AudioMapping map = content->audio_mapping ();
 436                 for (int i = 0; i < map.content_channels(); ++i) {
 437                         for (int j = 0; j < _film->audio_channels(); ++j) {
 438                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 439                                         dcp_mapped->accumulate_channel (
 440                                                 all->audio.get(),
 441                                                 i,
 442                                                 j,
 443                                                 map.get (i, static_cast<dcp::Channel> (j))
 444                                                 );
 445                                 }
 446                         }
 447                 }
 448
 449                 all->audio = dcp_mapped;
 450
 451                 /* Delay */
 452                 /* XXX
 453                 audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
 454                 if (audio->dcp_time < 0) {
 455                         int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
 456                         if (frames >= audio->audio->frames ()) {
 457                                 return;
 458                         }
 459
 460                         shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->audio->channels(), audio->audio->frames() - frames));
 461                         trimmed->copy_from (audio->audio.get(), audio->audio->frames() - frames, frames, 0);
 462
 463                         audio->audio = trimmed;
 464                         audio->dcp_time = 0;
 465                 }
 466                 */
 467
 468                 audio->accumulate_frames (all->audio.get(), all->frame - content_time, 0, min (AudioFrame (all->audio->frames()), length_frames));
 469         }
 470
 471         return audio;
 472 }
 473
 474 VideoFrame
 475 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 476 {
 477         /* s is the offset of t from the start position of this content */
 478         DCPTime s = t - piece->content->position ();
 479         s = DCPTime (max (int64_t (0), s.get ()));
 480         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 481
 482         /* Convert this to the content frame */
 483         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 484 }
 485
 486 AudioFrame
 487 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 488 {
 489         /* s is the offset of t from the start position of this content */
 490         DCPTime s = t - piece->content->position ();
 491         s = DCPTime (max (int64_t (0), s.get ()));
 492         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 493
 494         /* Convert this to the content frame */
 495         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 496 }
 497
 498 ContentTime
 499 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 500 {
 501         /* s is the offset of t from the start position of this content */
 502         DCPTime s = t - piece->content->position ();
 503         s = DCPTime (max (int64_t (0), s.get ()));
 504         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 505
 506         return ContentTime (s, piece->frc);
 507 }
 508
 509 void
 510 PlayerStatistics::dump (shared_ptr<Log> log) const
 511 {
 512         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
 513         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()));
 514 }
 515
 516 PlayerStatistics const &
 517 Player::statistics () const
 518 {
 519         return _statistics;
 520 }