src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "ratio.h"
  38 #include "log.h"
  39 #include "scaler.h"
  40 #include "render_subtitles.h"
  41 #include "dcp_video.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44
  45 using std::list;
  46 using std::cout;
  47 using std::min;
  48 using std::max;
  49 using std::min;
  50 using std::vector;
  51 using std::pair;
  52 using std::map;
  53 using std::make_pair;
  54 using boost::shared_ptr;
  55 using boost::weak_ptr;
  56 using boost::dynamic_pointer_cast;
  57 using boost::optional;
  58
  59 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  60         : _film (f)
  61         , _playlist (p)
  62         , _have_valid_pieces (false)
  63         , _approximate_size (false)
  64         , _burn_subtitles (false)
  65 {
  66         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  67         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  68         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  69         set_video_container_size (_film->frame_size ());
  70 }
  71
  72 void
  73 Player::setup_pieces ()
  74 {
  75         list<shared_ptr<Piece> > old_pieces = _pieces;
  76         _pieces.clear ();
  77
  78         ContentList content = _playlist->content ();
  79
  80         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  81
  82                 if (!(*i)->paths_valid ()) {
  83                         continue;
  84                 }
  85
  86                 shared_ptr<Decoder> decoder;
  87                 optional<FrameRateChange> frc;
  88
  89                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  90                 DCPTime best_overlap_t;
  91                 shared_ptr<VideoContent> best_overlap;
  92                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  93                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  94                         if (!vc) {
  95                                 continue;
  96                         }
  97
  98                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
  99                         if (overlap > best_overlap_t) {
 100                                 best_overlap = vc;
 101                                 best_overlap_t = overlap;
 102                         }
 103                 }
 104
 105                 optional<FrameRateChange> best_overlap_frc;
 106                 if (best_overlap) {
 107                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 108                 } else {
 109                         /* No video overlap; e.g. if the DCP is just audio */
 110                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 111                 }
 112
 113                 /* FFmpeg */
 114                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 115                 if (fc) {
 116                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 117                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 118                 }
 119
 120                 /* ImageContent */
 121                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 122                 if (ic) {
 123                         /* See if we can re-use an old ImageDecoder */
 124                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 125                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 126                                 if (imd && imd->content() == ic) {
 127                                         decoder = imd;
 128                                 }
 129                         }
 130
 131                         if (!decoder) {
 132                                 decoder.reset (new ImageDecoder (ic));
 133                         }
 134
 135                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 136                 }
 137
 138                 /* SndfileContent */
 139                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 140                 if (sc) {
 141                         decoder.reset (new SndfileDecoder (sc));
 142                         frc = best_overlap_frc;
 143                 }
 144
 145                 /* SubRipContent */
 146                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 147                 if (rc) {
 148                         decoder.reset (new SubRipDecoder (rc));
 149                         frc = best_overlap_frc;
 150                 }
 151
 152                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 153         }
 154
 155         _have_valid_pieces = true;
 156 }
 157
 158 void
 159 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 160 {
 161         shared_ptr<Content> c = w.lock ();
 162         if (!c) {
 163                 return;
 164         }
 165
 166         if (
 167                 property == ContentProperty::POSITION ||
 168                 property == ContentProperty::LENGTH ||
 169                 property == ContentProperty::TRIM_START ||
 170                 property == ContentProperty::TRIM_END ||
 171                 property == ContentProperty::PATH ||
 172                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 173                 ) {
 174
 175                 _have_valid_pieces = false;
 176                 Changed (frequent);
 177
 178         } else if (
 179                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 180                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 181                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 182                 property == VideoContentProperty::VIDEO_CROP ||
 183                 property == VideoContentProperty::VIDEO_SCALE ||
 184                 property == VideoContentProperty::VIDEO_FRAME_RATE
 185                 ) {
 186
 187                 Changed (frequent);
 188         }
 189 }
 190
 191 void
 192 Player::playlist_changed ()
 193 {
 194         _have_valid_pieces = false;
 195         Changed (false);
 196 }
 197
 198 void
 199 Player::set_video_container_size (dcp::Size s)
 200 {
 201         _video_container_size = s;
 202
 203         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 204         _black_image->make_black ();
 205 }
 206
 207 void
 208 Player::film_changed (Film::Property p)
 209 {
 210         /* Here we should notice Film properties that affect our output, and
 211            alert listeners that our output now would be different to how it was
 212            last time we were run.
 213         */
 214
 215         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 216                 Changed (false);
 217         }
 218 }
 219
 220 list<PositionImage>
 221 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs)
 222 {
 223         list<PositionImage> all;
 224
 225         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 226                 if (!(*i)->image) {
 227                         continue;
 228                 }
 229
 230                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 231                 dcp::Size scaled_size;
 232
 233                 in_rect.x += content->subtitle_x_offset ();
 234                 in_rect.y += content->subtitle_y_offset ();
 235
 236                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 237                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 238                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 239
 240                 /* Then we need a corrective translation, consisting of two parts:
 241                  *
 242                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 243                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 244                  *
 245                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 246                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 247                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 248                  *
 249                  * Combining these two translations gives these expressions.
 250                  */
 251
 252                 all.push_back (
 253                         PositionImage (
 254                                 (*i)->image->scale (
 255                                         scaled_size,
 256                                         Scaler::from_id ("bicubic"),
 257                                         (*i)->image->pixel_format (),
 258                                         true
 259                                         ),
 260                                 Position<int> (
 261                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 262                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 263                                         )
 264                                 )
 265                         );
 266         }
 267
 268         return all;
 269 }
 270
 271 list<PositionImage>
 272 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub)
 273 {
 274         list<PositionImage> all;
 275         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 276                 if (!(*i)->subs.empty ()) {
 277                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 278                 }
 279         }
 280
 281         return all;
 282 }
 283
 284 void
 285 Player::set_approximate_size ()
 286 {
 287         _approximate_size = true;
 288 }
 289
 290 shared_ptr<DCPVideo>
 291 Player::black_dcp_video (DCPTime time) const
 292 {
 293         return shared_ptr<DCPVideo> (
 294                 new DCPVideo (
 295                         _black_image,
 296                         EYES_BOTH,
 297                         Crop (),
 298                         _video_container_size,
 299                         _video_container_size,
 300                         Scaler::from_id ("bicubic"),
 301                         Config::instance()->colour_conversions().front().conversion,
 302                         time
 303                 )
 304         );
 305 }
 306
 307 shared_ptr<DCPVideo>
 308 Player::get_video (DCPTime time, bool accurate)
 309 {
 310         if (!_have_valid_pieces) {
 311                 setup_pieces ();
 312         }
 313
 314         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (time);
 315         if (ov.empty ()) {
 316                 /* No video content at this time */
 317                 return black_dcp_video (time);
 318         }
 319
 320         /* Create a DCPVideo from the content's video at this time */
 321
 322         shared_ptr<Piece> piece = ov.back ();
 323         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 324         assert (decoder);
 325         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 326         assert (content);
 327
 328         optional<ContentVideo> dec = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 329         if (!dec) {
 330                 return black_dcp_video (time);
 331         }
 332
 333         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 334         if (_approximate_size) {
 335                 image_size.width &= ~3;
 336                 image_size.height &= ~3;
 337         }
 338
 339         shared_ptr<DCPVideo> dcp_video (
 340                 new DCPVideo (
 341                         dec->image,
 342                         dec->eyes,
 343                         content->crop (),
 344                         image_size,
 345                         _video_container_size,
 346                         _film->scaler(),
 347                         content->colour_conversion (),
 348                         time
 349                         )
 350                 );
 351
 352         /* Add subtitles */
 353
 354         ov = overlaps<SubtitleContent> (time);
 355         list<PositionImage> sub_images;
 356
 357         for (list<shared_ptr<Piece> >::const_iterator i = ov.begin(); i != ov.end(); ++i) {
 358                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 359                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 360                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 361                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 362
 363                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (from, to);
 364                 if (!image_subtitles.empty ()) {
 365                         list<PositionImage> im = process_content_image_subtitles (
 366                                 subtitle_content,
 367                                 image_subtitles
 368                                 );
 369
 370                         copy (im.begin(), im.end(), back_inserter (sub_images));
 371                 }
 372
 373                 if (_burn_subtitles) {
 374                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (from, to);
 375                         if (!text_subtitles.empty ()) {
 376                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 377                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 378                         }
 379                 }
 380         }
 381
 382         if (!sub_images.empty ()) {
 383                 dcp_video->set_subtitle (merge (sub_images));
 384         }
 385
 386         return dcp_video;
 387 }
 388
 389 shared_ptr<AudioBuffers>
 390 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 391 {
 392         if (!_have_valid_pieces) {
 393                 setup_pieces ();
 394         }
 395
 396         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 397
 398         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 399         audio->make_silent ();
 400
 401         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time);
 402         if (ov.empty ()) {
 403                 return audio;
 404         }
 405
 406         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 407
 408                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 409                 assert (content);
 410                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 411                 assert (decoder);
 412
 413                 AudioFrame const content_time = dcp_to_content_audio (*i, time);
 414
 415                 /* Audio from this piece's decoder (which might be more than what we asked for) */
 416                 shared_ptr<ContentAudio> all = decoder->get_audio (content_time, length_frames, accurate);
 417
 418                 /* Gain */
 419                 if (content->audio_gain() != 0) {
 420                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 421                         gain->apply_gain (content->audio_gain ());
 422                         all->audio = gain;
 423                 }
 424
 425                 /* Remap channels */
 426                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 427                 dcp_mapped->make_silent ();
 428                 AudioMapping map = content->audio_mapping ();
 429                 for (int i = 0; i < map.content_channels(); ++i) {
 430                         for (int j = 0; j < _film->audio_channels(); ++j) {
 431                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 432                                         dcp_mapped->accumulate_channel (
 433                                                 all->audio.get(),
 434                                                 i,
 435                                                 j,
 436                                                 map.get (i, static_cast<dcp::Channel> (j))
 437                                                 );
 438                                 }
 439                         }
 440                 }
 441
 442                 all->audio = dcp_mapped;
 443
 444                 /* Delay */
 445                 /* XXX
 446                 audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
 447                 if (audio->dcp_time < 0) {
 448                         int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
 449                         if (frames >= audio->audio->frames ()) {
 450                                 return;
 451                         }
 452
 453                         shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->audio->channels(), audio->audio->frames() - frames));
 454                         trimmed->copy_from (audio->audio.get(), audio->audio->frames() - frames, frames, 0);
 455
 456                         audio->audio = trimmed;
 457                         audio->dcp_time = 0;
 458                 }
 459                 */
 460
 461                 audio->accumulate_frames (all->audio.get(), all->frame - content_time, 0, min (AudioFrame (all->audio->frames()), length_frames));
 462         }
 463
 464         return audio;
 465 }
 466
 467 VideoFrame
 468 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 469 {
 470         /* s is the offset of t from the start position of this content */
 471         DCPTime s = t - piece->content->position ();
 472         s = DCPTime (max (int64_t (0), s.get ()));
 473         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 474
 475         /* Convert this to the content frame */
 476         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 477 }
 478
 479 AudioFrame
 480 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 481 {
 482         /* s is the offset of t from the start position of this content */
 483         DCPTime s = t - piece->content->position ();
 484         s = DCPTime (max (int64_t (0), s.get ()));
 485         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 486
 487         /* Convert this to the content frame */
 488         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 489 }
 490
 491 ContentTime
 492 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 493 {
 494         /* s is the offset of t from the start position of this content */
 495         DCPTime s = t - piece->content->position ();
 496         s = DCPTime (max (int64_t (0), s.get ()));
 497         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 498
 499         return ContentTime (s, piece->frc);
 500 }
 501
 502 void
 503 PlayerStatistics::dump (shared_ptr<Log> log) const
 504 {
 505         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
 506         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()));
 507 }
 508
 509 PlayerStatistics const &
 510 Player::statistics () const
 511 {
 512         return _statistics;
 513 }