src/lib/ffmpeg_decoder.cc

   1 /*
   2     Copyright (C) 2012-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 /** @file  src/ffmpeg_decoder.cc
  23  *  @brief A decoder using FFmpeg to decode content.
  24  */
  25
  26
  27 #include "audio_buffers.h"
  28 #include "audio_content.h"
  29 #include "audio_decoder.h"
  30 #include "compose.hpp"
  31 #include "dcpomatic_log.h"
  32 #include "exceptions.h"
  33 #include "ffmpeg_audio_stream.h"
  34 #include "ffmpeg_content.h"
  35 #include "ffmpeg_decoder.h"
  36 #include "ffmpeg_subtitle_stream.h"
  37 #include "film.h"
  38 #include "filter.h"
  39 #include "frame_interval_checker.h"
  40 #include "image.h"
  41 #include "log.h"
  42 #include "raw_image_proxy.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "util.h"
  46 #include "video_decoder.h"
  47 #include "video_filter_graph.h"
  48 #include <dcp/subtitle_string.h>
  49 #include <sub/ssa_reader.h>
  50 #include <sub/subtitle.h>
  51 #include <sub/collect.h>
  52 extern "C" {
  53 #include <libavcodec/avcodec.h>
  54 #include <libavformat/avformat.h>
  55 }
  56 #include <boost/algorithm/string.hpp>
  57 #include <iomanip>
  58 #include <iostream>
  59 #include <vector>
  60 #include <stdint.h>
  61
  62 #include "i18n.h"
  63
  64
  65 using std::cout;
  66 using std::dynamic_pointer_cast;
  67 using std::make_shared;
  68 using std::min;
  69 using std::shared_ptr;
  70 using std::string;
  71 using std::vector;
  72 using boost::optional;
  73 using dcp::Size;
  74 using namespace dcpomatic;
  75
  76
  77 FFmpegDecoder::FFmpegDecoder (shared_ptr<const Film> film, shared_ptr<const FFmpegContent> c, bool fast)
  78         : FFmpeg (c)
  79         , Decoder (film)
  80         , _filter_graphs(c->filters(), dcp::Fraction(lrint(_ffmpeg_content->video_frame_rate().get_value_or(24) * 1000), 1000))
  81 {
  82         if (c->video && c->video->use()) {
  83                 video = make_shared<VideoDecoder>(this, c);
  84                 _pts_offset = pts_offset (c->ffmpeg_audio_streams(), c->first_video(), c->active_video_frame_rate(film));
  85                 /* It doesn't matter what size or pixel format this is, it just needs to be black */
  86                 _black_image = make_shared<Image>(AV_PIX_FMT_RGB24, dcp::Size (128, 128), Image::Alignment::PADDED);
  87                 _black_image->make_black ();
  88         } else {
  89                 _pts_offset = {};
  90         }
  91
  92         if (c->audio) {
  93                 audio = make_shared<AudioDecoder>(this, c->audio, fast);
  94         }
  95
  96         if (c->only_text()) {
  97                 text.push_back (make_shared<TextDecoder>(this, c->only_text()));
  98                 /* XXX: we should be calling maybe_set_position() on this TextDecoder, but we can't easily find
  99                  * the time of the first subtitle at this point.
 100                  */
 101         }
 102
 103         for (auto i: c->ffmpeg_audio_streams()) {
 104                 _next_time[i] = boost::optional<dcpomatic::ContentTime>();
 105         }
 106 }
 107
 108
 109 FFmpegDecoder::FlushResult
 110 FFmpegDecoder::flush ()
 111 {
 112         LOG_DEBUG_PLAYER("Flush FFmpeg decoder: current state %1", static_cast<int>(_flush_state));
 113
 114         switch (_flush_state) {
 115         case FlushState::CODECS:
 116                 if (flush_codecs() == FlushResult::DONE) {
 117                         LOG_DEBUG_PLAYER_NC("Finished flushing codecs");
 118                         _flush_state = FlushState::AUDIO_DECODER;
 119                 }
 120                 break;
 121         case FlushState::AUDIO_DECODER:
 122                 if (audio) {
 123                         audio->flush();
 124                 }
 125                 LOG_DEBUG_PLAYER_NC("Finished flushing audio decoder");
 126                 _flush_state = FlushState::FILL;
 127                 break;
 128         case FlushState::FILL:
 129                 if (flush_fill() == FlushResult::DONE) {
 130                         LOG_DEBUG_PLAYER_NC("Finished flushing fills");
 131                         return FlushResult::DONE;
 132                 }
 133                 break;
 134         }
 135
 136         return FlushResult::AGAIN;
 137 }
 138
 139
 140 /** @return true if we have finished flushing the codecs */
 141 FFmpegDecoder::FlushResult
 142 FFmpegDecoder::flush_codecs()
 143 {
 144         bool did_something = false;
 145         if (video) {
 146                 if (decode_and_process_video_packet(nullptr)) {
 147                         did_something = true;
 148                 }
 149         }
 150
 151         for (auto i: ffmpeg_content()->ffmpeg_audio_streams()) {
 152                 auto context = _codec_context[i->index(_format_context)];
 153                 int r = avcodec_send_packet (context, nullptr);
 154                 if (r < 0 && r != AVERROR_EOF) {
 155                         /* EOF can happen if we've already sent a flush packet */
 156                         throw DecodeError (N_("avcodec_send_packet"), N_("FFmpegDecoder::flush"), r);
 157                 }
 158                 r = avcodec_receive_frame (context, audio_frame(i));
 159                 if (r >= 0) {
 160                         process_audio_frame (i);
 161                         did_something = true;
 162                 }
 163         }
 164
 165         return did_something ? FlushResult::AGAIN : FlushResult::DONE;
 166 }
 167
 168
 169 FFmpegDecoder::FlushResult
 170 FFmpegDecoder::flush_fill()
 171 {
 172         /* Make sure all streams are the same length and round up to the next video frame */
 173
 174         bool did_something = false;
 175
 176         auto const frc = film()->active_frame_rate_change(_ffmpeg_content->position());
 177         ContentTime full_length (_ffmpeg_content->full_length(film()), frc);
 178         full_length = full_length.ceil (frc.source);
 179         if (video && !video->ignore()) {
 180                 double const vfr = _ffmpeg_content->video_frame_rate().get();
 181                 auto const f = full_length.frames_round (vfr);
 182                 auto const v = video->position(film()).get_value_or(ContentTime()).frames_round(vfr) + 1;
 183                 if (v < f) {
 184                         video->emit(film(), make_shared<const RawImageProxy>(_black_image), v);
 185                         did_something = true;
 186                 }
 187         }
 188
 189         if (audio && !audio->ignore()) {
 190                 for (auto i: _ffmpeg_content->ffmpeg_audio_streams ()) {
 191                         auto const a = audio->stream_position(film(), i);
 192                         /* Unfortunately if a is 0 that really means that we don't know the stream position since
 193                            there has been no data on it since the last seek.  In this case we'll just do nothing
 194                            here.  I'm not sure if that's the right idea.
 195                         */
 196                         if (a > ContentTime() && a < full_length) {
 197                                 LOG_DEBUG_PLAYER("Flush inserts silence at %1", to_string(a));
 198                                 auto to_do = min (full_length - a, ContentTime::from_seconds (0.1));
 199                                 auto silence = make_shared<AudioBuffers>(i->channels(), to_do.frames_ceil (i->frame_rate()));
 200                                 silence->make_silent ();
 201                                 audio->emit (film(), i, silence, a, true);
 202                                 did_something = true;
 203                         }
 204                 }
 205         }
 206
 207         return did_something ? FlushResult::AGAIN : FlushResult::DONE;
 208 }
 209
 210
 211 bool
 212 FFmpegDecoder::pass ()
 213 {
 214         auto packet = av_packet_alloc();
 215         DCPOMATIC_ASSERT (packet);
 216
 217         int r = av_read_frame (_format_context, packet);
 218
 219         /* AVERROR_INVALIDDATA can apparently be returned sometimes even when av_read_frame
 220            has pretty-much succeeded (and hence generated data which should be processed).
 221            Hence it makes sense to continue here in that case.
 222         */
 223         if (r < 0 && r != AVERROR_INVALIDDATA) {
 224                 LOG_DEBUG_PLAYER("FFpmegDecoder::pass flushes because av_read_frame returned %1", r);
 225                 if (r != AVERROR_EOF) {
 226                         /* Maybe we should fail here, but for now we'll just finish off instead */
 227                         char buf[256];
 228                         av_strerror (r, buf, sizeof(buf));
 229                         LOG_ERROR (N_("error on av_read_frame (%1) (%2)"), &buf[0], r);
 230                 }
 231
 232                 av_packet_free (&packet);
 233                 return flush() == FlushResult::DONE;
 234         }
 235
 236         int const si = packet->stream_index;
 237         auto fc = _ffmpeg_content;
 238
 239         if (_video_stream && si == _video_stream.get() && video && !video->ignore()) {
 240                 decode_and_process_video_packet (packet);
 241         } else if (fc->subtitle_stream() && fc->subtitle_stream()->uses_index(_format_context, si) && !only_text()->ignore()) {
 242                 decode_and_process_subtitle_packet (packet);
 243         } else {
 244                 decode_and_process_audio_packet (packet);
 245         }
 246
 247         av_packet_free (&packet);
 248         return false;
 249 }
 250
 251
 252 /** @param data pointer to array of pointers to buffers.
 253  *  Only the first buffer will be used for non-planar data, otherwise there will be one per channel.
 254  */
 255 static
 256 shared_ptr<AudioBuffers>
 257 deinterleave_audio(shared_ptr<FFmpegAudioStream> stream, AVFrame* frame)
 258 {
 259         auto format = static_cast<AVSampleFormat>(frame->format);
 260
 261         /* XXX: can't we use swr_convert() to do the format conversion? */
 262
 263         int const channels = frame->channels;
 264         int const frames = frame->nb_samples;
 265         int const total_samples = frames * channels;
 266         auto audio = make_shared<AudioBuffers>(channels, frames);
 267         auto data = audio->data();
 268
 269         switch (format) {
 270         case AV_SAMPLE_FMT_U8:
 271         {
 272                 auto p = reinterpret_cast<uint8_t *> (frame->data[0]);
 273                 int sample = 0;
 274                 int channel = 0;
 275                 for (int i = 0; i < total_samples; ++i) {
 276                         data[channel][sample] = float(*p++) / (1 << 23);
 277
 278                         ++channel;
 279                         if (channel == channels) {
 280                                 channel = 0;
 281                                 ++sample;
 282                         }
 283                 }
 284         }
 285         break;
 286
 287         case AV_SAMPLE_FMT_S16:
 288         {
 289                 auto p = reinterpret_cast<int16_t *> (frame->data[0]);
 290                 int sample = 0;
 291                 int channel = 0;
 292                 for (int i = 0; i < total_samples; ++i) {
 293                         data[channel][sample] = float(*p++) / (1 << 15);
 294
 295                         ++channel;
 296                         if (channel == channels) {
 297                                 channel = 0;
 298                                 ++sample;
 299                         }
 300                 }
 301         }
 302         break;
 303
 304         case AV_SAMPLE_FMT_S16P:
 305         {
 306                 auto p = reinterpret_cast<int16_t **> (frame->data);
 307                 for (int i = 0; i < channels; ++i) {
 308                         for (int j = 0; j < frames; ++j) {
 309                                 data[i][j] = static_cast<float>(p[i][j]) / (1 << 15);
 310                         }
 311                 }
 312         }
 313         break;
 314
 315         case AV_SAMPLE_FMT_S32:
 316         {
 317                 auto p = reinterpret_cast<int32_t *> (frame->data[0]);
 318                 int sample = 0;
 319                 int channel = 0;
 320                 for (int i = 0; i < total_samples; ++i) {
 321                         data[channel][sample] = static_cast<float>(*p++) / 2147483648;
 322
 323                         ++channel;
 324                         if (channel == channels) {
 325                                 channel = 0;
 326                                 ++sample;
 327                         }
 328                 }
 329         }
 330         break;
 331
 332         case AV_SAMPLE_FMT_S32P:
 333         {
 334                 auto p = reinterpret_cast<int32_t **> (frame->data);
 335                 for (int i = 0; i < channels; ++i) {
 336                         for (int j = 0; j < frames; ++j) {
 337                                 data[i][j] = static_cast<float>(p[i][j]) / 2147483648;
 338                         }
 339                 }
 340         }
 341         break;
 342
 343         case AV_SAMPLE_FMT_FLT:
 344         {
 345                 auto p = reinterpret_cast<float*> (frame->data[0]);
 346                 int sample = 0;
 347                 int channel = 0;
 348                 for (int i = 0; i < total_samples; ++i) {
 349                         data[channel][sample] = *p++;
 350
 351                         ++channel;
 352                         if (channel == channels) {
 353                                 channel = 0;
 354                                 ++sample;
 355                         }
 356                 }
 357         }
 358         break;
 359
 360         case AV_SAMPLE_FMT_FLTP:
 361         {
 362                 auto p = reinterpret_cast<float**> (frame->data);
 363                 DCPOMATIC_ASSERT(channels <= stream->channels());
 364                 for (int i = 0; i < channels; ++i) {
 365                         memcpy (data[i], p[i], frames * sizeof(float));
 366                 }
 367         }
 368         break;
 369
 370         default:
 371                 throw DecodeError (String::compose(_("Unrecognised audio sample format (%1)"), static_cast<int>(format)));
 372         }
 373
 374         return audio;
 375 }
 376
 377
 378 AVSampleFormat
 379 FFmpegDecoder::audio_sample_format (shared_ptr<FFmpegAudioStream> stream) const
 380 {
 381         return static_cast<AVSampleFormat>(stream->stream(_format_context)->codecpar->format);
 382 }
 383
 384
 385 int
 386 FFmpegDecoder::bytes_per_audio_sample (shared_ptr<FFmpegAudioStream> stream) const
 387 {
 388         return av_get_bytes_per_sample (audio_sample_format (stream));
 389 }
 390
 391
 392 void
 393 FFmpegDecoder::seek (ContentTime time, bool accurate)
 394 {
 395         Decoder::seek (time, accurate);
 396
 397         /* If we are doing an `accurate' seek, we need to use pre-roll, as
 398            we don't really know what the seek will give us.
 399         */
 400
 401         auto pre_roll = accurate ? ContentTime::from_seconds (2) : ContentTime (0);
 402         time -= pre_roll;
 403
 404         /* XXX: it seems debatable whether PTS should be used here...
 405            http://www.mjbshaw.com/2012/04/seeking-in-ffmpeg-know-your-timestamp.html
 406         */
 407
 408         optional<int> stream;
 409
 410         if (_video_stream) {
 411                 stream = _video_stream;
 412         } else {
 413                 DCPOMATIC_ASSERT (_ffmpeg_content->audio);
 414                 auto s = dynamic_pointer_cast<FFmpegAudioStream>(_ffmpeg_content->audio->stream());
 415                 if (s) {
 416                         stream = s->index (_format_context);
 417                 }
 418         }
 419
 420         DCPOMATIC_ASSERT (stream);
 421
 422         auto u = time - _pts_offset;
 423         if (u < ContentTime ()) {
 424                 u = ContentTime ();
 425         }
 426         av_seek_frame (
 427                 _format_context,
 428                 stream.get(),
 429                 u.seconds() / av_q2d (_format_context->streams[stream.get()]->time_base),
 430                 AVSEEK_FLAG_BACKWARD
 431                 );
 432
 433         /* Force re-creation of filter graphs to reset them and hence to make sure
 434            they don't have any pre-seek frames knocking about.
 435         */
 436         _filter_graphs.clear();
 437
 438         if (video_codec_context ()) {
 439                 avcodec_flush_buffers (video_codec_context());
 440         }
 441
 442         for (auto i: ffmpeg_content()->ffmpeg_audio_streams()) {
 443                 avcodec_flush_buffers (_codec_context[i->index(_format_context)]);
 444         }
 445
 446         if (subtitle_codec_context ()) {
 447                 avcodec_flush_buffers (subtitle_codec_context ());
 448         }
 449
 450         _have_current_subtitle = false;
 451
 452         for (auto& i: _next_time) {
 453                 i.second = boost::optional<dcpomatic::ContentTime>();
 454         }
 455 }
 456
 457
 458 shared_ptr<FFmpegAudioStream>
 459 FFmpegDecoder::audio_stream_from_index (int index) const
 460 {
 461         /* XXX: inefficient */
 462         auto streams = ffmpeg_content()->ffmpeg_audio_streams();
 463         auto stream = streams.begin();
 464         while (stream != streams.end() && !(*stream)->uses_index(_format_context, index)) {
 465                 ++stream;
 466         }
 467
 468         if (stream == streams.end ()) {
 469                 return {};
 470         }
 471
 472         return *stream;
 473 }
 474
 475
 476 void
 477 FFmpegDecoder::process_audio_frame (shared_ptr<FFmpegAudioStream> stream)
 478 {
 479         auto frame = audio_frame (stream);
 480         auto data = deinterleave_audio(stream, frame);
 481
 482         auto const time_base = stream->stream(_format_context)->time_base;
 483
 484         ContentTime ct;
 485         if (frame->pts == AV_NOPTS_VALUE) {
 486                 /* In some streams we see not every frame coming through with a timestamp; for those
 487                    that have AV_NOPTS_VALUE we need to work out the timestamp ourselves.  This is
 488                    particularly noticeable with TrueHD streams (see #1111).
 489                    */
 490                 if (_next_time[stream]) {
 491                         ct = *_next_time[stream];
 492                 }
 493         } else {
 494                 ct = ContentTime::from_seconds (
 495                         frame->best_effort_timestamp *
 496                         av_q2d(time_base))
 497                         + _pts_offset;
 498                 LOG_DEBUG_PLAYER(
 499                         "Process audio with timestamp %1 (BET %2, timebase %3/%4, (PTS offset %5)",
 500                         to_string(ct),
 501                         frame->best_effort_timestamp,
 502                         time_base.num,
 503                         time_base.den,
 504                         to_string(_pts_offset)
 505                         );
 506         }
 507
 508         _next_time[stream] = ct + ContentTime::from_frames(data->frames(), stream->frame_rate());
 509
 510         if (ct < ContentTime()) {
 511                 /* Discard audio data that comes before time 0 */
 512                 auto const remove = min (int64_t(data->frames()), (-ct).frames_ceil(double(stream->frame_rate())));
 513                 data->move (data->frames() - remove, remove, 0);
 514                 data->set_frames (data->frames() - remove);
 515                 ct += ContentTime::from_frames (remove, stream->frame_rate());
 516         }
 517
 518         if (ct < ContentTime()) {
 519                 LOG_WARNING (
 520                         "Crazy timestamp %1 for %2 samples in stream %3 (ts=%4 tb=%5, off=%6)",
 521                         to_string(ct),
 522                         data->frames(),
 523                         stream->id(),
 524                         frame->best_effort_timestamp,
 525                         av_q2d(time_base),
 526                         to_string(_pts_offset)
 527                         );
 528         }
 529
 530         /* Give this data provided there is some, and its time is sane */
 531         if (ct >= ContentTime() && data->frames() > 0) {
 532                 audio->emit (film(), stream, data, ct);
 533         }
 534 }
 535
 536
 537 void
 538 FFmpegDecoder::decode_and_process_audio_packet (AVPacket* packet)
 539 {
 540         auto stream = audio_stream_from_index (packet->stream_index);
 541         if (!stream) {
 542                 return;
 543         }
 544
 545         auto context = _codec_context[stream->index(_format_context)];
 546         auto frame = audio_frame (stream);
 547
 548         LOG_DEBUG_PLAYER("Send audio packet on stream %1", stream->index(_format_context));
 549         int r = avcodec_send_packet (context, packet);
 550         if (r < 0) {
 551                 LOG_WARNING("avcodec_send_packet returned %1 for an audio packet", r);
 552         }
 553         while (r >= 0) {
 554                 r = avcodec_receive_frame (context, frame);
 555                 if (r == AVERROR(EAGAIN)) {
 556                         /* More input is required */
 557                         LOG_DEBUG_PLAYER_NC("EAGAIN after trying to receive audio frame");
 558                         return;
 559                 }
 560
 561                 /* We choose to be relaxed here about other errors; it seems that there may be valid
 562                  * data to decode even if an error occurred.  #352 may be related (though this was
 563                  * when we were using an old version of the FFmpeg API).
 564                  */
 565                 process_audio_frame (stream);
 566         }
 567 }
 568
 569
 570 bool
 571 FFmpegDecoder::decode_and_process_video_packet (AVPacket* packet)
 572 {
 573         DCPOMATIC_ASSERT (_video_stream);
 574
 575         auto context = video_codec_context();
 576
 577         bool pending = false;
 578         do {
 579                 int r = avcodec_send_packet (context, packet);
 580                 if (r < 0) {
 581                         LOG_WARNING("avcodec_send_packet returned %1 for a video packet", r);
 582                 }
 583
 584                 /* EAGAIN means we should call avcodec_receive_frame and then re-send the same packet */
 585                 pending = r == AVERROR(EAGAIN);
 586
 587                 while (true) {
 588                         r = avcodec_receive_frame (context, _video_frame);
 589                         if (r == AVERROR(EAGAIN) || r == AVERROR_EOF || (r < 0 && !packet)) {
 590                                 /* More input is required, no more frames are coming, or we are flushing and there was
 591                                  * some error which we just want to ignore.
 592                                  */
 593                                 return false;
 594                         } else if (r < 0) {
 595                                 throw DecodeError (N_("avcodec_receive_frame"), N_("FFmpeg::decode_and_process_video_packet"), r);
 596                         }
 597
 598                         process_video_frame ();
 599                 }
 600         } while (pending);
 601
 602         return true;
 603 }
 604
 605
 606 void
 607 FFmpegDecoder::process_video_frame ()
 608 {
 609         auto graph = _filter_graphs.get(dcp::Size(_video_frame->width, _video_frame->height), static_cast<AVPixelFormat>(_video_frame->format));
 610         auto images = graph->process (_video_frame);
 611
 612         for (auto const& i: images) {
 613
 614                 auto image = i.first;
 615
 616                 if (i.second != AV_NOPTS_VALUE) {
 617                         double const pts = i.second * av_q2d(_format_context->streams[_video_stream.get()]->time_base) + _pts_offset.seconds();
 618
 619                         video->emit (
 620                                 film(),
 621                                 make_shared<RawImageProxy>(image),
 622                                 llrint(pts * _ffmpeg_content->active_video_frame_rate(film()))
 623                                 );
 624                 } else {
 625                         LOG_WARNING_NC ("Dropping frame without PTS");
 626                 }
 627         }
 628 }
 629
 630
 631 void
 632 FFmpegDecoder::decode_and_process_subtitle_packet (AVPacket* packet)
 633 {
 634         int got_subtitle;
 635         AVSubtitle sub;
 636         if (avcodec_decode_subtitle2 (subtitle_codec_context(), &sub, &got_subtitle, packet) < 0 || !got_subtitle) {
 637                 return;
 638         }
 639
 640         auto sub_period = subtitle_period (packet, ffmpeg_content()->subtitle_stream()->stream(_format_context), sub);
 641
 642         /* Stop any current subtitle, either at the time it was supposed to stop, or now if now is sooner */
 643         if (_have_current_subtitle) {
 644                 if (_current_subtitle_to) {
 645                         only_text()->emit_stop (min(*_current_subtitle_to, sub_period.from + _pts_offset));
 646                 } else {
 647                         only_text()->emit_stop (sub_period.from + _pts_offset);
 648                 }
 649                 _have_current_subtitle = false;
 650         }
 651
 652         if (sub.num_rects <= 0) {
 653                 /* Nothing new in this subtitle */
 654                 avsubtitle_free (&sub);
 655                 return;
 656         }
 657
 658         /* Subtitle PTS (within the source, not taking into account any of the
 659            source that we may have chopped off for the DCP).
 660         */
 661         ContentTime from;
 662         from = sub_period.from + _pts_offset;
 663         if (sub_period.to) {
 664                 _current_subtitle_to = *sub_period.to + _pts_offset;
 665         } else {
 666                 _current_subtitle_to = optional<ContentTime>();
 667                 _have_current_subtitle = true;
 668         }
 669
 670         ContentBitmapText bitmap_text(from);
 671         for (unsigned int i = 0; i < sub.num_rects; ++i) {
 672                 auto const rect = sub.rects[i];
 673
 674                 switch (rect->type) {
 675                 case SUBTITLE_NONE:
 676                         break;
 677                 case SUBTITLE_BITMAP:
 678                         bitmap_text.subs.push_back(process_bitmap_subtitle(rect));
 679                         break;
 680                 case SUBTITLE_TEXT:
 681                         cout << "XXX: SUBTITLE_TEXT " << rect->text << "\n";
 682                         break;
 683                 case SUBTITLE_ASS:
 684                         process_ass_subtitle (rect->ass, from);
 685                         break;
 686                 }
 687         }
 688
 689         if (!bitmap_text.subs.empty()) {
 690                 only_text()->emit_bitmap_start(bitmap_text);
 691         }
 692
 693         if (_current_subtitle_to) {
 694                 only_text()->emit_stop (*_current_subtitle_to);
 695         }
 696
 697         avsubtitle_free (&sub);
 698 }
 699
 700
 701 BitmapText
 702 FFmpegDecoder::process_bitmap_subtitle (AVSubtitleRect const * rect)
 703 {
 704         /* Note BGRA is expressed little-endian, so the first byte in the word is B, second
 705            G, third R, fourth A.
 706         */
 707         auto image = make_shared<Image>(AV_PIX_FMT_BGRA, dcp::Size (rect->w, rect->h), Image::Alignment::PADDED);
 708
 709 #ifdef DCPOMATIC_HAVE_AVSUBTITLERECT_PICT
 710         /* Start of the first line in the subtitle */
 711         auto sub_p = rect->pict.data[0];
 712         /* sub_p looks up into a BGRA palette which is at rect->pict.data[1];
 713            (i.e. first byte B, second G, third R, fourth A)
 714         */
 715         auto const palette = rect->pict.data[1];
 716 #else
 717         /* Start of the first line in the subtitle */
 718         auto sub_p = rect->data[0];
 719         /* sub_p looks up into a BGRA palette which is at rect->data[1].
 720            (first byte B, second G, third R, fourth A)
 721         */
 722         auto const* palette = rect->data[1];
 723 #endif
 724         /* And the stream has a map of those palette colours to colours
 725            chosen by the user; created a `mapped' palette from those settings.
 726         */
 727         auto colour_map = ffmpeg_content()->subtitle_stream()->colours();
 728         vector<RGBA> mapped_palette (rect->nb_colors);
 729         for (int i = 0; i < rect->nb_colors; ++i) {
 730                 RGBA c (palette[2], palette[1], palette[0], palette[3]);
 731                 auto j = colour_map.find (c);
 732                 if (j != colour_map.end ()) {
 733                         mapped_palette[i] = j->second;
 734                 } else {
 735                         /* This colour was not found in the FFmpegSubtitleStream's colour map; probably because
 736                            it is from a project that was created before this stuff was added.  Just use the
 737                            colour straight from the original palette.
 738                         */
 739                         mapped_palette[i] = c;
 740                 }
 741                 palette += 4;
 742         }
 743
 744         /* Start of the output data */
 745         auto out_p = image->data()[0];
 746
 747         for (int y = 0; y < rect->h; ++y) {
 748                 auto sub_line_p = sub_p;
 749                 auto out_line_p = out_p;
 750                 for (int x = 0; x < rect->w; ++x) {
 751                         auto const p = mapped_palette[*sub_line_p++];
 752                         *out_line_p++ = p.b;
 753                         *out_line_p++ = p.g;
 754                         *out_line_p++ = p.r;
 755                         *out_line_p++ = p.a;
 756                 }
 757 #ifdef DCPOMATIC_HAVE_AVSUBTITLERECT_PICT
 758                 sub_p += rect->pict.linesize[0];
 759 #else
 760                 sub_p += rect->linesize[0];
 761 #endif
 762                 out_p += image->stride()[0];
 763         }
 764
 765         int target_width = subtitle_codec_context()->width;
 766         if (target_width == 0 && video_codec_context()) {
 767                 /* subtitle_codec_context()->width == 0 has been seen in the wild but I don't
 768                    know if it's supposed to mean something from FFmpeg's point of view.
 769                 */
 770                 target_width = video_codec_context()->width;
 771         }
 772         int target_height = subtitle_codec_context()->height;
 773         if (target_height == 0 && video_codec_context()) {
 774                 target_height = video_codec_context()->height;
 775         }
 776         DCPOMATIC_ASSERT (target_width);
 777         DCPOMATIC_ASSERT (target_height);
 778         dcpomatic::Rect<double> const scaled_rect (
 779                 static_cast<double>(rect->x) / target_width,
 780                 static_cast<double>(rect->y) / target_height,
 781                 static_cast<double>(rect->w) / target_width,
 782                 static_cast<double>(rect->h) / target_height
 783                 );
 784
 785         return { image, scaled_rect };
 786 }
 787
 788
 789 void
 790 FFmpegDecoder::process_ass_subtitle (string ass, ContentTime from)
 791 {
 792         /* We have no styles and no Format: line, so I'm assuming that FFmpeg
 793            produces a single format of Dialogue: lines...
 794         */
 795
 796         int commas = 0;
 797         string text;
 798         for (size_t i = 0; i < ass.length(); ++i) {
 799                 if (commas < 9 && ass[i] == ',') {
 800                         ++commas;
 801                 } else if (commas == 9) {
 802                         text += ass[i];
 803                 }
 804         }
 805
 806         if (text.empty ()) {
 807                 return;
 808         }
 809
 810         sub::RawSubtitle base;
 811         auto raw = sub::SSAReader::parse_line (
 812                 base,
 813                 text,
 814                 _ffmpeg_content->video->size().width,
 815                 _ffmpeg_content->video->size().height,
 816                 sub::Colour(1, 1, 1)
 817                 );
 818
 819         for (auto const& i: sub::collect<vector<sub::Subtitle>>(raw)) {
 820                 only_text()->emit_plain_start (from, i);
 821         }
 822 }