src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "constants.h"
  27 #include "cross.h"
  28 #include "dcp_content_type.h"
  29 #include "dcp_video.h"
  30 #include "dcpomatic_log.h"
  31 #include "film.h"
  32 #include "film_util.h"
  33 #include "frame_info.h"
  34 #include "job.h"
  35 #include "log.h"
  36 #include "ratio.h"
  37 #include "reel_writer.h"
  38 #include "text_content.h"
  39 #include "util.h"
  40 #include "version.h"
  41 #include "writer.h"
  42 #include <dcp/cpl.h>
  43 #include <dcp/locale_convert.h>
  44 #include <dcp/raw_convert.h>
  45 #include <dcp/reel_closed_caption_asset.h>
  46 #include <dcp/reel_file_asset.h>
  47 #include <dcp/reel_subtitle_asset.h>
  48 #include <cerrno>
  49 #include <cfloat>
  50 #include <set>
  51
  52 #include "i18n.h"
  53
  54
  55 /* OS X strikes again */
  56 #undef set_key
  57
  58
  59 using std::cout;
  60 using std::dynamic_pointer_cast;
  61 using std::make_shared;
  62 using std::max;
  63 using std::min;
  64 using std::shared_ptr;
  65 using std::set;
  66 using std::string;
  67 using std::vector;
  68 using std::weak_ptr;
  69 using boost::optional;
  70 #if BOOST_VERSION >= 106100
  71 using namespace boost::placeholders;
  72 #endif
  73 using dcp::Data;
  74 using dcp::ArrayData;
  75 using namespace dcpomatic;
  76
  77
  78 /** @param weak_job Job to report progress to, or 0.
  79  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  80  */
  81 Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, bool text_only)
  82         : WeakConstFilm (weak_film)
  83         , _job(weak_job)
  84         /* These will be reset to sensible values when J2KEncoder is created */
  85         , _maximum_frames_in_memory (8)
  86         , _maximum_queue_size (8)
  87         , _text_only (text_only)
  88 {
  89         auto job = _job.lock ();
  90
  91         int reel_index = 0;
  92         auto const reels = film()->reels();
  93         for (auto p: reels) {
  94                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  95         }
  96
  97         _last_written.resize (reels.size());
  98
  99         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
 100            and captions arrive to the Writer in sequence.  This is not so for video.
 101         */
 102         _audio_reel = _reels.begin ();
 103         _subtitle_reel = _reels.begin ();
 104         for (auto i: film()->closed_caption_tracks()) {
 105                 _caption_reels[i] = _reels.begin ();
 106         }
 107         _atmos_reel = _reels.begin ();
 108
 109         /* Check that the signer is OK */
 110         string reason;
 111         if (!Config::instance()->signer_chain()->valid(&reason)) {
 112                 throw InvalidSignerError (reason);
 113         }
 114 }
 115
 116
 117 void
 118 Writer::start ()
 119 {
 120         if (!_text_only) {
 121                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 122 #ifdef DCPOMATIC_LINUX
 123                 pthread_setname_np (_thread.native_handle(), "writer");
 124 #endif
 125         }
 126 }
 127
 128
 129 Writer::~Writer ()
 130 {
 131         if (!_text_only) {
 132                 terminate_thread (false);
 133         }
 134 }
 135
 136
 137 /** Pass a video frame to the writer for writing to disk at some point.
 138  *  This method can be called with frames out of order.
 139  *  @param encoded JPEG2000-encoded data.
 140  *  @param frame Frame index within the DCP.
 141  *  @param eyes Eyes that this frame image is for.
 142  */
 143 void
 144 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 145 {
 146         boost::mutex::scoped_lock lock (_state_mutex);
 147
 148         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 149                 /* There are too many full frames in memory; wake the main writer thread and
 150                    wait until it sorts everything out */
 151                 _empty_condition.notify_all ();
 152                 _full_condition.wait (lock);
 153         }
 154
 155         QueueItem qi;
 156         qi.type = QueueItem::Type::FULL;
 157         qi.encoded = encoded;
 158         qi.reel = video_reel (frame);
 159         qi.frame = frame - _reels[qi.reel].start ();
 160
 161         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 162
 163         qi.eyes = eyes;
 164         _queue.push_back(qi);
 165         ++_queued_full_in_memory;
 166
 167         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 168         _empty_condition.notify_all ();
 169 }
 170
 171
 172 bool
 173 Writer::can_repeat (Frame frame) const
 174 {
 175         return frame > _reels[video_reel(frame)].start();
 176 }
 177
 178
 179 /** Repeat the last frame that was written to a reel as a new frame.
 180  *  @param frame Frame index within the DCP of the new (repeated) frame.
 181  *  @param eyes Eyes that this repeated frame image is for.
 182  */
 183 void
 184 Writer::repeat (Frame frame, Eyes eyes)
 185 {
 186         boost::mutex::scoped_lock lock (_state_mutex);
 187
 188         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 189                 /* The queue is too big, and the main writer thread can run and fix it, so
 190                    wake it and wait until it has done.
 191                 */
 192                 _empty_condition.notify_all ();
 193                 _full_condition.wait (lock);
 194         }
 195
 196         QueueItem qi;
 197         qi.type = QueueItem::Type::REPEAT;
 198         qi.reel = video_reel (frame);
 199         qi.frame = frame - _reels[qi.reel].start ();
 200         if (film()->three_d() && eyes == Eyes::BOTH) {
 201                 qi.eyes = Eyes::LEFT;
 202                 _queue.push_back (qi);
 203                 qi.eyes = Eyes::RIGHT;
 204                 _queue.push_back (qi);
 205         } else {
 206                 qi.eyes = eyes;
 207                 _queue.push_back (qi);
 208         }
 209
 210         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 211         _empty_condition.notify_all ();
 212 }
 213
 214
 215 void
 216 Writer::fake_write (Frame frame, Eyes eyes)
 217 {
 218         boost::mutex::scoped_lock lock (_state_mutex);
 219
 220         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 221                 /* The queue is too big, and the main writer thread can run and fix it, so
 222                    wake it and wait until it has done.
 223                 */
 224                 _empty_condition.notify_all ();
 225                 _full_condition.wait (lock);
 226         }
 227
 228         size_t const reel = video_reel (frame);
 229         Frame const frame_in_reel = frame - _reels[reel].start ();
 230
 231         QueueItem qi;
 232         qi.type = QueueItem::Type::FAKE;
 233         qi.size = J2KFrameInfo(film()->info_file_handle(_reels[reel].period(), true), frame_in_reel, eyes).size;
 234
 235         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 236
 237         qi.reel = reel;
 238         qi.frame = frame_in_reel;
 239         qi.eyes = eyes;
 240         _queue.push_back(qi);
 241
 242         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 243         _empty_condition.notify_all ();
 244 }
 245
 246
 247 /** Write some audio frames to the DCP.
 248  *  @param audio Audio data.
 249  *  @param time Time of this data within the DCP.
 250  *  This method is not thread safe.
 251  */
 252 void
 253 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 254 {
 255         DCPOMATIC_ASSERT (audio);
 256
 257         int const afr = film()->audio_frame_rate();
 258
 259         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 260
 261         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 262
 263         DCPTime t = time;
 264         while (t < end) {
 265
 266                 if (_audio_reel == _reels.end ()) {
 267                         /* This audio is off the end of the last reel; ignore it */
 268                         return;
 269                 }
 270
 271                 if (end <= _audio_reel->period().to) {
 272                         /* Easy case: we can write all the audio to this reel */
 273                         _audio_reel->write (audio);
 274                         t = end;
 275                 } else if (_audio_reel->period().to <= t) {
 276                         /* This reel is entirely before the start of our audio; just skip the reel */
 277                         ++_audio_reel;
 278                 } else {
 279                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 280                         DCPTime part_lengths[2] = {
 281                                 _audio_reel->period().to - t,
 282                                 end - _audio_reel->period().to
 283                         };
 284
 285                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 286                         Frame part_frames[2] = {
 287                                 part_lengths[0].frames_ceil(afr),
 288                                 part_lengths[1].frames_floor(afr)
 289                         };
 290
 291                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 292
 293                         if (part_frames[0]) {
 294                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 295                                 _audio_reel->write (part);
 296                         }
 297
 298                         if (part_frames[1]) {
 299                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 300                         } else {
 301                                 audio.reset ();
 302                         }
 303
 304                         ++_audio_reel;
 305                         t += part_lengths[0];
 306                 }
 307         }
 308 }
 309
 310
 311 void
 312 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 313 {
 314         if (_atmos_reel->period().to == time) {
 315                 ++_atmos_reel;
 316                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 317         }
 318
 319         /* We assume that we get a video frame's worth of data here */
 320         _atmos_reel->write (atmos, metadata);
 321 }
 322
 323
 324 /** Caller must hold a lock on _state_mutex */
 325 bool
 326 Writer::have_sequenced_image_at_queue_head ()
 327 {
 328         if (_queue.empty ()) {
 329                 return false;
 330         }
 331
 332         _queue.sort ();
 333         auto const & f = _queue.front();
 334         return _last_written[f.reel].next(f);
 335 }
 336
 337
 338 bool
 339 Writer::LastWritten::next (QueueItem qi) const
 340 {
 341         if (qi.eyes == Eyes::BOTH) {
 342                 /* 2D */
 343                 return qi.frame == (_frame + 1);
 344         }
 345
 346         /* 3D */
 347
 348         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 349                 return true;
 350         }
 351
 352         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 353                 return true;
 354         }
 355
 356         return false;
 357 }
 358
 359
 360 void
 361 Writer::LastWritten::update (QueueItem qi)
 362 {
 363         _frame = qi.frame;
 364         _eyes = qi.eyes;
 365 }
 366
 367
 368 void
 369 Writer::thread ()
 370 try
 371 {
 372         start_of_thread ("Writer");
 373
 374         while (true)
 375         {
 376                 boost::mutex::scoped_lock lock (_state_mutex);
 377
 378                 while (true) {
 379
 380                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 381                                 /* We've got something to do: go and do it */
 382                                 break;
 383                         }
 384
 385                         /* Nothing to do: wait until something happens which may indicate that we do */
 386                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 387                         _empty_condition.wait (lock);
 388                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 389                 }
 390
 391                 /* We stop here if we have been asked to finish, and if either the queue
 392                    is empty or we do not have a sequenced image at its head (if this is the
 393                    case we will never terminate as no new frames will be sent once
 394                    _finish is true).
 395                 */
 396                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 397                         /* (Hopefully temporarily) log anything that was not written */
 398                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 399                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 400                                 for (auto const& i: _queue) {
 401                                         if (i.type == QueueItem::Type::FULL) {
 402                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 403                                         } else {
 404                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 405                                         }
 406                                 }
 407                         }
 408                         return;
 409                 }
 410
 411                 /* Write any frames that we can write; i.e. those that are in sequence. */
 412                 while (have_sequenced_image_at_queue_head ()) {
 413                         auto qi = _queue.front ();
 414                         _last_written[qi.reel].update (qi);
 415                         _queue.pop_front ();
 416                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 417                                 --_queued_full_in_memory;
 418                         }
 419
 420                         lock.unlock ();
 421
 422                         auto& reel = _reels[qi.reel];
 423
 424                         switch (qi.type) {
 425                         case QueueItem::Type::FULL:
 426                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 427                                 if (!qi.encoded) {
 428                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 429                                 }
 430                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 431                                 ++_full_written;
 432                                 break;
 433                         case QueueItem::Type::FAKE:
 434                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 435                                 reel.fake_write (qi.size);
 436                                 ++_fake_written;
 437                                 break;
 438                         case QueueItem::Type::REPEAT:
 439                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 440                                 reel.repeat_write (qi.frame, qi.eyes);
 441                                 ++_repeat_written;
 442                                 break;
 443                         }
 444
 445                         lock.lock ();
 446                         _full_condition.notify_all ();
 447                 }
 448
 449                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 450                         /* Too many frames in memory which can't yet be written to the stream.
 451                            Write some FULL frames to disk.
 452                         */
 453
 454                         /* Find one from the back of the queue */
 455                         _queue.sort ();
 456                         auto item = _queue.rbegin();
 457                         while (item != _queue.rend() && (item->type != QueueItem::Type::FULL || !item->encoded)) {
 458                                 ++item;
 459                         }
 460
 461                         DCPOMATIC_ASSERT(item != _queue.rend());
 462                         ++_pushed_to_disk;
 463                         /* For the log message below */
 464                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 465                         lock.unlock ();
 466
 467                         /* i is valid here, even though we don't hold a lock on the mutex,
 468                            since list iterators are unaffected by insertion and only this
 469                            thread could erase the last item in the list.
 470                         */
 471
 472                         LOG_GENERAL("Writer full; pushes %1 to disk while awaiting %2", item->frame, awaiting);
 473
 474                         item->encoded->write_via_temp(
 475                                 film()->j2c_path(item->reel, item->frame, item->eyes, true),
 476                                 film()->j2c_path(item->reel, item->frame, item->eyes, false)
 477                                 );
 478
 479                         lock.lock ();
 480                         item->encoded.reset();
 481                         --_queued_full_in_memory;
 482                         _full_condition.notify_all ();
 483                 }
 484         }
 485 }
 486 catch (...)
 487 {
 488         store_current ();
 489 }
 490
 491
 492 void
 493 Writer::terminate_thread (bool can_throw)
 494 {
 495         boost::this_thread::disable_interruption dis;
 496
 497         boost::mutex::scoped_lock lock (_state_mutex);
 498
 499         _finish = true;
 500         _empty_condition.notify_all ();
 501         _full_condition.notify_all ();
 502         lock.unlock ();
 503
 504         try {
 505                 _thread.join ();
 506         } catch (...) {}
 507
 508         if (can_throw) {
 509                 rethrow ();
 510         }
 511 }
 512
 513
 514 void
 515 Writer::calculate_digests ()
 516 {
 517         auto job = _job.lock ();
 518         if (job) {
 519                 job->sub (_("Computing digests"));
 520         }
 521
 522         boost::asio::io_service service;
 523         boost::thread_group pool;
 524
 525         auto work = make_shared<boost::asio::io_service::work>(service);
 526
 527         int const threads = max (1, Config::instance()->master_encoding_threads());
 528
 529         for (int i = 0; i < threads; ++i) {
 530                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 531         }
 532
 533         std::function<void (int, int64_t, int64_t)> set_progress;
 534         if (job) {
 535                 set_progress = boost::bind(&Writer::set_digest_progress, this, job.get(), _1, _2, _3);
 536         } else {
 537                 set_progress = [](int, int64_t, int64_t) {
 538                         boost::this_thread::interruption_point();
 539                 };
 540         }
 541
 542         int index = 0;
 543
 544         for (auto& i: _reels) {
 545                 service.post(
 546                         boost::bind(
 547                                 &ReelWriter::calculate_digests,
 548                                 &i,
 549                                 std::function<void (int64_t, int64_t)>(boost::bind(set_progress, index, _1, _2))
 550                                 ));
 551                 ++index;
 552         }
 553         service.post(
 554                 boost::bind(
 555                         &Writer::calculate_referenced_digests,
 556                         this,
 557                         std::function<void (int64_t, int64_t)>(boost::bind(set_progress, index, _1, _2))
 558                         ));
 559
 560         work.reset ();
 561
 562         try {
 563                 pool.join_all ();
 564         } catch (boost::thread_interrupted) {
 565                 /* join_all was interrupted, so we need to interrupt the threads
 566                  * in our pool then try again to join them.
 567                  */
 568                 pool.interrupt_all ();
 569                 pool.join_all ();
 570         }
 571
 572         service.stop ();
 573 }
 574
 575
 576 /** @param output_dcp Path to DCP folder to write */
 577 void
 578 Writer::finish (boost::filesystem::path output_dcp)
 579 {
 580         if (_thread.joinable()) {
 581                 LOG_GENERAL_NC ("Terminating writer thread");
 582                 terminate_thread (true);
 583         }
 584
 585         LOG_GENERAL_NC ("Finishing ReelWriters");
 586
 587         for (auto& reel: _reels) {
 588                 write_hanging_text(reel);
 589                 reel.finish(output_dcp);
 590         }
 591
 592         LOG_GENERAL_NC ("Writing XML");
 593
 594         dcp::DCP dcp (output_dcp);
 595
 596         auto cpl = make_shared<dcp::CPL>(
 597                 film()->dcp_name(),
 598                 film()->dcp_content_type()->libdcp_kind(),
 599                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 600                 );
 601
 602         dcp.add (cpl);
 603
 604         calculate_digests ();
 605
 606         /* Add reels */
 607
 608         for (auto& i: _reels) {
 609                 cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
 610         }
 611
 612         /* Add metadata */
 613
 614         auto creator = Config::instance()->dcp_creator();
 615         if (creator.empty()) {
 616                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 617         }
 618
 619         auto issuer = Config::instance()->dcp_issuer();
 620         if (issuer.empty()) {
 621                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 622         }
 623
 624         cpl->set_creator (creator);
 625         cpl->set_issuer (issuer);
 626
 627         cpl->set_ratings (film()->ratings());
 628
 629         vector<dcp::ContentVersion> cv;
 630         for (auto i: film()->content_versions()) {
 631                 /* Make sure we don't end up writing an empty <LabelText> node as some validators
 632                  * complain about that.
 633                  */
 634                 cv.push_back(!i.empty() ? dcp::ContentVersion(i) : dcp::ContentVersion("1"));
 635         }
 636         if (cv.empty()) {
 637                 cv = { dcp::ContentVersion("1") };
 638         }
 639         cpl->set_content_versions (cv);
 640
 641         cpl->set_full_content_title_text (film()->name());
 642         cpl->set_full_content_title_text_language (film()->name_language());
 643         if (film()->release_territory()) {
 644                 cpl->set_release_territory (*film()->release_territory());
 645         }
 646         cpl->set_version_number (film()->version_number());
 647         cpl->set_status (film()->status());
 648         if (film()->chain()) {
 649                 cpl->set_chain (*film()->chain());
 650         }
 651         if (film()->distributor()) {
 652                 cpl->set_distributor (*film()->distributor());
 653         }
 654         if (film()->facility()) {
 655                 cpl->set_facility (*film()->facility());
 656         }
 657         if (film()->luminance()) {
 658                 cpl->set_luminance (*film()->luminance());
 659         }
 660         if (film()->sign_language_video_language()) {
 661                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 662         }
 663
 664         dcp::MCASoundField field;
 665         if (channel_is_mapped(film(), dcp::Channel::BSL) || channel_is_mapped(film(), dcp::Channel::BSR)) {
 666                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 667         } else {
 668                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 669         }
 670
 671         auto const audio_channels = film()->audio_channels();
 672         dcp::MainSoundConfiguration msc(field, audio_channels);
 673         for (auto i: film()->mapped_audio_channels()) {
 674                 if (i < audio_channels) {
 675                         msc.set_mapping(i, static_cast<dcp::Channel>(i));
 676                 }
 677         }
 678
 679         cpl->set_main_sound_configuration(msc);
 680         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 681         cpl->set_main_picture_stored_area (film()->frame_size());
 682
 683         auto active_area = film()->active_area();
 684         if (active_area.width > 0 && active_area.height > 0) {
 685                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 686                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 687         }
 688
 689         auto sl = film()->subtitle_languages().second;
 690         if (!sl.empty()) {
 691                 cpl->set_additional_subtitle_languages(sl);
 692         }
 693
 694         auto signer = Config::instance()->signer_chain();
 695         /* We did check earlier, but check again here to be on the safe side */
 696         string reason;
 697         if (!signer->valid (&reason)) {
 698                 throw InvalidSignerError (reason);
 699         }
 700
 701         dcp.set_issuer(issuer);
 702         dcp.set_creator(creator);
 703         dcp.set_annotation_text(film()->dcp_name());
 704
 705         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 706
 707         LOG_GENERAL (
 708                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 709                 );
 710
 711         write_cover_sheet (output_dcp);
 712 }
 713
 714
 715 void
 716 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 717 {
 718         auto const cover = film()->file("COVER_SHEET.txt");
 719         dcp::File file(cover, "w");
 720         if (!file) {
 721                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 722         }
 723
 724         auto text = Config::instance()->cover_sheet ();
 725         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 726         auto cpls = film()->cpls();
 727         if (!cpls.empty()) {
 728                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 729         }
 730         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 731         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 732
 733         auto audio_language = film()->audio_language();
 734         if (audio_language) {
 735                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 736         } else {
 737                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 738         }
 739
 740         auto subtitle_languages = film()->subtitle_languages();
 741         if (subtitle_languages.first) {
 742                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 743         } else {
 744                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 745         }
 746
 747         boost::uintmax_t size = 0;
 748         for (
 749                 auto i = dcp::filesystem::recursive_directory_iterator(output_dcp);
 750                 i != dcp::filesystem::recursive_directory_iterator();
 751                 ++i) {
 752                 if (dcp::filesystem::is_regular_file(i->path())) {
 753                         size += dcp::filesystem::file_size(i->path());
 754                 }
 755         }
 756
 757         if (size > (1000000000L)) {
 758                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 759         } else {
 760                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 761         }
 762
 763         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 764         auto description = String::compose("%1.%2", ch.first, ch.second);
 765
 766         if (description == "0.0") {
 767                 description = _("None");
 768         } else if (description == "1.0") {
 769                 description = _("Mono");
 770         } else if (description == "2.0") {
 771                 description = _("Stereo");
 772         }
 773         boost::algorithm::replace_all (text, "$AUDIO", description);
 774
 775         auto const hmsf = film()->length().split(film()->video_frame_rate());
 776         string length;
 777         if (hmsf.h == 0 && hmsf.m == 0) {
 778                 length = String::compose("%1s", hmsf.s);
 779         } else if (hmsf.h == 0 && hmsf.m > 0) {
 780                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 781         } else if (hmsf.h > 0 && hmsf.m > 0) {
 782                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 783         }
 784
 785         boost::algorithm::replace_all (text, "$LENGTH", length);
 786
 787         file.checked_write(text.c_str(), text.length());
 788 }
 789
 790
 791 /** @param frame Frame index within the whole DCP.
 792  *  @return true if we can fake-write this frame.
 793  */
 794 bool
 795 Writer::can_fake_write (Frame frame) const
 796 {
 797         if (film()->encrypted()) {
 798                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 799                 return false;
 800         }
 801
 802         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 803            parameters in the asset writer.
 804         */
 805
 806         auto const & reel = _reels[video_reel(frame)];
 807
 808         /* Make frame relative to the start of the reel */
 809         frame -= reel.start ();
 810         return (frame != 0 && frame < reel.first_nonexistent_frame());
 811 }
 812
 813
 814 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 815 void
 816 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 817 {
 818         vector<ReelWriter>::iterator* reel = nullptr;
 819
 820         switch (type) {
 821         case TextType::OPEN_SUBTITLE:
 822                 reel = &_subtitle_reel;
 823                 _have_subtitles = true;
 824                 break;
 825         case TextType::CLOSED_CAPTION:
 826                 DCPOMATIC_ASSERT (track);
 827                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 828                 reel = &_caption_reels[*track];
 829                 _have_closed_captions.insert (*track);
 830                 break;
 831         default:
 832                 DCPOMATIC_ASSERT (false);
 833         }
 834
 835         DCPOMATIC_ASSERT (*reel != _reels.end());
 836         while ((*reel)->period().to <= period.from) {
 837                 ++(*reel);
 838                 DCPOMATIC_ASSERT (*reel != _reels.end());
 839                 write_hanging_text (**reel);
 840         }
 841
 842         auto back_off = [this](DCPTimePeriod period) {
 843                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 844                 return period;
 845         };
 846
 847         if (period.to > (*reel)->period().to) {
 848                 /* This text goes off the end of the reel.  Store parts of it that should go into
 849                  * other reels.
 850                  */
 851                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 852                         auto overlap = i->period().overlap(period);
 853                         if (overlap) {
 854                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 855                         }
 856                 }
 857                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 858                  * for subtitles being too close together.
 859                  */
 860                 period.to = (*reel)->period().to;
 861                 period = back_off(period);
 862         }
 863
 864         (*reel)->write(text, type, track, period, _fonts, _chosen_interop_font);
 865 }
 866
 867
 868 void
 869 Writer::write (vector<shared_ptr<Font>> fonts)
 870 {
 871         if (fonts.empty()) {
 872                 return;
 873         }
 874
 875         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 876         auto fix_id = [](string id) {
 877                 return id.empty() ? "font" : id;
 878         };
 879
 880         if (film()->interop()) {
 881                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 882                  * even write them as they upset some validators.  Set up _fonts so that every
 883                  * font used by any subtitle will be written with the same ID.
 884                  */
 885                 for (size_t i = 0; i < fonts.size(); ++i) {
 886                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 887                 }
 888                 _chosen_interop_font = fonts[0];
 889         } else {
 890                 for (auto font: fonts) {
 891                         _fonts.put(font, fix_id(font->id()));
 892                 }
 893         }
 894 }
 895
 896
 897 bool
 898 operator< (QueueItem const & a, QueueItem const & b)
 899 {
 900         if (a.reel != b.reel) {
 901                 return a.reel < b.reel;
 902         }
 903
 904         if (a.frame != b.frame) {
 905                 return a.frame < b.frame;
 906         }
 907
 908         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 909 }
 910
 911
 912 bool
 913 operator== (QueueItem const & a, QueueItem const & b)
 914 {
 915         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 916 }
 917
 918
 919 void
 920 Writer::set_encoder_threads (int threads)
 921 {
 922         boost::mutex::scoped_lock lm (_state_mutex);
 923         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 924         _maximum_queue_size = threads * 16;
 925 }
 926
 927
 928 void
 929 Writer::write (ReferencedReelAsset asset)
 930 {
 931         _reel_assets.push_back (asset);
 932
 933         if (dynamic_pointer_cast<dcp::ReelSubtitleAsset>(asset.asset)) {
 934                 _have_subtitles = true;
 935         } else if (auto ccap = dynamic_pointer_cast<dcp::ReelClosedCaptionAsset>(asset.asset)) {
 936                 /* This feels quite fragile. We have a referenced reel and want to know if it's
 937                  * part of a given closed-caption track so that we can fill if it has any
 938                  * missing reels.  I guess for that purpose almost any DCPTextTrack values are
 939                  * fine so long as they are consistent.
 940                  */
 941                 DCPTextTrack track;
 942                 track.name = ccap->annotation_text().get_value_or("");
 943                 track.language = dcp::LanguageTag(ccap->language().get_value_or("en-US"));
 944                 if (_have_closed_captions.find(track) == _have_closed_captions.end()) {
 945                         _have_closed_captions.insert(track);
 946                 }
 947         }
 948 }
 949
 950
 951 size_t
 952 Writer::video_reel (int frame) const
 953 {
 954         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 955         size_t reel_index = 0;
 956         while (reel_index < _reels.size() && !_reels[reel_index].period().contains(t)) {
 957                 ++reel_index;
 958         }
 959
 960         DCPOMATIC_ASSERT(reel_index < _reels.size ());
 961         return reel_index;
 962 }
 963
 964
 965 /** Update job progress with information about the progress of a single digest calculation
 966  *  thread.
 967  *  @param id Unique identifier for the thread whose progress has changed.
 968  *  @param done Number of bytes that this thread has processed.
 969  *  @param size Total number of bytes that this thread must process.
 970  */
 971 void
 972 Writer::set_digest_progress(Job* job, int id, int64_t done, int64_t size)
 973 {
 974         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 975
 976         /* Update the progress for this thread */
 977         _digest_progresses[id] = std::make_pair(done, size);
 978
 979         /* Get the total progress across all threads and use it to set job progress */
 980         int64_t total_done = 0;
 981         int64_t total_size = 0;
 982         for (auto const& i: _digest_progresses) {
 983                 total_done += i.second.first;
 984                 total_size += i.second.second;
 985         }
 986
 987         job->set_progress(float(total_done) / total_size);
 988
 989         Waker waker;
 990         waker.nudge ();
 991
 992         boost::this_thread::interruption_point();
 993 }
 994
 995
 996 /** Calculate hashes for any referenced MXF assets which do not already have one */
 997 void
 998 Writer::calculate_referenced_digests(std::function<void (int64_t, int64_t)> set_progress)
 999 try
1000 {
1001         int64_t total_size = 0;
1002         for (auto const& i: _reel_assets) {
1003                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1004                 if (file && !file->hash()) {
1005                         auto filename = file->asset_ref().asset()->file();
1006                         DCPOMATIC_ASSERT(filename);
1007                         total_size += boost::filesystem::file_size(*filename);
1008                 }
1009         }
1010
1011         int64_t total_done = 0;
1012         for (auto const& i: _reel_assets) {
1013                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1014                 if (file && !file->hash()) {
1015                         file->asset_ref().asset()->hash([&total_done, total_size, set_progress](int64_t done, int64_t) {
1016                                 set_progress(total_done + done, total_size);
1017                         });
1018                         total_done += boost::filesystem::file_size(*file->asset_ref().asset()->file());
1019                         file->set_hash (file->asset_ref().asset()->hash());
1020                 }
1021         }
1022 } catch (boost::thread_interrupted) {
1023         /* set_progress contains an interruption_point, so any of these methods
1024          * may throw thread_interrupted, at which point we just give up.
1025          */
1026 }
1027
1028
1029 void
1030 Writer::write_hanging_text (ReelWriter& reel)
1031 {
1032         vector<HangingText> new_hanging_texts;
1033         for (auto i: _hanging_texts) {
1034                 if (i.period.from == reel.period().from) {
1035                         reel.write(i.text, i.type, i.track, i.period, _fonts, _chosen_interop_font);
1036                 } else {
1037                         new_hanging_texts.push_back (i);
1038                 }
1039         }
1040         _hanging_texts = new_hanging_texts;
1041 }