src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "constants.h"
  27 #include "cross.h"
  28 #include "dcp_content_type.h"
  29 #include "dcp_video.h"
  30 #include "dcpomatic_log.h"
  31 #include "film.h"
  32 #include "film_util.h"
  33 #include "job.h"
  34 #include "log.h"
  35 #include "ratio.h"
  36 #include "reel_writer.h"
  37 #include "text_content.h"
  38 #include "util.h"
  39 #include "version.h"
  40 #include "writer.h"
  41 #include <dcp/cpl.h>
  42 #include <dcp/locale_convert.h>
  43 #include <dcp/raw_convert.h>
  44 #include <dcp/reel_file_asset.h>
  45 #include <cerrno>
  46 #include <cfloat>
  47 #include <set>
  48
  49 #include "i18n.h"
  50
  51
  52 /* OS X strikes again */
  53 #undef set_key
  54
  55
  56 using std::cout;
  57 using std::dynamic_pointer_cast;
  58 using std::make_shared;
  59 using std::max;
  60 using std::min;
  61 using std::shared_ptr;
  62 using std::set;
  63 using std::string;
  64 using std::vector;
  65 using std::weak_ptr;
  66 using boost::optional;
  67 #if BOOST_VERSION >= 106100
  68 using namespace boost::placeholders;
  69 #endif
  70 using dcp::Data;
  71 using dcp::ArrayData;
  72 using namespace dcpomatic;
  73
  74
  75 /** @param weak_job Job to report progress to, or 0.
  76  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  77  */
  78 Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, bool text_only)
  79         : WeakConstFilm (weak_film)
  80         , _job(weak_job)
  81         /* These will be reset to sensible values when J2KEncoder is created */
  82         , _maximum_frames_in_memory (8)
  83         , _maximum_queue_size (8)
  84         , _text_only (text_only)
  85 {
  86         auto job = _job.lock ();
  87
  88         int reel_index = 0;
  89         auto const reels = film()->reels();
  90         for (auto p: reels) {
  91                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  92         }
  93
  94         _last_written.resize (reels.size());
  95
  96         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  97            and captions arrive to the Writer in sequence.  This is not so for video.
  98         */
  99         _audio_reel = _reels.begin ();
 100         _subtitle_reel = _reels.begin ();
 101         for (auto i: film()->closed_caption_tracks()) {
 102                 _caption_reels[i] = _reels.begin ();
 103         }
 104         _atmos_reel = _reels.begin ();
 105
 106         /* Check that the signer is OK */
 107         string reason;
 108         if (!Config::instance()->signer_chain()->valid(&reason)) {
 109                 throw InvalidSignerError (reason);
 110         }
 111 }
 112
 113
 114 void
 115 Writer::start ()
 116 {
 117         if (!_text_only) {
 118                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 119 #ifdef DCPOMATIC_LINUX
 120                 pthread_setname_np (_thread.native_handle(), "writer");
 121 #endif
 122         }
 123 }
 124
 125
 126 Writer::~Writer ()
 127 {
 128         if (!_text_only) {
 129                 terminate_thread (false);
 130         }
 131 }
 132
 133
 134 /** Pass a video frame to the writer for writing to disk at some point.
 135  *  This method can be called with frames out of order.
 136  *  @param encoded JPEG2000-encoded data.
 137  *  @param frame Frame index within the DCP.
 138  *  @param eyes Eyes that this frame image is for.
 139  */
 140 void
 141 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 142 {
 143         boost::mutex::scoped_lock lock (_state_mutex);
 144
 145         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 146                 /* There are too many full frames in memory; wake the main writer thread and
 147                    wait until it sorts everything out */
 148                 _empty_condition.notify_all ();
 149                 _full_condition.wait (lock);
 150         }
 151
 152         QueueItem qi;
 153         qi.type = QueueItem::Type::FULL;
 154         qi.encoded = encoded;
 155         qi.reel = video_reel (frame);
 156         qi.frame = frame - _reels[qi.reel].start ();
 157
 158         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 159
 160         qi.eyes = eyes;
 161         _queue.push_back(qi);
 162         ++_queued_full_in_memory;
 163
 164         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 165         _empty_condition.notify_all ();
 166 }
 167
 168
 169 bool
 170 Writer::can_repeat (Frame frame) const
 171 {
 172         return frame > _reels[video_reel(frame)].start();
 173 }
 174
 175
 176 /** Repeat the last frame that was written to a reel as a new frame.
 177  *  @param frame Frame index within the DCP of the new (repeated) frame.
 178  *  @param eyes Eyes that this repeated frame image is for.
 179  */
 180 void
 181 Writer::repeat (Frame frame, Eyes eyes)
 182 {
 183         boost::mutex::scoped_lock lock (_state_mutex);
 184
 185         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 186                 /* The queue is too big, and the main writer thread can run and fix it, so
 187                    wake it and wait until it has done.
 188                 */
 189                 _empty_condition.notify_all ();
 190                 _full_condition.wait (lock);
 191         }
 192
 193         QueueItem qi;
 194         qi.type = QueueItem::Type::REPEAT;
 195         qi.reel = video_reel (frame);
 196         qi.frame = frame - _reels[qi.reel].start ();
 197         if (film()->three_d() && eyes == Eyes::BOTH) {
 198                 qi.eyes = Eyes::LEFT;
 199                 _queue.push_back (qi);
 200                 qi.eyes = Eyes::RIGHT;
 201                 _queue.push_back (qi);
 202         } else {
 203                 qi.eyes = eyes;
 204                 _queue.push_back (qi);
 205         }
 206
 207         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 208         _empty_condition.notify_all ();
 209 }
 210
 211
 212 void
 213 Writer::fake_write (Frame frame, Eyes eyes)
 214 {
 215         boost::mutex::scoped_lock lock (_state_mutex);
 216
 217         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 218                 /* The queue is too big, and the main writer thread can run and fix it, so
 219                    wake it and wait until it has done.
 220                 */
 221                 _empty_condition.notify_all ();
 222                 _full_condition.wait (lock);
 223         }
 224
 225         size_t const reel = video_reel (frame);
 226         Frame const frame_in_reel = frame - _reels[reel].start ();
 227
 228         QueueItem qi;
 229         qi.type = QueueItem::Type::FAKE;
 230
 231         {
 232                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 233                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 234         }
 235
 236         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 237
 238         qi.reel = reel;
 239         qi.frame = frame_in_reel;
 240         qi.eyes = eyes;
 241         _queue.push_back(qi);
 242
 243         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 244         _empty_condition.notify_all ();
 245 }
 246
 247
 248 /** Write some audio frames to the DCP.
 249  *  @param audio Audio data.
 250  *  @param time Time of this data within the DCP.
 251  *  This method is not thread safe.
 252  */
 253 void
 254 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 255 {
 256         DCPOMATIC_ASSERT (audio);
 257
 258         int const afr = film()->audio_frame_rate();
 259
 260         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 261
 262         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 263
 264         DCPTime t = time;
 265         while (t < end) {
 266
 267                 if (_audio_reel == _reels.end ()) {
 268                         /* This audio is off the end of the last reel; ignore it */
 269                         return;
 270                 }
 271
 272                 if (end <= _audio_reel->period().to) {
 273                         /* Easy case: we can write all the audio to this reel */
 274                         _audio_reel->write (audio);
 275                         t = end;
 276                 } else if (_audio_reel->period().to <= t) {
 277                         /* This reel is entirely before the start of our audio; just skip the reel */
 278                         ++_audio_reel;
 279                 } else {
 280                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 281                         DCPTime part_lengths[2] = {
 282                                 _audio_reel->period().to - t,
 283                                 end - _audio_reel->period().to
 284                         };
 285
 286                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 287                         Frame part_frames[2] = {
 288                                 part_lengths[0].frames_ceil(afr),
 289                                 part_lengths[1].frames_floor(afr)
 290                         };
 291
 292                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 293
 294                         if (part_frames[0]) {
 295                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 296                                 _audio_reel->write (part);
 297                         }
 298
 299                         if (part_frames[1]) {
 300                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 301                         } else {
 302                                 audio.reset ();
 303                         }
 304
 305                         ++_audio_reel;
 306                         t += part_lengths[0];
 307                 }
 308         }
 309 }
 310
 311
 312 void
 313 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 314 {
 315         if (_atmos_reel->period().to == time) {
 316                 ++_atmos_reel;
 317                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 318         }
 319
 320         /* We assume that we get a video frame's worth of data here */
 321         _atmos_reel->write (atmos, metadata);
 322 }
 323
 324
 325 /** Caller must hold a lock on _state_mutex */
 326 bool
 327 Writer::have_sequenced_image_at_queue_head ()
 328 {
 329         if (_queue.empty ()) {
 330                 return false;
 331         }
 332
 333         _queue.sort ();
 334         auto const & f = _queue.front();
 335         return _last_written[f.reel].next(f);
 336 }
 337
 338
 339 bool
 340 Writer::LastWritten::next (QueueItem qi) const
 341 {
 342         if (qi.eyes == Eyes::BOTH) {
 343                 /* 2D */
 344                 return qi.frame == (_frame + 1);
 345         }
 346
 347         /* 3D */
 348
 349         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 350                 return true;
 351         }
 352
 353         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 354                 return true;
 355         }
 356
 357         return false;
 358 }
 359
 360
 361 void
 362 Writer::LastWritten::update (QueueItem qi)
 363 {
 364         _frame = qi.frame;
 365         _eyes = qi.eyes;
 366 }
 367
 368
 369 void
 370 Writer::thread ()
 371 try
 372 {
 373         start_of_thread ("Writer");
 374
 375         while (true)
 376         {
 377                 boost::mutex::scoped_lock lock (_state_mutex);
 378
 379                 while (true) {
 380
 381                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 382                                 /* We've got something to do: go and do it */
 383                                 break;
 384                         }
 385
 386                         /* Nothing to do: wait until something happens which may indicate that we do */
 387                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 388                         _empty_condition.wait (lock);
 389                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 390                 }
 391
 392                 /* We stop here if we have been asked to finish, and if either the queue
 393                    is empty or we do not have a sequenced image at its head (if this is the
 394                    case we will never terminate as no new frames will be sent once
 395                    _finish is true).
 396                 */
 397                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 398                         /* (Hopefully temporarily) log anything that was not written */
 399                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 400                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 401                                 for (auto const& i: _queue) {
 402                                         if (i.type == QueueItem::Type::FULL) {
 403                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 404                                         } else {
 405                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 406                                         }
 407                                 }
 408                         }
 409                         return;
 410                 }
 411
 412                 /* Write any frames that we can write; i.e. those that are in sequence. */
 413                 while (have_sequenced_image_at_queue_head ()) {
 414                         auto qi = _queue.front ();
 415                         _last_written[qi.reel].update (qi);
 416                         _queue.pop_front ();
 417                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 418                                 --_queued_full_in_memory;
 419                         }
 420
 421                         lock.unlock ();
 422
 423                         auto& reel = _reels[qi.reel];
 424
 425                         switch (qi.type) {
 426                         case QueueItem::Type::FULL:
 427                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 428                                 if (!qi.encoded) {
 429                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 430                                 }
 431                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 432                                 ++_full_written;
 433                                 break;
 434                         case QueueItem::Type::FAKE:
 435                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 436                                 reel.fake_write (qi.size);
 437                                 ++_fake_written;
 438                                 break;
 439                         case QueueItem::Type::REPEAT:
 440                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 441                                 reel.repeat_write (qi.frame, qi.eyes);
 442                                 ++_repeat_written;
 443                                 break;
 444                         }
 445
 446                         lock.lock ();
 447                         _full_condition.notify_all ();
 448                 }
 449
 450                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 451                         /* Too many frames in memory which can't yet be written to the stream.
 452                            Write some FULL frames to disk.
 453                         */
 454
 455                         /* Find one from the back of the queue */
 456                         _queue.sort ();
 457                         auto item = _queue.rbegin();
 458                         while (item != _queue.rend() && (item->type != QueueItem::Type::FULL || !item->encoded)) {
 459                                 ++item;
 460                         }
 461
 462                         DCPOMATIC_ASSERT(item != _queue.rend());
 463                         ++_pushed_to_disk;
 464                         /* For the log message below */
 465                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 466                         lock.unlock ();
 467
 468                         /* i is valid here, even though we don't hold a lock on the mutex,
 469                            since list iterators are unaffected by insertion and only this
 470                            thread could erase the last item in the list.
 471                         */
 472
 473                         LOG_GENERAL("Writer full; pushes %1 to disk while awaiting %2", item->frame, awaiting);
 474
 475                         item->encoded->write_via_temp(
 476                                 film()->j2c_path(item->reel, item->frame, item->eyes, true),
 477                                 film()->j2c_path(item->reel, item->frame, item->eyes, false)
 478                                 );
 479
 480                         lock.lock ();
 481                         item->encoded.reset();
 482                         --_queued_full_in_memory;
 483                         _full_condition.notify_all ();
 484                 }
 485         }
 486 }
 487 catch (...)
 488 {
 489         store_current ();
 490 }
 491
 492
 493 void
 494 Writer::terminate_thread (bool can_throw)
 495 {
 496         boost::this_thread::disable_interruption dis;
 497
 498         boost::mutex::scoped_lock lock (_state_mutex);
 499
 500         _finish = true;
 501         _empty_condition.notify_all ();
 502         _full_condition.notify_all ();
 503         lock.unlock ();
 504
 505         try {
 506                 _thread.join ();
 507         } catch (...) {}
 508
 509         if (can_throw) {
 510                 rethrow ();
 511         }
 512 }
 513
 514
 515 void
 516 Writer::calculate_digests ()
 517 {
 518         auto job = _job.lock ();
 519         if (job) {
 520                 job->sub (_("Computing digests"));
 521         }
 522
 523         boost::asio::io_service service;
 524         boost::thread_group pool;
 525
 526         auto work = make_shared<boost::asio::io_service::work>(service);
 527
 528         int const threads = max (1, Config::instance()->master_encoding_threads());
 529
 530         for (int i = 0; i < threads; ++i) {
 531                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 532         }
 533
 534         std::function<void (int, int64_t, int64_t)> set_progress;
 535         if (job) {
 536                 set_progress = boost::bind(&Writer::set_digest_progress, this, job.get(), _1, _2, _3);
 537         } else {
 538                 set_progress = [](int, int64_t, int64_t) {
 539                         boost::this_thread::interruption_point();
 540                 };
 541         }
 542
 543         int index = 0;
 544
 545         for (auto& i: _reels) {
 546                 service.post(
 547                         boost::bind(
 548                                 &ReelWriter::calculate_digests,
 549                                 &i,
 550                                 std::function<void (int64_t, int64_t)>(boost::bind(set_progress, index, _1, _2))
 551                                 ));
 552                 ++index;
 553         }
 554         service.post(
 555                 boost::bind(
 556                         &Writer::calculate_referenced_digests,
 557                         this,
 558                         std::function<void (int64_t, int64_t)>(boost::bind(set_progress, index, _1, _2))
 559                         ));
 560
 561         work.reset ();
 562
 563         try {
 564                 pool.join_all ();
 565         } catch (boost::thread_interrupted) {
 566                 /* join_all was interrupted, so we need to interrupt the threads
 567                  * in our pool then try again to join them.
 568                  */
 569                 pool.interrupt_all ();
 570                 pool.join_all ();
 571         }
 572
 573         service.stop ();
 574 }
 575
 576
 577 /** @param output_dcp Path to DCP folder to write */
 578 void
 579 Writer::finish (boost::filesystem::path output_dcp)
 580 {
 581         if (_thread.joinable()) {
 582                 LOG_GENERAL_NC ("Terminating writer thread");
 583                 terminate_thread (true);
 584         }
 585
 586         LOG_GENERAL_NC ("Finishing ReelWriters");
 587
 588         for (auto& reel: _reels) {
 589                 write_hanging_text(reel);
 590                 reel.finish(output_dcp);
 591         }
 592
 593         LOG_GENERAL_NC ("Writing XML");
 594
 595         dcp::DCP dcp (output_dcp);
 596
 597         auto cpl = make_shared<dcp::CPL>(
 598                 film()->dcp_name(),
 599                 film()->dcp_content_type()->libdcp_kind(),
 600                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 601                 );
 602
 603         dcp.add (cpl);
 604
 605         calculate_digests ();
 606
 607         /* Add reels */
 608
 609         for (auto& i: _reels) {
 610                 cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
 611         }
 612
 613         /* Add metadata */
 614
 615         auto creator = Config::instance()->dcp_creator();
 616         if (creator.empty()) {
 617                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 618         }
 619
 620         auto issuer = Config::instance()->dcp_issuer();
 621         if (issuer.empty()) {
 622                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 623         }
 624
 625         cpl->set_creator (creator);
 626         cpl->set_issuer (issuer);
 627
 628         cpl->set_ratings (film()->ratings());
 629
 630         vector<dcp::ContentVersion> cv;
 631         for (auto i: film()->content_versions()) {
 632                 /* Make sure we don't end up writing an empty <LabelText> node as some validators
 633                  * complain about that.
 634                  */
 635                 cv.push_back(!i.empty() ? dcp::ContentVersion(i) : dcp::ContentVersion("1"));
 636         }
 637         if (cv.empty()) {
 638                 cv = { dcp::ContentVersion("1") };
 639         }
 640         cpl->set_content_versions (cv);
 641
 642         cpl->set_full_content_title_text (film()->name());
 643         cpl->set_full_content_title_text_language (film()->name_language());
 644         if (film()->release_territory()) {
 645                 cpl->set_release_territory (*film()->release_territory());
 646         }
 647         cpl->set_version_number (film()->version_number());
 648         cpl->set_status (film()->status());
 649         if (film()->chain()) {
 650                 cpl->set_chain (*film()->chain());
 651         }
 652         if (film()->distributor()) {
 653                 cpl->set_distributor (*film()->distributor());
 654         }
 655         if (film()->facility()) {
 656                 cpl->set_facility (*film()->facility());
 657         }
 658         if (film()->luminance()) {
 659                 cpl->set_luminance (*film()->luminance());
 660         }
 661         if (film()->sign_language_video_language()) {
 662                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 663         }
 664
 665         dcp::MCASoundField field;
 666         if (channel_is_mapped(film(), dcp::Channel::BSL) || channel_is_mapped(film(), dcp::Channel::BSR)) {
 667                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 668         } else {
 669                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 670         }
 671
 672         auto const audio_channels = film()->audio_channels();
 673         dcp::MainSoundConfiguration msc(field, audio_channels);
 674         for (auto i: film()->mapped_audio_channels()) {
 675                 if (i < audio_channels) {
 676                         msc.set_mapping(i, static_cast<dcp::Channel>(i));
 677                 }
 678         }
 679
 680         cpl->set_main_sound_configuration(msc);
 681         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 682         cpl->set_main_picture_stored_area (film()->frame_size());
 683
 684         auto active_area = film()->active_area();
 685         if (active_area.width > 0 && active_area.height > 0) {
 686                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 687                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 688         }
 689
 690         auto sl = film()->subtitle_languages().second;
 691         if (!sl.empty()) {
 692                 cpl->set_additional_subtitle_languages(sl);
 693         }
 694
 695         auto signer = Config::instance()->signer_chain();
 696         /* We did check earlier, but check again here to be on the safe side */
 697         string reason;
 698         if (!signer->valid (&reason)) {
 699                 throw InvalidSignerError (reason);
 700         }
 701
 702         dcp.set_issuer(issuer);
 703         dcp.set_creator(creator);
 704         dcp.set_annotation_text(film()->dcp_name());
 705
 706         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 707
 708         LOG_GENERAL (
 709                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 710                 );
 711
 712         write_cover_sheet (output_dcp);
 713 }
 714
 715
 716 void
 717 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 718 {
 719         auto const cover = film()->file("COVER_SHEET.txt");
 720         dcp::File file(cover, "w");
 721         if (!file) {
 722                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 723         }
 724
 725         auto text = Config::instance()->cover_sheet ();
 726         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 727         auto cpls = film()->cpls();
 728         if (!cpls.empty()) {
 729                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 730         }
 731         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 732         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 733
 734         auto audio_language = film()->audio_language();
 735         if (audio_language) {
 736                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 737         } else {
 738                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 739         }
 740
 741         auto subtitle_languages = film()->subtitle_languages();
 742         if (subtitle_languages.first) {
 743                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 744         } else {
 745                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 746         }
 747
 748         boost::uintmax_t size = 0;
 749         for (
 750                 auto i = dcp::filesystem::recursive_directory_iterator(output_dcp);
 751                 i != dcp::filesystem::recursive_directory_iterator();
 752                 ++i) {
 753                 if (dcp::filesystem::is_regular_file(i->path())) {
 754                         size += dcp::filesystem::file_size(i->path());
 755                 }
 756         }
 757
 758         if (size > (1000000000L)) {
 759                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 760         } else {
 761                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 762         }
 763
 764         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 765         auto description = String::compose("%1.%2", ch.first, ch.second);
 766
 767         if (description == "0.0") {
 768                 description = _("None");
 769         } else if (description == "1.0") {
 770                 description = _("Mono");
 771         } else if (description == "2.0") {
 772                 description = _("Stereo");
 773         }
 774         boost::algorithm::replace_all (text, "$AUDIO", description);
 775
 776         auto const hmsf = film()->length().split(film()->video_frame_rate());
 777         string length;
 778         if (hmsf.h == 0 && hmsf.m == 0) {
 779                 length = String::compose("%1s", hmsf.s);
 780         } else if (hmsf.h == 0 && hmsf.m > 0) {
 781                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 782         } else if (hmsf.h > 0 && hmsf.m > 0) {
 783                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 784         }
 785
 786         boost::algorithm::replace_all (text, "$LENGTH", length);
 787
 788         file.checked_write(text.c_str(), text.length());
 789 }
 790
 791
 792 /** @param frame Frame index within the whole DCP.
 793  *  @return true if we can fake-write this frame.
 794  */
 795 bool
 796 Writer::can_fake_write (Frame frame) const
 797 {
 798         if (film()->encrypted()) {
 799                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 800                 return false;
 801         }
 802
 803         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 804            parameters in the asset writer.
 805         */
 806
 807         auto const & reel = _reels[video_reel(frame)];
 808
 809         /* Make frame relative to the start of the reel */
 810         frame -= reel.start ();
 811         return (frame != 0 && frame < reel.first_nonexistent_frame());
 812 }
 813
 814
 815 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 816 void
 817 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 818 {
 819         vector<ReelWriter>::iterator* reel = nullptr;
 820
 821         switch (type) {
 822         case TextType::OPEN_SUBTITLE:
 823                 reel = &_subtitle_reel;
 824                 _have_subtitles = true;
 825                 break;
 826         case TextType::CLOSED_CAPTION:
 827                 DCPOMATIC_ASSERT (track);
 828                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 829                 reel = &_caption_reels[*track];
 830                 _have_closed_captions.insert (*track);
 831                 break;
 832         default:
 833                 DCPOMATIC_ASSERT (false);
 834         }
 835
 836         DCPOMATIC_ASSERT (*reel != _reels.end());
 837         while ((*reel)->period().to <= period.from) {
 838                 ++(*reel);
 839                 DCPOMATIC_ASSERT (*reel != _reels.end());
 840                 write_hanging_text (**reel);
 841         }
 842
 843         auto back_off = [this](DCPTimePeriod period) {
 844                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 845                 return period;
 846         };
 847
 848         if (period.to > (*reel)->period().to) {
 849                 /* This text goes off the end of the reel.  Store parts of it that should go into
 850                  * other reels.
 851                  */
 852                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 853                         auto overlap = i->period().overlap(period);
 854                         if (overlap) {
 855                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 856                         }
 857                 }
 858                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 859                  * for subtitles being too close together.
 860                  */
 861                 period.to = (*reel)->period().to;
 862                 period = back_off(period);
 863         }
 864
 865         (*reel)->write(text, type, track, period, _fonts, _chosen_interop_font);
 866 }
 867
 868
 869 void
 870 Writer::write (vector<shared_ptr<Font>> fonts)
 871 {
 872         if (fonts.empty()) {
 873                 return;
 874         }
 875
 876         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 877         auto fix_id = [](string id) {
 878                 return id.empty() ? "font" : id;
 879         };
 880
 881         if (film()->interop()) {
 882                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 883                  * even write them as they upset some validators.  Set up _fonts so that every
 884                  * font used by any subtitle will be written with the same ID.
 885                  */
 886                 for (size_t i = 0; i < fonts.size(); ++i) {
 887                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 888                 }
 889                 _chosen_interop_font = fonts[0];
 890         } else {
 891                 for (auto font: fonts) {
 892                         _fonts.put(font, fix_id(font->id()));
 893                 }
 894         }
 895 }
 896
 897
 898 bool
 899 operator< (QueueItem const & a, QueueItem const & b)
 900 {
 901         if (a.reel != b.reel) {
 902                 return a.reel < b.reel;
 903         }
 904
 905         if (a.frame != b.frame) {
 906                 return a.frame < b.frame;
 907         }
 908
 909         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 910 }
 911
 912
 913 bool
 914 operator== (QueueItem const & a, QueueItem const & b)
 915 {
 916         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 917 }
 918
 919
 920 void
 921 Writer::set_encoder_threads (int threads)
 922 {
 923         boost::mutex::scoped_lock lm (_state_mutex);
 924         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 925         _maximum_queue_size = threads * 16;
 926 }
 927
 928
 929 void
 930 Writer::write (ReferencedReelAsset asset)
 931 {
 932         _reel_assets.push_back (asset);
 933 }
 934
 935
 936 size_t
 937 Writer::video_reel (int frame) const
 938 {
 939         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 940         size_t reel_index = 0;
 941         while (reel_index < _reels.size() && !_reels[reel_index].period().contains(t)) {
 942                 ++reel_index;
 943         }
 944
 945         DCPOMATIC_ASSERT(reel_index < _reels.size ());
 946         return reel_index;
 947 }
 948
 949
 950 /** Update job progress with information about the progress of a single digest calculation
 951  *  thread.
 952  *  @param id Unique identifier for the thread whose progress has changed.
 953  *  @param done Number of bytes that this thread has processed.
 954  *  @param size Total number of bytes that this thread must process.
 955  */
 956 void
 957 Writer::set_digest_progress(Job* job, int id, int64_t done, int64_t size)
 958 {
 959         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 960
 961         /* Update the progress for this thread */
 962         _digest_progresses[id] = std::make_pair(done, size);
 963
 964         /* Get the total progress across all threads and use it to set job progress */
 965         int64_t total_done = 0;
 966         int64_t total_size = 0;
 967         for (auto const& i: _digest_progresses) {
 968                 total_done += i.second.first;
 969                 total_size += i.second.second;
 970         }
 971
 972         job->set_progress(float(total_done) / total_size);
 973
 974         Waker waker;
 975         waker.nudge ();
 976
 977         boost::this_thread::interruption_point();
 978 }
 979
 980
 981 /** Calculate hashes for any referenced MXF assets which do not already have one */
 982 void
 983 Writer::calculate_referenced_digests(std::function<void (int64_t, int64_t)> set_progress)
 984 try
 985 {
 986         int64_t total_size = 0;
 987         for (auto const& i: _reel_assets) {
 988                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
 989                 if (file && !file->hash()) {
 990                         auto filename = file->asset_ref().asset()->file();
 991                         DCPOMATIC_ASSERT(filename);
 992                         total_size += boost::filesystem::file_size(*filename);
 993                 }
 994         }
 995
 996         int64_t total_done = 0;
 997         for (auto const& i: _reel_assets) {
 998                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
 999                 if (file && !file->hash()) {
1000                         file->asset_ref().asset()->hash([&total_done, total_size, set_progress](int64_t done, int64_t) {
1001                                 set_progress(total_done + done, total_size);
1002                         });
1003                         total_done += boost::filesystem::file_size(*file->asset_ref().asset()->file());
1004                         file->set_hash (file->asset_ref().asset()->hash());
1005                 }
1006         }
1007 } catch (boost::thread_interrupted) {
1008         /* set_progress contains an interruption_point, so any of these methods
1009          * may throw thread_interrupted, at which point we just give up.
1010          */
1011 }
1012
1013
1014 void
1015 Writer::write_hanging_text (ReelWriter& reel)
1016 {
1017         vector<HangingText> new_hanging_texts;
1018         for (auto i: _hanging_texts) {
1019                 if (i.period.from == reel.period().from) {
1020                         reel.write(i.text, i.type, i.track, i.period, _fonts, _chosen_interop_font);
1021                 } else {
1022                         new_hanging_texts.push_back (i);
1023                 }
1024         }
1025         _hanging_texts = new_hanging_texts;
1026 }