src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "constants.h"
  27 #include "cross.h"
  28 #include "dcp_content_type.h"
  29 #include "dcp_video.h"
  30 #include "dcpomatic_log.h"
  31 #include "film.h"
  32 #include "film_util.h"
  33 #include "job.h"
  34 #include "log.h"
  35 #include "ratio.h"
  36 #include "reel_writer.h"
  37 #include "text_content.h"
  38 #include "util.h"
  39 #include "version.h"
  40 #include "writer.h"
  41 #include <dcp/cpl.h>
  42 #include <dcp/locale_convert.h>
  43 #include <dcp/raw_convert.h>
  44 #include <dcp/reel_file_asset.h>
  45 #include <cerrno>
  46 #include <cfloat>
  47 #include <set>
  48
  49 #include "i18n.h"
  50
  51
  52 /* OS X strikes again */
  53 #undef set_key
  54
  55
  56 using std::cout;
  57 using std::dynamic_pointer_cast;
  58 using std::make_shared;
  59 using std::max;
  60 using std::min;
  61 using std::shared_ptr;
  62 using std::set;
  63 using std::string;
  64 using std::vector;
  65 using std::weak_ptr;
  66 using boost::optional;
  67 #if BOOST_VERSION >= 106100
  68 using namespace boost::placeholders;
  69 #endif
  70 using dcp::Data;
  71 using dcp::ArrayData;
  72 using namespace dcpomatic;
  73
  74
  75 /** @param weak_job Job to report progress to, or 0.
  76  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  77  */
  78 Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, bool text_only)
  79         : WeakConstFilm (weak_film)
  80         , _job(weak_job)
  81         /* These will be reset to sensible values when J2KEncoder is created */
  82         , _maximum_frames_in_memory (8)
  83         , _maximum_queue_size (8)
  84         , _text_only (text_only)
  85 {
  86         auto job = _job.lock ();
  87
  88         int reel_index = 0;
  89         auto const reels = film()->reels();
  90         for (auto p: reels) {
  91                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  92         }
  93
  94         _last_written.resize (reels.size());
  95
  96         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  97            and captions arrive to the Writer in sequence.  This is not so for video.
  98         */
  99         _audio_reel = _reels.begin ();
 100         _subtitle_reel = _reels.begin ();
 101         for (auto i: film()->closed_caption_tracks()) {
 102                 _caption_reels[i] = _reels.begin ();
 103         }
 104         _atmos_reel = _reels.begin ();
 105
 106         /* Check that the signer is OK */
 107         string reason;
 108         if (!Config::instance()->signer_chain()->valid(&reason)) {
 109                 throw InvalidSignerError (reason);
 110         }
 111 }
 112
 113
 114 void
 115 Writer::start ()
 116 {
 117         if (!_text_only) {
 118                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 119 #ifdef DCPOMATIC_LINUX
 120                 pthread_setname_np (_thread.native_handle(), "writer");
 121 #endif
 122         }
 123 }
 124
 125
 126 Writer::~Writer ()
 127 {
 128         if (!_text_only) {
 129                 terminate_thread (false);
 130         }
 131 }
 132
 133
 134 /** Pass a video frame to the writer for writing to disk at some point.
 135  *  This method can be called with frames out of order.
 136  *  @param encoded JPEG2000-encoded data.
 137  *  @param frame Frame index within the DCP.
 138  *  @param eyes Eyes that this frame image is for.
 139  */
 140 void
 141 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 142 {
 143         boost::mutex::scoped_lock lock (_state_mutex);
 144
 145         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 146                 /* There are too many full frames in memory; wake the main writer thread and
 147                    wait until it sorts everything out */
 148                 _empty_condition.notify_all ();
 149                 _full_condition.wait (lock);
 150         }
 151
 152         QueueItem qi;
 153         qi.type = QueueItem::Type::FULL;
 154         qi.encoded = encoded;
 155         qi.reel = video_reel (frame);
 156         qi.frame = frame - _reels[qi.reel].start ();
 157
 158         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 159
 160         qi.eyes = eyes;
 161         _queue.push_back(qi);
 162         ++_queued_full_in_memory;
 163
 164         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 165         _empty_condition.notify_all ();
 166 }
 167
 168
 169 bool
 170 Writer::can_repeat (Frame frame) const
 171 {
 172         return frame > _reels[video_reel(frame)].start();
 173 }
 174
 175
 176 /** Repeat the last frame that was written to a reel as a new frame.
 177  *  @param frame Frame index within the DCP of the new (repeated) frame.
 178  *  @param eyes Eyes that this repeated frame image is for.
 179  */
 180 void
 181 Writer::repeat (Frame frame, Eyes eyes)
 182 {
 183         boost::mutex::scoped_lock lock (_state_mutex);
 184
 185         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 186                 /* The queue is too big, and the main writer thread can run and fix it, so
 187                    wake it and wait until it has done.
 188                 */
 189                 _empty_condition.notify_all ();
 190                 _full_condition.wait (lock);
 191         }
 192
 193         QueueItem qi;
 194         qi.type = QueueItem::Type::REPEAT;
 195         qi.reel = video_reel (frame);
 196         qi.frame = frame - _reels[qi.reel].start ();
 197         if (film()->three_d() && eyes == Eyes::BOTH) {
 198                 qi.eyes = Eyes::LEFT;
 199                 _queue.push_back (qi);
 200                 qi.eyes = Eyes::RIGHT;
 201                 _queue.push_back (qi);
 202         } else {
 203                 qi.eyes = eyes;
 204                 _queue.push_back (qi);
 205         }
 206
 207         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 208         _empty_condition.notify_all ();
 209 }
 210
 211
 212 void
 213 Writer::fake_write (Frame frame, Eyes eyes)
 214 {
 215         boost::mutex::scoped_lock lock (_state_mutex);
 216
 217         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 218                 /* The queue is too big, and the main writer thread can run and fix it, so
 219                    wake it and wait until it has done.
 220                 */
 221                 _empty_condition.notify_all ();
 222                 _full_condition.wait (lock);
 223         }
 224
 225         size_t const reel = video_reel (frame);
 226         Frame const frame_in_reel = frame - _reels[reel].start ();
 227
 228         QueueItem qi;
 229         qi.type = QueueItem::Type::FAKE;
 230
 231         {
 232                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 233                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 234         }
 235
 236         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 237
 238         qi.reel = reel;
 239         qi.frame = frame_in_reel;
 240         qi.eyes = eyes;
 241         _queue.push_back(qi);
 242
 243         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 244         _empty_condition.notify_all ();
 245 }
 246
 247
 248 /** Write some audio frames to the DCP.
 249  *  @param audio Audio data.
 250  *  @param time Time of this data within the DCP.
 251  *  This method is not thread safe.
 252  */
 253 void
 254 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 255 {
 256         DCPOMATIC_ASSERT (audio);
 257
 258         int const afr = film()->audio_frame_rate();
 259
 260         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 261
 262         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 263
 264         DCPTime t = time;
 265         while (t < end) {
 266
 267                 if (_audio_reel == _reels.end ()) {
 268                         /* This audio is off the end of the last reel; ignore it */
 269                         return;
 270                 }
 271
 272                 if (end <= _audio_reel->period().to) {
 273                         /* Easy case: we can write all the audio to this reel */
 274                         _audio_reel->write (audio);
 275                         t = end;
 276                 } else if (_audio_reel->period().to <= t) {
 277                         /* This reel is entirely before the start of our audio; just skip the reel */
 278                         ++_audio_reel;
 279                 } else {
 280                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 281                         DCPTime part_lengths[2] = {
 282                                 _audio_reel->period().to - t,
 283                                 end - _audio_reel->period().to
 284                         };
 285
 286                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 287                         Frame part_frames[2] = {
 288                                 part_lengths[0].frames_ceil(afr),
 289                                 part_lengths[1].frames_floor(afr)
 290                         };
 291
 292                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 293
 294                         if (part_frames[0]) {
 295                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 296                                 _audio_reel->write (part);
 297                         }
 298
 299                         if (part_frames[1]) {
 300                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 301                         } else {
 302                                 audio.reset ();
 303                         }
 304
 305                         ++_audio_reel;
 306                         t += part_lengths[0];
 307                 }
 308         }
 309 }
 310
 311
 312 void
 313 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 314 {
 315         if (_atmos_reel->period().to == time) {
 316                 ++_atmos_reel;
 317                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 318         }
 319
 320         /* We assume that we get a video frame's worth of data here */
 321         _atmos_reel->write (atmos, metadata);
 322 }
 323
 324
 325 /** Caller must hold a lock on _state_mutex */
 326 bool
 327 Writer::have_sequenced_image_at_queue_head ()
 328 {
 329         if (_queue.empty ()) {
 330                 return false;
 331         }
 332
 333         _queue.sort ();
 334         auto const & f = _queue.front();
 335         return _last_written[f.reel].next(f);
 336 }
 337
 338
 339 bool
 340 Writer::LastWritten::next (QueueItem qi) const
 341 {
 342         if (qi.eyes == Eyes::BOTH) {
 343                 /* 2D */
 344                 return qi.frame == (_frame + 1);
 345         }
 346
 347         /* 3D */
 348
 349         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 350                 return true;
 351         }
 352
 353         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 354                 return true;
 355         }
 356
 357         return false;
 358 }
 359
 360
 361 void
 362 Writer::LastWritten::update (QueueItem qi)
 363 {
 364         _frame = qi.frame;
 365         _eyes = qi.eyes;
 366 }
 367
 368
 369 void
 370 Writer::thread ()
 371 try
 372 {
 373         start_of_thread ("Writer");
 374
 375         while (true)
 376         {
 377                 boost::mutex::scoped_lock lock (_state_mutex);
 378
 379                 while (true) {
 380
 381                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 382                                 /* We've got something to do: go and do it */
 383                                 break;
 384                         }
 385
 386                         /* Nothing to do: wait until something happens which may indicate that we do */
 387                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 388                         _empty_condition.wait (lock);
 389                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 390                 }
 391
 392                 /* We stop here if we have been asked to finish, and if either the queue
 393                    is empty or we do not have a sequenced image at its head (if this is the
 394                    case we will never terminate as no new frames will be sent once
 395                    _finish is true).
 396                 */
 397                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 398                         /* (Hopefully temporarily) log anything that was not written */
 399                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 400                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 401                                 for (auto const& i: _queue) {
 402                                         if (i.type == QueueItem::Type::FULL) {
 403                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 404                                         } else {
 405                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 406                                         }
 407                                 }
 408                         }
 409                         return;
 410                 }
 411
 412                 /* Write any frames that we can write; i.e. those that are in sequence. */
 413                 while (have_sequenced_image_at_queue_head ()) {
 414                         auto qi = _queue.front ();
 415                         _last_written[qi.reel].update (qi);
 416                         _queue.pop_front ();
 417                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 418                                 --_queued_full_in_memory;
 419                         }
 420
 421                         lock.unlock ();
 422
 423                         auto& reel = _reels[qi.reel];
 424
 425                         switch (qi.type) {
 426                         case QueueItem::Type::FULL:
 427                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 428                                 if (!qi.encoded) {
 429                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 430                                 }
 431                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 432                                 ++_full_written;
 433                                 break;
 434                         case QueueItem::Type::FAKE:
 435                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 436                                 reel.fake_write (qi.size);
 437                                 ++_fake_written;
 438                                 break;
 439                         case QueueItem::Type::REPEAT:
 440                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 441                                 reel.repeat_write (qi.frame, qi.eyes);
 442                                 ++_repeat_written;
 443                                 break;
 444                         }
 445
 446                         lock.lock ();
 447                         _full_condition.notify_all ();
 448                 }
 449
 450                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 451                         /* Too many frames in memory which can't yet be written to the stream.
 452                            Write some FULL frames to disk.
 453                         */
 454
 455                         /* Find one from the back of the queue */
 456                         _queue.sort ();
 457                         auto item = _queue.rbegin();
 458                         while (item != _queue.rend() && (item->type != QueueItem::Type::FULL || !item->encoded)) {
 459                                 ++item;
 460                         }
 461
 462                         DCPOMATIC_ASSERT(item != _queue.rend());
 463                         ++_pushed_to_disk;
 464                         /* For the log message below */
 465                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 466                         lock.unlock ();
 467
 468                         /* i is valid here, even though we don't hold a lock on the mutex,
 469                            since list iterators are unaffected by insertion and only this
 470                            thread could erase the last item in the list.
 471                         */
 472
 473                         LOG_GENERAL("Writer full; pushes %1 to disk while awaiting %2", item->frame, awaiting);
 474
 475                         item->encoded->write_via_temp(
 476                                 film()->j2c_path(item->reel, item->frame, item->eyes, true),
 477                                 film()->j2c_path(item->reel, item->frame, item->eyes, false)
 478                                 );
 479
 480                         lock.lock ();
 481                         item->encoded.reset();
 482                         --_queued_full_in_memory;
 483                         _full_condition.notify_all ();
 484                 }
 485         }
 486 }
 487 catch (...)
 488 {
 489         store_current ();
 490 }
 491
 492
 493 void
 494 Writer::terminate_thread (bool can_throw)
 495 {
 496         boost::this_thread::disable_interruption dis;
 497
 498         boost::mutex::scoped_lock lock (_state_mutex);
 499
 500         _finish = true;
 501         _empty_condition.notify_all ();
 502         _full_condition.notify_all ();
 503         lock.unlock ();
 504
 505         try {
 506                 _thread.join ();
 507         } catch (...) {}
 508
 509         if (can_throw) {
 510                 rethrow ();
 511         }
 512 }
 513
 514
 515 void
 516 Writer::calculate_digests ()
 517 {
 518         auto job = _job.lock ();
 519         if (job) {
 520                 job->sub (_("Computing digests"));
 521         }
 522
 523         boost::asio::io_service service;
 524         boost::thread_group pool;
 525
 526         auto work = make_shared<boost::asio::io_service::work>(service);
 527
 528         int const threads = max (1, Config::instance()->master_encoding_threads());
 529
 530         for (int i = 0; i < threads; ++i) {
 531                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 532         }
 533
 534         std::function<void (float)> set_progress;
 535         if (job) {
 536                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 537         } else {
 538                 set_progress = [](float) {
 539                         boost::this_thread::interruption_point();
 540                 };
 541         }
 542
 543         for (auto& i: _reels) {
 544                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 545         }
 546         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 547
 548         work.reset ();
 549
 550         try {
 551                 pool.join_all ();
 552         } catch (boost::thread_interrupted) {
 553                 /* join_all was interrupted, so we need to interrupt the threads
 554                  * in our pool then try again to join them.
 555                  */
 556                 pool.interrupt_all ();
 557                 pool.join_all ();
 558         }
 559
 560         service.stop ();
 561 }
 562
 563
 564 /** @param output_dcp Path to DCP folder to write */
 565 void
 566 Writer::finish (boost::filesystem::path output_dcp)
 567 {
 568         if (_thread.joinable()) {
 569                 LOG_GENERAL_NC ("Terminating writer thread");
 570                 terminate_thread (true);
 571         }
 572
 573         LOG_GENERAL_NC ("Finishing ReelWriters");
 574
 575         for (auto& reel: _reels) {
 576                 write_hanging_text(reel);
 577                 reel.finish(output_dcp);
 578         }
 579
 580         LOG_GENERAL_NC ("Writing XML");
 581
 582         dcp::DCP dcp (output_dcp);
 583
 584         auto cpl = make_shared<dcp::CPL>(
 585                 film()->dcp_name(),
 586                 film()->dcp_content_type()->libdcp_kind(),
 587                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 588                 );
 589
 590         dcp.add (cpl);
 591
 592         calculate_digests ();
 593
 594         /* Add reels */
 595
 596         for (auto& i: _reels) {
 597                 cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
 598         }
 599
 600         /* Add metadata */
 601
 602         auto creator = Config::instance()->dcp_creator();
 603         if (creator.empty()) {
 604                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 605         }
 606
 607         auto issuer = Config::instance()->dcp_issuer();
 608         if (issuer.empty()) {
 609                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 610         }
 611
 612         cpl->set_creator (creator);
 613         cpl->set_issuer (issuer);
 614
 615         cpl->set_ratings (film()->ratings());
 616
 617         vector<dcp::ContentVersion> cv;
 618         for (auto i: film()->content_versions()) {
 619                 /* Make sure we don't end up writing an empty <LabelText> node as some validators
 620                  * complain about that.
 621                  */
 622                 cv.push_back(!i.empty() ? dcp::ContentVersion(i) : dcp::ContentVersion("1"));
 623         }
 624         if (cv.empty()) {
 625                 cv = { dcp::ContentVersion("1") };
 626         }
 627         cpl->set_content_versions (cv);
 628
 629         cpl->set_full_content_title_text (film()->name());
 630         cpl->set_full_content_title_text_language (film()->name_language());
 631         if (film()->release_territory()) {
 632                 cpl->set_release_territory (*film()->release_territory());
 633         }
 634         cpl->set_version_number (film()->version_number());
 635         cpl->set_status (film()->status());
 636         if (film()->chain()) {
 637                 cpl->set_chain (*film()->chain());
 638         }
 639         if (film()->distributor()) {
 640                 cpl->set_distributor (*film()->distributor());
 641         }
 642         if (film()->facility()) {
 643                 cpl->set_facility (*film()->facility());
 644         }
 645         if (film()->luminance()) {
 646                 cpl->set_luminance (*film()->luminance());
 647         }
 648         if (film()->sign_language_video_language()) {
 649                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 650         }
 651
 652         dcp::MCASoundField field;
 653         if (channel_is_mapped(film(), dcp::Channel::BSL) || channel_is_mapped(film(), dcp::Channel::BSR)) {
 654                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 655         } else {
 656                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 657         }
 658
 659         auto const audio_channels = film()->audio_channels();
 660         dcp::MainSoundConfiguration msc(field, audio_channels);
 661         for (auto i: film()->mapped_audio_channels()) {
 662                 if (i < audio_channels) {
 663                         msc.set_mapping(i, static_cast<dcp::Channel>(i));
 664                 }
 665         }
 666
 667         cpl->set_main_sound_configuration(msc);
 668         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 669         cpl->set_main_picture_stored_area (film()->frame_size());
 670
 671         auto active_area = film()->active_area();
 672         if (active_area.width > 0 && active_area.height > 0) {
 673                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 674                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 675         }
 676
 677         auto sl = film()->subtitle_languages().second;
 678         if (!sl.empty()) {
 679                 cpl->set_additional_subtitle_languages(sl);
 680         }
 681
 682         auto signer = Config::instance()->signer_chain();
 683         /* We did check earlier, but check again here to be on the safe side */
 684         string reason;
 685         if (!signer->valid (&reason)) {
 686                 throw InvalidSignerError (reason);
 687         }
 688
 689         dcp.set_issuer(issuer);
 690         dcp.set_creator(creator);
 691         dcp.set_annotation_text(film()->dcp_name());
 692
 693         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 694
 695         LOG_GENERAL (
 696                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 697                 );
 698
 699         write_cover_sheet (output_dcp);
 700 }
 701
 702
 703 void
 704 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 705 {
 706         auto const cover = film()->file("COVER_SHEET.txt");
 707         dcp::File file(cover, "w");
 708         if (!file) {
 709                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 710         }
 711
 712         auto text = Config::instance()->cover_sheet ();
 713         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 714         auto cpls = film()->cpls();
 715         if (!cpls.empty()) {
 716                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 717         }
 718         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 719         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 720
 721         auto audio_language = film()->audio_language();
 722         if (audio_language) {
 723                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 724         } else {
 725                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 726         }
 727
 728         auto subtitle_languages = film()->subtitle_languages();
 729         if (subtitle_languages.first) {
 730                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 731         } else {
 732                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 733         }
 734
 735         boost::uintmax_t size = 0;
 736         for (
 737                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 738                 i != boost::filesystem::recursive_directory_iterator();
 739                 ++i) {
 740                 if (boost::filesystem::is_regular_file (i->path())) {
 741                         size += boost::filesystem::file_size (i->path());
 742                 }
 743         }
 744
 745         if (size > (1000000000L)) {
 746                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 747         } else {
 748                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 749         }
 750
 751         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 752         auto description = String::compose("%1.%2", ch.first, ch.second);
 753
 754         if (description == "0.0") {
 755                 description = _("None");
 756         } else if (description == "1.0") {
 757                 description = _("Mono");
 758         } else if (description == "2.0") {
 759                 description = _("Stereo");
 760         }
 761         boost::algorithm::replace_all (text, "$AUDIO", description);
 762
 763         auto const hmsf = film()->length().split(film()->video_frame_rate());
 764         string length;
 765         if (hmsf.h == 0 && hmsf.m == 0) {
 766                 length = String::compose("%1s", hmsf.s);
 767         } else if (hmsf.h == 0 && hmsf.m > 0) {
 768                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 769         } else if (hmsf.h > 0 && hmsf.m > 0) {
 770                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 771         }
 772
 773         boost::algorithm::replace_all (text, "$LENGTH", length);
 774
 775         file.checked_write(text.c_str(), text.length());
 776 }
 777
 778
 779 /** @param frame Frame index within the whole DCP.
 780  *  @return true if we can fake-write this frame.
 781  */
 782 bool
 783 Writer::can_fake_write (Frame frame) const
 784 {
 785         if (film()->encrypted()) {
 786                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 787                 return false;
 788         }
 789
 790         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 791            parameters in the asset writer.
 792         */
 793
 794         auto const & reel = _reels[video_reel(frame)];
 795
 796         /* Make frame relative to the start of the reel */
 797         frame -= reel.start ();
 798         return (frame != 0 && frame < reel.first_nonexistent_frame());
 799 }
 800
 801
 802 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 803 void
 804 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 805 {
 806         vector<ReelWriter>::iterator* reel = nullptr;
 807
 808         switch (type) {
 809         case TextType::OPEN_SUBTITLE:
 810                 reel = &_subtitle_reel;
 811                 _have_subtitles = true;
 812                 break;
 813         case TextType::CLOSED_CAPTION:
 814                 DCPOMATIC_ASSERT (track);
 815                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 816                 reel = &_caption_reels[*track];
 817                 _have_closed_captions.insert (*track);
 818                 break;
 819         default:
 820                 DCPOMATIC_ASSERT (false);
 821         }
 822
 823         DCPOMATIC_ASSERT (*reel != _reels.end());
 824         while ((*reel)->period().to <= period.from) {
 825                 ++(*reel);
 826                 DCPOMATIC_ASSERT (*reel != _reels.end());
 827                 write_hanging_text (**reel);
 828         }
 829
 830         auto back_off = [this](DCPTimePeriod period) {
 831                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 832                 return period;
 833         };
 834
 835         if (period.to > (*reel)->period().to) {
 836                 /* This text goes off the end of the reel.  Store parts of it that should go into
 837                  * other reels.
 838                  */
 839                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 840                         auto overlap = i->period().overlap(period);
 841                         if (overlap) {
 842                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 843                         }
 844                 }
 845                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 846                  * for subtitles being too close together.
 847                  */
 848                 period.to = (*reel)->period().to;
 849                 period = back_off(period);
 850         }
 851
 852         (*reel)->write(text, type, track, period, _fonts, _chosen_interop_font);
 853 }
 854
 855
 856 void
 857 Writer::write (vector<shared_ptr<Font>> fonts)
 858 {
 859         if (fonts.empty()) {
 860                 return;
 861         }
 862
 863         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 864         auto fix_id = [](string id) {
 865                 return id.empty() ? "font" : id;
 866         };
 867
 868         if (film()->interop()) {
 869                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 870                  * even write them as they upset some validators.  Set up _fonts so that every
 871                  * font used by any subtitle will be written with the same ID.
 872                  */
 873                 for (size_t i = 0; i < fonts.size(); ++i) {
 874                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 875                 }
 876                 _chosen_interop_font = fonts[0];
 877         } else {
 878                 set<string> used_ids;
 879
 880                 /* Return the index of a _N at the end of a string, or string::npos */
 881                 auto underscore_number_position = [](string s) {
 882                         auto last_underscore = s.find_last_of("_");
 883                         if (last_underscore == string::npos) {
 884                                 return string::npos;
 885                         }
 886
 887                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 888                                 if (!isdigit(s[i])) {
 889                                         return string::npos;
 890                                 }
 891                         }
 892
 893                         return last_underscore;
 894                 };
 895
 896                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 897                 for (auto font: fonts) {
 898                         auto id = fix_id(font->id());
 899                         if (used_ids.find(id) == used_ids.end()) {
 900                                 /* This ID is unique so we can just use it as-is */
 901                                 _fonts.put(font, id);
 902                                 used_ids.insert(id);
 903                         } else {
 904                                 auto end = underscore_number_position(id);
 905                                 if (end == string::npos) {
 906                                         /* This string has no _N suffix, so add one */
 907                                         id += "_0";
 908                                         end = underscore_number_position(id);
 909                                 }
 910
 911                                 ++end;
 912
 913                                 /* Increment the suffix until we find a unique one */
 914                                 auto number = dcp::raw_convert<int>(id.substr(end));
 915                                 while (used_ids.find(id) != used_ids.end()) {
 916                                         ++number;
 917                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 918                                 }
 919                                 used_ids.insert(id);
 920                         }
 921                         _fonts.put(font, id);
 922                 }
 923
 924                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 925         }
 926 }
 927
 928
 929 bool
 930 operator< (QueueItem const & a, QueueItem const & b)
 931 {
 932         if (a.reel != b.reel) {
 933                 return a.reel < b.reel;
 934         }
 935
 936         if (a.frame != b.frame) {
 937                 return a.frame < b.frame;
 938         }
 939
 940         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 941 }
 942
 943
 944 bool
 945 operator== (QueueItem const & a, QueueItem const & b)
 946 {
 947         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 948 }
 949
 950
 951 void
 952 Writer::set_encoder_threads (int threads)
 953 {
 954         boost::mutex::scoped_lock lm (_state_mutex);
 955         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 956         _maximum_queue_size = threads * 16;
 957 }
 958
 959
 960 void
 961 Writer::write (ReferencedReelAsset asset)
 962 {
 963         _reel_assets.push_back (asset);
 964 }
 965
 966
 967 size_t
 968 Writer::video_reel (int frame) const
 969 {
 970         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 971         size_t reel_index = 0;
 972         while (reel_index < _reels.size() && !_reels[reel_index].period().contains(t)) {
 973                 ++reel_index;
 974         }
 975
 976         DCPOMATIC_ASSERT(reel_index < _reels.size ());
 977         return reel_index;
 978 }
 979
 980
 981 void
 982 Writer::set_digest_progress (Job* job, float progress)
 983 {
 984         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 985
 986         _digest_progresses[boost::this_thread::get_id()] = progress;
 987         float min_progress = FLT_MAX;
 988         for (auto const& i: _digest_progresses) {
 989                 min_progress = min (min_progress, i.second);
 990         }
 991
 992         job->set_progress (min_progress);
 993
 994         Waker waker;
 995         waker.nudge ();
 996
 997         boost::this_thread::interruption_point();
 998 }
 999
1000
1001 /** Calculate hashes for any referenced MXF assets which do not already have one */
1002 void
1003 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
1004 try
1005 {
1006         for (auto const& i: _reel_assets) {
1007                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1008                 if (file && !file->hash()) {
1009                         file->asset_ref().asset()->hash (set_progress);
1010                         file->set_hash (file->asset_ref().asset()->hash());
1011                 }
1012         }
1013 } catch (boost::thread_interrupted) {
1014         /* set_progress contains an interruption_point, so any of these methods
1015          * may throw thread_interrupted, at which point we just give up.
1016          */
1017 }
1018
1019
1020 void
1021 Writer::write_hanging_text (ReelWriter& reel)
1022 {
1023         vector<HangingText> new_hanging_texts;
1024         for (auto i: _hanging_texts) {
1025                 if (i.period.from == reel.period().from) {
1026                         reel.write(i.text, i.type, i.track, i.period, _fonts, _chosen_interop_font);
1027                 } else {
1028                         new_hanging_texts.push_back (i);
1029                 }
1030         }
1031         _hanging_texts = new_hanging_texts;
1032 }