src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "constants.h"
  27 #include "cross.h"
  28 #include "dcp_content_type.h"
  29 #include "dcp_video.h"
  30 #include "dcpomatic_log.h"
  31 #include "film.h"
  32 #include "film_util.h"
  33 #include "job.h"
  34 #include "log.h"
  35 #include "ratio.h"
  36 #include "reel_writer.h"
  37 #include "text_content.h"
  38 #include "util.h"
  39 #include "version.h"
  40 #include "writer.h"
  41 #include <dcp/cpl.h>
  42 #include <dcp/locale_convert.h>
  43 #include <dcp/raw_convert.h>
  44 #include <dcp/reel_file_asset.h>
  45 #include <cerrno>
  46 #include <cfloat>
  47 #include <set>
  48
  49 #include "i18n.h"
  50
  51
  52 /* OS X strikes again */
  53 #undef set_key
  54
  55
  56 using std::cout;
  57 using std::dynamic_pointer_cast;
  58 using std::make_shared;
  59 using std::max;
  60 using std::min;
  61 using std::shared_ptr;
  62 using std::set;
  63 using std::string;
  64 using std::vector;
  65 using std::weak_ptr;
  66 using boost::optional;
  67 #if BOOST_VERSION >= 106100
  68 using namespace boost::placeholders;
  69 #endif
  70 using dcp::Data;
  71 using dcp::ArrayData;
  72 using namespace dcpomatic;
  73
  74
  75 /** @param weak_job Job to report progress to, or 0.
  76  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  77  */
  78 Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, bool text_only)
  79         : WeakConstFilm (weak_film)
  80         , _job(weak_job)
  81         /* These will be reset to sensible values when J2KEncoder is created */
  82         , _maximum_frames_in_memory (8)
  83         , _maximum_queue_size (8)
  84         , _text_only (text_only)
  85 {
  86         auto job = _job.lock ();
  87
  88         int reel_index = 0;
  89         auto const reels = film()->reels();
  90         for (auto p: reels) {
  91                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  92         }
  93
  94         _last_written.resize (reels.size());
  95
  96         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  97            and captions arrive to the Writer in sequence.  This is not so for video.
  98         */
  99         _audio_reel = _reels.begin ();
 100         _subtitle_reel = _reels.begin ();
 101         for (auto i: film()->closed_caption_tracks()) {
 102                 _caption_reels[i] = _reels.begin ();
 103         }
 104         _atmos_reel = _reels.begin ();
 105
 106         /* Check that the signer is OK */
 107         string reason;
 108         if (!Config::instance()->signer_chain()->valid(&reason)) {
 109                 throw InvalidSignerError (reason);
 110         }
 111 }
 112
 113
 114 void
 115 Writer::start ()
 116 {
 117         if (!_text_only) {
 118                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 119 #ifdef DCPOMATIC_LINUX
 120                 pthread_setname_np (_thread.native_handle(), "writer");
 121 #endif
 122         }
 123 }
 124
 125
 126 Writer::~Writer ()
 127 {
 128         if (!_text_only) {
 129                 terminate_thread (false);
 130         }
 131 }
 132
 133
 134 /** Pass a video frame to the writer for writing to disk at some point.
 135  *  This method can be called with frames out of order.
 136  *  @param encoded JPEG2000-encoded data.
 137  *  @param frame Frame index within the DCP.
 138  *  @param eyes Eyes that this frame image is for.
 139  */
 140 void
 141 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 142 {
 143         boost::mutex::scoped_lock lock (_state_mutex);
 144
 145         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 146                 /* There are too many full frames in memory; wake the main writer thread and
 147                    wait until it sorts everything out */
 148                 _empty_condition.notify_all ();
 149                 _full_condition.wait (lock);
 150         }
 151
 152         QueueItem qi;
 153         qi.type = QueueItem::Type::FULL;
 154         qi.encoded = encoded;
 155         qi.reel = video_reel (frame);
 156         qi.frame = frame - _reels[qi.reel].start ();
 157
 158         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 159
 160         qi.eyes = eyes;
 161         _queue.push_back(qi);
 162         ++_queued_full_in_memory;
 163
 164         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 165         _empty_condition.notify_all ();
 166 }
 167
 168
 169 bool
 170 Writer::can_repeat (Frame frame) const
 171 {
 172         return frame > _reels[video_reel(frame)].start();
 173 }
 174
 175
 176 /** Repeat the last frame that was written to a reel as a new frame.
 177  *  @param frame Frame index within the DCP of the new (repeated) frame.
 178  *  @param eyes Eyes that this repeated frame image is for.
 179  */
 180 void
 181 Writer::repeat (Frame frame, Eyes eyes)
 182 {
 183         boost::mutex::scoped_lock lock (_state_mutex);
 184
 185         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 186                 /* The queue is too big, and the main writer thread can run and fix it, so
 187                    wake it and wait until it has done.
 188                 */
 189                 _empty_condition.notify_all ();
 190                 _full_condition.wait (lock);
 191         }
 192
 193         QueueItem qi;
 194         qi.type = QueueItem::Type::REPEAT;
 195         qi.reel = video_reel (frame);
 196         qi.frame = frame - _reels[qi.reel].start ();
 197         if (film()->three_d() && eyes == Eyes::BOTH) {
 198                 qi.eyes = Eyes::LEFT;
 199                 _queue.push_back (qi);
 200                 qi.eyes = Eyes::RIGHT;
 201                 _queue.push_back (qi);
 202         } else {
 203                 qi.eyes = eyes;
 204                 _queue.push_back (qi);
 205         }
 206
 207         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 208         _empty_condition.notify_all ();
 209 }
 210
 211
 212 void
 213 Writer::fake_write (Frame frame, Eyes eyes)
 214 {
 215         boost::mutex::scoped_lock lock (_state_mutex);
 216
 217         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 218                 /* The queue is too big, and the main writer thread can run and fix it, so
 219                    wake it and wait until it has done.
 220                 */
 221                 _empty_condition.notify_all ();
 222                 _full_condition.wait (lock);
 223         }
 224
 225         size_t const reel = video_reel (frame);
 226         Frame const frame_in_reel = frame - _reels[reel].start ();
 227
 228         QueueItem qi;
 229         qi.type = QueueItem::Type::FAKE;
 230
 231         {
 232                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 233                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 234         }
 235
 236         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 237
 238         qi.reel = reel;
 239         qi.frame = frame_in_reel;
 240         qi.eyes = eyes;
 241         _queue.push_back(qi);
 242
 243         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 244         _empty_condition.notify_all ();
 245 }
 246
 247
 248 /** Write some audio frames to the DCP.
 249  *  @param audio Audio data.
 250  *  @param time Time of this data within the DCP.
 251  *  This method is not thread safe.
 252  */
 253 void
 254 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 255 {
 256         DCPOMATIC_ASSERT (audio);
 257
 258         int const afr = film()->audio_frame_rate();
 259
 260         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 261
 262         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 263
 264         DCPTime t = time;
 265         while (t < end) {
 266
 267                 if (_audio_reel == _reels.end ()) {
 268                         /* This audio is off the end of the last reel; ignore it */
 269                         return;
 270                 }
 271
 272                 if (end <= _audio_reel->period().to) {
 273                         /* Easy case: we can write all the audio to this reel */
 274                         _audio_reel->write (audio);
 275                         t = end;
 276                 } else if (_audio_reel->period().to <= t) {
 277                         /* This reel is entirely before the start of our audio; just skip the reel */
 278                         ++_audio_reel;
 279                 } else {
 280                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 281                         DCPTime part_lengths[2] = {
 282                                 _audio_reel->period().to - t,
 283                                 end - _audio_reel->period().to
 284                         };
 285
 286                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 287                         Frame part_frames[2] = {
 288                                 part_lengths[0].frames_ceil(afr),
 289                                 part_lengths[1].frames_floor(afr)
 290                         };
 291
 292                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 293
 294                         if (part_frames[0]) {
 295                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 296                                 _audio_reel->write (part);
 297                         }
 298
 299                         if (part_frames[1]) {
 300                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 301                         } else {
 302                                 audio.reset ();
 303                         }
 304
 305                         ++_audio_reel;
 306                         t += part_lengths[0];
 307                 }
 308         }
 309 }
 310
 311
 312 void
 313 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 314 {
 315         if (_atmos_reel->period().to == time) {
 316                 ++_atmos_reel;
 317                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 318         }
 319
 320         /* We assume that we get a video frame's worth of data here */
 321         _atmos_reel->write (atmos, metadata);
 322 }
 323
 324
 325 /** Caller must hold a lock on _state_mutex */
 326 bool
 327 Writer::have_sequenced_image_at_queue_head ()
 328 {
 329         if (_queue.empty ()) {
 330                 return false;
 331         }
 332
 333         _queue.sort ();
 334         auto const & f = _queue.front();
 335         return _last_written[f.reel].next(f);
 336 }
 337
 338
 339 bool
 340 Writer::LastWritten::next (QueueItem qi) const
 341 {
 342         if (qi.eyes == Eyes::BOTH) {
 343                 /* 2D */
 344                 return qi.frame == (_frame + 1);
 345         }
 346
 347         /* 3D */
 348
 349         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 350                 return true;
 351         }
 352
 353         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 354                 return true;
 355         }
 356
 357         return false;
 358 }
 359
 360
 361 void
 362 Writer::LastWritten::update (QueueItem qi)
 363 {
 364         _frame = qi.frame;
 365         _eyes = qi.eyes;
 366 }
 367
 368
 369 void
 370 Writer::thread ()
 371 try
 372 {
 373         start_of_thread ("Writer");
 374
 375         while (true)
 376         {
 377                 boost::mutex::scoped_lock lock (_state_mutex);
 378
 379                 while (true) {
 380
 381                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 382                                 /* We've got something to do: go and do it */
 383                                 break;
 384                         }
 385
 386                         /* Nothing to do: wait until something happens which may indicate that we do */
 387                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 388                         _empty_condition.wait (lock);
 389                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 390                 }
 391
 392                 /* We stop here if we have been asked to finish, and if either the queue
 393                    is empty or we do not have a sequenced image at its head (if this is the
 394                    case we will never terminate as no new frames will be sent once
 395                    _finish is true).
 396                 */
 397                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 398                         /* (Hopefully temporarily) log anything that was not written */
 399                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 400                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 401                                 for (auto const& i: _queue) {
 402                                         if (i.type == QueueItem::Type::FULL) {
 403                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 404                                         } else {
 405                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 406                                         }
 407                                 }
 408                         }
 409                         return;
 410                 }
 411
 412                 /* Write any frames that we can write; i.e. those that are in sequence. */
 413                 while (have_sequenced_image_at_queue_head ()) {
 414                         auto qi = _queue.front ();
 415                         _last_written[qi.reel].update (qi);
 416                         _queue.pop_front ();
 417                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 418                                 --_queued_full_in_memory;
 419                         }
 420
 421                         lock.unlock ();
 422
 423                         auto& reel = _reels[qi.reel];
 424
 425                         switch (qi.type) {
 426                         case QueueItem::Type::FULL:
 427                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 428                                 if (!qi.encoded) {
 429                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 430                                 }
 431                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 432                                 ++_full_written;
 433                                 break;
 434                         case QueueItem::Type::FAKE:
 435                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 436                                 reel.fake_write (qi.size);
 437                                 ++_fake_written;
 438                                 break;
 439                         case QueueItem::Type::REPEAT:
 440                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 441                                 reel.repeat_write (qi.frame, qi.eyes);
 442                                 ++_repeat_written;
 443                                 break;
 444                         }
 445
 446                         lock.lock ();
 447                         _full_condition.notify_all ();
 448                 }
 449
 450                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 451                         /* Too many frames in memory which can't yet be written to the stream.
 452                            Write some FULL frames to disk.
 453                         */
 454
 455                         /* Find one from the back of the queue */
 456                         _queue.sort ();
 457                         auto item = _queue.rbegin();
 458                         while (item != _queue.rend() && (item->type != QueueItem::Type::FULL || !item->encoded)) {
 459                                 ++item;
 460                         }
 461
 462                         DCPOMATIC_ASSERT(item != _queue.rend());
 463                         ++_pushed_to_disk;
 464                         /* For the log message below */
 465                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 466                         lock.unlock ();
 467
 468                         /* i is valid here, even though we don't hold a lock on the mutex,
 469                            since list iterators are unaffected by insertion and only this
 470                            thread could erase the last item in the list.
 471                         */
 472
 473                         LOG_GENERAL("Writer full; pushes %1 to disk while awaiting %2", item->frame, awaiting);
 474
 475                         item->encoded->write_via_temp(
 476                                 film()->j2c_path(item->reel, item->frame, item->eyes, true),
 477                                 film()->j2c_path(item->reel, item->frame, item->eyes, false)
 478                                 );
 479
 480                         lock.lock ();
 481                         item->encoded.reset();
 482                         --_queued_full_in_memory;
 483                         _full_condition.notify_all ();
 484                 }
 485         }
 486 }
 487 catch (...)
 488 {
 489         store_current ();
 490 }
 491
 492
 493 void
 494 Writer::terminate_thread (bool can_throw)
 495 {
 496         boost::this_thread::disable_interruption dis;
 497
 498         boost::mutex::scoped_lock lock (_state_mutex);
 499
 500         _finish = true;
 501         _empty_condition.notify_all ();
 502         _full_condition.notify_all ();
 503         lock.unlock ();
 504
 505         try {
 506                 _thread.join ();
 507         } catch (...) {}
 508
 509         if (can_throw) {
 510                 rethrow ();
 511         }
 512 }
 513
 514
 515 void
 516 Writer::calculate_digests ()
 517 {
 518         auto job = _job.lock ();
 519         if (job) {
 520                 job->sub (_("Computing digests"));
 521         }
 522
 523         boost::asio::io_service service;
 524         boost::thread_group pool;
 525
 526         auto work = make_shared<boost::asio::io_service::work>(service);
 527
 528         int const threads = max (1, Config::instance()->master_encoding_threads());
 529
 530         for (int i = 0; i < threads; ++i) {
 531                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 532         }
 533
 534         std::function<void (float)> set_progress;
 535         if (job) {
 536                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 537         } else {
 538                 set_progress = [](float) {
 539                         boost::this_thread::interruption_point();
 540                 };
 541         }
 542
 543         for (auto& i: _reels) {
 544                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 545         }
 546         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 547
 548         work.reset ();
 549
 550         try {
 551                 pool.join_all ();
 552         } catch (boost::thread_interrupted) {
 553                 /* join_all was interrupted, so we need to interrupt the threads
 554                  * in our pool then try again to join them.
 555                  */
 556                 pool.interrupt_all ();
 557                 pool.join_all ();
 558         }
 559
 560         service.stop ();
 561 }
 562
 563
 564 /** @param output_dcp Path to DCP folder to write */
 565 void
 566 Writer::finish (boost::filesystem::path output_dcp)
 567 {
 568         if (_thread.joinable()) {
 569                 LOG_GENERAL_NC ("Terminating writer thread");
 570                 terminate_thread (true);
 571         }
 572
 573         LOG_GENERAL_NC ("Finishing ReelWriters");
 574
 575         for (auto& reel: _reels) {
 576                 write_hanging_text(reel);
 577                 reel.finish(output_dcp);
 578         }
 579
 580         LOG_GENERAL_NC ("Writing XML");
 581
 582         dcp::DCP dcp (output_dcp);
 583
 584         auto cpl = make_shared<dcp::CPL>(
 585                 film()->dcp_name(),
 586                 film()->dcp_content_type()->libdcp_kind(),
 587                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 588                 );
 589
 590         dcp.add (cpl);
 591
 592         calculate_digests ();
 593
 594         /* Add reels */
 595
 596         for (auto& i: _reels) {
 597                 cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
 598         }
 599
 600         /* Add metadata */
 601
 602         auto creator = Config::instance()->dcp_creator();
 603         if (creator.empty()) {
 604                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 605         }
 606
 607         auto issuer = Config::instance()->dcp_issuer();
 608         if (issuer.empty()) {
 609                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 610         }
 611
 612         cpl->set_creator (creator);
 613         cpl->set_issuer (issuer);
 614
 615         cpl->set_ratings (film()->ratings());
 616
 617         vector<dcp::ContentVersion> cv;
 618         for (auto i: film()->content_versions()) {
 619                 cv.push_back (dcp::ContentVersion(i));
 620         }
 621         if (cv.empty()) {
 622                 cv = { dcp::ContentVersion("1") };
 623         }
 624         cpl->set_content_versions (cv);
 625
 626         cpl->set_full_content_title_text (film()->name());
 627         cpl->set_full_content_title_text_language (film()->name_language());
 628         if (film()->release_territory()) {
 629                 cpl->set_release_territory (*film()->release_territory());
 630         }
 631         cpl->set_version_number (film()->version_number());
 632         cpl->set_status (film()->status());
 633         if (film()->chain()) {
 634                 cpl->set_chain (*film()->chain());
 635         }
 636         if (film()->distributor()) {
 637                 cpl->set_distributor (*film()->distributor());
 638         }
 639         if (film()->facility()) {
 640                 cpl->set_facility (*film()->facility());
 641         }
 642         if (film()->luminance()) {
 643                 cpl->set_luminance (*film()->luminance());
 644         }
 645         if (film()->sign_language_video_language()) {
 646                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 647         }
 648
 649         dcp::MCASoundField field;
 650         if (channel_is_mapped(film(), dcp::Channel::BSL) || channel_is_mapped(film(), dcp::Channel::BSR)) {
 651                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 652         } else {
 653                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 654         }
 655
 656         auto const audio_channels = film()->audio_channels();
 657         dcp::MainSoundConfiguration msc(field, audio_channels);
 658         for (auto i: film()->mapped_audio_channels()) {
 659                 if (i < audio_channels) {
 660                         msc.set_mapping(i, static_cast<dcp::Channel>(i));
 661                 }
 662         }
 663
 664         cpl->set_main_sound_configuration(msc);
 665         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 666         cpl->set_main_picture_stored_area (film()->frame_size());
 667
 668         auto active_area = film()->active_area();
 669         if (active_area.width > 0 && active_area.height > 0) {
 670                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 671                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 672         }
 673
 674         auto sl = film()->subtitle_languages().second;
 675         if (!sl.empty()) {
 676                 cpl->set_additional_subtitle_languages(sl);
 677         }
 678
 679         auto signer = Config::instance()->signer_chain();
 680         /* We did check earlier, but check again here to be on the safe side */
 681         string reason;
 682         if (!signer->valid (&reason)) {
 683                 throw InvalidSignerError (reason);
 684         }
 685
 686         dcp.set_issuer(issuer);
 687         dcp.set_creator(creator);
 688         dcp.set_annotation_text(film()->dcp_name());
 689
 690         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 691
 692         LOG_GENERAL (
 693                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 694                 );
 695
 696         write_cover_sheet (output_dcp);
 697 }
 698
 699
 700 void
 701 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 702 {
 703         auto const cover = film()->file("COVER_SHEET.txt");
 704         dcp::File file(cover, "w");
 705         if (!file) {
 706                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 707         }
 708
 709         auto text = Config::instance()->cover_sheet ();
 710         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 711         auto cpls = film()->cpls();
 712         if (!cpls.empty()) {
 713                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 714         }
 715         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 716         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 717
 718         auto audio_language = film()->audio_language();
 719         if (audio_language) {
 720                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 721         } else {
 722                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 723         }
 724
 725         auto subtitle_languages = film()->subtitle_languages();
 726         if (subtitle_languages.first) {
 727                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 728         } else {
 729                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 730         }
 731
 732         boost::uintmax_t size = 0;
 733         for (
 734                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 735                 i != boost::filesystem::recursive_directory_iterator();
 736                 ++i) {
 737                 if (boost::filesystem::is_regular_file (i->path())) {
 738                         size += boost::filesystem::file_size (i->path());
 739                 }
 740         }
 741
 742         if (size > (1000000000L)) {
 743                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 744         } else {
 745                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 746         }
 747
 748         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 749         auto description = String::compose("%1.%2", ch.first, ch.second);
 750
 751         if (description == "0.0") {
 752                 description = _("None");
 753         } else if (description == "1.0") {
 754                 description = _("Mono");
 755         } else if (description == "2.0") {
 756                 description = _("Stereo");
 757         }
 758         boost::algorithm::replace_all (text, "$AUDIO", description);
 759
 760         auto const hmsf = film()->length().split(film()->video_frame_rate());
 761         string length;
 762         if (hmsf.h == 0 && hmsf.m == 0) {
 763                 length = String::compose("%1s", hmsf.s);
 764         } else if (hmsf.h == 0 && hmsf.m > 0) {
 765                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 766         } else if (hmsf.h > 0 && hmsf.m > 0) {
 767                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 768         }
 769
 770         boost::algorithm::replace_all (text, "$LENGTH", length);
 771
 772         file.checked_write(text.c_str(), text.length());
 773 }
 774
 775
 776 /** @param frame Frame index within the whole DCP.
 777  *  @return true if we can fake-write this frame.
 778  */
 779 bool
 780 Writer::can_fake_write (Frame frame) const
 781 {
 782         if (film()->encrypted()) {
 783                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 784                 return false;
 785         }
 786
 787         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 788            parameters in the asset writer.
 789         */
 790
 791         auto const & reel = _reels[video_reel(frame)];
 792
 793         /* Make frame relative to the start of the reel */
 794         frame -= reel.start ();
 795         return (frame != 0 && frame < reel.first_nonexistent_frame());
 796 }
 797
 798
 799 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 800 void
 801 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 802 {
 803         vector<ReelWriter>::iterator* reel = nullptr;
 804
 805         switch (type) {
 806         case TextType::OPEN_SUBTITLE:
 807                 reel = &_subtitle_reel;
 808                 _have_subtitles = true;
 809                 break;
 810         case TextType::CLOSED_CAPTION:
 811                 DCPOMATIC_ASSERT (track);
 812                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 813                 reel = &_caption_reels[*track];
 814                 _have_closed_captions.insert (*track);
 815                 break;
 816         default:
 817                 DCPOMATIC_ASSERT (false);
 818         }
 819
 820         DCPOMATIC_ASSERT (*reel != _reels.end());
 821         while ((*reel)->period().to <= period.from) {
 822                 ++(*reel);
 823                 DCPOMATIC_ASSERT (*reel != _reels.end());
 824                 write_hanging_text (**reel);
 825         }
 826
 827         auto back_off = [this](DCPTimePeriod period) {
 828                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 829                 return period;
 830         };
 831
 832         if (period.to > (*reel)->period().to) {
 833                 /* This text goes off the end of the reel.  Store parts of it that should go into
 834                  * other reels.
 835                  */
 836                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 837                         auto overlap = i->period().overlap(period);
 838                         if (overlap) {
 839                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 840                         }
 841                 }
 842                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 843                  * for subtitles being too close together.
 844                  */
 845                 period.to = (*reel)->period().to;
 846                 period = back_off(period);
 847         }
 848
 849         (*reel)->write(text, type, track, period, _fonts, _chosen_interop_font);
 850 }
 851
 852
 853 void
 854 Writer::write (vector<shared_ptr<Font>> fonts)
 855 {
 856         if (fonts.empty()) {
 857                 return;
 858         }
 859
 860         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 861         auto fix_id = [](string id) {
 862                 return id.empty() ? "font" : id;
 863         };
 864
 865         if (film()->interop()) {
 866                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 867                  * even write them as they upset some validators.  Set up _fonts so that every
 868                  * font used by any subtitle will be written with the same ID.
 869                  */
 870                 for (size_t i = 0; i < fonts.size(); ++i) {
 871                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 872                 }
 873                 _chosen_interop_font = fonts[0];
 874         } else {
 875                 set<string> used_ids;
 876
 877                 /* Return the index of a _N at the end of a string, or string::npos */
 878                 auto underscore_number_position = [](string s) {
 879                         auto last_underscore = s.find_last_of("_");
 880                         if (last_underscore == string::npos) {
 881                                 return string::npos;
 882                         }
 883
 884                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 885                                 if (!isdigit(s[i])) {
 886                                         return string::npos;
 887                                 }
 888                         }
 889
 890                         return last_underscore;
 891                 };
 892
 893                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 894                 for (auto font: fonts) {
 895                         auto id = fix_id(font->id());
 896                         if (used_ids.find(id) == used_ids.end()) {
 897                                 /* This ID is unique so we can just use it as-is */
 898                                 _fonts.put(font, id);
 899                                 used_ids.insert(id);
 900                         } else {
 901                                 auto end = underscore_number_position(id);
 902                                 if (end == string::npos) {
 903                                         /* This string has no _N suffix, so add one */
 904                                         id += "_0";
 905                                         end = underscore_number_position(id);
 906                                 }
 907
 908                                 ++end;
 909
 910                                 /* Increment the suffix until we find a unique one */
 911                                 auto number = dcp::raw_convert<int>(id.substr(end));
 912                                 while (used_ids.find(id) != used_ids.end()) {
 913                                         ++number;
 914                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 915                                 }
 916                                 used_ids.insert(id);
 917                         }
 918                         _fonts.put(font, id);
 919                 }
 920
 921                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 922         }
 923 }
 924
 925
 926 bool
 927 operator< (QueueItem const & a, QueueItem const & b)
 928 {
 929         if (a.reel != b.reel) {
 930                 return a.reel < b.reel;
 931         }
 932
 933         if (a.frame != b.frame) {
 934                 return a.frame < b.frame;
 935         }
 936
 937         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 938 }
 939
 940
 941 bool
 942 operator== (QueueItem const & a, QueueItem const & b)
 943 {
 944         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 945 }
 946
 947
 948 void
 949 Writer::set_encoder_threads (int threads)
 950 {
 951         boost::mutex::scoped_lock lm (_state_mutex);
 952         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 953         _maximum_queue_size = threads * 16;
 954 }
 955
 956
 957 void
 958 Writer::write (ReferencedReelAsset asset)
 959 {
 960         _reel_assets.push_back (asset);
 961 }
 962
 963
 964 size_t
 965 Writer::video_reel (int frame) const
 966 {
 967         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 968         size_t reel_index = 0;
 969         while (reel_index < _reels.size() && !_reels[reel_index].period().contains(t)) {
 970                 ++reel_index;
 971         }
 972
 973         DCPOMATIC_ASSERT(reel_index < _reels.size ());
 974         return reel_index;
 975 }
 976
 977
 978 void
 979 Writer::set_digest_progress (Job* job, float progress)
 980 {
 981         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 982
 983         _digest_progresses[boost::this_thread::get_id()] = progress;
 984         float min_progress = FLT_MAX;
 985         for (auto const& i: _digest_progresses) {
 986                 min_progress = min (min_progress, i.second);
 987         }
 988
 989         job->set_progress (min_progress);
 990
 991         Waker waker;
 992         waker.nudge ();
 993
 994         boost::this_thread::interruption_point();
 995 }
 996
 997
 998 /** Calculate hashes for any referenced MXF assets which do not already have one */
 999 void
1000 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
1001 try
1002 {
1003         for (auto const& i: _reel_assets) {
1004                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1005                 if (file && !file->hash()) {
1006                         file->asset_ref().asset()->hash (set_progress);
1007                         file->set_hash (file->asset_ref().asset()->hash());
1008                 }
1009         }
1010 } catch (boost::thread_interrupted) {
1011         /* set_progress contains an interruption_point, so any of these methods
1012          * may throw thread_interrupted, at which point we just give up.
1013          */
1014 }
1015
1016
1017 void
1018 Writer::write_hanging_text (ReelWriter& reel)
1019 {
1020         vector<HangingText> new_hanging_texts;
1021         for (auto i: _hanging_texts) {
1022                 if (i.period.from == reel.period().from) {
1023                         reel.write(i.text, i.type, i.track, i.period, _fonts, _chosen_interop_font);
1024                 } else {
1025                         new_hanging_texts.push_back (i);
1026                 }
1027         }
1028         _hanging_texts = new_hanging_texts;
1029 }