src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "constants.h"
  27 #include "cross.h"
  28 #include "dcp_content_type.h"
  29 #include "dcp_video.h"
  30 #include "dcpomatic_log.h"
  31 #include "film.h"
  32 #include "film_util.h"
  33 #include "job.h"
  34 #include "log.h"
  35 #include "ratio.h"
  36 #include "reel_writer.h"
  37 #include "text_content.h"
  38 #include "util.h"
  39 #include "version.h"
  40 #include "writer.h"
  41 #include <dcp/cpl.h>
  42 #include <dcp/locale_convert.h>
  43 #include <dcp/raw_convert.h>
  44 #include <dcp/reel_file_asset.h>
  45 #include <cerrno>
  46 #include <cfloat>
  47 #include <set>
  48
  49 #include "i18n.h"
  50
  51
  52 /* OS X strikes again */
  53 #undef set_key
  54
  55
  56 using std::cout;
  57 using std::dynamic_pointer_cast;
  58 using std::make_shared;
  59 using std::max;
  60 using std::min;
  61 using std::shared_ptr;
  62 using std::set;
  63 using std::string;
  64 using std::vector;
  65 using std::weak_ptr;
  66 using boost::optional;
  67 #if BOOST_VERSION >= 106100
  68 using namespace boost::placeholders;
  69 #endif
  70 using dcp::Data;
  71 using dcp::ArrayData;
  72 using namespace dcpomatic;
  73
  74
  75 /** @param j Job to report progress to, or 0.
  76  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  77  */
  78 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
  79         : WeakConstFilm (weak_film)
  80         , _job (j)
  81         /* These will be reset to sensible values when J2KEncoder is created */
  82         , _maximum_frames_in_memory (8)
  83         , _maximum_queue_size (8)
  84         , _text_only (text_only)
  85 {
  86         auto job = _job.lock ();
  87
  88         int reel_index = 0;
  89         auto const reels = film()->reels();
  90         for (auto p: reels) {
  91                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  92         }
  93
  94         _last_written.resize (reels.size());
  95
  96         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  97            and captions arrive to the Writer in sequence.  This is not so for video.
  98         */
  99         _audio_reel = _reels.begin ();
 100         _subtitle_reel = _reels.begin ();
 101         for (auto i: film()->closed_caption_tracks()) {
 102                 _caption_reels[i] = _reels.begin ();
 103         }
 104         _atmos_reel = _reels.begin ();
 105
 106         /* Check that the signer is OK */
 107         string reason;
 108         if (!Config::instance()->signer_chain()->valid(&reason)) {
 109                 throw InvalidSignerError (reason);
 110         }
 111 }
 112
 113
 114 void
 115 Writer::start ()
 116 {
 117         if (!_text_only) {
 118                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 119 #ifdef DCPOMATIC_LINUX
 120                 pthread_setname_np (_thread.native_handle(), "writer");
 121 #endif
 122         }
 123 }
 124
 125
 126 Writer::~Writer ()
 127 {
 128         if (!_text_only) {
 129                 terminate_thread (false);
 130         }
 131 }
 132
 133
 134 /** Pass a video frame to the writer for writing to disk at some point.
 135  *  This method can be called with frames out of order.
 136  *  @param encoded JPEG2000-encoded data.
 137  *  @param frame Frame index within the DCP.
 138  *  @param eyes Eyes that this frame image is for.
 139  */
 140 void
 141 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 142 {
 143         boost::mutex::scoped_lock lock (_state_mutex);
 144
 145         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 146                 /* There are too many full frames in memory; wake the main writer thread and
 147                    wait until it sorts everything out */
 148                 _empty_condition.notify_all ();
 149                 _full_condition.wait (lock);
 150         }
 151
 152         QueueItem qi;
 153         qi.type = QueueItem::Type::FULL;
 154         qi.encoded = encoded;
 155         qi.reel = video_reel (frame);
 156         qi.frame = frame - _reels[qi.reel].start ();
 157
 158         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 159
 160         qi.eyes = eyes;
 161         _queue.push_back(qi);
 162         ++_queued_full_in_memory;
 163
 164         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 165         _empty_condition.notify_all ();
 166 }
 167
 168
 169 bool
 170 Writer::can_repeat (Frame frame) const
 171 {
 172         return frame > _reels[video_reel(frame)].start();
 173 }
 174
 175
 176 /** Repeat the last frame that was written to a reel as a new frame.
 177  *  @param frame Frame index within the DCP of the new (repeated) frame.
 178  *  @param eyes Eyes that this repeated frame image is for.
 179  */
 180 void
 181 Writer::repeat (Frame frame, Eyes eyes)
 182 {
 183         boost::mutex::scoped_lock lock (_state_mutex);
 184
 185         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 186                 /* The queue is too big, and the main writer thread can run and fix it, so
 187                    wake it and wait until it has done.
 188                 */
 189                 _empty_condition.notify_all ();
 190                 _full_condition.wait (lock);
 191         }
 192
 193         QueueItem qi;
 194         qi.type = QueueItem::Type::REPEAT;
 195         qi.reel = video_reel (frame);
 196         qi.frame = frame - _reels[qi.reel].start ();
 197         if (film()->three_d() && eyes == Eyes::BOTH) {
 198                 qi.eyes = Eyes::LEFT;
 199                 _queue.push_back (qi);
 200                 qi.eyes = Eyes::RIGHT;
 201                 _queue.push_back (qi);
 202         } else {
 203                 qi.eyes = eyes;
 204                 _queue.push_back (qi);
 205         }
 206
 207         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 208         _empty_condition.notify_all ();
 209 }
 210
 211
 212 void
 213 Writer::fake_write (Frame frame, Eyes eyes)
 214 {
 215         boost::mutex::scoped_lock lock (_state_mutex);
 216
 217         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 218                 /* The queue is too big, and the main writer thread can run and fix it, so
 219                    wake it and wait until it has done.
 220                 */
 221                 _empty_condition.notify_all ();
 222                 _full_condition.wait (lock);
 223         }
 224
 225         size_t const reel = video_reel (frame);
 226         Frame const frame_in_reel = frame - _reels[reel].start ();
 227
 228         QueueItem qi;
 229         qi.type = QueueItem::Type::FAKE;
 230
 231         {
 232                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 233                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 234         }
 235
 236         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 237
 238         qi.reel = reel;
 239         qi.frame = frame_in_reel;
 240         qi.eyes = eyes;
 241         _queue.push_back(qi);
 242
 243         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 244         _empty_condition.notify_all ();
 245 }
 246
 247
 248 /** Write some audio frames to the DCP.
 249  *  @param audio Audio data.
 250  *  @param time Time of this data within the DCP.
 251  *  This method is not thread safe.
 252  */
 253 void
 254 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 255 {
 256         DCPOMATIC_ASSERT (audio);
 257
 258         int const afr = film()->audio_frame_rate();
 259
 260         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 261
 262         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 263
 264         DCPTime t = time;
 265         while (t < end) {
 266
 267                 if (_audio_reel == _reels.end ()) {
 268                         /* This audio is off the end of the last reel; ignore it */
 269                         return;
 270                 }
 271
 272                 if (end <= _audio_reel->period().to) {
 273                         /* Easy case: we can write all the audio to this reel */
 274                         _audio_reel->write (audio);
 275                         t = end;
 276                 } else if (_audio_reel->period().to <= t) {
 277                         /* This reel is entirely before the start of our audio; just skip the reel */
 278                         ++_audio_reel;
 279                 } else {
 280                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 281                         DCPTime part_lengths[2] = {
 282                                 _audio_reel->period().to - t,
 283                                 end - _audio_reel->period().to
 284                         };
 285
 286                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 287                         Frame part_frames[2] = {
 288                                 part_lengths[0].frames_ceil(afr),
 289                                 part_lengths[1].frames_floor(afr)
 290                         };
 291
 292                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 293
 294                         if (part_frames[0]) {
 295                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 296                                 _audio_reel->write (part);
 297                         }
 298
 299                         if (part_frames[1]) {
 300                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 301                         } else {
 302                                 audio.reset ();
 303                         }
 304
 305                         ++_audio_reel;
 306                         t += part_lengths[0];
 307                 }
 308         }
 309 }
 310
 311
 312 void
 313 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 314 {
 315         if (_atmos_reel->period().to == time) {
 316                 ++_atmos_reel;
 317                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 318         }
 319
 320         /* We assume that we get a video frame's worth of data here */
 321         _atmos_reel->write (atmos, metadata);
 322 }
 323
 324
 325 /** Caller must hold a lock on _state_mutex */
 326 bool
 327 Writer::have_sequenced_image_at_queue_head ()
 328 {
 329         if (_queue.empty ()) {
 330                 return false;
 331         }
 332
 333         _queue.sort ();
 334         auto const & f = _queue.front();
 335         return _last_written[f.reel].next(f);
 336 }
 337
 338
 339 bool
 340 Writer::LastWritten::next (QueueItem qi) const
 341 {
 342         if (qi.eyes == Eyes::BOTH) {
 343                 /* 2D */
 344                 return qi.frame == (_frame + 1);
 345         }
 346
 347         /* 3D */
 348
 349         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 350                 return true;
 351         }
 352
 353         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 354                 return true;
 355         }
 356
 357         return false;
 358 }
 359
 360
 361 void
 362 Writer::LastWritten::update (QueueItem qi)
 363 {
 364         _frame = qi.frame;
 365         _eyes = qi.eyes;
 366 }
 367
 368
 369 void
 370 Writer::thread ()
 371 try
 372 {
 373         start_of_thread ("Writer");
 374
 375         while (true)
 376         {
 377                 boost::mutex::scoped_lock lock (_state_mutex);
 378
 379                 while (true) {
 380
 381                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 382                                 /* We've got something to do: go and do it */
 383                                 break;
 384                         }
 385
 386                         /* Nothing to do: wait until something happens which may indicate that we do */
 387                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 388                         _empty_condition.wait (lock);
 389                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 390                 }
 391
 392                 /* We stop here if we have been asked to finish, and if either the queue
 393                    is empty or we do not have a sequenced image at its head (if this is the
 394                    case we will never terminate as no new frames will be sent once
 395                    _finish is true).
 396                 */
 397                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 398                         /* (Hopefully temporarily) log anything that was not written */
 399                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 400                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 401                                 for (auto const& i: _queue) {
 402                                         if (i.type == QueueItem::Type::FULL) {
 403                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 404                                         } else {
 405                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 406                                         }
 407                                 }
 408                         }
 409                         return;
 410                 }
 411
 412                 /* Write any frames that we can write; i.e. those that are in sequence. */
 413                 while (have_sequenced_image_at_queue_head ()) {
 414                         auto qi = _queue.front ();
 415                         _last_written[qi.reel].update (qi);
 416                         _queue.pop_front ();
 417                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 418                                 --_queued_full_in_memory;
 419                         }
 420
 421                         lock.unlock ();
 422
 423                         auto& reel = _reels[qi.reel];
 424
 425                         switch (qi.type) {
 426                         case QueueItem::Type::FULL:
 427                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 428                                 if (!qi.encoded) {
 429                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 430                                 }
 431                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 432                                 ++_full_written;
 433                                 break;
 434                         case QueueItem::Type::FAKE:
 435                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 436                                 reel.fake_write (qi.size);
 437                                 ++_fake_written;
 438                                 break;
 439                         case QueueItem::Type::REPEAT:
 440                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 441                                 reel.repeat_write (qi.frame, qi.eyes);
 442                                 ++_repeat_written;
 443                                 break;
 444                         }
 445
 446                         lock.lock ();
 447                         _full_condition.notify_all ();
 448                 }
 449
 450                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 451                         /* Too many frames in memory which can't yet be written to the stream.
 452                            Write some FULL frames to disk.
 453                         */
 454
 455                         /* Find one from the back of the queue */
 456                         _queue.sort ();
 457                         auto i = _queue.rbegin ();
 458                         while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
 459                                 ++i;
 460                         }
 461
 462                         DCPOMATIC_ASSERT (i != _queue.rend());
 463                         ++_pushed_to_disk;
 464                         /* For the log message below */
 465                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 466                         lock.unlock ();
 467
 468                         /* i is valid here, even though we don't hold a lock on the mutex,
 469                            since list iterators are unaffected by insertion and only this
 470                            thread could erase the last item in the list.
 471                         */
 472
 473                         LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
 474
 475                         i->encoded->write_via_temp (
 476                                 film()->j2c_path(i->reel, i->frame, i->eyes, true),
 477                                 film()->j2c_path(i->reel, i->frame, i->eyes, false)
 478                                 );
 479
 480                         lock.lock ();
 481                         i->encoded.reset ();
 482                         --_queued_full_in_memory;
 483                         _full_condition.notify_all ();
 484                 }
 485         }
 486 }
 487 catch (...)
 488 {
 489         store_current ();
 490 }
 491
 492
 493 void
 494 Writer::terminate_thread (bool can_throw)
 495 {
 496         boost::this_thread::disable_interruption dis;
 497
 498         boost::mutex::scoped_lock lock (_state_mutex);
 499
 500         _finish = true;
 501         _empty_condition.notify_all ();
 502         _full_condition.notify_all ();
 503         lock.unlock ();
 504
 505         try {
 506                 _thread.join ();
 507         } catch (...) {}
 508
 509         if (can_throw) {
 510                 rethrow ();
 511         }
 512 }
 513
 514
 515 void
 516 Writer::calculate_digests ()
 517 {
 518         auto job = _job.lock ();
 519         if (job) {
 520                 job->sub (_("Computing digests"));
 521         }
 522
 523         boost::asio::io_service service;
 524         boost::thread_group pool;
 525
 526         auto work = make_shared<boost::asio::io_service::work>(service);
 527
 528         int const threads = max (1, Config::instance()->master_encoding_threads());
 529
 530         for (int i = 0; i < threads; ++i) {
 531                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 532         }
 533
 534         std::function<void (float)> set_progress;
 535         if (job) {
 536                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 537         } else {
 538                 set_progress = [](float) {
 539                         boost::this_thread::interruption_point();
 540                 };
 541         }
 542
 543         for (auto& i: _reels) {
 544                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 545         }
 546         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 547
 548         work.reset ();
 549
 550         try {
 551                 pool.join_all ();
 552         } catch (boost::thread_interrupted) {
 553                 /* join_all was interrupted, so we need to interrupt the threads
 554                  * in our pool then try again to join them.
 555                  */
 556                 pool.interrupt_all ();
 557                 pool.join_all ();
 558         }
 559
 560         service.stop ();
 561 }
 562
 563
 564 /** @param output_dcp Path to DCP folder to write */
 565 void
 566 Writer::finish (boost::filesystem::path output_dcp)
 567 {
 568         if (_thread.joinable()) {
 569                 LOG_GENERAL_NC ("Terminating writer thread");
 570                 terminate_thread (true);
 571         }
 572
 573         LOG_GENERAL_NC ("Finishing ReelWriters");
 574
 575         for (auto& i: _reels) {
 576                 write_hanging_text (i);
 577                 i.finish (output_dcp);
 578         }
 579
 580         LOG_GENERAL_NC ("Writing XML");
 581
 582         dcp::DCP dcp (output_dcp);
 583
 584         auto cpl = make_shared<dcp::CPL>(
 585                 film()->dcp_name(),
 586                 film()->dcp_content_type()->libdcp_kind(),
 587                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 588                 );
 589
 590         dcp.add (cpl);
 591
 592         calculate_digests ();
 593
 594         /* Add reels */
 595
 596         for (auto& i: _reels) {
 597                 cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
 598         }
 599
 600         /* Add metadata */
 601
 602         auto creator = Config::instance()->dcp_creator();
 603         if (creator.empty()) {
 604                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 605         }
 606
 607         auto issuer = Config::instance()->dcp_issuer();
 608         if (issuer.empty()) {
 609                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 610         }
 611
 612         cpl->set_creator (creator);
 613         cpl->set_issuer (issuer);
 614
 615         cpl->set_ratings (film()->ratings());
 616
 617         vector<dcp::ContentVersion> cv;
 618         for (auto i: film()->content_versions()) {
 619                 cv.push_back (dcp::ContentVersion(i));
 620         }
 621         if (cv.empty()) {
 622                 cv = { dcp::ContentVersion("1") };
 623         }
 624         cpl->set_content_versions (cv);
 625
 626         cpl->set_full_content_title_text (film()->name());
 627         cpl->set_full_content_title_text_language (film()->name_language());
 628         if (film()->release_territory()) {
 629                 cpl->set_release_territory (*film()->release_territory());
 630         }
 631         cpl->set_version_number (film()->version_number());
 632         cpl->set_status (film()->status());
 633         if (film()->chain()) {
 634                 cpl->set_chain (*film()->chain());
 635         }
 636         if (film()->distributor()) {
 637                 cpl->set_distributor (*film()->distributor());
 638         }
 639         if (film()->facility()) {
 640                 cpl->set_facility (*film()->facility());
 641         }
 642         if (film()->luminance()) {
 643                 cpl->set_luminance (*film()->luminance());
 644         }
 645         if (film()->sign_language_video_language()) {
 646                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 647         }
 648
 649         dcp::MCASoundField field;
 650         if (channel_is_mapped(film(), dcp::Channel::BSL) || channel_is_mapped(film(), dcp::Channel::BSR)) {
 651                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 652         } else {
 653                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 654         }
 655
 656         dcp::MainSoundConfiguration msc(field, MAX_DCP_AUDIO_CHANNELS);
 657         for (auto i: film()->mapped_audio_channels()) {
 658                 msc.set_mapping(i, static_cast<dcp::Channel>(i));
 659         }
 660
 661         cpl->set_main_sound_configuration(msc);
 662         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 663         cpl->set_main_picture_stored_area (film()->frame_size());
 664
 665         auto active_area = film()->active_area();
 666         if (active_area.width > 0 && active_area.height > 0) {
 667                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 668                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 669         }
 670
 671         auto sl = film()->subtitle_languages().second;
 672         if (!sl.empty()) {
 673                 cpl->set_additional_subtitle_languages(sl);
 674         }
 675
 676         auto signer = Config::instance()->signer_chain();
 677         /* We did check earlier, but check again here to be on the safe side */
 678         string reason;
 679         if (!signer->valid (&reason)) {
 680                 throw InvalidSignerError (reason);
 681         }
 682
 683         dcp.set_issuer(issuer);
 684         dcp.set_creator(creator);
 685         dcp.set_annotation_text(film()->dcp_name());
 686
 687         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 688
 689         LOG_GENERAL (
 690                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 691                 );
 692
 693         write_cover_sheet (output_dcp);
 694 }
 695
 696
 697 void
 698 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 699 {
 700         auto const cover = film()->file("COVER_SHEET.txt");
 701         dcp::File f(cover, "w");
 702         if (!f) {
 703                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 704         }
 705
 706         auto text = Config::instance()->cover_sheet ();
 707         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 708         auto cpls = film()->cpls();
 709         if (!cpls.empty()) {
 710                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 711         }
 712         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 713         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 714
 715         auto audio_language = film()->audio_language();
 716         if (audio_language) {
 717                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 718         } else {
 719                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 720         }
 721
 722         auto subtitle_languages = film()->subtitle_languages();
 723         if (subtitle_languages.first) {
 724                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 725         } else {
 726                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 727         }
 728
 729         boost::uintmax_t size = 0;
 730         for (
 731                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 732                 i != boost::filesystem::recursive_directory_iterator();
 733                 ++i) {
 734                 if (boost::filesystem::is_regular_file (i->path())) {
 735                         size += boost::filesystem::file_size (i->path());
 736                 }
 737         }
 738
 739         if (size > (1000000000L)) {
 740                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 741         } else {
 742                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 743         }
 744
 745         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 746         auto description = String::compose("%1.%2", ch.first, ch.second);
 747
 748         if (description == "0.0") {
 749                 description = _("None");
 750         } else if (description == "1.0") {
 751                 description = _("Mono");
 752         } else if (description == "2.0") {
 753                 description = _("Stereo");
 754         }
 755         boost::algorithm::replace_all (text, "$AUDIO", description);
 756
 757         auto const hmsf = film()->length().split(film()->video_frame_rate());
 758         string length;
 759         if (hmsf.h == 0 && hmsf.m == 0) {
 760                 length = String::compose("%1s", hmsf.s);
 761         } else if (hmsf.h == 0 && hmsf.m > 0) {
 762                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 763         } else if (hmsf.h > 0 && hmsf.m > 0) {
 764                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 765         }
 766
 767         boost::algorithm::replace_all (text, "$LENGTH", length);
 768
 769         f.checked_write(text.c_str(), text.length());
 770 }
 771
 772
 773 /** @param frame Frame index within the whole DCP.
 774  *  @return true if we can fake-write this frame.
 775  */
 776 bool
 777 Writer::can_fake_write (Frame frame) const
 778 {
 779         if (film()->encrypted()) {
 780                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 781                 return false;
 782         }
 783
 784         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 785            parameters in the asset writer.
 786         */
 787
 788         auto const & reel = _reels[video_reel(frame)];
 789
 790         /* Make frame relative to the start of the reel */
 791         frame -= reel.start ();
 792         return (frame != 0 && frame < reel.first_nonexistent_frame());
 793 }
 794
 795
 796 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 797 void
 798 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 799 {
 800         vector<ReelWriter>::iterator* reel = nullptr;
 801
 802         switch (type) {
 803         case TextType::OPEN_SUBTITLE:
 804                 reel = &_subtitle_reel;
 805                 _have_subtitles = true;
 806                 break;
 807         case TextType::CLOSED_CAPTION:
 808                 DCPOMATIC_ASSERT (track);
 809                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 810                 reel = &_caption_reels[*track];
 811                 _have_closed_captions.insert (*track);
 812                 break;
 813         default:
 814                 DCPOMATIC_ASSERT (false);
 815         }
 816
 817         DCPOMATIC_ASSERT (*reel != _reels.end());
 818         while ((*reel)->period().to <= period.from) {
 819                 ++(*reel);
 820                 DCPOMATIC_ASSERT (*reel != _reels.end());
 821                 write_hanging_text (**reel);
 822         }
 823
 824         auto back_off = [this](DCPTimePeriod period) {
 825                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 826                 return period;
 827         };
 828
 829         if (period.to > (*reel)->period().to) {
 830                 /* This text goes off the end of the reel.  Store parts of it that should go into
 831                  * other reels.
 832                  */
 833                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 834                         auto overlap = i->period().overlap(period);
 835                         if (overlap) {
 836                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 837                         }
 838                 }
 839                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 840                  * for subtitles being too close together.
 841                  */
 842                 period.to = (*reel)->period().to;
 843                 period = back_off(period);
 844         }
 845
 846         (*reel)->write(text, type, track, period, _fonts, _chosen_interop_font);
 847 }
 848
 849
 850 void
 851 Writer::write (vector<shared_ptr<Font>> fonts)
 852 {
 853         if (fonts.empty()) {
 854                 return;
 855         }
 856
 857         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 858         auto fix_id = [](string id) {
 859                 return id.empty() ? "font" : id;
 860         };
 861
 862         if (film()->interop()) {
 863                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 864                  * even write them as they upset some validators.  Set up _fonts so that every
 865                  * font used by any subtitle will be written with the same ID.
 866                  */
 867                 for (size_t i = 0; i < fonts.size(); ++i) {
 868                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 869                 }
 870                 _chosen_interop_font = fonts[0];
 871         } else {
 872                 set<string> used_ids;
 873
 874                 /* Return the index of a _N at the end of a string, or string::npos */
 875                 auto underscore_number_position = [](string s) {
 876                         auto last_underscore = s.find_last_of("_");
 877                         if (last_underscore == string::npos) {
 878                                 return string::npos;
 879                         }
 880
 881                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 882                                 if (!isdigit(s[i])) {
 883                                         return string::npos;
 884                                 }
 885                         }
 886
 887                         return last_underscore;
 888                 };
 889
 890                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 891                 for (auto font: fonts) {
 892                         auto id = fix_id(font->id());
 893                         if (used_ids.find(id) == used_ids.end()) {
 894                                 /* This ID is unique so we can just use it as-is */
 895                                 _fonts.put(font, id);
 896                                 used_ids.insert(id);
 897                         } else {
 898                                 auto end = underscore_number_position(id);
 899                                 if (end == string::npos) {
 900                                         /* This string has no _N suffix, so add one */
 901                                         id += "_0";
 902                                         end = underscore_number_position(id);
 903                                 }
 904
 905                                 ++end;
 906
 907                                 /* Increment the suffix until we find a unique one */
 908                                 auto number = dcp::raw_convert<int>(id.substr(end));
 909                                 while (used_ids.find(id) != used_ids.end()) {
 910                                         ++number;
 911                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 912                                 }
 913                                 used_ids.insert(id);
 914                         }
 915                         _fonts.put(font, id);
 916                 }
 917
 918                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 919         }
 920 }
 921
 922
 923 bool
 924 operator< (QueueItem const & a, QueueItem const & b)
 925 {
 926         if (a.reel != b.reel) {
 927                 return a.reel < b.reel;
 928         }
 929
 930         if (a.frame != b.frame) {
 931                 return a.frame < b.frame;
 932         }
 933
 934         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 935 }
 936
 937
 938 bool
 939 operator== (QueueItem const & a, QueueItem const & b)
 940 {
 941         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 942 }
 943
 944
 945 void
 946 Writer::set_encoder_threads (int threads)
 947 {
 948         boost::mutex::scoped_lock lm (_state_mutex);
 949         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 950         _maximum_queue_size = threads * 16;
 951 }
 952
 953
 954 void
 955 Writer::write (ReferencedReelAsset asset)
 956 {
 957         _reel_assets.push_back (asset);
 958 }
 959
 960
 961 size_t
 962 Writer::video_reel (int frame) const
 963 {
 964         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 965         size_t i = 0;
 966         while (i < _reels.size() && !_reels[i].period().contains (t)) {
 967                 ++i;
 968         }
 969
 970         DCPOMATIC_ASSERT (i < _reels.size ());
 971         return i;
 972 }
 973
 974
 975 void
 976 Writer::set_digest_progress (Job* job, float progress)
 977 {
 978         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 979
 980         _digest_progresses[boost::this_thread::get_id()] = progress;
 981         float min_progress = FLT_MAX;
 982         for (auto const& i: _digest_progresses) {
 983                 min_progress = min (min_progress, i.second);
 984         }
 985
 986         job->set_progress (min_progress);
 987
 988         Waker waker;
 989         waker.nudge ();
 990
 991         boost::this_thread::interruption_point();
 992 }
 993
 994
 995 /** Calculate hashes for any referenced MXF assets which do not already have one */
 996 void
 997 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
 998 try
 999 {
1000         for (auto const& i: _reel_assets) {
1001                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1002                 if (file && !file->hash()) {
1003                         file->asset_ref().asset()->hash (set_progress);
1004                         file->set_hash (file->asset_ref().asset()->hash());
1005                 }
1006         }
1007 } catch (boost::thread_interrupted) {
1008         /* set_progress contains an interruption_point, so any of these methods
1009          * may throw thread_interrupted, at which point we just give up.
1010          */
1011 }
1012
1013
1014 void
1015 Writer::write_hanging_text (ReelWriter& reel)
1016 {
1017         vector<HangingText> new_hanging_texts;
1018         for (auto i: _hanging_texts) {
1019                 if (i.period.from == reel.period().from) {
1020                         reel.write(i.text, i.type, i.track, i.period, _fonts, _chosen_interop_font);
1021                 } else {
1022                         new_hanging_texts.push_back (i);
1023                 }
1024         }
1025         _hanging_texts = new_hanging_texts;
1026 }