src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "cross.h"
  27 #include "dcp_content_type.h"
  28 #include "dcp_video.h"
  29 #include "dcpomatic_log.h"
  30 #include "film.h"
  31 #include "job.h"
  32 #include "log.h"
  33 #include "ratio.h"
  34 #include "reel_writer.h"
  35 #include "text_content.h"
  36 #include "util.h"
  37 #include "version.h"
  38 #include "writer.h"
  39 #include <dcp/cpl.h>
  40 #include <dcp/locale_convert.h>
  41 #include <dcp/raw_convert.h>
  42 #include <dcp/reel_file_asset.h>
  43 #include <cerrno>
  44 #include <cfloat>
  45 #include <set>
  46
  47 #include "i18n.h"
  48
  49
  50 /* OS X strikes again */
  51 #undef set_key
  52
  53
  54 using std::cout;
  55 using std::dynamic_pointer_cast;
  56 using std::make_shared;
  57 using std::max;
  58 using std::min;
  59 using std::shared_ptr;
  60 using std::set;
  61 using std::string;
  62 using std::vector;
  63 using std::weak_ptr;
  64 using boost::optional;
  65 #if BOOST_VERSION >= 106100
  66 using namespace boost::placeholders;
  67 #endif
  68 using dcp::Data;
  69 using dcp::ArrayData;
  70 using namespace dcpomatic;
  71
  72
  73 /** @param j Job to report progress to, or 0.
  74  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  75  */
  76 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
  77         : WeakConstFilm (weak_film)
  78         , _job (j)
  79         /* These will be reset to sensible values when J2KEncoder is created */
  80         , _maximum_frames_in_memory (8)
  81         , _maximum_queue_size (8)
  82         , _text_only (text_only)
  83 {
  84         auto job = _job.lock ();
  85
  86         int reel_index = 0;
  87         auto const reels = film()->reels();
  88         for (auto p: reels) {
  89                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  90         }
  91
  92         _last_written.resize (reels.size());
  93
  94         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  95            and captions arrive to the Writer in sequence.  This is not so for video.
  96         */
  97         _audio_reel = _reels.begin ();
  98         _subtitle_reel = _reels.begin ();
  99         for (auto i: film()->closed_caption_tracks()) {
 100                 _caption_reels[i] = _reels.begin ();
 101         }
 102         _atmos_reel = _reels.begin ();
 103
 104         /* Check that the signer is OK */
 105         string reason;
 106         if (!Config::instance()->signer_chain()->valid(&reason)) {
 107                 throw InvalidSignerError (reason);
 108         }
 109 }
 110
 111
 112 void
 113 Writer::start ()
 114 {
 115         if (!_text_only) {
 116                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 117 #ifdef DCPOMATIC_LINUX
 118                 pthread_setname_np (_thread.native_handle(), "writer");
 119 #endif
 120         }
 121 }
 122
 123
 124 Writer::~Writer ()
 125 {
 126         if (!_text_only) {
 127                 terminate_thread (false);
 128         }
 129 }
 130
 131
 132 /** Pass a video frame to the writer for writing to disk at some point.
 133  *  This method can be called with frames out of order.
 134  *  @param encoded JPEG2000-encoded data.
 135  *  @param frame Frame index within the DCP.
 136  *  @param eyes Eyes that this frame image is for.
 137  */
 138 void
 139 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 140 {
 141         boost::mutex::scoped_lock lock (_state_mutex);
 142
 143         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 144                 /* There are too many full frames in memory; wake the main writer thread and
 145                    wait until it sorts everything out */
 146                 _empty_condition.notify_all ();
 147                 _full_condition.wait (lock);
 148         }
 149
 150         QueueItem qi;
 151         qi.type = QueueItem::Type::FULL;
 152         qi.encoded = encoded;
 153         qi.reel = video_reel (frame);
 154         qi.frame = frame - _reels[qi.reel].start ();
 155
 156         if (film()->three_d() && eyes == Eyes::BOTH) {
 157                 /* 2D material in a 3D DCP; fake the 3D */
 158                 qi.eyes = Eyes::LEFT;
 159                 _queue.push_back (qi);
 160                 ++_queued_full_in_memory;
 161                 qi.eyes = Eyes::RIGHT;
 162                 _queue.push_back (qi);
 163                 ++_queued_full_in_memory;
 164         } else {
 165                 qi.eyes = eyes;
 166                 _queue.push_back (qi);
 167                 ++_queued_full_in_memory;
 168         }
 169
 170         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 171         _empty_condition.notify_all ();
 172 }
 173
 174
 175 bool
 176 Writer::can_repeat (Frame frame) const
 177 {
 178         return frame > _reels[video_reel(frame)].start();
 179 }
 180
 181
 182 /** Repeat the last frame that was written to a reel as a new frame.
 183  *  @param frame Frame index within the DCP of the new (repeated) frame.
 184  *  @param eyes Eyes that this repeated frame image is for.
 185  */
 186 void
 187 Writer::repeat (Frame frame, Eyes eyes)
 188 {
 189         boost::mutex::scoped_lock lock (_state_mutex);
 190
 191         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 192                 /* The queue is too big, and the main writer thread can run and fix it, so
 193                    wake it and wait until it has done.
 194                 */
 195                 _empty_condition.notify_all ();
 196                 _full_condition.wait (lock);
 197         }
 198
 199         QueueItem qi;
 200         qi.type = QueueItem::Type::REPEAT;
 201         qi.reel = video_reel (frame);
 202         qi.frame = frame - _reels[qi.reel].start ();
 203         if (film()->three_d() && eyes == Eyes::BOTH) {
 204                 qi.eyes = Eyes::LEFT;
 205                 _queue.push_back (qi);
 206                 qi.eyes = Eyes::RIGHT;
 207                 _queue.push_back (qi);
 208         } else {
 209                 qi.eyes = eyes;
 210                 _queue.push_back (qi);
 211         }
 212
 213         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 214         _empty_condition.notify_all ();
 215 }
 216
 217
 218 void
 219 Writer::fake_write (Frame frame, Eyes eyes)
 220 {
 221         boost::mutex::scoped_lock lock (_state_mutex);
 222
 223         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 224                 /* The queue is too big, and the main writer thread can run and fix it, so
 225                    wake it and wait until it has done.
 226                 */
 227                 _empty_condition.notify_all ();
 228                 _full_condition.wait (lock);
 229         }
 230
 231         size_t const reel = video_reel (frame);
 232         Frame const frame_in_reel = frame - _reels[reel].start ();
 233
 234         QueueItem qi;
 235         qi.type = QueueItem::Type::FAKE;
 236
 237         {
 238                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 239                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 240         }
 241
 242         qi.reel = reel;
 243         qi.frame = frame_in_reel;
 244         if (film()->three_d() && eyes == Eyes::BOTH) {
 245                 qi.eyes = Eyes::LEFT;
 246                 _queue.push_back (qi);
 247                 qi.eyes = Eyes::RIGHT;
 248                 _queue.push_back (qi);
 249         } else {
 250                 qi.eyes = eyes;
 251                 _queue.push_back (qi);
 252         }
 253
 254         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 255         _empty_condition.notify_all ();
 256 }
 257
 258
 259 /** Write some audio frames to the DCP.
 260  *  @param audio Audio data.
 261  *  @param time Time of this data within the DCP.
 262  *  This method is not thread safe.
 263  */
 264 void
 265 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 266 {
 267         DCPOMATIC_ASSERT (audio);
 268
 269         int const afr = film()->audio_frame_rate();
 270
 271         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 272
 273         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 274
 275         DCPTime t = time;
 276         while (t < end) {
 277
 278                 if (_audio_reel == _reels.end ()) {
 279                         /* This audio is off the end of the last reel; ignore it */
 280                         return;
 281                 }
 282
 283                 if (end <= _audio_reel->period().to) {
 284                         /* Easy case: we can write all the audio to this reel */
 285                         _audio_reel->write (audio);
 286                         t = end;
 287                 } else if (_audio_reel->period().to <= t) {
 288                         /* This reel is entirely before the start of our audio; just skip the reel */
 289                         ++_audio_reel;
 290                 } else {
 291                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 292                         DCPTime part_lengths[2] = {
 293                                 _audio_reel->period().to - t,
 294                                 end - _audio_reel->period().to
 295                         };
 296
 297                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 298                         Frame part_frames[2] = {
 299                                 part_lengths[0].frames_ceil(afr),
 300                                 part_lengths[1].frames_floor(afr)
 301                         };
 302
 303                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 304
 305                         if (part_frames[0]) {
 306                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 307                                 _audio_reel->write (part);
 308                         }
 309
 310                         if (part_frames[1]) {
 311                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 312                         } else {
 313                                 audio.reset ();
 314                         }
 315
 316                         ++_audio_reel;
 317                         t += part_lengths[0];
 318                 }
 319         }
 320 }
 321
 322
 323 void
 324 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 325 {
 326         if (_atmos_reel->period().to == time) {
 327                 ++_atmos_reel;
 328                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 329         }
 330
 331         /* We assume that we get a video frame's worth of data here */
 332         _atmos_reel->write (atmos, metadata);
 333 }
 334
 335
 336 /** Caller must hold a lock on _state_mutex */
 337 bool
 338 Writer::have_sequenced_image_at_queue_head ()
 339 {
 340         if (_queue.empty ()) {
 341                 return false;
 342         }
 343
 344         _queue.sort ();
 345         auto const & f = _queue.front();
 346         return _last_written[f.reel].next(f);
 347 }
 348
 349
 350 bool
 351 Writer::LastWritten::next (QueueItem qi) const
 352 {
 353         if (qi.eyes == Eyes::BOTH) {
 354                 /* 2D */
 355                 return qi.frame == (_frame + 1);
 356         }
 357
 358         /* 3D */
 359
 360         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 361                 return true;
 362         }
 363
 364         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 365                 return true;
 366         }
 367
 368         return false;
 369 }
 370
 371
 372 void
 373 Writer::LastWritten::update (QueueItem qi)
 374 {
 375         _frame = qi.frame;
 376         _eyes = qi.eyes;
 377 }
 378
 379
 380 void
 381 Writer::thread ()
 382 try
 383 {
 384         start_of_thread ("Writer");
 385
 386         while (true)
 387         {
 388                 boost::mutex::scoped_lock lock (_state_mutex);
 389
 390                 while (true) {
 391
 392                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 393                                 /* We've got something to do: go and do it */
 394                                 break;
 395                         }
 396
 397                         /* Nothing to do: wait until something happens which may indicate that we do */
 398                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 399                         _empty_condition.wait (lock);
 400                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 401                 }
 402
 403                 /* We stop here if we have been asked to finish, and if either the queue
 404                    is empty or we do not have a sequenced image at its head (if this is the
 405                    case we will never terminate as no new frames will be sent once
 406                    _finish is true).
 407                 */
 408                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 409                         /* (Hopefully temporarily) log anything that was not written */
 410                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 411                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 412                                 for (auto const& i: _queue) {
 413                                         if (i.type == QueueItem::Type::FULL) {
 414                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 415                                         } else {
 416                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 417                                         }
 418                                 }
 419                         }
 420                         return;
 421                 }
 422
 423                 /* Write any frames that we can write; i.e. those that are in sequence. */
 424                 while (have_sequenced_image_at_queue_head ()) {
 425                         auto qi = _queue.front ();
 426                         _last_written[qi.reel].update (qi);
 427                         _queue.pop_front ();
 428                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 429                                 --_queued_full_in_memory;
 430                         }
 431
 432                         lock.unlock ();
 433
 434                         auto& reel = _reels[qi.reel];
 435
 436                         switch (qi.type) {
 437                         case QueueItem::Type::FULL:
 438                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 439                                 if (!qi.encoded) {
 440                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 441                                 }
 442                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 443                                 ++_full_written;
 444                                 break;
 445                         case QueueItem::Type::FAKE:
 446                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 447                                 reel.fake_write (qi.size);
 448                                 ++_fake_written;
 449                                 break;
 450                         case QueueItem::Type::REPEAT:
 451                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 452                                 reel.repeat_write (qi.frame, qi.eyes);
 453                                 ++_repeat_written;
 454                                 break;
 455                         }
 456
 457                         lock.lock ();
 458                         _full_condition.notify_all ();
 459                 }
 460
 461                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 462                         /* Too many frames in memory which can't yet be written to the stream.
 463                            Write some FULL frames to disk.
 464                         */
 465
 466                         /* Find one from the back of the queue */
 467                         _queue.sort ();
 468                         auto i = _queue.rbegin ();
 469                         while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
 470                                 ++i;
 471                         }
 472
 473                         DCPOMATIC_ASSERT (i != _queue.rend());
 474                         ++_pushed_to_disk;
 475                         /* For the log message below */
 476                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 477                         lock.unlock ();
 478
 479                         /* i is valid here, even though we don't hold a lock on the mutex,
 480                            since list iterators are unaffected by insertion and only this
 481                            thread could erase the last item in the list.
 482                         */
 483
 484                         LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
 485
 486                         i->encoded->write_via_temp (
 487                                 film()->j2c_path(i->reel, i->frame, i->eyes, true),
 488                                 film()->j2c_path(i->reel, i->frame, i->eyes, false)
 489                                 );
 490
 491                         lock.lock ();
 492                         i->encoded.reset ();
 493                         --_queued_full_in_memory;
 494                         _full_condition.notify_all ();
 495                 }
 496         }
 497 }
 498 catch (...)
 499 {
 500         store_current ();
 501 }
 502
 503
 504 void
 505 Writer::terminate_thread (bool can_throw)
 506 {
 507         boost::this_thread::disable_interruption dis;
 508
 509         boost::mutex::scoped_lock lock (_state_mutex);
 510
 511         _finish = true;
 512         _empty_condition.notify_all ();
 513         _full_condition.notify_all ();
 514         lock.unlock ();
 515
 516         try {
 517                 _thread.join ();
 518         } catch (...) {}
 519
 520         if (can_throw) {
 521                 rethrow ();
 522         }
 523 }
 524
 525
 526 void
 527 Writer::calculate_digests ()
 528 {
 529         auto job = _job.lock ();
 530         if (job) {
 531                 job->sub (_("Computing digests"));
 532         }
 533
 534         boost::asio::io_service service;
 535         boost::thread_group pool;
 536
 537         auto work = make_shared<boost::asio::io_service::work>(service);
 538
 539         int const threads = max (1, Config::instance()->master_encoding_threads());
 540
 541         for (int i = 0; i < threads; ++i) {
 542                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 543         }
 544
 545         std::function<void (float)> set_progress;
 546         if (job) {
 547                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 548         } else {
 549                 set_progress = [](float) {
 550                         boost::this_thread::interruption_point();
 551                 };
 552         }
 553
 554         for (auto& i: _reels) {
 555                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 556         }
 557         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 558
 559         work.reset ();
 560
 561         try {
 562                 pool.join_all ();
 563         } catch (boost::thread_interrupted) {
 564                 /* join_all was interrupted, so we need to interrupt the threads
 565                  * in our pool then try again to join them.
 566                  */
 567                 pool.interrupt_all ();
 568                 pool.join_all ();
 569         }
 570
 571         service.stop ();
 572 }
 573
 574
 575 /** @param output_dcp Path to DCP folder to write */
 576 void
 577 Writer::finish (boost::filesystem::path output_dcp)
 578 {
 579         if (_thread.joinable()) {
 580                 LOG_GENERAL_NC ("Terminating writer thread");
 581                 terminate_thread (true);
 582         }
 583
 584         LOG_GENERAL_NC ("Finishing ReelWriters");
 585
 586         for (auto& i: _reels) {
 587                 write_hanging_text (i);
 588                 i.finish (output_dcp);
 589         }
 590
 591         LOG_GENERAL_NC ("Writing XML");
 592
 593         dcp::DCP dcp (output_dcp);
 594
 595         auto cpl = make_shared<dcp::CPL>(
 596                 film()->dcp_name(),
 597                 film()->dcp_content_type()->libdcp_kind(),
 598                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 599                 );
 600
 601         dcp.add (cpl);
 602
 603         calculate_digests ();
 604
 605         /* Add reels */
 606
 607         for (auto& i: _reels) {
 608                 cpl->add (i.create_reel(_reel_assets, _fonts, _chosen_interop_font, output_dcp, _have_subtitles, _have_closed_captions));
 609         }
 610
 611         /* Add metadata */
 612
 613         auto creator = Config::instance()->dcp_creator();
 614         if (creator.empty()) {
 615                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 616         }
 617
 618         auto issuer = Config::instance()->dcp_issuer();
 619         if (issuer.empty()) {
 620                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 621         }
 622
 623         cpl->set_creator (creator);
 624         cpl->set_issuer (issuer);
 625
 626         cpl->set_ratings (film()->ratings());
 627
 628         vector<dcp::ContentVersion> cv;
 629         for (auto i: film()->content_versions()) {
 630                 cv.push_back (dcp::ContentVersion(i));
 631         }
 632         if (cv.empty()) {
 633                 cv = { dcp::ContentVersion("1") };
 634         }
 635         cpl->set_content_versions (cv);
 636
 637         cpl->set_full_content_title_text (film()->name());
 638         cpl->set_full_content_title_text_language (film()->name_language());
 639         if (film()->release_territory()) {
 640                 cpl->set_release_territory (*film()->release_territory());
 641         }
 642         cpl->set_version_number (film()->version_number());
 643         cpl->set_status (film()->status());
 644         if (film()->chain()) {
 645                 cpl->set_chain (*film()->chain());
 646         }
 647         if (film()->distributor()) {
 648                 cpl->set_distributor (*film()->distributor());
 649         }
 650         if (film()->facility()) {
 651                 cpl->set_facility (*film()->facility());
 652         }
 653         if (film()->luminance()) {
 654                 cpl->set_luminance (*film()->luminance());
 655         }
 656         if (film()->sign_language_video_language()) {
 657                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 658         }
 659
 660         auto ac = film()->mapped_audio_channels();
 661         dcp::MCASoundField field = (
 662                 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSL)) != ac.end() ||
 663                 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSR)) != ac.end()
 664                 ) ? dcp::MCASoundField::SEVEN_POINT_ONE : dcp::MCASoundField::FIVE_POINT_ONE;
 665
 666         dcp::MainSoundConfiguration msc (field, film()->audio_channels());
 667         for (auto i: ac) {
 668                 if (static_cast<int>(i) < film()->audio_channels()) {
 669                         msc.set_mapping (i, static_cast<dcp::Channel>(i));
 670                 }
 671         }
 672
 673         cpl->set_main_sound_configuration (msc.to_string());
 674         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 675         cpl->set_main_picture_stored_area (film()->frame_size());
 676
 677         auto active_area = film()->active_area();
 678         if (active_area.width > 0 && active_area.height > 0) {
 679                 /* It's not allowed to have a zero active area width or height */
 680                 cpl->set_main_picture_active_area (active_area);
 681         }
 682
 683         auto sl = film()->subtitle_languages().second;
 684         if (!sl.empty()) {
 685                 cpl->set_additional_subtitle_languages(sl);
 686         }
 687
 688         auto signer = Config::instance()->signer_chain();
 689         /* We did check earlier, but check again here to be on the safe side */
 690         string reason;
 691         if (!signer->valid (&reason)) {
 692                 throw InvalidSignerError (reason);
 693         }
 694
 695         dcp.set_issuer(issuer);
 696         dcp.set_creator(creator);
 697         dcp.set_annotation_text(film()->dcp_name());
 698
 699         dcp.write_xml (signer, Config::instance()->dcp_metadata_filename_format());
 700
 701         LOG_GENERAL (
 702                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 703                 );
 704
 705         write_cover_sheet (output_dcp);
 706 }
 707
 708
 709 void
 710 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 711 {
 712         auto const cover = film()->file("COVER_SHEET.txt");
 713         dcp::File f(cover, "w");
 714         if (!f) {
 715                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 716         }
 717
 718         auto text = Config::instance()->cover_sheet ();
 719         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 720         auto cpls = film()->cpls();
 721         if (!cpls.empty()) {
 722                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 723         }
 724         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 725         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 726
 727         auto audio_language = film()->audio_language();
 728         if (audio_language) {
 729                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 730         } else {
 731                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 732         }
 733
 734         auto subtitle_languages = film()->subtitle_languages();
 735         if (subtitle_languages.first) {
 736                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 737         } else {
 738                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 739         }
 740
 741         boost::uintmax_t size = 0;
 742         for (
 743                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 744                 i != boost::filesystem::recursive_directory_iterator();
 745                 ++i) {
 746                 if (boost::filesystem::is_regular_file (i->path())) {
 747                         size += boost::filesystem::file_size (i->path());
 748                 }
 749         }
 750
 751         if (size > (1000000000L)) {
 752                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 753         } else {
 754                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 755         }
 756
 757         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 758         auto description = String::compose("%1.%2", ch.first, ch.second);
 759
 760         if (description == "0.0") {
 761                 description = _("None");
 762         } else if (description == "1.0") {
 763                 description = _("Mono");
 764         } else if (description == "2.0") {
 765                 description = _("Stereo");
 766         }
 767         boost::algorithm::replace_all (text, "$AUDIO", description);
 768
 769         auto const hmsf = film()->length().split(film()->video_frame_rate());
 770         string length;
 771         if (hmsf.h == 0 && hmsf.m == 0) {
 772                 length = String::compose("%1s", hmsf.s);
 773         } else if (hmsf.h == 0 && hmsf.m > 0) {
 774                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 775         } else if (hmsf.h > 0 && hmsf.m > 0) {
 776                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 777         }
 778
 779         boost::algorithm::replace_all (text, "$LENGTH", length);
 780
 781         f.checked_write(text.c_str(), text.length());
 782 }
 783
 784
 785 /** @param frame Frame index within the whole DCP.
 786  *  @return true if we can fake-write this frame.
 787  */
 788 bool
 789 Writer::can_fake_write (Frame frame) const
 790 {
 791         if (film()->encrypted()) {
 792                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 793                 return false;
 794         }
 795
 796         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 797            parameters in the asset writer.
 798         */
 799
 800         auto const & reel = _reels[video_reel(frame)];
 801
 802         /* Make frame relative to the start of the reel */
 803         frame -= reel.start ();
 804         return (frame != 0 && frame < reel.first_nonexistant_frame());
 805 }
 806
 807
 808 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 809 void
 810 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 811 {
 812         vector<ReelWriter>::iterator* reel = nullptr;
 813
 814         switch (type) {
 815         case TextType::OPEN_SUBTITLE:
 816                 reel = &_subtitle_reel;
 817                 _have_subtitles = true;
 818                 break;
 819         case TextType::CLOSED_CAPTION:
 820                 DCPOMATIC_ASSERT (track);
 821                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 822                 reel = &_caption_reels[*track];
 823                 _have_closed_captions.insert (*track);
 824                 break;
 825         default:
 826                 DCPOMATIC_ASSERT (false);
 827         }
 828
 829         DCPOMATIC_ASSERT (*reel != _reels.end());
 830         while ((*reel)->period().to <= period.from) {
 831                 ++(*reel);
 832                 DCPOMATIC_ASSERT (*reel != _reels.end());
 833                 write_hanging_text (**reel);
 834         }
 835
 836         auto back_off = [this](DCPTimePeriod period) {
 837                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 838                 return period;
 839         };
 840
 841         if (period.to > (*reel)->period().to) {
 842                 /* This text goes off the end of the reel.  Store parts of it that should go into
 843                  * other reels.
 844                  */
 845                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 846                         auto overlap = i->period().overlap(period);
 847                         if (overlap) {
 848                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 849                         }
 850                 }
 851                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 852                  * for subtitles being too close together.
 853                  */
 854                 period.to = (*reel)->period().to;
 855                 period = back_off(period);
 856         }
 857
 858         (*reel)->write(text, type, track, period, _fonts);
 859 }
 860
 861
 862 void
 863 Writer::write (vector<shared_ptr<Font>> fonts)
 864 {
 865         if (fonts.empty()) {
 866                 return;
 867         }
 868
 869         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 870         auto fix_id = [](string id) {
 871                 return id.empty() ? "font" : id;
 872         };
 873
 874         if (film()->interop()) {
 875                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 876                  * even write them as they upset some validators.  Set up _fonts so that every
 877                  * font used by any subtitle will be written with the same ID.
 878                  */
 879                 for (size_t i = 0; i < fonts.size(); ++i) {
 880                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 881                 }
 882                 _chosen_interop_font = fonts[0];
 883         } else {
 884                 set<string> used_ids;
 885
 886                 /* Return the index of a _N at the end of a string, or string::npos */
 887                 auto underscore_number_position = [](string s) {
 888                         auto last_underscore = s.find_last_of("_");
 889                         if (last_underscore == string::npos) {
 890                                 return string::npos;
 891                         }
 892
 893                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 894                                 if (!isdigit(s[i])) {
 895                                         return string::npos;
 896                                 }
 897                         }
 898
 899                         return last_underscore;
 900                 };
 901
 902                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 903                 for (auto font: fonts) {
 904                         auto id = fix_id(font->id());
 905                         if (used_ids.find(id) == used_ids.end()) {
 906                                 /* This ID is unique so we can just use it as-is */
 907                                 _fonts.put(font, id);
 908                                 used_ids.insert(id);
 909                         } else {
 910                                 auto end = underscore_number_position(id);
 911                                 if (end == string::npos) {
 912                                         /* This string has no _N suffix, so add one */
 913                                         id += "_0";
 914                                         end = underscore_number_position(id);
 915                                 }
 916
 917                                 ++end;
 918
 919                                 /* Increment the suffix until we find a unique one */
 920                                 auto number = dcp::raw_convert<int>(id.substr(end));
 921                                 while (used_ids.find(id) != used_ids.end()) {
 922                                         ++number;
 923                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 924                                 }
 925                                 used_ids.insert(id);
 926                         }
 927                         _fonts.put(font, id);
 928                 }
 929
 930                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 931         }
 932 }
 933
 934
 935 bool
 936 operator< (QueueItem const & a, QueueItem const & b)
 937 {
 938         if (a.reel != b.reel) {
 939                 return a.reel < b.reel;
 940         }
 941
 942         if (a.frame != b.frame) {
 943                 return a.frame < b.frame;
 944         }
 945
 946         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 947 }
 948
 949
 950 bool
 951 operator== (QueueItem const & a, QueueItem const & b)
 952 {
 953         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 954 }
 955
 956
 957 void
 958 Writer::set_encoder_threads (int threads)
 959 {
 960         boost::mutex::scoped_lock lm (_state_mutex);
 961         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 962         _maximum_queue_size = threads * 16;
 963 }
 964
 965
 966 void
 967 Writer::write (ReferencedReelAsset asset)
 968 {
 969         _reel_assets.push_back (asset);
 970 }
 971
 972
 973 size_t
 974 Writer::video_reel (int frame) const
 975 {
 976         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 977         size_t i = 0;
 978         while (i < _reels.size() && !_reels[i].period().contains (t)) {
 979                 ++i;
 980         }
 981
 982         DCPOMATIC_ASSERT (i < _reels.size ());
 983         return i;
 984 }
 985
 986
 987 void
 988 Writer::set_digest_progress (Job* job, float progress)
 989 {
 990         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 991
 992         _digest_progresses[boost::this_thread::get_id()] = progress;
 993         float min_progress = FLT_MAX;
 994         for (auto const& i: _digest_progresses) {
 995                 min_progress = min (min_progress, i.second);
 996         }
 997
 998         job->set_progress (min_progress);
 999
1000         Waker waker;
1001         waker.nudge ();
1002
1003         boost::this_thread::interruption_point();
1004 }
1005
1006
1007 /** Calculate hashes for any referenced MXF assets which do not already have one */
1008 void
1009 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
1010 try
1011 {
1012         for (auto const& i: _reel_assets) {
1013                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1014                 if (file && !file->hash()) {
1015                         file->asset_ref().asset()->hash (set_progress);
1016                         file->set_hash (file->asset_ref().asset()->hash());
1017                 }
1018         }
1019 } catch (boost::thread_interrupted) {
1020         /* set_progress contains an interruption_point, so any of these methods
1021          * may throw thread_interrupted, at which point we just give up.
1022          */
1023 }
1024
1025
1026 void
1027 Writer::write_hanging_text (ReelWriter& reel)
1028 {
1029         vector<HangingText> new_hanging_texts;
1030         for (auto i: _hanging_texts) {
1031                 if (i.period.from == reel.period().from) {
1032                         reel.write (i.text, i.type, i.track, i.period, _fonts);
1033                 } else {
1034                         new_hanging_texts.push_back (i);
1035                 }
1036         }
1037         _hanging_texts = new_hanging_texts;
1038 }