src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "constants.h"
  27 #include "cross.h"
  28 #include "dcp_content_type.h"
  29 #include "dcp_video.h"
  30 #include "dcpomatic_log.h"
  31 #include "film.h"
  32 #include "job.h"
  33 #include "log.h"
  34 #include "ratio.h"
  35 #include "reel_writer.h"
  36 #include "text_content.h"
  37 #include "util.h"
  38 #include "version.h"
  39 #include "writer.h"
  40 #include <dcp/cpl.h>
  41 #include <dcp/locale_convert.h>
  42 #include <dcp/raw_convert.h>
  43 #include <dcp/reel_file_asset.h>
  44 #include <cerrno>
  45 #include <cfloat>
  46 #include <set>
  47
  48 #include "i18n.h"
  49
  50
  51 /* OS X strikes again */
  52 #undef set_key
  53
  54
  55 using std::cout;
  56 using std::dynamic_pointer_cast;
  57 using std::make_shared;
  58 using std::max;
  59 using std::min;
  60 using std::shared_ptr;
  61 using std::set;
  62 using std::string;
  63 using std::vector;
  64 using std::weak_ptr;
  65 using boost::optional;
  66 #if BOOST_VERSION >= 106100
  67 using namespace boost::placeholders;
  68 #endif
  69 using dcp::Data;
  70 using dcp::ArrayData;
  71 using namespace dcpomatic;
  72
  73
  74 /** @param j Job to report progress to, or 0.
  75  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  76  */
  77 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
  78         : WeakConstFilm (weak_film)
  79         , _job (j)
  80         /* These will be reset to sensible values when J2KEncoder is created */
  81         , _maximum_frames_in_memory (8)
  82         , _maximum_queue_size (8)
  83         , _text_only (text_only)
  84 {
  85         auto job = _job.lock ();
  86
  87         int reel_index = 0;
  88         auto const reels = film()->reels();
  89         for (auto p: reels) {
  90                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  91         }
  92
  93         _last_written.resize (reels.size());
  94
  95         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  96            and captions arrive to the Writer in sequence.  This is not so for video.
  97         */
  98         _audio_reel = _reels.begin ();
  99         _subtitle_reel = _reels.begin ();
 100         for (auto i: film()->closed_caption_tracks()) {
 101                 _caption_reels[i] = _reels.begin ();
 102         }
 103         _atmos_reel = _reels.begin ();
 104
 105         /* Check that the signer is OK */
 106         string reason;
 107         if (!Config::instance()->signer_chain()->valid(&reason)) {
 108                 throw InvalidSignerError (reason);
 109         }
 110 }
 111
 112
 113 void
 114 Writer::start ()
 115 {
 116         if (!_text_only) {
 117                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 118 #ifdef DCPOMATIC_LINUX
 119                 pthread_setname_np (_thread.native_handle(), "writer");
 120 #endif
 121         }
 122 }
 123
 124
 125 Writer::~Writer ()
 126 {
 127         if (!_text_only) {
 128                 terminate_thread (false);
 129         }
 130 }
 131
 132
 133 /** Pass a video frame to the writer for writing to disk at some point.
 134  *  This method can be called with frames out of order.
 135  *  @param encoded JPEG2000-encoded data.
 136  *  @param frame Frame index within the DCP.
 137  *  @param eyes Eyes that this frame image is for.
 138  */
 139 void
 140 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 141 {
 142         boost::mutex::scoped_lock lock (_state_mutex);
 143
 144         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 145                 /* There are too many full frames in memory; wake the main writer thread and
 146                    wait until it sorts everything out */
 147                 _empty_condition.notify_all ();
 148                 _full_condition.wait (lock);
 149         }
 150
 151         QueueItem qi;
 152         qi.type = QueueItem::Type::FULL;
 153         qi.encoded = encoded;
 154         qi.reel = video_reel (frame);
 155         qi.frame = frame - _reels[qi.reel].start ();
 156
 157         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 158
 159         qi.eyes = eyes;
 160         _queue.push_back(qi);
 161         ++_queued_full_in_memory;
 162
 163         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 164         _empty_condition.notify_all ();
 165 }
 166
 167
 168 bool
 169 Writer::can_repeat (Frame frame) const
 170 {
 171         return frame > _reels[video_reel(frame)].start();
 172 }
 173
 174
 175 /** Repeat the last frame that was written to a reel as a new frame.
 176  *  @param frame Frame index within the DCP of the new (repeated) frame.
 177  *  @param eyes Eyes that this repeated frame image is for.
 178  */
 179 void
 180 Writer::repeat (Frame frame, Eyes eyes)
 181 {
 182         boost::mutex::scoped_lock lock (_state_mutex);
 183
 184         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 185                 /* The queue is too big, and the main writer thread can run and fix it, so
 186                    wake it and wait until it has done.
 187                 */
 188                 _empty_condition.notify_all ();
 189                 _full_condition.wait (lock);
 190         }
 191
 192         QueueItem qi;
 193         qi.type = QueueItem::Type::REPEAT;
 194         qi.reel = video_reel (frame);
 195         qi.frame = frame - _reels[qi.reel].start ();
 196         if (film()->three_d() && eyes == Eyes::BOTH) {
 197                 qi.eyes = Eyes::LEFT;
 198                 _queue.push_back (qi);
 199                 qi.eyes = Eyes::RIGHT;
 200                 _queue.push_back (qi);
 201         } else {
 202                 qi.eyes = eyes;
 203                 _queue.push_back (qi);
 204         }
 205
 206         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 207         _empty_condition.notify_all ();
 208 }
 209
 210
 211 void
 212 Writer::fake_write (Frame frame, Eyes eyes)
 213 {
 214         boost::mutex::scoped_lock lock (_state_mutex);
 215
 216         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 217                 /* The queue is too big, and the main writer thread can run and fix it, so
 218                    wake it and wait until it has done.
 219                 */
 220                 _empty_condition.notify_all ();
 221                 _full_condition.wait (lock);
 222         }
 223
 224         size_t const reel = video_reel (frame);
 225         Frame const frame_in_reel = frame - _reels[reel].start ();
 226
 227         QueueItem qi;
 228         qi.type = QueueItem::Type::FAKE;
 229
 230         {
 231                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 232                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 233         }
 234
 235         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 236
 237         qi.reel = reel;
 238         qi.frame = frame_in_reel;
 239         qi.eyes = eyes;
 240         _queue.push_back(qi);
 241
 242         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 243         _empty_condition.notify_all ();
 244 }
 245
 246
 247 /** Write some audio frames to the DCP.
 248  *  @param audio Audio data.
 249  *  @param time Time of this data within the DCP.
 250  *  This method is not thread safe.
 251  */
 252 void
 253 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 254 {
 255         DCPOMATIC_ASSERT (audio);
 256
 257         int const afr = film()->audio_frame_rate();
 258
 259         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 260
 261         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 262
 263         DCPTime t = time;
 264         while (t < end) {
 265
 266                 if (_audio_reel == _reels.end ()) {
 267                         /* This audio is off the end of the last reel; ignore it */
 268                         return;
 269                 }
 270
 271                 if (end <= _audio_reel->period().to) {
 272                         /* Easy case: we can write all the audio to this reel */
 273                         _audio_reel->write (audio);
 274                         t = end;
 275                 } else if (_audio_reel->period().to <= t) {
 276                         /* This reel is entirely before the start of our audio; just skip the reel */
 277                         ++_audio_reel;
 278                 } else {
 279                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 280                         DCPTime part_lengths[2] = {
 281                                 _audio_reel->period().to - t,
 282                                 end - _audio_reel->period().to
 283                         };
 284
 285                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 286                         Frame part_frames[2] = {
 287                                 part_lengths[0].frames_ceil(afr),
 288                                 part_lengths[1].frames_floor(afr)
 289                         };
 290
 291                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 292
 293                         if (part_frames[0]) {
 294                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 295                                 _audio_reel->write (part);
 296                         }
 297
 298                         if (part_frames[1]) {
 299                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 300                         } else {
 301                                 audio.reset ();
 302                         }
 303
 304                         ++_audio_reel;
 305                         t += part_lengths[0];
 306                 }
 307         }
 308 }
 309
 310
 311 void
 312 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 313 {
 314         if (_atmos_reel->period().to == time) {
 315                 ++_atmos_reel;
 316                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 317         }
 318
 319         /* We assume that we get a video frame's worth of data here */
 320         _atmos_reel->write (atmos, metadata);
 321 }
 322
 323
 324 /** Caller must hold a lock on _state_mutex */
 325 bool
 326 Writer::have_sequenced_image_at_queue_head ()
 327 {
 328         if (_queue.empty ()) {
 329                 return false;
 330         }
 331
 332         _queue.sort ();
 333         auto const & f = _queue.front();
 334         return _last_written[f.reel].next(f);
 335 }
 336
 337
 338 bool
 339 Writer::LastWritten::next (QueueItem qi) const
 340 {
 341         if (qi.eyes == Eyes::BOTH) {
 342                 /* 2D */
 343                 return qi.frame == (_frame + 1);
 344         }
 345
 346         /* 3D */
 347
 348         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 349                 return true;
 350         }
 351
 352         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 353                 return true;
 354         }
 355
 356         return false;
 357 }
 358
 359
 360 void
 361 Writer::LastWritten::update (QueueItem qi)
 362 {
 363         _frame = qi.frame;
 364         _eyes = qi.eyes;
 365 }
 366
 367
 368 void
 369 Writer::thread ()
 370 try
 371 {
 372         start_of_thread ("Writer");
 373
 374         while (true)
 375         {
 376                 boost::mutex::scoped_lock lock (_state_mutex);
 377
 378                 while (true) {
 379
 380                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 381                                 /* We've got something to do: go and do it */
 382                                 break;
 383                         }
 384
 385                         /* Nothing to do: wait until something happens which may indicate that we do */
 386                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 387                         _empty_condition.wait (lock);
 388                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 389                 }
 390
 391                 /* We stop here if we have been asked to finish, and if either the queue
 392                    is empty or we do not have a sequenced image at its head (if this is the
 393                    case we will never terminate as no new frames will be sent once
 394                    _finish is true).
 395                 */
 396                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 397                         /* (Hopefully temporarily) log anything that was not written */
 398                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 399                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 400                                 for (auto const& i: _queue) {
 401                                         if (i.type == QueueItem::Type::FULL) {
 402                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 403                                         } else {
 404                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 405                                         }
 406                                 }
 407                         }
 408                         return;
 409                 }
 410
 411                 /* Write any frames that we can write; i.e. those that are in sequence. */
 412                 while (have_sequenced_image_at_queue_head ()) {
 413                         auto qi = _queue.front ();
 414                         _last_written[qi.reel].update (qi);
 415                         _queue.pop_front ();
 416                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 417                                 --_queued_full_in_memory;
 418                         }
 419
 420                         lock.unlock ();
 421
 422                         auto& reel = _reels[qi.reel];
 423
 424                         switch (qi.type) {
 425                         case QueueItem::Type::FULL:
 426                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 427                                 if (!qi.encoded) {
 428                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 429                                 }
 430                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 431                                 ++_full_written;
 432                                 break;
 433                         case QueueItem::Type::FAKE:
 434                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 435                                 reel.fake_write (qi.size);
 436                                 ++_fake_written;
 437                                 break;
 438                         case QueueItem::Type::REPEAT:
 439                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 440                                 reel.repeat_write (qi.frame, qi.eyes);
 441                                 ++_repeat_written;
 442                                 break;
 443                         }
 444
 445                         lock.lock ();
 446                         _full_condition.notify_all ();
 447                 }
 448
 449                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 450                         /* Too many frames in memory which can't yet be written to the stream.
 451                            Write some FULL frames to disk.
 452                         */
 453
 454                         /* Find one from the back of the queue */
 455                         _queue.sort ();
 456                         auto i = _queue.rbegin ();
 457                         while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
 458                                 ++i;
 459                         }
 460
 461                         DCPOMATIC_ASSERT (i != _queue.rend());
 462                         ++_pushed_to_disk;
 463                         /* For the log message below */
 464                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 465                         lock.unlock ();
 466
 467                         /* i is valid here, even though we don't hold a lock on the mutex,
 468                            since list iterators are unaffected by insertion and only this
 469                            thread could erase the last item in the list.
 470                         */
 471
 472                         LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
 473
 474                         i->encoded->write_via_temp (
 475                                 film()->j2c_path(i->reel, i->frame, i->eyes, true),
 476                                 film()->j2c_path(i->reel, i->frame, i->eyes, false)
 477                                 );
 478
 479                         lock.lock ();
 480                         i->encoded.reset ();
 481                         --_queued_full_in_memory;
 482                         _full_condition.notify_all ();
 483                 }
 484         }
 485 }
 486 catch (...)
 487 {
 488         store_current ();
 489 }
 490
 491
 492 void
 493 Writer::terminate_thread (bool can_throw)
 494 {
 495         boost::this_thread::disable_interruption dis;
 496
 497         boost::mutex::scoped_lock lock (_state_mutex);
 498
 499         _finish = true;
 500         _empty_condition.notify_all ();
 501         _full_condition.notify_all ();
 502         lock.unlock ();
 503
 504         try {
 505                 _thread.join ();
 506         } catch (...) {}
 507
 508         if (can_throw) {
 509                 rethrow ();
 510         }
 511 }
 512
 513
 514 void
 515 Writer::calculate_digests ()
 516 {
 517         auto job = _job.lock ();
 518         if (job) {
 519                 job->sub (_("Computing digests"));
 520         }
 521
 522         boost::asio::io_service service;
 523         boost::thread_group pool;
 524
 525         auto work = make_shared<boost::asio::io_service::work>(service);
 526
 527         int const threads = max (1, Config::instance()->master_encoding_threads());
 528
 529         for (int i = 0; i < threads; ++i) {
 530                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 531         }
 532
 533         std::function<void (float)> set_progress;
 534         if (job) {
 535                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 536         } else {
 537                 set_progress = [](float) {
 538                         boost::this_thread::interruption_point();
 539                 };
 540         }
 541
 542         for (auto& i: _reels) {
 543                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 544         }
 545         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 546
 547         work.reset ();
 548
 549         try {
 550                 pool.join_all ();
 551         } catch (boost::thread_interrupted) {
 552                 /* join_all was interrupted, so we need to interrupt the threads
 553                  * in our pool then try again to join them.
 554                  */
 555                 pool.interrupt_all ();
 556                 pool.join_all ();
 557         }
 558
 559         service.stop ();
 560 }
 561
 562
 563 /** @param output_dcp Path to DCP folder to write */
 564 void
 565 Writer::finish (boost::filesystem::path output_dcp)
 566 {
 567         if (_thread.joinable()) {
 568                 LOG_GENERAL_NC ("Terminating writer thread");
 569                 terminate_thread (true);
 570         }
 571
 572         LOG_GENERAL_NC ("Finishing ReelWriters");
 573
 574         for (auto& i: _reels) {
 575                 write_hanging_text (i);
 576                 i.finish (output_dcp);
 577         }
 578
 579         LOG_GENERAL_NC ("Writing XML");
 580
 581         dcp::DCP dcp (output_dcp);
 582
 583         auto cpl = make_shared<dcp::CPL>(
 584                 film()->dcp_name(),
 585                 film()->dcp_content_type()->libdcp_kind(),
 586                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 587                 );
 588
 589         dcp.add (cpl);
 590
 591         calculate_digests ();
 592
 593         /* Add reels */
 594
 595         for (auto& i: _reels) {
 596                 cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
 597         }
 598
 599         /* Add metadata */
 600
 601         auto creator = Config::instance()->dcp_creator();
 602         if (creator.empty()) {
 603                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 604         }
 605
 606         auto issuer = Config::instance()->dcp_issuer();
 607         if (issuer.empty()) {
 608                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 609         }
 610
 611         cpl->set_creator (creator);
 612         cpl->set_issuer (issuer);
 613
 614         cpl->set_ratings (film()->ratings());
 615
 616         vector<dcp::ContentVersion> cv;
 617         for (auto i: film()->content_versions()) {
 618                 cv.push_back (dcp::ContentVersion(i));
 619         }
 620         if (cv.empty()) {
 621                 cv = { dcp::ContentVersion("1") };
 622         }
 623         cpl->set_content_versions (cv);
 624
 625         cpl->set_full_content_title_text (film()->name());
 626         cpl->set_full_content_title_text_language (film()->name_language());
 627         if (film()->release_territory()) {
 628                 cpl->set_release_territory (*film()->release_territory());
 629         }
 630         cpl->set_version_number (film()->version_number());
 631         cpl->set_status (film()->status());
 632         if (film()->chain()) {
 633                 cpl->set_chain (*film()->chain());
 634         }
 635         if (film()->distributor()) {
 636                 cpl->set_distributor (*film()->distributor());
 637         }
 638         if (film()->facility()) {
 639                 cpl->set_facility (*film()->facility());
 640         }
 641         if (film()->luminance()) {
 642                 cpl->set_luminance (*film()->luminance());
 643         }
 644         if (film()->sign_language_video_language()) {
 645                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 646         }
 647
 648         dcp::MCASoundField field;
 649         if (film()->audio_channels() <= 6) {
 650                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 651         } else {
 652                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 653         }
 654
 655         dcp::MainSoundConfiguration msc(field, MAX_DCP_AUDIO_CHANNELS);
 656         for (auto i: film()->mapped_audio_channels()) {
 657                 msc.set_mapping(i, static_cast<dcp::Channel>(i));
 658         }
 659
 660         cpl->set_main_sound_configuration(msc);
 661         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 662         cpl->set_main_picture_stored_area (film()->frame_size());
 663
 664         auto active_area = film()->active_area();
 665         if (active_area.width > 0 && active_area.height > 0) {
 666                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 667                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 668         }
 669
 670         auto sl = film()->subtitle_languages().second;
 671         if (!sl.empty()) {
 672                 cpl->set_additional_subtitle_languages(sl);
 673         }
 674
 675         auto signer = Config::instance()->signer_chain();
 676         /* We did check earlier, but check again here to be on the safe side */
 677         string reason;
 678         if (!signer->valid (&reason)) {
 679                 throw InvalidSignerError (reason);
 680         }
 681
 682         dcp.set_issuer(issuer);
 683         dcp.set_creator(creator);
 684         dcp.set_annotation_text(film()->dcp_name());
 685
 686         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 687
 688         LOG_GENERAL (
 689                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 690                 );
 691
 692         write_cover_sheet (output_dcp);
 693 }
 694
 695
 696 void
 697 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 698 {
 699         auto const cover = film()->file("COVER_SHEET.txt");
 700         dcp::File f(cover, "w");
 701         if (!f) {
 702                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 703         }
 704
 705         auto text = Config::instance()->cover_sheet ();
 706         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 707         auto cpls = film()->cpls();
 708         if (!cpls.empty()) {
 709                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 710         }
 711         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 712         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 713
 714         auto audio_language = film()->audio_language();
 715         if (audio_language) {
 716                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 717         } else {
 718                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 719         }
 720
 721         auto subtitle_languages = film()->subtitle_languages();
 722         if (subtitle_languages.first) {
 723                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 724         } else {
 725                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 726         }
 727
 728         boost::uintmax_t size = 0;
 729         for (
 730                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 731                 i != boost::filesystem::recursive_directory_iterator();
 732                 ++i) {
 733                 if (boost::filesystem::is_regular_file (i->path())) {
 734                         size += boost::filesystem::file_size (i->path());
 735                 }
 736         }
 737
 738         if (size > (1000000000L)) {
 739                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 740         } else {
 741                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 742         }
 743
 744         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 745         auto description = String::compose("%1.%2", ch.first, ch.second);
 746
 747         if (description == "0.0") {
 748                 description = _("None");
 749         } else if (description == "1.0") {
 750                 description = _("Mono");
 751         } else if (description == "2.0") {
 752                 description = _("Stereo");
 753         }
 754         boost::algorithm::replace_all (text, "$AUDIO", description);
 755
 756         auto const hmsf = film()->length().split(film()->video_frame_rate());
 757         string length;
 758         if (hmsf.h == 0 && hmsf.m == 0) {
 759                 length = String::compose("%1s", hmsf.s);
 760         } else if (hmsf.h == 0 && hmsf.m > 0) {
 761                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 762         } else if (hmsf.h > 0 && hmsf.m > 0) {
 763                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 764         }
 765
 766         boost::algorithm::replace_all (text, "$LENGTH", length);
 767
 768         f.checked_write(text.c_str(), text.length());
 769 }
 770
 771
 772 /** @param frame Frame index within the whole DCP.
 773  *  @return true if we can fake-write this frame.
 774  */
 775 bool
 776 Writer::can_fake_write (Frame frame) const
 777 {
 778         if (film()->encrypted()) {
 779                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 780                 return false;
 781         }
 782
 783         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 784            parameters in the asset writer.
 785         */
 786
 787         auto const & reel = _reels[video_reel(frame)];
 788
 789         /* Make frame relative to the start of the reel */
 790         frame -= reel.start ();
 791         return (frame != 0 && frame < reel.first_nonexistent_frame());
 792 }
 793
 794
 795 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 796 void
 797 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 798 {
 799         vector<ReelWriter>::iterator* reel = nullptr;
 800
 801         switch (type) {
 802         case TextType::OPEN_SUBTITLE:
 803                 reel = &_subtitle_reel;
 804                 _have_subtitles = true;
 805                 break;
 806         case TextType::CLOSED_CAPTION:
 807                 DCPOMATIC_ASSERT (track);
 808                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 809                 reel = &_caption_reels[*track];
 810                 _have_closed_captions.insert (*track);
 811                 break;
 812         default:
 813                 DCPOMATIC_ASSERT (false);
 814         }
 815
 816         DCPOMATIC_ASSERT (*reel != _reels.end());
 817         while ((*reel)->period().to <= period.from) {
 818                 ++(*reel);
 819                 DCPOMATIC_ASSERT (*reel != _reels.end());
 820                 write_hanging_text (**reel);
 821         }
 822
 823         auto back_off = [this](DCPTimePeriod period) {
 824                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 825                 return period;
 826         };
 827
 828         if (period.to > (*reel)->period().to) {
 829                 /* This text goes off the end of the reel.  Store parts of it that should go into
 830                  * other reels.
 831                  */
 832                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 833                         auto overlap = i->period().overlap(period);
 834                         if (overlap) {
 835                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 836                         }
 837                 }
 838                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 839                  * for subtitles being too close together.
 840                  */
 841                 period.to = (*reel)->period().to;
 842                 period = back_off(period);
 843         }
 844
 845         (*reel)->write(text, type, track, period, _fonts, _chosen_interop_font);
 846 }
 847
 848
 849 void
 850 Writer::write (vector<shared_ptr<Font>> fonts)
 851 {
 852         if (fonts.empty()) {
 853                 return;
 854         }
 855
 856         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 857         auto fix_id = [](string id) {
 858                 return id.empty() ? "font" : id;
 859         };
 860
 861         if (film()->interop()) {
 862                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 863                  * even write them as they upset some validators.  Set up _fonts so that every
 864                  * font used by any subtitle will be written with the same ID.
 865                  */
 866                 for (size_t i = 0; i < fonts.size(); ++i) {
 867                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 868                 }
 869                 _chosen_interop_font = fonts[0];
 870         } else {
 871                 set<string> used_ids;
 872
 873                 /* Return the index of a _N at the end of a string, or string::npos */
 874                 auto underscore_number_position = [](string s) {
 875                         auto last_underscore = s.find_last_of("_");
 876                         if (last_underscore == string::npos) {
 877                                 return string::npos;
 878                         }
 879
 880                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 881                                 if (!isdigit(s[i])) {
 882                                         return string::npos;
 883                                 }
 884                         }
 885
 886                         return last_underscore;
 887                 };
 888
 889                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 890                 for (auto font: fonts) {
 891                         auto id = fix_id(font->id());
 892                         if (used_ids.find(id) == used_ids.end()) {
 893                                 /* This ID is unique so we can just use it as-is */
 894                                 _fonts.put(font, id);
 895                                 used_ids.insert(id);
 896                         } else {
 897                                 auto end = underscore_number_position(id);
 898                                 if (end == string::npos) {
 899                                         /* This string has no _N suffix, so add one */
 900                                         id += "_0";
 901                                         end = underscore_number_position(id);
 902                                 }
 903
 904                                 ++end;
 905
 906                                 /* Increment the suffix until we find a unique one */
 907                                 auto number = dcp::raw_convert<int>(id.substr(end));
 908                                 while (used_ids.find(id) != used_ids.end()) {
 909                                         ++number;
 910                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 911                                 }
 912                                 used_ids.insert(id);
 913                         }
 914                         _fonts.put(font, id);
 915                 }
 916
 917                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 918         }
 919 }
 920
 921
 922 bool
 923 operator< (QueueItem const & a, QueueItem const & b)
 924 {
 925         if (a.reel != b.reel) {
 926                 return a.reel < b.reel;
 927         }
 928
 929         if (a.frame != b.frame) {
 930                 return a.frame < b.frame;
 931         }
 932
 933         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 934 }
 935
 936
 937 bool
 938 operator== (QueueItem const & a, QueueItem const & b)
 939 {
 940         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 941 }
 942
 943
 944 void
 945 Writer::set_encoder_threads (int threads)
 946 {
 947         boost::mutex::scoped_lock lm (_state_mutex);
 948         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 949         _maximum_queue_size = threads * 16;
 950 }
 951
 952
 953 void
 954 Writer::write (ReferencedReelAsset asset)
 955 {
 956         _reel_assets.push_back (asset);
 957 }
 958
 959
 960 size_t
 961 Writer::video_reel (int frame) const
 962 {
 963         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 964         size_t i = 0;
 965         while (i < _reels.size() && !_reels[i].period().contains (t)) {
 966                 ++i;
 967         }
 968
 969         DCPOMATIC_ASSERT (i < _reels.size ());
 970         return i;
 971 }
 972
 973
 974 void
 975 Writer::set_digest_progress (Job* job, float progress)
 976 {
 977         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 978
 979         _digest_progresses[boost::this_thread::get_id()] = progress;
 980         float min_progress = FLT_MAX;
 981         for (auto const& i: _digest_progresses) {
 982                 min_progress = min (min_progress, i.second);
 983         }
 984
 985         job->set_progress (min_progress);
 986
 987         Waker waker;
 988         waker.nudge ();
 989
 990         boost::this_thread::interruption_point();
 991 }
 992
 993
 994 /** Calculate hashes for any referenced MXF assets which do not already have one */
 995 void
 996 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
 997 try
 998 {
 999         for (auto const& i: _reel_assets) {
1000                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1001                 if (file && !file->hash()) {
1002                         file->asset_ref().asset()->hash (set_progress);
1003                         file->set_hash (file->asset_ref().asset()->hash());
1004                 }
1005         }
1006 } catch (boost::thread_interrupted) {
1007         /* set_progress contains an interruption_point, so any of these methods
1008          * may throw thread_interrupted, at which point we just give up.
1009          */
1010 }
1011
1012
1013 void
1014 Writer::write_hanging_text (ReelWriter& reel)
1015 {
1016         vector<HangingText> new_hanging_texts;
1017         for (auto i: _hanging_texts) {
1018                 if (i.period.from == reel.period().from) {
1019                         reel.write(i.text, i.type, i.track, i.period, _fonts, _chosen_interop_font);
1020                 } else {
1021                         new_hanging_texts.push_back (i);
1022                 }
1023         }
1024         _hanging_texts = new_hanging_texts;
1025 }