From 557afd0041173da9d67a850c04b6c8f7ce674860 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 15 Mar 2024 00:41:20 +0100 Subject: [PATCH] WIP: stop using video directory and hard-linking (#2756). --- src/lib/dcp_film_encoder.cc | 7 +- src/lib/film.cc | 90 +++++++++++++--------- src/lib/film.h | 11 ++- src/lib/hints.cc | 4 +- src/lib/reel_writer.cc | 124 +++++++++++------------------- src/lib/reel_writer.h | 6 +- src/lib/remembered_asset.cc | 97 +++++++++++++++++++++++ src/lib/remembered_asset.h | 79 +++++++++++++++++++ src/lib/writer.cc | 22 +++--- src/lib/writer.h | 7 +- src/lib/wscript | 1 + src/tools/dcpomatic.cc | 13 ++-- src/tools/dcpomatic_batch.cc | 5 +- test/j2k_encode_threading_test.cc | 4 +- test/j2k_encoder_test.cc | 2 +- test/j2k_video_bit_rate_test.cc | 5 +- test/recover_test.cc | 32 ++++---- test/reel_writer_test.cc | 2 +- test/writer_test.cc | 6 +- 19 files changed, 344 insertions(+), 173 deletions(-) create mode 100644 src/lib/remembered_asset.cc create mode 100644 src/lib/remembered_asset.h diff --git a/src/lib/dcp_film_encoder.cc b/src/lib/dcp_film_encoder.cc index bdcd17f38..878ef3c6f 100644 --- a/src/lib/dcp_film_encoder.cc +++ b/src/lib/dcp_film_encoder.cc @@ -67,7 +67,7 @@ using namespace dcpomatic; */ DCPFilmEncoder::DCPFilmEncoder(shared_ptr film, weak_ptr job) : FilmEncoder(film, job) - , _writer(film, job) + , _writer(film, job, film->dir(film->dcp_name())) , _finishing (false) , _non_burnt_subtitles (false) { @@ -82,6 +82,9 @@ DCPFilmEncoder::DCPFilmEncoder(shared_ptr film, weak_ptr job) DCPOMATIC_ASSERT(false); } + /* Now that we have a Writer we can clear out the assets directory */ + clean_up_asset_directory(film->assets_path()); + _player_video_connection = _player.Video.connect(bind(&DCPFilmEncoder::video, this, _1, _2)); _player_audio_connection = _player.Audio.connect(bind(&DCPFilmEncoder::audio, this, _1, _2)); _player_text_connection = _player.Text.connect(bind(&DCPFilmEncoder::text, this, _1, _2, _3, _4)); @@ -129,7 +132,7 @@ DCPFilmEncoder::go() _finishing = true; _encoder->end(); - _writer.finish(_film->dir(_film->dcp_name())); + _writer.finish(); } diff --git a/src/lib/film.cc b/src/lib/film.cc index a3e78e877..12902d14d 100644 --- a/src/lib/film.cc +++ b/src/lib/film.cc @@ -116,6 +116,7 @@ using namespace dcpomatic; static constexpr char metadata_file[] = "metadata.xml"; static constexpr char ui_state_file[] = "ui.xml"; +static constexpr char assets_file[] = "assets.xml"; /* 5 -> 6 @@ -288,17 +289,6 @@ Film::info_file (DCPTimePeriod period) const return file (p); } -boost::filesystem::path -Film::internal_video_asset_dir () const -{ - return dir ("video"); -} - -boost::filesystem::path -Film::internal_video_asset_filename (DCPTimePeriod p) const -{ - return video_identifier() + "_" + raw_convert (p.from.get()) + "_" + raw_convert (p.to.get()) + ".mxf"; -} boost::filesystem::path Film::audio_analysis_path (shared_ptr playlist) const @@ -342,6 +332,13 @@ Film::audio_analysis_path (shared_ptr playlist) const } +boost::filesystem::path +Film::assets_path() const +{ + return dir("assets"); +} + + boost::filesystem::path Film::subtitle_analysis_path (shared_ptr content) const { @@ -1818,30 +1815,11 @@ Film::required_disk_space () const * Note: the decision made by this method isn't, of course, 100% reliable. */ bool -Film::should_be_enough_disk_space (double& required, double& available, bool& can_hard_link) const -{ - /* Create a test file and see if we can hard-link it */ - boost::filesystem::path test = internal_video_asset_dir() / "test"; - boost::filesystem::path test2 = internal_video_asset_dir() / "test2"; - can_hard_link = true; - dcp::File f(test, "w"); - if (f) { - f.close(); - boost::system::error_code ec; - dcp::filesystem::create_hard_link(test, test2, ec); - if (ec) { - can_hard_link = false; - } - dcp::filesystem::remove(test); - dcp::filesystem::remove(test2); - } - - auto s = dcp::filesystem::space(internal_video_asset_dir()); - required = double (required_disk_space ()) / 1073741824.0f; - if (!can_hard_link) { - required *= 2; - } - available = double (s.available) / 1073741824.0f; +Film::should_be_enough_disk_space(double& required, double& available) const +{ + DCPOMATIC_ASSERT(directory()); + required = required_disk_space() / 1073741824.0f; + available = dcp::filesystem::space(*directory()).available / 1073741824.0f; return (available - required) > 1; } @@ -2383,3 +2361,45 @@ Film::read_ui_state() } } catch (...) {} } + + +vector +Film::read_remembered_assets() const +{ + vector assets; + + try { + cxml::Document xml("Assets"); + xml.read_file(dcp::filesystem::fix_long_path(file(assets_file))); + for (auto node: xml.node_children("Asset")) { + assets.push_back(RememberedAsset(node)); + } + } catch (std::exception& e) { + LOG_ERROR("Could not read assets file %1 (%2)", file(assets_file), e.what()); + } catch (...) { + LOG_ERROR("Could not read assets file %1", file(assets_file)); + } + + return assets; +} + + +void +Film::write_remembered_assets(vector const& assets) const +{ + auto doc = make_shared(); + auto root = doc->create_root_node("Assets"); + + for (auto asset: assets) { + asset.as_xml(root->add_child("Asset")); + } + + try { + doc->write_to_file_formatted(dcp::filesystem::fix_long_path(file(assets_file)).string()); + } catch (std::exception& e) { + LOG_ERROR("Could not write assets file %1 (%2)", file(assets_file), e.what()); + } catch (...) { + LOG_ERROR("Could not write assets file %1", file(assets_file)); + } +} + diff --git a/src/lib/film.h b/src/lib/film.h index e2e88e2c9..baac7c4fe 100644 --- a/src/lib/film.h +++ b/src/lib/film.h @@ -36,6 +36,7 @@ #include "film_property.h" #include "frame_rate_change.h" #include "named_channel.h" +#include "remembered_asset.h" #include "resolution.h" #include "signaller.h" #include "territory_type.h" @@ -118,11 +119,10 @@ public: std::shared_ptr info_file_handle (dcpomatic::DCPTimePeriod period, bool read) const; boost::filesystem::path j2c_path (int, Frame, Eyes, bool) const; - boost::filesystem::path internal_video_asset_dir () const; - boost::filesystem::path internal_video_asset_filename (dcpomatic::DCPTimePeriod p) const; boost::filesystem::path audio_analysis_path (std::shared_ptr) const; boost::filesystem::path subtitle_analysis_path (std::shared_ptr) const; + boost::filesystem::path assets_path() const; void send_dcp_to_tms (); @@ -162,7 +162,7 @@ public: std::list closed_caption_tracks () const; uint64_t required_disk_space () const; - bool should_be_enough_disk_space (double& required, double& available, bool& can_hard_link) const; + bool should_be_enough_disk_space(double& required, double& available) const; bool has_sign_language_video_channel () const; @@ -450,6 +450,10 @@ public: boost::optional ui_state(std::string key) const; void read_ui_state(); + std::vector read_remembered_assets() const; + void write_remembered_assets(std::vector const& assets) const; + std::string video_identifier() const; + /** Emitted when some property has of the Film is about to change or has changed */ mutable boost::signals2::signal Change; @@ -483,7 +487,6 @@ private: void signal_change (ChangeType, FilmProperty); void signal_change (ChangeType, int); - std::string video_identifier () const; void playlist_change (ChangeType); void playlist_order_changed (); void playlist_content_change (ChangeType type, std::weak_ptr, int, bool frequent); diff --git a/src/lib/hints.cc b/src/lib/hints.cc index ca69832a9..f9f87221e 100644 --- a/src/lib/hints.cc +++ b/src/lib/hints.cc @@ -77,7 +77,7 @@ using namespace boost::placeholders; Hints::Hints (weak_ptr weak_film) : WeakConstFilm (weak_film) - , _writer (new Writer(weak_film, weak_ptr(), true)) + , _writer(new Writer(weak_film, weak_ptr(), film()->dir("hints") / dcpomatic::get_process_id(), true)) , _analyser (film(), film()->playlist(), true, [](float) {}) , _stop (false) { @@ -495,7 +495,7 @@ try auto dcp_dir = film->dir("hints") / dcpomatic::get_process_id(); dcp::filesystem::remove_all(dcp_dir); - _writer->finish (film->dir("hints") / dcpomatic::get_process_id()); + _writer->finish(); dcp::DCP dcp (dcp_dir); dcp.read (); diff --git a/src/lib/reel_writer.cc b/src/lib/reel_writer.cc index 8dc0d7f06..950cb38db 100644 --- a/src/lib/reel_writer.cc +++ b/src/lib/reel_writer.cc @@ -34,6 +34,7 @@ #include "job.h" #include "log.h" #include "reel_writer.h" +#include "remembered_asset.h" #include #include #include @@ -106,9 +107,10 @@ mxf_metadata () * subtitle / closed caption files. */ ReelWriter::ReelWriter ( - weak_ptr weak_film, DCPTimePeriod period, shared_ptr job, int reel_index, int reel_count, bool text_only + weak_ptr weak_film, DCPTimePeriod period, shared_ptr job, int reel_index, int reel_count, bool text_only, boost::filesystem::path output_dir ) - : WeakConstFilm (weak_film) + : WeakConstFilm(weak_film) + , _output_dir(std::move(output_dir)) , _period (period) , _reel_index (reel_index) , _reel_count (reel_count) @@ -117,34 +119,24 @@ ReelWriter::ReelWriter ( , _text_only (text_only) , _font_metrics(film()->frame_size().height) { - /* Create or find our picture asset in a subdirectory, named - according to those film's parameters which affect the video - output. We will hard-link it into the DCP later. - */ + _default_font = dcp::ArrayData(default_font_file()); + + if (text_only) { + return; + } auto const standard = film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE; - boost::filesystem::path const asset = - film()->internal_video_asset_dir() / film()->internal_video_asset_filename(_period); + auto remembered_assets = film()->read_remembered_assets(); + DCPOMATIC_ASSERT(film()->directory()); - _first_nonexistent_frame = check_existing_picture_asset (asset); + auto existing_asset_filename = find_asset(remembered_assets, *film()->directory(), period, film()->video_identifier()); + if (existing_asset_filename) { + _first_nonexistent_frame = check_existing_picture_asset(*existing_asset_filename); + } if (_first_nonexistent_frame < period.duration().frames_round(film()->video_frame_rate())) { - /* We do not have a complete picture asset. If there is an - existing asset, break any hard links to it as we are about - to change its contents (if only by changing the IDs); see - #1126. - */ - if (dcp::filesystem::exists(asset) && dcp::filesystem::hard_link_count(asset) > 1) { - if (job) { - job->sub (_("Copying old video file")); - copy_in_bits (asset, asset.string() + ".tmp", bind(&Job::set_progress, job.get(), _1, false)); - } else { - dcp::filesystem::copy_file(asset, asset.string() + ".tmp"); - } - dcp::filesystem::remove(asset); - dcp::filesystem::rename(asset.string() + ".tmp", asset); - } + /* No existing asset, or an incomplete one */ auto const rate = dcp::Fraction(film()->video_frame_rate(), 1); @@ -158,6 +150,8 @@ ReelWriter::ReelWriter ( } }; + shared_ptr picture_asset; + if (film()->video_encoding() == VideoEncoding::JPEG2000) { if (film()->three_d()) { _j2k_picture_asset = std::make_shared(rate, standard); @@ -165,26 +159,45 @@ ReelWriter::ReelWriter ( _j2k_picture_asset = std::make_shared(rate, standard); } setup(_j2k_picture_asset); - _j2k_picture_asset->set_file(asset); - _j2k_picture_asset_writer = _j2k_picture_asset->start_write(asset, _first_nonexistent_frame > 0 ? dcp::Behaviour::OVERWRITE_EXISTING : dcp::Behaviour::MAKE_NEW); + picture_asset = _j2k_picture_asset; } else { _mpeg2_picture_asset = std::make_shared(rate); setup(_mpeg2_picture_asset); - _mpeg2_picture_asset->set_file(asset); - _mpeg2_picture_asset_writer = _mpeg2_picture_asset->start_write(asset, _first_nonexistent_frame > 0 ? dcp::Behaviour::OVERWRITE_EXISTING : dcp::Behaviour::MAKE_NEW); + picture_asset = _mpeg2_picture_asset; } - } else if (!text_only) { + auto new_asset_filename = _output_dir / video_asset_filename(picture_asset, _reel_index, _reel_count, _content_summary); + if (_first_nonexistent_frame > 0) { + LOG_GENERAL("Re-using partial asset %1: has frames up to %2", *existing_asset_filename, _first_nonexistent_frame); + dcp::filesystem::rename(*existing_asset_filename, new_asset_filename); + } + remembered_assets.push_back(RememberedAsset(new_asset_filename.filename(), period, film()->video_identifier())); + film()->write_remembered_assets(remembered_assets); + picture_asset->set_file(new_asset_filename); + + dcp::Behaviour const behaviour = _first_nonexistent_frame > 0 ? dcp::Behaviour::OVERWRITE_EXISTING : dcp::Behaviour::MAKE_NEW; + if (_j2k_picture_asset) { + _j2k_picture_asset_writer = _j2k_picture_asset->start_write(new_asset_filename, behaviour); + } else { + _mpeg2_picture_asset_writer = _mpeg2_picture_asset->start_write(new_asset_filename, behaviour); + } + } else { + LOG_GENERAL("Re-using complete asset %1", *existing_asset_filename); /* We already have a complete picture asset that we can just re-use */ /* XXX: what about if the encryption key changes? */ + auto new_asset_filename = _output_dir / existing_asset_filename->filename(); + dcp::filesystem::copy(*existing_asset_filename, new_asset_filename); + remembered_assets.push_back(RememberedAsset(new_asset_filename, period, film()->video_identifier())); + film()->write_remembered_assets(remembered_assets); + if (film()->video_encoding() == VideoEncoding::JPEG2000) { if (film()->three_d()) { - _j2k_picture_asset = make_shared(asset); + _j2k_picture_asset = make_shared(new_asset_filename); } else { - _j2k_picture_asset = make_shared(asset); + _j2k_picture_asset = make_shared(new_asset_filename); } } else { - _mpeg2_picture_asset = make_shared(asset); + _mpeg2_picture_asset = make_shared(new_asset_filename); } } @@ -223,8 +236,6 @@ ReelWriter::ReelWriter ( film()->limit_to_smpte_bv20() ? dcp::SoundAsset::MCASubDescriptors::DISABLED : dcp::SoundAsset::MCASubDescriptors::ENABLED ); } - - _default_font = dcp::ArrayData(default_font_file()); } @@ -366,51 +377,6 @@ ReelWriter::finish (boost::filesystem::path output_dcp) _sound_asset.reset (); } - shared_ptr picture_asset; - if (_j2k_picture_asset) { - picture_asset = _j2k_picture_asset; - } else if (_mpeg2_picture_asset) { - picture_asset = _mpeg2_picture_asset; - } - - /* Hard-link any video asset file into the DCP */ - if (picture_asset) { - auto const file = picture_asset->file(); - DCPOMATIC_ASSERT(file); - - auto video_from = *file; - auto video_to = output_dcp; - video_to /= video_asset_filename(picture_asset, _reel_index, _reel_count, _content_summary); - /* There may be an existing "to" file if we are recreating a DCP in the same place without - changing any video. - */ - boost::system::error_code ec; - dcp::filesystem::remove(video_to, ec); - - dcp::filesystem::create_hard_link(video_from, video_to, ec); - if (ec) { - LOG_WARNING("Hard-link failed (%1); copying instead", error_details(ec)); - auto job = _job.lock (); - if (job) { - job->sub (_("Copying video file into DCP")); - try { - copy_in_bits (video_from, video_to, bind(&Job::set_progress, job.get(), _1, false)); - } catch (exception& e) { - LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), e.what()); - throw FileError (e.what(), video_from); - } - } else { - dcp::filesystem::copy_file(video_from, video_to, ec); - if (ec) { - LOG_ERROR("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), error_details(ec)); - throw FileError (ec.message(), video_from); - } - } - } - - picture_asset->set_file(video_to); - } - /* Move the audio asset into the DCP */ if (_sound_asset) { boost::filesystem::path audio_to = output_dcp; diff --git a/src/lib/reel_writer.h b/src/lib/reel_writer.h index f6273f8e9..0b243b264 100644 --- a/src/lib/reel_writer.h +++ b/src/lib/reel_writer.h @@ -70,7 +70,8 @@ public: std::shared_ptr job, int reel_index, int reel_count, - bool text_only + bool text_only, + boost::filesystem::path output_dir ); void write (std::shared_ptr encoded, Frame frame, Eyes eyes); @@ -121,9 +122,10 @@ private: void create_reel_markers (std::shared_ptr reel) const; float convert_vertical_position(StringText const& subtitle, dcp::SubtitleStandard to) const; + boost::filesystem::path _output_dir; dcpomatic::DCPTimePeriod _period; /** the first picture frame index that does not already exist in our MXF */ - int _first_nonexistent_frame; + int _first_nonexistent_frame = 0; /** the data of the last written frame, if there is one */ EnumIndexedVector, Eyes> _last_written; /** index of this reel within the DCP (starting from 0) */ diff --git a/src/lib/remembered_asset.cc b/src/lib/remembered_asset.cc new file mode 100644 index 000000000..f9c550af4 --- /dev/null +++ b/src/lib/remembered_asset.cc @@ -0,0 +1,97 @@ +/* + Copyright (C) 2024 Carl Hetherington + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + +#include "dcpomatic_assert.h" +#include "remembered_asset.h" +#include +#include +#include +LIBDCP_DISABLE_WARNINGS +#include +LIBDCP_ENABLE_WARNINGS + + +using std::string; +using std::vector; + + +RememberedAsset::RememberedAsset(cxml::ConstNodePtr node) +{ + _filename = node->string_child("Filename"); + auto period_node = node->node_child("Period"); + DCPOMATIC_ASSERT(period_node); + + _period = { + dcpomatic::DCPTime(period_node->number_child("From")), + dcpomatic::DCPTime(period_node->number_child("To")) + }; + + _identifier = node->string_child("Identifier"); +} + + +void +RememberedAsset::as_xml(xmlpp::Element* parent) const +{ + cxml::add_text_child(parent, "Filename", _filename.string()); + auto period_node = parent->add_child("Period"); + cxml::add_text_child(period_node, "From", dcp::raw_convert(_period.from.get())); + cxml::add_text_child(period_node, "To", dcp::raw_convert(_period.to.get())); + cxml::add_text_child(parent, "Identifier", _identifier); +} + + +boost::optional +find_asset(vector const& assets, boost::filesystem::path directory, dcpomatic::DCPTimePeriod period, string identifier) +{ + for (auto path: dcp::filesystem::recursive_directory_iterator(directory)) { + auto iter = std::find_if(assets.begin(), assets.end(), [period, identifier, path](RememberedAsset const& asset) { + return asset.filename() == path.path().filename() && asset.period() == period && asset.identifier() == identifier; + }); + if (iter != assets.end()) { + return path.path(); + } + } + + return {}; +} + + +void +clean_up_asset_directory(boost::filesystem::path directory) +{ + /* We could do something more advanced here (e.g. keep the last N assets) but for now + * let's just clean the whole thing out. + */ + boost::system::error_code ec; + dcp::filesystem::remove_all(directory, ec); +} + + +void +preserve_assets(boost::filesystem::path search, boost::filesystem::path assets_path) +{ + for (auto const& path: boost::filesystem::directory_iterator(search)) { + if (path.path().extension() == ".mxf") { + dcp::filesystem::rename(path.path(), assets_path / path.path().filename()); + } + } +} diff --git a/src/lib/remembered_asset.h b/src/lib/remembered_asset.h new file mode 100644 index 000000000..ec9344e24 --- /dev/null +++ b/src/lib/remembered_asset.h @@ -0,0 +1,79 @@ +/* + Copyright (C) 2024 Carl Hetherington + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + +#ifndef DCPOMATIC_REMEMBERED_ASSET_H +#define DCPOMATIC_REMEMBERED_ASSET_H + + +#include "dcpomatic_time.h" +#include +#include +LIBDCP_DISABLE_WARNINGS +#include +LIBDCP_ENABLE_WARNINGS +#include + + +class RememberedAsset +{ +public: + explicit RememberedAsset(cxml::ConstNodePtr node); + + RememberedAsset(boost::filesystem::path filename, dcpomatic::DCPTimePeriod period, std::string identifier) + : _filename(filename) + , _period(period) + , _identifier(std::move(identifier)) + {} + + void as_xml(xmlpp::Element* parent) const; + + boost::filesystem::path filename() const { + return _filename; + } + + dcpomatic::DCPTimePeriod period() const { + return _period; + } + + std::string identifier() const { + return _identifier; + } + +private: + boost::filesystem::path _filename; + dcpomatic::DCPTimePeriod _period; + std::string _identifier; +}; + + +boost::optional find_asset( + std::vector const& assets, boost::filesystem::path directory, dcpomatic::DCPTimePeriod period, std::string identifier + ); + + +void clean_up_asset_directory(boost::filesystem::path directory); + + +void preserve_assets(boost::filesystem::path search, boost::filesystem::path assets_path); + + +#endif + diff --git a/src/lib/writer.cc b/src/lib/writer.cc index c0b083ff0..f9293ed09 100644 --- a/src/lib/writer.cc +++ b/src/lib/writer.cc @@ -79,9 +79,10 @@ using namespace dcpomatic; /** @param weak_job Job to report progress to, or 0. * @param text_only true to enable only the text (subtitle/ccap) parts of the writer. */ -Writer::Writer(weak_ptr weak_film, weak_ptr weak_job, bool text_only) - : WeakConstFilm (weak_film) +Writer::Writer(weak_ptr weak_film, weak_ptr weak_job, boost::filesystem::path output_dir, bool text_only) + : WeakConstFilm(weak_film) , _job(weak_job) + , _output_dir(output_dir) /* These will be reset to sensible values when J2KEncoder is created */ , _maximum_frames_in_memory (8) , _maximum_queue_size (8) @@ -92,7 +93,7 @@ Writer::Writer(weak_ptr weak_film, weak_ptr weak_job, bool text int reel_index = 0; auto const reels = film()->reels(); for (auto p: reels) { - _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only)); + _reels.push_back(ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only, _output_dir)); } _last_written.resize (reels.size()); @@ -588,9 +589,8 @@ Writer::calculate_digests () } -/** @param output_dcp Path to DCP folder to write */ void -Writer::finish (boost::filesystem::path output_dcp) +Writer::finish() { if (_thread.joinable()) { LOG_GENERAL_NC ("Terminating writer thread"); @@ -601,12 +601,12 @@ Writer::finish (boost::filesystem::path output_dcp) for (auto& reel: _reels) { write_hanging_text(reel); - reel.finish(output_dcp); + reel.finish(_output_dir); } LOG_GENERAL_NC ("Writing XML"); - dcp::DCP dcp (output_dcp); + dcp::DCP dcp(_output_dir); auto cpl = make_shared( film()->dcp_name(), @@ -621,7 +621,7 @@ Writer::finish (boost::filesystem::path output_dcp) /* Add reels */ for (auto& i: _reels) { - cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions)); + cpl->add(i.create_reel(_reel_assets, _output_dir, _have_subtitles, _have_closed_captions)); } /* Add metadata */ @@ -723,12 +723,12 @@ Writer::finish (boost::filesystem::path output_dcp) N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk ); - write_cover_sheet (output_dcp); + write_cover_sheet(); } void -Writer::write_cover_sheet (boost::filesystem::path output_dcp) +Writer::write_cover_sheet() { auto const cover = film()->file("COVER_SHEET.txt"); dcp::File file(cover, "w"); @@ -761,7 +761,7 @@ Writer::write_cover_sheet (boost::filesystem::path output_dcp) boost::uintmax_t size = 0; for ( - auto i = dcp::filesystem::recursive_directory_iterator(output_dcp); + auto i = dcp::filesystem::recursive_directory_iterator(_output_dir); i != dcp::filesystem::recursive_directory_iterator(); ++i) { if (dcp::filesystem::is_regular_file(i->path())) { diff --git a/src/lib/writer.h b/src/lib/writer.h index f9ec0b88c..3e93c9b7b 100644 --- a/src/lib/writer.h +++ b/src/lib/writer.h @@ -106,7 +106,7 @@ bool operator== (QueueItem const & a, QueueItem const & b); class Writer : public ExceptionStore, public WeakConstFilm { public: - Writer (std::weak_ptr, std::weak_ptr, bool text_only = false); + Writer(std::weak_ptr, std::weak_ptr, boost::filesystem::path output_dir, bool text_only = false); ~Writer (); Writer (Writer const &) = delete; @@ -126,7 +126,7 @@ public: void write (ReferencedReelAsset asset); void write (std::shared_ptr atmos, dcpomatic::DCPTime time, AtmosMetadata metadata); void write (std::shared_ptr image, Frame frame); - void finish (boost::filesystem::path output_dcp); + void finish(); void set_encoder_threads (int threads); @@ -142,7 +142,7 @@ private: bool have_sequenced_image_at_queue_head (); size_t video_reel (int frame) const; void set_digest_progress(Job* job, int id, int64_t done, int64_t size); - void write_cover_sheet (boost::filesystem::path output_dcp); + void write_cover_sheet(); void calculate_referenced_digests(std::function set_progress); void write_hanging_text (ReelWriter& reel); void calculate_digests (); @@ -154,6 +154,7 @@ private: std::map::iterator> _caption_reels; std::vector::iterator _atmos_reel; + boost::filesystem::path _output_dir; /** our thread */ boost::thread _thread; /** true if our thread should finish */ diff --git a/src/lib/wscript b/src/lib/wscript index 68b988eb3..33e68a108 100644 --- a/src/lib/wscript +++ b/src/lib/wscript @@ -169,6 +169,7 @@ sources = """ reel_writer.cc referenced_reel_asset.cc release_notes.cc + remembered_asset.cc render_text.cc remote_j2k_encoder_thread.cc resampler.cc diff --git a/src/tools/dcpomatic.cc b/src/tools/dcpomatic.cc index 8f65fa83d..63802279f 100644 --- a/src/tools/dcpomatic.cc +++ b/src/tools/dcpomatic.cc @@ -102,6 +102,7 @@ #include #include #include +#include #include LIBDCP_DISABLE_WARNINGS #include @@ -816,15 +817,9 @@ private: { double required; double available; - bool can_hard_link; - if (!_film->should_be_enough_disk_space (required, available, can_hard_link)) { - wxString message; - if (can_hard_link) { - message = wxString::Format (_("The DCP for this film will take up about %.1f GB, and the disk that you are using only has %.1f GB available. Do you want to continue anyway?"), required, available); - } else { - message = wxString::Format (_("The DCP and intermediate files for this film will take up about %.1f GB, and the disk that you are using only has %.1f GB available. You would need half as much space if the filesystem supported hard links, but it does not. Do you want to continue anyway?"), required, available); - } + if (!_film->should_be_enough_disk_space(required, available)) { + auto const message = wxString::Format(_("The DCP for this film will take up about %.1f GB, and the disk that you are using only has %.1f GB available. Do you want to continue anyway?"), required, available); if (!confirm_dialog (this, message)) { return; } @@ -854,6 +849,8 @@ private: if (!confirm_dialog (this, wxString::Format (_("Do you want to overwrite the existing DCP %s?"), std_to_wx(dcp_dir.string()).data()))) { return; } + + preserve_assets(dcp_dir, _film->assets_path()); dcp::filesystem::remove_all(dcp_dir); } diff --git a/src/tools/dcpomatic_batch.cc b/src/tools/dcpomatic_batch.cc index 32e8dec08..51d700d20 100644 --- a/src/tools/dcpomatic_batch.cc +++ b/src/tools/dcpomatic_batch.cc @@ -220,9 +220,8 @@ public: double total_required; double available; - bool can_hard_link; - film->should_be_enough_disk_space (total_required, available, can_hard_link); + film->should_be_enough_disk_space(total_required, available); set> films; @@ -239,7 +238,7 @@ public: } double required; - i->should_be_enough_disk_space (required, available, can_hard_link); + i->should_be_enough_disk_space(required, available); total_required += (1 - progress) * required; } diff --git a/test/j2k_encode_threading_test.cc b/test/j2k_encode_threading_test.cc index ee219fbe0..f63f7c829 100644 --- a/test/j2k_encode_threading_test.cc +++ b/test/j2k_encode_threading_test.cc @@ -44,7 +44,7 @@ using std::list; BOOST_AUTO_TEST_CASE(local_threads_created_and_destroyed) { auto film = new_test_film2("local_threads_created_and_destroyed", {}); - Writer writer(film, {}); + Writer writer(film, {}, "foo"); J2KEncoder encoder(film, writer); encoder.remake_threads(32, 0, {}); @@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(local_threads_created_and_destroyed) BOOST_AUTO_TEST_CASE(remote_threads_created_and_destroyed) { auto film = new_test_film2("remote_threads_created_and_destroyed", {}); - Writer writer(film, {}); + Writer writer(film, {}, "foo"); J2KEncoder encoder(film, writer); list servers = { diff --git a/test/j2k_encoder_test.cc b/test/j2k_encoder_test.cc index 39e3f9135..e231dce88 100644 --- a/test/j2k_encoder_test.cc +++ b/test/j2k_encoder_test.cc @@ -45,7 +45,7 @@ BOOST_AUTO_TEST_CASE(j2k_encoder_deadlock_test) auto film = new_test_film2("j2k_encoder_deadlock_test"); /* Don't call ::start() on this Writer, so it can never write anything */ - Writer writer(film, {}); + Writer writer(film, {}, {}); writer.set_encoder_threads(4); /* We want to test the case where the writer queue fills, and this can't happen unless there diff --git a/test/j2k_video_bit_rate_test.cc b/test/j2k_video_bit_rate_test.cc index b8388ca4c..8cb16387b 100644 --- a/test/j2k_video_bit_rate_test.cc +++ b/test/j2k_video_bit_rate_test.cc @@ -65,10 +65,7 @@ check (int target_bits_per_second) target_bits_per_second <= 250000000 ); - boost::filesystem::directory_iterator i (boost::filesystem::path("build") / "test" / name / "video"); - boost::filesystem::path test = *i++; - BOOST_REQUIRE (i == boost::filesystem::directory_iterator()); - + auto test = find_file(film->dir(film->dcp_name()), "j2c_"); double actual_bits_per_second = boost::filesystem::file_size(test) * 8.0 / duration; /* Check that we're within 85% to 115% of target on average */ diff --git a/test/recover_test.cc b/test/recover_test.cc index 696a2c36a..30090b4a3 100644 --- a/test/recover_test.cc +++ b/test/recover_test.cc @@ -76,13 +76,16 @@ BOOST_AUTO_TEST_CASE (recover_test_2d) dcp::VerificationNote::Code::MISSING_FFEC_IN_FEATURE }); - boost::filesystem::path const video = "build/test/recover_test_2d/video/185_2K_4650f318cea570763a0c6411c8c098ce_24_100000000_P_S_L21_0_1200000.mxf"; + auto video = [film]() { + return find_file(boost::filesystem::path("build/test/recover_test_2d") / film->dcp_name(false), "j2c_"); + }; + boost::filesystem::copy_file ( - video, + video(), "build/test/recover_test_2d/original.mxf" ); - boost::filesystem::resize_file (video, 2 * 1024 * 1024); + boost::filesystem::resize_file(video(), 2 * 1024 * 1024); make_and_verify_dcp( film, @@ -96,7 +99,7 @@ BOOST_AUTO_TEST_CASE (recover_test_2d) ); auto A = make_shared("build/test/recover_test_2d/original.mxf"); - auto B = make_shared(video); + auto B = make_shared(video()); dcp::EqualityOptions eq; BOOST_CHECK (A->equals (B, eq, boost::bind (¬e, _1, _2))); @@ -120,14 +123,16 @@ BOOST_AUTO_TEST_CASE (recover_test_3d, * boost::unit_test::depends_on("recover_t make_and_verify_dcp (film, { dcp::VerificationNote::Code::MISSING_FFEC_IN_FEATURE, dcp::VerificationNote::Code::MISSING_FFMC_IN_FEATURE }); - boost::filesystem::path const video = "build/test/recover_test_3d/video/185_2K_60a75a531ca9546bdd513163117e2214_24_100000000_P_S_L21_3D_0_96000.mxf"; + auto video = [film]() { + return find_file(boost::filesystem::path("build/test/recover_test_3d") / film->dcp_name(false), "j2c_"); + }; boost::filesystem::copy_file ( - video, + video(), "build/test/recover_test_3d/original.mxf" ); - boost::filesystem::resize_file (video, 2 * 1024 * 1024); + boost::filesystem::resize_file(video(), 2 * 1024 * 1024); make_and_verify_dcp( film, @@ -140,7 +145,7 @@ BOOST_AUTO_TEST_CASE (recover_test_3d, * boost::unit_test::depends_on("recover_t ); auto A = make_shared("build/test/recover_test_3d/original.mxf"); - auto B = make_shared(video); + auto B = make_shared(video()); dcp::EqualityOptions eq; BOOST_CHECK (A->equals (B, eq, boost::bind (¬e, _1, _2))); @@ -164,15 +169,16 @@ BOOST_AUTO_TEST_CASE (recover_test_2d_encrypted, * boost::unit_test::depends_on( make_and_verify_dcp (film, { dcp::VerificationNote::Code::MISSING_FFEC_IN_FEATURE, dcp::VerificationNote::Code::MISSING_FFMC_IN_FEATURE }); - boost::filesystem::path const video = - "build/test/recover_test_2d_encrypted/video/185_2K_4650f318cea570763a0c6411c8c098ce_24_100000000_Eeafcb91c9f5472edf01f3a2404c57258_S_L21_0_1200000.mxf"; + auto video = [film]() { + return find_file(boost::filesystem::path("build/test/recover_test_2d_encrypted") / film->dcp_name(false), "j2c_"); + }; boost::filesystem::copy_file ( - video, + video(), "build/test/recover_test_2d_encrypted/original.mxf" ); - boost::filesystem::resize_file (video, 2 * 1024 * 1024); + boost::filesystem::resize_file(video(), 2 * 1024 * 1024); make_and_verify_dcp( film, @@ -186,7 +192,7 @@ BOOST_AUTO_TEST_CASE (recover_test_2d_encrypted, * boost::unit_test::depends_on( auto A = make_shared("build/test/recover_test_2d_encrypted/original.mxf"); A->set_key (film->key ()); - auto B = make_shared(video); + auto B = make_shared(video()); B->set_key (film->key ()); dcp::EqualityOptions eq; diff --git a/test/reel_writer_test.cc b/test/reel_writer_test.cc index f81e8e333..9a756bd46 100644 --- a/test/reel_writer_test.cc +++ b/test/reel_writer_test.cc @@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE (write_frame_info_test) { auto film = new_test_film2 ("write_frame_info_test"); dcpomatic::DCPTimePeriod const period (dcpomatic::DCPTime(0), dcpomatic::DCPTime(96000)); - ReelWriter writer (film, period, shared_ptr(), 0, 1, false); + ReelWriter writer(film, period, shared_ptr(), 0, 1, false, "foo"); /* Write the first one */ diff --git a/test/writer_test.cc b/test/writer_test.cc index 2d4da570f..b98e553ed 100644 --- a/test/writer_test.cc +++ b/test/writer_test.cc @@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE (test_write_odd_amount_of_silence) auto content = content_factory("test/data/flat_red.png"); auto film = new_test_film2 ("test_write_odd_amount_of_silence", content); content[0]->video->set_length(24); - auto writer = make_shared(film, shared_ptr()); + auto writer = make_shared(film, shared_ptr(), "foo"); auto audio = make_shared(6, 48000); audio->make_silent (); @@ -82,7 +82,7 @@ BOOST_AUTO_TEST_CASE (interrupt_writer) auto video_ptr = make_shared(video.data(), video.size()); auto audio = make_shared(6, 48000 / 24); - auto writer = make_shared(film, shared_ptr()); + auto writer = make_shared(film, shared_ptr(), film->dir(film->dcp_name())); writer->start (); for (int i = 0; i < frames; ++i) { @@ -92,7 +92,7 @@ BOOST_AUTO_TEST_CASE (interrupt_writer) /* Start digest calculations then abort them; there should be no crash or error */ boost::thread thread([film, writer]() { - writer->finish(film->dir(film->dcp_name())); + writer->finish(); }); dcpomatic_sleep_seconds (1); -- 2.30.2