WIP: stop using video directory and hard-linking (#2756).
authorCarl Hetherington <cth@carlh.net>
Thu, 14 Mar 2024 23:41:20 +0000 (00:41 +0100)
committerCarl Hetherington <cth@carlh.net>
Fri, 10 May 2024 22:18:55 +0000 (00:18 +0200)
19 files changed:
src/lib/dcp_film_encoder.cc
src/lib/film.cc
src/lib/film.h
src/lib/hints.cc
src/lib/reel_writer.cc
src/lib/reel_writer.h
src/lib/remembered_asset.cc [new file with mode: 0644]
src/lib/remembered_asset.h [new file with mode: 0644]
src/lib/writer.cc
src/lib/writer.h
src/lib/wscript
src/tools/dcpomatic.cc
src/tools/dcpomatic_batch.cc
test/j2k_encode_threading_test.cc
test/j2k_encoder_test.cc
test/j2k_video_bit_rate_test.cc
test/recover_test.cc
test/reel_writer_test.cc
test/writer_test.cc

index bdcd17f387954023c6d1a1300f3de8a5291af0d0..878ef3c6f63f7fe98e0d9cecadf0995999a16631 100644 (file)
@@ -67,7 +67,7 @@ using namespace dcpomatic;
  */
 DCPFilmEncoder::DCPFilmEncoder(shared_ptr<const Film> film, weak_ptr<Job> job)
        : FilmEncoder(film, job)
-       , _writer(film, job)
+       , _writer(film, job, film->dir(film->dcp_name()))
        , _finishing (false)
        , _non_burnt_subtitles (false)
 {
@@ -82,6 +82,9 @@ DCPFilmEncoder::DCPFilmEncoder(shared_ptr<const Film> film, weak_ptr<Job> job)
                DCPOMATIC_ASSERT(false);
        }
 
+       /* Now that we have a Writer we can clear out the assets directory */
+       clean_up_asset_directory(film->assets_path());
+
        _player_video_connection = _player.Video.connect(bind(&DCPFilmEncoder::video, this, _1, _2));
        _player_audio_connection = _player.Audio.connect(bind(&DCPFilmEncoder::audio, this, _1, _2));
        _player_text_connection = _player.Text.connect(bind(&DCPFilmEncoder::text, this, _1, _2, _3, _4));
@@ -129,7 +132,7 @@ DCPFilmEncoder::go()
 
        _finishing = true;
        _encoder->end();
-       _writer.finish(_film->dir(_film->dcp_name()));
+       _writer.finish();
 }
 
 
index a3e78e8773e557cbb00b345d33c469b910537af0..12902d14d5d9f8a8ccd34680057a1e11210fb57e 100644 (file)
@@ -116,6 +116,7 @@ using namespace dcpomatic;
 
 static constexpr char metadata_file[] = "metadata.xml";
 static constexpr char ui_state_file[] = "ui.xml";
+static constexpr char assets_file[] = "assets.xml";
 
 
 /* 5 -> 6
@@ -288,17 +289,6 @@ Film::info_file (DCPTimePeriod period) const
        return file (p);
 }
 
-boost::filesystem::path
-Film::internal_video_asset_dir () const
-{
-       return dir ("video");
-}
-
-boost::filesystem::path
-Film::internal_video_asset_filename (DCPTimePeriod p) const
-{
-       return video_identifier() + "_" + raw_convert<string> (p.from.get()) + "_" + raw_convert<string> (p.to.get()) + ".mxf";
-}
 
 boost::filesystem::path
 Film::audio_analysis_path (shared_ptr<const Playlist> playlist) const
@@ -342,6 +332,13 @@ Film::audio_analysis_path (shared_ptr<const Playlist> playlist) const
 }
 
 
+boost::filesystem::path
+Film::assets_path() const
+{
+       return dir("assets");
+}
+
+
 boost::filesystem::path
 Film::subtitle_analysis_path (shared_ptr<const Content> content) const
 {
@@ -1818,30 +1815,11 @@ Film::required_disk_space () const
  *  Note: the decision made by this method isn't, of course, 100% reliable.
  */
 bool
-Film::should_be_enough_disk_space (double& required, double& available, bool& can_hard_link) const
-{
-       /* Create a test file and see if we can hard-link it */
-       boost::filesystem::path test = internal_video_asset_dir() / "test";
-       boost::filesystem::path test2 = internal_video_asset_dir() / "test2";
-       can_hard_link = true;
-       dcp::File f(test, "w");
-       if (f) {
-               f.close();
-               boost::system::error_code ec;
-               dcp::filesystem::create_hard_link(test, test2, ec);
-               if (ec) {
-                       can_hard_link = false;
-               }
-               dcp::filesystem::remove(test);
-               dcp::filesystem::remove(test2);
-       }
-
-       auto s = dcp::filesystem::space(internal_video_asset_dir());
-       required = double (required_disk_space ()) / 1073741824.0f;
-       if (!can_hard_link) {
-               required *= 2;
-       }
-       available = double (s.available) / 1073741824.0f;
+Film::should_be_enough_disk_space(double& required, double& available) const
+{
+       DCPOMATIC_ASSERT(directory());
+       required = required_disk_space() / 1073741824.0f;
+       available = dcp::filesystem::space(*directory()).available / 1073741824.0f;
        return (available - required) > 1;
 }
 
@@ -2383,3 +2361,45 @@ Film::read_ui_state()
                }
        } catch (...) {}
 }
+
+
+vector<RememberedAsset>
+Film::read_remembered_assets() const
+{
+       vector<RememberedAsset> assets;
+
+       try {
+               cxml::Document xml("Assets");
+               xml.read_file(dcp::filesystem::fix_long_path(file(assets_file)));
+               for (auto node: xml.node_children("Asset")) {
+                       assets.push_back(RememberedAsset(node));
+               }
+       } catch (std::exception& e) {
+               LOG_ERROR("Could not read assets file %1 (%2)", file(assets_file), e.what());
+       } catch (...) {
+               LOG_ERROR("Could not read assets file %1", file(assets_file));
+       }
+
+       return assets;
+}
+
+
+void
+Film::write_remembered_assets(vector<RememberedAsset> const& assets) const
+{
+       auto doc = make_shared<xmlpp::Document>();
+       auto root = doc->create_root_node("Assets");
+
+       for (auto asset: assets) {
+               asset.as_xml(root->add_child("Asset"));
+       }
+
+       try {
+               doc->write_to_file_formatted(dcp::filesystem::fix_long_path(file(assets_file)).string());
+       } catch (std::exception& e) {
+               LOG_ERROR("Could not write assets file %1 (%2)", file(assets_file), e.what());
+       } catch (...) {
+               LOG_ERROR("Could not write assets file %1", file(assets_file));
+       }
+}
+
index e2e88e2c9fe294569c40b165e88f51e7e68a68dd..baac7c4fead5241b978f7fef7668326a0f308770 100644 (file)
@@ -36,6 +36,7 @@
 #include "film_property.h"
 #include "frame_rate_change.h"
 #include "named_channel.h"
+#include "remembered_asset.h"
 #include "resolution.h"
 #include "signaller.h"
 #include "territory_type.h"
@@ -118,11 +119,10 @@ public:
 
        std::shared_ptr<InfoFileHandle> info_file_handle (dcpomatic::DCPTimePeriod period, bool read) const;
        boost::filesystem::path j2c_path (int, Frame, Eyes, bool) const;
-       boost::filesystem::path internal_video_asset_dir () const;
-       boost::filesystem::path internal_video_asset_filename (dcpomatic::DCPTimePeriod p) const;
 
        boost::filesystem::path audio_analysis_path (std::shared_ptr<const Playlist>) const;
        boost::filesystem::path subtitle_analysis_path (std::shared_ptr<const Content>) const;
+       boost::filesystem::path assets_path() const;
 
        void send_dcp_to_tms ();
 
@@ -162,7 +162,7 @@ public:
        std::list<DCPTextTrack> closed_caption_tracks () const;
 
        uint64_t required_disk_space () const;
-       bool should_be_enough_disk_space (double& required, double& available, bool& can_hard_link) const;
+       bool should_be_enough_disk_space(double& required, double& available) const;
 
        bool has_sign_language_video_channel () const;
 
@@ -450,6 +450,10 @@ public:
        boost::optional<std::string> ui_state(std::string key) const;
        void read_ui_state();
 
+       std::vector<RememberedAsset> read_remembered_assets() const;
+       void write_remembered_assets(std::vector<RememberedAsset> const& assets) const;
+       std::string video_identifier() const;
+
        /** Emitted when some property has of the Film is about to change or has changed */
        mutable boost::signals2::signal<void (ChangeType, FilmProperty)> Change;
 
@@ -483,7 +487,6 @@ private:
 
        void signal_change (ChangeType, FilmProperty);
        void signal_change (ChangeType, int);
-       std::string video_identifier () const;
        void playlist_change (ChangeType);
        void playlist_order_changed ();
        void playlist_content_change (ChangeType type, std::weak_ptr<Content>, int, bool frequent);
index ca69832a9d3a77d21677d339d7cd473aa08d4ca4..f9f87221eeedaf79c7214696758241fc6e12a095 100644 (file)
@@ -77,7 +77,7 @@ using namespace boost::placeholders;
 
 Hints::Hints (weak_ptr<const Film> weak_film)
        : WeakConstFilm (weak_film)
-       , _writer (new Writer(weak_film, weak_ptr<Job>(), true))
+       , _writer(new Writer(weak_film, weak_ptr<Job>(), film()->dir("hints") / dcpomatic::get_process_id(), true))
        , _analyser (film(), film()->playlist(), true, [](float) {})
        , _stop (false)
 {
@@ -495,7 +495,7 @@ try
        auto dcp_dir = film->dir("hints") / dcpomatic::get_process_id();
        dcp::filesystem::remove_all(dcp_dir);
 
-       _writer->finish (film->dir("hints") / dcpomatic::get_process_id());
+       _writer->finish();
 
        dcp::DCP dcp (dcp_dir);
        dcp.read ();
index 8dc0d7f06ff9936530c0520458ecd08977f38cf8..950cb38db67759390e79906b60ed6ab235d66aac 100644 (file)
@@ -34,6 +34,7 @@
 #include "job.h"
 #include "log.h"
 #include "reel_writer.h"
+#include "remembered_asset.h"
 #include <dcp/atmos_asset.h>
 #include <dcp/atmos_asset_writer.h>
 #include <dcp/certificate_chain.h>
@@ -106,9 +107,10 @@ mxf_metadata ()
  *  subtitle / closed caption files.
  */
 ReelWriter::ReelWriter (
-       weak_ptr<const Film> weak_film, DCPTimePeriod period, shared_ptr<Job> job, int reel_index, int reel_count, bool text_only
+       weak_ptr<const Film> weak_film, DCPTimePeriod period, shared_ptr<Job> job, int reel_index, int reel_count, bool text_only, boost::filesystem::path output_dir
        )
-       : WeakConstFilm (weak_film)
+       : WeakConstFilm(weak_film)
+       , _output_dir(std::move(output_dir))
        , _period (period)
        , _reel_index (reel_index)
        , _reel_count (reel_count)
@@ -117,34 +119,24 @@ ReelWriter::ReelWriter (
        , _text_only (text_only)
        , _font_metrics(film()->frame_size().height)
 {
-       /* Create or find our picture asset in a subdirectory, named
-          according to those film's parameters which affect the video
-          output.  We will hard-link it into the DCP later.
-       */
+       _default_font = dcp::ArrayData(default_font_file());
+
+       if (text_only) {
+               return;
+       }
 
        auto const standard = film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE;
 
-       boost::filesystem::path const asset =
-               film()->internal_video_asset_dir() / film()->internal_video_asset_filename(_period);
+       auto remembered_assets = film()->read_remembered_assets();
+       DCPOMATIC_ASSERT(film()->directory());
 
-       _first_nonexistent_frame = check_existing_picture_asset (asset);
+       auto existing_asset_filename = find_asset(remembered_assets, *film()->directory(), period, film()->video_identifier());
+       if (existing_asset_filename) {
+               _first_nonexistent_frame = check_existing_picture_asset(*existing_asset_filename);
+       }
 
        if (_first_nonexistent_frame < period.duration().frames_round(film()->video_frame_rate())) {
-               /* We do not have a complete picture asset.  If there is an
-                  existing asset, break any hard links to it as we are about
-                  to change its contents (if only by changing the IDs); see
-                  #1126.
-               */
-               if (dcp::filesystem::exists(asset) && dcp::filesystem::hard_link_count(asset) > 1) {
-                       if (job) {
-                               job->sub (_("Copying old video file"));
-                               copy_in_bits (asset, asset.string() + ".tmp", bind(&Job::set_progress, job.get(), _1, false));
-                       } else {
-                               dcp::filesystem::copy_file(asset, asset.string() + ".tmp");
-                       }
-                       dcp::filesystem::remove(asset);
-                       dcp::filesystem::rename(asset.string() + ".tmp", asset);
-               }
+               /* No existing asset, or an incomplete one */
 
                auto const rate = dcp::Fraction(film()->video_frame_rate(), 1);
 
@@ -158,6 +150,8 @@ ReelWriter::ReelWriter (
                        }
                };
 
+               shared_ptr<dcp::PictureAsset> picture_asset;
+
                if (film()->video_encoding() == VideoEncoding::JPEG2000) {
                        if (film()->three_d()) {
                                _j2k_picture_asset = std::make_shared<dcp::StereoJ2KPictureAsset>(rate, standard);
@@ -165,26 +159,45 @@ ReelWriter::ReelWriter (
                                _j2k_picture_asset = std::make_shared<dcp::MonoJ2KPictureAsset>(rate, standard);
                        }
                        setup(_j2k_picture_asset);
-                       _j2k_picture_asset->set_file(asset);
-                       _j2k_picture_asset_writer = _j2k_picture_asset->start_write(asset, _first_nonexistent_frame > 0 ? dcp::Behaviour::OVERWRITE_EXISTING : dcp::Behaviour::MAKE_NEW);
+                       picture_asset = _j2k_picture_asset;
                } else {
                        _mpeg2_picture_asset = std::make_shared<dcp::MonoMPEG2PictureAsset>(rate);
                        setup(_mpeg2_picture_asset);
-                       _mpeg2_picture_asset->set_file(asset);
-                       _mpeg2_picture_asset_writer = _mpeg2_picture_asset->start_write(asset, _first_nonexistent_frame > 0 ? dcp::Behaviour::OVERWRITE_EXISTING : dcp::Behaviour::MAKE_NEW);
+                       picture_asset = _mpeg2_picture_asset;
                }
 
-       } else if (!text_only) {
+               auto new_asset_filename = _output_dir / video_asset_filename(picture_asset, _reel_index, _reel_count, _content_summary);
+               if (_first_nonexistent_frame > 0) {
+                       LOG_GENERAL("Re-using partial asset %1: has frames up to %2", *existing_asset_filename, _first_nonexistent_frame);
+                       dcp::filesystem::rename(*existing_asset_filename, new_asset_filename);
+               }
+               remembered_assets.push_back(RememberedAsset(new_asset_filename.filename(), period, film()->video_identifier()));
+               film()->write_remembered_assets(remembered_assets);
+               picture_asset->set_file(new_asset_filename);
+
+               dcp::Behaviour const behaviour = _first_nonexistent_frame > 0 ? dcp::Behaviour::OVERWRITE_EXISTING : dcp::Behaviour::MAKE_NEW;
+               if (_j2k_picture_asset) {
+                       _j2k_picture_asset_writer = _j2k_picture_asset->start_write(new_asset_filename, behaviour);
+               } else {
+                       _mpeg2_picture_asset_writer = _mpeg2_picture_asset->start_write(new_asset_filename, behaviour);
+               }
+       } else {
+               LOG_GENERAL("Re-using complete asset %1", *existing_asset_filename);
                /* We already have a complete picture asset that we can just re-use */
                /* XXX: what about if the encryption key changes? */
+               auto new_asset_filename = _output_dir / existing_asset_filename->filename();
+               dcp::filesystem::copy(*existing_asset_filename, new_asset_filename);
+               remembered_assets.push_back(RememberedAsset(new_asset_filename, period, film()->video_identifier()));
+               film()->write_remembered_assets(remembered_assets);
+
                if (film()->video_encoding() == VideoEncoding::JPEG2000) {
                        if (film()->three_d()) {
-                               _j2k_picture_asset = make_shared<dcp::StereoJ2KPictureAsset>(asset);
+                               _j2k_picture_asset = make_shared<dcp::StereoJ2KPictureAsset>(new_asset_filename);
                        } else {
-                               _j2k_picture_asset = make_shared<dcp::MonoJ2KPictureAsset>(asset);
+                               _j2k_picture_asset = make_shared<dcp::MonoJ2KPictureAsset>(new_asset_filename);
                        }
                } else {
-                       _mpeg2_picture_asset = make_shared<dcp::MonoMPEG2PictureAsset>(asset);
+                       _mpeg2_picture_asset = make_shared<dcp::MonoMPEG2PictureAsset>(new_asset_filename);
                }
        }
 
@@ -223,8 +236,6 @@ ReelWriter::ReelWriter (
                        film()->limit_to_smpte_bv20() ? dcp::SoundAsset::MCASubDescriptors::DISABLED : dcp::SoundAsset::MCASubDescriptors::ENABLED
                        );
        }
-
-       _default_font = dcp::ArrayData(default_font_file());
 }
 
 
@@ -366,51 +377,6 @@ ReelWriter::finish (boost::filesystem::path output_dcp)
                _sound_asset.reset ();
        }
 
-       shared_ptr<dcp::PictureAsset> picture_asset;
-       if (_j2k_picture_asset) {
-               picture_asset = _j2k_picture_asset;
-       } else if (_mpeg2_picture_asset) {
-               picture_asset = _mpeg2_picture_asset;
-       }
-
-       /* Hard-link any video asset file into the DCP */
-       if (picture_asset) {
-               auto const file = picture_asset->file();
-               DCPOMATIC_ASSERT(file);
-
-               auto video_from = *file;
-               auto video_to = output_dcp;
-               video_to /= video_asset_filename(picture_asset, _reel_index, _reel_count, _content_summary);
-               /* There may be an existing "to" file if we are recreating a DCP in the same place without
-                  changing any video.
-               */
-               boost::system::error_code ec;
-               dcp::filesystem::remove(video_to, ec);
-
-               dcp::filesystem::create_hard_link(video_from, video_to, ec);
-               if (ec) {
-                       LOG_WARNING("Hard-link failed (%1); copying instead", error_details(ec));
-                       auto job = _job.lock ();
-                       if (job) {
-                               job->sub (_("Copying video file into DCP"));
-                               try {
-                                       copy_in_bits (video_from, video_to, bind(&Job::set_progress, job.get(), _1, false));
-                               } catch (exception& e) {
-                                       LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), e.what());
-                                       throw FileError (e.what(), video_from);
-                               }
-                       } else {
-                               dcp::filesystem::copy_file(video_from, video_to, ec);
-                               if (ec) {
-                                       LOG_ERROR("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), error_details(ec));
-                                       throw FileError (ec.message(), video_from);
-                               }
-                       }
-               }
-
-               picture_asset->set_file(video_to);
-       }
-
        /* Move the audio asset into the DCP */
        if (_sound_asset) {
                boost::filesystem::path audio_to = output_dcp;
index f6273f8e9349b320d826e9bfa79f2d88afdf50b4..0b243b264405807bc84de4e9db338fc84aaeaecd 100644 (file)
@@ -70,7 +70,8 @@ public:
                std::shared_ptr<Job> job,
                int reel_index,
                int reel_count,
-               bool text_only
+               bool text_only,
+               boost::filesystem::path output_dir
                );
 
        void write (std::shared_ptr<const dcp::Data> encoded, Frame frame, Eyes eyes);
@@ -121,9 +122,10 @@ private:
        void create_reel_markers (std::shared_ptr<dcp::Reel> reel) const;
        float convert_vertical_position(StringText const& subtitle, dcp::SubtitleStandard to) const;
 
+       boost::filesystem::path _output_dir;
        dcpomatic::DCPTimePeriod _period;
        /** the first picture frame index that does not already exist in our MXF */
-       int _first_nonexistent_frame;
+       int _first_nonexistent_frame = 0;
        /** the data of the last written frame, if there is one */
        EnumIndexedVector<std::shared_ptr<const dcp::Data>, Eyes> _last_written;
        /** index of this reel within the DCP (starting from 0) */
diff --git a/src/lib/remembered_asset.cc b/src/lib/remembered_asset.cc
new file mode 100644 (file)
index 0000000..f9c550a
--- /dev/null
@@ -0,0 +1,97 @@
+/*
+    Copyright (C) 2024 Carl Hetherington <cth@carlh.net>
+
+    This file is part of DCP-o-matic.
+
+    DCP-o-matic is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    DCP-o-matic is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+
+#include "dcpomatic_assert.h"
+#include "remembered_asset.h"
+#include <dcp/filesystem.h>
+#include <dcp/raw_convert.h>
+#include <libcxml/cxml.h>
+LIBDCP_DISABLE_WARNINGS
+#include <libxml++/libxml++.h>
+LIBDCP_ENABLE_WARNINGS
+
+
+using std::string;
+using std::vector;
+
+
+RememberedAsset::RememberedAsset(cxml::ConstNodePtr node)
+{
+       _filename = node->string_child("Filename");
+       auto period_node = node->node_child("Period");
+       DCPOMATIC_ASSERT(period_node);
+
+       _period = {
+               dcpomatic::DCPTime(period_node->number_child<int64_t>("From")),
+               dcpomatic::DCPTime(period_node->number_child<int64_t>("To"))
+       };
+
+       _identifier = node->string_child("Identifier");
+}
+
+
+void
+RememberedAsset::as_xml(xmlpp::Element* parent) const
+{
+       cxml::add_text_child(parent, "Filename", _filename.string());
+       auto period_node = parent->add_child("Period");
+       cxml::add_text_child(period_node, "From", dcp::raw_convert<string>(_period.from.get()));
+       cxml::add_text_child(period_node, "To", dcp::raw_convert<string>(_period.to.get()));
+       cxml::add_text_child(parent, "Identifier", _identifier);
+}
+
+
+boost::optional<boost::filesystem::path>
+find_asset(vector<RememberedAsset> const& assets, boost::filesystem::path directory, dcpomatic::DCPTimePeriod period, string identifier)
+{
+       for (auto path: dcp::filesystem::recursive_directory_iterator(directory)) {
+               auto iter = std::find_if(assets.begin(), assets.end(), [period, identifier, path](RememberedAsset const& asset) {
+                       return asset.filename() == path.path().filename() && asset.period() == period && asset.identifier() == identifier;
+               });
+               if (iter != assets.end()) {
+                       return path.path();
+               }
+       }
+
+       return {};
+}
+
+
+void
+clean_up_asset_directory(boost::filesystem::path directory)
+{
+       /* We could do something more advanced here (e.g. keep the last N assets) but for now
+        * let's just clean the whole thing out.
+        */
+       boost::system::error_code ec;
+       dcp::filesystem::remove_all(directory, ec);
+}
+
+
+void
+preserve_assets(boost::filesystem::path search, boost::filesystem::path assets_path)
+{
+       for (auto const& path: boost::filesystem::directory_iterator(search)) {
+               if (path.path().extension() == ".mxf") {
+                       dcp::filesystem::rename(path.path(), assets_path / path.path().filename());
+               }
+       }
+}
diff --git a/src/lib/remembered_asset.h b/src/lib/remembered_asset.h
new file mode 100644 (file)
index 0000000..ec9344e
--- /dev/null
@@ -0,0 +1,79 @@
+/*
+    Copyright (C) 2024 Carl Hetherington <cth@carlh.net>
+
+    This file is part of DCP-o-matic.
+
+    DCP-o-matic is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    DCP-o-matic is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+
+#ifndef DCPOMATIC_REMEMBERED_ASSET_H
+#define DCPOMATIC_REMEMBERED_ASSET_H
+
+
+#include "dcpomatic_time.h"
+#include <libcxml/cxml.h>
+#include <dcp/warnings.h>
+LIBDCP_DISABLE_WARNINGS
+#include <libxml++/libxml++.h>
+LIBDCP_ENABLE_WARNINGS
+#include <boost/filesystem.hpp>
+
+
+class RememberedAsset
+{
+public:
+       explicit RememberedAsset(cxml::ConstNodePtr node);
+
+       RememberedAsset(boost::filesystem::path filename, dcpomatic::DCPTimePeriod period, std::string identifier)
+               : _filename(filename)
+               , _period(period)
+               , _identifier(std::move(identifier))
+       {}
+
+       void as_xml(xmlpp::Element* parent) const;
+
+       boost::filesystem::path filename() const {
+               return _filename;
+       }
+
+       dcpomatic::DCPTimePeriod period() const {
+               return _period;
+       }
+
+       std::string identifier() const {
+               return _identifier;
+       }
+
+private:
+       boost::filesystem::path _filename;
+       dcpomatic::DCPTimePeriod _period;
+       std::string _identifier;
+};
+
+
+boost::optional<boost::filesystem::path> find_asset(
+       std::vector<RememberedAsset> const& assets, boost::filesystem::path directory, dcpomatic::DCPTimePeriod period, std::string identifier
+       );
+
+
+void clean_up_asset_directory(boost::filesystem::path directory);
+
+
+void preserve_assets(boost::filesystem::path search, boost::filesystem::path assets_path);
+
+
+#endif
+
index c0b083ff06adc080f25b61c416211e4398c34ea8..f9293ed091c04a60280a1c97fe1eaca5b8ddb4c9 100644 (file)
@@ -79,9 +79,10 @@ using namespace dcpomatic;
 /** @param weak_job Job to report progress to, or 0.
  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  */
-Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, bool text_only)
-       : WeakConstFilm (weak_film)
+Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, boost::filesystem::path output_dir, bool text_only)
+       : WeakConstFilm(weak_film)
        , _job(weak_job)
+       , _output_dir(output_dir)
        /* These will be reset to sensible values when J2KEncoder is created */
        , _maximum_frames_in_memory (8)
        , _maximum_queue_size (8)
@@ -92,7 +93,7 @@ Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, bool text
        int reel_index = 0;
        auto const reels = film()->reels();
        for (auto p: reels) {
-               _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
+               _reels.push_back(ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only, _output_dir));
        }
 
        _last_written.resize (reels.size());
@@ -588,9 +589,8 @@ Writer::calculate_digests ()
 }
 
 
-/** @param output_dcp Path to DCP folder to write */
 void
-Writer::finish (boost::filesystem::path output_dcp)
+Writer::finish()
 {
        if (_thread.joinable()) {
                LOG_GENERAL_NC ("Terminating writer thread");
@@ -601,12 +601,12 @@ Writer::finish (boost::filesystem::path output_dcp)
 
        for (auto& reel: _reels) {
                write_hanging_text(reel);
-               reel.finish(output_dcp);
+               reel.finish(_output_dir);
        }
 
        LOG_GENERAL_NC ("Writing XML");
 
-       dcp::DCP dcp (output_dcp);
+       dcp::DCP dcp(_output_dir);
 
        auto cpl = make_shared<dcp::CPL>(
                film()->dcp_name(),
@@ -621,7 +621,7 @@ Writer::finish (boost::filesystem::path output_dcp)
        /* Add reels */
 
        for (auto& i: _reels) {
-               cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
+               cpl->add(i.create_reel(_reel_assets, _output_dir, _have_subtitles, _have_closed_captions));
        }
 
        /* Add metadata */
@@ -723,12 +723,12 @@ Writer::finish (boost::filesystem::path output_dcp)
                N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
                );
 
-       write_cover_sheet (output_dcp);
+       write_cover_sheet();
 }
 
 
 void
-Writer::write_cover_sheet (boost::filesystem::path output_dcp)
+Writer::write_cover_sheet()
 {
        auto const cover = film()->file("COVER_SHEET.txt");
        dcp::File file(cover, "w");
@@ -761,7 +761,7 @@ Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 
        boost::uintmax_t size = 0;
        for (
-               auto i = dcp::filesystem::recursive_directory_iterator(output_dcp);
+               auto i = dcp::filesystem::recursive_directory_iterator(_output_dir);
                i != dcp::filesystem::recursive_directory_iterator();
                ++i) {
                if (dcp::filesystem::is_regular_file(i->path())) {
index f9ec0b88cb5dae2b35e5502cea427508d657f7cc..3e93c9b7b56f4c5e9c8d293982dea52951cd2008 100644 (file)
@@ -106,7 +106,7 @@ bool operator== (QueueItem const & a, QueueItem const & b);
 class Writer : public ExceptionStore, public WeakConstFilm
 {
 public:
-       Writer (std::weak_ptr<const Film>, std::weak_ptr<Job>, bool text_only = false);
+       Writer(std::weak_ptr<const Film>, std::weak_ptr<Job>, boost::filesystem::path output_dir, bool text_only = false);
        ~Writer ();
 
        Writer (Writer const &) = delete;
@@ -126,7 +126,7 @@ public:
        void write (ReferencedReelAsset asset);
        void write (std::shared_ptr<const dcp::AtmosFrame> atmos, dcpomatic::DCPTime time, AtmosMetadata metadata);
        void write (std::shared_ptr<dcp::MonoMPEG2PictureFrame> image, Frame frame);
-       void finish (boost::filesystem::path output_dcp);
+       void finish();
 
        void set_encoder_threads (int threads);
 
@@ -142,7 +142,7 @@ private:
        bool have_sequenced_image_at_queue_head ();
        size_t video_reel (int frame) const;
        void set_digest_progress(Job* job, int id, int64_t done, int64_t size);
-       void write_cover_sheet (boost::filesystem::path output_dcp);
+       void write_cover_sheet();
        void calculate_referenced_digests(std::function<void (int64_t, int64_t)> set_progress);
        void write_hanging_text (ReelWriter& reel);
        void calculate_digests ();
@@ -154,6 +154,7 @@ private:
        std::map<DCPTextTrack, std::vector<ReelWriter>::iterator> _caption_reels;
        std::vector<ReelWriter>::iterator _atmos_reel;
 
+       boost::filesystem::path _output_dir;
        /** our thread */
        boost::thread _thread;
        /** true if our thread should finish */
index 68b988eb3faffdf3e812d1f2a67e2ddc012d4abc..33e68a108660c1eb5ddd137dbee51301956e6202 100644 (file)
@@ -169,6 +169,7 @@ sources = """
           reel_writer.cc
           referenced_reel_asset.cc
           release_notes.cc
+          remembered_asset.cc
           render_text.cc
           remote_j2k_encoder_thread.cc
           resampler.cc
index 8f65fa83db9ff786efe92f6bf58f8d5bf5c0d036..63802279f925f99e7316f37bb5030b6f9177f914 100644 (file)
 #include <dcp/exceptions.h>
 #include <dcp/filesystem.h>
 #include <dcp/raw_convert.h>
+#include <dcp/scope_guard.h>
 #include <dcp/warnings.h>
 LIBDCP_DISABLE_WARNINGS
 #include <wx/cmdline.h>
@@ -816,15 +817,9 @@ private:
        {
                double required;
                double available;
-               bool can_hard_link;
 
-               if (!_film->should_be_enough_disk_space (required, available, can_hard_link)) {
-                       wxString message;
-                       if (can_hard_link) {
-                               message = wxString::Format (_("The DCP for this film will take up about %.1f GB, and the disk that you are using only has %.1f GB available.  Do you want to continue anyway?"), required, available);
-                       } else {
-                               message = wxString::Format (_("The DCP and intermediate files for this film will take up about %.1f GB, and the disk that you are using only has %.1f GB available.  You would need half as much space if the filesystem supported hard links, but it does not.  Do you want to continue anyway?"), required, available);
-                       }
+               if (!_film->should_be_enough_disk_space(required, available)) {
+                       auto const message = wxString::Format(_("The DCP for this film will take up about %.1f GB, and the disk that you are using only has %.1f GB available.  Do you want to continue anyway?"), required, available);
                        if (!confirm_dialog (this, message)) {
                                return;
                        }
@@ -854,6 +849,8 @@ private:
                        if (!confirm_dialog (this, wxString::Format (_("Do you want to overwrite the existing DCP %s?"), std_to_wx(dcp_dir.string()).data()))) {
                                return;
                        }
+
+                       preserve_assets(dcp_dir, _film->assets_path());
                        dcp::filesystem::remove_all(dcp_dir);
                }
 
index 32e8dec084656bae1dd118b070b894e341d08455..51d700d204931549ac53bba41c22ce66413fc9ab 100644 (file)
@@ -220,9 +220,8 @@ public:
 
                        double total_required;
                        double available;
-                       bool can_hard_link;
 
-                       film->should_be_enough_disk_space (total_required, available, can_hard_link);
+                       film->should_be_enough_disk_space(total_required, available);
 
                        set<shared_ptr<const Film>> films;
 
@@ -239,7 +238,7 @@ public:
                                }
 
                                double required;
-                               i->should_be_enough_disk_space (required, available, can_hard_link);
+                               i->should_be_enough_disk_space(required, available);
                                total_required += (1 - progress) * required;
                        }
 
index ee219fbe0faf4944fc01c4bf334e05c5972b2b70..f63f7c82976cef42a44a301bb272344b521da19e 100644 (file)
@@ -44,7 +44,7 @@ using std::list;
 BOOST_AUTO_TEST_CASE(local_threads_created_and_destroyed)
 {
        auto film = new_test_film2("local_threads_created_and_destroyed", {});
-       Writer writer(film, {});
+       Writer writer(film, {}, "foo");
        J2KEncoder encoder(film, writer);
 
        encoder.remake_threads(32, 0, {});
@@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(local_threads_created_and_destroyed)
 BOOST_AUTO_TEST_CASE(remote_threads_created_and_destroyed)
 {
        auto film = new_test_film2("remote_threads_created_and_destroyed", {});
-       Writer writer(film, {});
+       Writer writer(film, {}, "foo");
        J2KEncoder encoder(film, writer);
 
        list<EncodeServerDescription> servers = {
index 39e3f9135f7bb17386f4e5d38b7ae96630e3beb0..e231dce88c38e7ecfe41f123d448bf7f37537cfe 100644 (file)
@@ -45,7 +45,7 @@ BOOST_AUTO_TEST_CASE(j2k_encoder_deadlock_test)
        auto film = new_test_film2("j2k_encoder_deadlock_test");
 
        /* Don't call ::start() on this Writer, so it can never write anything */
-       Writer writer(film, {});
+       Writer writer(film, {}, {});
        writer.set_encoder_threads(4);
 
        /* We want to test the case where the writer queue fills, and this can't happen unless there
index b8388ca4cd3d4ba6f861312a2943b03b3eee8d72..8cb16387b1d0e8f81fa65d77a99e847575e1fb8d 100644 (file)
@@ -65,10 +65,7 @@ check (int target_bits_per_second)
                target_bits_per_second <= 250000000
                );
 
-       boost::filesystem::directory_iterator i (boost::filesystem::path("build") / "test" / name / "video");
-       boost::filesystem::path test = *i++;
-       BOOST_REQUIRE (i == boost::filesystem::directory_iterator());
-
+       auto test = find_file(film->dir(film->dcp_name()), "j2c_");
        double actual_bits_per_second = boost::filesystem::file_size(test) * 8.0 / duration;
 
        /* Check that we're within 85% to 115% of target on average */
index 696a2c36a38c6afb59d2df8d7b2e7a31b6061123..30090b4a3e9c930292487290dc6f64ae28ff6463 100644 (file)
@@ -76,13 +76,16 @@ BOOST_AUTO_TEST_CASE (recover_test_2d)
                        dcp::VerificationNote::Code::MISSING_FFEC_IN_FEATURE
                });
 
-       boost::filesystem::path const video = "build/test/recover_test_2d/video/185_2K_4650f318cea570763a0c6411c8c098ce_24_100000000_P_S_L21_0_1200000.mxf";
+       auto video = [film]() {
+               return find_file(boost::filesystem::path("build/test/recover_test_2d") / film->dcp_name(false), "j2c_");
+       };
+
        boost::filesystem::copy_file (
-               video,
+               video(),
                "build/test/recover_test_2d/original.mxf"
                );
 
-       boost::filesystem::resize_file (video, 2 * 1024 * 1024);
+       boost::filesystem::resize_file(video(), 2 * 1024 * 1024);
 
        make_and_verify_dcp(
                film,
@@ -96,7 +99,7 @@ BOOST_AUTO_TEST_CASE (recover_test_2d)
                );
 
        auto A = make_shared<dcp::MonoJ2KPictureAsset>("build/test/recover_test_2d/original.mxf");
-       auto B = make_shared<dcp::MonoJ2KPictureAsset>(video);
+       auto B = make_shared<dcp::MonoJ2KPictureAsset>(video());
 
        dcp::EqualityOptions eq;
        BOOST_CHECK (A->equals (B, eq, boost::bind (&note, _1, _2)));
@@ -120,14 +123,16 @@ BOOST_AUTO_TEST_CASE (recover_test_3d, * boost::unit_test::depends_on("recover_t
 
        make_and_verify_dcp (film, { dcp::VerificationNote::Code::MISSING_FFEC_IN_FEATURE, dcp::VerificationNote::Code::MISSING_FFMC_IN_FEATURE });
 
-       boost::filesystem::path const video = "build/test/recover_test_3d/video/185_2K_60a75a531ca9546bdd513163117e2214_24_100000000_P_S_L21_3D_0_96000.mxf";
+       auto video = [film]() {
+               return find_file(boost::filesystem::path("build/test/recover_test_3d") / film->dcp_name(false), "j2c_");
+       };
 
        boost::filesystem::copy_file (
-               video,
+               video(),
                "build/test/recover_test_3d/original.mxf"
                );
 
-       boost::filesystem::resize_file (video, 2 * 1024 * 1024);
+       boost::filesystem::resize_file(video(), 2 * 1024 * 1024);
 
        make_and_verify_dcp(
                film,
@@ -140,7 +145,7 @@ BOOST_AUTO_TEST_CASE (recover_test_3d, * boost::unit_test::depends_on("recover_t
                );
 
        auto A = make_shared<dcp::StereoJ2KPictureAsset>("build/test/recover_test_3d/original.mxf");
-       auto B = make_shared<dcp::StereoJ2KPictureAsset>(video);
+       auto B = make_shared<dcp::StereoJ2KPictureAsset>(video());
 
        dcp::EqualityOptions eq;
        BOOST_CHECK (A->equals (B, eq, boost::bind (&note, _1, _2)));
@@ -164,15 +169,16 @@ BOOST_AUTO_TEST_CASE (recover_test_2d_encrypted, * boost::unit_test::depends_on(
 
        make_and_verify_dcp (film, { dcp::VerificationNote::Code::MISSING_FFEC_IN_FEATURE, dcp::VerificationNote::Code::MISSING_FFMC_IN_FEATURE });
 
-       boost::filesystem::path const video =
-               "build/test/recover_test_2d_encrypted/video/185_2K_4650f318cea570763a0c6411c8c098ce_24_100000000_Eeafcb91c9f5472edf01f3a2404c57258_S_L21_0_1200000.mxf";
+       auto video = [film]() {
+               return find_file(boost::filesystem::path("build/test/recover_test_2d_encrypted") / film->dcp_name(false), "j2c_");
+       };
 
        boost::filesystem::copy_file (
-               video,
+               video(),
                "build/test/recover_test_2d_encrypted/original.mxf"
                );
 
-       boost::filesystem::resize_file (video, 2 * 1024 * 1024);
+       boost::filesystem::resize_file(video(), 2 * 1024 * 1024);
 
        make_and_verify_dcp(
                film,
@@ -186,7 +192,7 @@ BOOST_AUTO_TEST_CASE (recover_test_2d_encrypted, * boost::unit_test::depends_on(
 
        auto A = make_shared<dcp::MonoJ2KPictureAsset>("build/test/recover_test_2d_encrypted/original.mxf");
        A->set_key (film->key ());
-       auto B = make_shared<dcp::MonoJ2KPictureAsset>(video);
+       auto B = make_shared<dcp::MonoJ2KPictureAsset>(video());
        B->set_key (film->key ());
 
        dcp::EqualityOptions eq;
index f81e8e333af111bd60aab5cb162d21031d8ae36b..9a756bd4652c0f7ca3dc12296da7ec479f6a3be7 100644 (file)
@@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE (write_frame_info_test)
 {
        auto film = new_test_film2 ("write_frame_info_test");
        dcpomatic::DCPTimePeriod const period (dcpomatic::DCPTime(0), dcpomatic::DCPTime(96000));
-       ReelWriter writer (film, period, shared_ptr<Job>(), 0, 1, false);
+       ReelWriter writer(film, period, shared_ptr<Job>(), 0, 1, false, "foo");
 
        /* Write the first one */
 
index 2d4da570f9ee7f7dd711651a5a5cbb0cf27cf73a..b98e553edda75397c83cf44177d909ad3df40f41 100644 (file)
@@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE (test_write_odd_amount_of_silence)
        auto content = content_factory("test/data/flat_red.png");
        auto film = new_test_film2 ("test_write_odd_amount_of_silence", content);
        content[0]->video->set_length(24);
-       auto writer = make_shared<Writer>(film, shared_ptr<Job>());
+       auto writer = make_shared<Writer>(film, shared_ptr<Job>(), "foo");
 
        auto audio = make_shared<AudioBuffers>(6, 48000);
        audio->make_silent ();
@@ -82,7 +82,7 @@ BOOST_AUTO_TEST_CASE (interrupt_writer)
        auto video_ptr = make_shared<dcp::ArrayData>(video.data(), video.size());
        auto audio = make_shared<AudioBuffers>(6, 48000 / 24);
 
-       auto writer = make_shared<Writer>(film, shared_ptr<Job>());
+       auto writer = make_shared<Writer>(film, shared_ptr<Job>(), film->dir(film->dcp_name()));
        writer->start ();
 
        for (int i = 0; i < frames; ++i) {
@@ -92,7 +92,7 @@ BOOST_AUTO_TEST_CASE (interrupt_writer)
 
        /* Start digest calculations then abort them; there should be no crash or error */
        boost::thread thread([film, writer]() {
-               writer->finish(film->dir(film->dcp_name()));
+               writer->finish();
        });
 
        dcpomatic_sleep_seconds (1);