Allow multiple DCPs to be written to a disk (#1756).
authorCarl Hetherington <cth@carlh.net>
Tue, 19 Jul 2022 22:51:23 +0000 (00:51 +0200)
committerCarl Hetherington <cth@carlh.net>
Wed, 20 Jul 2022 08:18:17 +0000 (10:18 +0200)
src/lib/copy_to_drive_job.cc
src/lib/copy_to_drive_job.h
src/lib/ext.cc
src/lib/ext.h
src/tools/dcpomatic_disk.cc
src/tools/dcpomatic_disk_writer.cc
test/disk_writer_test.cc

index 9ddedfe62dfc8c946f38a1fd7a65755bf8bccd7d..c6f9c84b0cea67867ad9f6cc8835a6aae4cfea16 100644 (file)
 
 */
 
-#include "disk_writer_messages.h"
-#include "copy_to_drive_job.h"
+
 #include "compose.hpp"
-#include "exceptions.h"
+#include "copy_to_drive_job.h"
 #include "dcpomatic_log.h"
+#include "disk_writer_messages.h"
+#include "exceptions.h"
 #include <dcp/raw_convert.h>
 #include <nanomsg/nn.h>
-#include <unistd.h>
 #include <fcntl.h>
+#include <unistd.h>
+#include <iostream>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <iostream>
 
 #include "i18n.h"
 
-using std::string;
+
 using std::cout;
 using std::min;
 using std::shared_ptr;
+using std::string;
 using boost::optional;
 using dcp::raw_convert;
 
-CopyToDriveJob::CopyToDriveJob (boost::filesystem::path dcp, Drive drive, Nanomsg& nanomsg)
+
+CopyToDriveJob::CopyToDriveJob(std::vector<boost::filesystem::path> const& dcps, Drive drive, Nanomsg& nanomsg)
        : Job (shared_ptr<Film>())
-       , _dcp (dcp)
+       , _dcps (dcps)
        , _drive (drive)
        , _nanomsg (nanomsg)
 {
 
 }
 
+
 string
 CopyToDriveJob::name () const
 {
-       return String::compose (_("Copying %1\nto %2"), _dcp.filename().string(), _drive.description());
+       if (_dcps.size() == 1) {
+               return String::compose(_("Copying %1\nto %2"), _dcps[0].filename().string(), _drive.description());
+       }
+
+       return String::compose(_("Copying DCPs to %1"), _drive.description());
 }
 
+
 string
 CopyToDriveJob::json_name () const
 {
@@ -65,8 +74,17 @@ CopyToDriveJob::json_name () const
 void
 CopyToDriveJob::run ()
 {
-       LOG_DISK("Sending write request to disk writer for %1 %2", _dcp.string(), _drive.device());
-       if (!_nanomsg.send(String::compose(DISK_WRITER_WRITE "\n%1\n%2\n", _dcp.string(), _drive.device()), 2000)) {
+       LOG_DISK("Sending write requests to disk %1 for:", _drive.device());
+       for (auto dcp: _dcps) {
+               LOG_DISK("%1", dcp.string());
+       }
+
+       string request = String::compose(DISK_WRITER_WRITE "\n%1\n", _drive.device());
+       for (auto dcp: _dcps) {
+               request += String::compose("%1\n", dcp.string());
+       }
+       request += "\n";
+       if (!_nanomsg.send(request, 2000)) {
                LOG_DISK_NC("Failed to send write request.");
                throw CommunicationFailedError ();
        }
index cb91195c26208fecf0b9ea0cd2c647252ea88752..8616a05d22ffe3f6fe8be9e9f6f8e0067e736fa6 100644 (file)
@@ -25,7 +25,7 @@
 class CopyToDriveJob : public Job
 {
 public:
-       CopyToDriveJob (boost::filesystem::path dcp, Drive drive, Nanomsg& nanomsg);
+       CopyToDriveJob (std::vector<boost::filesystem::path> const& dcps, Drive drive, Nanomsg& nanomsg);
 
        std::string name () const override;
        std::string json_name () const override;
@@ -37,7 +37,7 @@ public:
 private:
        void count (boost::filesystem::path dir, uint64_t& total_bytes);
        void copy (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_remaining, uint64_t total);
-       boost::filesystem::path _dcp;
+       std::vector<boost::filesystem::path> _dcps;
        Drive _drive;
        Nanomsg& _nanomsg;
 };
index 5e2ff7d9cbd93a4463f4c5f05ae661a7f955b7b6..49e63d6481c1c6300f5d6927aa4e6b0776fc80b5 100644 (file)
@@ -78,16 +78,18 @@ uint64_t constexpr block_size = 4096 * 4096;
 
 static
 void
-count (boost::filesystem::path dir, uint64_t& total_bytes)
+count (std::vector<boost::filesystem::path> dirs, uint64_t& total_bytes)
 {
-       dir = dcp::fix_long_path (dir);
-
        using namespace boost::filesystem;
-       for (auto i: directory_iterator(dir)) {
-               if (is_directory(i)) {
-                       count (i, total_bytes);
-               } else {
-                       total_bytes += file_size (i);
+
+       for (auto dir: dirs) {
+               dir = dcp::fix_long_path(dir);
+               for (auto path: directory_iterator(dir)) {
+                       if (is_directory(path)) {
+                               count({path}, total_bytes);
+                       } else {
+                               total_bytes += file_size(path);
+                       }
                }
        }
 }
@@ -277,9 +279,9 @@ format_progress (void* context, float progress)
 
 void
 #ifdef DCPOMATIC_WINDOWS
-dcpomatic::write (boost::filesystem::path dcp_path, string device, string, Nanomsg* nanomsg)
+dcpomatic::write (vector<boost::filesystem::path> dcp_paths, string device, string, Nanomsg* nanomsg)
 #else
-dcpomatic::write (boost::filesystem::path dcp_path, string device, string posix_partition, Nanomsg* nanomsg)
+dcpomatic::write (vector<boost::filesystem::path> dcp_paths, string device, string posix_partition, Nanomsg* nanomsg)
 #endif
 try
 {
@@ -390,11 +392,13 @@ try
        LOG_DISK_NC ("Mounted device");
 
        uint64_t total_bytes = 0;
-       count (dcp_path, total_bytes);
+       count (dcp_paths, total_bytes);
 
        uint64_t total_remaining = total_bytes;
        vector<CopiedFile> copied_files;
-       copy (dcp_path, "/mp", total_remaining, total_bytes, copied_files, nanomsg);
+       for (auto dcp_path: dcp_paths) {
+               copy (dcp_path, "/mp", total_remaining, total_bytes, copied_files, nanomsg);
+       }
 
        /* Unmount and re-mount to make sure the write has finished */
        r = ext4_umount("/mp/");
index 7c8afd58f7383ee4575bd4bc585eaaefa1a8bac9..367a71ba944df120e2dd9fa743df59dadd781ff6 100644 (file)
@@ -29,7 +29,7 @@ class Nanomsg;
 namespace dcpomatic {
 
 
-extern void write (boost::filesystem::path dcp_path, std::string device, std::string posix_partition, Nanomsg* nanomsg);
+extern void write (std::vector<boost::filesystem::path> dcp_paths, std::string device, std::string posix_partition, Nanomsg* nanomsg);
 
 
 }
index e4cc6c264e65df68d1e4d5c33934c853af075f83..a517dde65203a4d7dcbb28b0ec4864d092f13891 100644 (file)
@@ -21,6 +21,7 @@
 
 #include "wx/disk_warning_dialog.h"
 #include "wx/drive_wipe_warning_dialog.h"
+#include "wx/editable_list.h"
 #include "wx/job_manager_view.h"
 #include "wx/message_dialog.h"
 #include "wx/try_unmount_dialog.h"
@@ -57,6 +58,7 @@ using std::exception;
 using std::make_shared;
 using std::shared_ptr;
 using std::string;
+using std::vector;
 using boost::optional;
 #if BOOST_VERSION >= 106100
 using namespace boost::placeholders;
@@ -70,6 +72,33 @@ enum {
 #endif
 
 
+class DirDialogWrapper : public wxDirDialog
+{
+public:
+       DirDialogWrapper (wxWindow* parent)
+               : wxDirDialog (parent, _("Choose a DCP folder"), wxT(""), wxDD_DIR_MUST_EXIST)
+       {
+
+       }
+
+       boost::optional<boost::filesystem::path> get () const
+       {
+               auto const dcp = boost::filesystem::path(wx_to_std(GetPath()));
+               if (!boost::filesystem::exists(dcp / "ASSETMAP") && !boost::filesystem::exists(dcp / "ASSETMAP.xml")) {
+                       error_dialog (nullptr, _("No ASSETMAP or ASSETMAP.xml found in this folder.  Please choose a DCP folder."));
+                       return {};
+               }
+
+               return dcp;
+       }
+
+       void set (boost::filesystem::path)
+       {
+               /* Not used */
+       }
+};
+
+
 class DOMFrame : public wxFrame
 {
 public:
@@ -99,12 +128,19 @@ public:
 
                int r = 0;
                add_label_to_sizer (grid, overall_panel, _("DCP"), true, wxGBPosition(r, 0));
-               auto dcp_name_sizer = new wxBoxSizer (wxHORIZONTAL);
-               _dcp_name = new wxStaticText (overall_panel, wxID_ANY, wxEmptyString);
-               dcp_name_sizer->Add (_dcp_name, 1, wxALIGN_CENTER_VERTICAL | wxRIGHT, DCPOMATIC_SIZER_X_GAP);
-               _dcp_open = new wxButton (overall_panel, wxID_ANY, _("Open..."));
-               dcp_name_sizer->Add (_dcp_open, 0);
-               grid->Add (dcp_name_sizer, wxGBPosition(r, 1), wxDefaultSpan, wxEXPAND);
+               auto dcp_sizer = new wxBoxSizer (wxHORIZONTAL);
+               auto dcps = new EditableList<boost::filesystem::path, DirDialogWrapper>(
+                       overall_panel,
+                       { EditableListColumn(_("DCP"), 300, true) },
+                       boost::bind(&DOMFrame::dcp_paths, this),
+                       boost::bind(&DOMFrame::set_dcp_paths, this, _1),
+                       [](boost::filesystem::path p, int) { return p.filename().string(); },
+                       false,
+                       EditableListButton::NEW | EditableListButton::REMOVE
+                       );
+
+               dcp_sizer->Add(dcps, 1, wxALIGN_CENTER_VERTICAL, DCPOMATIC_SIZER_X_GAP);
+               grid->Add(dcp_sizer, wxGBPosition(r, 1), wxDefaultSpan, wxEXPAND);
                ++r;
 
                add_label_to_sizer (grid, overall_panel, _("Drive"), true, wxGBPosition(r, 0));
@@ -126,7 +162,6 @@ public:
 
                grid->AddGrowableCol (1);
 
-               _dcp_open->Bind (wxEVT_BUTTON, boost::bind(&DOMFrame::open, this));
                _copy->Bind (wxEVT_BUTTON, boost::bind(&DOMFrame::copy, this));
                _drive->Bind (wxEVT_CHOICE, boost::bind(&DOMFrame::setup_sensitivity, this));
                _drive_refresh->Bind (wxEVT_BUTTON, boost::bind(&DOMFrame::drive_refresh, this));
@@ -180,18 +215,18 @@ public:
                dcpomatic_sleep_seconds (1);
        }
 
-       void set_dcp (boost::filesystem::path dcp)
+       void set_dcp_paths (vector<boost::filesystem::path> dcps)
        {
-               if (!boost::filesystem::exists(dcp / "ASSETMAP") && !boost::filesystem::exists(dcp / "ASSETMAP.xml")) {
-                       error_dialog (nullptr, _("No ASSETMAP or ASSETMAP.xml found in this folder.  Please choose a DCP folder."));
-                       return;
-               }
-
-               _dcp_path = dcp;
-               _dcp_name->SetLabel (std_to_wx(dcp.filename().string()));
+               _dcp_paths = dcps;
+               setup_sensitivity();
        }
 
 private:
+       vector<boost::filesystem::path> dcp_paths() const
+       {
+               return _dcp_paths;
+       }
+
        void sized (wxSizeEvent& ev)
        {
                _sizer->Layout ();
@@ -236,22 +271,6 @@ private:
                ev.Skip ();
        }
 
-
-       void open ()
-       {
-               auto d = new wxDirDialog (this, _("Choose a DCP folder"), wxT(""), wxDD_DIR_MUST_EXIST);
-               int r = d->ShowModal ();
-               boost::filesystem::path const path (wx_to_std(d->GetPath()));
-               d->Destroy ();
-
-               if (r != wxID_OK) {
-                       return;
-               }
-
-               set_dcp (path);
-               setup_sensitivity ();
-       }
-
        void copy ()
        {
                /* Check that the selected drive still exists and update its properties if so */
@@ -262,7 +281,7 @@ private:
                }
 
                DCPOMATIC_ASSERT (_drive->GetSelection() != wxNOT_FOUND);
-               DCPOMATIC_ASSERT (static_cast<bool>(_dcp_path));
+               DCPOMATIC_ASSERT (!_dcp_paths.empty());
 
                auto ping = [this](int attempt) {
                        if (_nanomsg.send(DISK_WRITER_PING "\n", 1000)) {
@@ -356,7 +375,7 @@ private:
                        return;
                }
 
-               JobManager::instance()->add(make_shared<CopyToDriveJob>(*_dcp_path, _drives[_drive->GetSelection()], _nanomsg));
+               JobManager::instance()->add(make_shared<CopyToDriveJob>(_dcp_paths, _drives[_drive->GetSelection()], _nanomsg));
                setup_sensitivity ();
        }
 
@@ -385,16 +404,14 @@ private:
 
        void setup_sensitivity ()
        {
-               _copy->Enable (static_cast<bool>(_dcp_path) && _drive->GetSelection() != wxNOT_FOUND && !JobManager::instance()->work_to_do());
+               _copy->Enable (!_dcp_paths.empty() && _drive->GetSelection() != wxNOT_FOUND && !JobManager::instance()->work_to_do());
        }
 
-       wxStaticText* _dcp_name;
-       wxButton* _dcp_open;
        wxChoice* _drive;
        wxButton* _drive_refresh;
        wxButton* _copy;
        JobManagerView* _jobs;
-       boost::optional<boost::filesystem::path> _dcp_path;
+       std::vector<boost::filesystem::path> _dcp_paths;
        std::vector<Drive> _drives;
 #ifndef DCPOMATIC_OSX
        boost::process::child* _writer;
@@ -474,7 +491,7 @@ public:
                        _frame->Show ();
 
                        if (_dcp_to_write) {
-                               _frame->set_dcp (*_dcp_to_write);
+                               _frame->set_dcp_paths({*_dcp_to_write});
                        }
 
                        signal_manager = new wxSignalManager (this);
index dca09f729ad0ee37bbd4e12906ceac98d4fe146a..ef3bf2f776c86936b6ad443e26b3d717eca1d26b 100644 (file)
@@ -195,16 +195,27 @@ try
                                }
                        });
        } else if (*s == DISK_WRITER_WRITE) {
-               auto dcp_path_opt = nanomsg->receive (LONG_TIMEOUT);
                auto device_opt = nanomsg->receive (LONG_TIMEOUT);
-               if (!dcp_path_opt || !device_opt) {
+               if (!device_opt) {
                        LOG_DISK_NC("Failed to receive write request");
                        throw CommunicationFailedError();
                }
-
-               auto dcp_path = *dcp_path_opt;
                auto device = *device_opt;
 
+               vector<boost::filesystem::path> dcp_paths;
+               while (true) {
+                       auto dcp_path_opt = nanomsg->receive (LONG_TIMEOUT);
+                       if (!dcp_path_opt) {
+                               LOG_DISK_NC("Failed to receive write request");
+                               throw CommunicationFailedError();
+                       }
+                       if (*dcp_path_opt != "") {
+                               dcp_paths.push_back(*dcp_path_opt);
+                       } else {
+                               break;
+                       }
+               }
+
                /* Do some basic sanity checks; this is a bit belt-and-braces but it can't hurt... */
 
 #ifdef DCPOMATIC_OSX
@@ -249,11 +260,14 @@ try
                        return true;
                }
 
-               LOG_DISK ("Here we go writing %1 to %2", dcp_path, device);
+               LOG_DISK("Here we go writing these to %1", device);
+               for (auto dcp: dcp_paths) {
+                       LOG_DISK("  %1", dcp);
+               }
 
                request_privileges (
                        "com.dcpomatic.write-drive",
-                       [dcp_path, device]() {
+                       [dcp_paths, device]() {
 #if defined(DCPOMATIC_LINUX)
                                auto posix_partition = device;
                                /* XXX: don't know if this logic is sensible */
@@ -262,12 +276,12 @@ try
                                } else {
                                        posix_partition += "1";
                                }
-                               dcpomatic::write (dcp_path, device, posix_partition, nanomsg);
+                               dcpomatic::write (dcp_paths, device, posix_partition, nanomsg);
 #elif defined(DCPOMATIC_OSX)
                                auto fast_device = boost::algorithm::replace_first_copy (device, "/dev/disk", "/dev/rdisk");
-                               dcpomatic::write (dcp_path, fast_device, fast_device + "s1", nanomsg);
+                               dcpomatic::write (dcp_paths, fast_device, fast_device + "s1", nanomsg);
 #elif defined(DCPOMATIC_WINDOWS)
-                               dcpomatic::write (dcp_path, device, "", nanomsg);
+                               dcpomatic::write (dcp_paths, device, "", nanomsg);
 #endif
                        },
                        []() {
index dae991e5870ba34f0bf0ea2928cd844df14e6a23..553adcae7188693f8c330a0b7018e6c0f1894a43 100644 (file)
@@ -100,7 +100,7 @@ BOOST_AUTO_TEST_CASE (disk_writer_test1)
        /* Some arbitrary file size here */
        make_random_file (dcp / "foo", 1024 * 1024 * 32 - 6128);
 
-       dcpomatic::write (dcp, disk.string(), partition.string(), nullptr);
+       dcpomatic::write ({dcp}, disk.string(), partition.string(), nullptr);
 
        BOOST_CHECK_EQUAL (system("/sbin/e2fsck -fn build/test/disk_writer_test1.partition"), 0);
 
@@ -164,7 +164,7 @@ BOOST_AUTO_TEST_CASE (disk_writer_test2)
        make_empty_file(partition, 31043571712LL);
 
        auto const dcp = TestPaths::private_data() / "xm";
-       dcpomatic::write(dcp, disk.string(), partition.string(), nullptr);
+       dcpomatic::write({dcp}, disk.string(), partition.string(), nullptr);
 
        BOOST_CHECK_EQUAL(system("/sbin/e2fsck -fn build/test/disk_writer_test2.partition"), 0);
 
@@ -183,6 +183,53 @@ BOOST_AUTO_TEST_CASE (disk_writer_test2)
 }
 
 
+
+BOOST_AUTO_TEST_CASE (disk_writer_test3)
+{
+       using namespace boost::filesystem;
+       using namespace boost::process;
+
+       remove_all("build/test/disk_writer_test3.disk");
+       remove_all("build/test/disk_writer_test3.partition");
+       remove_all("build/test/disk_writer_test3");
+
+       Cleanup cl;
+
+       path const disk = "build/test/disk_writer_test3.disk";
+       path const partition = "build/test/disk_writer_test3.partition";
+
+       cl.add(disk);
+       cl.add(partition);
+
+       /* Using empty files here still triggers the bug and is much quicker than using random data */
+       make_empty_file(disk,      31043616768LL);
+       make_empty_file(partition, 31043571712LL);
+
+       vector<boost::filesystem::path> const dcps = {
+               TestPaths::private_data() / "xm",
+               TestPaths::private_data() / "JourneyToJah_TLR-1_F_EN-DE-FR_CH_51_2K_LOK_20140225_DGL_SMPTE_OV"
+       };
+       dcpomatic::write(dcps, disk.string(), partition.string(), nullptr);
+
+       BOOST_CHECK_EQUAL(system("/sbin/e2fsck -fn build/test/disk_writer_test3.partition"), 0);
+
+       path const check = "build/test/disk_writer_test3";
+       create_directory(check);
+       cl.add(check);
+
+       for (auto dcp: dcps) {
+               for (auto original: directory_iterator(dcp)) {
+                       auto path_in_copy = dcp.filename() / original.path().filename();
+                       auto path_in_check = check / original.path().filename();
+                       system("e2cp " + partition.string() + ":" + path_in_copy.string() + " " + path_in_check.string());
+                       check_file(original.path(), path_in_check);
+               }
+       }
+
+       cl.run();
+}
+
+
 BOOST_AUTO_TEST_CASE (osx_drive_identification_test)
 {
        vector<OSXDisk> disks;