From 4bdaccb5ede7d9a1066e0a0665fcfce9b1e3241e Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 20 Jul 2022 00:51:23 +0200 Subject: [PATCH] Allow multiple DCPs to be written to a disk (#1756). --- src/lib/copy_to_drive_job.cc | 40 +++++++++---- src/lib/copy_to_drive_job.h | 4 +- src/lib/ext.cc | 28 +++++---- src/lib/ext.h | 2 +- src/tools/dcpomatic_disk.cc | 93 ++++++++++++++++++------------ src/tools/dcpomatic_disk_writer.cc | 32 +++++++--- test/disk_writer_test.cc | 51 +++++++++++++++- 7 files changed, 175 insertions(+), 75 deletions(-) diff --git a/src/lib/copy_to_drive_job.cc b/src/lib/copy_to_drive_job.cc index 9ddedfe62..c6f9c84b0 100644 --- a/src/lib/copy_to_drive_job.cc +++ b/src/lib/copy_to_drive_job.cc @@ -18,44 +18,53 @@ */ -#include "disk_writer_messages.h" -#include "copy_to_drive_job.h" + #include "compose.hpp" -#include "exceptions.h" +#include "copy_to_drive_job.h" #include "dcpomatic_log.h" +#include "disk_writer_messages.h" +#include "exceptions.h" #include #include -#include #include +#include +#include #include #include #include -#include #include "i18n.h" -using std::string; + using std::cout; using std::min; using std::shared_ptr; +using std::string; using boost::optional; using dcp::raw_convert; -CopyToDriveJob::CopyToDriveJob (boost::filesystem::path dcp, Drive drive, Nanomsg& nanomsg) + +CopyToDriveJob::CopyToDriveJob(std::vector const& dcps, Drive drive, Nanomsg& nanomsg) : Job (shared_ptr()) - , _dcp (dcp) + , _dcps (dcps) , _drive (drive) , _nanomsg (nanomsg) { } + string CopyToDriveJob::name () const { - return String::compose (_("Copying %1\nto %2"), _dcp.filename().string(), _drive.description()); + if (_dcps.size() == 1) { + return String::compose(_("Copying %1\nto %2"), _dcps[0].filename().string(), _drive.description()); + } + + return String::compose(_("Copying DCPs to %1"), _drive.description()); } + string CopyToDriveJob::json_name () const { @@ -65,8 +74,17 @@ CopyToDriveJob::json_name () const void CopyToDriveJob::run () { - LOG_DISK("Sending write request to disk writer for %1 %2", _dcp.string(), _drive.device()); - if (!_nanomsg.send(String::compose(DISK_WRITER_WRITE "\n%1\n%2\n", _dcp.string(), _drive.device()), 2000)) { + LOG_DISK("Sending write requests to disk %1 for:", _drive.device()); + for (auto dcp: _dcps) { + LOG_DISK("%1", dcp.string()); + } + + string request = String::compose(DISK_WRITER_WRITE "\n%1\n", _drive.device()); + for (auto dcp: _dcps) { + request += String::compose("%1\n", dcp.string()); + } + request += "\n"; + if (!_nanomsg.send(request, 2000)) { LOG_DISK_NC("Failed to send write request."); throw CommunicationFailedError (); } diff --git a/src/lib/copy_to_drive_job.h b/src/lib/copy_to_drive_job.h index cb91195c2..8616a05d2 100644 --- a/src/lib/copy_to_drive_job.h +++ b/src/lib/copy_to_drive_job.h @@ -25,7 +25,7 @@ class CopyToDriveJob : public Job { public: - CopyToDriveJob (boost::filesystem::path dcp, Drive drive, Nanomsg& nanomsg); + CopyToDriveJob (std::vector const& dcps, Drive drive, Nanomsg& nanomsg); std::string name () const override; std::string json_name () const override; @@ -37,7 +37,7 @@ public: private: void count (boost::filesystem::path dir, uint64_t& total_bytes); void copy (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_remaining, uint64_t total); - boost::filesystem::path _dcp; + std::vector _dcps; Drive _drive; Nanomsg& _nanomsg; }; diff --git a/src/lib/ext.cc b/src/lib/ext.cc index 5e2ff7d9c..49e63d648 100644 --- a/src/lib/ext.cc +++ b/src/lib/ext.cc @@ -78,16 +78,18 @@ uint64_t constexpr block_size = 4096 * 4096; static void -count (boost::filesystem::path dir, uint64_t& total_bytes) +count (std::vector dirs, uint64_t& total_bytes) { - dir = dcp::fix_long_path (dir); - using namespace boost::filesystem; - for (auto i: directory_iterator(dir)) { - if (is_directory(i)) { - count (i, total_bytes); - } else { - total_bytes += file_size (i); + + for (auto dir: dirs) { + dir = dcp::fix_long_path(dir); + for (auto path: directory_iterator(dir)) { + if (is_directory(path)) { + count({path}, total_bytes); + } else { + total_bytes += file_size(path); + } } } } @@ -277,9 +279,9 @@ format_progress (void* context, float progress) void #ifdef DCPOMATIC_WINDOWS -dcpomatic::write (boost::filesystem::path dcp_path, string device, string, Nanomsg* nanomsg) +dcpomatic::write (vector dcp_paths, string device, string, Nanomsg* nanomsg) #else -dcpomatic::write (boost::filesystem::path dcp_path, string device, string posix_partition, Nanomsg* nanomsg) +dcpomatic::write (vector dcp_paths, string device, string posix_partition, Nanomsg* nanomsg) #endif try { @@ -390,11 +392,13 @@ try LOG_DISK_NC ("Mounted device"); uint64_t total_bytes = 0; - count (dcp_path, total_bytes); + count (dcp_paths, total_bytes); uint64_t total_remaining = total_bytes; vector copied_files; - copy (dcp_path, "/mp", total_remaining, total_bytes, copied_files, nanomsg); + for (auto dcp_path: dcp_paths) { + copy (dcp_path, "/mp", total_remaining, total_bytes, copied_files, nanomsg); + } /* Unmount and re-mount to make sure the write has finished */ r = ext4_umount("/mp/"); diff --git a/src/lib/ext.h b/src/lib/ext.h index 7c8afd58f..367a71ba9 100644 --- a/src/lib/ext.h +++ b/src/lib/ext.h @@ -29,7 +29,7 @@ class Nanomsg; namespace dcpomatic { -extern void write (boost::filesystem::path dcp_path, std::string device, std::string posix_partition, Nanomsg* nanomsg); +extern void write (std::vector dcp_paths, std::string device, std::string posix_partition, Nanomsg* nanomsg); } diff --git a/src/tools/dcpomatic_disk.cc b/src/tools/dcpomatic_disk.cc index e4cc6c264..a517dde65 100644 --- a/src/tools/dcpomatic_disk.cc +++ b/src/tools/dcpomatic_disk.cc @@ -21,6 +21,7 @@ #include "wx/disk_warning_dialog.h" #include "wx/drive_wipe_warning_dialog.h" +#include "wx/editable_list.h" #include "wx/job_manager_view.h" #include "wx/message_dialog.h" #include "wx/try_unmount_dialog.h" @@ -57,6 +58,7 @@ using std::exception; using std::make_shared; using std::shared_ptr; using std::string; +using std::vector; using boost::optional; #if BOOST_VERSION >= 106100 using namespace boost::placeholders; @@ -70,6 +72,33 @@ enum { #endif +class DirDialogWrapper : public wxDirDialog +{ +public: + DirDialogWrapper (wxWindow* parent) + : wxDirDialog (parent, _("Choose a DCP folder"), wxT(""), wxDD_DIR_MUST_EXIST) + { + + } + + boost::optional get () const + { + auto const dcp = boost::filesystem::path(wx_to_std(GetPath())); + if (!boost::filesystem::exists(dcp / "ASSETMAP") && !boost::filesystem::exists(dcp / "ASSETMAP.xml")) { + error_dialog (nullptr, _("No ASSETMAP or ASSETMAP.xml found in this folder. Please choose a DCP folder.")); + return {}; + } + + return dcp; + } + + void set (boost::filesystem::path) + { + /* Not used */ + } +}; + + class DOMFrame : public wxFrame { public: @@ -99,12 +128,19 @@ public: int r = 0; add_label_to_sizer (grid, overall_panel, _("DCP"), true, wxGBPosition(r, 0)); - auto dcp_name_sizer = new wxBoxSizer (wxHORIZONTAL); - _dcp_name = new wxStaticText (overall_panel, wxID_ANY, wxEmptyString); - dcp_name_sizer->Add (_dcp_name, 1, wxALIGN_CENTER_VERTICAL | wxRIGHT, DCPOMATIC_SIZER_X_GAP); - _dcp_open = new wxButton (overall_panel, wxID_ANY, _("Open...")); - dcp_name_sizer->Add (_dcp_open, 0); - grid->Add (dcp_name_sizer, wxGBPosition(r, 1), wxDefaultSpan, wxEXPAND); + auto dcp_sizer = new wxBoxSizer (wxHORIZONTAL); + auto dcps = new EditableList( + overall_panel, + { EditableListColumn(_("DCP"), 300, true) }, + boost::bind(&DOMFrame::dcp_paths, this), + boost::bind(&DOMFrame::set_dcp_paths, this, _1), + [](boost::filesystem::path p, int) { return p.filename().string(); }, + false, + EditableListButton::NEW | EditableListButton::REMOVE + ); + + dcp_sizer->Add(dcps, 1, wxALIGN_CENTER_VERTICAL, DCPOMATIC_SIZER_X_GAP); + grid->Add(dcp_sizer, wxGBPosition(r, 1), wxDefaultSpan, wxEXPAND); ++r; add_label_to_sizer (grid, overall_panel, _("Drive"), true, wxGBPosition(r, 0)); @@ -126,7 +162,6 @@ public: grid->AddGrowableCol (1); - _dcp_open->Bind (wxEVT_BUTTON, boost::bind(&DOMFrame::open, this)); _copy->Bind (wxEVT_BUTTON, boost::bind(&DOMFrame::copy, this)); _drive->Bind (wxEVT_CHOICE, boost::bind(&DOMFrame::setup_sensitivity, this)); _drive_refresh->Bind (wxEVT_BUTTON, boost::bind(&DOMFrame::drive_refresh, this)); @@ -180,18 +215,18 @@ public: dcpomatic_sleep_seconds (1); } - void set_dcp (boost::filesystem::path dcp) + void set_dcp_paths (vector dcps) { - if (!boost::filesystem::exists(dcp / "ASSETMAP") && !boost::filesystem::exists(dcp / "ASSETMAP.xml")) { - error_dialog (nullptr, _("No ASSETMAP or ASSETMAP.xml found in this folder. Please choose a DCP folder.")); - return; - } - - _dcp_path = dcp; - _dcp_name->SetLabel (std_to_wx(dcp.filename().string())); + _dcp_paths = dcps; + setup_sensitivity(); } private: + vector dcp_paths() const + { + return _dcp_paths; + } + void sized (wxSizeEvent& ev) { _sizer->Layout (); @@ -236,22 +271,6 @@ private: ev.Skip (); } - - void open () - { - auto d = new wxDirDialog (this, _("Choose a DCP folder"), wxT(""), wxDD_DIR_MUST_EXIST); - int r = d->ShowModal (); - boost::filesystem::path const path (wx_to_std(d->GetPath())); - d->Destroy (); - - if (r != wxID_OK) { - return; - } - - set_dcp (path); - setup_sensitivity (); - } - void copy () { /* Check that the selected drive still exists and update its properties if so */ @@ -262,7 +281,7 @@ private: } DCPOMATIC_ASSERT (_drive->GetSelection() != wxNOT_FOUND); - DCPOMATIC_ASSERT (static_cast(_dcp_path)); + DCPOMATIC_ASSERT (!_dcp_paths.empty()); auto ping = [this](int attempt) { if (_nanomsg.send(DISK_WRITER_PING "\n", 1000)) { @@ -356,7 +375,7 @@ private: return; } - JobManager::instance()->add(make_shared(*_dcp_path, _drives[_drive->GetSelection()], _nanomsg)); + JobManager::instance()->add(make_shared(_dcp_paths, _drives[_drive->GetSelection()], _nanomsg)); setup_sensitivity (); } @@ -385,16 +404,14 @@ private: void setup_sensitivity () { - _copy->Enable (static_cast(_dcp_path) && _drive->GetSelection() != wxNOT_FOUND && !JobManager::instance()->work_to_do()); + _copy->Enable (!_dcp_paths.empty() && _drive->GetSelection() != wxNOT_FOUND && !JobManager::instance()->work_to_do()); } - wxStaticText* _dcp_name; - wxButton* _dcp_open; wxChoice* _drive; wxButton* _drive_refresh; wxButton* _copy; JobManagerView* _jobs; - boost::optional _dcp_path; + std::vector _dcp_paths; std::vector _drives; #ifndef DCPOMATIC_OSX boost::process::child* _writer; @@ -474,7 +491,7 @@ public: _frame->Show (); if (_dcp_to_write) { - _frame->set_dcp (*_dcp_to_write); + _frame->set_dcp_paths({*_dcp_to_write}); } signal_manager = new wxSignalManager (this); diff --git a/src/tools/dcpomatic_disk_writer.cc b/src/tools/dcpomatic_disk_writer.cc index dca09f729..ef3bf2f77 100644 --- a/src/tools/dcpomatic_disk_writer.cc +++ b/src/tools/dcpomatic_disk_writer.cc @@ -195,16 +195,27 @@ try } }); } else if (*s == DISK_WRITER_WRITE) { - auto dcp_path_opt = nanomsg->receive (LONG_TIMEOUT); auto device_opt = nanomsg->receive (LONG_TIMEOUT); - if (!dcp_path_opt || !device_opt) { + if (!device_opt) { LOG_DISK_NC("Failed to receive write request"); throw CommunicationFailedError(); } - - auto dcp_path = *dcp_path_opt; auto device = *device_opt; + vector dcp_paths; + while (true) { + auto dcp_path_opt = nanomsg->receive (LONG_TIMEOUT); + if (!dcp_path_opt) { + LOG_DISK_NC("Failed to receive write request"); + throw CommunicationFailedError(); + } + if (*dcp_path_opt != "") { + dcp_paths.push_back(*dcp_path_opt); + } else { + break; + } + } + /* Do some basic sanity checks; this is a bit belt-and-braces but it can't hurt... */ #ifdef DCPOMATIC_OSX @@ -249,11 +260,14 @@ try return true; } - LOG_DISK ("Here we go writing %1 to %2", dcp_path, device); + LOG_DISK("Here we go writing these to %1", device); + for (auto dcp: dcp_paths) { + LOG_DISK(" %1", dcp); + } request_privileges ( "com.dcpomatic.write-drive", - [dcp_path, device]() { + [dcp_paths, device]() { #if defined(DCPOMATIC_LINUX) auto posix_partition = device; /* XXX: don't know if this logic is sensible */ @@ -262,12 +276,12 @@ try } else { posix_partition += "1"; } - dcpomatic::write (dcp_path, device, posix_partition, nanomsg); + dcpomatic::write (dcp_paths, device, posix_partition, nanomsg); #elif defined(DCPOMATIC_OSX) auto fast_device = boost::algorithm::replace_first_copy (device, "/dev/disk", "/dev/rdisk"); - dcpomatic::write (dcp_path, fast_device, fast_device + "s1", nanomsg); + dcpomatic::write (dcp_paths, fast_device, fast_device + "s1", nanomsg); #elif defined(DCPOMATIC_WINDOWS) - dcpomatic::write (dcp_path, device, "", nanomsg); + dcpomatic::write (dcp_paths, device, "", nanomsg); #endif }, []() { diff --git a/test/disk_writer_test.cc b/test/disk_writer_test.cc index dae991e58..553adcae7 100644 --- a/test/disk_writer_test.cc +++ b/test/disk_writer_test.cc @@ -100,7 +100,7 @@ BOOST_AUTO_TEST_CASE (disk_writer_test1) /* Some arbitrary file size here */ make_random_file (dcp / "foo", 1024 * 1024 * 32 - 6128); - dcpomatic::write (dcp, disk.string(), partition.string(), nullptr); + dcpomatic::write ({dcp}, disk.string(), partition.string(), nullptr); BOOST_CHECK_EQUAL (system("/sbin/e2fsck -fn build/test/disk_writer_test1.partition"), 0); @@ -164,7 +164,7 @@ BOOST_AUTO_TEST_CASE (disk_writer_test2) make_empty_file(partition, 31043571712LL); auto const dcp = TestPaths::private_data() / "xm"; - dcpomatic::write(dcp, disk.string(), partition.string(), nullptr); + dcpomatic::write({dcp}, disk.string(), partition.string(), nullptr); BOOST_CHECK_EQUAL(system("/sbin/e2fsck -fn build/test/disk_writer_test2.partition"), 0); @@ -183,6 +183,53 @@ BOOST_AUTO_TEST_CASE (disk_writer_test2) } + +BOOST_AUTO_TEST_CASE (disk_writer_test3) +{ + using namespace boost::filesystem; + using namespace boost::process; + + remove_all("build/test/disk_writer_test3.disk"); + remove_all("build/test/disk_writer_test3.partition"); + remove_all("build/test/disk_writer_test3"); + + Cleanup cl; + + path const disk = "build/test/disk_writer_test3.disk"; + path const partition = "build/test/disk_writer_test3.partition"; + + cl.add(disk); + cl.add(partition); + + /* Using empty files here still triggers the bug and is much quicker than using random data */ + make_empty_file(disk, 31043616768LL); + make_empty_file(partition, 31043571712LL); + + vector const dcps = { + TestPaths::private_data() / "xm", + TestPaths::private_data() / "JourneyToJah_TLR-1_F_EN-DE-FR_CH_51_2K_LOK_20140225_DGL_SMPTE_OV" + }; + dcpomatic::write(dcps, disk.string(), partition.string(), nullptr); + + BOOST_CHECK_EQUAL(system("/sbin/e2fsck -fn build/test/disk_writer_test3.partition"), 0); + + path const check = "build/test/disk_writer_test3"; + create_directory(check); + cl.add(check); + + for (auto dcp: dcps) { + for (auto original: directory_iterator(dcp)) { + auto path_in_copy = dcp.filename() / original.path().filename(); + auto path_in_check = check / original.path().filename(); + system("e2cp " + partition.string() + ":" + path_in_copy.string() + " " + path_in_check.string()); + check_file(original.path(), path_in_check); + } + } + + cl.run(); +} + + BOOST_AUTO_TEST_CASE (osx_drive_identification_test) { vector disks; -- 2.30.2