X-Git-Url: https://git.carlh.net/gitweb/?p=dcpomatic.git;a=blobdiff_plain;f=src%2Flib%2Fext.cc;h=5e2ff7d9cbd93a4463f4c5f05ae661a7f955b7b6;hp=4aa8af5e87f30ec02d72065e971ec7315a9d288f;hb=fe933ebb2c55b4235fcba5d02af2ba91c272fc88;hpb=d4f0bd6523854be0e03b6b14d00d8def3c1ce384 diff --git a/src/lib/ext.cc b/src/lib/ext.cc index 4aa8af5e8..5e2ff7d9c 100644 --- a/src/lib/ext.cc +++ b/src/lib/ext.cc @@ -27,6 +27,7 @@ #include "exceptions.h" #include "ext.h" #include "nanomsg.h" +#include #ifdef DCPOMATIC_LINUX #include @@ -57,7 +58,7 @@ extern "C" { #include } #include -#include +#include #include @@ -72,23 +73,37 @@ using std::vector; /* Use quite a big block size here, as ext4's fwrite() has quite a bit of overhead */ -static uint64_t const block_size = 4096 * 4096; +uint64_t constexpr block_size = 4096 * 4096; static void count (boost::filesystem::path dir, uint64_t& total_bytes) { + dir = dcp::fix_long_path (dir); + using namespace boost::filesystem; - for (directory_iterator i = directory_iterator(dir); i != directory_iterator(); ++i) { - if (is_directory(*i)) { - count (*i, total_bytes); + for (auto i: directory_iterator(dir)) { + if (is_directory(i)) { + count (i, total_bytes); } else { - total_bytes += file_size (*i); + total_bytes += file_size (i); } } } + +static +void +set_timestamps_to_now (boost::filesystem::path path) +{ + auto const now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + ext4_mtime_set (path.generic_string().c_str(), now); + ext4_ctime_set (path.generic_string().c_str(), now); + ext4_atime_set (path.generic_string().c_str(), now); +} + + static string write (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_remaining, uint64_t total, Nanomsg* nanomsg) @@ -99,13 +114,13 @@ write (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total throw CopyError (String::compose("Failed to open file %1", to.generic_string()), r); } - FILE* in = fopen_boost (from, "rb"); + dcp::File in(from, "rb"); if (!in) { ext4_fclose (&out); throw CopyError (String::compose("Failed to open file %1", from.string()), 0); } - uint8_t* buffer = new uint8_t[block_size]; + std::vector buffer(block_size); Digester digester; int progress_frequency = 1; @@ -113,42 +128,36 @@ write (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total uint64_t remaining = file_size (from); while (remaining > 0) { uint64_t const this_time = min(remaining, block_size); - size_t read = fread (buffer, 1, this_time, in); + size_t read = in.read(buffer.data(), 1, this_time); if (read != this_time) { - fclose (in); ext4_fclose (&out); - delete[] buffer; throw CopyError (String::compose("Short read; expected %1 but read %2", this_time, read), 0); } - digester.add (buffer, this_time); + digester.add (buffer.data(), this_time); size_t written; - r = ext4_fwrite (&out, buffer, this_time, &written); + r = ext4_fwrite (&out, buffer.data(), this_time, &written); if (r != EOK) { - fclose (in); ext4_fclose (&out); - delete[] buffer; throw CopyError ("Write failed", r); } if (written != this_time) { - fclose (in); ext4_fclose (&out); - delete[] buffer; throw CopyError (String::compose("Short write; expected %1 but wrote %2", this_time, written), 0); } remaining -= this_time; total_remaining -= this_time; ++progress_count; - if ((progress_count % progress_frequency) == 0) { + if ((progress_count % progress_frequency) == 0 && nanomsg) { nanomsg->send(String::compose(DISK_WRITER_COPY_PROGRESS "\n%1\n", (1 - float(total_remaining) / total)), SHORT_TIMEOUT); } } - fclose (in); ext4_fclose (&out); - delete[] buffer; + + set_timestamps_to_now (to); return digester.get (); } @@ -166,28 +175,28 @@ read (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_ } LOG_DISK("Opened %1 for read", to.generic_string()); - uint8_t* buffer = new uint8_t[block_size]; + std::vector buffer(block_size); Digester digester; uint64_t remaining = file_size (from); while (remaining > 0) { uint64_t const this_time = min(remaining, block_size); size_t read; - r = ext4_fread (&in, buffer, this_time, &read); + r = ext4_fread (&in, buffer.data(), this_time, &read); if (read != this_time) { ext4_fclose (&in); - delete[] buffer; throw VerifyError (String::compose("Short read; expected %1 but read %2", this_time, read), 0); } - digester.add (buffer, this_time); + digester.add (buffer.data(), this_time); remaining -= this_time; total_remaining -= this_time; - nanomsg->send(String::compose(DISK_WRITER_VERIFY_PROGRESS "\n%1\n", (1 - float(total_remaining) / total)), SHORT_TIMEOUT); + if (nanomsg) { + nanomsg->send(String::compose(DISK_WRITER_VERIFY_PROGRESS "\n%1\n", (1 - float(total_remaining) / total)), SHORT_TIMEOUT); + } } ext4_fclose (&in); - delete[] buffer; return digester.get (); } @@ -217,6 +226,7 @@ void copy (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_remaining, uint64_t total, vector& copied_files, Nanomsg* nanomsg) { LOG_DISK ("Copy %1 -> %2", from.string(), to.generic_string()); + from = dcp::fix_long_path (from); using namespace boost::filesystem; @@ -227,9 +237,10 @@ copy (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_ if (r != EOK) { throw CopyError (String::compose("Failed to create directory %1", cr.generic_string()), r); } + set_timestamps_to_now (cr); - for (directory_iterator i = directory_iterator(from); i != directory_iterator(); ++i) { - copy (i->path(), cr, total_remaining, total, copied_files, nanomsg); + for (auto i: directory_iterator(from)) { + copy (i.path(), cr, total_remaining, total, copied_files, nanomsg); } } else { string const write_digest = write (from, cr, total_remaining, total, nanomsg); @@ -244,7 +255,7 @@ void verify (vector const& copied_files, uint64_t total, Nanomsg* nanomsg) { uint64_t total_remaining = total; - BOOST_FOREACH (CopiedFile const& i, copied_files) { + for (auto const& i: copied_files) { string const read_digest = read (i.from, i.to, total_remaining, total, nanomsg); LOG_DISK ("Read %1 %2 was %3 on write, now %4", i.from.string(), i.to.generic_string(), i.write_digest, read_digest); if (read_digest != i.write_digest) { @@ -254,6 +265,16 @@ verify (vector const& copied_files, uint64_t total, Nanomsg* nanomsg } +static +void +format_progress (void* context, float progress) +{ + if (context) { + reinterpret_cast(context)->send(String::compose(DISK_WRITER_FORMAT_PROGRESS "\n%1\n", progress), SHORT_TIMEOUT); + } +} + + void #ifdef DCPOMATIC_WINDOWS dcpomatic::write (boost::filesystem::path dcp_path, string device, string, Nanomsg* nanomsg) @@ -264,12 +285,26 @@ try { ext4_dmask_set (DEBUG_ALL); - /* We rely on static initialization for these */ - static struct ext4_fs fs; - static struct ext4_mkfs_info info; + struct ext4_fs fs; + fs.read_only = false; + fs.bdev = nullptr; + fs.last_inode_bg_id = 0; + fs.jbd_fs = nullptr; + fs.jbd_journal = nullptr; + fs.curr_trans = nullptr; + struct ext4_mkfs_info info; + info.len = 0; info.block_size = 4096; + info.blocks_per_group = 0; info.inode_size = 128; + info.inodes = 0; + info.journal_blocks = 0; + info.dsc_size = 0; + for (int i = 0; i < UUID_SIZE; ++i) { + info.uuid[i] = rand() & 0xff; + } info.journal = false; + info.label = nullptr; #ifdef WIN32 file_windows_name_set(device.c_str()); @@ -290,10 +325,6 @@ try parts.division[2] = 0; parts.division[3] = 0; -#ifdef DCPOMATIC_LINUX - PrivilegeEscalator e; -#endif - /* XXX: not sure if disk_id matters */ int r = ext4_mbr_write (bd, &parts, 0); if (r) { @@ -320,6 +351,18 @@ try file_windows_partition_set (bdevs.partitions[0].part_offset, bdevs.partitions[0].part_size); #else file_dev_name_set (posix_partition.c_str()); + + /* On macOS (at least) if you try to write to a drive that is sleeping the ext4_mkfs call + * below is liable to return EIO because it can't open the device. Try to work around that + * here by opening and closing the device, waiting 5 seconds if it fails. + */ + int wake = open(posix_partition.c_str(), O_RDWR); + if (wake == -1) { + dcpomatic_sleep_seconds (5); + } else { + close(wake); + } + bd = file_dev_get (); #endif @@ -328,9 +371,7 @@ try } LOG_DISK_NC ("Opened partition"); - nanomsg->send(DISK_WRITER_FORMATTING "\n", SHORT_TIMEOUT); - - r = ext4_mkfs(&fs, bd, &info, F_SET_EXT2); + r = ext4_mkfs(&fs, bd, &info, F_SET_EXT2, format_progress, nanomsg); if (r != EOK) { throw CopyError ("Failed to make filesystem", r); } @@ -374,20 +415,26 @@ try } ext4_device_unregister("ext4_fs"); - if (!nanomsg->send(DISK_WRITER_OK "\n", LONG_TIMEOUT)) { + if (nanomsg && !nanomsg->send(DISK_WRITER_OK "\n", LONG_TIMEOUT)) { throw CommunicationFailedError (); } disk_write_finished (); } catch (CopyError& e) { LOG_DISK("CopyError (from write): %1 %2", e.message(), e.number().get_value_or(0)); - nanomsg->send(String::compose(DISK_WRITER_ERROR "\n%1\n%2\n", e.message(), e.number().get_value_or(0)), LONG_TIMEOUT); + if (nanomsg) { + nanomsg->send(String::compose(DISK_WRITER_ERROR "\n%1\n%2\n", e.message(), e.number().get_value_or(0)), LONG_TIMEOUT); + } } catch (VerifyError& e) { LOG_DISK("VerifyError (from write): %1 %2", e.message(), e.number()); - nanomsg->send(String::compose(DISK_WRITER_ERROR "\n%1\n%2\n", e.message(), e.number()), LONG_TIMEOUT); + if (nanomsg) { + nanomsg->send(String::compose(DISK_WRITER_ERROR "\n%1\n%2\n", e.message(), e.number()), LONG_TIMEOUT); + } } catch (exception& e) { LOG_DISK("Exception (from write): %1", e.what()); - nanomsg->send(String::compose(DISK_WRITER_ERROR "\n%1\n0\n", e.what()), LONG_TIMEOUT); + if (nanomsg) { + nanomsg->send(String::compose(DISK_WRITER_ERROR "\n%1\n0\n", e.what()), LONG_TIMEOUT); + } }