Add an advanced option to check data against the frame info file when making hashes (#2758).

This should show up corruption that happens on the disk/storage, i.e.
between DCP-o-matic writing the MXF and it being read back in again
for hashing.

6 files changed, 135 insertions, 3 deletions

diff --git a/src/lib/config.cc b/src/lib/config.cc
index 180ce5c55..4a96584fc 100644
--- a/src/lib/config.cc
+++ b/src/lib/config.cc
@@ -213,6 +213,7 @@ Config::set_defaults ()
 	_auto_crop_threshold = 0.1;
 	_last_release_notes_version = boost::none;
 	_allow_smpte_bv20 = false;
+	_check_disk_writes = false;
 	_isdcf_name_part_length = 14;
 	_enable_player_http_server = false;
 	_player_http_server_port = 8080;
@@ -652,6 +653,7 @@ try
 	}
 
 	_allow_smpte_bv20 = f.optional_bool_child("AllowSMPTEBv20").get_value_or(false);
+	_check_disk_writes = f.optional_bool_child("CheckDiskWrites").get_value_or(false);
 	_isdcf_name_part_length = f.optional_number_child<int>("ISDCFNamePartLength").get_value_or(14);
 	_enable_player_http_server = f.optional_bool_child("EnablePlayerHTTPServer").get_value_or(false);
 	_player_http_server_port = f.optional_number_child<int>("PlayerHTTPServerPort").get_value_or(8080);
@@ -1124,6 +1126,8 @@ Config::write_config () const
 
 	/* [XML] AllowSMPTEBv20 1 to allow the user to choose SMPTE (Bv2.0 only) as a standard, otherwise 0 */
 	cxml::add_text_child(root, "AllowSMPTEBv20", _allow_smpte_bv20 ? "1" : "0");
+	/* [XML] CheckDiskWrites 1 to check for disk corruption when creating MXF hashes, otherwise 0 */
+	cxml::add_text_child(root, "CheckDiskWrites", _check_disk_writes ? "1" : "0");
 	/* [XML] ISDCFNamePartLength Maximum length of the "name" part of an ISDCF name, which should be 14 according to the standard */
 	cxml::add_text_child(root, "ISDCFNamePartLength", raw_convert<string>(_isdcf_name_part_length));
 	/* [XML] EnablePlayerHTTPServer 1 to enable a HTTP server to control the player, otherwise 0 */
diff --git a/src/lib/config.h b/src/lib/config.h
index 1f13381b7..8394f612e 100644
--- a/src/lib/config.h
+++ b/src/lib/config.h
@@ -115,6 +115,7 @@ public:
 		AUDIO_MAPPING,
 		AUTO_CROP_THRESHOLD,
 		ALLOW_SMPTE_BV20,
+		CHECK_DISK_WRITES,
 		ISDCF_NAME_PART_LENGTH,
 		ALLOW_ANY_CONTAINER,
 #ifdef DCPOMATIC_GROK
@@ -644,6 +645,10 @@ public:
 		return _allow_smpte_bv20;
 	}
 
+	bool check_disk_writes() const {
+		return _check_disk_writes;
+	}
+
 #ifdef DCPOMATIC_GROK
 	class Grok
 	{
@@ -1209,6 +1214,10 @@ public:
 		maybe_set(_allow_smpte_bv20, allow, ALLOW_SMPTE_BV20);
 	}
 
+	void set_check_disk_writes(bool allow) {
+		maybe_set(_check_disk_writes, allow, CHECK_DISK_WRITES);
+	}
+
 #ifdef DCPOMATIC_GROK
 	void set_grok(Grok const& grok);
 #endif
@@ -1462,6 +1471,7 @@ private:
 	boost::optional<int> _main_content_divider_sash_position;
 	DefaultAddFileLocation _default_add_file_location;
 	bool _allow_smpte_bv20;
+	bool _check_disk_writes;
 	int _isdcf_name_part_length;
 	bool _enable_player_http_server;
 	int _player_http_server_port;
diff --git a/src/lib/info_file_checker.cc b/src/lib/info_file_checker.cc
new file mode 100644
index 000000000..d2a07def4
--- /dev/null
+++ b/src/lib/info_file_checker.cc
@@ -0,0 +1,62 @@
+/*
+    Copyright (C) 2024 Carl Hetherington <cth@carlh.net>
+
+    This file is part of DCP-o-matic.
+
+    DCP-o-matic is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    DCP-o-matic is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+
+#include "info_file_checker.h"
+#include <dcp/filesystem.h>
+
+#include "i18n.h"
+
+
+using std::shared_ptr;
+
+
+InfoFileChecker::InfoFileChecker(shared_ptr<InfoFileHandle> file)
+	: _file(file)
+	, _next(_file)
+{
+
+}
+
+
+void
+InfoFileChecker::check(int64_t offset, uint8_t const* data, int length)
+{
+	while (_next && offset <= static_cast<int64_t>(_next->offset) && _next->offset < static_cast<uint64_t>(offset + length)) {
+		auto hash_start_offset = _next->offset - offset;
+		auto hash_length = std::min(_next->size, length - hash_start_offset);
+		_digester.add(data + hash_start_offset, hash_length);
+		_next->offset += hash_length;
+		_next->size -= hash_length;
+		if (_next->size == 0) {
+			if (_digester.get_vector() != _next->hash) {
+				throw EncodeError(_("File on disk is corrupted"));
+			}
+
+			_next = boost::none;
+			try {
+				_next = J2KFrameInfo(_file);
+			} catch (...) {}
+
+			_digester = Digester();
+		}
+	}
+}
+
diff --git a/src/lib/info_file_checker.h b/src/lib/info_file_checker.h
new file mode 100644
index 000000000..c4cc2fe5c
--- /dev/null
+++ b/src/lib/info_file_checker.h
@@ -0,0 +1,41 @@
+/*
+    Copyright (C) 2024 Carl Hetherington <cth@carlh.net>
+
+    This file is part of DCP-o-matic.
+
+    DCP-o-matic is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    DCP-o-matic is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+
+#include "digester.h"
+#include "info_file_handle.h"
+#include "frame_info.h"
+#include <dcp/file.h>
+#include <boost/filesystem.hpp>
+
+
+class InfoFileChecker
+{
+public:
+	InfoFileChecker(std::shared_ptr<InfoFileHandle> file);
+
+	void check(int64_t offset, uint8_t const* data, int length);
+
+private:
+	std::shared_ptr<InfoFileHandle> _file;
+	boost::optional<J2KFrameInfo> _next;
+	Digester _digester;
+};
+
diff --git a/src/lib/reel_writer.cc b/src/lib/reel_writer.cc
index 17ac02c0c..af6334473 100644
--- a/src/lib/reel_writer.cc
+++ b/src/lib/reel_writer.cc
@@ -31,6 +31,7 @@
 #include "frame_info.h"
 #include "image.h"
 #include "image_png.h"
+#include "info_file_checker.h"
 #include "job.h"
 #include "log.h"
 #include "reel_writer.h"
@@ -758,10 +759,23 @@ try
 	}
 
 	int64_t total_done = 0;
+
+	auto progress = [&total_done, total_size, set_progress](int64_t done, int64_t) {
+		set_progress(total_done + done, total_size);
+	};
+
 	for (auto asset: assets) {
-		asset->hash([&total_done, total_size, set_progress](int64_t done, int64_t) {
-			set_progress(total_done + done, total_size);
-		});
+		if (asset == _j2k_picture_asset && Config::instance()->check_disk_writes()) {
+			try {
+				InfoFileChecker checker(film()->info_file_handle(_period, true));
+				asset->hash(progress, boost::bind(&InfoFileChecker::check, &checker, _1, _2, _3));
+			} catch (OpenFileError&) {
+				LOG_ERROR_NC("Could not open info file to check disk writes");
+				asset->hash(progress);
+			}
+		} else {
+			asset->hash(progress);
+		}
 		total_done += asset->file() ? boost::filesystem::file_size(*asset->file()) : 0;
 	}
 
diff --git a/src/lib/wscript b/src/lib/wscript
index 6ca49a05d..f559e2cc5 100644
--- a/src/lib/wscript
+++ b/src/lib/wscript
@@ -139,6 +139,7 @@ sources = """
           image_png.cc
           image_proxy.cc
           image_store.cc
+          info_file_checker.cc
           info_file_handle.cc
           internal_player_server.cc
           j2k_image_proxy.cc