Unfinished attempt to overwrite existing; tricky because you need to delay writes of the MXF header until you know lots of stuff about the JP2K file (to fill in the picture descriptor).

12 files changed, 219 insertions, 17 deletions

diff --git a/asdcplib/src/AS_DCP.h b/asdcplib/src/AS_DCP.h
index d5675586..2681bf14 100755
--- a/asdcplib/src/AS_DCP.h
+++ b/asdcplib/src/AS_DCP.h
@@ -1165,11 +1165,11 @@ namespace ASDCP {
 	  virtual MXF::OPAtomHeader& OPAtomHeader();
 	  virtual MXF::OPAtomIndexFooter& OPAtomIndexFooter();
 
-	  // Open the file for writing. The file must not exist. Returns error if
+	  // Open the file for writing. The file must not exist unless overwrite is true. Returns error if
 	  // the operation cannot be completed or if nonsensical data is discovered
 	  // in the essence descriptor.
 	  Result_t OpenWrite(const char* filename, const WriterInfo&,
-			     const PictureDescriptor&, ui32_t HeaderSize = 16384);
+			     const PictureDescriptor&, ui32_t HeaderSize, bool overwrite);
 
 	  // Writes a frame of essence to the MXF file. If the optional AESEncContext
 	  // argument is present, the essence is encrypted prior to writing.
@@ -1178,6 +1178,8 @@ namespace ASDCP {
 	  // error occurs.
 	  Result_t WriteFrame(const FrameBuffer&, AESEncContext* = 0, HMACContext* = 0, std::string* hash = 0);
 
+	  Result_t FakeWriteFrame(int size);
+
 	  // Closes the MXF file, writing the index and revised header.
 	  Result_t Finalize();
 
diff --git a/asdcplib/src/AS_DCP_JP2K.cpp b/asdcplib/src/AS_DCP_JP2K.cpp
index 359cd910..05f5e005 100755
--- a/asdcplib/src/AS_DCP_JP2K.cpp
+++ b/asdcplib/src/AS_DCP_JP2K.cpp
@@ -1,3 +1,5 @@
+/* -*- c-basic-offset: 2; -*- */
+
 /*
 Copyright (c) 2004-2012, John Hurst
 All rights reserved.
@@ -33,6 +35,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iostream>
 #include <iomanip>
 
+using std::cout;
 using namespace ASDCP::JP2K;
 using Kumu::GenRandomValue;
 
@@ -812,10 +815,11 @@ public:
 
   ~lh__Writer(){}
 
-  Result_t OpenWrite(const char*, EssenceType_t type, ui32_t HeaderSize);
+  Result_t OpenWrite(const char*, EssenceType_t type, ui32_t HeaderSize, bool);
   Result_t SetSourceStream(const PictureDescriptor&, const std::string& label,
 			   ASDCP::Rational LocalEditRate = ASDCP::Rational(0,0));
   Result_t WriteFrame(const JP2K::FrameBuffer&, bool add_index, AESEncContext*, HMACContext*, std::string* hash = 0);
+  Result_t FakeWriteFrame(int size, bool add_index);
   Result_t Finalize();
   Result_t JP2K_PDesc_to_MD(JP2K::PictureDescriptor& PDesc);
 };
@@ -900,15 +904,21 @@ lh__Writer::JP2K_PDesc_to_MD(JP2K::PictureDescriptor& PDesc)
 }
 
 
-// Open the file for writing. The file must not exist. Returns error if
+// Open the file for writing. The file must not exist unless overwrite is true. Returns error if
 // the operation cannot be completed.
 ASDCP::Result_t
-lh__Writer::OpenWrite(const char* filename, EssenceType_t type, ui32_t HeaderSize)
+lh__Writer::OpenWrite(const char* filename, EssenceType_t type, ui32_t HeaderSize, bool overwrite)
 {
   if ( ! m_State.Test_BEGIN() )
     return RESULT_STATE;
 
-  Result_t result = m_File.OpenWrite(filename);
+  Result_t result = RESULT_OK;
+  if (overwrite) {
+    result = m_File.OpenModify(filename);
+    m_File.Seek(0);
+  } else {
+    result = m_File.OpenWrite(filename);
+  }
 
   if ( ASDCP_SUCCESS(result) )
     {
@@ -986,6 +996,8 @@ lh__Writer::WriteFrame(const JP2K::FrameBuffer& FrameBuf, bool add_index,
     result = m_State.Goto_RUNNING(); // first time through
  
   ui64_t StreamOffset = m_StreamOffset;
+  cout << "Real write @ " << StreamOffset << " (header is " << m_HeaderSize << ")\n";
+  cout << "\tfile @ " << m_File.Tell() << "\n";
 
   if ( ASDCP_SUCCESS(result) )
     result = WriteEKLVPacket(FrameBuf, m_EssenceUL, Ctx, HMAC, hash);
@@ -1001,6 +1013,32 @@ lh__Writer::WriteFrame(const JP2K::FrameBuffer& FrameBuf, bool add_index,
   return result;
 }
 
+Result_t
+lh__Writer::FakeWriteFrame(int size, bool add_index)
+{
+  Result_t result = RESULT_OK;
+
+  if ( m_State.Test_READY() )
+    result = m_State.Goto_RUNNING();
+
+  ui64_t StreamOffset = m_StreamOffset;
+  cout << "Fake write @ " << StreamOffset << " (header is " << m_HeaderSize << ")\n";
+  cout << "\tfile @ " << m_File.Tell() << "\n";
+
+  if ( ASDCP_SUCCESS(result) )
+    result = FakeWriteEKLVPacket(size);
+
+  if ( ASDCP_SUCCESS(result) && add_index )
+    {  
+      IndexTableSegment::IndexEntry Entry;
+      Entry.StreamOffset = StreamOffset;
+      m_FooterPart.PushIndexEntry(Entry);
+    }
+
+  m_FramesWritten++;
+  return result;
+}
+
 
 // Closes the MXF file, writing the index and other closing information.
 //
@@ -1069,11 +1107,11 @@ ASDCP::JP2K::MXFWriter::OPAtomIndexFooter()
   return m_Writer->m_FooterPart;
 }
 
-// Open the file for writing. The file must not exist. Returns error if
+// Open the file for writing. The file must not exist unless overwrite is true. Returns error if
 // the operation cannot be completed.
 ASDCP::Result_t
 ASDCP::JP2K::MXFWriter::OpenWrite(const char* filename, const WriterInfo& Info,
-				  const PictureDescriptor& PDesc, ui32_t HeaderSize)
+				  const PictureDescriptor& PDesc, ui32_t HeaderSize, bool overwrite)
 {
   if ( Info.LabelSetType == LS_MXF_SMPTE )
     m_Writer = new h__Writer(DefaultSMPTEDict());
@@ -1082,7 +1120,7 @@ ASDCP::JP2K::MXFWriter::OpenWrite(const char* filename, const WriterInfo& Info,
 
   m_Writer->m_Info = Info;
 
-  Result_t result = m_Writer->OpenWrite(filename, ASDCP::ESS_JPEG_2000, HeaderSize);
+  Result_t result = m_Writer->OpenWrite(filename, ASDCP::ESS_JPEG_2000, HeaderSize, overwrite);
 
   if ( ASDCP_SUCCESS(result) )
     result = m_Writer->SetSourceStream(PDesc, JP2K_PACKAGE_LABEL);
@@ -1107,6 +1145,15 @@ ASDCP::JP2K::MXFWriter::WriteFrame(const FrameBuffer& FrameBuf, AESEncContext* C
   return m_Writer->WriteFrame(FrameBuf, true, Ctx, HMAC, hash);
 }
 
+ASDCP::Result_t
+ASDCP::JP2K::MXFWriter::FakeWriteFrame(int size)
+{
+  if ( m_Writer.empty() )
+    return RESULT_INIT;
+
+  return m_Writer->FakeWriteFrame(size, true);
+}
+
 // Closes the MXF file, writing the index and other closing information.
 ASDCP::Result_t
 ASDCP::JP2K::MXFWriter::Finalize()
@@ -1233,7 +1280,7 @@ ASDCP::JP2K::MXFSWriter::OpenWrite(const char* filename, const WriterInfo& Info,
 
   m_Writer->m_Info = Info;
 
-  Result_t result = m_Writer->OpenWrite(filename, ASDCP::ESS_JPEG_2000_S, HeaderSize);
+  Result_t result = m_Writer->OpenWrite(filename, ASDCP::ESS_JPEG_2000_S, HeaderSize, false);
 
   if ( ASDCP_SUCCESS(result) )
     {
diff --git a/asdcplib/src/AS_DCP_internal.h b/asdcplib/src/AS_DCP_internal.h
index 15b52822..73106a26 100755
--- a/asdcplib/src/AS_DCP_internal.h
+++ b/asdcplib/src/AS_DCP_internal.h
@@ -239,6 +239,8 @@ namespace ASDCP
       Result_t WriteEKLVPacket(const ASDCP::FrameBuffer& FrameBuf,
 			       const byte_t* EssenceUL, AESEncContext* Ctx, HMACContext* HMAC, std::string* hash = 0);
 
+      Result_t FakeWriteEKLVPacket(int size);
+	    
       Result_t WriteMXFFooter();
 
    };
diff --git a/asdcplib/src/KM_prng.cpp b/asdcplib/src/KM_prng.cpp
index 06b22d91..463ae157 100755
--- a/asdcplib/src/KM_prng.cpp
+++ b/asdcplib/src/KM_prng.cpp
@@ -1,3 +1,5 @@
+/* -*- c-basic-offset: 2; -*- */
+
 /*
 Copyright (c) 2006-2009, John Hurst
 All rights reserved.
@@ -98,7 +100,9 @@ public:
 
     set_key(rng_key);
 
-    m_libdcp_test_rng_state = 1;
+#ifdef LIBDCP_POSIX    
+    reset();
+#endif    
   }
 	
   //
@@ -153,6 +157,13 @@ public:
     /* XXX */
 #endif
   }
+
+#ifdef LIBDCP_POSIX  
+  void reset ()
+  {
+    m_libdcp_test_rng_state = 1;
+  }
+#endif  
 };
 
 
@@ -206,6 +217,14 @@ Kumu::FortunaRNG::FillRandom(Kumu::ByteString& Buffer)
   return Buffer.Data();
 }
 
+#ifdef LIBDCP_POSIX
+void
+Kumu::FortunaRNG::Reset()
+{
+  s_RNG->reset();
+}
+#endif
+
 //------------------------------------------------------------------------------------------
 
 //
diff --git a/asdcplib/src/KM_prng.h b/asdcplib/src/KM_prng.h
index 4d7ab2b7..0b941f3b 100755
--- a/asdcplib/src/KM_prng.h
+++ b/asdcplib/src/KM_prng.h
@@ -45,6 +45,9 @@ namespace Kumu
       ~FortunaRNG();
       const byte_t* FillRandom(byte_t* buf, ui32_t len);
       const byte_t* FillRandom(ByteString&);
+#ifdef LIBDCP_POSIX	    
+      void Reset();
+#endif	    
     };
 
 
diff --git a/asdcplib/src/KM_util.cpp b/asdcplib/src/KM_util.cpp
index 23e8a1a6..b1814840 100755
--- a/asdcplib/src/KM_util.cpp
+++ b/asdcplib/src/KM_util.cpp
@@ -534,6 +534,15 @@ Kumu::GenRandomValue(UUID& ID)
   ID.Set(tmp_buf);
 }
 
+#ifdef LIBDCP_POSIX
+void
+Kumu::ResetTestRNG()
+{
+  FortunaRNG RNG;
+  RNG.Reset();
+}
+#endif
+
 //
 void
 Kumu::GenRandomUUID(byte_t* buf)
diff --git a/asdcplib/src/KM_util.h b/asdcplib/src/KM_util.h
index 892670e6..a9793ba0 100755
--- a/asdcplib/src/KM_util.h
+++ b/asdcplib/src/KM_util.h
@@ -381,6 +381,9 @@ namespace Kumu
   
   void GenRandomUUID(byte_t* buf); // buf must be UUID_Length or longer
   void GenRandomValue(UUID&);
+#ifdef LIBDCP_POSIX	
+  void ResetTestRNG();
+#endif	
   
   typedef ArchivableList<UUID> UUIDList;
 
diff --git a/asdcplib/src/h__Writer.cpp b/asdcplib/src/h__Writer.cpp
index 662e0f82..d743e300 100755
--- a/asdcplib/src/h__Writer.cpp
+++ b/asdcplib/src/h__Writer.cpp
@@ -32,6 +32,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "AS_DCP_internal.h"
 #include "KLV.h"
 
+using std::cout;
 using namespace ASDCP;
 using namespace ASDCP::MXF;
 
@@ -644,6 +645,17 @@ ASDCP::h__Writer::WriteEKLVPacket(const ASDCP::FrameBuffer& FrameBuf, const byte
   return result;
 }
 
+Result_t
+ASDCP::h__Writer::FakeWriteEKLVPacket(int size)
+{
+  Result_t result = RESULT_OK;
+
+  m_StreamOffset += size;
+  m_File.Seek(size, Kumu::SP_POS);
+
+  return result;
+}
+
 
 // standard method of writing the header and footer of a completed MXF file
 //
diff --git a/run-tests.sh b/run-tests.sh
index 7e464329..888968ad 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -63,5 +63,11 @@ if [ "$?" != "0" ]; then
   echo "FAIL: dcpinfo output from rewrite incorrect"
   exit 1
 fi
+
+diff build/test/baz/video1.mxf build/test/baz/video2.mxf
+if [ "$?" != "0" ]; then
+  echo "FAIL: MXFs from recovery incorrect"
+  exit 1
+fi
     
 echo "PASS"
diff --git a/src/picture_asset.cc b/src/picture_asset.cc
index d242777c..11815761 100644
--- a/src/picture_asset.cc
+++ b/src/picture_asset.cc
@@ -207,7 +207,7 @@ MonoPictureAsset::construct (boost::function<string (int)> get_path)
 	fill_writer_info (&writer_info, _uuid);
 	
 	ASDCP::JP2K::MXFWriter mxf_writer;
-	if (ASDCP_FAILURE (mxf_writer.OpenWrite (path().string().c_str(), writer_info, picture_desc))) {
+	if (ASDCP_FAILURE (mxf_writer.OpenWrite (path().string().c_str(), writer_info, picture_desc, 16384, false))) {
 		throw MXFFileError ("could not open MXF file for writing", path().string());
 	}
 
@@ -399,7 +399,14 @@ shared_ptr<MonoPictureAssetWriter>
 MonoPictureAsset::start_write ()
 {
 	/* XXX: can't we use a shared_ptr here? */
-	return shared_ptr<MonoPictureAssetWriter> (new MonoPictureAssetWriter (this));
+	return shared_ptr<MonoPictureAssetWriter> (new MonoPictureAssetWriter (this, false));
+}
+
+shared_ptr<MonoPictureAssetWriter>
+MonoPictureAsset::start_overwrite ()
+{
+	/* XXX: can't we use a shared_ptr here? */
+	return shared_ptr<MonoPictureAssetWriter> (new MonoPictureAssetWriter (this, true));
 }
 
 FrameInfo::FrameInfo (istream& s)
@@ -455,8 +462,8 @@ MonoPictureAssetWriter::write (uint8_t* data, int size)
 		_state->picture_descriptor.EditRate = ASDCP::Rational (_asset->edit_rate(), 1);
 	
 		MXFAsset::fill_writer_info (&_state->writer_info, _asset->uuid());
-		
-		if (ASDCP_FAILURE (_state->mxf_writer.OpenWrite (_asset->path().string().c_str(), _state->writer_info, _state->picture_descriptor))) {
+
+		if (ASDCP_FAILURE (_state->mxf_writer.OpenWrite (_asset->path().string().c_str(), _state->writer_info, _state->picture_descriptor, 16384, false))) {
 			throw MXFFileError ("could not open MXF file for writing", _asset->path().string());
 		}
 	}
@@ -468,11 +475,36 @@ MonoPictureAssetWriter::write (uint8_t* data, int size)
 		throw MiscError ("error in writing video MXF");
 	}
 
-	_frames_written++;
+	++_frames_written;
 	return FrameInfo (before_offset, _state->mxf_writer.Tell() - before_offset, hash);
 }
 
 void
+MonoPictureAssetWriter::fake_write (int size)
+{
+	assert (!_finalized);
+
+	if (_frames_written == 0) {
+		/* This is our first frame; set up the writer */
+		
+		_state->j2k_parser.FillPictureDescriptor (_state->picture_descriptor);
+		_state->picture_descriptor.EditRate = ASDCP::Rational (_asset->edit_rate(), 1);
+	
+		MXFAsset::fill_writer_info (&_state->writer_info, _asset->uuid());
+
+		if (ASDCP_FAILURE (_state->mxf_writer.OpenWrite (_asset->path().string().c_str(), _state->writer_info, _state->picture_descriptor, 16384, true))) {
+			throw MXFFileError ("could not open MXF file for writing", _asset->path().string());
+		}
+	}
+
+	if (ASDCP_FAILURE (_state->mxf_writer.FakeWriteFrame (size))) {
+		throw MiscError ("error in writing video MXF");
+	}
+
+	++_frames_written;
+}
+
+void
 MonoPictureAssetWriter::finalize ()
 {
 	if (ASDCP_FAILURE (_state->mxf_writer.Finalize())) {
diff --git a/src/picture_asset.h b/src/picture_asset.h
index 4e9e1dd7..9df85bad 100644
--- a/src/picture_asset.h
+++ b/src/picture_asset.h
@@ -115,12 +115,13 @@ public:
 	~MonoPictureAssetWriter ();
 
 	FrameInfo write (uint8_t* data, int size);
+	void fake_write (int size);
 	void finalize ();
 
 private:
 	friend class MonoPictureAsset;
 
-	MonoPictureAssetWriter (MonoPictureAsset *);
+	MonoPictureAssetWriter (MonoPictureAsset *, bool);
 
 	/* no copy construction */
 	MonoPictureAssetWriter (MonoPictureAssetWriter const &);
@@ -205,6 +206,8 @@ public:
 	/** Start a progressive write to a MonoPictureAsset */
 	boost::shared_ptr<MonoPictureAssetWriter> start_write ();
 
+	boost::shared_ptr<MonoPictureAssetWriter> start_overwrite ();
+
 	boost::shared_ptr<const MonoPictureFrame> get_frame (int n) const;
 	bool equals (boost::shared_ptr<const Asset> other, EqualityOptions opt, std::list<std::string>& notes) const;
 
diff --git a/test/tests.cc b/test/tests.cc
index 2d21ae86..c9243b88 100644
--- a/test/tests.cc
+++ b/test/tests.cc
@@ -35,6 +35,7 @@
 #include <boost/test/unit_test.hpp>
 
 using std::string;
+using std::cout;
 using std::vector;
 using std::list;
 using boost::shared_ptr;
@@ -578,3 +579,66 @@ BOOST_AUTO_TEST_CASE (color)
 	BOOST_CHECK_EQUAL (c.to_argb_string(), "FF0000FF");
 	
 }
+
+BOOST_AUTO_TEST_CASE (recovery)
+{
+	Kumu::libdcp_test = true;
+
+	cout << "=== recovery.\n";
+
+	string const picture = "test/data/32x32_red_square.j2c";
+	int const length = boost::filesystem::file_size (picture);
+	uint8_t* data = new uint8_t[length];
+	{
+		FILE* f = fopen (picture.c_str(), "rb");
+		BOOST_CHECK (f);
+		fread (data, 1, length, f);
+		fclose (f);
+	}
+
+	Kumu::ResetTestRNG ();
+	
+	boost::filesystem::remove_all ("build/test/baz");
+	boost::filesystem::create_directories ("build/test/baz");
+	shared_ptr<libdcp::MonoPictureAsset> mp (new libdcp::MonoPictureAsset ("build/test/baz", "video1.mxf", 24, libdcp::Size (32, 32)));
+	shared_ptr<libdcp::MonoPictureAssetWriter> writer = mp->start_write ();
+
+	int written_length = 0;
+	for (int i = 0; i < 24; ++i) {
+		libdcp::FrameInfo info = writer->write (data, length);
+		written_length = info.length;
+		cout << "- written length " << written_length << "\n";
+	}
+
+	writer->finalize ();
+	writer.reset ();
+
+	cout << "=== recovery part 2.\n";
+	
+	boost::filesystem::copy_file ("build/test/baz/video1.mxf", "build/test/baz/video2.mxf");
+	boost::filesystem::resize_file ("build/test/baz/video2.mxf", 16384 + 353 * 11);
+
+	{
+		FILE* f = fopen ("build/test/baz/video2.mxf", "wa");
+		rewind (f);
+		char zeros[256];
+		memset (zeros, 0, 256);
+		fwrite (zeros, 1, 256, f);
+		fclose (f);
+	}
+
+	Kumu::ResetTestRNG ();
+
+	mp.reset (new libdcp::MonoPictureAsset ("build/test/baz", "video2.mxf", 24, libdcp::Size (32, 32)));
+	writer = mp->start_overwrite ();
+
+	for (int i = 0; i < 4; ++i) {
+		writer->fake_write (written_length);
+	}
+
+	for (int i = 0; i < 20; ++i) {
+		writer->write (data, length);
+	}
+	
+	writer->finalize ();
+}