From 0b760c0526b0b9d13def519ab8afba1e511d8111 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 23 Jan 2013 20:49:26 +0000 Subject: [PATCH] Try to fix up paths for video MXFs, hashes and temporarily-stored frames. --- .gitignore | 5 +- src/lib/check_hashes_job.cc | 120 ------------------------------------ src/lib/check_hashes_job.h | 39 ------------ src/lib/dcp_video_frame.cc | 12 ++-- src/lib/encoder.cc | 26 -------- src/lib/encoder.h | 6 +- src/lib/film.cc | 90 +++++++++++++-------------- src/lib/film.h | 10 +-- src/lib/transcode_job.cc | 5 -- src/lib/writer.cc | 6 +- src/lib/wscript | 1 - 11 files changed, 61 insertions(+), 259 deletions(-) delete mode 100644 src/lib/check_hashes_job.cc delete mode 100644 src/lib/check_hashes_job.h diff --git a/.gitignore b/.gitignore index 4c0c1926d..cc3351558 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,7 @@ doc/manual/pdf doc/manual/extensions.ent .be/id-cache *.pyc - +GPATH +GRTAGS +GSYMS +GTAGS diff --git a/src/lib/check_hashes_job.cc b/src/lib/check_hashes_job.cc deleted file mode 100644 index 55a744552..000000000 --- a/src/lib/check_hashes_job.cc +++ /dev/null @@ -1,120 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -#include -#include -#include -#include "check_hashes_job.h" -#include "options.h" -#include "log.h" -#include "job_manager.h" -#include "ab_transcode_job.h" -#include "transcode_job.h" -#include "film.h" -#include "exceptions.h" - -using std::string; -using std::stringstream; -using std::ifstream; -using boost::shared_ptr; - -CheckHashesJob::CheckHashesJob (shared_ptr f, DecodeOptions o, shared_ptr req) - : Job (f, req) - , _decode_opt (o) - , _bad (0) -{ - -} - -string -CheckHashesJob::name () const -{ - return String::compose ("Check hashes of %1", _film->name()); -} - -void -CheckHashesJob::run () -{ - _bad = 0; - - if (!_film->dcp_intrinsic_duration()) { - throw EncodeError ("cannot check hashes of a DCP with unknown intrinsic duration"); - } - - int const N = _film->dcp_intrinsic_duration().get(); - for (int i = 0; i < N; ++i) { - string const j2k_file = _film->frame_out_path (i, false); - string const hash_file = _film->hash_out_path (i, false); - - if (!boost::filesystem::exists (j2k_file)) { - _film->log()->log (String::compose ("Frame %1 has a missing J2K file.", i)); - boost::filesystem::remove (hash_file); - ++_bad; - } else if (!boost::filesystem::exists (hash_file)) { - _film->log()->log (String::compose ("Frame %1 has a missing hash file.", i)); - boost::filesystem::remove (j2k_file); - ++_bad; - } else { - ifstream ref (hash_file.c_str ()); - string hash; - ref >> hash; - if (hash != md5_digest (j2k_file)) { - _film->log()->log (String::compose ("Frame %1 has wrong hash; deleting.", i)); - boost::filesystem::remove (j2k_file); - boost::filesystem::remove (hash_file); - ++_bad; - } - } - - set_progress (float (i) / N); - } - - if (_bad) { - shared_ptr tc; - - if (_film->dcp_ab()) { - tc.reset (new ABTranscodeJob (_film, _decode_opt, shared_from_this())); - } else { - tc.reset (new TranscodeJob (_film, _decode_opt, shared_from_this())); - } - - JobManager::instance()->add_after (shared_from_this(), tc); - JobManager::instance()->add_after (tc, shared_ptr (new CheckHashesJob (_film, _decode_opt, tc))); - } - - set_progress (1); - set_state (FINISHED_OK); -} - -string -CheckHashesJob::status () const -{ - stringstream s; - s << Job::status (); - if (overall_progress() > 0) { - if (_bad == 0) { - s << "; no bad frames found"; - } else if (_bad == 1) { - s << "; 1 bad frame found"; - } else { - s << "; " << _bad << " bad frames found"; - } - } - return s.str (); -} diff --git a/src/lib/check_hashes_job.h b/src/lib/check_hashes_job.h deleted file mode 100644 index 5fa17382d..000000000 --- a/src/lib/check_hashes_job.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -#include "job.h" -#include "options.h" - -class CheckHashesJob : public Job -{ -public: - CheckHashesJob ( - boost::shared_ptr f, - DecodeOptions od, - boost::shared_ptr req - ); - - std::string name () const; - void run (); - std::string status () const; - -private: - DecodeOptions _decode_opt; - int _bad; -}; diff --git a/src/lib/dcp_video_frame.cc b/src/lib/dcp_video_frame.cc index c3f0fed03..5df2d7a46 100644 --- a/src/lib/dcp_video_frame.cc +++ b/src/lib/dcp_video_frame.cc @@ -407,27 +407,27 @@ EncodedData::~EncodedData () void EncodedData::write (shared_ptr film, int frame) const { - string const tmp_j2k = film->frame_out_path (frame, true); + string const tmp_j2c = film->j2c_path (frame, true); - FILE* f = fopen (tmp_j2k.c_str (), "wb"); + FILE* f = fopen (tmp_j2c.c_str (), "wb"); if (!f) { - throw WriteFileError (tmp_j2k, errno); + throw WriteFileError (tmp_j2c, errno); } fwrite (_data, 1, _size, f); fclose (f); - string const real_j2k = film->frame_out_path (frame, false); + string const real_j2c = film->j2c_path (frame, false); /* Rename the file from foo.j2c.tmp to foo.j2c now that it is complete */ - boost::filesystem::rename (tmp_j2k, real_j2k); + boost::filesystem::rename (tmp_j2c, real_j2c); } void EncodedData::write_hash (shared_ptr film, int frame) const { - string const hash = film->hash_out_path (frame, false); + string const hash = film->hash_path (frame); ofstream h (hash.c_str()); h << md5_digest (_data, _size) << "\n"; } diff --git a/src/lib/encoder.cc b/src/lib/encoder.cc index 6b14b2698..978d787e8 100644 --- a/src/lib/encoder.cc +++ b/src/lib/encoder.cc @@ -55,7 +55,6 @@ int const Encoder::_history_size = 25; */ Encoder::Encoder (shared_ptr f) : _film (f) - , _just_skipped (false) , _video_frames_in (0) , _video_frames_out (0) #ifdef HAVE_SWRESAMPLE @@ -207,14 +206,6 @@ Encoder::current_frames_per_second () const return _history_size / (seconds (now) - seconds (_time_history.back ())); } -/** @return true if the last frame to be processed was skipped as it already existed */ -bool -Encoder::skipping () const -{ - boost::mutex::scoped_lock (_history_mutex); - return _just_skipped; -} - /** @return Number of video frames that have been sent out */ int Encoder::video_frames_out () const @@ -230,7 +221,6 @@ void Encoder::frame_done () { boost::mutex::scoped_lock lock (_history_mutex); - _just_skipped = false; struct timeval tv; gettimeofday (&tv, 0); @@ -240,16 +230,6 @@ Encoder::frame_done () } } -/** Called by a subclass when it has just skipped the processing - of a frame because it has already been done. -*/ -void -Encoder::frame_skipped () -{ - boost::mutex::scoped_lock lock (_history_mutex); - _just_skipped = true; -} - void Encoder::process_video (shared_ptr image, bool same, boost::shared_ptr sub) { @@ -273,12 +253,6 @@ Encoder::process_video (shared_ptr image, bool same, boost::shared_ptrframe_out_path (_video_frames_out, false))) { - frame_skipped (); - return; - } - if (same && _have_a_real_frame) { /* Use the last frame that we encoded. */ _writer->repeat (_video_frames_out); diff --git a/src/lib/encoder.h b/src/lib/encoder.h index 429b46a18..8b02f7004 100644 --- a/src/lib/encoder.h +++ b/src/lib/encoder.h @@ -82,13 +82,11 @@ public: virtual void process_end (); float current_frames_per_second () const; - bool skipping () const; int video_frames_out () const; private: void frame_done (); - void frame_skipped (); void write_audio (boost::shared_ptr audio); @@ -98,7 +96,7 @@ private: /** Film that we are encoding */ boost::shared_ptr _film; - /** Mutex for _time_history, _just_skipped and _last_frame */ + /** Mutex for _time_history and _last_frame */ mutable boost::mutex _history_mutex; /** List of the times of completion of the last _history_size frames; first is the most recently completed. @@ -106,8 +104,6 @@ private: std::list _time_history; /** Number of frames that we should keep history for */ static int const _history_size; - /** true if the last frame we processed was skipped (because it was already done) */ - bool _just_skipped; /** Number of video frames received so far */ SourceFrame _video_frames_in; diff --git a/src/lib/film.cc b/src/lib/film.cc index ae9edbfdb..dc308078c 100644 --- a/src/lib/film.cc +++ b/src/lib/film.cc @@ -46,7 +46,6 @@ #include "scaler.h" #include "decoder_factory.h" #include "config.h" -#include "check_hashes_job.h" #include "version.h" #include "ui_signaller.h" #include "video_decoder.h" @@ -194,24 +193,14 @@ Film::~Film () { delete _log; } - -/** @return The path to the directory to write JPEG2000 files to */ + string -Film::j2k_dir () const +Film::video_state_identifier () const { - assert (format()); - - boost::filesystem::path p; - - /* Start with j2c */ - p /= "j2c"; + assert (format ()); pair f = Filter::ffmpeg_strings (filters()); - /* Write stuff to specify the filter / post-processing settings that are in use, - so that we don't get confused about J2K files generated using different - settings. - */ stringstream s; s << format()->id() << "_" << content_digest() @@ -221,19 +210,33 @@ Film::j2k_dir () const << "_" << j2k_bandwidth() << "_" << boost::lexical_cast (colour_lut()); - p /= s.str (); - - /* Similarly for the A/B case */ if (dcp_ab()) { - stringstream s; pair fa = Filter::ffmpeg_strings (Config::instance()->reference_filters()); s << "ab_" << Config::instance()->reference_scaler()->id() << "_" << fa.first << "_" << fa.second; - p /= s.str (); } - + + return s.str (); +} + +/** @return The path to the directory to write video frame hash files to */ +string +Film::hash_dir () const +{ + boost::filesystem::path p; + p /= "hash"; + p /= video_state_identifier (); return dir (p.string()); } +string +Film::video_mxf_path () const +{ + boost::filesystem::path p; + p /= "video"; + p /= video_state_identifier (); + return file (p.string()); +} + /** Add suitable Jobs to the JobManager to create a DCP for this Film. * @param true to transcode, false to use the WAV and J2K files that are already there. */ @@ -302,8 +305,6 @@ Film::make_dcp (bool transcode) r = JobManager::instance()->add (shared_ptr (new TranscodeJob (shared_from_this(), od, shared_ptr ()))); } } - - // r = JobManager::instance()->add (shared_ptr (new CheckHashesJob (shared_from_this(), od, r))); } /** Start a job to examine our content file */ @@ -344,7 +345,7 @@ Film::encoded_frames () const } int N = 0; - for (boost::filesystem::directory_iterator i = boost::filesystem::directory_iterator (j2k_dir ()); i != boost::filesystem::directory_iterator(); ++i) { + for (boost::filesystem::directory_iterator i = boost::filesystem::directory_iterator (hash_dir ()); i != boost::filesystem::directory_iterator(); ++i) { ++N; boost::this_thread::interruption_point (); } @@ -1391,44 +1392,37 @@ Film::audio_stream () const return _external_audio_stream; } -/** @param f DCP frame index. - * @param t true to return a temporary file path, otherwise a permanent one. - * @return The path to write this video frame to. - */ string -Film::frame_out_path (int f, bool t) const +Film::hash_path (int f) const { + boost::filesystem::path p; + p /= hash_dir (); + stringstream s; - s << j2k_dir() << "/"; s.width (8); - s << std::setfill('0') << f << ".j2c"; + s << setfill('0') << f << ".md5"; - if (t) { - s << ".tmp"; - } - - return s.str (); + p /= s.str(); + return p.string (); } string -Film::hash_out_path (int f, bool t) const +Film::j2c_path (int f, bool t) const { - return frame_out_path (f, t) + ".md5"; -} + boost::filesystem::path p; + p /= "j2c"; + p /= video_state_identifier (); -/** @param c Audio channel index. - * @param t true to return a temporary file path, otherwise a permanent one. - * @return The path to write this audio file to. - */ -string -Film::multichannel_audio_out_path (int c, bool t) const -{ stringstream s; - s << dir ("wavs") << "/" << (c + 1) << ".wav"; + s.width (8); + s << setfill('0') << f << ".j2c"; + if (t) { s << ".tmp"; } - - return s.str (); + + p /= s.str(); + return p.string (); } + diff --git a/src/lib/film.h b/src/lib/film.h index 60646b0c8..4d2819be3 100644 --- a/src/lib/film.h +++ b/src/lib/film.h @@ -59,7 +59,10 @@ public: Film (Film const &); ~Film (); - std::string j2k_dir () const; + std::string hash_dir () const; + std::string j2c_path (int f, bool t) const; + std::string hash_path (int f) const; + std::string video_mxf_path () const; void examine_content (); void send_dcp_to_tms (); @@ -78,10 +81,6 @@ public: std::string file (std::string f) const; std::string dir (std::string d) const; - std::string frame_out_path (int f, bool t) const; - std::string hash_out_path (int f, bool t) const; - std::string multichannel_audio_out_path (int c, bool t) const; - std::string content_path () const; ContentType content_type () const; @@ -413,6 +412,7 @@ private: void signal_changed (Property); void examine_content_finished (); + std::string video_state_identifier () const; /** Complete path to directory containing the film metadata; * must not be relative. diff --git a/src/lib/transcode_job.cc b/src/lib/transcode_job.cc index 6dd74c36c..e9a59c743 100644 --- a/src/lib/transcode_job.cc +++ b/src/lib/transcode_job.cc @@ -89,11 +89,6 @@ TranscodeJob::status () const return "0%"; } - if (_encoder->skipping () && !finished ()) { - return "skipping already-encoded frames"; - } - - float const fps = _encoder->current_frames_per_second (); if (fps == 0) { return Job::status (); diff --git a/src/lib/writer.cc b/src/lib/writer.cc index a434db0ec..16ed5c349 100644 --- a/src/lib/writer.cc +++ b/src/lib/writer.cc @@ -42,7 +42,7 @@ Writer::Writer (shared_ptr f) _picture_asset.reset ( new libdcp::MonoPictureAsset ( _film->dir (_film->dcp_name()), - "video.mxf", + _film->video_mxf_path(), DCPFrameRate (_film->frames_per_second()).frames_per_second, _film->format()->dcp_size() ) @@ -164,9 +164,9 @@ Writer::thread () lock.unlock (); _film->log()->log (String::compose ("Writer pulls %1 back from disk", fetch)); shared_ptr encoded; - if (boost::filesystem::exists (_film->frame_out_path (fetch, false))) { + if (boost::filesystem::exists (_film->j2c_path (fetch, false))) { /* It's an actual frame (not a repeat-last); load it in */ - encoded.reset (new EncodedData (_film->frame_out_path (fetch, false))); + encoded.reset (new EncodedData (_film->j2c_path (fetch, false))); } lock.lock (); diff --git a/src/lib/wscript b/src/lib/wscript index 02c426373..5d676f249 100644 --- a/src/lib/wscript +++ b/src/lib/wscript @@ -14,7 +14,6 @@ def build(bld): ab_transcoder.cc audio_decoder.cc audio_source.cc - check_hashes_job.cc config.cc combiner.cc cross.cc -- 2.30.2